taodeng/memory-continuity
1
0
Fork 0
mirror of https://github.com/dtzp555-max/memory-continuity.git synced 2026-07-21 21:15:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: hoist CJK regex and poison patterns, fix surrogate slicing

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
taodeng
2026-03-31 14:23:58 +10:00
co-authored by Claude Sonnet 4.6
parent a5feb5e0e5
commit cc0aad38cb
1 changed files with 26 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
index.js
+26 -17
View File
@@ -14,18 +14,35 @@ const PLACEHOLDER_VALUES = new Set([
"[exactly what should happen next]",
]);
// Matches a single CJK character (no `g` flag — used per-char in estimateTokens)
const CJK_RE = /[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff\u3000-\u303f\uff00-\uffef]/;
// Patterns that indicate error/garbage assistant responses — kept at module scope
// so the array is not re-created on every extractStateFromMessages call.
const POISON_PATTERNS = [
/not logged in/i,
/please run \/login/i,
/unknown skill/i,
/session expired/i,
/auth.*failed/i,
/error.*timeout/i,
];
const isPoisoned = (text) => POISON_PATTERNS.some(p => p.test(text));
/**
* Estimate token count with CJK awareness.
* CJK characters ≈ 1.5 tokens each; Latin words ≈ 1 token per ~4 chars.
* CJK characters ≈ 1.5 tokens each; Latin/other chars ≈ 1 token per ~4 chars.
* Uses a single-pass for...of loop so surrogate pairs are iterated by code point.
*/
function estimateTokens(text) {
if (!text) return 0;
// Count CJK characters (CJK Unified Ideographs + common CJK ranges)
const cjkCount = (text.match(/[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff\u3000-\u303f\uff00-\uffef]/g) || []).length;
// Remove CJK chars, count remaining as ~1 token per 4 chars
const nonCjk = text.replace(/[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff\u3000-\u303f\uff00-\uffef]/g, "");
const latinTokens = Math.ceil(nonCjk.length / 4);
return Math.ceil(cjkCount * 1.5) + latinTokens;
let cjkCount = 0;
let nonCjkLen = 0;
for (const ch of text) {
if (CJK_RE.test(ch)) cjkCount++;
else nonCjkLen++;
}
return Math.ceil(cjkCount * 1.5) + Math.ceil(nonCjkLen / 4);
}
const STATE_TEMPLATE = `# Current State
@@ -195,20 +212,12 @@ function extractStateFromMessages(messages) {
if (estimateTokens(s.slice(0, mid)) <= maxTokens) lo = mid;
else hi = mid - 1;
}
// Avoid splitting surrogate pairs
while (lo > 0 && lo < s.length && s.charCodeAt(lo) >= 0xDC00 && s.charCodeAt(lo) <= 0xDFFF) lo--;
return s.slice(0, lo) + "...";
};
// Filter out error/garbage responses that would poison future sessions
const POISON_PATTERNS = [
/not logged in/i,
/please run \/login/i,
/unknown skill/i,
/session expired/i,
/auth.*failed/i,
/error.*timeout/i,
];
const isPoisoned = (text) => POISON_PATTERNS.some(p => p.test(text));
if (isPoisoned(lastAssistant)) return null;
if (isPoisoned(lastUser) && !lastAssistant) return null;