完整差異
diff --git a/pipeline/text_spacing.py b/pipeline/text_spacing.py\nindex 598c3f05e..db2b308a8 100644\n--- a/pipeline/text_spacing.py\n+++ b/pipeline/text_spacing.py\n@@ -7,6 +7,9 @@ from html.parser import HTMLParser\n \n CJK = r"\u2e80-\u9fff\uf900-\ufaff\uff01-\uff60\U00020000-\U000323af"\n SPACES = re.compile(rf"(?<=[{CJK}]) +| +(?=[{CJK}])")\n+# Keep sentence separation even when source copy omitted the space. Include\n+# closing quotes/brackets, but never split repeated punctuation or a paragraph.\n+SENTENCE_END = re.compile(r'[。！？][」』”’）)】\]〕〉》]*(?=[^\s。！？!?，,、；;：:．.…」』”’）)】\]〕〉》])')\n BLOCKS = set("html head body address article aside blockquote br button dd details dialog div dl dt fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 header hr li main nav ol option p section summary table tbody td th thead tr ul".split())\n SKIP = {"script", "style", "pre", "code", "textarea"}\n TEXT_ATTRS = {"alt", "title", "placeholder", "aria-label", "aria-description"}\n@@ -15,8 +18,19 @@ META_TEXT = {"description", "og:title", "og:description", "og:image:alt",\n ATTR = re.compile(r'''([\w:-]+)(\s*=\s*)(["'])(.*?)\3''', re.DOTALL)\n \n \n+def _spacing_changes(text: str):\n+ removed = {i for match in SPACES.finditer(text)\n+ for i in range(match.start(), match.end())}\n+ positions = [i for i in range(len(text)) if i not in removed]\n+ compacted = "".join(text[i] for i in positions)\n+ inserted = {positions[match.end()] for match in SENTENCE_END.finditer(compacted)}\n+ return removed, inserted\n+\n+\n def compact_text(text: str) -> str:\n- return SPACES.sub("", text)\n+ removed, inserted = _spacing_changes(text)\n+ return "".join((" " if i in inserted else "") + char\n+ for i, char in enumerate(text) if i not in removed)\n \n \n class _DisplaySpacing(HTMLParser):\n@@ -34,16 +48,17 @@ class _DisplaySpacing(HTMLParser):\n \n def flush(self):\n text = "".join(value for _, _, value in self.run)\n- removed = set()\n- for match in SPACES.finditer(text):\n- removed.update(range(match.start(), match.end()))\n+ removed, inserted = _spacing_changes(text)\n cursor = 0\n- if removed:\n+ if removed or inserted:\n for start, raw, value in self.run:\n- cleaned = "".join(c for i, c in enumerate(value)\n+ cleaned = "".join((" " if cursor + i in inserted else "") + c\n+ for i, c in enumerate(value)\n if cursor + i not in removed)\n if cleaned != value:\n # Entities are individual tokens; untouched ones stay verbatim.\n+ if raw != value:\n+ cleaned = html.escape(cleaned, quote=False)\n self.edits.append((start, start + len(raw), cleaned))\n cursor += len(value)\n self.run.clear()\ndiff --git a/static/text-spacing.js b/static/text-spacing.js\nindex ff040fefa..b46aa0f86 100644\n--- a/static/text-spacing.js\n+++ b/static/text-spacing.js\n@@ -2,28 +2,47 @@\n "use strict";\n var cjk = "\\u2e80-\\u9fff\\uf900-\\ufaff\\uff01-\\uff60\\u{20000}-\\u{323af}";\n var spaces = new RegExp("(?<=[" + cjk + "]) +| +(?=[" + cjk + "])", "gu");\n+ var sentenceEnd = /[。！？][」』”’）)】\]〕〉》]*(?=[^\s。！？!?，,、；;：:．.…」』”’）)】\]〕〉》])/gu;\n var blocks = new Set("HTML HEAD BODY ADDRESS ARTICLE ASIDE BLOCKQUOTE BR BUTTON DD DETAILS DIALOG DIV DL DT FIELDSET FIGCAPTION FIGURE FOOTER FORM H1 H2 H3 H4 H5 H6 HEADER HR LI MAIN NAV OL OPTION P SECTION SUMMARY TABLE TBODY TD TH THEAD TR UL".split(" "));\n var skip = "script,style,pre,code,textarea,[contenteditable]:not([contenteditable=false])";\n var attrs = ["alt", "title", "placeholder", "aria-label", "aria-description"];\n \n- function compact(text) { return text.replace(spaces, ""); }\n+ function changes(text) {\n+ var removed = new Set(), positions = [];\n+ text.replace(spaces, function (match, offset) {\n+ for (var i = 0; i < match.length; i++) removed.add(offset + i);\n+ return "";\n+ });\n+ for (var i = 0; i < text.length; i++) if (!removed.has(i)) positions.push(i);\n+ var compacted = positions.map(function (index) { return text[index]; }).join("");\n+ var inserted = new Set();\n+ compacted.replace(sentenceEnd, function (match, offset) {\n+ inserted.add(positions[offset + match.length]);\n+ return match;\n+ });\n+ return { removed: removed, inserted: inserted };\n+ }\n+\n+ function cleanNode(text, offset, edits) {\n+ return text.split("").map(function (char, index) {\n+ var position = offset + index;\n+ return edits.removed.has(position) ? "" :\n+ (edits.inserted.has(position) ? " " : "") + char;\n+ }).join("");\n+ }\n+\n+ function compact(text) { return cleanNode(text, 0, changes(text)); }\n \n function normalize(root) {\n var run = [];\n function flush() {\n var text = run.map(function (node) { return node.data; }).join("");\n- var removed = new Set();\n- text.replace(spaces, function (match, offset) {\n- for (var i = 0; i < match.length; i++) removed.add(offset + i);\n- return "";\n- });\n+ var edits = changes(text);\n var offset = 0;\n- if (removed.size) run.forEach(function (node) {\n+ if (edits.removed.size || edits.inserted.size) run.forEach(function (node) {\n var original = node.data;\n // String offsets and regex indices both use UTF-16 code units.\n- var clean = original.split("").filter(function (_, index) {\n- return !removed.has(offset + index);\n- }).join("");\n+ var clean = cleanNode(original, offset, edits);\n offset += original.length;\n if (clean !== original) node.data = clean;\n });\ndiff --git a/tests/test_text_spacing.py b/tests/test_text_spacing.py\nindex e0d17e793..c60902a1c 100644\n--- a/tests/test_text_spacing.py\n+++ b/tests/test_text_spacing.py\n@@ -15,6 +15,13 @@ def test_display_spacing():\n "中文 123 航班（ TPE ）": "中文123航班（TPE）",\n "中文\u00a0English": "中文\u00a0English",\n "𠀀 English": "𠀀English",\n+ "完成試飛。G400於2024年首飛。": "完成試飛。 G400於2024年首飛。",\n+ "第一句。 第二句。 第三句。": "第一句。 第二句。 第三句。",\n+ "已起飛！下一班呢？仍在準備。": "已起飛！ 下一班呢？ 仍在準備。",\n+ "他說：「已完成。」下一步試飛。": "他說：「已完成。」 下一步試飛。",\n+ "真的！？下一句。（附註。）後續。": "真的！？ 下一句。 （附註。） 後續。",\n+ "第一句。\n第二段。": "第一句。\n第二段。",\n+ "English sentence. Another with 0.89 and U.S. aviation.": "English sentence. Another with 0.89 and U.S. aviation.",\n }\n for original, expected in cases.items():\n assert compact_text(original) == expected, (original, compact_text(original))\n@@ -38,8 +45,30 @@ def test_display_spacing():\n '<style>/* 中文 AI */</style>', '<!-- 中文 AI -->'):\n assert preserved in result\n assert compact_html(result) == result\n+\n+ # Inserting before an entity must not turn escaped text into HTML markup.\n+ escaped = '<p>第一句。<Next>。&後續。</p>'\n+ assert compact_html(escaped) == '<p>第一句。 <Next>。 &後續。</p>'\n print("test_text_spacing: OK")\n \n \n+def test_sentence_spacing_across_markup():\n+ source = '''<p>完成試飛。<strong>G400</strong> 於2024年首飛。 下一句。</p>\n+<p>他說：「已完成。<em>」</em>後續試飛。</p>\n+<p>第一句。<a href="/中文 path/">第二句</a>。 第三句。</p>\n+<p>第一句。</p><p>第二段。</p>\n+<p>English sentence. <strong>Another</strong> sentence.</p>\n+<script>const text = "第一句。第二句。";</script>'''\n+ result = compact_html(source)\n+ assert '完成試飛。<strong> G400</strong>於2024年首飛。 下一句。' in result\n+ assert '已完成。<em>」</em> 後續試飛。' in result\n+ assert '。<a href="/中文 path/"> 第二句</a>。 第三句。' in result\n+ assert '<p>第一句。</p><p>第二段。</p>' in result\n+ assert '<p>English sentence. <strong>Another</strong> sentence.</p>' in result\n+ assert '<script>const text = "第一句。第二句。";</script>' in result\n+ assert compact_html(result) == result\n+\n+\n if __name__ == "__main__":\n test_display_spacing()\n+ test_sentence_spacing_across_markup()\ndiff --git a/tests/test_text_spacing_client.js b/tests/test_text_spacing_client.js\nindex fa7fcb3eb..425585777 100644\n--- a/tests/test_text_spacing_client.js\n+++ b/tests/test_text_spacing_client.js\n@@ -8,6 +8,34 @@ for (const [source, expected] of [\n ["Latest News and Air France", "Latest News and Air France"],\n ["中文 123 航班（ TPE ）", "中文123航班（TPE）"],\n ["中文\u00a0English", "中文\u00a0English"],\n- ["𠀀 English", "𠀀English"]\n-]) assert.strictEqual(spacing.compact(source), expected);\n+ ["𠀀 English", "𠀀English"],\n+ ["完成試飛。G400於2024年首飛。", "完成試飛。 G400於2024年首飛。"],\n+ ["第一句。 第二句。 第三句。", "第一句。 第二句。 第三句。"],\n+ ["已起飛！下一班呢？仍在準備。", "已起飛！ 下一班呢？ 仍在準備。"],\n+ ["他說：「已完成。」下一步試飛。", "他說：「已完成。」 下一步試飛。"],\n+ ["真的！？下一句。（附註。）後續。", "真的！？ 下一句。 （附註。） 後續。"],\n+ ["第一句。\n第二段。", "第一句。\n第二段。"],\n+ ["English sentence. Another with 0.89 and U.S. aviation.", "English sentence. Another with 0.89 and U.S. aviation."],\n+ ["𠀀完成。𠀀 下一句。", "𠀀完成。 𠀀下一句。"]\n+]) {\n+ assert.strictEqual(spacing.compact(source), expected);\n+ assert.strictEqual(spacing.compact(expected), expected);\n+}\n+\n+// Exercise the actual DOM path, including text split by inline elements.\n+function element(tagName, childNodes) {\n+ return { nodeType: 1, tagName, childNodes,\n+ matches() { return tagName === "CODE"; }, hasAttribute() { return false; } };\n+}\n+function text(data) { return { nodeType: 3, data }; }\n+const first = text("完成試飛。"), next = text("G400"), tail = text(" 於2024年首飛。 下一句。");\n+const second = text("第二段。"), skipped = text("中文 AI。下一句。");\n+const root = element("DIV", [element("P", [first, element("STRONG", [next]), tail]),\n+ element("P", [second]), element("CODE", [skipped])]);\n+spacing.normalize(root);\n+assert.strictEqual(first.data + next.data + tail.data, "完成試飛。 G400於2024年首飛。 下一句。");\n+assert.strictEqual(second.data, "第二段。");\n+assert.strictEqual(skipped.data, "中文 AI。下一句。");\n+spacing.normalize(root);\n+assert.strictEqual(first.data + next.data + tail.data, "完成試飛。 G400於2024年首飛。 下一句。");\n console.log("test_text_spacing_client: OK");
