From 5bbdfa19edb60def40c6d733bfc5750645ad454d Mon Sep 17 00:00:00 2001 From: liwenhao2 Date: Wed, 16 Sep 2026 09:43:38 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BF=BB=E8=AF=91=E5=BC=80=E5=A4=B4=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E6=AE=8B=E7=89=87=E6=B8=85=E7=90=86:cleanTrans=20?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=20transNoisePrefix(=E5=9B=B4=E6=A0=8F?= =?UTF-8?q?=E7=AC=A6/=E5=8D=8A=E6=88=AAHTML=E6=A0=87=E7=AD=BE/=E6=A8=A1?= =?UTF-8?q?=E6=9D=BF=E8=AF=8D=E5=85=83/html=E5=B0=BE=E5=B7=B4),=E4=B8=8E?= =?UTF-8?q?=E7=AD=94=E6=A1=88=E6=A0=87=E7=AD=BE=E5=89=A5=E9=99=A4=E4=BA=A4?= =?UTF-8?q?=E6=9B=BF=E5=BE=AA=E7=8E=AF=E7=9B=B4=E8=87=B3=E7=A8=B3=E5=AE=9A?= =?UTF-8?q?;=E7=8E=B0=E5=9C=BA'html>'=3DGemma=E8=AF=8D=E8=A1=A8''?= =?UTF-8?q?=E5=B0=BE=E9=83=A8BPE=E8=AF=8D=E5=85=83,=E5=9B=B0=E6=83=91?= =?UTF-8?q?=E6=80=81=E5=86=B7=E5=90=AF=E5=8A=A8=E9=A6=96=E8=AF=8D=E5=85=83?= =?UTF-8?q?=E8=90=BD=E6=AE=8B=E7=95=99=E3=80=81=E5=90=8E=E7=BB=AD=E8=AF=91?= =?UTF-8?q?=E6=96=87=E6=AD=A3=E5=B8=B8=E5=85=A8=E9=83=A8=E8=B4=A8=E6=A3=80?= =?UTF-8?q?=E6=94=BE=E8=A1=8C=E5=AF=BC=E8=87=B4=E4=B8=8A=E5=B1=8F;?= =?UTF-8?q?=E5=89=A5=E7=A9=BA=E2=86=92null=E8=B5=B0=E9=87=8D=E8=AF=95;TAP-?= =?UTF-8?q?NOISE=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/k2fsa/sherpa/onnx/MainActivity.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index ea8ac62..59cc346 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -2068,6 +2068,14 @@ class MainActivity : AppCompatActivity() { "^\\s*(\\*\\*|__|#+\\s*)?\\s*[「\"“'『]?\\s*(以下是?)?\\s*(翻訳|翻译|訳文|译文|" + "translation|translate)\\s*(结果|結果)?\\s*(如下)?\\s*(\\*\\*|__)?\\s*" + "(は|的|为|:)?\\s*[：:—\\-]\\s*[」\"”'』]?\\s*", RegexOption.IGNORE_CASE) + // 开头格式残片(2026-09-16 现场:翻译开头出现 "html>"):Gemma 词表里 + // "" 的尾巴是独立 BPE 词元,模型困惑态冷启动偶尔第一个词元就 + // 落在这类"文档结尾"残留上,后面译文正常、全部质检放行 → 噪声上屏。 + // 保守集合:围栏符/半截标签/模板词元字面/html 尾巴;循环里与标签剥除 + // 交替直至稳定。剥到空 → null 走重试,不外发。 + private val transNoisePrefix = Regex( + "^(`{3,}\\s*[A-Za-z]*>?||\\.?html?>)", + RegexOption.IGNORE_CASE) private fun cleanTrans(raw: String?): String? { if (raw.isNullOrBlank()) return null @@ -2079,6 +2087,12 @@ class MainActivity : AppCompatActivity() { do { prev = s s = transLabelPrefix.replaceFirst(s, "").trim() + val s1 = s + s = transNoisePrefix.replaceFirst(s, "").trim() + if (s != s1) { + Log.i(TAG, "TAP-NOISE stripped: " + + s1.take(16).replace("\n", " ")) + } } while (s != prev && s.isNotEmpty()) if (s.length >= 2 && ((s.startsWith("「") && s.endsWith("」")) || (s.startsWith("\"") && s.endsWith("\"")) ||