From 2bdeee21476b251c9ab868ab88a113fe1bf648f6 Mon Sep 17 00:00:00 2001 From: liwenhao2 Date: Wed, 16 Sep 2026 09:46:29 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BF=BB=E8=AF=91=E5=BC=80=E5=A4=B4=20ASCII=20?= =?UTF-8?q?=E6=AE=8B=E7=89=87=E6=B8=85=E7=90=86(aby/nope/fewest=20?= =?UTF-8?q?=E6=97=8F):cleanTrans=20=E5=8A=A0=E5=8E=9F=E6=96=87=E5=AF=B9?= =?UTF-8?q?=E7=85=A7=E5=89=A5=E9=99=A4=E2=80=94=E2=80=94=E5=BC=80=E5=A4=B4?= =?UTF-8?q?2~6=E4=BD=8D=E5=B0=8F=E5=86=99=E5=AD=97=E6=AF=8D+=E5=88=86?= =?UTF-8?q?=E9=9A=94=E7=AC=A6+CJK/=E5=81=87=E5=90=8D,=E4=B8=94=E8=AF=A5?= =?UTF-8?q?=E7=89=87=E6=AE=B5=E4=B8=8D=E5=9C=A8=E5=8E=9F=E6=96=87=E4=B8=AD?= =?UTF-8?q?(=E5=8E=9F=E6=96=87=E7=9C=9F=E8=AF=8D=E6=88=96=E5=A4=A7?= =?UTF-8?q?=E5=86=99=E5=BC=80=E5=A4=B4=E5=A6=82AI/OK=E4=B8=8D=E8=AF=AF?= =?UTF-8?q?=E4=BC=A4);=E5=9B=9B=E5=A4=84=E8=B0=83=E7=94=A8=E7=82=B9?= =?UTF-8?q?=E4=BC=A0=20src(=E5=85=9C=E5=BA=95/=E5=8D=95=E5=8F=A5/=E7=BC=96?= =?UTF-8?q?=E5=8F=B7=E6=89=B9/=E9=80=90=E6=AE=B5=E5=85=9C=E5=BA=95);'aby'?= =?UTF-8?q?=E4=B8=8E'html>'=E5=90=8C=E6=97=8F=3D=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E5=9B=B0=E6=83=91=E6=80=81=E9=A6=96=E8=AF=8D=E5=85=83=E8=90=BD?= =?UTF-8?q?=E8=AE=AD=E7=BB=83=E6=AE=8B=E7=89=87,=E5=90=8E=E7=BB=AD?= =?UTF-8?q?=E8=AF=91=E6=96=87=E5=81=A5=E5=BA=B7=E6=97=B6=E6=97=A7=E9=97=B8?= =?UTF-8?q?=E9=97=A8=E5=85=A8=E6=94=BE=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/k2fsa/sherpa/onnx/MainActivity.kt | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index 59cc346..a04a9a9 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -2077,7 +2077,13 @@ class MainActivity : AppCompatActivity() { "^(`{3,}\\s*[A-Za-z]*>?||\\.?html?>)", RegexOption.IGNORE_CASE) - private fun cleanTrans(raw: String?): String? { + // 小写 ASCII 残片(aby/nope/fewest 族):2~6 个小写字母+空白/标点后随 + // CJK/假名。必须"原文里没有该片段"才剥——原件含 iPhone/PDF 等真词时 + // 不误伤;大写开头(AI/OK)天然不在此列 + private val transAsciiNoise = Regex( + "^([a-z]{2,6})[\\s,、，。!！?？]+(?=[\\u4e00-\\u9fff\\u3040-\\u30ff])") + + private fun cleanTrans(raw: String?, src: String? = null): String? { if (raw.isNullOrBlank()) return null val ls = raw.trim().lines().toMutableList() while (ls.isNotEmpty() && transLabelLine.matches(ls.first().trim())) @@ -2089,6 +2095,14 @@ class MainActivity : AppCompatActivity() { s = transLabelPrefix.replaceFirst(s, "").trim() val s1 = s s = transNoisePrefix.replaceFirst(s, "").trim() + if (s == s1 && src != null) { + transAsciiNoise.find(s)?.let { m -> + val frag = m.groupValues[1] + if (!src.contains(frag, ignoreCase = true)) { + s = s.removeRange(m.range).trim() + } + } + } if (s != s1) { Log.i(TAG, "TAP-NOISE stripped: " + s1.take(16).replace("\n", " ")) @@ -2214,7 +2228,7 @@ class MainActivity : AppCompatActivity() { maxNewTokens = (src.length * 2 + 96).coerceAtMost(1200), temperature = 0.2f, repeatPenalty = 1.1f, ) - val tr = cleanTrans(out) + val tr = cleanTrans(out, src) when { tr.isNullOrBlank() || isEcho(src, tr, echoJaccardTh(src)) || !targetLangOk(tr) -> { @@ -2344,7 +2358,7 @@ class MainActivity : AppCompatActivity() { } catch (t: Throwable) { Log.w(TAG, "TRANSLATE single failed: " + t.message); null } - val tr = cleanTrans(one) + val tr = cleanTrans(one, segKeys[0].second) if (!tr.isNullOrEmpty()) got[segKeys[0].first] = tr if (got.isEmpty()) return null } else { @@ -2370,7 +2384,7 @@ class MainActivity : AppCompatActivity() { } val byIdx = extractNumberedPairs(out) segKeys.forEachIndexed { i, st -> - byIdx?.get((i + 1).toString())?.let { cleanTrans(it) } + byIdx?.get((i + 1).toString())?.let { cleanTrans(it, st.second) } ?.takeIf { it.isNotBlank() } ?.let { got[st.first] = it } } @@ -2399,7 +2413,7 @@ class MainActivity : AppCompatActivity() { temperature = 0f, ) } catch (t: Throwable) { null } - cleanTrans(one)?.let { got[st.first] = it } + cleanTrans(one, st.second)?.let { got[st.first] = it } } } if (got.isEmpty()) return null