liwenhao2
/
ASR-Android
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

翻译开头 ASCII 残片清理(aby/nope/fewest 族):cleanTrans 加原文对照剥除——开头2~6位小写字母+分隔符+CJK/假名,且该片段不在原文中(原文真词或大写开头如AI/OK不误伤);四处调用点传 src(兜底/单句/编号批/逐段兜底);'aby'与'html>'同族=模型困惑态首词元落训练残片,后续译文健康时旧闸门全放行

This commit is contained in:
liwenhao2 2026-09-16 09:46:29 +08:00
parent 5bbdfa19ed
commit 2bdeee2147
1 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt
View File

@ -2077,7 +2077,13 @@ class MainActivity : AppCompatActivity() {
"^(`{3,}\\s*[A-Za-z]*>?|</?[A-Za-z0-9_|]{1,18}>|\\.?html?>)", "^(`{3,}\\s*[A-Za-z]*>?|</?[A-Za-z0-9_|]{1,18}>|\\.?html?>)",
RegexOption.IGNORE_CASE) RegexOption.IGNORE_CASE)
private fun cleanTrans(raw: String?): String? { // 小写 ASCII 残片(aby/nope/fewest 族):2~6 个小写字母+空白/标点后随
// CJK/假名。必须"原文里没有该片段"才剥——原件含 iPhone/PDF 等真词时
// 不误伤;大写开头(AI/OK)天然不在此列
private val transAsciiNoise = Regex(
"^([a-z]{2,6})[\\s,、，。!！?？]+(?=[\\u4e00-\\u9fff\\u3040-\\u30ff])")
private fun cleanTrans(raw: String?, src: String? = null): String? {
if (raw.isNullOrBlank()) return null if (raw.isNullOrBlank()) return null
val ls = raw.trim().lines().toMutableList() val ls = raw.trim().lines().toMutableList()
while (ls.isNotEmpty() && transLabelLine.matches(ls.first().trim())) while (ls.isNotEmpty() && transLabelLine.matches(ls.first().trim()))
@ -2089,6 +2095,14 @@ class MainActivity : AppCompatActivity() {
s = transLabelPrefix.replaceFirst(s, "").trim() s = transLabelPrefix.replaceFirst(s, "").trim()
val s1 = s val s1 = s
s = transNoisePrefix.replaceFirst(s, "").trim() s = transNoisePrefix.replaceFirst(s, "").trim()
if (s == s1 && src != null) {
transAsciiNoise.find(s)?.let { m ->
val frag = m.groupValues[1]
if (!src.contains(frag, ignoreCase = true)) {
s = s.removeRange(m.range).trim()
}
}
}
if (s != s1) { if (s != s1) {
Log.i(TAG, "TAP-NOISE stripped: " + Log.i(TAG, "TAP-NOISE stripped: " +
s1.take(16).replace("\n", " ")) s1.take(16).replace("\n", " "))
@ -2214,7 +2228,7 @@ class MainActivity : AppCompatActivity() {
maxNewTokens = (src.length * 2 + 96).coerceAtMost(1200), maxNewTokens = (src.length * 2 + 96).coerceAtMost(1200),
temperature = 0.2f, repeatPenalty = 1.1f, temperature = 0.2f, repeatPenalty = 1.1f,
) )
val tr = cleanTrans(out) val tr = cleanTrans(out, src)
when { when {
tr.isNullOrBlank() || isEcho(src, tr, echoJaccardTh(src)) || tr.isNullOrBlank() || isEcho(src, tr, echoJaccardTh(src)) ||
!targetLangOk(tr) -> { !targetLangOk(tr) -> {
@ -2344,7 +2358,7 @@ class MainActivity : AppCompatActivity() {
} catch (t: Throwable) { } catch (t: Throwable) {
Log.w(TAG, "TRANSLATE single failed: " + t.message); null Log.w(TAG, "TRANSLATE single failed: " + t.message); null
} }
val tr = cleanTrans(one) val tr = cleanTrans(one, segKeys[0].second)
if (!tr.isNullOrEmpty()) got[segKeys[0].first] = tr if (!tr.isNullOrEmpty()) got[segKeys[0].first] = tr
if (got.isEmpty()) return null if (got.isEmpty()) return null
} else { } else {
@ -2370,7 +2384,7 @@ class MainActivity : AppCompatActivity() {
} }
val byIdx = extractNumberedPairs(out) val byIdx = extractNumberedPairs(out)
segKeys.forEachIndexed { i, st -> segKeys.forEachIndexed { i, st ->
byIdx?.get((i + 1).toString())?.let { cleanTrans(it) } byIdx?.get((i + 1).toString())?.let { cleanTrans(it, st.second) }
?.takeIf { it.isNotBlank() } ?.takeIf { it.isNotBlank() }
?.let { got[st.first] = it } ?.let { got[st.first] = it }
} }
@ -2399,7 +2413,7 @@ class MainActivity : AppCompatActivity() {
temperature = 0f, temperature = 0f,
) )
} catch (t: Throwable) { null } } catch (t: Throwable) { null }
cleanTrans(one)?.let { got[st.first] = it } cleanTrans(one, st.second)?.let { got[st.first] = it }
} }
} }
if (got.isEmpty()) return null if (got.isEmpty()) return null