翻译开头 ASCII 残片清理(aby/nope/fewest 族):cleanTrans 加原文对照剥除——开头2~6位小写字母+分隔符+CJK/假名,且该片段不在原文中(原文真词或大写开头如AI/OK不误伤);四处调用点传 src(兜底/单句/编号批/逐段兜底);'aby'与'html>'同族=模型困惑态首词元落训练残片,后续译文健康时旧闸门全放行
This commit is contained in:
parent
5bbdfa19ed
commit
2bdeee2147
|
|
@ -2077,7 +2077,13 @@ class MainActivity : AppCompatActivity() {
|
|||
"^(`{3,}\\s*[A-Za-z]*>?|</?[A-Za-z0-9_|]{1,18}>|\\.?html?>)",
|
||||
RegexOption.IGNORE_CASE)
|
||||
|
||||
private fun cleanTrans(raw: String?): String? {
|
||||
// 小写 ASCII 残片(aby/nope/fewest 族):2~6 个小写字母+空白/标点后随
|
||||
// CJK/假名。必须"原文里没有该片段"才剥——原件含 iPhone/PDF 等真词时
|
||||
// 不误伤;大写开头(AI/OK)天然不在此列
|
||||
private val transAsciiNoise = Regex(
|
||||
"^([a-z]{2,6})[\\s,、,。!!??]+(?=[\\u4e00-\\u9fff\\u3040-\\u30ff])")
|
||||
|
||||
private fun cleanTrans(raw: String?, src: String? = null): String? {
|
||||
if (raw.isNullOrBlank()) return null
|
||||
val ls = raw.trim().lines().toMutableList()
|
||||
while (ls.isNotEmpty() && transLabelLine.matches(ls.first().trim()))
|
||||
|
|
@ -2089,6 +2095,14 @@ class MainActivity : AppCompatActivity() {
|
|||
s = transLabelPrefix.replaceFirst(s, "").trim()
|
||||
val s1 = s
|
||||
s = transNoisePrefix.replaceFirst(s, "").trim()
|
||||
if (s == s1 && src != null) {
|
||||
transAsciiNoise.find(s)?.let { m ->
|
||||
val frag = m.groupValues[1]
|
||||
if (!src.contains(frag, ignoreCase = true)) {
|
||||
s = s.removeRange(m.range).trim()
|
||||
}
|
||||
}
|
||||
}
|
||||
if (s != s1) {
|
||||
Log.i(TAG, "TAP-NOISE stripped: " +
|
||||
s1.take(16).replace("\n", " "))
|
||||
|
|
@ -2214,7 +2228,7 @@ class MainActivity : AppCompatActivity() {
|
|||
maxNewTokens = (src.length * 2 + 96).coerceAtMost(1200),
|
||||
temperature = 0.2f, repeatPenalty = 1.1f,
|
||||
)
|
||||
val tr = cleanTrans(out)
|
||||
val tr = cleanTrans(out, src)
|
||||
when {
|
||||
tr.isNullOrBlank() || isEcho(src, tr, echoJaccardTh(src)) ||
|
||||
!targetLangOk(tr) -> {
|
||||
|
|
@ -2344,7 +2358,7 @@ class MainActivity : AppCompatActivity() {
|
|||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "TRANSLATE single failed: " + t.message); null
|
||||
}
|
||||
val tr = cleanTrans(one)
|
||||
val tr = cleanTrans(one, segKeys[0].second)
|
||||
if (!tr.isNullOrEmpty()) got[segKeys[0].first] = tr
|
||||
if (got.isEmpty()) return null
|
||||
} else {
|
||||
|
|
@ -2370,7 +2384,7 @@ class MainActivity : AppCompatActivity() {
|
|||
}
|
||||
val byIdx = extractNumberedPairs(out)
|
||||
segKeys.forEachIndexed { i, st ->
|
||||
byIdx?.get((i + 1).toString())?.let { cleanTrans(it) }
|
||||
byIdx?.get((i + 1).toString())?.let { cleanTrans(it, st.second) }
|
||||
?.takeIf { it.isNotBlank() }
|
||||
?.let { got[st.first] = it }
|
||||
}
|
||||
|
|
@ -2399,7 +2413,7 @@ class MainActivity : AppCompatActivity() {
|
|||
temperature = 0f,
|
||||
)
|
||||
} catch (t: Throwable) { null }
|
||||
cleanTrans(one)?.let { got[st.first] = it }
|
||||
cleanTrans(one, st.second)?.let { got[st.first] = it }
|
||||
}
|
||||
}
|
||||
if (got.isEmpty()) return null
|
||||
|
|
|
|||
Loading…
Reference in New Issue