语言漂移治理: 长句拆段≤140字段级编号一次生成再拼回(治简→繁→异语中途掉线)、目标语健全检查(ja须假名/zh须简体/en无CJK)、漂移带温度0.3重试一次、提示词注入目标语硬约束、MISS langdrift/seg-drift日志

This commit is contained in:
liwenhao2 2026-09-15 10:46:20 +08:00
parent 902084e175
commit 679b88bec3
1 changed files with 153 additions and 66 deletions

View File

@ -1946,6 +1946,8 @@ class MainActivity : AppCompatActivity() {
Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-empty")
isEcho(s2.text, tr, echoJaccardTh(s2.text)) ->
Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-echo")
!targetLangOk(tr) ->
Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-drift")
else -> if (s2.trans != tr) {
s2.trans = tr
s2.line?.let { touched.add(it) }
@ -2020,99 +2022,145 @@ class MainActivity : AppCompatActivity() {
if (initialPending.isEmpty() || translateDisabled) return
val batch = ArrayList(initialPending)
initialPending.clear()
translateHandler.post {
val map = translateNumbered(batch.map {
Pair(it.first.toString(), it.second)
})
translateHandler.post { runTranslateBatch(batch, 0f) }
}
// 翻译线程执行:直出/编号批 → 逐条质检(空/复读/语言漂移)→ 主线程应用。
// 生成失败或语言漂移 → 带温度重试一次(每句仅一次,retriedSids 全部在本线程读写)。
private fun runTranslateBatch(batch: List<Pair<Long, String>>, temp: Float) {
val map = translateNumbered(
batch.map { Pair(it.first.toString(), it.second) }, temp
)
if (map == null || !plausibleTranslations(map.values)) {
Log.i(TAG, "MISS reason=genfail sids=" +
batch.joinToString(",") { it.first.toString() })
// 失败自动重试一次(仅一次;判重在主线程做,避免竞态)
uiHandler.post {
if (temp == 0f) {
val retry = batch.filter { retriedSids.add(it.first) }
if (retry.isNotEmpty()) {
initialPending.addAll(retry)
flushInitialBatch()
if (retry.isNotEmpty()) translateHandler.post {
runTranslateBatch(retry, 0.3f)
}
}
return@post
return
}
uiHandler.post {
val touched = HashSet<SubtitleLine>()
batch.forEach { (sid, src) ->
val good = HashMap<Long, String>()
val drift = ArrayList<Pair<Long, String>>()
for ((sid, src) in batch) {
val tr = map[sid.toString()]
when {
tr == null || tr.isBlank() ->
Log.i(TAG, "MISS sid=$sid reason=empty")
Log.i(TAG, "MISS sid=" + sid + " reason=empty")
isEcho(src, tr, echoJaccardTh(src)) ->
Log.i(TAG, "MISS sid=$sid reason=echo")
else -> findSentence(sid)?.let { s2 ->
Log.i(TAG, "MISS sid=" + sid + " reason=echo")
!targetLangOk(tr) -> {
Log.i(TAG, "MISS sid=" + sid + " reason=langdrift")
if (temp == 0f && retriedSids.add(sid)) drift.add(Pair(sid, src))
}
else -> good[sid] = tr
}
}
uiHandler.post {
val touched = HashSet<SubtitleLine>()
good.forEach { (sid, tr) ->
findSentence(sid)?.let { s2 ->
if (s2.trans != tr) {
s2.trans = tr
s2.line?.let { touched.add(it) }
}
}
}
}
touched.forEach { notifyLine(it) }
}
if (drift.isNotEmpty()) translateHandler.post {
runTranslateBatch(drift, 0.3f)
}
}
// 一批 = 一个全新引擎(nativeGenerate 不清 KV 的 workaround)。
// items: key(通常是 sid 字符串) × 原文;返回 key × 译文,失败 null。
private fun translateNumbered(items: List<Pair<String, String>>): Map<String, String>? {
private fun translateNumbered(
items: List<Pair<String, String>>, temp: Float = 0f,
): Map<String, String>? {
if (items.isEmpty()) return emptyMap()
val targetName = when (uiLanguage()) {
"ja" -> "日本語"; "en" -> "English"; else -> "简体中文"
}
val langHint = when (uiLanguage()) {
"ja" -> "译文必须是日语(须含假名,不要输出中文或繁体)"
"zh" -> "译文必须是简体中文(不得用繁体字,不要输出日语或英语)"
else -> "The translation must be English only (no CJK characters)"
}
translateEngine?.let { runCatching { it.release() } }
translateEngine = null
val engine = ensureTranslateEngine()
?: run { translateDisabled = true; return null }
// 动态预算:按原文总长给输出额度(译文常比原文长,原 160/句 会把
// 长句截断成残缺 JSON,整批被静默丢弃——"第一句不翻"的直接死因)
val budget = (items.sumOf { it.second.length } * 2 + 96)
// 长句拆段:小模型对短输入才稳定;长句译到中途会"掉语言"
// (简体→繁体→别的语言)。拆 ≤140 字段一次 generate,段序拼回。
val segKeys = ArrayList<Pair<String, String>>()
val segCount = HashMap<String, Int>()
for ((key, t) in items) {
val segs = splitSegments(t)
if (segs.size > 1) segCount[key] = segs.size
segs.forEachIndexed { i, sg ->
val sk = if (segs.size == 1) key else key + "#s" + i
segKeys.add(sk to sg)
}
}
val budget = (segKeys.sumOf { it.second.length } * 2 + 96)
.coerceAtMost(1200)
// 单句 = 直出模式:不套 JSON 包装,无格式可坏,截断也仍是有效译文
if (items.size == 1) {
val got = HashMap<String, String>()
if (segKeys.size == 1) {
val one = try {
engine.generate(
systemPrompt = "你是专业翻译。把给出的句子翻译成" + targetName +
"。只输出译文本身,不要解释、不要引号、不要抄写原文。",
userPrompt = items[0].second,
maxNewTokens = budget, temperature = 0f,
"。只输出译文本身,不要解释、不要引号、不要抄写原文。" +
langHint + "。",
userPrompt = segKeys[0].second,
maxNewTokens = budget, temperature = temp,
)
} catch (t: Throwable) {
Log.w(TAG, "TRANSLATE single failed: ${t.message}"); null
Log.w(TAG, "TRANSLATE single failed: " + t.message); null
}
val tr = one?.trim()?.lines()?.firstOrNull { it.isNotBlank() }?.trim()
return if (tr.isNullOrEmpty()) null
else mapOf(items[0].first to tr)
}
val numbered = items.mapIndexed { i, (_, t) -> "${i + 1}. $t" }
.joinToString("\n")
if (!tr.isNullOrEmpty()) got[segKeys[0].first] = tr
if (got.isEmpty()) return null
} else {
val numbered = segKeys.mapIndexed { i, st ->
(i + 1).toString() + ". " + st.second
}.joinToString("\n")
val out = try {
engine.generate(
systemPrompt = "你是专业翻译。逐句翻译下面编号句子为" + targetName +
"。各句相互独立,必须逐句对应翻译,不得合并、不得重复、不得解释。" +
systemPrompt = "你是专业翻译。逐段翻译下面编号文本为" + targetName +
"。各段相互独立,逐段对应,不得合并、不得重复、不得解释。" +
"只输出JSON对象,键是编号字符串,值是译文," +
"例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。" +
"严禁把原句原样作为译文输出;确实无法翻译的句子," +
"值输出空字符串。",
userPrompt = "【句子】\n$numbered",
maxNewTokens = budget,
temperature = 0f,
"严禁把原文照抄为译文;无法翻译的段值输出空字符串。" +
langHint + "。",
userPrompt = "【文本】\n" + numbered,
maxNewTokens = budget, temperature = temp,
)
} catch (t: Throwable) {
Log.w(TAG, "TRANSLATE batch failed: ${t.message}"); null
Log.w(TAG, "TRANSLATE batch failed: " + t.message); null
}
val byIdx = extractNumberedPairs(out) ?: return null
val res = HashMap<String, String>()
items.forEachIndexed { i, (key, _) ->
byIdx[(i + 1).toString()]?.let { res[key] = it }
segKeys.forEachIndexed { i, st ->
byIdx[(i + 1).toString()]?.takeIf { it.isNotBlank() }
?.let { got[st.first] = it }
}
return res
if (got.isEmpty()) return null
}
val res = HashMap<String, String>()
for ((key, _) in items) {
val n = segCount[key]
if (n == null) got[key]?.let { res[key] = it }
else {
val sb = StringBuilder()
for (i in 0 until n) {
got[key + "#s" + i]?.let { sb.append(it) }
}
if (sb.isNotEmpty()) res[key] = sb.toString()
}
}
return if (res.isEmpty()) null else res
}
// 复读/退化检测:一半以上译文雷同 → 判退化,保留旧译文
@ -2134,6 +2182,45 @@ class MainActivity : AppCompatActivity() {
return if ((d == "ja" || d == "zh") && (t == "ja" || t == "zh")) 0.88 else 0.72
}
// 长文按句读切成 ≤140 字的段(优先在句末标点断开,兜底硬切)
private fun splitSegments(t: String, max: Int = 140): List<String> {
if (t.length <= max) return listOf(t)
val parts = ArrayList<String>()
val cur = StringBuilder()
for (ch in t) {
cur.append(ch)
if (cur.length >= max * 3 / 4 && (ch == '。' || ch == '!' ||
ch == '?' || ch == '!' || ch == '?' ||
ch == ';' || ch == ';' || ch == '\n')
) {
parts.add(cur.toString()); cur.setLength(0)
}
}
if (cur.isNotBlank()) parts.add(cur.toString())
val out = ArrayList<String>()
for (raw in parts) {
var x = raw
while (x.length > max) {
out.add(x.substring(0, max)); x = x.substring(max)
}
if (x.isNotBlank()) out.add(x)
}
return if (out.isEmpty()) listOf(t) else out
}
// 目标语健全检查(治"译着译着变繁体/变第三种语言"):
// ja 必须含足量假名;zh 不得有假名且繁体专用字 ≤1;en 不得含 CJK。
private val tradOnlyChars = "們這來說對學體豐華讓裡國變顯點單據觀"
private fun targetLangOk(tr: String): Boolean {
if (tr.length < 12) return true
return when (uiLanguage()) {
"ja" -> tr.count { it.code in 0x3040..0x30FF } * 25 >= tr.length
"zh" -> tr.none { it.code in 0x3040..0x30FF } &&
tr.count { c -> tradOnlyChars.contains(c) } <= 1
else -> tr.none { it.code >= 0x2E80 }
}
}
private fun isEcho(src: String, tr: String, th: Double = 0.72): Boolean {
val a = normalizeForEcho(src)
val b = normalizeForEcho(tr)