From 679b88bec3166fd996f1b22b5b088eb66ad3409b Mon Sep 17 00:00:00 2001 From: liwenhao2 Date: Tue, 15 Sep 2026 10:46:20 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=AD=E8=A8=80=E6=BC=82=E7=A7=BB=E6=B2=BB?= =?UTF-8?q?=E7=90=86:=20=E9=95=BF=E5=8F=A5=E6=8B=86=E6=AE=B5=E2=89=A4140?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E7=BA=A7=E7=BC=96=E5=8F=B7=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E7=94=9F=E6=88=90=E5=86=8D=E6=8B=BC=E5=9B=9E(=E6=B2=BB?= =?UTF-8?q?=E7=AE=80=E2=86=92=E7=B9=81=E2=86=92=E5=BC=82=E8=AF=AD=E4=B8=AD?= =?UTF-8?q?=E9=80=94=E6=8E=89=E7=BA=BF)=E3=80=81=E7=9B=AE=E6=A0=87?= =?UTF-8?q?=E8=AF=AD=E5=81=A5=E5=85=A8=E6=A3=80=E6=9F=A5(ja=E9=A1=BB?= =?UTF-8?q?=E5=81=87=E5=90=8D/zh=E9=A1=BB=E7=AE=80=E4=BD=93/en=E6=97=A0CJK?= =?UTF-8?q?)=E3=80=81=E6=BC=82=E7=A7=BB=E5=B8=A6=E6=B8=A9=E5=BA=A60.3?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E4=B8=80=E6=AC=A1=E3=80=81=E6=8F=90=E7=A4=BA?= =?UTF-8?q?=E8=AF=8D=E6=B3=A8=E5=85=A5=E7=9B=AE=E6=A0=87=E8=AF=AD=E7=A1=AC?= =?UTF-8?q?=E7=BA=A6=E6=9D=9F=E3=80=81MISS=20langdrift/seg-drift=E6=97=A5?= =?UTF-8?q?=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/k2fsa/sherpa/onnx/MainActivity.kt | 219 ++++++++++++------ 1 file changed, 153 insertions(+), 66 deletions(-) diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index 6d6b8a9..5438739 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -1946,6 +1946,8 @@ class MainActivity : AppCompatActivity() { Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-empty") isEcho(s2.text, tr, echoJaccardTh(s2.text)) -> Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-echo") + !targetLangOk(tr) -> + Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-drift") else -> if (s2.trans != tr) { s2.trans = tr s2.line?.let { touched.add(it) } @@ -2020,99 +2022,145 @@ class MainActivity : AppCompatActivity() { if (initialPending.isEmpty() || translateDisabled) return val batch = ArrayList(initialPending) initialPending.clear() - translateHandler.post { - val map = translateNumbered(batch.map { - Pair(it.first.toString(), it.second) - }) - if (map == null || !plausibleTranslations(map.values)) { - Log.i(TAG, "MISS reason=genfail sids=" + - batch.joinToString(",") { it.first.toString() }) - // 失败自动重试一次(仅一次;判重在主线程做,避免竞态) - uiHandler.post { - val retry = batch.filter { retriedSids.add(it.first) } - if (retry.isNotEmpty()) { - initialPending.addAll(retry) - flushInitialBatch() + translateHandler.post { runTranslateBatch(batch, 0f) } + } + + // 翻译线程执行:直出/编号批 → 逐条质检(空/复读/语言漂移)→ 主线程应用。 + // 生成失败或语言漂移 → 带温度重试一次(每句仅一次,retriedSids 全部在本线程读写)。 + private fun runTranslateBatch(batch: List>, temp: Float) { + val map = translateNumbered( + batch.map { Pair(it.first.toString(), it.second) }, temp + ) + if (map == null || !plausibleTranslations(map.values)) { + Log.i(TAG, "MISS reason=genfail sids=" + + batch.joinToString(",") { it.first.toString() }) + if (temp == 0f) { + val retry = batch.filter { retriedSids.add(it.first) } + if (retry.isNotEmpty()) translateHandler.post { + runTranslateBatch(retry, 0.3f) + } + } + return + } + val good = HashMap() + val drift = ArrayList>() + for ((sid, src) in batch) { + val tr = map[sid.toString()] + when { + tr == null || tr.isBlank() -> + Log.i(TAG, "MISS sid=" + sid + " reason=empty") + isEcho(src, tr, echoJaccardTh(src)) -> + Log.i(TAG, "MISS sid=" + sid + " reason=echo") + !targetLangOk(tr) -> { + Log.i(TAG, "MISS sid=" + sid + " reason=langdrift") + if (temp == 0f && retriedSids.add(sid)) drift.add(Pair(sid, src)) + } + else -> good[sid] = tr + } + } + uiHandler.post { + val touched = HashSet() + good.forEach { (sid, tr) -> + findSentence(sid)?.let { s2 -> + if (s2.trans != tr) { + s2.trans = tr + s2.line?.let { touched.add(it) } } } - return@post - } - uiHandler.post { - val touched = HashSet() - batch.forEach { (sid, src) -> - val tr = map[sid.toString()] - when { - tr == null || tr.isBlank() -> - Log.i(TAG, "MISS sid=$sid reason=empty") - isEcho(src, tr, echoJaccardTh(src)) -> - Log.i(TAG, "MISS sid=$sid reason=echo") - else -> findSentence(sid)?.let { s2 -> - if (s2.trans != tr) { - s2.trans = tr - s2.line?.let { touched.add(it) } - } - } - } - } - touched.forEach { notifyLine(it) } } + touched.forEach { notifyLine(it) } + } + if (drift.isNotEmpty()) translateHandler.post { + runTranslateBatch(drift, 0.3f) } } // 一批 = 一个全新引擎(nativeGenerate 不清 KV 的 workaround)。 // items: key(通常是 sid 字符串) × 原文;返回 key × 译文,失败 null。 - private fun translateNumbered(items: List>): Map? { + private fun translateNumbered( + items: List>, temp: Float = 0f, + ): Map? { if (items.isEmpty()) return emptyMap() val targetName = when (uiLanguage()) { "ja" -> "日本語"; "en" -> "English"; else -> "简体中文" } + val langHint = when (uiLanguage()) { + "ja" -> "译文必须是日语(须含假名,不要输出中文或繁体)" + "zh" -> "译文必须是简体中文(不得用繁体字,不要输出日语或英语)" + else -> "The translation must be English only (no CJK characters)" + } translateEngine?.let { runCatching { it.release() } } translateEngine = null val engine = ensureTranslateEngine() ?: run { translateDisabled = true; return null } - // 动态预算:按原文总长给输出额度(译文常比原文长,原 160/句 会把 - // 长句截断成残缺 JSON,整批被静默丢弃——"第一句不翻"的直接死因) - val budget = (items.sumOf { it.second.length } * 2 + 96) + // 长句拆段:小模型对短输入才稳定;长句译到中途会"掉语言" + // (简体→繁体→别的语言)。拆 ≤140 字段一次 generate,段序拼回。 + val segKeys = ArrayList>() + val segCount = HashMap() + for ((key, t) in items) { + val segs = splitSegments(t) + if (segs.size > 1) segCount[key] = segs.size + segs.forEachIndexed { i, sg -> + val sk = if (segs.size == 1) key else key + "#s" + i + segKeys.add(sk to sg) + } + } + val budget = (segKeys.sumOf { it.second.length } * 2 + 96) .coerceAtMost(1200) - // 单句 = 直出模式:不套 JSON 包装,无格式可坏,截断也仍是有效译文 - if (items.size == 1) { + val got = HashMap() + if (segKeys.size == 1) { val one = try { engine.generate( systemPrompt = "你是专业翻译。把给出的句子翻译成" + targetName + - "。只输出译文本身,不要解释、不要引号、不要抄写原文。", - userPrompt = items[0].second, - maxNewTokens = budget, temperature = 0f, + "。只输出译文本身,不要解释、不要引号、不要抄写原文。" + + langHint + "。", + userPrompt = segKeys[0].second, + maxNewTokens = budget, temperature = temp, ) } catch (t: Throwable) { - Log.w(TAG, "TRANSLATE single failed: ${t.message}"); null + Log.w(TAG, "TRANSLATE single failed: " + t.message); null } val tr = one?.trim()?.lines()?.firstOrNull { it.isNotBlank() }?.trim() - return if (tr.isNullOrEmpty()) null - else mapOf(items[0].first to tr) + if (!tr.isNullOrEmpty()) got[segKeys[0].first] = tr + if (got.isEmpty()) return null + } else { + val numbered = segKeys.mapIndexed { i, st -> + (i + 1).toString() + ". " + st.second + }.joinToString("\n") + val out = try { + engine.generate( + systemPrompt = "你是专业翻译。逐段翻译下面编号文本为" + targetName + + "。各段相互独立,逐段对应,不得合并、不得重复、不得解释。" + + "只输出JSON对象,键是编号字符串,值是译文," + + "例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。" + + "严禁把原文照抄为译文;无法翻译的段值输出空字符串。" + + langHint + "。", + userPrompt = "【文本】\n" + numbered, + maxNewTokens = budget, temperature = temp, + ) + } catch (t: Throwable) { + Log.w(TAG, "TRANSLATE batch failed: " + t.message); null + } + val byIdx = extractNumberedPairs(out) ?: return null + segKeys.forEachIndexed { i, st -> + byIdx[(i + 1).toString()]?.takeIf { it.isNotBlank() } + ?.let { got[st.first] = it } + } + if (got.isEmpty()) return null } - val numbered = items.mapIndexed { i, (_, t) -> "${i + 1}. $t" } - .joinToString("\n") - val out = try { - engine.generate( - systemPrompt = "你是专业翻译。逐句翻译下面编号句子为" + targetName + - "。各句相互独立,必须逐句对应翻译,不得合并、不得重复、不得解释。" + - "只输出JSON对象,键是编号字符串,值是译文," + - "例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。" + - "严禁把原句原样作为译文输出;确实无法翻译的句子," + - "值输出空字符串。", - userPrompt = "【句子】\n$numbered", - maxNewTokens = budget, - temperature = 0f, - ) - } catch (t: Throwable) { - Log.w(TAG, "TRANSLATE batch failed: ${t.message}"); null - } - val byIdx = extractNumberedPairs(out) ?: return null val res = HashMap() - items.forEachIndexed { i, (key, _) -> - byIdx[(i + 1).toString()]?.let { res[key] = it } + for ((key, _) in items) { + val n = segCount[key] + if (n == null) got[key]?.let { res[key] = it } + else { + val sb = StringBuilder() + for (i in 0 until n) { + got[key + "#s" + i]?.let { sb.append(it) } + } + if (sb.isNotEmpty()) res[key] = sb.toString() + } } - return res + return if (res.isEmpty()) null else res } // 复读/退化检测:一半以上译文雷同 → 判退化,保留旧译文 @@ -2134,6 +2182,45 @@ class MainActivity : AppCompatActivity() { return if ((d == "ja" || d == "zh") && (t == "ja" || t == "zh")) 0.88 else 0.72 } + // 长文按句读切成 ≤140 字的段(优先在句末标点断开,兜底硬切) + private fun splitSegments(t: String, max: Int = 140): List { + if (t.length <= max) return listOf(t) + val parts = ArrayList() + val cur = StringBuilder() + for (ch in t) { + cur.append(ch) + if (cur.length >= max * 3 / 4 && (ch == '。' || ch == '！' || + ch == '？' || ch == '!' || ch == '?' || + ch == '；' || ch == ';' || ch == '\n') + ) { + parts.add(cur.toString()); cur.setLength(0) + } + } + if (cur.isNotBlank()) parts.add(cur.toString()) + val out = ArrayList() + for (raw in parts) { + var x = raw + while (x.length > max) { + out.add(x.substring(0, max)); x = x.substring(max) + } + if (x.isNotBlank()) out.add(x) + } + return if (out.isEmpty()) listOf(t) else out + } + + // 目标语健全检查(治"译着译着变繁体/变第三种语言"): + // ja 必须含足量假名;zh 不得有假名且繁体专用字 ≤1;en 不得含 CJK。 + private val tradOnlyChars = "們這來說對學體豐華讓裡國變顯點單據觀" + private fun targetLangOk(tr: String): Boolean { + if (tr.length < 12) return true + return when (uiLanguage()) { + "ja" -> tr.count { it.code in 0x3040..0x30FF } * 25 >= tr.length + "zh" -> tr.none { it.code in 0x3040..0x30FF } && + tr.count { c -> tradOnlyChars.contains(c) } <= 1 + else -> tr.none { it.code >= 0x2E80 } + } + } + private fun isEcho(src: String, tr: String, th: Double = 0.72): Boolean { val a = normalizeForEcho(src) val b = normalizeForEcho(tr)