语言漂移治理: 长句拆段≤140字段级编号一次生成再拼回(治简→繁→异语中途掉线)、目标语健全检查(ja须假名/zh须简体/en无CJK)、漂移带温度0.3重试一次、提示词注入目标语硬约束、MISS langdrift/seg-drift日志
This commit is contained in:
parent
902084e175
commit
679b88bec3
|
|
@ -1946,6 +1946,8 @@ class MainActivity : AppCompatActivity() {
|
||||||
Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-empty")
|
Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-empty")
|
||||||
isEcho(s2.text, tr, echoJaccardTh(s2.text)) ->
|
isEcho(s2.text, tr, echoJaccardTh(s2.text)) ->
|
||||||
Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-echo")
|
Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-echo")
|
||||||
|
!targetLangOk(tr) ->
|
||||||
|
Log.i(TAG, "MISS sid=" + s2.id + " reason=seg-drift")
|
||||||
else -> if (s2.trans != tr) {
|
else -> if (s2.trans != tr) {
|
||||||
s2.trans = tr
|
s2.trans = tr
|
||||||
s2.line?.let { touched.add(it) }
|
s2.line?.let { touched.add(it) }
|
||||||
|
|
@ -2020,99 +2022,145 @@ class MainActivity : AppCompatActivity() {
|
||||||
if (initialPending.isEmpty() || translateDisabled) return
|
if (initialPending.isEmpty() || translateDisabled) return
|
||||||
val batch = ArrayList(initialPending)
|
val batch = ArrayList(initialPending)
|
||||||
initialPending.clear()
|
initialPending.clear()
|
||||||
translateHandler.post {
|
translateHandler.post { runTranslateBatch(batch, 0f) }
|
||||||
val map = translateNumbered(batch.map {
|
}
|
||||||
Pair(it.first.toString(), it.second)
|
|
||||||
})
|
// 翻译线程执行:直出/编号批 → 逐条质检(空/复读/语言漂移)→ 主线程应用。
|
||||||
|
// 生成失败或语言漂移 → 带温度重试一次(每句仅一次,retriedSids 全部在本线程读写)。
|
||||||
|
private fun runTranslateBatch(batch: List<Pair<Long, String>>, temp: Float) {
|
||||||
|
val map = translateNumbered(
|
||||||
|
batch.map { Pair(it.first.toString(), it.second) }, temp
|
||||||
|
)
|
||||||
if (map == null || !plausibleTranslations(map.values)) {
|
if (map == null || !plausibleTranslations(map.values)) {
|
||||||
Log.i(TAG, "MISS reason=genfail sids=" +
|
Log.i(TAG, "MISS reason=genfail sids=" +
|
||||||
batch.joinToString(",") { it.first.toString() })
|
batch.joinToString(",") { it.first.toString() })
|
||||||
// 失败自动重试一次(仅一次;判重在主线程做,避免竞态)
|
if (temp == 0f) {
|
||||||
uiHandler.post {
|
|
||||||
val retry = batch.filter { retriedSids.add(it.first) }
|
val retry = batch.filter { retriedSids.add(it.first) }
|
||||||
if (retry.isNotEmpty()) {
|
if (retry.isNotEmpty()) translateHandler.post {
|
||||||
initialPending.addAll(retry)
|
runTranslateBatch(retry, 0.3f)
|
||||||
flushInitialBatch()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return@post
|
return
|
||||||
}
|
}
|
||||||
uiHandler.post {
|
val good = HashMap<Long, String>()
|
||||||
val touched = HashSet<SubtitleLine>()
|
val drift = ArrayList<Pair<Long, String>>()
|
||||||
batch.forEach { (sid, src) ->
|
for ((sid, src) in batch) {
|
||||||
val tr = map[sid.toString()]
|
val tr = map[sid.toString()]
|
||||||
when {
|
when {
|
||||||
tr == null || tr.isBlank() ->
|
tr == null || tr.isBlank() ->
|
||||||
Log.i(TAG, "MISS sid=$sid reason=empty")
|
Log.i(TAG, "MISS sid=" + sid + " reason=empty")
|
||||||
isEcho(src, tr, echoJaccardTh(src)) ->
|
isEcho(src, tr, echoJaccardTh(src)) ->
|
||||||
Log.i(TAG, "MISS sid=$sid reason=echo")
|
Log.i(TAG, "MISS sid=" + sid + " reason=echo")
|
||||||
else -> findSentence(sid)?.let { s2 ->
|
!targetLangOk(tr) -> {
|
||||||
|
Log.i(TAG, "MISS sid=" + sid + " reason=langdrift")
|
||||||
|
if (temp == 0f && retriedSids.add(sid)) drift.add(Pair(sid, src))
|
||||||
|
}
|
||||||
|
else -> good[sid] = tr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
uiHandler.post {
|
||||||
|
val touched = HashSet<SubtitleLine>()
|
||||||
|
good.forEach { (sid, tr) ->
|
||||||
|
findSentence(sid)?.let { s2 ->
|
||||||
if (s2.trans != tr) {
|
if (s2.trans != tr) {
|
||||||
s2.trans = tr
|
s2.trans = tr
|
||||||
s2.line?.let { touched.add(it) }
|
s2.line?.let { touched.add(it) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
touched.forEach { notifyLine(it) }
|
touched.forEach { notifyLine(it) }
|
||||||
}
|
}
|
||||||
|
if (drift.isNotEmpty()) translateHandler.post {
|
||||||
|
runTranslateBatch(drift, 0.3f)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 一批 = 一个全新引擎(nativeGenerate 不清 KV 的 workaround)。
|
// 一批 = 一个全新引擎(nativeGenerate 不清 KV 的 workaround)。
|
||||||
// items: key(通常是 sid 字符串) × 原文;返回 key × 译文,失败 null。
|
// items: key(通常是 sid 字符串) × 原文;返回 key × 译文,失败 null。
|
||||||
private fun translateNumbered(items: List<Pair<String, String>>): Map<String, String>? {
|
private fun translateNumbered(
|
||||||
|
items: List<Pair<String, String>>, temp: Float = 0f,
|
||||||
|
): Map<String, String>? {
|
||||||
if (items.isEmpty()) return emptyMap()
|
if (items.isEmpty()) return emptyMap()
|
||||||
val targetName = when (uiLanguage()) {
|
val targetName = when (uiLanguage()) {
|
||||||
"ja" -> "日本語"; "en" -> "English"; else -> "简体中文"
|
"ja" -> "日本語"; "en" -> "English"; else -> "简体中文"
|
||||||
}
|
}
|
||||||
|
val langHint = when (uiLanguage()) {
|
||||||
|
"ja" -> "译文必须是日语(须含假名,不要输出中文或繁体)"
|
||||||
|
"zh" -> "译文必须是简体中文(不得用繁体字,不要输出日语或英语)"
|
||||||
|
else -> "The translation must be English only (no CJK characters)"
|
||||||
|
}
|
||||||
translateEngine?.let { runCatching { it.release() } }
|
translateEngine?.let { runCatching { it.release() } }
|
||||||
translateEngine = null
|
translateEngine = null
|
||||||
val engine = ensureTranslateEngine()
|
val engine = ensureTranslateEngine()
|
||||||
?: run { translateDisabled = true; return null }
|
?: run { translateDisabled = true; return null }
|
||||||
// 动态预算:按原文总长给输出额度(译文常比原文长,原 160/句 会把
|
// 长句拆段:小模型对短输入才稳定;长句译到中途会"掉语言"
|
||||||
// 长句截断成残缺 JSON,整批被静默丢弃——"第一句不翻"的直接死因)
|
// (简体→繁体→别的语言)。拆 ≤140 字段一次 generate,段序拼回。
|
||||||
val budget = (items.sumOf { it.second.length } * 2 + 96)
|
val segKeys = ArrayList<Pair<String, String>>()
|
||||||
|
val segCount = HashMap<String, Int>()
|
||||||
|
for ((key, t) in items) {
|
||||||
|
val segs = splitSegments(t)
|
||||||
|
if (segs.size > 1) segCount[key] = segs.size
|
||||||
|
segs.forEachIndexed { i, sg ->
|
||||||
|
val sk = if (segs.size == 1) key else key + "#s" + i
|
||||||
|
segKeys.add(sk to sg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val budget = (segKeys.sumOf { it.second.length } * 2 + 96)
|
||||||
.coerceAtMost(1200)
|
.coerceAtMost(1200)
|
||||||
// 单句 = 直出模式:不套 JSON 包装,无格式可坏,截断也仍是有效译文
|
val got = HashMap<String, String>()
|
||||||
if (items.size == 1) {
|
if (segKeys.size == 1) {
|
||||||
val one = try {
|
val one = try {
|
||||||
engine.generate(
|
engine.generate(
|
||||||
systemPrompt = "你是专业翻译。把给出的句子翻译成" + targetName +
|
systemPrompt = "你是专业翻译。把给出的句子翻译成" + targetName +
|
||||||
"。只输出译文本身,不要解释、不要引号、不要抄写原文。",
|
"。只输出译文本身,不要解释、不要引号、不要抄写原文。" +
|
||||||
userPrompt = items[0].second,
|
langHint + "。",
|
||||||
maxNewTokens = budget, temperature = 0f,
|
userPrompt = segKeys[0].second,
|
||||||
|
maxNewTokens = budget, temperature = temp,
|
||||||
)
|
)
|
||||||
} catch (t: Throwable) {
|
} catch (t: Throwable) {
|
||||||
Log.w(TAG, "TRANSLATE single failed: ${t.message}"); null
|
Log.w(TAG, "TRANSLATE single failed: " + t.message); null
|
||||||
}
|
}
|
||||||
val tr = one?.trim()?.lines()?.firstOrNull { it.isNotBlank() }?.trim()
|
val tr = one?.trim()?.lines()?.firstOrNull { it.isNotBlank() }?.trim()
|
||||||
return if (tr.isNullOrEmpty()) null
|
if (!tr.isNullOrEmpty()) got[segKeys[0].first] = tr
|
||||||
else mapOf(items[0].first to tr)
|
if (got.isEmpty()) return null
|
||||||
}
|
} else {
|
||||||
val numbered = items.mapIndexed { i, (_, t) -> "${i + 1}. $t" }
|
val numbered = segKeys.mapIndexed { i, st ->
|
||||||
.joinToString("\n")
|
(i + 1).toString() + ". " + st.second
|
||||||
|
}.joinToString("\n")
|
||||||
val out = try {
|
val out = try {
|
||||||
engine.generate(
|
engine.generate(
|
||||||
systemPrompt = "你是专业翻译。逐句翻译下面编号句子为" + targetName +
|
systemPrompt = "你是专业翻译。逐段翻译下面编号文本为" + targetName +
|
||||||
"。各句相互独立,必须逐句对应翻译,不得合并、不得重复、不得解释。" +
|
"。各段相互独立,逐段对应,不得合并、不得重复、不得解释。" +
|
||||||
"只输出JSON对象,键是编号字符串,值是译文," +
|
"只输出JSON对象,键是编号字符串,值是译文," +
|
||||||
"例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。" +
|
"例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。" +
|
||||||
"严禁把原句原样作为译文输出;确实无法翻译的句子," +
|
"严禁把原文照抄为译文;无法翻译的段值输出空字符串。" +
|
||||||
"值输出空字符串。",
|
langHint + "。",
|
||||||
userPrompt = "【句子】\n$numbered",
|
userPrompt = "【文本】\n" + numbered,
|
||||||
maxNewTokens = budget,
|
maxNewTokens = budget, temperature = temp,
|
||||||
temperature = 0f,
|
|
||||||
)
|
)
|
||||||
} catch (t: Throwable) {
|
} catch (t: Throwable) {
|
||||||
Log.w(TAG, "TRANSLATE batch failed: ${t.message}"); null
|
Log.w(TAG, "TRANSLATE batch failed: " + t.message); null
|
||||||
}
|
}
|
||||||
val byIdx = extractNumberedPairs(out) ?: return null
|
val byIdx = extractNumberedPairs(out) ?: return null
|
||||||
val res = HashMap<String, String>()
|
segKeys.forEachIndexed { i, st ->
|
||||||
items.forEachIndexed { i, (key, _) ->
|
byIdx[(i + 1).toString()]?.takeIf { it.isNotBlank() }
|
||||||
byIdx[(i + 1).toString()]?.let { res[key] = it }
|
?.let { got[st.first] = it }
|
||||||
}
|
}
|
||||||
return res
|
if (got.isEmpty()) return null
|
||||||
|
}
|
||||||
|
val res = HashMap<String, String>()
|
||||||
|
for ((key, _) in items) {
|
||||||
|
val n = segCount[key]
|
||||||
|
if (n == null) got[key]?.let { res[key] = it }
|
||||||
|
else {
|
||||||
|
val sb = StringBuilder()
|
||||||
|
for (i in 0 until n) {
|
||||||
|
got[key + "#s" + i]?.let { sb.append(it) }
|
||||||
|
}
|
||||||
|
if (sb.isNotEmpty()) res[key] = sb.toString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return if (res.isEmpty()) null else res
|
||||||
}
|
}
|
||||||
|
|
||||||
// 复读/退化检测:一半以上译文雷同 → 判退化,保留旧译文
|
// 复读/退化检测:一半以上译文雷同 → 判退化,保留旧译文
|
||||||
|
|
@ -2134,6 +2182,45 @@ class MainActivity : AppCompatActivity() {
|
||||||
return if ((d == "ja" || d == "zh") && (t == "ja" || t == "zh")) 0.88 else 0.72
|
return if ((d == "ja" || d == "zh") && (t == "ja" || t == "zh")) 0.88 else 0.72
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 长文按句读切成 ≤140 字的段(优先在句末标点断开,兜底硬切)
|
||||||
|
private fun splitSegments(t: String, max: Int = 140): List<String> {
|
||||||
|
if (t.length <= max) return listOf(t)
|
||||||
|
val parts = ArrayList<String>()
|
||||||
|
val cur = StringBuilder()
|
||||||
|
for (ch in t) {
|
||||||
|
cur.append(ch)
|
||||||
|
if (cur.length >= max * 3 / 4 && (ch == '。' || ch == '！' ||
|
||||||
|
ch == '？' || ch == '!' || ch == '?' ||
|
||||||
|
ch == '；' || ch == ';' || ch == '\n')
|
||||||
|
) {
|
||||||
|
parts.add(cur.toString()); cur.setLength(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (cur.isNotBlank()) parts.add(cur.toString())
|
||||||
|
val out = ArrayList<String>()
|
||||||
|
for (raw in parts) {
|
||||||
|
var x = raw
|
||||||
|
while (x.length > max) {
|
||||||
|
out.add(x.substring(0, max)); x = x.substring(max)
|
||||||
|
}
|
||||||
|
if (x.isNotBlank()) out.add(x)
|
||||||
|
}
|
||||||
|
return if (out.isEmpty()) listOf(t) else out
|
||||||
|
}
|
||||||
|
|
||||||
|
// 目标语健全检查(治"译着译着变繁体/变第三种语言"):
|
||||||
|
// ja 必须含足量假名;zh 不得有假名且繁体专用字 ≤1;en 不得含 CJK。
|
||||||
|
private val tradOnlyChars = "們這來說對學體豐華讓裡國變顯點單據觀"
|
||||||
|
private fun targetLangOk(tr: String): Boolean {
|
||||||
|
if (tr.length < 12) return true
|
||||||
|
return when (uiLanguage()) {
|
||||||
|
"ja" -> tr.count { it.code in 0x3040..0x30FF } * 25 >= tr.length
|
||||||
|
"zh" -> tr.none { it.code in 0x3040..0x30FF } &&
|
||||||
|
tr.count { c -> tradOnlyChars.contains(c) } <= 1
|
||||||
|
else -> tr.none { it.code >= 0x2E80 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun isEcho(src: String, tr: String, th: Double = 0.72): Boolean {
|
private fun isEcho(src: String, tr: String, th: Double = 0.72): Boolean {
|
||||||
val a = normalizeForEcho(src)
|
val a = normalizeForEcho(src)
|
||||||
val b = normalizeForEcho(tr)
|
val b = normalizeForEcho(tr)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue