翻译复读治理: 初译批次4→2、逐条isEcho复读检测(归一化+二元组Jaccard)拒写抄原文、二次翻译同规则、提示词显式禁抄并允许空串

This commit is contained in:
liwenhao2 2026-09-15 09:57:08 +08:00
parent 2e1de4da48
commit da45878c54
1 changed files with 35 additions and 5 deletions

View File

@ -309,7 +309,7 @@ class MainActivity : AppCompatActivity() {
// KV 污染 workaround(2026-09-14,不改 .so):nativeGenerate 不清 KV,
// 同引擎连续 generate 会复读上一条 → 每"批"generate 前重建引擎。
// 初译从逐句改为小批量编号翻译(凑 4 句或 2.5s 定时封批)。
private val initialBatchMax = 4
private val initialBatchMax = 2 // 4 句批后期句常退化成抄原文;2 句批明显更稳
private val initialBatchDelayMs = 2_500L
private val initialPending = ArrayList<Pair<Long, String>>()
private var initialBatchTimer: Runnable? = null
@ -1935,7 +1935,9 @@ class MainActivity : AppCompatActivity() {
uiHandler.post {
val touched = HashSet<SubtitleLine>()
refs.forEach { s2 ->
map[s2.id.toString()]?.takeIf { it.isNotBlank() }?.let { tr ->
map[s2.id.toString()]?.takeIf {
it.isNotBlank() && !isEcho(s2.text, it)
}?.let { tr ->
if (s2.trans != tr) {
s2.trans = tr
s2.line?.let { touched.add(it) }
@ -1995,8 +1997,10 @@ class MainActivity : AppCompatActivity() {
if (map == null || !plausibleTranslations(map.values)) return@post
uiHandler.post {
val touched = HashSet<SubtitleLine>()
batch.forEach { (sid, _) ->
map[sid.toString()]?.takeIf { it.isNotBlank() }?.let { tr ->
batch.forEach { (sid, src) ->
map[sid.toString()]?.takeIf {
it.isNotBlank() && !isEcho(src, it)
}?.let { tr ->
findSentence(sid)?.let { s2 ->
if (s2.trans != tr) {
s2.trans = tr
@ -2028,7 +2032,9 @@ class MainActivity : AppCompatActivity() {
systemPrompt = "你是专业翻译。逐句翻译下面编号句子为" + targetName +
"。各句相互独立,必须逐句对应翻译,不得合并、不得重复、不得解释。" +
"只输出JSON对象,键是编号字符串,值是译文," +
"例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。",
"例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。" +
"严禁把原句原样作为译文输出;确实无法翻译的句子," +
"值输出空字符串。",
userPrompt = "【句子】\n$numbered",
maxNewTokens = (160 * items.size).coerceAtMost(900),
temperature = 0f,
@ -2050,6 +2056,30 @@ class MainActivity : AppCompatActivity() {
return v.size < 2 || v.distinct().size * 2 >= v.size
}
// 复读检测:译文与原文归一化后全等/长串包含/字符二元组 Jaccard≥0.72
// → 判为"把原文抄进译文",该条丢弃(维持原状),不写入字幕
private fun normalizeForEcho(x: String): String =
x.lowercase().filter { it.isLetterOrDigit() || it.code >= 0x3040 }
private fun isEcho(src: String, tr: String): Boolean {
val a = normalizeForEcho(src)
val b = normalizeForEcho(tr)
if (a.isEmpty() || b.isEmpty()) return false
if (a == b) return true
if (a.length >= 8 && (a.contains(b) || b.contains(a))) return true
if (a.length < 6 || b.length < 6) return false
fun bigrams(x: String): HashSet<String> {
val set = HashSet<String>()
for (i in 0 until x.length - 1) set.add(x.substring(i, i + 2))
return set
}
val ba = bigrams(a); val bb = bigrams(b)
if (ba.isEmpty() || bb.isEmpty()) return false
val inter = bb.count { ba.contains(it) }
val union = ba.size + bb.size - inter
return union > 0 && inter.toDouble() / union >= 0.72
}
private val unloadRunnable = Runnable {
val e = translateEngine ?: return@Runnable
if (initialPending.isEmpty()) {