翻译复读治理: 初译批次4→2、逐条isEcho复读检测(归一化+二元组Jaccard)拒写抄原文、二次翻译同规则、提示词显式禁抄并允许空串
This commit is contained in:
parent
2e1de4da48
commit
da45878c54
|
|
@ -309,7 +309,7 @@ class MainActivity : AppCompatActivity() {
|
|||
// KV 污染 workaround(2026-09-14,不改 .so):nativeGenerate 不清 KV,
|
||||
// 同引擎连续 generate 会复读上一条 → 每"批"generate 前重建引擎。
|
||||
// 初译从逐句改为小批量编号翻译(凑 4 句或 2.5s 定时封批)。
|
||||
private val initialBatchMax = 4
|
||||
private val initialBatchMax = 2 // 4 句批后期句常退化成抄原文;2 句批明显更稳
|
||||
private val initialBatchDelayMs = 2_500L
|
||||
private val initialPending = ArrayList<Pair<Long, String>>()
|
||||
private var initialBatchTimer: Runnable? = null
|
||||
|
|
@ -1935,7 +1935,9 @@ class MainActivity : AppCompatActivity() {
|
|||
uiHandler.post {
|
||||
val touched = HashSet<SubtitleLine>()
|
||||
refs.forEach { s2 ->
|
||||
map[s2.id.toString()]?.takeIf { it.isNotBlank() }?.let { tr ->
|
||||
map[s2.id.toString()]?.takeIf {
|
||||
it.isNotBlank() && !isEcho(s2.text, it)
|
||||
}?.let { tr ->
|
||||
if (s2.trans != tr) {
|
||||
s2.trans = tr
|
||||
s2.line?.let { touched.add(it) }
|
||||
|
|
@ -1995,8 +1997,10 @@ class MainActivity : AppCompatActivity() {
|
|||
if (map == null || !plausibleTranslations(map.values)) return@post
|
||||
uiHandler.post {
|
||||
val touched = HashSet<SubtitleLine>()
|
||||
batch.forEach { (sid, _) ->
|
||||
map[sid.toString()]?.takeIf { it.isNotBlank() }?.let { tr ->
|
||||
batch.forEach { (sid, src) ->
|
||||
map[sid.toString()]?.takeIf {
|
||||
it.isNotBlank() && !isEcho(src, it)
|
||||
}?.let { tr ->
|
||||
findSentence(sid)?.let { s2 ->
|
||||
if (s2.trans != tr) {
|
||||
s2.trans = tr
|
||||
|
|
@ -2028,7 +2032,9 @@ class MainActivity : AppCompatActivity() {
|
|||
systemPrompt = "你是专业翻译。逐句翻译下面编号句子为" + targetName +
|
||||
"。各句相互独立,必须逐句对应翻译,不得合并、不得重复、不得解释。" +
|
||||
"只输出JSON对象,键是编号字符串,值是译文," +
|
||||
"例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。",
|
||||
"例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。" +
|
||||
"严禁把原句原样作为译文输出;确实无法翻译的句子," +
|
||||
"值输出空字符串。",
|
||||
userPrompt = "【句子】\n$numbered",
|
||||
maxNewTokens = (160 * items.size).coerceAtMost(900),
|
||||
temperature = 0f,
|
||||
|
|
@ -2050,6 +2056,30 @@ class MainActivity : AppCompatActivity() {
|
|||
return v.size < 2 || v.distinct().size * 2 >= v.size
|
||||
}
|
||||
|
||||
// 复读检测:译文与原文归一化后全等/长串包含/字符二元组 Jaccard≥0.72
|
||||
// → 判为"把原文抄进译文",该条丢弃(维持原状),不写入字幕
|
||||
private fun normalizeForEcho(x: String): String =
|
||||
x.lowercase().filter { it.isLetterOrDigit() || it.code >= 0x3040 }
|
||||
|
||||
private fun isEcho(src: String, tr: String): Boolean {
|
||||
val a = normalizeForEcho(src)
|
||||
val b = normalizeForEcho(tr)
|
||||
if (a.isEmpty() || b.isEmpty()) return false
|
||||
if (a == b) return true
|
||||
if (a.length >= 8 && (a.contains(b) || b.contains(a))) return true
|
||||
if (a.length < 6 || b.length < 6) return false
|
||||
fun bigrams(x: String): HashSet<String> {
|
||||
val set = HashSet<String>()
|
||||
for (i in 0 until x.length - 1) set.add(x.substring(i, i + 2))
|
||||
return set
|
||||
}
|
||||
val ba = bigrams(a); val bb = bigrams(b)
|
||||
if (ba.isEmpty() || bb.isEmpty()) return false
|
||||
val inter = bb.count { ba.contains(it) }
|
||||
val union = ba.size + bb.size - inter
|
||||
return union > 0 && inter.toDouble() / union >= 0.72
|
||||
}
|
||||
|
||||
private val unloadRunnable = Runnable {
|
||||
val e = translateEngine ?: return@Runnable
|
||||
if (initialPending.isEmpty()) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue