翻译复读治理: 初译批次4→2、逐条isEcho复读检测(归一化+二元组Jaccard)拒写抄原文、二次翻译同规则、提示词显式禁抄并允许空串
This commit is contained in:
parent
2e1de4da48
commit
da45878c54
|
|
@ -309,7 +309,7 @@ class MainActivity : AppCompatActivity() {
|
||||||
// KV 污染 workaround(2026-09-14,不改 .so):nativeGenerate 不清 KV,
|
// KV 污染 workaround(2026-09-14,不改 .so):nativeGenerate 不清 KV,
|
||||||
// 同引擎连续 generate 会复读上一条 → 每"批"generate 前重建引擎。
|
// 同引擎连续 generate 会复读上一条 → 每"批"generate 前重建引擎。
|
||||||
// 初译从逐句改为小批量编号翻译(凑 4 句或 2.5s 定时封批)。
|
// 初译从逐句改为小批量编号翻译(凑 4 句或 2.5s 定时封批)。
|
||||||
private val initialBatchMax = 4
|
private val initialBatchMax = 2 // 4 句批后期句常退化成抄原文;2 句批明显更稳
|
||||||
private val initialBatchDelayMs = 2_500L
|
private val initialBatchDelayMs = 2_500L
|
||||||
private val initialPending = ArrayList<Pair<Long, String>>()
|
private val initialPending = ArrayList<Pair<Long, String>>()
|
||||||
private var initialBatchTimer: Runnable? = null
|
private var initialBatchTimer: Runnable? = null
|
||||||
|
|
@ -1935,7 +1935,9 @@ class MainActivity : AppCompatActivity() {
|
||||||
uiHandler.post {
|
uiHandler.post {
|
||||||
val touched = HashSet<SubtitleLine>()
|
val touched = HashSet<SubtitleLine>()
|
||||||
refs.forEach { s2 ->
|
refs.forEach { s2 ->
|
||||||
map[s2.id.toString()]?.takeIf { it.isNotBlank() }?.let { tr ->
|
map[s2.id.toString()]?.takeIf {
|
||||||
|
it.isNotBlank() && !isEcho(s2.text, it)
|
||||||
|
}?.let { tr ->
|
||||||
if (s2.trans != tr) {
|
if (s2.trans != tr) {
|
||||||
s2.trans = tr
|
s2.trans = tr
|
||||||
s2.line?.let { touched.add(it) }
|
s2.line?.let { touched.add(it) }
|
||||||
|
|
@ -1995,8 +1997,10 @@ class MainActivity : AppCompatActivity() {
|
||||||
if (map == null || !plausibleTranslations(map.values)) return@post
|
if (map == null || !plausibleTranslations(map.values)) return@post
|
||||||
uiHandler.post {
|
uiHandler.post {
|
||||||
val touched = HashSet<SubtitleLine>()
|
val touched = HashSet<SubtitleLine>()
|
||||||
batch.forEach { (sid, _) ->
|
batch.forEach { (sid, src) ->
|
||||||
map[sid.toString()]?.takeIf { it.isNotBlank() }?.let { tr ->
|
map[sid.toString()]?.takeIf {
|
||||||
|
it.isNotBlank() && !isEcho(src, it)
|
||||||
|
}?.let { tr ->
|
||||||
findSentence(sid)?.let { s2 ->
|
findSentence(sid)?.let { s2 ->
|
||||||
if (s2.trans != tr) {
|
if (s2.trans != tr) {
|
||||||
s2.trans = tr
|
s2.trans = tr
|
||||||
|
|
@ -2028,7 +2032,9 @@ class MainActivity : AppCompatActivity() {
|
||||||
systemPrompt = "你是专业翻译。逐句翻译下面编号句子为" + targetName +
|
systemPrompt = "你是专业翻译。逐句翻译下面编号句子为" + targetName +
|
||||||
"。各句相互独立,必须逐句对应翻译,不得合并、不得重复、不得解释。" +
|
"。各句相互独立,必须逐句对应翻译,不得合并、不得重复、不得解释。" +
|
||||||
"只输出JSON对象,键是编号字符串,值是译文," +
|
"只输出JSON对象,键是编号字符串,值是译文," +
|
||||||
"例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。",
|
"例如 {\"1\":\"…\",\"2\":\"…\"}。不得增删编号。" +
|
||||||
|
"严禁把原句原样作为译文输出;确实无法翻译的句子," +
|
||||||
|
"值输出空字符串。",
|
||||||
userPrompt = "【句子】\n$numbered",
|
userPrompt = "【句子】\n$numbered",
|
||||||
maxNewTokens = (160 * items.size).coerceAtMost(900),
|
maxNewTokens = (160 * items.size).coerceAtMost(900),
|
||||||
temperature = 0f,
|
temperature = 0f,
|
||||||
|
|
@ -2050,6 +2056,30 @@ class MainActivity : AppCompatActivity() {
|
||||||
return v.size < 2 || v.distinct().size * 2 >= v.size
|
return v.size < 2 || v.distinct().size * 2 >= v.size
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 复读检测:译文与原文归一化后全等/长串包含/字符二元组 Jaccard≥0.72
|
||||||
|
// → 判为"把原文抄进译文",该条丢弃(维持原状),不写入字幕
|
||||||
|
private fun normalizeForEcho(x: String): String =
|
||||||
|
x.lowercase().filter { it.isLetterOrDigit() || it.code >= 0x3040 }
|
||||||
|
|
||||||
|
private fun isEcho(src: String, tr: String): Boolean {
|
||||||
|
val a = normalizeForEcho(src)
|
||||||
|
val b = normalizeForEcho(tr)
|
||||||
|
if (a.isEmpty() || b.isEmpty()) return false
|
||||||
|
if (a == b) return true
|
||||||
|
if (a.length >= 8 && (a.contains(b) || b.contains(a))) return true
|
||||||
|
if (a.length < 6 || b.length < 6) return false
|
||||||
|
fun bigrams(x: String): HashSet<String> {
|
||||||
|
val set = HashSet<String>()
|
||||||
|
for (i in 0 until x.length - 1) set.add(x.substring(i, i + 2))
|
||||||
|
return set
|
||||||
|
}
|
||||||
|
val ba = bigrams(a); val bb = bigrams(b)
|
||||||
|
if (ba.isEmpty() || bb.isEmpty()) return false
|
||||||
|
val inter = bb.count { ba.contains(it) }
|
||||||
|
val union = ba.size + bb.size - inter
|
||||||
|
return union > 0 && inter.toDouble() / union >= 0.72
|
||||||
|
}
|
||||||
|
|
||||||
private val unloadRunnable = Runnable {
|
private val unloadRunnable = Runnable {
|
||||||
val e = translateEngine ?: return@Runnable
|
val e = translateEngine ?: return@Runnable
|
||||||
if (initialPending.isEmpty()) {
|
if (initialPending.isEmpty()) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue