liwenhao2
/
ASR-Android
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

翻译开头格式残片清理:cleanTrans 新增 transNoisePrefix(围栏符/半截HTML标签/模板词元/html尾巴),与答案标签剥除交替循环直至稳定;现场'html>'=Gemma词表'</html>'尾部BPE词元,困惑态冷启动首词元落残留、后续译文正常全部质检放行导致上屏;剥空→null走重试;TAP-NOISE日志

This commit is contained in:
liwenhao2 2026-09-16 09:43:38 +08:00
parent 0997a21461
commit 5bbdfa19ed
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt
View File

@ -2068,6 +2068,14 @@ class MainActivity : AppCompatActivity() {
"^\\s*(\\*\\*|__|#+\\s*)?\\s*[「\"“'『]?\\s*(以下是?)?\\s*(翻訳|翻译|訳文|译文|" +
"translation|translate)\\s*(结果|結果)?\\s*(如下)?\\s*(\\*\\*|__)?\\s*" +
"(は|的|为|:)?\\s*[：:—\\-]\\s*[」\"”'』]?\\s*", RegexOption.IGNORE_CASE)
// 开头格式残片(2026-09-16 现场:翻译开头出现 "html>"):Gemma 词表里
// "</html>" 的尾巴是独立 BPE 词元,模型困惑态冷启动偶尔第一个词元就
// 落在这类"文档结尾"残留上,后面译文正常、全部质检放行 → 噪声上屏。
// 保守集合:围栏符/半截标签/模板词元字面/html 尾巴;循环里与标签剥除
// 交替直至稳定。剥到空 → null 走重试,不外发。
private val transNoisePrefix = Regex(
"^(`{3,}\\s*[A-Za-z]*>?|</?[A-Za-z0-9_|]{1,18}>|\\.?html?>)",
RegexOption.IGNORE_CASE)
private fun cleanTrans(raw: String?): String? {
if (raw.isNullOrBlank()) return null
@ -2079,6 +2087,12 @@ class MainActivity : AppCompatActivity() {
do {
prev = s
s = transLabelPrefix.replaceFirst(s, "").trim()
val s1 = s
s = transNoisePrefix.replaceFirst(s, "").trim()
if (s != s1) {
Log.i(TAG, "TAP-NOISE stripped: " +
s1.take(16).replace("\n", " "))
}
} while (s != prev && s.isNotEmpty())
if (s.length >= 2 && ((s.startsWith("「") && s.endsWith("」")) ||
(s.startsWith("\"") && s.endsWith("\"")) ||