From 7e7206a8d4965a349b6c44ed61cfcfd94cfe4c43 Mon Sep 17 00:00:00 2001 From: liwenhao2 Date: Thu, 10 Sep 2026 18:26:20 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E6=A8=A1=E5=9E=8B+=E8=AF=B4=E8=AF=9D?= =?UTF-8?q?=E4=BA=BA=E5=8C=BA=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/k2fsa/sherpa/onnx/MainActivity.kt | 368 ++++++++++++++---- .../k2fsa/sherpa/onnx/OfflineRecognizer.kt | 7 +- .../com/k2fsa/sherpa/onnx/OnlineRecognizer.kt | 3 + .../com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt | 70 +++- 4 files changed, 368 insertions(+), 80 deletions(-) diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index 95c3e94..beefd27 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -12,6 +12,7 @@ import android.os.Handler import android.os.HandlerThread import android.os.Looper import android.os.Process +import android.os.SystemClock import android.text.SpannableStringBuilder import android.text.Spanned import android.text.method.ScrollingMovementMethod @@ -99,6 +100,10 @@ class MainActivity : AppCompatActivity() { private var colorSeq = 0 private var lastConfirmedLabel: String? = null // partial 行沿用的最近确认说话人 private var lastSettledSid = -1L // 主线程:最近定格的句子槽位 + // 同人停顿超时封行:距上一句定格 > 该时长再开口,同一人也另起新行 + // (会议记录里算新一轮发言)。仅作用于"最底行接句"判断。 + private val lineIdleTimeoutMs = 60_000L + private var lastSentenceSettledAt = 0L private lateinit var subtitleRecycler: RecyclerView private lateinit var subtitleAdapter: SubtitleAdapter @@ -142,6 +147,8 @@ class MainActivity : AppCompatActivity() { private val speakerThreshold = 0.5f private var speakersEnabled = false private var lastEmbedding: FloatArray? = null + // 同一句话在 fast(轻量)空间的镜像嵌入,命名时双写进 fastRegistry + private var lastEmbeddingFast: FloatArray? = null private lateinit var nameSpeakerButton: Button private lateinit var clearSpeakersButton: Button @@ -150,11 +157,26 @@ class MainActivity : AppCompatActivity() { // compares it (cosine) with the window reference taken at the start of // the current utterance. A big similarity drop == voice changed mid // utterance == force an endpoint cut at the switch point. - private class SpeakerSwitch(val label: String?, val cutSamples: Int) + private class SpeakerSwitch( + val label: String?, + val cutSamples: Int, + val ownerSid: Long, // 判定属于哪句话;消费时句已结算则作废 + ) private val speakerCheckThread = HandlerThread("speaker-window").also { it.start() } private val speakerCheckHandler by lazy { Handler(speakerCheckThread.looper) } - private val speakerSwitchDrop = 0.35f // cos(window, reference) below this => switch + private val speakerSwitchDrop = 0.30f // cos(window, reference) below this => switch + // 滚动检测在 fast 空间查"这人是谁"仅用于**显示标签**,阈值放宽到 0.4; + // endpoint 权威判定仍用准确模型 + speakerThreshold(0.5)。 + private val fastSearchThreshold = 0.4f + // 最速档:1.2s 周期 + 单票(switchStrikesNeeded=1)。误断的代价只是 + // 多分行+流重置,不丢音频(快照平铺保证时间轴连续)。 + private val switchStrikesNeeded = 1 + private val minWindowRms = 0.004f // 静音主导窗不做判定(embedding 不可靠) + private val enableRollingSwitch = true // A/B 诊断:关掉=只用自然 endpoint 断句 + @Volatile private var rollingLowHits = 0 + // C(最新者胜):每次投递滚动检测都刷新序号,队列里被超额的旧任务直接跳过 + @Volatile private var rollingSeq = 0L @Volatile private var pendingSwitch: SpeakerSwitch? = null @Volatile private var windowRefEmb: FloatArray? = null @Volatile private var activeLabelForWindow: String? = null @@ -240,6 +262,7 @@ class MainActivity : AppCompatActivity() { clearSpeakersButton.setOnClickListener { SpeakerRecognition.resetSpeakers() lastEmbedding = null + lastEmbeddingFast = null lastLabel = null lastConfirmedLabel = null activeLabelForWindow = null @@ -260,8 +283,9 @@ class MainActivity : AppCompatActivity() { subtitleHandler.post { Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND) } + // D:声纹检测线程优先级高于定稿线程(nano),换行判定不被定稿挤慢 speakerCheckHandler.post { - Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND) + Process.setThreadPriority(Process.THREAD_PRIORITY_DEFAULT) } // Loading ~1.6 GB of ONNX weights takes tens of seconds. Do it OFF the @@ -276,6 +300,11 @@ class MainActivity : AppCompatActivity() { initOfflineRecognizer() setStatus("正在加载声纹模型…") initSpeakerIdentification() + if (speakersEnabled) { + initFastSpeakerModel( + application.assets.list("").orEmpty() + ) + } modelsReady = true runOnUiThread { recordButton.isEnabled = true } val speakerNote = if (speakersEnabled) "说话人功能已启用" @@ -349,6 +378,7 @@ class MainActivity : AppCompatActivity() { resetBuffer() pendingSwitch = null windowRefEmb = null + rollingLowHits = 0 checkSerial++ activeLabelForWindow = null lastConfirmedLabel = null @@ -389,8 +419,10 @@ class MainActivity : AppCompatActivity() { var sentenceId = -1L // rolling speaker-check state (recording-thread only) - val window = RollingWindow(sampleRateInHz * 2) // last 2 s, 0.5 s overlap - val checkStep = (sampleRateInHz * 1.5).toInt() // re-check every 1.5 s + // v8.1 定点回滚:窗口 2s / 周期 1.2s(v8 的 1.2s/0.7s 单票导致误断 + // 刷屏灰"未识别");单票保留,配合误断防护使用 + val window = RollingWindow(sampleRateInHz * 2) + val checkStep = (sampleRateInHz * 1.2f).toInt() var newSinceCheck = 0 while (isRecording) { @@ -400,31 +432,71 @@ class MainActivity : AppCompatActivity() { pushChunk(samples) window.push(samples) - if (speakersEnabled && sentenceId != -1L) { + if (enableRollingSwitch && speakersEnabled && sentenceId != -1L) { newSinceCheck += ret if (newSinceCheck >= checkStep) { newSinceCheck = 0 val win = window.snapshot() val cutAt = bufferTotal val seq = checkSerial - speakerCheckHandler.post { - if (seq != checkSerial || pendingSwitch != null) return@post - val emb = SpeakerRecognition.computeEmbedding( - win, sampleRateInHz - ) ?: return@post - val ref = windowRefEmb - if (ref == null) { - // first window of this utterance becomes the - // voice reference - windowRefEmb = emb - } else if (SpeakerRecognition.similarity(emb, ref) - < speakerSwitchDrop - ) { - val newLabel = SpeakerRecognition.registry.search( - emb, speakerThreshold - ) - if (pendingSwitch == null) { - pendingSwitch = SpeakerSwitch(newLabel, cutAt) + val owner = sentenceId + if (rmsOf(win) >= minWindowRms) { + val rseq = ++rollingSeq + speakerCheckHandler.post { + // C:被更新的检测投递超过 → 直接作废(防积压) + if (rseq != rollingSeq) return@post + if (seq != checkSerial || pendingSwitch != null) return@post + // E:滚动检测走轻量模型(base),推理快 ~4 倍; + // 匹配也只在 fast 空间的镜像 registry 里查 + val emb = SpeakerRecognition.computeEmbeddingFast( + win, sampleRateInHz + ) ?: return@post + if (rseq != rollingSeq) return@post + // 计算耗时内句子可能已结算:回写前二次核销 + if (seq != checkSerial) return@post + val ref = windowRefEmb + if (ref == null) { + // first window of this utterance becomes the + // voice reference + windowRefEmb = emb + rollingLowHits = 0 + } else { + val sim = SpeakerRecognition.similarity(emb, ref) + if (sim >= speakerSwitchDrop) { + // 同人缓漂:参考嵌入 EMA 慢速更新 + windowRefEmb = blendEmb(ref, emb) + rollingLowHits = 0 + } else { + rollingLowHits += 1 + if (rollingLowHits >= switchStrikesNeeded && + pendingSwitch == null + ) { + rollingLowHits = 0 + val newLabel = + rollingSpeakerSearch(emb) + val active = activeLabelForWindow + when { + // ③ 误断防护:查出来就是当前人 → + // 相似度毛刺,不断行,参考回拉 + newLabel != null && + active != null && + newLabel == active -> { + windowRefEmb = + blendEmb(ref, emb) + } + // ③ 当前行还没归属、而窗口恰好匹配到 + // 某个已知人 → 直接冠名,不必断行 + newLabel != null && active == null -> { + uiHandler.post { + adoptActiveLabel(newLabel) + } + } + else -> pendingSwitch = SpeakerSwitch( + newLabel, cutAt, owner + ) + } + } + } } } } @@ -438,10 +510,15 @@ class MainActivity : AppCompatActivity() { // ---- 声纹判定换人:强制 endpoint(需求一.4a/b/c) ---- val ps = pendingSwitch - if (ps != null) { + if (ps != null && ps.ownerSid == sentenceId) { pendingSwitch = null checkSerial++ // drop in-flight window jobs val oldLabel = activeLabelForWindow // 被切断句子的归属人 + Log.i(TAG, "speaker-switch FIRED newLabel=" + + (ps.label?.let { displayName(it) } ?: "未识别") + + " owner=sid${ps.ownerSid} cut@" + ps.cutSamples + + " (active=" + (oldLabel?.let { displayName(it) } + ?: "null") + ")") val size = samplesBuffer.size if (size == 0) { uiHandler.post { onSpeakerSwitched(ps.label) } @@ -472,6 +549,7 @@ class MainActivity : AppCompatActivity() { val sid = sentenceId sentenceId = -1L windowRefEmb = null + rollingLowHits = 0 newSinceCheck = 0 if (sid != -1L) { @@ -497,6 +575,11 @@ class MainActivity : AppCompatActivity() { dropPrefixChunks(k) // 切换点之后的音频留在缓冲区,归入新说话人(需求一.4c) } + } else if (ps != null) { + // 判定属于已结算的旧句子:纯过期,直接丢弃 —— + // 不 reset、不丢音频(v5 误断第一遍的根因修复) + pendingSwitch = null + Log.i(TAG, "SPEAKER-SWITCH ignored: stale owner=${ps.ownerSid} current=$sentenceId") } val isEndpoint = onlineRecognizer.isEndpoint(stream) @@ -510,6 +593,7 @@ class MainActivity : AppCompatActivity() { if (sentenceId == -1L) { sentenceId = lineIdSeq.incrementAndGet() windowRefEmb = null + rollingLowHits = 0 } val sid = sentenceId uiHandler.post { onPartial(sid, text) } @@ -522,13 +606,16 @@ class MainActivity : AppCompatActivity() { val sid = sentenceId sentenceId = -1L windowRefEmb = null + rollingLowHits = 0 newSinceCheck = 0 checkSerial++ val snap = flattenSamples() // fresh copy, 绝不共享引用 val dur = bufferTotal / sampleRateInHz.toFloat() val prompt = rawAll // 未过滤底稿作提示(需求二:不丢词) - keepTailChunks(5) // 0.5 s 尾作下一片段上下文 + // 时间轴严格平铺:本段全部音频已交给 snap,缓冲区清零, + // 下一段从该断点开始(不再回含 0.5s 重叠) + resetBuffer() // 整句声纹判定移出录音线程(停摆根因):后台算完 // 回主线程 confirmSpeaker 修正归属 @@ -536,14 +623,30 @@ class MainActivity : AppCompatActivity() { val emb = SpeakerRecognition.computeEmbedding( snap, sampleRateInHz ) + // 镜像嵌入:② 用句尾 2s 切片(与滚动检测同视角), + // 整句 embedding 会拉低窗口匹配度,导致换行时查不到人 + val fastEmb = if (emb != null && + SpeakerRecognition.fastReady() + ) { + val tailSlice = if (snap.size > 32000) + snap.copyOfRange( + snap.size - 32000, snap.size + ) else snap + SpeakerRecognition.computeEmbeddingFast( + tailSlice, sampleRateInHz + ) + } else null val label = if (emb == null) null else SpeakerRecognition.registry.search( emb, speakerThreshold ) ?: SpeakerRecognition.nextAutoLabel().also { SpeakerRecognition.registry.addAuto(it, emb) + if (fastEmb != null) + SpeakerRecognition.fastRegistry + .addAuto(it, fastEmb) } - uiHandler.post { confirmSpeaker(sid, label, emb) } + uiHandler.post { confirmSpeaker(sid, label, emb, fastEmb) } } // 定格 + 送检立即走主线程队列,不等声纹 uiHandler.post { @@ -625,6 +728,30 @@ class MainActivity : AppCompatActivity() { activeSpeakerModel = null } + // E:轻量声纹模型(eres2net base)专供滚动换人检测,推理快 ~4 倍。 + // 找不到该文件或准确模型本来就是 base 时跳过,滚动检测自动回退到准确模型。 + private fun initFastSpeakerModel(topAssets: Array) { + val fastName = speakerModels[1] + if (activeSpeakerModel == fastName) return // 准确模型已是 base + try { + SpeakerRecognition.initFastExtractor( + null, File(externalModelDir, fastName).absolutePath + ) + Log.i(TAG, "fast speaker model ready (external): $fastName") + return + } catch (t: Throwable) { + Log.i(TAG, "fast model '$fastName' not in external dir") + } + if (topAssets.contains(fastName)) { + try { + SpeakerRecognition.initFastExtractor(application.assets, fastName) + Log.i(TAG, "fast speaker model ready (assets): $fastName") + } catch (t: Throwable) { + Log.e(TAG, "fast model '$fastName' failed: $t") + } + } + } + private fun onNameSpeakerClick() { val emb = lastEmbedding if (emb == null) { @@ -639,6 +766,9 @@ class MainActivity : AppCompatActivity() { val name = input.text.toString().trim() if (name.isNotEmpty()) { SpeakerRecognition.registry.addNamed(name, emb) + lastEmbeddingFast?.let { + SpeakerRecognition.fastRegistry.addNamed(name, it) + } relabelSpeakerLines(name) Toast.makeText(this, "之后「$name」说话时会显示真名", Toast.LENGTH_SHORT).show() } @@ -748,9 +878,9 @@ class MainActivity : AppCompatActivity() { } // Runs on subtitleHandler (serial): never touches UI state, only decodes - // a snapshot of one completed utterance. Trims trailing audio that is - // usually the silence/gap that ended the segment; short snapshots keep - // their tail (it doubles as 前段上下文补偿, 需求二.2). + // a snapshot of one completed utterance. Trims a graded amount of trailing + // audio (usually the gap/silence that ended the segment); segments are + // strictly contiguous (v6: no 0.5 s overlap between clips). // // 需求二(prompt):第一遍草稿经 setConfig 注入 funasrNano.hotwords —— // native 会把它拼进模型训练时学过的"上下文信息/热词列表"提示模板。 @@ -869,13 +999,6 @@ class MainActivity : AppCompatActivity() { bufferTotal = 0 } - private fun keepTailChunks(n: Int) { - while (samplesBuffer.size > n) { - bufferTotal -= samplesBuffer.removeAt(0).size - chunkRms.removeAt(0) - } - } - private fun dropPrefixChunks(n: Int) { repeat(minOf(n, samplesBuffer.size)) { bufferTotal -= samplesBuffer.removeAt(0).size @@ -890,6 +1013,13 @@ class MainActivity : AppCompatActivity() { return Math.sqrt(acc / s.size).toFloat() } + // 句内参考 embedding 的 EMA 混合(7:3),让基准缓慢跟随说话人的 + // 音量/音色自然漂移,避免"越说越像别人"的连锁误判 + private fun blendEmb(a: FloatArray, b: FloatArray): FloatArray { + val n = minOf(a.size, b.size) + return FloatArray(n) { a[it] * 0.7f + b[it] * 0.3f } + } + private fun flattenSamples(): FloatArray = flattenPrefix(samplesBuffer.size) // fresh copy of the first k chunks — never hand the shared buffer to threads @@ -1034,6 +1164,7 @@ class MainActivity : AppCompatActivity() { labelColorIdx.clear() colorSeq = 0 lastLabel = null + lastSentenceSettledAt = SystemClock.elapsedRealtime() if (n > 0) subtitleAdapter.notifyItemRangeRemoved(0, n) textView.visibility = View.GONE subtitleRecycler.visibility = View.VISIBLE @@ -1053,6 +1184,16 @@ class MainActivity : AppCompatActivity() { labelColorIdx.getOrPut(label) { colorSeq++ % speakerPalette.size } private fun appendNewLine(label: String?): SubtitleLine { + // 新行出现即封掉之前的开放行,并把其中"未定格的进行中句子"整体迁移 + // 过来 —— 硬保证 in-progress 只可能发生在最底行(修复:声纹判定搬家后, + // 旧行里还在长字的句子被孤立在上方的问题) + val old = subtitleLines.lastOrNull() + val migrating = if (old != null && old.open) + old.sentences.filter { !it.finalized } else emptyList() + old?.open = false + if (migrating.isNotEmpty()) { + old?.sentences?.removeAll(migrating.toSet()) + } val line = SubtitleLine( id = lineIdSeq.incrementAndGet(), label = label, @@ -1060,6 +1201,10 @@ class MainActivity : AppCompatActivity() { ) subtitleLines.add(line) subtitleAdapter.notifyItemInserted(subtitleLines.size - 1) + // 进行中句子接在新行尾部,继续实时更新 + line.sentences.addAll(migrating) + for (s in migrating) s.line = line + if (migrating.isNotEmpty()) old?.let { notifyLine(it) } return line } @@ -1069,8 +1214,12 @@ class MainActivity : AppCompatActivity() { private fun onPartial(sid: Long, partial: String) { val existing = sentenceById[sid] if (existing == null) { - val active = subtitleLines.lastOrNull()?.takeIf { it.open } - ?: appendNewLine(lastConfirmedLabel) + // 只接"最底部的开放行";同人停顿超时则另起新行(用户确认的规则) + val last = subtitleLines.lastOrNull() + val fresh = SystemClock.elapsedRealtime() - + lastSentenceSettledAt < lineIdleTimeoutMs + val active = if (last != null && last.open && fresh) last + else appendNewLine(lastConfirmedLabel) val sentence = Sentence(sid, partial, finalized = false) sentence.line = active active.sentences.add(sentence) @@ -1089,6 +1238,7 @@ class MainActivity : AppCompatActivity() { // v5 起声纹判定已移出录音线程)。 private fun finalizeSentence(sid: Long, willRefine: Boolean) { lastSettledSid = sid + lastSentenceSettledAt = SystemClock.elapsedRealtime() val sentence = sentenceById[sid] ?: return if (sentence.finalized) return sentence.finalized = true @@ -1114,36 +1264,70 @@ class MainActivity : AppCompatActivity() { return null } - // 声纹线程回传的整句权威判定(需求一.2/一.4): - // * 与所在行标签不符 → 该句整体迁入新说话人行(原行不含此句); - // * embedding 回写供"命名说话人"; - // * 仅当该槽仍是最近 settle 的句子才更新全局活跃归属(防旧判定覆盖更新的换人状态)。 - private fun confirmSpeaker(sid: Long, label: String?, emb: FloatArray?) { + // 声纹线程回传的整句权威判定: + // * 无判定或已同人 → 不动; + // * 占位行首次判定 → 就地冠名;若满足"保守并回"条件(本行为底行、 + // 上一行正是判定同人)→ 整行并回上一行,撤销误切的那次换行; + // * 行已归属他人 → 句子迁出;若误切行因此清空、且它是底行、上一行 + // 正是判定同人 → 同样并回上方,不再另起新行; + // * 其余情况维持原行为(并回底部同人行,没有则在最下方新建)。 + // * embedding 回写供"命名说话人";仅当该槽是最近 settle 的句子才更新全局归属。 + private fun confirmSpeaker( + sid: Long, label: String?, emb: FloatArray?, fastEmb: FloatArray?, + ) { Log.i(TAG, "SLOT sid=$sid VERDICT speaker=" + (label?.let { displayName(it) } ?: "未匹配/无判定")) if (emb != null) { lastEmbedding = emb + lastEmbeddingFast = fastEmb nameSpeakerButton.isEnabled = true } val sentence = findSentence(sid) ?: return - val line = sentence.line ?: return + var line = sentence.line ?: return if (label != null && label != line.label) { - line.sentences.remove(sentence) - line.open = false - if (line.sentences.isEmpty()) { - val pos = subtitleLines.indexOf(line) - if (pos >= 0) { - subtitleLines.removeAt(pos) - subtitleAdapter.notifyItemRemoved(pos) + if (line.label == null) { + // 占位行首次判定:整行就地冠名 + line.label = label + line.colorIdx = colorIdxFor(label) + if (!mergeLineIntoPrevIfSame(line, label)) { + notifyLine(line) } } else { - notifyLine(line) + line.sentences.remove(sentence) + val pos = subtitleLines.indexOf(line) + val wasLast = pos == subtitleLines.size - 1 + val prevSame = if (pos > 0) + subtitleLines[pos - 1].takeIf { it.label == label } else null + val lineEmpty = line.sentences.isEmpty() + if (lineEmpty && pos >= 0) { + subtitleLines.removeAt(pos) + subtitleAdapter.notifyItemRemoved(pos) + } else if (pos >= 0) { + notifyLine(line) + } + if (lineEmpty && wasLast && prevSame != null) { + // 保守并回:误切行清空后撤销该行,句子回到上方同人行 + Log.i(TAG, "verdict merge-back sid=$sid -> line above") + prevSame.sentences.add(sentence) + sentence.line = prevSame + line = prevSame + notifyLine(prevSame) + autoScroll() + } else { + // 只并回"最底部且同人"的行;否则在最下方新建 + val target = subtitleLines.lastOrNull() + ?.takeIf { it.open && it.label == label } + ?: appendNewLine(label) + // 定格句插到该行"进行中句子"之前,保持行内时间序 + val ins = target.sentences.indexOfFirst { !it.finalized } + if (ins >= 0) target.sentences.add(ins, sentence) + else target.sentences.add(sentence) + sentence.line = target + line = target + notifyLine(target) + autoScroll() + } } - val nl = appendNewLine(label) - nl.sentences.add(sentence) - sentence.line = nl - notifyLine(nl) - autoScroll() } if (label != null && sid == lastSettledSid) { lastConfirmedLabel = label @@ -1152,6 +1336,30 @@ class MainActivity : AppCompatActivity() { } } + // 保守并回(用户确认的规则,2026-09-10):误切行仍是底行、且它的上一条 + // 恰好就是判定出的同一个人 → 整行并回上一行后删除。in-progress 句子随行 + // 迁移,合并后 prev 成为底行,"进行中只存在于最底行"不变量保持成立。 + // 只撤销"被判定翻转的误切行";60s 超时产生的相邻同人行不在此列 + // (超时行建档时就带着标签,永远走不到本分支)。 + private fun mergeLineIntoPrevIfSame(line: SubtitleLine, verdict: String): Boolean { + val pos = subtitleLines.indexOf(line) + if (pos <= 0 || pos != subtitleLines.size - 1) return false + val prev = subtitleLines[pos - 1] + if (prev.label != verdict) return false + Log.i(TAG, "verdict merge-back: whole line@$pos -> prev(same speaker)") + val snapshot = line.sentences.toList() + for (s in snapshot) { + s.line = prev + prev.sentences.add(s) + } + line.sentences.clear() + subtitleLines.removeAt(pos) + subtitleAdapter.notifyItemRemoved(pos) + notifyLine(prev) + autoScroll() + return true + } + // second-pass refine, 需求二加固: // * 底稿铁律:refine 只能覆盖对应句;空/异常/超时(看门狗)/折叠误杀/ // 明显缩短 → 一律保留第一遍文字,该句标记 sentence.refined=false。 @@ -1174,15 +1382,27 @@ class MainActivity : AppCompatActivity() { result = "REJECT_EXCEPTION" note = "底稿保留" } else { - val collapsed = collapseRepetitions(refined) - val candidate = - if (collapsed.isEmpty() || collapsed.length * 2 < refined.length) { - note = "折叠误杀回退原文;" - refined - } else { - collapsed + // 长度锚定(v8.4):底稿是"实时音频能装下多少字"的自然上界。 + // 精修 ≤ 底稿×1.5+8 → 无膨胀迹象,原文直接用(真说出口的重叠词 + // 如"谢谢谢谢"原样保留,不再误折叠); + // 超出 → 必是 LLM 复读,折叠;折叠后仍超界或为空 → 整条拒绝留底稿。 + // (修复:旧规则按"折叠量>50%判误杀回退原文",恰好给真复读开了后门) + val ceiling = first.length * 3 / 2 + 8 + var candidate = "" + if (refined.length <= ceiling) { + candidate = refined + } else { + note = "精修超长(${refined.length}>ceil $ceiling)疑似复读;" + val collapsed = collapseRepetitions(refined) + if (collapsed.isNotEmpty() && collapsed.length <= ceiling) { + candidate = collapsed + note += "折叠后采用" } + } when { + candidate.isEmpty() -> { + result = "REJECT_TOO_LONG"; note += "底稿保留" + } candidate.isBlank() -> { result = "REJECT_EMPTY"; note = "底稿保留" } @@ -1213,6 +1433,24 @@ class MainActivity : AppCompatActivity() { activeLabelForWindow = label } + // 滚动窗口查"这是谁":有 fast 模型时查 fast 镜像(0.4 宽松阈值,仅供显示); + // 没有 base 文件时滚动检测本身回退在准确模型空间 → 直查准确 registry。 + // 修复:base 缺失时镜像库恒空、换行标签全 null、误断防护失效的问题。 + private fun rollingSpeakerSearch(emb: FloatArray): String? = + if (SpeakerRecognition.fastReady()) + SpeakerRecognition.fastRegistry.search(emb, fastSearchThreshold) + else + SpeakerRecognition.registry.search(emb, speakerThreshold) + + // ③ 滚动检测查到了已知人、但当前行还没有归属(占位"未识别"): + // 直接把活跃归属采纳为他,等 endpoint 权威判定给行冠名,不再多切一行 + private fun adoptActiveLabel(label: String) { + if (lastConfirmedLabel == null) { + lastConfirmedLabel = label + activeLabelForWindow = label + } + } + private fun notifyLine(line: SubtitleLine) { trimOldLines() val pos = subtitleLines.indexOf(line) diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt index 6183c56..0cd6803 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt @@ -826,9 +826,10 @@ fun getOfflineModelConfig(type: Int, modelPrefix: String = ""): OfflineModelConf tokenizer = "$modelDir/Qwen3-0.6B", ), tokens = "", - // 2 threads: leave cores for the streaming pass + eres2net so - // nano decoding can't starve the first pass (v5 停摆加固) - numThreads = 2, + // 定稿提速:LLM 解码 4 线程(≈快一倍)。线程预算:nemotron 2 + + // nano 4 + 声纹 v2 2 + 滚动 base 1;若机身发烫/第一遍变卡, + // 退回 2 并改上"积压保险"方案 + numThreads = 4, ) } diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt index 43d539b..799bd5a 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt @@ -614,6 +614,9 @@ fun getModelConfig(type: Int, modelPrefix: String = ""): OnlineModelConfig? { joiner = "$modelDir/joiner.int8.onnx", ), tokens = "$modelDir/tokens.txt", + // 0.6B 流式模型单线程解码实时余量太薄,定稿/声纹一跑就掉帧; + // 2 线程保障第一遍实时性(纯资源参数,不影响精度) + numThreads = 2, ) } diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt index 563c64d..550d6bf 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt @@ -159,31 +159,45 @@ class SpeakerRegistry { } // --------------------------------------------------------------------------- -// Singleton holder shared by the UI thread and the recording thread. +// Singleton holder shared by the UI thread, recording thread and the rolling +// speaker-check thread. +// +// 双模型方案(E):准确模型(eres2netv2)负责 endpoint 权威判定与注册; +// 轻量模型(eres2net base,可选)只负责滚动换人检测(推理快 ~4 倍)。 +// ⚠ 两个模型的 embedding 不互通:各自配一份 registry,注册时双写镜像。 // --------------------------------------------------------------------------- object SpeakerRecognition { - private var _extractor: SpeakerEmbeddingExtractor? = null + @Volatile private var _extractor: SpeakerEmbeddingExtractor? = null + @Volatile private var _fastExtractor: SpeakerEmbeddingExtractor? = null val extractor: SpeakerEmbeddingExtractor get() = _extractor!! + /** accurate-space registry (endpoint verdict / naming source of truth) */ val registry = SpeakerRegistry() + /** fast-space mirror (rolling-check search only; may stay empty) */ + val fastRegistry = SpeakerRegistry() + var autoCounter: Int = 0 private set - // autoCounter is touched from the recording thread (identifySpeaker) while - // resetSpeakers runs on the UI thread — keep them serialized. + private val accLock = Any() + private val fastLock = Any() + + // autoCounter is touched from worker threads while resetSpeakers runs on + // the UI thread — keep them serialized. @Synchronized fun nextAutoLabel(): String = "S${autoCounter++}" @Synchronized fun resetSpeakers() { registry.clear() + fastRegistry.clear() autoCounter = 0 } // assetManager = null -> `model` is treated as an absolute file system path fun initExtractor(assetManager: AssetManager?, model: String) { - synchronized(this) { + synchronized(accLock) { if (_extractor != null) return Log.i(TAG, "Initializing speaker embedding extractor with $model") _extractor = SpeakerEmbeddingExtractor( @@ -198,13 +212,45 @@ object SpeakerRecognition { } } - // Compute one embedding from a complete utterance (float mono @ 16 kHz). - // Returns null when the utterance is too short for the model to consume. - // Serialized: the native extractor holds one session shared by the - // recording thread and the rolling-window checker thread. - @Synchronized - fun computeEmbedding(samples: FloatArray, sampleRate: Int): FloatArray? { - val ext = _extractor ?: return null + // Optional lightweight model for the rolling speaker-change detector. + fun initFastExtractor(assetManager: AssetManager?, model: String) { + synchronized(fastLock) { + if (_fastExtractor != null) return + Log.i(TAG, "Initializing FAST speaker embedding extractor with $model") + _fastExtractor = SpeakerEmbeddingExtractor( + assetManager = assetManager, + config = SpeakerEmbeddingExtractorConfig( + model = model, + numThreads = 1, + debug = false, + provider = "cpu", + ) + ) + } + } + + fun fastReady(): Boolean = _fastExtractor != null + + // Compute one embedding with the accurate extractor (endpoint verdicts). + fun computeEmbedding(samples: FloatArray, sampleRate: Int): FloatArray? = + synchronized(accLock) { + val ext = _extractor ?: return null + runCompute(ext, samples, sampleRate) + } + + // Rolling-check embedding: fast space when available, else accurate space + // (in the latter case both registries see the same space anyway). + fun computeEmbeddingFast(samples: FloatArray, sampleRate: Int): FloatArray? { + val fast = _fastExtractor + return if (fast == null) computeEmbedding(samples, sampleRate) + else synchronized(fastLock) { runCompute(fast, samples, sampleRate) } + } + + private fun runCompute( + ext: SpeakerEmbeddingExtractor, + samples: FloatArray, + sampleRate: Int, + ): FloatArray? { if (samples.isEmpty()) return null val stream = ext.createStream() stream.acceptWaveform(samples = samples, sampleRate = sampleRate)