From 236037a0b0e77741824a78e4e3160f385f812825 Mon Sep 17 00:00:00 2001 From: liwenhao2 Date: Fri, 11 Sep 2026 17:27:08 +0800 Subject: [PATCH] =?UTF-8?q?=E9=98=B6=E6=AE=B5=E6=80=A7=E5=9F=BA=E7=BA=BF:?= =?UTF-8?q?=E5=AD=97=E5=B9=95=E8=AF=B4=E8=AF=9D=E4=BA=BA=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E4=BA=A4=E4=BA=92+=E6=8D=A2=E4=BA=BA=E6=96=AD=E5=8F=A5/?= =?UTF-8?q?=E5=AE=9A=E7=A8=BF=E5=8A=A0=E5=9B=BA(v2-v8.4)=E3=80=81=E7=A6=BB?= =?UTF-8?q?=E7=BA=BF=E7=BA=AA=E8=A6=81=E5=AD=90=E7=B3=BB=E7=BB=9F=E3=80=81?= =?UTF-8?q?=E5=81=A5=E5=A3=AE=E6=80=A7(=E6=81=AF=E5=B1=8F=E4=BF=9D?= =?UTF-8?q?=E6=B4=BB/=E4=BC=9A=E8=AF=9D=E4=BF=9D=E7=95=9950=E5=9C=BA/?= =?UTF-8?q?=E6=AD=BB=E4=BB=A3=E7=A0=81=E6=B8=85=E7=90=86/=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=8D=95=E6=B5=8B)=E2=80=94=E2=80=94=E5=BE=85?= =?UTF-8?q?=E7=9C=9F=E6=9C=BA=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 6 + .idea/AndroidProjectSystem.xml | 10 +- .idea/deploymentTargetSelector.xml | 20 +- .idea/migrations.xml | 18 +- .idea/runConfigurations.xml | 32 +- app/build.gradle | 2 + app/src/main/assets/summarize/prompts.md | 139 ++++++ .../java/com/k2fsa/sherpa/onnx/LlmEngine.kt | 151 ++++++ .../com/k2fsa/sherpa/onnx/MainActivity.kt | 457 +++++++++++++++--- .../com/k2fsa/sherpa/onnx/MeetingNotes.kt | 63 +++ .../k2fsa/sherpa/onnx/SessionTranscript.kt | 216 +++++++++ .../com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt | 23 + .../com/k2fsa/sherpa/onnx/SubtitleFormat.kt | 16 + .../k2fsa/sherpa/onnx/SummarizePipeline.kt | 412 ++++++++++++++++ app/src/main/res/layout/activity_main.xml | 76 ++- app/src/main/res/layout/item_subtitle.xml | 1 - app/src/main/res/values/strings.xml | 9 +- .../com/k2fsa/sherpa/onnx/MeetingNotesTest.kt | 45 ++ .../sherpa/onnx/SubtitleAndSpeakerTest.kt | 78 +++ .../k2fsa/sherpa/onnx/SummarizeLogicTest.kt | 176 +++++++ docs/M2_STRESS_TEST.md | 57 +++ gradlew.bat | 178 +++---- tools/llama-jni/CMakeLists.txt | 22 + tools/llama-jni/build_arm64.sh | 38 ++ tools/llama-jni/exports.map | 6 + tools/llama-jni/src/summarize_llm_jni.cpp | 312 ++++++++++++ 26 files changed, 2342 insertions(+), 221 deletions(-) create mode 100644 app/src/main/assets/summarize/prompts.md create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/LlmEngine.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/MeetingNotes.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/SessionTranscript.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/SubtitleFormat.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/SummarizePipeline.kt create mode 100644 app/src/test/java/com/k2fsa/sherpa/onnx/MeetingNotesTest.kt create mode 100644 app/src/test/java/com/k2fsa/sherpa/onnx/SubtitleAndSpeakerTest.kt create mode 100644 app/src/test/java/com/k2fsa/sherpa/onnx/SummarizeLogicTest.kt create mode 100644 docs/M2_STRESS_TEST.md create mode 100644 tools/llama-jni/CMakeLists.txt create mode 100644 tools/llama-jni/build_arm64.sh create mode 100644 tools/llama-jni/exports.map create mode 100644 tools/llama-jni/src/summarize_llm_jni.cpp diff --git a/.gitignore b/.gitignore index b984051..504e73b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,11 @@ .cxx local.properties +# 构建产物与日志不入库 +/build-check/ +*.log +/tools/llama-jni/llama.cpp/ +/tools/llama-jni/build-*/ + # 模型文件体积大，不入库；运行时手动放入 app/src/main/assets/ *.onnx diff --git a/.idea/AndroidProjectSystem.xml b/.idea/AndroidProjectSystem.xml index 4a53bee..d58d49b 100644 --- a/.idea/AndroidProjectSystem.xml +++ b/.idea/AndroidProjectSystem.xml @@ -1,6 +1,6 @@ - - - - + + + + \ No newline at end of file diff --git a/.idea/deploymentTargetSelector.xml b/.idea/deploymentTargetSelector.xml index ca16a99..47d9235 100644 --- a/.idea/deploymentTargetSelector.xml +++ b/.idea/deploymentTargetSelector.xml @@ -1,11 +1,11 @@ - - - - - - - - + + + + + + + + \ No newline at end of file diff --git a/.idea/migrations.xml b/.idea/migrations.xml index f8051a6..48052b2 100644 --- a/.idea/migrations.xml +++ b/.idea/migrations.xml @@ -1,10 +1,10 @@ - - - - - + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml index 16660f1..5bd6771 100644 --- a/.idea/runConfigurations.xml +++ b/.idea/runConfigurations.xml @@ -1,17 +1,17 @@ - - - - - + + + + + \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index f1f48ca..434e496 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -52,6 +52,8 @@ dependencies { implementation 'com.google.android.material:material:1.7.0' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' testImplementation 'junit:junit:4.13.2' + // 摘要流水线的 JVM 单测需要真实 org.json(android.jar 里的是会抛 Stub! 的桩) + testImplementation 'org.json:json:20180813' androidTestImplementation 'androidx.test.ext:junit:1.1.4' androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.0' } \ No newline at end of file diff --git a/app/src/main/assets/summarize/prompts.md b/app/src/main/assets/summarize/prompts.md new file mode 100644 index 0000000..25f078c --- /dev/null +++ b/app/src/main/assets/summarize/prompts.md @@ -0,0 +1,139 @@ +# 摘要功能三段提示词(里程碑 1) + +> 适用模型:`gemma-3-1b-it`(ggml-org/gemma-3-1b-it-GGUF,Q4_K_M;2026-09-11 由 +> gemma-4-E2B 换小提速。此前:`google/gemma-4-E2B-it-qat-q4_0-gguf`,架构 gemma4,Apache-2.0) +> 本文件是唯一事实源:App 内 `SummarizePipeline.kt` 逐字引用这里的文本。改提示词先改本文件。 +> PC 调试方法见文末「用 PC llama.cpp 复现」。 + +## 0. 运行参数(与 App 实现一致) + +| 步骤 | temperature | maxNewTokens | 备注 | +|---|---|---|---| +| P1 滚动笔记更新 | 0(Greedy) | 900 | 可复现;笔记要求确定性 | +| P2 成稿 | 0.5 / topK 64 / topP 0.95 | 2500 | Gemma 4 官方推荐采样档位,低温折中 | + +思考(thinking)关闭:App 渲染模板时不注入 `<|think|>` 标记;模型若仍输出 `<|channel>thought…` 块,App 侧正则剔除。 + +## 1. 聊天模板(App 手工渲染,与 Gemma 4 canonical template 单轮路径等价) + +``` +(tokenizer 自动) +<|turn>system +{SYSTEM} +<|turn>user +{USER} +<|turn>model +``` + +特殊 token id:``=2,``=1,`<|turn>`=105,``=106。生成遇 ``、``、或 8192 上下文耗尽即停。 + +--- + +## P1 — 滚动笔记更新 + +### SYSTEM(固定,勿动态改写) + +``` +あなたは会議の文字起こしを処理する記録係です。与えられた「現在のメモ」に「新しい断片」の内容を統合し、更新後のメモをJSONだけで出力してください。 + +厳守事項: +- 出力は指定スキーマのJSONオブジェクトのみ。説明文・コードフェンス・前置き禁止。 +- メモは日本語で書く。ただし人名・組織名・製品名・固有名詞は原文表記をそのまま維持。 +- 数字、金額、日時、期限、回数、単位は原文どおり正確に転記。ざっくり要約しない。 +- 否定・保留・反対の意図を消さない。「見送り」「再検討」「反対意見」はそれ自体が重要な情報。 +- 各項目に根拠となる発話のタイムレンジを「出典」に付ける(形式: "mm:ss-mm:ss"、複数可 ";" 区切り)。 +- 断片に新情報がない場合はメモをほぼ変更しない(前回末尾は必ず更新)。 +- 全体で500字を超えそうな場合、重要度の低い過去の項目を統合・削除して圧縮する。削除時は行動項目と已定事項を最後に守る。 +- 「前回末尾」には、次の断片と文脈が繋がるための直近話題を1文で書く。 +- JSONの値の中にダブルクォートが必要なら \" にエスケープ。改行は \n を使う。 + +スキーマ(キー名は変更禁止): +{"已定事項":[{"内容":"","出典":""}],"討論中":[{"内容":"","出典":""}],"行動項目":[{"内容":"","担当者":"","期限":"","出典":""}],"人物":[{"名前":"","立場":""}],"前回末尾":""} +``` + +### USER(每块渲染;`{}` 为占位符) + +``` +【現在のメモ】 +{notes_json} + +【新しい断片】(タイムスタンプは会議開始からの分:秒、[]内は話者ラベル) +{chunk_text} + +更新後のメモJSONを出力してください。 +``` + +--- + +## P2 — 最终成稿 + +### SYSTEM(固定) + +``` +あなたは会議の公式議事録作成者です。会議メモ(JSON)と用語集を受け取ったら、指定言語で完成した議事録をMarkdownで出力してください。 + +厳守事項: +- 出力言語は指定された「成稿言語」を厳守。日本語メモの内容をそのまま指定言語の書き言葉文体で記述する。 +- 本文は成稿言語の文字のみで書く:中文稿なら仮名を一切出さない、English稿なら漢字・仮名を一切出さない、日本語稿なら製品名以外に英語の文を残さない。 +- ただし人物・会社・ブランドなどの固有名詞は、メモ・用語集にある原文表記のまま保留してよい(上記の文字制限の例外)。 +- 見出しは次の4つを必ずこの順で:「議題と経過」「決定事項」「行動項目」「未決事項・持ち帰り」。該当ゼロなら「(なし)」と明記。 +- 行動項目は表形式(担当者 / 内容 / 期限 / 出典)。担当者が不明なら「未定」と書く。 +- 各項目の末尾に出典を付ける。形式: `出典: mm:ss-mm:ss`。出典はメモJSONの「出典」フィールドのみを使う。捏造禁止。 +- メモにない事実・推測・補足説明を追加しない。 +- 人名・製品名は用語集の正式表記に統一する(用語集にない表記はメモのまま採用)。 +- 冒頭にメタ情報を置く: 会議日時(指定があれば)、話者数、使用言語、文字起こし総尺。 +- Markdown本文のみ出力。コードフェンスで全体を囲まない。 +``` + +### USER + +``` +【成稿言語】{language_name} + +【会議メタ情報】開始: {meeting_start} / 話者数: {n_speakers} / 総尺: {total_duration} + +【用語集(正式表記の一覧。人名・製品名の誤変換をこれに統一)】 +{glossary_lines} + +【会議メモJSON】 +{notes_json} + +上記に基づき、議事録Markdownを出力してください。 +``` + +--- + +## 语言名映射(P2 `{language_name}`) + +| 会议语言 | language_name | +|---|---| +| ja | 日本語 | +| zh | 中文(简体书面语) | +| en | English (formal written English) | + +## 分块参数(Kotlin 侧,不进提示词) + +- 目标 2500±500 字/块,按句子边界聚合,块间重叠 1 句;同一说话人轮次不拆开。 +- 行格式:`(mm:ss-mm:ss)[话者名] 文本`,时间戳来自 SessionTranscript。 +- 首轮 `{notes_json}` = 空 schema(`{"已定事項":[],"討論中":[],"行動項目":[],"人物":[],"前回末尾":""}`)。 +- 单块+笔记的总输入若超过 ~5000 字符,Kotlin 先把该块对半拆成两次 P1 调用(块长自适应,不截断内容)。 + +## 用 PC llama.cpp 复现(提示词调优用) + +1. 取 llama.cpp master(≥2026-09;gemma3 架构早已支持,App 的 `libsummarize-llm-jni.so` 无需重编)。 +2. 模型:`gemma-3-1b-it-Q4_K_M.gguf`(ggml-org/gemma-3-1b-it-GGUF)。 +3. 跑 P1 示例(替换 sys/user 两段;`--jinja` 走 GGUF 内置模板,与 App 的 + `renderGemma3Prompt`——system 并入首个 user turn——等价): + +```bash +llama-cli -m gemma-3-1b-it-Q4_K_M.gguf -c 8192 -ctk q8_0 -ctv f16 --jinja \ + --temp 0 -n 900 \ + --sys "$(sed -n '/^### SYSTEM/,/^```$/p' prompts.md | sed '1,2d;$d')" \ + -p "$USER_PROMPT" +``` + +4. 调优结果(好例/坏例)记录到本文件末尾「调优日志」。 + +## 调优日志 + +(空——等 PC/真机验证后追加) diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/LlmEngine.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/LlmEngine.kt new file mode 100644 index 0000000..3f26fd5 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/LlmEngine.kt @@ -0,0 +1,151 @@ +package com.k2fsa.sherpa.onnx + +import android.util.Log + +// ============================================================================ +// LlmEngine — libsummarize-llm-jni.so(llama.cpp + 薄 JNI)的 Kotlin 封装。 +// +// 与工程其它 native wrapper(OnlineRecognizer/OfflineRecognizer/Extractor) +// 同范式:ptr + require + finalize()/release()。但注意: +// * 本库绝不能进录音/主线程路径 —— generate() 是长阻塞调用,设计为只在 +// 摘要专用后台线程上使用,一次只能有一个 generate 在跑(JNI 内有锁)。 +// * 模型文件必须由用户 adb push,App 不下载;调用 nativeNewFromFile 前 +// 必须在 Kotlin 侧确认文件存在(工程教训:缺文件时有的 native 会 exit(), +// 本 .so 已做 access() 双保险并抛异常,但仍按规矩先查)。 +// ============================================================================ + +/** 摘要 LLM 的可用性探测。so 只编了 arm64-v8a,x86_64 模拟器上自动不可用。 */ +object LlmRuntime { + private const val LIB = "summarize-llm-jni" + @Volatile private var tried = false + @Volatile var nativeAvailable = false + private set + + fun ensure(): Boolean { + if (!tried) { + synchronized(this) { + if (!tried) { + tried = true + nativeAvailable = try { + System.loadLibrary(LIB) + true + } catch (t: Throwable) { + Log.w("sherpa-onnx", "summarize lib $LIB not loadable: $t") + false + } + } + } + } + return nativeAvailable + } +} + +class LlmEngine( + modelPath: String, + ctxSize: Int = 8192, // 滚动笔记+单块输入远小于此(§3 资源账) + nThreads: Int = 6, // 8 Gen3 大核 4 个,给摘要留余量:6 线程够用 + kvTypeK: String = "q8_0", // KV 用 q8_0 压显存/内存(§3) + kvTypeV: String = "f16", // V 保持 f16:K 量化到 q8_0 是 llama.cpp 推荐档 + chatTemplate: String = "gemma4", // "gemma4" | "gemma3",须与 GGUF 内嵌模板一致 +) { + companion object { + private const val TAG = "sherpa-onnx" + + // ---- Gemma 4 canonical template 的单轮(system+user)渲染 ---- + // 与 GGUF 内嵌模板(enable_thinking=false, 无 tools)逐字等价: + // bos <|turn>system\nSYS\n <|turn>user\nUSR\n <|turn>model\n + // bos 由 tokenizer 的 add_bos_token 自动添加(native 侧 parse_special=true)。 + fun renderGemma4Prompt(system: String, user: String): String = + buildString { + append("<|turn>system\n").append(system.trim()).append("\n") + append("<|turn>user\n").append(user.trim()).append("\n") + append("<|turn>model\n") + } + + // ---- Gemma 3 官方模板的单轮渲染(ggml-org/gemma-3-1b-it-GGUF)---- + // 官方 jinja:system 不作为独立 turn,而是拼进第一个 user turn; + // user\n{system}\n\n{user}\nmodel\n + // EOG token 是 (gemma3 的 eot),依赖 native 按 eog 停止。 + fun renderGemma3Prompt(system: String, user: String): String = + buildString { + append("user\n") + append(system.trim()).append("\n\n") + append(user.trim()) + append("\n") + append("model\n") + } + + fun renderChatPrompt(system: String, user: String, + template: String = "gemma4"): String = + when (template) { + "gemma3" -> renderGemma3Prompt(system, user) + else -> renderGemma4Prompt(system, user) + } + } + + private val chatTemplate: String = chatTemplate + private var ptr: Long = 0L + + init { + require(LlmRuntime.ensure()) { "摘要推理库不可用(设备 ABI 不支持或未打包)" } + require(existsFile(modelPath)) { "找不到模型文件: $modelPath" } + ptr = nativeNewFromFile(modelPath, ctxSize, nThreads, kvTypeK, kvTypeV) + require(ptr != 0L) { "LLM 模型加载失败: $modelPath" } + } + + // 与工程教训对齐的存在性检查:加载任何模型文件前必须先确认文件在 + private fun existsFile(path: String): Boolean = + java.io.File(path).let { it.isFile && it.length() > 100L } + + /** + * 阻塞式生成(仅供摘要后台线程调用)。返回模型输出文本(不含输入)。 + * temperature=0 为 Greedy。abort() 可从其它线程安全打断。 + */ + fun generate( + systemPrompt: String, + userPrompt: String, + maxNewTokens: Int = 900, + temperature: Float = 0f, + topK: Int = 64, + topP: Float = 0.95f, + ): String { + require(ptr != 0L) { "LlmEngine already released" } + return nativeGenerate( + ptr, renderChatPrompt(systemPrompt, userPrompt, chatTemplate), + maxNewTokens, temperature, topK, topP + ) + } + + fun abort() { + if (ptr != 0L) nativeAbort(ptr) + } + + /** 模型内嵌的 jinja 聊天模板(调试用,验证渲染一致性)。 */ + fun chatTemplate(): String? { + if (ptr == 0L) return null + return nativeGetStringMeta(ptr, "tokenizer.chat_template") + } + + protected fun finalize() { + if (ptr != 0L) { + nativeFree(ptr) // JNI 内部等待在途 generate 结束后再释放 + ptr = 0L + } + } + + fun release() = finalize() + + // ---- native(实现在 tools/llama-jni/src/summarize_llm_jni.cpp)---- + private external fun nativeNewFromFile( + path: String, ctxSize: Int, nThreads: Int, typeK: String, typeV: String + ): Long + + private external fun nativeGenerate( + ptr: Long, prompt: String, maxNewTokens: Int, + temperature: Float, topK: Int, topP: Float + ): String + + private external fun nativeAbort(ptr: Long) + private external fun nativeFree(ptr: Long) + private external fun nativeGetStringMeta(ptr: Long, key: String): String? +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index beefd27..ffab623 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -15,13 +15,17 @@ import android.os.Process import android.os.SystemClock import android.text.SpannableStringBuilder import android.text.Spanned +import android.text.TextPaint +import android.text.method.LinkMovementMethod import android.text.method.ScrollingMovementMethod +import android.text.style.ClickableSpan import android.text.style.ForegroundColorSpan import android.text.style.StyleSpan import android.util.Log import android.view.LayoutInflater import android.view.View import android.view.ViewGroup +import android.view.WindowManager import android.widget.Button import android.widget.EditText import android.widget.TextView @@ -146,11 +150,6 @@ class MainActivity : AppCompatActivity() { private var activeSpeakerModel: String? = null private val speakerThreshold = 0.5f private var speakersEnabled = false - private var lastEmbedding: FloatArray? = null - // 同一句话在 fast(轻量)空间的镜像嵌入,命名时双写进 fastRegistry - private var lastEmbeddingFast: FloatArray? = null - private lateinit var nameSpeakerButton: Button - private lateinit var clearSpeakersButton: Button // ---- rolling speaker-change detection (需求一.4) ------------------------- // Every ~1.5 s the speaker-check thread embeds the last 2 s window and @@ -208,9 +207,6 @@ class MainActivity : AppCompatActivity() { @Volatile private var isRecording: Boolean = false - @Volatile - private var modelsReady: Boolean = false - // ---- External model loading (adb push) --------------------------------- // Models live in Android/data//files/models/ instead of assets: // adb push /sdcard/Android/data/com.k2fsa.sherpa.onnx/files/models/ @@ -226,6 +222,31 @@ class MainActivity : AppCompatActivity() { private var loadFromAssets = true private var modelPrefix = "" + // ---- Summarize (离线 LLM 会议纪要,里程碑 1) ------------------------------ + // §7 红线:总开关 false 时不建会话记录、不显示按钮,App 行为与加功能前 + // 完全一致。摘要是独立世界:专用后台线程 + 独立 .so(llama.cpp), + // 不 import 任何 sherpa-onnx ASR native 对象。 + private val summarizeEnabled = true + private val summarizeModelDirName = "gemma4-e2b" + private val summarizeGgufName = "gemma-4-E2B_q4_0-it.gguf" + private val summarizeChatTemplate = "gemma4" + private lateinit var summarizeButton: Button + private lateinit var copyMinutesButton: Button + private lateinit var minutesStatus: TextView + private lateinit var minutesText: TextView + private lateinit var minutesHint: TextView + private var sessionTranscript: SessionTranscript? = null + @Volatile private var isSummarizing = false + @Volatile private var summarizeCancelled = false + // generate 阻塞在 native,取消靠 abort;引擎只在摘要线程持有 + @Volatile private var summarizeEngine: LlmEngine? = null + + companion object { + private val SUMMARY_THREADS = 6 // 空闲时:大核全用 + private val SUMMARY_THREADS_BUSY = 4 // 录音中成稿:给 ASR 让核(实测零丢帧) + private val SUMMARY_CTX = 8192 // §3 资源账:笔记+单块远小于此 + } + override fun onRequestPermissionsResult( requestCode: Int, permissions: Array, grantResults: IntArray ) { @@ -254,21 +275,24 @@ class MainActivity : AppCompatActivity() { recordButton.isEnabled = false recordButton.setOnClickListener { onclick() } - nameSpeakerButton = findViewById(R.id.name_speaker_button) - nameSpeakerButton.isEnabled = false - nameSpeakerButton.setOnClickListener { onNameSpeakerClick() } - - clearSpeakersButton = findViewById(R.id.clear_speakers_button) - clearSpeakersButton.setOnClickListener { - SpeakerRecognition.resetSpeakers() - lastEmbedding = null - lastEmbeddingFast = null - lastLabel = null - lastConfirmedLabel = null - activeLabelForWindow = null - nameSpeakerButton.isEnabled = false - Toast.makeText(this, "已清除所有说话人记录", Toast.LENGTH_SHORT).show() + summarizeButton = findViewById(R.id.summarize_button) + copyMinutesButton = findViewById(R.id.copy_minutes_button) + minutesStatus = findViewById(R.id.minutes_status) + minutesText = findViewById(R.id.minutes_text) + minutesHint = findViewById(R.id.minutes_hint) + if (summarizeEnabled) { + // 生成中再点同一按钮 = 取消(按钮文案随之切换) + summarizeButton.setOnClickListener { + if (isSummarizing) cancelSummarize() else onSummarizeClick() + } + copyMinutesButton.isEnabled = false + copyMinutesButton.setOnClickListener { onCopyMinutesClick() } + } else { + summarizeButton.visibility = View.GONE + copyMinutesButton.visibility = View.GONE } + loadLastMinutes() + hintUnfinishedMeeting() textView = findViewById(R.id.my_text) textView.movementMethod = ScrollingMovementMethod() @@ -305,7 +329,6 @@ class MainActivity : AppCompatActivity() { application.assets.list("").orEmpty() ) } - modelsReady = true runOnUiThread { recordButton.isEnabled = true } val speakerNote = if (speakersEnabled) "说话人功能已启用" else "⚠ 说话人功能未启用:缺少声纹模型 ${speakerModels[0]}" + @@ -360,6 +383,11 @@ class MainActivity : AppCompatActivity() { override fun onDestroy() { super.onDestroy() isRecording = false + sessionTranscript?.close() + if (isSummarizing) { + summarizeCancelled = true + summarizeEngine?.abort() + } subtitleThread.quitSafely() speakerCheckThread.quitSafely() } @@ -375,6 +403,8 @@ class MainActivity : AppCompatActivity() { audioRecord!!.startRecording() recordButton.setText(R.string.stop) isRecording = true + // 会议记录场景:录音期间保持屏幕常亮,防息屏冻结/MIUI 杀后台 + window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) resetBuffer() pendingSwitch = null windowRefEmb = null @@ -383,6 +413,11 @@ class MainActivity : AppCompatActivity() { activeLabelForWindow = null lastConfirmedLabel = null uiHandler.post { resetSubtitles() } + if (summarizeEnabled) { + // 每次录音 = 一个新会话文件(§4.2);主线程操作 + sessionTranscript?.close() + sessionTranscript = SessionTranscript.begin(filesDir) + } recordingThread = thread(true) { processSamples() @@ -390,10 +425,16 @@ class MainActivity : AppCompatActivity() { Log.i(TAG, "Started recording") } else { isRecording = false + window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) audioRecord!!.stop() audioRecord!!.release() audioRecord = null recordButton.setText(R.string.start) + if (summarizeEnabled) { + // 保留 sessionTranscript 引用直到下次开始录音: + // 「刚停止」是点生成纪要最常见的时间窗 + sessionTranscript?.flush() + } Log.i(TAG, "Stopped recording") } } @@ -752,54 +793,81 @@ class MainActivity : AppCompatActivity() { } } - private fun onNameSpeakerClick() { - val emb = lastEmbedding - if (emb == null) { - Toast.makeText(this, "先录一句话,再为这位说话人命名", Toast.LENGTH_SHORT).show() + // 点击行首说话人标签(ClickableSpan)→ 命名或改名。 + // internalKey:该行的内部声纹键("S0"/真名);null=归属未定,提示稍后再命名。 + private fun showNameSpeakerDialog(internalKey: String?) { + if (internalKey == null) { + Toast.makeText(this, "该行声纹归属尚未确定,先说一句完整的话再命名", + Toast.LENGTH_SHORT).show() return } - val input = EditText(this).apply { hint = "说话人名字，如 张三" } + val input = EditText(this).apply { + hint = "「${displayName(internalKey)}」的真名,如 张三" + } AlertDialog.Builder(this) - .setTitle("为说话人「${displayName(lastLabel)}」命名") + .setTitle(if (autoLabelRegex.matches(internalKey)) + "命名说话人「${displayName(internalKey)}」" + else "将「$internalKey」改名为…") .setView(input) - .setPositiveButton("注册") { _, _ -> + .setPositiveButton("确定") { _, _ -> val name = input.text.toString().trim() - if (name.isNotEmpty()) { - SpeakerRecognition.registry.addNamed(name, emb) - lastEmbeddingFast?.let { - SpeakerRecognition.fastRegistry.addNamed(name, it) - } - relabelSpeakerLines(name) - Toast.makeText(this, "之后「$name」说话时会显示真名", Toast.LENGTH_SHORT).show() + if (name.isNotEmpty() && name != internalKey) { + renameSpeaker(internalKey, name) } } .setNegativeButton("取消", null) .show() } - // 命名后:当前会话里这位说话人的所有行立即改显真名(颜色跟人走); - // 之后的新行也会用真名(声纹 search 命名优先返回真名)。 - private fun relabelSpeakerLines(name: String) { - val old = lastLabel - if (old == null) { - lastLabel = name + private fun renameSpeaker(oldKey: String, name: String) { + // 真名不得长得像内部键/显示名,避免与 S{n}→未识别N 的映射冲突 + if (autoLabelRegex.matches(name)) { + Toast.makeText(this, "名字不能是 S+数字 形式", Toast.LENGTH_SHORT).show() return } - if (old != name) { - val idx = labelColorIdx.remove(old) ?: colorIdxFor(name) - labelColorIdx[name] = idx - for (l in subtitleLines) { - if (l.label == old) { - l.label = name - l.colorIdx = idx - val pos = subtitleLines.indexOf(l) - if (pos >= 0) subtitleAdapter.notifyItemChanged(pos) - } + // 声纹库升级:S{n} 的嵌入转为真名注册(真名可多组嵌入并存, + // 即"两个误判声纹归为同一人",今后 search 对两组嵌入都返回同名) + SpeakerRecognition.registry.renameToName(oldKey, name) + SpeakerRecognition.fastRegistry.renameToName(oldKey, name) + for (l in subtitleLines) { + if (l.label == oldKey) { + l.label = name + l.colorIdx = colorIdxFor(name) + } + } + labelColorIdx.remove(oldKey) + if (lastConfirmedLabel == oldKey) lastConfirmedLabel = name + if (lastLabel == oldKey) lastLabel = name + if (activeLabelForWindow == oldKey) activeLabelForWindow = name + if (summarizeEnabled) sessionTranscript?.onRename(oldKey, name) + mergeAdjacentSameLabel() + subtitleAdapter.notifyItemRangeChanged(0, subtitleLines.size) + Log.i(TAG, "speaker renamed: $oldKey -> $name") + Toast.makeText(this, "已命名「$name」,相邻同名行已合并", + Toast.LENGTH_SHORT).show() + } + + // 命名归并(用户确认的规则:只并相邻同名行,时间线不回改旧行)。 + // 含未定格句子的行不并入 —— 保护"in-progress 只存在于最底行"不变量。 + private fun mergeAdjacentSameLabel() { + var i = 0 + while (i < subtitleLines.size - 1) { + val a = subtitleLines[i] + val b = subtitleLines[i + 1] + if (a.label != null && a.label == b.label && + b.sentences.none { !it.finalized } + ) { + Log.i(TAG, "rename merge: line@${i + 1} -> line@$i (${a.label})") + for (s in b.sentences.toList()) { + s.line = a + a.sentences.add(s) + } + subtitleLines.removeAt(i + 1) + subtitleAdapter.notifyItemRemoved(i + 1) + } else { + i++ } } - lastLabel = name - lastConfirmedLabel = name - activeLabelForWindow = name } private fun initMicrophone(): Boolean { @@ -1106,10 +1174,15 @@ class MainActivity : AppCompatActivity() { private inner class SubtitleAdapter : RecyclerView.Adapter() { inner class VH(val text: TextView) : RecyclerView.ViewHolder(text) - override fun onCreateViewHolder(parent: ViewGroup, type: Int): VH = VH( - LayoutInflater.from(parent.context) - .inflate(R.layout.item_subtitle, parent, false) as TextView - ) + override fun onCreateViewHolder(parent: ViewGroup, type: Int): VH { + val vh = VH( + LayoutInflater.from(parent.context) + .inflate(R.layout.item_subtitle, parent, false) as TextView + ) + // 行首标签是 ClickableSpan;没有 movementMethod 点击不会生效 + vh.text.movementMethod = LinkMovementMethod.getInstance() + return vh + } override fun getItemCount(): Int = subtitleLines.size @@ -1132,6 +1205,18 @@ class MainActivity : AppCompatActivity() { StyleSpan(Typeface.BOLD), start, start + head.length, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE ) + // 点击标签 → 命名/改名(绑定时捕获内部键,重命名后整表重绑) + val internalKey = line.label + sb.setSpan(object : ClickableSpan() { + override fun onClick(widget: View) { + showNameSpeakerDialog(internalKey) + } + + override fun updateDrawState(ds: TextPaint) { + ds.color = color + ds.isUnderlineText = true + } + }, start, start + head.length, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE) for ((i, s) in line.sentences.withIndex()) { if (s.text.isBlank()) continue if (sb.length > start + head.length) { @@ -1146,16 +1231,7 @@ class MainActivity : AppCompatActivity() { return sb } - // 句间衔接:已带标点则空格;CJK/假名结尾用句号;拉丁结尾用英文句号。 - private fun joinerAfter(prev: String): String { - val last = prev.trimEnd().lastOrNull() ?: return " " - return when { - last in "。！？；.!?;…\"”’』】》）)" -> " " - last.code >= 0x2E80 -> "。" - Character.isLetterOrDigit(last) -> ". " - else -> " " - } - } + // 句间衔接已提出为顶层函数 SubtitleFormat.kt#joinerAfter(可 JVM 单测) private fun resetSubtitles() { val n = subtitleLines.size @@ -1242,6 +1318,12 @@ class MainActivity : AppCompatActivity() { val sentence = sentenceById[sid] ?: return if (sentence.finalized) return sentence.finalized = true + if (summarizeEnabled) { + // 会话记录器挂钩①(settle):时长取 slotBook 记账(SKIP_SHORT 时无 + // 记录,按 0 处理 —— startMs≈endMs,不影响纪要内容) + val durMs = ((slotBook[sid]?.durSec ?: 0f) * 1000f).toLong() + sessionTranscript?.onSettled(sid, durMs, sentence.text) + } if (!speakersEnabled) { // 声纹缺失:退化为按 endpoint 分行(显示"未识别") sentence.line?.open = false @@ -1277,10 +1359,9 @@ class MainActivity : AppCompatActivity() { ) { Log.i(TAG, "SLOT sid=$sid VERDICT speaker=" + (label?.let { displayName(it) } ?: "未匹配/无判定")) - if (emb != null) { - lastEmbedding = emb - lastEmbeddingFast = fastEmb - nameSpeakerButton.isEnabled = true + if (summarizeEnabled && label != null) { + // 会话记录器挂钩③(说话人归属;改名走 renameSpeaker 的 RENAME 事件) + sessionTranscript?.onSpeaker(sid, label) } val sentence = findSentence(sid) ?: return var line = sentence.line ?: return @@ -1424,6 +1505,10 @@ class MainActivity : AppCompatActivity() { } else { sentence.refined = false // 未精修标记 } + if (summarizeEnabled) { + // 会话记录器挂钩②(终稿文本):与字幕层看到同一最终结果 + sessionTranscript?.onRefined(sid, sentence.text, sentence.refined) + } } // 换人强制断句后:封掉旧行,之后的 partial 开新行并归属新说话人。 @@ -1474,4 +1559,228 @@ class MainActivity : AppCompatActivity() { subtitleRecycler.scrollToPosition(last) } } + + // ==================== Summarize(会中/会后一键成稿,§4.2)================ + // 录音中点击 = 对截至当时已输出的全部定稿字幕生成纪要,录音转写全程不停。 + // 进度与结果内嵌在下半屏「会议纪要」区(生成中旧纪要不清空);成稿成功 + // 落 <会话>.notes.json,下次点击只处理新增句子续写笔记,纪要始终覆盖 + // "开会至今"全量。模型点击才 mmap、完成即 release(§4.2 模型生命周期)。 + private fun onSummarizeClick() { + if (isSummarizing) { + Toast.makeText(this, "纪要正在生成中,再点一次可取消", Toast.LENGTH_SHORT).show() + return + } + if (!LlmRuntime.ensure()) { + Toast.makeText(this, "本机 ABI 未包含摘要运行时库(仅 arm64-v8a 真机可用)", + Toast.LENGTH_LONG).show() + return + } + val modelDir = File(externalModelDir, summarizeModelDirName) + val gguf = File(modelDir, summarizeGgufName) + // 存在性检查必须在任何 native 调用之前(工程教训:缺文件会让 native exit()) + if (!gguf.isFile) { + Toast.makeText( + this, + "缺少摘要模型。请执行:\nadb push $summarizeGgufName " + + "/sdcard/Android/data/com.k2fsa.sherpa.onnx/files/models/$summarizeModelDirName/", + Toast.LENGTH_LONG + ).show() + return + } + + // 会话选择(2026-09-11 修订):只要存在当前会话,它就是唯一数据源 + // (含刚 Stop 未久的会话——供会后整理用)。新会话还没有定格句子时 + // 如实提示,绝不回退到上一场的 jsonl/notes,避免"新会议按旧笔记摘要"。 + // 仅当本进程没有会话上下文(如重启 app)时,才取盘上最新一场会议。 + val entries: List + val wallStart: Long + val baseName: String + val cur = sessionTranscript + if (cur != null) { + val snap = cur.snapshot() + if (snap.isEmpty()) { + Toast.makeText(this, "当前会议还没有定格的转写,先录一段再生成纪要", + Toast.LENGTH_SHORT).show() + return + } + entries = snap; wallStart = cur.startWallMs + baseName = cur.file.nameWithoutExtension + } else { + val f = SessionTranscript.latestFile(filesDir) + if (f == null) { + Toast.makeText(this, "还没有任何会议转写,请先录一场会议", + Toast.LENGTH_SHORT).show() + return + } + entries = SessionTranscript.load(f) + if (entries.isEmpty()) { + Toast.makeText(this, "最近一场会议没有定格句子", Toast.LENGTH_SHORT).show() + return + } + wallStart = SessionTranscript.fileStartWallMs(f) + baseName = f.nameWithoutExtension + } + if (isRecording) { + Toast.makeText(this, "纪要生成中,录音转写不会中断", Toast.LENGTH_SHORT).show() + } + // 增量断点:上次成稿留下的 <会话>.notes.json → 只处理新增句子续写笔记 + val saved = MeetingNotesIO.load( + MeetingNotesIO.pathFor(SessionTranscript.dirOf(filesDir), baseName), baseName) + var resumeIndex = 0 + var resumeNotes: String? = null + if (saved != null && saved.entriesP1Done > 0 && saved.notes != SummarizePipeline.EMPTY_NOTES) { + resumeIndex = saved.entriesP1Done.coerceAtMost(entries.size) + resumeNotes = saved.notes + Log.i(TAG, "SUMMARIZE resume from notes at $resumeIndex/${entries.size}") + } + isSummarizing = true + summarizeCancelled = false + runOnUiThread { setMinutesBusyUi(true, "正在加载摘要模型(约 3.2 GB)…") } + + val entriesRef = entries // 快照已在主线程拷贝,线程安全 + thread(true) { + Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND) + var engine: LlmEngine? = null + try { + val tLoad = System.currentTimeMillis() + engine = LlmEngine(gguf.absolutePath, SUMMARY_CTX, + if (isRecording) SUMMARY_THREADS_BUSY else SUMMARY_THREADS, + chatTemplate = summarizeChatTemplate) + Log.i(TAG, "SUMMARIZE engine loaded ms=" + + (System.currentTimeMillis() - tLoad)) + summarizeEngine = engine + runOnUiThread { + setMinutesBusyUi(true, if (resumeNotes != null) + "模型就绪,从上次断点续跑($resumeIndex/${entriesRef.size} 句已消化)…" + else "模型就绪,共 ${entriesRef.size} 句,开始滚动笔记…") + } + val prompts = PromptsLoader.load(application.assets) + val glossary = File(modelDir, "glossary.txt").let { + if (it.isFile) runCatching { it.readText() }.getOrNull() ?: "" else "" + } + val pipeline = SummarizePipeline(engine, prompts) + val result = pipeline.run( + labelOf = { l -> displayName(l) }, // 纯函数,跨线程安全 + entries = entriesRef, + meetingStartWallMs = wallStart, + glossary = glossary, + resumeNotes = resumeNotes, + resumeFromIndex = resumeIndex, + listener = object : SummarizeListener { + override fun onProgress(stage: String, done: Int, total: Int) { + runOnUiThread { + minutesStatus.text = "正在$stage: $done/$total" + } + } + override fun isCancelled(): Boolean = summarizeCancelled + }, + ) + val mdFile = File(SessionTranscript.dirOf(filesDir), "$baseName.md") + mdFile.writeText(result.markdown) + // 增量断点落盘:本次条目全部进了笔记,下次点击只消化新增句子 + MeetingNotesIO.save( + MeetingNotesIO.pathFor(SessionTranscript.dirOf(filesDir), baseName), + MeetingNotes(result.notesJson, entriesRef.size, + entriesRef.last().endMs, result.failedChunks, baseName)) + Log.i(TAG, "SUMMARIZE done: ${result.totalChunks} chunks, " + + "failed=${result.failedChunks}, saved=${mdFile.absolutePath}") + runOnUiThread { + minutesText.text = result.markdown + minutesHint.visibility = View.GONE + minutesStatus.visibility = View.VISIBLE + minutesStatus.text = "纪要已保存(${result.language}," + + "${result.totalChunks} 块" + + (if (result.failedChunks > 0) ",${result.failedChunks} 块解析失败" else "") + + "):${mdFile.name}" + copyMinutesButton.isEnabled = true + } + } catch (t: Throwable) { + Log.e(TAG, "SUMMARIZE failed", t) + runOnUiThread { + minutesStatus.visibility = View.VISIBLE + minutesStatus.text = if (t is SummarizeCancelledException) + "已取消生成(保留上一次的纪要)" + else "生成失败:${t.message}" + Toast.makeText(this, + if (t is SummarizeCancelledException) "已取消生成" + else "纪要生成失败:${t.message}", Toast.LENGTH_LONG).show() + } + } finally { + summarizeEngine = null + engine?.release() // 用完必须显式释放,勿依赖 finalize() + isSummarizing = false + runOnUiThread { setMinutesBusyUi(false, null) } + } + } + } + + /** 生成中:纪要按钮变「取消生成」+ 状态行显示 msg;结束:复原按钮。 */ + private fun setMinutesBusyUi(busy: Boolean, msg: String?) { + if (busy) { + summarizeButton.setText(R.string.cancel_minutes) + if (msg != null) { + minutesStatus.visibility = View.VISIBLE + minutesStatus.text = msg + } + } else { + summarizeButton.setText(R.string.generate_minutes) + } + } + + private fun cancelSummarize() { + summarizeCancelled = true + summarizeEngine?.abort() // 打断在途 generate;finally 里统一收尾 + minutesStatus.text = "正在取消…" + minutesStatus.visibility = View.VISIBLE + } + + /** 把当前纪要正文复制到剪贴板(方便粘到 PC/邮件)。 */ + private fun onCopyMinutesClick() { + val body = minutesText.text?.toString().orEmpty() + if (body.isBlank()) { + Toast.makeText(this, "尚无纪要可复制", Toast.LENGTH_SHORT).show() + return + } + try { + val cm = getSystemService(CLIPBOARD_SERVICE) as android.content.ClipboardManager + cm.setPrimaryClip(android.content.ClipData.newPlainText("meeting-minutes", body)) + Toast.makeText(this, "纪要全文已复制(${body.length} 字)", Toast.LENGTH_SHORT).show() + } catch (t: Throwable) { + Toast.makeText(this, "复制失败:$t", Toast.LENGTH_SHORT).show() + } + } + + /** 冷启动把上一场最近的纪要填进下方区(.md 在盘上,读文件即可)。 */ + private fun loadLastMinutes() { + try { + val latest = SessionTranscript.dirOf(filesDir) + .listFiles { f -> f.isFile && f.name.endsWith(".md") } + ?.maxByOrNull { it.lastModified() } ?: return + val body = latest.readText() + if (body.isNotBlank()) { + minutesText.text = body + minutesHint.visibility = View.GONE + copyMinutesButton.isEnabled = true + minutesStatus.visibility = View.VISIBLE + minutesStatus.text = "上次纪要:${latest.name}(生成新纪要后会覆盖)" + } + } catch (t: Throwable) { + Log.w(TAG, "loadLastMinutes: $t") + } + } + + /** + * ②息屏生存能力(重启提示部分):本进程没有会话上下文时,若盘上 + * 最新会议的 .md 缺失或比转写旧(生成中途被杀/没点生成),状态行提示可续写。 + */ + private fun hintUnfinishedMeeting() { + if (!summarizeEnabled || sessionTranscript != null) return + val f = SessionTranscript.latestFile(filesDir) ?: return + val md = File(SessionTranscript.dirOf(filesDir), f.nameWithoutExtension + ".md") + if (!md.isFile || md.lastModified() < f.lastModified()) { + minutesStatus.visibility = View.VISIBLE + minutesStatus.text = "上一场会议「${f.nameWithoutExtension}」尚未出纪要," + + "点「生成纪要」可续写(增量,只处理新增句子)" + } + } } diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/MeetingNotes.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/MeetingNotes.kt new file mode 100644 index 0000000..8c2e35f --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/MeetingNotes.kt @@ -0,0 +1,63 @@ +package com.k2fsa.sherpa.onnx + +import android.util.Log +import org.json.JSONObject +import java.io.File + +// ============================================================================ +// MeetingNotes — 会议滚动笔记的盘上断点(meetings/<会话>.notes.json) +// +// 「会中手动成稿」的增量基础:每次成稿成功后把 P1 笔记 + 已消化句子数写入; +// 下次点「生成纪要」只处理新增句子(接同一份笔记续跑),长会第 N 次点击仍是 +// 分钟级。笔记文件缺失/校验不过 → 自动退回全量重跑(里程碑 1 行为)。 +// 取代了已取消的"会中自动滚动"(RollingSummarizer,2026-09-11 需求变更)。 +// ============================================================================ + +data class MeetingNotes( + val notes: String, + val entriesP1Done: Int, + val lastEndMs: Long, + val failedChunks: Int, + val sessionId: String, +) + +object MeetingNotesIO { + private const val TAG = "sherpa-onnx" + + fun pathFor(meetingDir: File, sessionBaseName: String): File = + File(meetingDir, "$sessionBaseName.notes.json") + + /** sessionId 必须与文件一致(防止把别场会议的笔记接错);笔记 JSON 校验必需键。 */ + fun load(file: File, sessionId: String): MeetingNotes? { + if (!file.isFile) return null + return try { + val o = JSONObject(file.readText()) + if (o.optString("sessionId") != sessionId) return null + val notes = o.optString("notes") + if (SummarizePipeline.extractJson(notes) == null) return null + MeetingNotes( + notes, o.optInt("entriesP1Done"), o.optLong("lastEndMs"), + o.optInt("failedChunks"), sessionId, + ) + } catch (t: Throwable) { + Log.w(TAG, "notes file unreadable: $t") + null + } + } + + /** tmp+rename 原子写:成稿线程调用(会议目录私有 IO,毫秒级)。 */ + fun save(file: File, p: MeetingNotes) { + try { + val tmp = File(file.parentFile, file.name + ".tmp") + tmp.writeText(JSONObject().apply { + put("sessionId", p.sessionId); put("notes", p.notes) + put("entriesP1Done", p.entriesP1Done); put("lastEndMs", p.lastEndMs) + put("failedChunks", p.failedChunks); put("savedAt", System.currentTimeMillis()) + }.toString()) + if (file.exists()) file.delete() + if (!tmp.renameTo(file)) tmp.copyTo(file, overwrite = true) + } catch (t: Throwable) { + Log.w(TAG, "notes persist failed: $t") + } + } +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/SessionTranscript.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/SessionTranscript.kt new file mode 100644 index 0000000..aa1abe2 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/SessionTranscript.kt @@ -0,0 +1,216 @@ +package com.k2fsa.sherpa.onnx + +import android.os.SystemClock +import android.util.Log +import org.json.JSONObject +import java.io.BufferedWriter +import java.io.File +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale + +// ============================================================================ +// SessionTranscript — 全量会议转写记录器(摘要功能·里程碑 1,§4.2) +// +// 独立于字幕 UI 的追加式事件流:字幕层受 300 行上限并会被迁移/合并/改名, +// 这里以 sid 为主键记录每条定格句的 (startMs, endMs, speaker, text, refined), +// 落盘 JSONL(filesDir/meetings/.jsonl),回放事件重建全文。 +// +// 线程规矩:所有 on* 方法只在主线程被调用(finalizeSentence / confirmSpeaker / +// applyRefined / relabelSpeakerLines 本身就在主线程);JSONL 逐行 append 是 +// 毫秒级 IO,每句几秒一次的频率不会卡 UI。snapshot()/load() 在摘要后台线程 +// 读文件,与写入的逐行 flush 兼容(读侧容忍最后一行残缺)。 +// ============================================================================ + +class TranscriptEntry( + val sid: Long, + val startMs: Long, // 相对会话开始(ms) + val endMs: Long, // 相对会话开始(ms) + var speaker: String?,// 内部标签 S{n}/真名;null=未识别 + var text: String, + var refined: Boolean, +) + +class SessionTranscript private constructor( + val file: File, + val sessionStartElapsed: Long, +) { + val startWallMs: Long = System.currentTimeMillis() + private val entries = ArrayList() + private val bySid = HashMap() + private var writer: BufferedWriter? = null + + companion object { + private const val TAG = "sherpa-onnx" + const val DIR_NAME = "meetings" + /** 本地最多保留的会议场数(超出自动清理 jsonl/notes/md) */ + const val KEEP_SESSIONS = 50 + + private fun tsFmt() = SimpleDateFormat("yyyyMMdd-HHmm", Locale.US) + + fun dirOf(filesDir: File): File = File(filesDir, DIR_NAME).apply { mkdirs() } + + fun latestFile(filesDir: File): File? = + dirOf(filesDir).listFiles { f -> f.name.endsWith(".jsonl") } + ?.maxByOrNull { it.lastModified() } + + /** 从文件第一行 H 事件读会议真实开始时间;缺失时退回文件 mtime。 */ + fun fileStartWallMs(file: File): Long { + return try { + file.bufferedReader().use { r -> + val line = r.readLine() ?: return@use file.lastModified() + val o = JSONObject(line) + if (o.optString("t") == "H") o.optLong("wall", file.lastModified()) + else file.lastModified() + } + } catch (t: Throwable) { + file.lastModified() + } + } + + /** 录音开始时在主线程调用,新建会话文件。 */ + fun begin(filesDir: File): SessionTranscript { + val dir = dirOf(filesDir) + pruneOldSessions(dir, KEEP_SESSIONS) + val name = tsFmt().format(Date()) + ".jsonl" + var f = File(dir, name) + var i = 1 + while (f.exists()) { f = File(dir, name.removeSuffix(".jsonl") + "-$i.jsonl"); i++ } + val t = SessionTranscript(f, SystemClock.elapsedRealtime()) + t.openWriter() + t.appendLine(JSONObject().apply { + put("t", "H"); put("wall", System.currentTimeMillis()) + }) + return t + } + + /** + * ③ 保留策略:按会话名分组(jsonl/notes/md 同前缀归一组), + * 按组内最新修改时间排序,只留最近 KEEP_SESSIONS 场。 + * 删除失败静默(文件可能被并发读),不影响新会话创建。 + */ + internal fun pruneOldSessions(dir: File, keep: Int = KEEP_SESSIONS) { + runCatching { + dir.listFiles { f -> f.isFile } + ?.groupBy { it.name.substringBeforeLast('.') } + ?.entries + ?.sortedByDescending { (_, files) -> files.maxOf { it.lastModified() } } + ?.drop(keep) + ?.forEach { (_, files) -> files.forEach { runCatching { it.delete() } } } + } + } + + /** 从 JSONL 回放(摘要后台线程用)。容忍最后一行残缺。 */ + fun load(file: File): List { + val out = ArrayList() + val idx = HashMap() + var renames = ArrayList() + file.forEachLine { line -> + val o = try { JSONObject(line) } catch (t: Throwable) { return@forEachLine } + when (o.optString("t")) { + "S" -> { + val e = TranscriptEntry( + sid = o.optLong("sid"), + startMs = o.optLong("b"), endMs = o.optLong("e"), + speaker = o.optString("sp").takeIf { it.isNotEmpty() && it != "null" }, + text = o.optString("x"), refined = o.optInt("r") == 1, + ) + idx[e.sid] = e; out.add(e) + } + "R" -> idx[o.optLong("sid")]?.let { + it.text = o.optString("x", it.text); it.refined = o.optInt("r") == 1 + } + "P" -> idx[o.optLong("sid")]?.let { + val sp = o.optString("sp").takeIf { s -> s.isNotEmpty() && s != "null" } + if (sp != null) it.speaker = sp + } + "N" -> renames.add(o) + } + } + // 改名事件最后统一回放(顺序与录制时一致) + for (rn in renames) { + val from = rn.optString("from"); val to = rn.optString("to") + for (e in out) if (e.speaker == from) e.speaker = to + } + return out + } + } + + // ---- 主线程挂钩点 -------------------------------------------------------- + + /** finalizeSentence 处调用:endMs=定格时刻(相对会话开始),startMs=endMs-dur。 */ + fun onSettled(sid: Long, durMs: Long, text: String) { + if (text.isBlank()) return + val end = SystemClock.elapsedRealtime() - sessionStartElapsed + val e = TranscriptEntry(sid, (end - durMs).coerceAtLeast(0), end, null, text, false) + bySid[sid] = e; entries.add(e) + appendLine(JSONObject().apply { + put("t", "S"); put("sid", sid); put("b", e.startMs); put("e", end) + put("x", text) + }) + } + + /** applyRefined 处调用(每槽恰好一次,与字幕层幂等规则一致)。 */ + fun onRefined(sid: Long, text: String, refined: Boolean) { + val e = bySid[sid] ?: return + if (text.isBlank()) return + e.text = text; e.refined = refined + appendLine(JSONObject().apply { + put("t", "R"); put("sid", sid); put("x", text); put("r", if (refined) 1 else 0) + }) + } + + /** confirmSpeaker 处调用(含声纹线程回传后的所有修正路径)。 */ + fun onSpeaker(sid: Long, label: String?) { + val e = bySid[sid] ?: return + if (label != null && e.speaker != label) { + e.speaker = label + appendLine(JSONObject().apply { + put("t", "P"); put("sid", sid); put("sp", label) + }) + } + } + + /** renameSpeaker(点击标签命名/改名)处调用:S{n}→真名。 */ + fun onRename(from: String, to: String) { + var touched = false + for (e in entries) if (e.speaker == from) { e.speaker = to; touched = true } + if (touched) appendLine(JSONObject().apply { + put("t", "N"); put("from", from); put("to", to) + }) + } + + // ---- 读取 / 收尾 ---------------------------------------------------------- + + /** 主线程取快照(拷贝,供摘要后台线程安全读)。 */ + fun snapshot(): List = synchronized(entries) { + entries.map { + TranscriptEntry(it.sid, it.startMs, it.endMs, it.speaker, it.text, it.refined) + } + } + + fun flush() { + try { writer?.flush() } catch (t: Throwable) { Log.w(TAG, "transcript flush: $t") } + } + + fun close() { + flush() + try { writer?.close() } catch (_: Throwable) {} + writer = null + } + + // ---- 内部 ----------------------------------------------------------------- + + private fun openWriter() { + try { writer = file.bufferedWriter() } catch (t: Throwable) { + Log.e(TAG, "SessionTranscript cannot open $file", t) + } + } + + private fun appendLine(o: JSONObject) { + val w = writer ?: return + try { + w.write(o.toString()); w.write("\n"); w.flush() + } catch (t: Throwable) { Log.w(TAG, "transcript append: $t") } + } +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt index 550d6bf..f714f5d 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt @@ -123,6 +123,29 @@ class SpeakerRegistry { return best } + /** + * 点击标签命名用:把内部键 S{n}(自动注册、未命名)升级为真名, + * 或把已有真名改成另一个真名。同名允许多条嵌入并存 —— 之前误判成 + * 两个说话人、后被命名为同一名字的两组声纹,自此 search 都返回同一名。 + */ + @Synchronized + fun renameToName(oldKey: String, name: String): Boolean { + val ai = auto.indexOfFirst { it.first == oldKey } + if (ai >= 0) { + named.add(Pair(name, auto[ai].second)) + auto.removeAt(ai) + return true + } + var changed = false + for (i in named.indices) { + if (named[i].first == oldKey) { + named[i] = Pair(name, named[i].second) + changed = true + } + } + return changed + } + private fun match( list: List>, e: FloatArray, diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/SubtitleFormat.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/SubtitleFormat.kt new file mode 100644 index 0000000..cdc3984 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/SubtitleFormat.kt @@ -0,0 +1,16 @@ +package com.k2fsa.sherpa.onnx + +// ============================================================================ +// SubtitleFormat — 字幕行内句间衔接的纯函数(从 MainActivity 提出以便 JVM 单测) +// ============================================================================ + +/** 句间衔接:已带标点则空格;CJK/假名结尾用句号;拉丁结尾用英文句号。 */ +internal fun joinerAfter(prev: String): String { + val last = prev.trimEnd().lastOrNull() ?: return " " + return when { + last in "。！？；.!?;…\"”’』】》）)" -> " " + last.code >= 0x2E80 -> "。" + Character.isLetterOrDigit(last) -> ". " + else -> " " + } +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/SummarizePipeline.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/SummarizePipeline.kt new file mode 100644 index 0000000..e7bc8b3 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/SummarizePipeline.kt @@ -0,0 +1,412 @@ +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager +import android.util.Log +import org.json.JSONObject + +// ============================================================================ +// SummarizePipeline — §5 摘要流水线(分块 → 滚动笔记 → 成稿) +// +// 提示词唯一事实源是 assets/summarize/prompts.md(用户可直接编辑/复制到 PC +// llama.cpp 调试)。解析失败时退回本文件内置的同源副本 —— 两份不一致以 md +// 为准,改提示词请同步改 md。 +// +// 只在摘要专用后台线程上 new 出来跑完就丢,绝不触碰录音/主线程对象。 +// ============================================================================ + +class SummarizeCancelledException : Exception("summarize cancelled") + +data class Prompts( + val p1System: String, val p1UserTpl: String, + val p2System: String, val p2UserTpl: String, +) + +data class MeetingSummary( + val markdown: String, + val language: String, // ja / zh / en + val notesJson: String, + val totalChunks: Int, + val failedChunks: Int, +) + +interface SummarizeListener { + /** stage 人话描述("正在加载模型…"、"第 3/12 块…")。在摘要线程回调。 */ + fun onProgress(stage: String, done: Int, total: Int) + fun isCancelled(): Boolean +} + +// ---- prompts.md 解析 -------------------------------------------------------- + +object PromptsLoader { + private const val TAG = "sherpa-onnx" + private const val ASSET = "summarize/prompts.md" + + fun load(assets: AssetManager): Prompts { + val md = try { + assets.open(ASSET).bufferedReader().use { it.readText() } + } catch (t: Throwable) { + Log.w(TAG, "prompts.md missing/unreadable, using builtin: $t") + return BUILTIN + } + return try { + parse(md) + } catch (t: Throwable) { + Log.e(TAG, "prompts.md parse failed, using builtin", t) + BUILTIN + } + } + + // "## P1 …" 小节里前两个代码围栏块依次是 SYSTEM、USER;"## P2" 同理。 + internal fun parse(md: String): Prompts { + fun section(marker: String): Pair { + val start = md.indexOf(marker) + require(start >= 0) { "section $marker not found" } + val end = md.indexOf("\n## ", start + marker.length).let { + if (it < 0) md.length else it + } + val body = md.substring(start, end) + val fences = ArrayList() + var i = 0 + while (true) { + val open = body.indexOf("```", i) + if (open < 0) break + val close = body.indexOf("```", open + 3) + if (close < 0) break + fences.add(body.substring(open + 3, close).trim('\n').trim()) + i = close + 3 + } + require(fences.size >= 2) { "$marker: need 2 fenced blocks, got ${fences.size}" } + return fences[0] to fences[1] + } + val (p1s, p1u) = section("## P1") + val (p2s, p2u) = section("## P2") + require(p1s.isNotBlank() && p2s.isNotBlank()) + return Prompts(p1s, p1u, p2s, p2u) + } + + // 与 prompts.md 同源的内置副本(md 丢失/被改坏时的兜底)。 + val BUILTIN = Prompts( + p1System = "あなたは会議の文字起こしを処理する記録係です。与えられた「現在のメモ」に「新しい断片」の内容を統合し、更新後のメモをJSONだけで出力してください。\n\n" + + "厳守事項:\n" + + "- 出力は指定スキーマのJSONオブジェクトのみ。説明文・コードフェンス・前置き禁止。\n" + + "- メモは日本語で書く。ただし人名・組織名・製品名・固有名詞は原文表記をそのまま維持。\n" + + "- 数字、金額、日時、期限、回数、単位は原文どおり正確に転記。ざっくり要約しない。\n" + + "- 否定・保留・反対の意図を消さない。「見送り」「再検討」「反対意見」はそれ自体が重要な情報。\n" + + "- 各項目に根拠となる発話のタイムレンジを「出典」に付ける(形式: \"mm:ss-mm:ss\"、複数可 \";\" 区切り)。\n" + + "- 断片に新情報がない場合はメモをほぼ変更しない(前回末尾は必ず更新)。\n" + + "- 全体で500字を超えそうな場合、重要度の低い過去の項目を統合・削除して圧縮する。削除時は行動項目と已定事項を最後に守る。\n" + + "- 「前回末尾」には、次の断片と文脈が繋がるための直近話題を1文で書く。\n" + + "- JSONの値の中にダブルクォートが必要なら \\\" にエスケープ。改行は \\n を使う。\n\n" + + "スキーマ(キー名は変更禁止):\n" + + "{\"已定事項\":[{\"内容\":\"\",\"出典\":\"\"}],\"討論中\":[{\"内容\":\"\",\"出典\":\"\"}],\"行動項目\":[{\"内容\":\"\",\"担当者\":\"\",\"期限\":\"\",\"出典\":\"\"}],\"人物\":[{\"名前\":\"\",\"立場\":\"\"}],\"前回末尾\":\"\"}", + p1UserTpl = "【現在のメモ】\n{notes_json}\n\n" + + "【新しい断片】(タイムスタンプは会議開始からの分:秒、[]内は話者ラベル)\n{chunk_text}\n\n" + + "更新後のメモJSONを出力してください。", + p2System = "あなたは会議の公式議事録作成者です。会議メモ(JSON)と用語集を受け取ったら、指定言語で完成した議事録をMarkdownで出力してください。\n\n" + + "厳守事項:\n" + + "- 出力言語は指定された「成稿言語」を厳守。日本語メモの内容をそのまま指定言語の書き言葉文体で記述する。\n" + + "- 本文は成稿言語の文字のみで書く:中文稿なら仮名を一切出さない、English稿なら漢字・仮名を一切出さない、日本語稿なら製品名以外に英語の文を残さない。\n" + + "- ただし人物・会社・ブランドなどの固有名詞は、メモ・用語集にある原文表記のまま保留してよい(上記の文字制限の例外)。\n" + + "- 見出しは次の4つを必ずこの順で:「議題と経過」「決定事項」「行動項目」「未決事項・持ち帰り」。該当ゼロなら「(なし)」と明記。\n" + + "- 行動項目は表形式(担当者 / 内容 / 期限 / 出典)。担当者が不明なら「未定」と書く。\n" + + "- 各項目の末尾に出典を付ける。形式: `出典: mm:ss-mm:ss`。出典はメモJSONの「出典」フィールドのみを使う。捏造禁止。\n" + + "- メモにない事実・推測・補足説明を追加しない。\n" + + "- 人名・製品名は用語集の正式表記に統一する(用語集にない表記はメモのまま採用)。\n" + + "- 冒頭にメタ情報を置く: 会議日時(指定があれば)、話者数、使用言語、文字起こし総尺。\n" + + "- Markdown本文のみ出力。コードフェンスで全体を囲まない。", + p2UserTpl = "【成稿言語】{language_name}\n\n" + + "【会議メタ情報】開始: {meeting_start} / 話者数: {n_speakers} / 総尺: {total_duration}\n\n" + + "【用語集(正式表記の一覧。人名・製品名の誤変換をこれに統一)】\n{glossary_lines}\n\n" + + "【会議メモJSON】\n{notes_json}\n\n" + + "上記に基づき、議事録Markdownを出力してください。", + ) +} + +// ---- 流水线主体 -------------------------------------------------------------- + +class SummarizePipeline( + private val engine: LlmEngine, + private val prompts: Prompts, +) { + companion object { + private const val TAG = "sherpa-onnx" + private const val CHUNK_TARGET = 2500 // §5:2500±500 字/块 + private const val CHUNK_MIN = 2000 + private const val CHUNK_MAX = 3000 + // P1 输入(笔记+块+固定文案)字符预算:超了就对半拆块, + // 保证 ≤8192 ctx 永远够用(日语 1 字 ≈ 1 token 多一点) + private const val P1_MAX_PROMPT_CHARS = 6500 + private const val NOTES_RESERVE = 1200 // 笔记+模板文案预留 + val EMPTY_NOTES = + "{\"已定事項\":[],\"討論中\":[],\"行動項目\":[],\"人物\":[],\"前回末尾\":\"\"}" + + // 脚本占比判语种:假名出现即日语信号(日语里汉字占比也高,不能只看汉字); + // 无假名时看汉字 vs 拉丁;全兜底 en。 + fun detectLanguage(text: String): String { + var kana = 0 + var han = 0 + var latin = 0 + for (ch in text) { + val cp = ch.code + when { + cp in 0x3040..0x30FF || cp in 0x31F0..0x31FF || cp in 0xFF66..0xFF9F -> kana++ + cp in 0x3400..0x4DBF || cp in 0x4E00..0x9FFF || cp in 0xF900..0xFAFF -> han++ + cp in 0x41..0x5A || cp in 0x61..0x7A -> latin++ + } + } + return when { + kana > 0 && kana * 20 >= kana + han -> "ja" // 假名/汉字 ≥ 5% + han >= latin && han > 0 -> "zh" + else -> "en" + } + } + + fun languageName(lang: String) = when (lang) { + "ja" -> "日本語" + "zh" -> "中文(简体书面语)" + else -> "English (formal written English)" + } + + fun mmss(ms: Long): String { + val t = ms / 1000 + return "%02d:%02d".format(t / 60, t % 60) + } + + fun formatDate(wallMs: Long): String = + java.text.SimpleDateFormat("yyyy-MM-dd HH:mm", java.util.Locale.US) + .format(java.util.Date(wallMs)) + + /** P2 成稿 user 提示词构造(滚动成稿与里程碑 1 共用,保证两路输出同构)。 */ + fun buildDraftUserPrompt( + prompts: Prompts, lang: String, meetingStartWallMs: Long, + nSpeakers: Int, totalDurMs: Long, glossary: String, notes: String, + ): String = prompts.p2UserTpl + .replace("{language_name}", languageName(lang)) + .replace("{meeting_start}", formatDate(meetingStartWallMs)) + .replace("{n_speakers}", nSpeakers.toString()) + .replace("{total_duration}", mmss(totalDurMs)) + .replace("{glossary_lines}", glossary.ifBlank { "(なし)" }) + .replace("{notes_json}", notes) + + // §5 分块铁律:句子边界聚合到 2500±500 字;块间重叠 1 句; + // 同一说话人连续段(轮次)不拆开:切点落在同人连续段中间则延到段尾。 + // 另:超 P1 预算的块按行二分一次,保证 prompt ≤ ctx。 + internal fun chunk(lines: List): List { + val blocks = ArrayList() + var i = 0 + var overlap: String? = null + while (i < lines.size) { + var j = i + var len = 0 + while (j < lines.size) { + val l = lines[j] + if (len > 0 && len + l.length > CHUNK_MAX) break + if (len >= CHUNK_TARGET) { + // 与上一句同一说话人 → 轮次未完,继续收 + if (j == i || speakerOf(l) != speakerOf(lines[j - 1])) { + if (len >= CHUNK_MIN) break + } + } + len += l.length + 1 + j++ + } + if (j == i) j++ // 单句超长的极端情况:至少吞一句 + blocks.add( + StringBuilder().apply { + overlap?.let { append(it).append('\n') } + for (k in i until j) append(lines[k]).append('\n') + }.toString().trimEnd() + ) + overlap = if (j < lines.size) lines[j - 1] else null + i = j + } + + val fine = ArrayList() + for (c in blocks) { + if (c.length + NOTES_RESERVE <= P1_MAX_PROMPT_CHARS) { + fine.add(c) + } else { + val ls = c.split('\n').filter { it.isNotBlank() } + val half = (ls.size / 2).coerceAtLeast(1) + val a = ls.subList(0, half).joinToString("\n") + val b = ls.subList(half, ls.size).joinToString("\n") + if (a.isNotBlank()) fine.add(a) + if (b.isNotBlank()) fine.add(b) + } + } + return fine + } + + private fun speakerOf(line: String): String { + val a = line.indexOf('[') + val b = line.indexOf(']', a + 1) + return if (a >= 0 && b > a) line.substring(a, b) else line + } + + // 笔记输出健壮性:取首个平衡 {...}(前后的客套话/代码围栏自然被忽略)、 + // 必需键检查 + internal fun extractJson(raw: String): String? { + val start = raw.indexOf('{') + if (start < 0) return null + var depth = 0 + var inStr = false + var esc = false + for (i in start until raw.length) { + val c = raw[i] + if (esc) { esc = false; continue } + when { + c == '\\' && inStr -> esc = true + c == '"' -> inStr = !inStr + !inStr && c == '{' -> depth++ + !inStr && c == '}' -> { + depth-- + if (depth == 0) { + val cand = raw.substring(start, i + 1) + return try { + val o = JSONObject(cand) + if (o.has("已定事項") && o.has("行動項目")) cand else null + } catch (t: Throwable) { + null + } + } + } + } + } + return null + } + + internal fun cleanDraft(md: String): String { + var s = md.trim() + if (s.startsWith("```")) { + s = s.substringAfter('\n').substringBeforeLast("```") + } + s = s.replace( + Regex("<\\|channel>thought.*?", RegexOption.DOT_MATCHES_ALL), "" + ) + s = s.replace(Regex("\n{3,}"), "\n\n") + return s.trim() + } + + /** + * 输出语种纯度统计(2026-09-11 起只用于对账日志,不再触发整篇重写): + * zh → 假名 > 8 个判违规;en → 汉字+假名 > 6 个判违规;ja → 不检查。 + * 返回违规样本描述,null = 通过。 + */ + internal fun scriptViolation(lang: String, text: String): String? { + var kana = 0 + var han = 0 + var firstBad = -1 + for (i in text.indices) { + val cp = text[i].code + when { + cp in 0x3040..0x30FF || cp in 0xFF66..0xFF9F -> { kana++; if (firstBad < 0) firstBad = i } + cp in 0x3400..0x4DBF || cp in 0x4E00..0x9FFF || cp in 0xF900..0xFAFF -> { + han++ + if (firstBad < 0) firstBad = i + } + } + } + fun sample(): String = text.substring( + firstBad, (firstBad + 16).coerceAtMost(text.length) + ).replace("\n", " ") + return when (lang) { + "zh" -> if (kana > 8) "假名 $kana 个(如「${sample()}」)" else null + "en" -> if (kana + han > 6) "中日文字 $han+$kana 个(around \"${sample()}\")" else null + else -> null + } + } + } + + /** + * @param labelOf 内部说话人键(S{n}/真名)→ 显示名(未识别N/真名) + * @param entries SessionTranscript 快照(定格顺序,不可为空) + * @param resumeNotes/resumeFromIndex M2:会中滚动笔记已覆盖前 resumeFromIndex 句, + * 仅补跑尾部(进度、成稿复用同一份笔记) + */ + fun run( + labelOf: (String?) -> String, + entries: List, + meetingStartWallMs: Long, + glossary: String, + listener: SummarizeListener, + resumeNotes: String? = null, + resumeFromIndex: Int = 0, + ): MeetingSummary { + if (entries.isEmpty()) throw IllegalArgumentException("会议转写为空") + val lang = detectLanguage(entries.joinToString("") { it.text }) + + // ---- 分块(从断点句开始;断点前借 1 句做上下文衔接,同块间重叠规则)---- + val startIdx = resumeFromIndex.coerceIn(0, entries.size) + fun renderLine(e: TranscriptEntry) = "(%s-%s)[%s] %s".format( + mmss(e.startMs), mmss(e.endMs), labelOf(e.speaker), e.text.trim() + ) + val lines = ArrayList() + if (startIdx > 0 && startIdx < entries.size) lines.add(renderLine(entries[startIdx - 1])) + for (e in entries.drop(startIdx)) lines.add(renderLine(e)) + val chunks = chunk(lines) + val total = chunks.size + Log.i(TAG, "SUMMARIZE: ${entries.size} utterances -> $total chunks, " + + "lang=$lang, resumeIdx=$startIdx") + + // ---- 滚动笔记(§5.2)---- + var notes = resumeNotes ?: EMPTY_NOTES + var failed = 0 + val tP1Start = System.currentTimeMillis() + chunks.forEachIndexed { idx, chunkText -> + if (listener.isCancelled()) throw SummarizeCancelledException() + val user = prompts.p1UserTpl + .replace("{notes_json}", notes) + .replace("{chunk_text}", chunkText) + listener.onProgress("滚动笔记", idx + 1, total) + val t0 = System.currentTimeMillis() + var out = engine.generate( + systemPrompt = prompts.p1System, userPrompt = user, + maxNewTokens = 900, temperature = 0f, + ) + var parsed = extractJson(out) + if (parsed == null) { + // 一次修复重试:把畸形输出递回去(仍失败则保留旧笔记,计 failed) + if (listener.isCancelled()) throw SummarizeCancelledException() + val fix = user + "\n\n(前回の出力はJSONとして不正でした: " + + out.take(200).replace("\n", " ") + " 正しいJSONのみを再出力してください。)" + out = engine.generate( + systemPrompt = prompts.p1System, userPrompt = fix, + maxNewTokens = 900, temperature = 0f, + ) + parsed = extractJson(out) + } + if (parsed != null) notes = parsed else failed++ + // 测量点(baseline,不改行为):raw≫json 说明解码浪费在废话/思考上; + // ms 是单块真实耗时,决定 diff 化/大块合并的参数 + Log.i(TAG, "P1 chunk=${idx + 1}/$total prompt_len=${user.length} " + + "raw=${out.length} json=${parsed?.length ?: 0} " + + "retry=${if (parsed == null || out.length < 1) 1 else 0} " + + "ms=${System.currentTimeMillis() - t0}") + } + Log.i(TAG, "P1 phase total_ms=${System.currentTimeMillis() - tP1Start} " + + "chunks=$total failed=$failed") + + // ---- 成稿(§5.3)---- + if (listener.isCancelled()) throw SummarizeCancelledException() + listener.onProgress("成稿", total, total) + val nSpeakers = entries.map { it.speaker ?: "?" }.distinct().size + val user2 = buildDraftUserPrompt( + prompts, lang, meetingStartWallMs, nSpeakers, + entries.last().endMs, glossary, notes + ) + val tP2 = System.currentTimeMillis() + val md = cleanDraft(engine.generate( + systemPrompt = prompts.p2System, userPrompt = user2, + maxNewTokens = 2500, temperature = 0.5f, topK = 64, topP = 0.95f, + )) + Log.i(TAG, "P2 draft prompt_len=${user2.length} draft_len=${md.length} " + + "ms=${System.currentTimeMillis() - tP2}") + // 语种"校正"(整篇重写)已按用户要求移除:识别是什么语言,纪要就存 + // 什么语言,不再为混语倾向多花一次 2500 token 生成。这里只留一行 + // 纯统计日志(零成本),纪要中的混语留给人工处理。 + val viol = scriptViolation(lang, md) + if (viol != null) Log.i(TAG, "SUMMARIZE langcheck NOTE($lang): $viol (no rewrite)") + Log.i(TAG, "SUMMARIZE wall clock total_ms=${System.currentTimeMillis() - tP1Start}") + return MeetingSummary(md, lang, notes, total, failed) + } +} diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index 29abe2d..205c48a 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -9,7 +9,6 @@ - + + + android:orientation="horizontal" + android:paddingTop="4dp" + android:paddingBottom="4dp">