From 80d3f59ecfa2c5d238f72e65a52d3babda3a39f4 Mon Sep 17 00:00:00 2001 From: liwenhao2 Date: Thu, 10 Sep 2026 14:47:15 +0800 Subject: [PATCH] =?UTF-8?q?Initial=20commit:=20SherpaOnnx2Pass=20=E5=8F=8C?= =?UTF-8?q?=E9=80=9A=E9=81=93=20ASR=20=E5=AE=89=E5=8D=93=E5=B7=A5=E7=A8=8B?= =?UTF-8?q?=EF=BC=88=E5=90=AB=E8=AF=B4=E8=AF=9D=E4=BA=BA=E5=8C=BA=E5=88=86?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 18 + .idea/.gitignore | 3 + .idea/AndroidProjectSystem.xml | 6 + .idea/compiler.xml | 6 + .idea/deploymentTargetSelector.xml | 11 + .idea/gradle.xml | 19 + .idea/migrations.xml | 10 + .idea/misc.xml | 10 + .idea/runConfigurations.xml | 17 + .idea/vcs.xml | 7 + app/.gitignore | 1 + app/build.gradle | 57 + app/proguard-rules.pro | 21 + .../sherpa/onnx/ExampleInstrumentedTest.kt | 24 + app/src/main/.gitignore | 1 + app/src/main/AndroidManifest.xml | 33 + app/src/main/assets/.gitkeep | 0 .../com/k2fsa/sherpa/onnx/FeatureConfig.kt | 11 + .../sherpa/onnx/HomophoneReplacerConfig.kt | 7 + .../com/k2fsa/sherpa/onnx/MainActivity.kt | 1239 ++++++++++++++ .../k2fsa/sherpa/onnx/OfflineRecognizer.kt | 1514 +++++++++++++++++ .../com/k2fsa/sherpa/onnx/OfflineStream.kt | 42 + .../com/k2fsa/sherpa/onnx/OnlineRecognizer.kt | 812 +++++++++ .../com/k2fsa/sherpa/onnx/OnlineStream.kt | 46 + .../java/com/k2fsa/sherpa/onnx/QnnConfig.kt | 7 + .../com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt | 235 +++ .../onnx/SpeakerEmbeddingExtractorConfig.kt | 8 + app/src/main/jniLibs/.gitkeep | 0 app/src/main/jniLibs/arm64-v8a/.gitkeep | 0 app/src/main/jniLibs/armeabi-v7a/.gitkeep | 0 app/src/main/jniLibs/x86/.gitkeep | 0 app/src/main/jniLibs/x86_64/.gitkeep | 0 .../drawable-v24/ic_launcher_foreground.xml | 30 + .../res/drawable/ic_launcher_background.xml | 170 ++ app/src/main/res/layout/activity_main.xml | 76 + app/src/main/res/layout/item_subtitle.xml | 8 + .../res/mipmap-anydpi-v26/ic_launcher.xml | 5 + .../mipmap-anydpi-v26/ic_launcher_round.xml | 5 + app/src/main/res/mipmap-hdpi/ic_launcher.webp | Bin 0 -> 1404 bytes .../res/mipmap-hdpi/ic_launcher_round.webp | Bin 0 -> 2898 bytes app/src/main/res/mipmap-mdpi/ic_launcher.webp | Bin 0 -> 982 bytes .../res/mipmap-mdpi/ic_launcher_round.webp | Bin 0 -> 1772 bytes .../main/res/mipmap-xhdpi/ic_launcher.webp | Bin 0 -> 1900 bytes .../res/mipmap-xhdpi/ic_launcher_round.webp | Bin 0 -> 3918 bytes .../main/res/mipmap-xxhdpi/ic_launcher.webp | Bin 0 -> 2884 bytes .../res/mipmap-xxhdpi/ic_launcher_round.webp | Bin 0 -> 5914 bytes .../main/res/mipmap-xxxhdpi/ic_launcher.webp | Bin 0 -> 3844 bytes .../res/mipmap-xxxhdpi/ic_launcher_round.webp | Bin 0 -> 7778 bytes app/src/main/res/values-night/themes.xml | 16 + app/src/main/res/values/colors.xml | 10 + app/src/main/res/values/strings.xml | 15 + app/src/main/res/values/themes.xml | 16 + app/src/main/res/xml/backup_rules.xml | 13 + .../main/res/xml/data_extraction_rules.xml | 19 + .../com/k2fsa/sherpa/onnx/ExampleUnitTest.kt | 17 + build.gradle | 6 + gradle.properties | 25 + gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes gradle/wrapper/gradle-wrapper.properties | 6 + gradlew | 185 ++ gradlew.bat | 89 + settings.gradle | 16 + 62 files changed, 4892 insertions(+) create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/AndroidProjectSystem.xml create mode 100644 .idea/compiler.xml create mode 100644 .idea/deploymentTargetSelector.xml create mode 100644 .idea/gradle.xml create mode 100644 .idea/migrations.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/runConfigurations.xml create mode 100644 .idea/vcs.xml create mode 100644 app/.gitignore create mode 100644 app/build.gradle create mode 100644 app/proguard-rules.pro create mode 100644 app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt create mode 100644 app/src/main/.gitignore create mode 100644 app/src/main/AndroidManifest.xml create mode 100644 app/src/main/assets/.gitkeep create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt create mode 100644 app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.kt create mode 100644 app/src/main/jniLibs/.gitkeep create mode 100644 app/src/main/jniLibs/arm64-v8a/.gitkeep create mode 100644 app/src/main/jniLibs/armeabi-v7a/.gitkeep create mode 100644 app/src/main/jniLibs/x86/.gitkeep create mode 100644 app/src/main/jniLibs/x86_64/.gitkeep create mode 100644 app/src/main/res/drawable-v24/ic_launcher_foreground.xml create mode 100644 app/src/main/res/drawable/ic_launcher_background.xml create mode 100644 app/src/main/res/layout/activity_main.xml create mode 100644 app/src/main/res/layout/item_subtitle.xml create mode 100644 app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml create mode 100644 app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml create mode 100644 app/src/main/res/mipmap-hdpi/ic_launcher.webp create mode 100644 app/src/main/res/mipmap-hdpi/ic_launcher_round.webp create mode 100644 app/src/main/res/mipmap-mdpi/ic_launcher.webp create mode 100644 app/src/main/res/mipmap-mdpi/ic_launcher_round.webp create mode 100644 app/src/main/res/mipmap-xhdpi/ic_launcher.webp create mode 100644 app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp create mode 100644 app/src/main/res/mipmap-xxhdpi/ic_launcher.webp create mode 100644 app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp create mode 100644 app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp create mode 100644 app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp create mode 100644 app/src/main/res/values-night/themes.xml create mode 100644 app/src/main/res/values/colors.xml create mode 100644 app/src/main/res/values/strings.xml create mode 100644 app/src/main/res/values/themes.xml create mode 100644 app/src/main/res/xml/backup_rules.xml create mode 100644 app/src/main/res/xml/data_extraction_rules.xml create mode 100644 app/src/test/java/com/k2fsa/sherpa/onnx/ExampleUnitTest.kt create mode 100644 build.gradle create mode 100644 gradle.properties create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100644 gradlew create mode 100644 gradlew.bat create mode 100644 settings.gradle diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b984051 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties + +# 模型文件体积大，不入库；运行时手动放入 app/src/main/assets/ +*.onnx diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/AndroidProjectSystem.xml b/.idea/AndroidProjectSystem.xml new file mode 100644 index 0000000..4a53bee --- /dev/null +++ b/.idea/AndroidProjectSystem.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..b589d56 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/deploymentTargetSelector.xml b/.idea/deploymentTargetSelector.xml new file mode 100644 index 0000000..ca16a99 --- /dev/null +++ b/.idea/deploymentTargetSelector.xml @@ -0,0 +1,11 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/gradle.xml b/.idea/gradle.xml new file mode 100644 index 0000000..639c779 --- /dev/null +++ b/.idea/gradle.xml @@ -0,0 +1,19 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/migrations.xml b/.idea/migrations.xml new file mode 100644 index 0000000..f8051a6 --- /dev/null +++ b/.idea/migrations.xml @@ -0,0 +1,10 @@ + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..976cf21 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml new file mode 100644 index 0000000..16660f1 --- /dev/null +++ b/.idea/runConfigurations.xml @@ -0,0 +1,17 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..fdf1fc8 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 0000000..42afabf --- /dev/null +++ b/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle new file mode 100644 index 0000000..f1f48ca --- /dev/null +++ b/app/build.gradle @@ -0,0 +1,57 @@ +plugins { + id 'com.android.application' + id 'org.jetbrains.kotlin.android' +} + +android { + namespace 'com.k2fsa.sherpa.onnx' + compileSdk 32 + + defaultConfig { + applicationId "com.k2fsa.sherpa.onnx" + minSdk 21 + targetSdk 32 + versionCode 20260901 + versionName "1.13.7" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + + // Only ship the ABIs actually needed: arm64 real devices + x86_64 + // emulator. (Drop 'x86_64' too if you never use the emulator.) + ndk { + abiFilters 'arm64-v8a', 'x86_64' + } + } + + // Store .onnx models uncompressed in the APK. Compressed assets must be + // inflated stream-by-stream at load time, which makes the 2+ GB model set + // dramatically slower to initialize (and can trip the 5s ANR timer). + androidResources { + noCompress 'onnx' + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = '1.8' + } +} + +dependencies { + + implementation 'androidx.core:core-ktx:1.7.0' + implementation 'androidx.appcompat:appcompat:1.5.1' + implementation 'com.google.android.material:material:1.7.0' + implementation 'androidx.constraintlayout:constraintlayout:2.1.4' + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'androidx.test.ext:junit:1.1.4' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.0' +} \ No newline at end of file diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro new file mode 100644 index 0000000..481bb43 --- /dev/null +++ b/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt b/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt new file mode 100644 index 0000000..1833832 --- /dev/null +++ b/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.k2fsa.sherpa.onnx + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.k2fsa.sherpa.onnx", appContext.packageName) + } +} \ No newline at end of file diff --git a/app/src/main/.gitignore b/app/src/main/.gitignore new file mode 100644 index 0000000..140f8cf --- /dev/null +++ b/app/src/main/.gitignore @@ -0,0 +1 @@ +*.so diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..0cbbfaf --- /dev/null +++ b/app/src/main/AndroidManifest.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/assets/.gitkeep b/app/src/main/assets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt new file mode 100644 index 0000000..2d2eba4 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt @@ -0,0 +1,11 @@ +package com.k2fsa.sherpa.onnx + +data class FeatureConfig( + var sampleRate: Int = 16000, + var featureDim: Int = 80, + var dither: Float = 0.0f +) + +fun getFeatureConfig(sampleRate: Int, featureDim: Int): FeatureConfig { + return FeatureConfig(sampleRate = sampleRate, featureDim = featureDim) +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt new file mode 100644 index 0000000..8baa3ff --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt @@ -0,0 +1,7 @@ +package com.k2fsa.sherpa.onnx + +data class HomophoneReplacerConfig( + var dictDir: String = "", // unused + var lexicon: String = "", + var ruleFsts: String = "", +) diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt new file mode 100644 index 0000000..95c3e94 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -0,0 +1,1239 @@ +package com.k2fsa.sherpa.onnx + +import android.Manifest +import android.content.pm.PackageManager +import android.graphics.Color +import android.graphics.Typeface +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaRecorder +import android.os.Bundle +import android.os.Handler +import android.os.HandlerThread +import android.os.Looper +import android.os.Process +import android.text.SpannableStringBuilder +import android.text.Spanned +import android.text.method.ScrollingMovementMethod +import android.text.style.ForegroundColorSpan +import android.text.style.StyleSpan +import android.util.Log +import android.view.LayoutInflater +import android.view.View +import android.view.ViewGroup +import android.widget.Button +import android.widget.EditText +import android.widget.TextView +import android.widget.Toast +import androidx.appcompat.app.AlertDialog +import androidx.appcompat.app.AppCompatActivity +import androidx.core.app.ActivityCompat +import androidx.recyclerview.widget.LinearLayoutManager +import androidx.recyclerview.widget.RecyclerView +import java.io.File +import java.util.concurrent.atomic.AtomicLong +import kotlin.concurrent.thread + +private const val TAG = "sherpa-onnx" +private const val REQUEST_RECORD_AUDIO_PERMISSION = 200 + +// adb emu avd hostmicon +// to enable microphone inside the emulator +class MainActivity : AppCompatActivity() { + private val permissions: Array = arrayOf(Manifest.permission.RECORD_AUDIO) + + private lateinit var onlineRecognizer: OnlineRecognizer + private lateinit var offlineRecognizer: OfflineRecognizer + + // 精修提示注入用(setConfig 复用同一个 Kotlin 配置对象;只在 + // subtitleHandler 串行队列上被改写,nano 解码期间不会有人读它) + private lateinit var offlineConfig: OfflineRecognizerConfig + private var audioRecord: AudioRecord? = null + private lateinit var recordButton: Button + private lateinit var textView: TextView + private var recordingThread: Thread? = null + + private val audioSource = MediaRecorder.AudioSource.MIC + private val sampleRateInHz = 16000 + private val channelConfig = AudioFormat.CHANNEL_IN_MONO + + private var samplesBuffer = arrayListOf() + + // Note: We don't use AudioFormat.ENCODING_PCM_FLOAT + // since the AudioRecord.read(float[]) needs API level >= 23 + // but we are targeting API level >= 21 + private val audioFormat = AudioFormat.ENCODING_PCM_16BIT + + // ---- Subtitle dialog stream (UI layer) ---------------------------------- + // A SubtitleLine is one speaker's turn: a list of finalized/in-progress + // sentences rendered as a single colored paragraph. A new line is started + // ONLY when the endpoint voice print says "a different speaker" (or when + // speaker id is disabled => one sentence per line). + // ALL of this state is touched on the main thread only (worker threads + // post updates through uiHandler). + private class Sentence( + val id: Long, // utterance id allocated on the recording thread + var text: String, + var finalized: Boolean, + ) { + var line: SubtitleLine? = null + var refined: Boolean = false // false => 底稿(第一遍)未被精修覆盖 + } + + private class SubtitleLine( + val id: Long, + var label: String?, // speaker display name; null => 未识别 + var colorIdx: Int, // index into speakerPalette + ) { + val sentences = ArrayList() + var open = true // active line: still accepting sentences + } + + private val uiHandler = Handler(Looper.getMainLooper()) + private val subtitleThread = HandlerThread("second-pass").also { it.start() } + private val subtitleHandler by lazy { Handler(subtitleThread.looper) } + private val lineIdSeq = AtomicLong(0) + private val subtitleLines = ArrayList() + private val sentenceById = HashMap() + private val labelColorIdx = HashMap() + private var colorSeq = 0 + private var lastConfirmedLabel: String? = null // partial 行沿用的最近确认说话人 + private var lastSettledSid = -1L // 主线程:最近定格的句子槽位 + private lateinit var subtitleRecycler: RecyclerView + private lateinit var subtitleAdapter: SubtitleAdapter + + // main-thread only: most recent confirmed speaker label (naming dialog, + // new-line inheritance). Written on the main thread in finalizeSentence. + private var lastLabel: String? = null + + private val speakerPalette = intArrayOf( + Color.parseColor("#1565C0"), // blue + Color.parseColor("#2E7D32"), // green + Color.parseColor("#E65100"), // orange + Color.parseColor("#6A1B9A"), // purple + Color.parseColor("#00695C"), // teal + Color.parseColor("#C62828"), // red + Color.parseColor("#283593"), // indigo + Color.parseColor("#5D4037"), // brown + ) + + // keep at most this many rendered lines; older ones are recycled so long + // sessions never grow the data or the view tree unboundedly + private val maxSubtitleLines = 300 + + // Language of the first-pass (nemotron-3.5) model. The model selects ONE + // language prompt per stream; there is no native "subset of languages" + // option, so we combine: + // 1. the stream option "language": "auto" (detect), "zh"/"zh-CN"/"zh-TW", + // "ja"/"ja-JP", "en" (see prompt_dictionary in encoder.int8.onnx); + // 2. when "auto", a zh/ja/en post-filter (isInAllowedLanguages) so that + // segments decoded into other languages are dropped. + private var firstPassLanguage = "auto" + + // ---- Speaker separation via voice-print embeddings --------------------- + // Embedding model candidates, tried in order. Place one in the external + // models dir (preferred) or at the ROOT of assets/. Download: + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models + private val speakerModels = listOf( + "3dspeaker_speech_eres2netv2_sv_zh-cn_16k-common.onnx", // more accurate + "3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx", // smaller fallback + ) + private var activeSpeakerModel: String? = null + private val speakerThreshold = 0.5f + private var speakersEnabled = false + private var lastEmbedding: FloatArray? = null + private lateinit var nameSpeakerButton: Button + private lateinit var clearSpeakersButton: Button + + // ---- rolling speaker-change detection (需求一.4) ------------------------- + // Every ~1.5 s the speaker-check thread embeds the last 2 s window and + // compares it (cosine) with the window reference taken at the start of + // the current utterance. A big similarity drop == voice changed mid + // utterance == force an endpoint cut at the switch point. + private class SpeakerSwitch(val label: String?, val cutSamples: Int) + + private val speakerCheckThread = HandlerThread("speaker-window").also { it.start() } + private val speakerCheckHandler by lazy { Handler(speakerCheckThread.looper) } + private val speakerSwitchDrop = 0.35f // cos(window, reference) below this => switch + @Volatile private var pendingSwitch: SpeakerSwitch? = null + @Volatile private var windowRefEmb: FloatArray? = null + @Volatile private var activeLabelForWindow: String? = null + @Volatile private var checkSerial = 0L // bumped when an utterance settles; + // stale in-flight jobs are dropped + + // ---- buffer accounting (chunk == 100 ms) -------------------------------- + // Parallel per-chunk RMS lets the forced cut snap to a low-energy (natural) + // chunk boundary within ±3 chunks (±0.3 s) of the detection point. + private val chunkRms = ArrayList() + private var bufferTotal = 0 + + // ---- refine-slot bookkeeping (需求二.4/二.5) ----------------------------- + // 每个句子槽位一条全字段对账日志(时长/路径/声纹判定/底稿/精修/结果)。 + // slotBook 仅主线程访问(submit/finalize/timeout 都经 uiHandler 排队)。 + private class SlotRec( + val path: String, + val durSec: Float, + val speaker: String?, + val first: String, + val prompt: String, + var watchdog: Runnable? = null, + ) + + private val slotBook = LinkedHashMap() + private val minRefineSamples = 6400 // 0.4 s: below this keep 第一遍底稿 + private val refineTimeoutMs = 8000L // 第二遍超时 → 底稿铁律兜底 + + @Volatile + private var isRecording: Boolean = false + + @Volatile + private var modelsReady: Boolean = false + + // ---- External model loading (adb push) --------------------------------- + // Models live in Android/data//files/models/ instead of assets: + // adb push /sdcard/Android/data/com.k2fsa.sherpa.onnx/files/models/ + // If the first-pass model dir is found there we load everything from the + // file system (assetManager = null, absolute paths); otherwise we fall + // back to the copy inside the APK's assets. + private val firstModelDirName = + "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11" + private val secondModelDirName = "sherpa-onnx-funasr-nano-int8-2025-12-30" + private val externalModelDir: File by lazy { + File(getExternalFilesDir(null), "models") + } + private var loadFromAssets = true + private var modelPrefix = "" + + override fun onRequestPermissionsResult( + requestCode: Int, permissions: Array, grantResults: IntArray + ) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + val permissionToRecordAccepted = if (requestCode == REQUEST_RECORD_AUDIO_PERMISSION) { + grantResults[0] == PackageManager.PERMISSION_GRANTED + } else { + false + } + + if (!permissionToRecordAccepted) { + Log.e(TAG, "Audio record is disallowed") + finish() + } + + Log.i(TAG, "Audio record is permitted") + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION) + + recordButton = findViewById(R.id.record_button) + recordButton.isEnabled = false + recordButton.setOnClickListener { onclick() } + + nameSpeakerButton = findViewById(R.id.name_speaker_button) + nameSpeakerButton.isEnabled = false + nameSpeakerButton.setOnClickListener { onNameSpeakerClick() } + + clearSpeakersButton = findViewById(R.id.clear_speakers_button) + clearSpeakersButton.setOnClickListener { + SpeakerRecognition.resetSpeakers() + lastEmbedding = null + lastLabel = null + lastConfirmedLabel = null + activeLabelForWindow = null + nameSpeakerButton.isEnabled = false + Toast.makeText(this, "已清除所有说话人记录", Toast.LENGTH_SHORT).show() + } + + textView = findViewById(R.id.my_text) + textView.movementMethod = ScrollingMovementMethod() + + subtitleRecycler = findViewById(R.id.subtitle_list) + subtitleRecycler.layoutManager = LinearLayoutManager(this) + subtitleAdapter = SubtitleAdapter() + subtitleRecycler.adapter = subtitleAdapter + + // 后台工作线程降为 background 优先级:定稿/声纹再忙也不能抢占录音 + // 回调与第一遍解码的 CPU(需求一.1 停摆加固)。 + subtitleHandler.post { + Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND) + } + speakerCheckHandler.post { + Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND) + } + + // Loading ~1.6 GB of ONNX weights takes tens of seconds. Do it OFF the + // main thread, otherwise Android shows an ANR ("app not responding") + // after 5 s and may kill the app. + thread(true) { + try { + decideModelSource() + setStatus("正在加载第一遍模型(nemotron-3.5)…") + initOnlineRecognizer() + setStatus("正在加载第二遍模型(funasr-nano)…") + initOfflineRecognizer() + setStatus("正在加载声纹模型…") + initSpeakerIdentification() + modelsReady = true + runOnUiThread { recordButton.isEnabled = true } + val speakerNote = if (speakersEnabled) "说话人功能已启用" + else "⚠ 说话人功能未启用:缺少声纹模型 ${speakerModels[0]}" + + "(请 adb push 到 ${externalModelDir.absolutePath})" + setStatus( + "模型加载完成（${if (loadFromAssets) "APK 内置模型" else "外部模型"}）。" + + "$speakerNote。\n点击 Start 开始识别" + ) + } catch (t: Throwable) { + Log.e(TAG, "Failed to load models", t) + setStatus( + "模型加载失败：${t.message}\n\n" + + "如使用外部模型，请在模型所在目录执行：\n" + + "adb push $firstModelDirName $secondModelDirName " + + "${speakerModels[0]} " + + "/sdcard/Android/data/com.k2fsa.sherpa.onnx/files/models/" + ) + } + } + } + + // Prefer models pushed to Android/data//files/models/; fall back to + // the assets copy when the first-pass model directory is not found there. + private fun decideModelSource() { + if (File(externalModelDir, firstModelDirName).isDirectory) { + loadFromAssets = false + modelPrefix = externalModelDir.absolutePath + "/" + Log.i(TAG, "Loading models from external dir: ${externalModelDir.absolutePath}") + if (!File(externalModelDir, secondModelDirName).isDirectory) { + Log.w(TAG, "Missing $secondModelDirName in external model dir") + } + } else { + loadFromAssets = true + modelPrefix = "" + Log.i(TAG, "External models not found, loading from assets") + // Native code calls exit() (uncatchable!) when an asset is missing, + // so verify up front and fail gracefully on screen instead. + val top = application.assets.list("").orEmpty() + if (!top.contains(firstModelDirName)) { + throw IllegalStateException( + "外部目录与 APK 内置 assets 中都找不到第一遍模型。" + ) + } + } + } + + private fun setStatus(msg: String) { + Log.i(TAG, msg) + runOnUiThread { if (!isRecording) textView.text = msg } + } + + override fun onDestroy() { + super.onDestroy() + isRecording = false + subtitleThread.quitSafely() + speakerCheckThread.quitSafely() + } + + private fun onclick() { + if (!isRecording) { + val ret = initMicrophone() + if (!ret) { + Log.e(TAG, "Failed to initialize microphone") + return + } + Log.i(TAG, "state: ${audioRecord?.state}") + audioRecord!!.startRecording() + recordButton.setText(R.string.stop) + isRecording = true + resetBuffer() + pendingSwitch = null + windowRefEmb = null + checkSerial++ + activeLabelForWindow = null + lastConfirmedLabel = null + uiHandler.post { resetSubtitles() } + + recordingThread = thread(true) { + processSamples() + } + Log.i(TAG, "Started recording") + } else { + isRecording = false + audioRecord!!.stop() + audioRecord!!.release() + audioRecord = null + recordButton.setText(R.string.start) + Log.i(TAG, "Stopped recording") + } + } + + private fun processSamples() { + Log.i(TAG, "processing samples") + // 录音回调线程提优:定稿/声纹再重也不能拖累音频读取节奏(需求一.1) + Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO) + val stream = onlineRecognizer.createStream() + + // Nemotron reads this option on every decoded chunk, so it even can be + // changed while recording (e.g. from a UI button). + stream.setOption("language", firstPassLanguage) + + val interval = 0.1 // i.e., 100 ms + val chunkN = (interval * sampleRateInHz).toInt() // in samples + val buffer = ShortArray(chunkN) + + // sentenceId of the utterance currently being spoken (-1 = none yet). + // ids are allocated here (recording thread) and shared with the main + // thread, so partials / endpoint / async refine all hit the same + // sentence inside whichever line it ended up on. + var sentenceId = -1L + + // rolling speaker-check state (recording-thread only) + val window = RollingWindow(sampleRateInHz * 2) // last 2 s, 0.5 s overlap + val checkStep = (sampleRateInHz * 1.5).toInt() // re-check every 1.5 s + var newSinceCheck = 0 + + while (isRecording) { + val ret = audioRecord?.read(buffer, 0, buffer.size) + if (ret != null && ret > 0) { + val samples = FloatArray(ret) { buffer[it] / 32768.0f } + pushChunk(samples) + window.push(samples) + + if (speakersEnabled && sentenceId != -1L) { + newSinceCheck += ret + if (newSinceCheck >= checkStep) { + newSinceCheck = 0 + val win = window.snapshot() + val cutAt = bufferTotal + val seq = checkSerial + speakerCheckHandler.post { + if (seq != checkSerial || pendingSwitch != null) return@post + val emb = SpeakerRecognition.computeEmbedding( + win, sampleRateInHz + ) ?: return@post + val ref = windowRefEmb + if (ref == null) { + // first window of this utterance becomes the + // voice reference + windowRefEmb = emb + } else if (SpeakerRecognition.similarity(emb, ref) + < speakerSwitchDrop + ) { + val newLabel = SpeakerRecognition.registry.search( + emb, speakerThreshold + ) + if (pendingSwitch == null) { + pendingSwitch = SpeakerSwitch(newLabel, cutAt) + } + } + } + } + } + + stream.acceptWaveform(samples, sampleRate = sampleRateInHz) + while (onlineRecognizer.isReady(stream)) { + onlineRecognizer.decode(stream) + } + + // ---- 声纹判定换人:强制 endpoint(需求一.4a/b/c) ---- + val ps = pendingSwitch + if (ps != null) { + pendingSwitch = null + checkSerial++ // drop in-flight window jobs + val oldLabel = activeLabelForWindow // 被切断句子的归属人 + val size = samplesBuffer.size + if (size == 0) { + uiHandler.post { onSpeakerSwitched(ps.label) } + } else { + var k = ((ps.cutSamples + chunkN / 2) / chunkN) + .coerceIn(1, size) + // 在 ±3 块(±0.3s)内吸附到低能量(自然间隙)边界 + var bestK = k + var bestRms = Float.MAX_VALUE + for (c in maxOf(1, k - 3)..minOf(size, k + 3)) { + val r = (chunkRms.getOrNull(c - 1) ?: 0f) + + (chunkRms.getOrNull(c) ?: 0f) + if (r < bestRms) { + bestRms = r + bestK = c + } + } + k = bestK + var cutSamples = 0 + for (c in 0 until k) cutSamples += samplesBuffer[c].size + + val rawText = onlineRecognizer.getResult(stream).text + val dispText = + if (rawText.isNotBlank() && isInAllowedLanguages(rawText)) + rawText else "" + onlineRecognizer.reset(stream) // OnlineStream 复位 + + val sid = sentenceId + sentenceId = -1L + windowRefEmb = null + newSinceCheck = 0 + + if (sid != -1L) { + if (dispText.isNotBlank()) { + val snap = flattenPrefix(k) // fresh copy 供后台 + val dur = + cutSamples / sampleRateInHz.toFloat() + uiHandler.post { + val accepted = submitRefineTask( + sid, snap, dur, "FORCE_SWITCH", + oldLabel, dispText, rawText + ) + finalizeSentence(sid, accepted) + } + } else { + // 过滤/尚无文字:丢弃该段,不产生空行 + uiHandler.post { + finalizeSentence(sid, false) + } + } + } + uiHandler.post { onSpeakerSwitched(ps.label) } + dropPrefixChunks(k) + // 切换点之后的音频留在缓冲区,归入新说话人(需求一.4c) + } + } + + val isEndpoint = onlineRecognizer.isEndpoint(stream) + + val rawAll = onlineRecognizer.getResult(stream).text + var text = rawAll + if (text.isNotBlank() && !isInAllowedLanguages(text)) { + text = "" // 被过滤的文字不进字幕 + } + if (text.isNotBlank()) { + if (sentenceId == -1L) { + sentenceId = lineIdSeq.incrementAndGet() + windowRefEmb = null + } + val sid = sentenceId + uiHandler.post { onPartial(sid, text) } + } + + if (isEndpoint) { + onlineRecognizer.reset(stream) + + if (sentenceId != -1L) { + val sid = sentenceId + sentenceId = -1L + windowRefEmb = null + newSinceCheck = 0 + checkSerial++ + + val snap = flattenSamples() // fresh copy, 绝不共享引用 + val dur = bufferTotal / sampleRateInHz.toFloat() + val prompt = rawAll // 未过滤底稿作提示(需求二:不丢词) + keepTailChunks(5) // 0.5 s 尾作下一片段上下文 + + // 整句声纹判定移出录音线程(停摆根因):后台算完 + // 回主线程 confirmSpeaker 修正归属 + speakerCheckHandler.post { + val emb = SpeakerRecognition.computeEmbedding( + snap, sampleRateInHz + ) + val label = if (emb == null) null else + SpeakerRecognition.registry.search( + emb, speakerThreshold + ) + ?: SpeakerRecognition.nextAutoLabel().also { + SpeakerRecognition.registry.addAuto(it, emb) + } + uiHandler.post { confirmSpeaker(sid, label, emb) } + } + // 定格 + 送检立即走主线程队列,不等声纹 + uiHandler.post { + val accepted = submitRefineTask( + sid, snap, dur, "NATURAL", null, text, prompt + ) + finalizeSentence(sid, accepted) + } + } else { + resetBuffer() + } + } + } + } + stream.release() + } + + // Keep only zh/ja/en output when firstPassLanguage == "auto": + // letters must be ASCII latin (en), kana (ja), or CJK hanzi (zh/ja); + // everything else (hangul, cyrillic, thai, arabic, accented latin ...) + // means the model decoded another language, so the segment is dropped. + // When a single language is pinned via the prompt, the filter is bypassed. + private fun isInAllowedLanguages(text: String): Boolean { + if (firstPassLanguage != "auto") { + return true + } + for (ch in text) { + if (!Character.isLetter(ch)) { + continue + } + val cp = ch.code + val allowed = + (cp in 0x0041..0x005A) || // A-Z + (cp in 0x0061..0x007A) || // a-z + (cp in 0x3040..0x30FF) || // hiragana + katakana + (cp in 0x31F0..0x31FF) || // katakana phonetic ext. + (cp in 0x3400..0x4DBF) || // CJK ext. A + (cp in 0x4E00..0x9FFF) || // CJK unified + (cp in 0xF900..0xFAFF) || // CJK compatibility + (cp in 0xFF66..0xFF9F) // halfwidth katakana + if (!allowed) { + return false + } + } + return true + } + + private fun initSpeakerIdentification() { + val topAssets = application.assets.list("").orEmpty() + for (model in speakerModels) { + // 1) external models dir; a missing file throws a catchable error + try { + SpeakerRecognition.initExtractor( + assetManager = null, + model = File(externalModelDir, model).absolutePath, + ) + speakersEnabled = true + activeSpeakerModel = model + Log.i(TAG, "Speaker extractor ready (external: $model)") + return + } catch (t: Throwable) { + Log.i(TAG, "Speaker model '$model' not usable from external dir") + } + // 2) assets -- but ONLY if the file is really there: a missing + // asset makes the native code call exit() (uncatchable). + if (topAssets.contains(model)) { + try { + SpeakerRecognition.initExtractor(application.assets, model = model) + speakersEnabled = true + activeSpeakerModel = model + Log.i(TAG, "Speaker extractor ready (assets: $model)") + return + } catch (t: Throwable) { + Log.e(TAG, "Speaker model '$model' failed to load from assets: $t") + } + } + } + speakersEnabled = false + activeSpeakerModel = null + } + + private fun onNameSpeakerClick() { + val emb = lastEmbedding + if (emb == null) { + Toast.makeText(this, "先录一句话,再为这位说话人命名", Toast.LENGTH_SHORT).show() + return + } + val input = EditText(this).apply { hint = "说话人名字，如 张三" } + AlertDialog.Builder(this) + .setTitle("为说话人「${displayName(lastLabel)}」命名") + .setView(input) + .setPositiveButton("注册") { _, _ -> + val name = input.text.toString().trim() + if (name.isNotEmpty()) { + SpeakerRecognition.registry.addNamed(name, emb) + relabelSpeakerLines(name) + Toast.makeText(this, "之后「$name」说话时会显示真名", Toast.LENGTH_SHORT).show() + } + } + .setNegativeButton("取消", null) + .show() + } + + // 命名后:当前会话里这位说话人的所有行立即改显真名(颜色跟人走); + // 之后的新行也会用真名(声纹 search 命名优先返回真名)。 + private fun relabelSpeakerLines(name: String) { + val old = lastLabel + if (old == null) { + lastLabel = name + return + } + if (old != name) { + val idx = labelColorIdx.remove(old) ?: colorIdxFor(name) + labelColorIdx[name] = idx + for (l in subtitleLines) { + if (l.label == old) { + l.label = name + l.colorIdx = idx + val pos = subtitleLines.indexOf(l) + if (pos >= 0) subtitleAdapter.notifyItemChanged(pos) + } + } + } + lastLabel = name + lastConfirmedLabel = name + activeLabelForWindow = name + } + + private fun initMicrophone(): Boolean { + if (ActivityCompat.checkSelfPermission( + this, Manifest.permission.RECORD_AUDIO + ) != PackageManager.PERMISSION_GRANTED + ) { + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION) + return false + } + + val numBytes = AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat) + Log.i( + TAG, "buffer size in milliseconds: ${numBytes * 1000.0f / sampleRateInHz}" + ) + + audioRecord = AudioRecord( + audioSource, + sampleRateInHz, + channelConfig, + audioFormat, + // 4x: ~400 ms of slack so occasional scheduling hiccups on the + // recording thread never overflow the OS capture ring (v5 停摆加固) + numBytes * 4 + ) + return true + } + + private fun initOnlineRecognizer() { + // Please change getModelConfig() to add new models + // See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html + // for a list of available models + val firstType = 34 + val firstRuleFsts: String? + firstRuleFsts = null + Log.i(TAG, "Select model type $firstType for the first pass") + val config = OnlineRecognizerConfig( + featConfig = getFeatureConfig(sampleRate = sampleRateInHz, featureDim = 80), + modelConfig = getModelConfig(type = firstType, modelPrefix = modelPrefix)!!, + endpointConfig = getEndpointConfig(), + enableEndpoint = true, + ) + if (firstRuleFsts != null) { + config.ruleFsts = firstRuleFsts; + } + + onlineRecognizer = OnlineRecognizer( + assetManager = if (loadFromAssets) application.assets else null, + config = config, + ) + } + + private fun initOfflineRecognizer() { + // Please change getOfflineModelConfig() to add new models + // See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html + // for a list of available models + val secondType = 46 + var secondRuleFsts: String? + secondRuleFsts = null + Log.i(TAG, "Select model type $secondType for the second pass") + + val config = OfflineRecognizerConfig( + featConfig = getFeatureConfig(sampleRate = sampleRateInHz, featureDim = 80), + modelConfig = getOfflineModelConfig(type = secondType, modelPrefix = modelPrefix)!!, + ) + + if (secondRuleFsts != null) { + config.ruleFsts = secondRuleFsts + } + + offlineConfig = config + offlineRecognizer = OfflineRecognizer( + assetManager = if (loadFromAssets) application.assets else null, + config = config, + ) + } + + // Runs on subtitleHandler (serial): never touches UI state, only decodes + // a snapshot of one completed utterance. Trims trailing audio that is + // usually the silence/gap that ended the segment; short snapshots keep + // their tail (it doubles as 前段上下文补偿, 需求二.2). + // + // 需求二(prompt):第一遍草稿经 setConfig 注入 funasrNano.hotwords —— + // native 会把它拼进模型训练时学过的"上下文信息/热词列表"提示模板。 + // 本函数只在精修串行线程上跑,解码前后各 setConfig 一次,无并发读者。 + private fun decodeSecondPass(snapshot: FloatArray, prompt: String): String { + val trim = when { + snapshot.size >= 24000 -> 8000 + snapshot.size >= 12000 -> 4000 + else -> 0 + } + val end = snapshot.size - trim + if (end <= 0) return "" + + val nano = offlineConfig.modelConfig.funasrNano + val injected = prompt.isNotBlank() + if (injected) { + nano.hotwords = sanitizePrompt(prompt) + offlineRecognizer.setConfig(offlineConfig) + } + val stream = offlineRecognizer.createStream() + try { + stream.acceptWaveform(snapshot.copyOfRange(0, end), sampleRateInHz) + offlineRecognizer.decode(stream) + return offlineRecognizer.getResult(stream).text + } finally { + stream.release() + if (injected) { + nano.hotwords = "" + offlineRecognizer.setConfig(offlineConfig) + } + } + } + + // ---- refine task 统一入口(需求二.4:快照 copyOf + 槽位只产生一次任务) ---- + // 仅主线程调用。返回 true 表示精修任务已排队。超时看门狗:精修未在 + // refineTimeoutMs 内回来 → 记 TIMEOUT、收回槽位、底稿原地保留(需求二.1)。 + private fun submitRefineTask( + sid: Long, + snapshot: FloatArray, + durSec: Float, + path: String, + speaker: String?, + firstText: String, + prompt: String = "", + ): Boolean { + if (slotBook.containsKey(sid)) { + Log.w(TAG, "SLOT sid=$sid path=$path RESULT=DUP_IGNORED (一槽两位防护)") + return false + } + val rec = SlotRec(path, durSec, speaker, firstText, prompt) + slotBook[sid] = rec + if (slotBook.size > 512) { + for (k in slotBook.keys.take(256).toList()) slotBook.remove(k) + } + if (snapshot.size < minRefineSamples) { + slotBook.remove(sid) + Log.i(TAG, slotLog(sid, rec, "SKIP_SHORT", null, "底稿保留")) + return false + } + val wd = Runnable { + if (sentenceById.remove(sid) != null) { + slotBook.remove(sid) + Log.w(TAG, slotLog(sid, rec, "TIMEOUT", null, "底稿保留")) + } + } + rec.watchdog = wd + uiHandler.postDelayed(wd, refineTimeoutMs) + subtitleHandler.post { + val refined: String? = try { + decodeSecondPass(snapshot, prompt) + } catch (t: Throwable) { + Log.e(TAG, "SLOT sid=$sid decode exception: $t") + null + } + uiHandler.post { applyRefined(sid, refined) } + } + return true + } + + // 一槽一条、全字段对账日志(需求二.5;prompt=实际传入的提示文本) + private fun slotLog( + sid: Long, + rec: SlotRec?, + result: String, + second: String?, + note: String = "", + ): String { + val sp = rec?.speaker?.let { displayName(it) } ?: "未匹配" + val pr = if (rec?.prompt?.isNotEmpty() == true) + " prompt=\"${rec.prompt}\"" else "" + return ("SLOT sid=$sid dur=${"%.1f".format(rec?.durSec ?: 0f)}s " + + "path=${rec?.path ?: "?"} speaker=$sp$pr " + + "first=\"${rec?.first ?: ""}\" " + + "second=${second?.let { "\"$it\"" } ?: "-"} " + + "RESULT=$result${if (note.isEmpty()) "" else " ($note)"}") + } + + // hotwords 通道以逗号/分号切词:统一替换成空格,避免草稿被拆碎; + // 限长 80 字,防止提示 token 挤占 max_total_len 的音频 KV 空间 + // (native 侧对超长 prompt 有截断告警)。 + private fun sanitizePrompt(s: String): String { + val flat = s.replace(Regex("[,;，；\\r\\n\\t]"), " ").trim() + return if (flat.length <= 80) flat else flat.take(80) + } + + // ---- chunked buffer helpers (recording thread only) ---------------------- + private fun pushChunk(s: FloatArray) { + samplesBuffer.add(s) + chunkRms.add(rmsOf(s)) + bufferTotal += s.size + } + + private fun resetBuffer() { + samplesBuffer.clear() + chunkRms.clear() + bufferTotal = 0 + } + + private fun keepTailChunks(n: Int) { + while (samplesBuffer.size > n) { + bufferTotal -= samplesBuffer.removeAt(0).size + chunkRms.removeAt(0) + } + } + + private fun dropPrefixChunks(n: Int) { + repeat(minOf(n, samplesBuffer.size)) { + bufferTotal -= samplesBuffer.removeAt(0).size + chunkRms.removeAt(0) + } + } + + private fun rmsOf(s: FloatArray): Float { + if (s.isEmpty()) return 0f + var acc = 0.0 + for (x in s) acc += x.toDouble() * x + return Math.sqrt(acc / s.size).toFloat() + } + + private fun flattenSamples(): FloatArray = flattenPrefix(samplesBuffer.size) + + // fresh copy of the first k chunks — never hand the shared buffer to threads + private fun flattenPrefix(k: Int): FloatArray { + var total = 0 + for (i in 0 until minOf(k, samplesBuffer.size)) total += samplesBuffer[i].size + val flat = FloatArray(total) + var i = 0 + for (c in 0 until minOf(k, samplesBuffer.size)) { + samplesBuffer[c].copyInto(flat, i) + i += samplesBuffer[c].size + } + return flat + } + + private fun bufferDurationSec(): Float = + bufferTotal / sampleRateInHz.toFloat() + + // fixed-size sliding window over recent audio (recording thread only) + private class RollingWindow(private val size: Int) { + private val buf = FloatArray(size) + private var len = 0 + + fun push(s: FloatArray) { + if (s.size >= size) { + System.arraycopy(s, s.size - size, buf, 0, size) + len = size + return + } + val overflow = len + s.size - size + var base = len + if (overflow > 0) { + System.arraycopy(buf, overflow, buf, 0, len - overflow) + base = len - overflow + } + System.arraycopy(s, 0, buf, base, s.size) + len = minOf(size, len + s.size) + } + + fun snapshot(): FloatArray = buf.copyOf(len) + } + + // funasr-nano (Qwen3-0.6B decoder) can fall into repetition loops, + // especially when the clip starts with the 0.5 s tail re-fed from the + // previous utterance. Collapse adjacent repeats conservatively so that + // legitimate echoes ("非常感谢", "哈哈哈") survive: + // - >=3 copies of a unit of >=4 chars (>=12 chars total), or + // - exactly 2 copies of a long unit (>=10 chars, e.g. a whole clause). + private fun collapseRepetitions(s: String): String { + if (s.length < 8) return s + var cur = s + var changed = true + while (changed) { + changed = false + outer@ for (unit in cur.length / 2 downTo 4) { + var i = 0 + while (i + unit <= cur.length) { + val base = cur.substring(i, i + unit) + var count = 1 + var j = i + unit + while (j + unit <= cur.length && + cur.substring(j, j + unit) == base + ) { + count++ + j += unit + } + val hit = (count >= 3 && unit >= 4) || + (count == 2 && unit >= 10) + if (hit) { + Log.i(TAG, "Collapsed ${count}x repeat: $base") + cur = cur.substring(0, i + unit) + cur.substring(j) + changed = true + break@outer + } + i = if (count > 1) j else i + 1 + } + } + } + return cur + } + + // ==================== Subtitle stream (main thread only) ==================== + + private inner class SubtitleAdapter : RecyclerView.Adapter() { + inner class VH(val text: TextView) : RecyclerView.ViewHolder(text) + + override fun onCreateViewHolder(parent: ViewGroup, type: Int): VH = VH( + LayoutInflater.from(parent.context) + .inflate(R.layout.item_subtitle, parent, false) as TextView + ) + + override fun getItemCount(): Int = subtitleLines.size + + override fun onBindViewHolder(holder: VH, position: Int) { + holder.text.text = buildLineSpannable(subtitleLines[position]) + } + } + + // "[张三] 第一句。第二句"——同一行内句子按语言习惯衔接。 + private fun buildLineSpannable(line: SubtitleLine): CharSequence { + val sb = SpannableStringBuilder() + val label = displayName(line.label) + val color = + if (line.label == null) Color.parseColor("#757575") + else speakerPalette[line.colorIdx] + val start = sb.length + val head = "[$label] " + sb.append(head) + sb.setSpan( + StyleSpan(Typeface.BOLD), start, start + head.length, + Spanned.SPAN_EXCLUSIVE_EXCLUSIVE + ) + for ((i, s) in line.sentences.withIndex()) { + if (s.text.isBlank()) continue + if (sb.length > start + head.length) { + sb.append(joinerAfter(line.sentences[i - 1].text)) + } + sb.append(s.text.lowercase()) + } + sb.setSpan( + ForegroundColorSpan(color), start, sb.length, + Spanned.SPAN_EXCLUSIVE_EXCLUSIVE + ) + return sb + } + + // 句间衔接:已带标点则空格;CJK/假名结尾用句号;拉丁结尾用英文句号。 + private fun joinerAfter(prev: String): String { + val last = prev.trimEnd().lastOrNull() ?: return " " + return when { + last in "。！？；.!?;…\"”’』】》）)" -> " " + last.code >= 0x2E80 -> "。" + Character.isLetterOrDigit(last) -> ". " + else -> " " + } + } + + private fun resetSubtitles() { + val n = subtitleLines.size + subtitleLines.clear() + sentenceById.clear() + labelColorIdx.clear() + colorSeq = 0 + lastLabel = null + if (n > 0) subtitleAdapter.notifyItemRangeRemoved(0, n) + textView.visibility = View.GONE + subtitleRecycler.visibility = View.VISIBLE + } + + // 内部标签 S0/S1/…(注册键,可被命名为真名)对外显示为 未识别1/未识别2…; + // 已命名者直接显示真名。 + private val autoLabelRegex = Regex("^S(\\d+)$") + + private fun displayName(label: String?): String { + if (label == null) return "未识别" + val m = autoLabelRegex.matchEntire(label) + return if (m != null) "未识别${m.groupValues[1].toInt() + 1}" else label + } + + private fun colorIdxFor(label: String): Int = + labelColorIdx.getOrPut(label) { colorSeq++ % speakerPalette.size } + + private fun appendNewLine(label: String?): SubtitleLine { + val line = SubtitleLine( + id = lineIdSeq.incrementAndGet(), + label = label, + colorIdx = colorIdxFor(label ?: "__unknown__"), + ) + subtitleLines.add(line) + subtitleAdapter.notifyItemInserted(subtitleLines.size - 1) + return line + } + + // first-pass partial:实时刷新"当前活跃句"。新 endpoint 之前的句子先挂在 + // 活跃行(=最后一条 open 行,沿用最近确认说话人);定格时若判定换人, + // 该整句会被移到新说话人的新行(finalizeSentence)。 + private fun onPartial(sid: Long, partial: String) { + val existing = sentenceById[sid] + if (existing == null) { + val active = subtitleLines.lastOrNull()?.takeIf { it.open } + ?: appendNewLine(lastConfirmedLabel) + val sentence = Sentence(sid, partial, finalized = false) + sentence.line = active + active.sentences.add(sentence) + sentenceById[sid] = sentence + notifyLine(active) + } else if (existing.text != partial) { + existing.text = partial + existing.line?.let { notifyLine(it) } + } + autoScroll() + } + + // endpoint: 定格当前句。声纹判定换人 → 该句整体移入新说话人的新行 + // (原行不含此句,哪怕 partial 阶段曾短暂挂错);同人 → 句子并入当前行。 + // endpoint/强制断句的 settle:该句就地定格(归属修正走 confirmSpeaker, + // v5 起声纹判定已移出录音线程)。 + private fun finalizeSentence(sid: Long, willRefine: Boolean) { + lastSettledSid = sid + val sentence = sentenceById[sid] ?: return + if (sentence.finalized) return + sentence.finalized = true + if (!speakersEnabled) { + // 声纹缺失:退化为按 endpoint 分行(显示"未识别") + sentence.line?.open = false + } + if (!willRefine) { + // 不会再有精修回来:立即收回句子索引 + // (SKIP_SHORT 等情况 submitRefineTask 已写对账日志,不重复记) + sentenceById.remove(sid) + } + sentence.line?.let { notifyLine(it) } + autoScroll() + } + + private fun findSentence(sid: Long): Sentence? { + for (l in subtitleLines) { + for (s in l.sentences) { + if (s.id == sid) return s + } + } + return null + } + + // 声纹线程回传的整句权威判定(需求一.2/一.4): + // * 与所在行标签不符 → 该句整体迁入新说话人行(原行不含此句); + // * embedding 回写供"命名说话人"; + // * 仅当该槽仍是最近 settle 的句子才更新全局活跃归属(防旧判定覆盖更新的换人状态)。 + private fun confirmSpeaker(sid: Long, label: String?, emb: FloatArray?) { + Log.i(TAG, "SLOT sid=$sid VERDICT speaker=" + + (label?.let { displayName(it) } ?: "未匹配/无判定")) + if (emb != null) { + lastEmbedding = emb + nameSpeakerButton.isEnabled = true + } + val sentence = findSentence(sid) ?: return + val line = sentence.line ?: return + if (label != null && label != line.label) { + line.sentences.remove(sentence) + line.open = false + if (line.sentences.isEmpty()) { + val pos = subtitleLines.indexOf(line) + if (pos >= 0) { + subtitleLines.removeAt(pos) + subtitleAdapter.notifyItemRemoved(pos) + } + } else { + notifyLine(line) + } + val nl = appendNewLine(label) + nl.sentences.add(sentence) + sentence.line = nl + notifyLine(nl) + autoScroll() + } + if (label != null && sid == lastSettledSid) { + lastConfirmedLabel = label + lastLabel = label + activeLabelForWindow = label + } + } + + // second-pass refine, 需求二加固: + // * 底稿铁律:refine 只能覆盖对应句;空/异常/超时(看门狗)/折叠误杀/ + // 明显缩短 → 一律保留第一遍文字,该句标记 sentence.refined=false。 + // * collapseRepetitions 仅作用于第二遍输出;折叠后 <50% 判误杀回退原文。 + // * 每槽恰好应用一次(sentenceById/slotBook 双 remove 幂等), + // 终结时输出一条全字段对账日志。 + private fun applyRefined(sid: Long, refined: String?) { + val sentence = sentenceById.remove(sid) + val rec = slotBook.remove(sid) + rec?.watchdog?.let { uiHandler.removeCallbacks(it) } + if (sentence == null) { + Log.w(TAG, slotLog(sid, rec, "LATE_OR_DUP", refined, "槽位已收回,结果丢弃")) + return + } + val first = sentence.text.ifEmpty { rec?.first ?: "" } + var second: String? = null + var result = "APPLIED" + var note = "" + if (refined == null) { + result = "REJECT_EXCEPTION" + note = "底稿保留" + } else { + val collapsed = collapseRepetitions(refined) + val candidate = + if (collapsed.isEmpty() || collapsed.length * 2 < refined.length) { + note = "折叠误杀回退原文;" + refined + } else { + collapsed + } + when { + candidate.isBlank() -> { + result = "REJECT_EMPTY"; note = "底稿保留" + } + !isInAllowedLanguages(candidate) -> { + result = "REJECT_LANGUAGE"; note = "底稿保留" + } + first.isNotBlank() && candidate.length * 2 < first.length -> { + result = "REJECT_TOO_SHORT"; note = "不足底稿50%,底稿保留" + } + else -> second = candidate + } + } + Log.i(TAG, slotLog(sid, rec, result, second, note)) + if (second != null && second != sentence.text) { + sentence.text = second + sentence.refined = true + sentence.line?.let { notifyLine(it) } + autoScroll() + } else { + sentence.refined = false // 未精修标记 + } + } + + // 换人强制断句后:封掉旧行,之后的 partial 开新行并归属新说话人。 + private fun onSpeakerSwitched(label: String?) { + subtitleLines.lastOrNull()?.takeIf { it.open }?.open = false + lastConfirmedLabel = label + activeLabelForWindow = label + } + + private fun notifyLine(line: SubtitleLine) { + trimOldLines() + val pos = subtitleLines.indexOf(line) + if (pos >= 0) subtitleAdapter.notifyItemChanged(pos) + } + + // 超长会话:丢最旧的行(数据+视图一起回收),句子索引同步清理 + private fun trimOldLines() { + while (subtitleLines.size > maxSubtitleLines) { + val old = subtitleLines.removeAt(0) + for (s in old.sentences) sentenceById.remove(s.id) + subtitleAdapter.notifyItemRemoved(0) + } + } + + private fun autoScroll() { + if (subtitleRecycler.visibility != View.VISIBLE) return + val lm = subtitleRecycler.layoutManager as LinearLayoutManager + val last = subtitleAdapter.itemCount - 1 + if (last >= 0 && lm.findLastCompletelyVisibleItemPosition() >= last - 1) { + subtitleRecycler.scrollToPosition(last) + } + } +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt new file mode 100644 index 0000000..6183c56 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt @@ -0,0 +1,1514 @@ +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class OfflineRecognizerResult( + val text: String, + val tokens: Array, + val timestamps: FloatArray, + val lang: String, + val emotion: String, + val event: String, + + // valid only for TDT models + val durations: FloatArray, +) + +data class OfflineTransducerModelConfig( + var encoder: String = "", + var decoder: String = "", + var joiner: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineParaformerModelConfig( + var model: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineNemoEncDecCtcModelConfig( + var model: String = "", +) + +data class OfflineDolphinModelConfig( + var model: String = "", +) + +data class OfflineZipformerCtcModelConfig( + var model: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineWenetCtcModelConfig( + var model: String = "", +) + +data class OfflineOmnilingualAsrCtcModelConfig( + var model: String = "", +) + +data class OfflineMedAsrCtcModelConfig( + var model: String = "", +) + +data class OfflineFireRedAsrCtcModelConfig( + var model: String = "", +) + +data class OfflineFunAsrNanoModelConfig( + var encoderAdaptor: String = "", + var llm: String = "", + var embedding: String = "", + var tokenizer: String = "", + var systemPrompt: String = "You are a helpful assistant.", + var userPrompt: String = "语音转写：", + var maxNewTokens: Int = 512, + var temperature: Float = 1e-6f, + var topP: Float = 0.8f, + var seed: Int = 42, + var language: String = "", + var itn: Boolean = true, + var hotwords: String = "", +) + +data class OfflineQwen3AsrModelConfig( + var convFrontend: String = "", + var encoder: String = "", + var decoder: String = "", + var tokenizer: String = "", + var maxTotalLen: Int = 512, + var maxNewTokens: Int = 128, + var temperature: Float = 1e-6f, + var topP: Float = 0.8f, + var seed: Int = 42, + var hotwords: String = "", +) + +data class OfflineWhisperModelConfig( + var encoder: String = "", + var decoder: String = "", + var language: String = "en", // Used with multilingual model + var task: String = "transcribe", // transcribe or translate + var tailPaddings: Int = 1000, // Padding added at the end of the samples + var enableTokenTimestamps: Boolean = false, + var enableSegmentTimestamps: Boolean = false, +) + +data class OfflineCanaryModelConfig( + var encoder: String = "", + var decoder: String = "", + var srcLang: String = "en", + var tgtLang: String = "en", + var usePnc: Boolean = true, +) + +data class OfflineCohereTranscribeModelConfig( + var encoder: String = "", + var decoder: String = "", + var language: String = "", + var usePunct: Boolean = true, + var useItn: Boolean = true, +) + +data class OfflineFireRedAsrModelConfig( + var encoder: String = "", + var decoder: String = "", +) + +// For moonshine v1, you need four models. +// For moonshine v2, you need two models. +// - v1: preprocessor, encoder, uncachedDecoder, cachedDecoder +// - v2: encoder, mergedDecoder +data class OfflineMoonshineModelConfig( + var preprocessor: String = "", + var encoder: String = "", + var uncachedDecoder: String = "", + var cachedDecoder: String = "", + var mergedDecoder: String = "", +) + +data class OfflineSenseVoiceModelConfig( + var model: String = "", + var language: String = "", + var useInverseTextNormalization: Boolean = true, + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineModelConfig( + var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(), + var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(), + var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(), + var fireRedAsr: OfflineFireRedAsrModelConfig = OfflineFireRedAsrModelConfig(), + var moonshine: OfflineMoonshineModelConfig = OfflineMoonshineModelConfig(), + var nemo: OfflineNemoEncDecCtcModelConfig = OfflineNemoEncDecCtcModelConfig(), + var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(), + var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(), + var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(), + var wenetCtc: OfflineWenetCtcModelConfig = OfflineWenetCtcModelConfig(), + var omnilingual: OfflineOmnilingualAsrCtcModelConfig = OfflineOmnilingualAsrCtcModelConfig(), + var medasr: OfflineMedAsrCtcModelConfig = OfflineMedAsrCtcModelConfig(), + var funasrNano: OfflineFunAsrNanoModelConfig = OfflineFunAsrNanoModelConfig(), + var qwen3Asr: OfflineQwen3AsrModelConfig = OfflineQwen3AsrModelConfig(), + var fireRedAsrCtc: OfflineFireRedAsrCtcModelConfig = OfflineFireRedAsrCtcModelConfig(), + var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(), + var cohereTranscribe: OfflineCohereTranscribeModelConfig = + OfflineCohereTranscribeModelConfig(), + var teleSpeech: String = "", + var numThreads: Int = 1, + var debug: Boolean = false, + var provider: String = "cpu", + var modelType: String = "", + var tokens: String = "", + var modelingUnit: String = "", + var bpeVocab: String = "", +) + +data class OfflineRecognizerConfig( + var featConfig: FeatureConfig = FeatureConfig(), + var modelConfig: OfflineModelConfig = OfflineModelConfig(), + // var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it + var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(), + var decodingMethod: String = "greedy_search", + var maxActivePaths: Int = 4, + var hotwordsFile: String = "", + var hotwordsScore: Float = 1.5f, + var ruleFsts: String = "", + var ruleFars: String = "", + var blankPenalty: Float = 0.0f, +) + +class OfflineRecognizer( + assetManager: AssetManager? = null, + val config: OfflineRecognizerConfig, +) { + private var ptr: Long + + init { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + require(ptr != 0L) { + "Invalid OfflineRecognizerConfig: failed to create native OfflineRecognizer" + } + } + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun createStream(): OfflineStream { + val p = createStream(ptr) + return OfflineStream(p) + } + + fun createStream(hotwords: String): OfflineStream { + val p = createStreamWithHotwords(ptr, hotwords) + return OfflineStream(p) + } + + fun getResult(stream: OfflineStream): OfflineRecognizerResult { + return getResult(stream.ptr) + } + + fun decode(stream: OfflineStream) = decode(ptr, stream.ptr) + + fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config) + + private external fun delete(ptr: Long) + + private external fun createStream(ptr: Long): Long + + private external fun createStreamWithHotwords(ptr: Long, hotwords: String): Long + + private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig) + + private external fun newFromAsset( + assetManager: AssetManager, + config: OfflineRecognizerConfig, + ): Long + + private external fun newFromFile( + config: OfflineRecognizerConfig, + ): Long + + private external fun decode(ptr: Long, streamPtr: Long) + + private external fun getResult(streamPtr: Long): OfflineRecognizerResult + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + + @JvmStatic + external fun prependAdspLibraryPath(newPath: String) // for qnn + } +} + +/* +Please see +https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html +for a list of pre-trained models. + +We only add a few here. Please change the following code +to add your own. (It should be straightforward to add a new model +by following the code) + +@param type + +0 - csukuangfj/sherpa-onnx-paraformer-zh-2023-09-14 (Chinese) + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2023-09-14-chinese + int8 + +1 - icefall-asr-multidataset-pruned_transducer_stateless7-2023-05-04 (English) + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#icefall-asr-multidataset-pruned-transducer-stateless7-2023-05-04-english + encoder int8, decoder/joiner float32 + +2 - sherpa-onnx-whisper-tiny.en + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html#tiny-en + encoder int8, decoder int8 + +3 - sherpa-onnx-whisper-base.en + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html#tiny-en + encoder int8, decoder int8 + +4 - pkufool/icefall-asr-zipformer-wenetspeech-20230615 (Chinese) + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#pkufool-icefall-asr-zipformer-wenetspeech-20230615-chinese + encoder/joiner int8, decoder fp32 + + */ +fun getOfflineModelConfig(type: Int, modelPrefix: String = ""): OfflineModelConfig? { + when (type) { + 0 -> { + val modelDir = "sherpa-onnx-paraformer-zh-2023-09-14" + return OfflineModelConfig( + paraformer = OfflineParaformerModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "paraformer", + ) + } + + 1 -> { + val modelDir = "icefall-asr-multidataset-pruned_transducer_stateless7-2023-05-04" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-30-avg-4.int8.onnx", + decoder = "$modelDir/decoder-epoch-30-avg-4.onnx", + joiner = "$modelDir/joiner-epoch-30-avg-4.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 2 -> { + val modelDir = "sherpa-onnx-whisper-tiny.en" + return OfflineModelConfig( + whisper = OfflineWhisperModelConfig( + encoder = "$modelDir/tiny.en-encoder.int8.onnx", + decoder = "$modelDir/tiny.en-decoder.int8.onnx", + ), + tokens = "$modelDir/tiny.en-tokens.txt", + modelType = "whisper", + ) + } + + 3 -> { + val modelDir = "sherpa-onnx-whisper-base.en" + return OfflineModelConfig( + whisper = OfflineWhisperModelConfig( + encoder = "$modelDir/base.en-encoder.int8.onnx", + decoder = "$modelDir/base.en-decoder.int8.onnx", + ), + tokens = "$modelDir/base.en-tokens.txt", + modelType = "whisper", + ) + } + + + 4 -> { + val modelDir = "icefall-asr-zipformer-wenetspeech-20230615" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-12-avg-4.int8.onnx", + decoder = "$modelDir/decoder-epoch-12-avg-4.onnx", + joiner = "$modelDir/joiner-epoch-12-avg-4.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 5 -> { + val modelDir = "sherpa-onnx-zipformer-multi-zh-hans-2023-9-2" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-20-avg-1.int8.onnx", + decoder = "$modelDir/decoder-epoch-20-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-20-avg-1.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 6 -> { + val modelDir = "sherpa-onnx-nemo-ctc-en-citrinet-512" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 7 -> { + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 8 -> { + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-24500" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9 -> { + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 10 -> { + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-es-1424" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 11 -> { + val modelDir = "sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04" + return OfflineModelConfig( + teleSpeech = "$modelDir/model.int8.onnx", + tokens = "$modelDir/tokens.txt", + modelType = "telespeech_ctc", + ) + } + + 12 -> { + val modelDir = "sherpa-onnx-zipformer-thai-2024-06-20" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-12-avg-5.int8.onnx", + decoder = "$modelDir/decoder-epoch-12-avg-5.onnx", + joiner = "$modelDir/joiner-epoch-12-avg-5.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 13 -> { + val modelDir = "sherpa-onnx-zipformer-korean-2024-06-24" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 14 -> { + val modelDir = "sherpa-onnx-paraformer-zh-small-2024-03-09" + return OfflineModelConfig( + paraformer = OfflineParaformerModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "paraformer", + ) + } + + 15 -> { + val modelDir = "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17" + return OfflineModelConfig( + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 16 -> { + val modelDir = "sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 17 -> { + val modelDir = "sherpa-onnx-zipformer-ru-2024-09-18" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 18 -> { + val modelDir = "sherpa-onnx-small-zipformer-ru-2024-09-18" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 19 -> { + val modelDir = "sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 20 -> { + val modelDir = "sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "nemo_transducer", + ) + } + + 21 -> { + val modelDir = "sherpa-onnx-moonshine-tiny-en-int8" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + preprocessor = "$modelDir/preprocess.onnx", + encoder = "$modelDir/encode.int8.onnx", + uncachedDecoder = "$modelDir/uncached_decode.int8.onnx", + cachedDecoder = "$modelDir/cached_decode.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 22 -> { + val modelDir = "sherpa-onnx-moonshine-base-en-int8" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + preprocessor = "$modelDir/preprocess.onnx", + encoder = "$modelDir/encode.int8.onnx", + uncachedDecoder = "$modelDir/uncached_decode.int8.onnx", + cachedDecoder = "$modelDir/cached_decode.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 23 -> { + val modelDir = "sherpa-onnx-zipformer-zh-en-2023-11-22" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-34-avg-19.int8.onnx", + decoder = "$modelDir/decoder-epoch-34-avg-19.onnx", + joiner = "$modelDir/joiner-epoch-34-avg-19.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 24 -> { + val modelDir = "sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16" + return OfflineModelConfig( + fireRedAsr = OfflineFireRedAsrModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 25 -> { + val modelDir = "sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02" + return OfflineModelConfig( + dolphin = OfflineDolphinModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 26 -> { + val modelDir = "sherpa-onnx-zipformer-vi-int8-2025-04-20" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-12-avg-8.int8.onnx", + decoder = "$modelDir/decoder-epoch-12-avg-8.onnx", + joiner = "$modelDir/joiner-epoch-12-avg-8.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 27 -> { + val modelDir = "sherpa-onnx-nemo-ctc-giga-am-v2-russian-2025-04-19" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 28 -> { + val modelDir = "sherpa-onnx-nemo-transducer-giga-am-v2-russian-2025-04-19" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "nemo_transducer", + ) + } + + 29 -> { + val modelDir = "sherpa-onnx-zipformer-ru-int8-2025-04-20" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 30 -> { + val modelDir = "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "nemo_transducer", + ) + } + + 31 -> { + val modelDir = "sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03" + return OfflineModelConfig( + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 32 -> { + val modelDir = "sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8" + return OfflineModelConfig( + canary = OfflineCanaryModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + srcLang = "en", + tgtLang = "en", + usePnc = true, + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 33 -> { + val modelDir = "sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 34 -> { + val modelDir = "sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 35 -> { + val modelDir = "sherpa-onnx-nemo-transducer-stt_pt_fastconformer_hybrid_large_pc-int8" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "nemo_transducer", + ) + } + + 36 -> { + val modelDir = "sherpa-onnx-nemo-stt_pt_fastconformer_hybrid_large_pc-int8" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 37 -> { + val modelDir = "sherpa-onnx-nemo-transducer-stt_de_fastconformer_hybrid_large_pc-int8" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "nemo_transducer", + ) + } + + 38 -> { + val modelDir = "sherpa-onnx-nemo-stt_de_fastconformer_hybrid_large_pc-int8" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 39 -> { + val modelDir = "sherpa-onnx-zipformer-ctc-small-zh-int8-2025-07-16" + return OfflineModelConfig( + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 40 -> { + val modelDir = "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "nemo_transducer", + ) + } + + 41 -> { + val modelDir = "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09" + return OfflineModelConfig( + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 42 -> { + val modelDir = + "sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10" + return OfflineModelConfig( + wenetCtc = OfflineWenetCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 43 -> { + val modelDir = "sherpa-onnx-paraformer-zh-int8-2025-10-07" + return OfflineModelConfig( + paraformer = OfflineParaformerModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "paraformer", + ) + } + + 44 -> { + val modelDir = "sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12" + return OfflineModelConfig( + omnilingual = OfflineOmnilingualAsrCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 45 -> { + val modelDir = "sherpa-onnx-medasr-ctc-en-int8-2025-12-25" + return OfflineModelConfig( + medasr = OfflineMedAsrCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 46 -> { + val modelDir = modelPrefix + "sherpa-onnx-funasr-nano-int8-2025-12-30" + return OfflineModelConfig( + funasrNano = OfflineFunAsrNanoModelConfig( + encoderAdaptor = "$modelDir/encoder_adaptor.int8.onnx", + llm = "$modelDir/llm.int8.onnx", + embedding = "$modelDir/embedding.int8.onnx", + tokenizer = "$modelDir/Qwen3-0.6B", + ), + tokens = "", + // 2 threads: leave cores for the streaming pass + eres2net so + // nano decoding can't starve the first pass (v5 停摆加固) + numThreads = 2, + ) + } + + 47 -> { + val modelDir = "sherpa-onnx-wenetspeech-wu-u2pp-conformer-ctc-zh-int8-2026-02-03" + return OfflineModelConfig( + wenetCtc = OfflineWenetCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 48 -> { + val modelDir = "sherpa-onnx-wenetspeech-wu-u2pp-conformer-ctc-zh-2026-02-03" + return OfflineModelConfig( + wenetCtc = OfflineWenetCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 49 -> { + val modelDir = "sherpa-onnx-zipformer-vi-30M-int8-2026-02-09" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } + + 50 -> { + val modelDir = "sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25" + return OfflineModelConfig( + fireRedAsrCtc = OfflineFireRedAsrCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 51 -> { + val modelDir = "sherpa-onnx-moonshine-tiny-ko-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 52 -> { + val modelDir = "sherpa-onnx-moonshine-tiny-ja-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 53 -> { + val modelDir = "sherpa-onnx-moonshine-tiny-en-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 54 -> { + val modelDir = "sherpa-onnx-moonshine-base-zh-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 55 -> { + val modelDir = "sherpa-onnx-moonshine-base-vi-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 56 -> { + val modelDir = "sherpa-onnx-moonshine-base-uk-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 57 -> { + val modelDir = "sherpa-onnx-moonshine-base-ja-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 58 -> { + val modelDir = "sherpa-onnx-moonshine-base-es-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 59 -> { + val modelDir = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 60 -> { + val modelDir = "sherpa-onnx-moonshine-base-ar-quantized-2026-02-27" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + encoder = "$modelDir/encoder_model.ort", + mergedDecoder = "$modelDir/decoder_model_merged.ort", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 61 -> { + val modelDir = "sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25" + return OfflineModelConfig( + qwen3Asr = OfflineQwen3AsrModelConfig( + convFrontend = "$modelDir/conv_frontend.onnx", + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + tokenizer = "$modelDir/tokenizer", + ), + tokens = "", + numThreads=3, + ) + } + + 62 -> { + val modelDir = "sherpa-onnx-nemo-parakeet-unified-en-0.6b-int8-non-streaming" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "nemo_transducer", + ) + } + + 9000 -> { + val modelDir = + "sherpa-onnx-qnn-5-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + // Please copy libQnnHtp.so and libQnnSystem.so to jniLibs/arm64-v8a by yourself + // + // model.bin is created in the first run and is used from the second run + // to speed up the initialization + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9001 -> { + val modelDir = + "sherpa-onnx-qnn-8-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9002 -> { + val modelDir = + "sherpa-onnx-qnn-10-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9003 -> { + val modelDir = + "sherpa-onnx-qnn-13-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9004 -> { + val modelDir = + "sherpa-onnx-qnn-15-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9005 -> { + val modelDir = + "sherpa-onnx-qnn-18-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9006 -> { + val modelDir = + "sherpa-onnx-qnn-20-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9007 -> { + val modelDir = + "sherpa-onnx-qnn-23-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9008 -> { + val modelDir = + "sherpa-onnx-qnn-25-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9009 -> { + val modelDir = + "sherpa-onnx-qnn-28-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9010 -> { + val modelDir = + "sherpa-onnx-qnn-30-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9011 -> { + val modelDir = + "sherpa-onnx-qnn-5-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9012 -> { + val modelDir = + "sherpa-onnx-qnn-8-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9013 -> { + val modelDir = + "sherpa-onnx-qnn-10-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9014 -> { + val modelDir = + "sherpa-onnx-qnn-13-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9015 -> { + val modelDir = + "sherpa-onnx-qnn-15-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9016 -> { + val modelDir = + "sherpa-onnx-qnn-18-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9017 -> { + val modelDir = + "sherpa-onnx-qnn-20-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9018 -> { + val modelDir = + "sherpa-onnx-qnn-23-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9019 -> { + val modelDir = + "sherpa-onnx-qnn-25-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9020 -> { + val modelDir = + "sherpa-onnx-qnn-28-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9021 -> { + val modelDir = + "sherpa-onnx-qnn-30-seconds-zipformer-ctc-zh-2025-07-03-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "$modelDir/libmodel.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9022 -> { + // for Xiaomi 17 Pro + val modelDir = + "sherpa-onnx-qnn-SM8850-binary-10-seconds-sense-voice-zh-en-ja-ko-yue-2024-07-17-int8" + return OfflineModelConfig( + provider = "qnn", + senseVoice = OfflineSenseVoiceModelConfig( + qnnConfig = QnnConfig( + // Please copy libQnnHtp.so and libQnnSystem.so to jniLibs/arm64-v8a by yourself + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/model.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9023 -> { + val modelDir = "sherpa-onnx-qnn-5-seconds-paraformer-zh-2023-03-28-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + paraformer = OfflineParaformerModelConfig( + model = "$modelDir/libencoder.so,$modelDir/libpredictor.so,$modelDir/libdecoder.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + // The following three *.bin files are generated during the first run + // and are used to replace the corresponding *.so files in later runs + contextBinary = "$modelDir/encoder.bin,$modelDir/predictor.bin,$modelDir/decoder.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9024 -> { + val modelDir = "sherpa-onnx-qnn-5-seconds-paraformer-zh-2025-10-07-int8-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + paraformer = OfflineParaformerModelConfig( + model = "$modelDir/libencoder.so,$modelDir/libpredictor.so,$modelDir/libdecoder.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + // The following three *.bin files are generated during the first run + // and are used to replace the corresponding *.so files in later runs + contextBinary = "$modelDir/encoder.bin,$modelDir/predictor.bin,$modelDir/decoder.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9025 -> { + // for Xiaomi 17 Pro + val modelDir = "sherpa-onnx-qnn-SM8850-binary-5-seconds-paraformer-zh-2023-03-28-int8" + return OfflineModelConfig( + provider = "qnn", + paraformer = OfflineParaformerModelConfig( + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/encoder.bin,$modelDir/predictor.bin,$modelDir/decoder.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + debug = true, + ) + } + + 9026 -> { + val modelDir = + "sherpa-onnx-qnn-reazonspeech-zipformer-transducer-ja-5s-2024-08-01-android-aarch64" + return OfflineModelConfig( + provider = "qnn", + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/libencoder.so", + decoder = "$modelDir/libdecoder.so", + joiner = "$modelDir/libjoiner.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/encoder.bin,$modelDir/decoder.bin,$modelDir/joiner.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + debug = true, + ) + } + + 9027 -> { + // for Xiaomi 17 Pro + val modelDir = + "sherpa-onnx-qnn-SM8850-binary-reazonspeech-zipformer-transducer-ja-5s-2024-08-01" + return OfflineModelConfig( + provider = "qnn", + transducer = OfflineTransducerModelConfig( + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/encoder.bin,$modelDir/decoder.bin,$modelDir/joiner.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + debug = true, + ) + } + } + return null +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt new file mode 100644 index 0000000..c354228 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt @@ -0,0 +1,42 @@ +package com.k2fsa.sherpa.onnx + +class OfflineStream(var ptr: Long) { + init { + require(ptr != 0L) { "Failed to create native OfflineStream" } + } + + fun acceptWaveform(samples: FloatArray, sampleRate: Int) = + acceptWaveform(ptr, samples, sampleRate) + + fun setOption(key: String, value: String) = setOption(ptr, key, value) + + fun getOption(key: String): String = getOption(ptr, key) + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun use(block: (OfflineStream) -> Unit) { + try { + block(this) + } finally { + release() + } + } + + private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int) + private external fun setOption(ptr: Long, key: String, value: String) + private external fun getOption(ptr: Long, key: String): String + private external fun delete(ptr: Long) + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt new file mode 100644 index 0000000..43d539b --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt @@ -0,0 +1,812 @@ +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class EndpointRule( + var mustContainNonSilence: Boolean, + var minTrailingSilence: Float, + var minUtteranceLength: Float, +) + +data class EndpointConfig( + var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f), + var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f), + var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f) +) + +data class OnlineTransducerModelConfig( + var encoder: String = "", + var decoder: String = "", + var joiner: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OnlineParaformerModelConfig( + var encoder: String = "", + var decoder: String = "", +) + +data class OnlineZipformer2CtcModelConfig( + var model: String = "", +) + +data class OnlineNeMoCtcModelConfig( + var model: String = "", +) + +data class OnlineToneCtcModelConfig( + var model: String = "", +) + +data class OnlineModelConfig( + var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(), + var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(), + var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(), + var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(), + var toneCtc: OnlineToneCtcModelConfig = OnlineToneCtcModelConfig(), + var tokens: String = "", + var numThreads: Int = 1, + var debug: Boolean = false, + var provider: String = "cpu", + var modelType: String = "", + var modelingUnit: String = "", + var bpeVocab: String = "", +) + +data class OnlineLMConfig( + var model: String = "", + var scale: Float = 0.5f, +) + +data class OnlineCtcFstDecoderConfig( + var graph: String = "", + var maxActive: Int = 3000, +) + +data class OnlineRecognizerConfig( + var featConfig: FeatureConfig = FeatureConfig(), + var modelConfig: OnlineModelConfig = OnlineModelConfig(), + var lmConfig: OnlineLMConfig = OnlineLMConfig(), + var ctcFstDecoderConfig: OnlineCtcFstDecoderConfig = OnlineCtcFstDecoderConfig(), + var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(), + var endpointConfig: EndpointConfig = EndpointConfig(), + var enableEndpoint: Boolean = true, + var decodingMethod: String = "greedy_search", + var maxActivePaths: Int = 4, + var hotwordsFile: String = "", + var hotwordsScore: Float = 1.5f, + var ruleFsts: String = "", + var ruleFars: String = "", + var blankPenalty: Float = 0.0f, +) + +data class OnlineRecognizerResult( + val text: String, + val tokens: Array, + val timestamps: FloatArray, + val ysProbs: FloatArray, + // TODO(fangjun): Add more fields +) + +class OnlineRecognizer( + assetManager: AssetManager? = null, + val config: OnlineRecognizerConfig, +) { + private var ptr: Long + + init { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + require(ptr != 0L) { + "Invalid OnlineRecognizerConfig: failed to create native OnlineRecognizer" + } + } + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun createStream(hotwords: String = ""): OnlineStream { + val p = createStream(ptr, hotwords) + return OnlineStream(p) + } + + fun reset(stream: OnlineStream) = reset(ptr, stream.ptr) + fun decode(stream: OnlineStream) = decode(ptr, stream.ptr) + fun isEndpoint(stream: OnlineStream) = isEndpoint(ptr, stream.ptr) + fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr) + fun getResult(stream: OnlineStream): OnlineRecognizerResult { + return getResult(ptr, stream.ptr) + } + + private external fun delete(ptr: Long) + + private external fun newFromAsset( + assetManager: AssetManager, + config: OnlineRecognizerConfig, + ): Long + + private external fun newFromFile( + config: OnlineRecognizerConfig, + ): Long + + private external fun createStream(ptr: Long, hotwords: String): Long + private external fun reset(ptr: Long, streamPtr: Long) + private external fun decode(ptr: Long, streamPtr: Long) + private external fun isEndpoint(ptr: Long, streamPtr: Long): Boolean + private external fun isReady(ptr: Long, streamPtr: Long): Boolean + private external fun getResult(ptr: Long, streamPtr: Long): OnlineRecognizerResult + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + + @JvmStatic + external fun prependAdspLibraryPath(newPath: String) // for qnn + } +} + + +/* +Please see +https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html +for a list of pre-trained models. + +We only add a few here. Please change the following code +to add your own. (It should be straightforward to add a new model +by following the code) + +@param type +0 - sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 (Bilingual, Chinese + English) + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english + +1 - csukuangfj/sherpa-onnx-lstm-zh-2023-02-20 (Chinese) + + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/lstm-transducer-models.html#csukuangfj-sherpa-onnx-lstm-zh-2023-02-20-chinese + +2 - csukuangfj/sherpa-onnx-lstm-en-2023-02-17 (English) + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/lstm-transducer-models.html#csukuangfj-sherpa-onnx-lstm-en-2023-02-17-english + +3,4 - pkufool/icefall-asr-zipformer-streaming-wenetspeech-20230615 + https://huggingface.co/pkufool/icefall-asr-zipformer-streaming-wenetspeech-20230615 + 3 - int8 encoder + 4 - float32 encoder + +5 - csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en + https://huggingface.co/csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en + +6 - sherpa-onnx-streaming-zipformer-en-2023-06-26 + https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26 + +7 - shaojieli/sherpa-onnx-streaming-zipformer-fr-2023-04-14 (French) + https://huggingface.co/shaojieli/sherpa-onnx-streaming-zipformer-fr-2023-04-14 + +8 - csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 (Bilingual, Chinese + English) + https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + encoder int8, decoder/joiner float32 + +9025 - sherpa-onnx-qnn-streaming-zipformer-transducer-zh-en-2023-03-20-chunk-size-32-android-aarch64 + QNN model libs with writable binary context files + libencoder.so, libdecoder.so, libjoiner.so + +9026 - sherpa-onnx-qnn-SM8850-binary-streaming-zipformer-transducer-zh-en-2023-03-20-chunk-size-32 + QNN binary context files only + encoder.bin, decoder.bin, joiner.bin + + */ +fun getModelConfig(type: Int, modelPrefix: String = ""): OnlineModelConfig? { + when (type) { + 0 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer", + ) + } + + 1 -> { + val modelDir = "sherpa-onnx-lstm-zh-2023-02-20" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-11-avg-1.onnx", + decoder = "$modelDir/decoder-epoch-11-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-11-avg-1.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "lstm", + ) + } + + 2 -> { + val modelDir = "sherpa-onnx-lstm-en-2023-02-17" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "lstm", + ) + } + + 3 -> { + val modelDir = "icefall-asr-zipformer-streaming-wenetspeech-20230615" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/exp/encoder-epoch-12-avg-4-chunk-16-left-128.int8.onnx", + decoder = "$modelDir/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx", + joiner = "$modelDir/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx", + ), + tokens = "$modelDir/data/lang_char/tokens.txt", + modelType = "zipformer2", + ) + } + + 4 -> { + val modelDir = "icefall-asr-zipformer-streaming-wenetspeech-20230615" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/exp/encoder-epoch-12-avg-4-chunk-16-left-128.onnx", + decoder = "$modelDir/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx", + joiner = "$modelDir/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx", + ), + tokens = "$modelDir/data/lang_char/tokens.txt", + modelType = "zipformer2", + ) + } + + 5 -> { + val modelDir = "sherpa-onnx-streaming-paraformer-bilingual-zh-en" + return OnlineModelConfig( + paraformer = OnlineParaformerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "paraformer", + ) + } + + 6 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-en-2023-06-26" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1-chunk-16-left-128.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1-chunk-16-left-128.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 7 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-fr-2023-04-14" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-29-avg-9-with-averaged-model.int8.onnx", + decoder = "$modelDir/decoder-epoch-29-avg-9-with-averaged-model.onnx", + joiner = "$modelDir/joiner-epoch-29-avg-9-with-averaged-model.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer", + ) + } + + 8 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer", + ) + } + + 9 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer", + ) + } + + 10 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer", + ) + } + + 11 -> { + val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms" + return OnlineModelConfig( + neMoCtc = OnlineNeMoCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 12 -> { + val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-480ms" + return OnlineModelConfig( + neMoCtc = OnlineNeMoCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 13 -> { + val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-1040ms" + return OnlineModelConfig( + neMoCtc = OnlineNeMoCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 14 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-korean-2024-06-16" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer", + ) + } + + 15 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-small-ctc-zh-int8-2025-04-01" + return OnlineModelConfig( + zipformer2Ctc = OnlineZipformer2CtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 16 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-small-ctc-zh-2025-04-01" + return OnlineModelConfig( + zipformer2Ctc = OnlineZipformer2CtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 17 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-ctc-zh-int8-2025-06-30" + return OnlineModelConfig( + zipformer2Ctc = OnlineZipformer2CtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 18 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-ctc-zh-2025-06-30" + return OnlineModelConfig( + zipformer2Ctc = OnlineZipformer2CtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 19 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-ctc-zh-fp16-2025-06-30" + return OnlineModelConfig( + zipformer2Ctc = OnlineZipformer2CtcModelConfig( + model = "$modelDir/model.fp16.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 20 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 21 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 22 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-es-kroko-2025-08-06" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 23 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-fr-kroko-2025-08-06" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 24 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-de-kroko-2025-08-06" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 25 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-small-ru-vosk-int8-2025-08-16" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 26 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-small-ru-vosk-2025-08-16" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 27 -> { + val modelDir = "sherpa-onnx-streaming-t-one-russian-2025-09-08" + return OnlineModelConfig( + toneCtc = OnlineToneCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 28 -> { + val modelDir = "sherpa-onnx-nemotron-speech-streaming-en-0.6b-int8-2026-01-14" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 29 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-bn-vosk-2026-02-09" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer2", + ) + } + + 30 -> { + val modelDir = "sherpa-onnx-nemotron-speech-streaming-en-0.6b-80ms-int8-2026-04-25" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 31 -> { + val modelDir = "sherpa-onnx-nemotron-speech-streaming-en-0.6b-160ms-int8-2026-04-25" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 32 -> { + val modelDir = "sherpa-onnx-nemotron-speech-streaming-en-0.6b-560ms-int8-2026-04-25" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 34 -> { + val modelDir = modelPrefix + + "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 33 -> { + val modelDir = "sherpa-onnx-nemotron-speech-streaming-en-0.6b-1120ms-int8-2026-04-25" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + joiner = "$modelDir/joiner.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9025 -> { + val modelDir = + "sherpa-onnx-qnn-streaming-zipformer-transducer-zh-en-2023-03-20-chunk-size-32-android-aarch64" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/libencoder.so", + decoder = "$modelDir/libdecoder.so", + joiner = "$modelDir/libjoiner.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + // The following three *.bin files are generated during the first run + // and are used to replace the corresponding *.so files in later runs + contextBinary = "$modelDir/encoder.bin,$modelDir/decoder.bin,$modelDir/joiner.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + provider = "qnn", + modelType = "zipformer", + ) + } + + 9026 -> { + // for Xiaomi 17 Pro + val modelDir = + "sherpa-onnx-qnn-SM8850-binary-streaming-zipformer-transducer-zh-en-2023-03-20-chunk-size-32" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/encoder.bin,$modelDir/decoder.bin,$modelDir/joiner.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + provider = "qnn", + modelType = "zipformer", + ) + } + + 9027 -> { + val modelDir = + "sherpa-onnx-qnn-x-asr-streaming-zipformer-transducer-zh-en-2026-06-05-chunk-size-480ms-android-aarch64" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/libencoder.so", + decoder = "$modelDir/libdecoder.so", + joiner = "$modelDir/libjoiner.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/encoder.bin,$modelDir/decoder.bin,$modelDir/joiner.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + provider = "qnn", + modelType = "zipformer", + ) + } + + 9028 -> { + val modelDir = + "sherpa-onnx-qnn-x-asr-streaming-zipformer-transducer-zh-en-punct-2026-06-05-chunk-size-480ms-android-aarch64" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/libencoder.so", + decoder = "$modelDir/libdecoder.so", + joiner = "$modelDir/libjoiner.so", + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/encoder.bin,$modelDir/decoder.bin,$modelDir/joiner.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + provider = "qnn", + modelType = "zipformer", + ) + } + + 9029 -> { + // for Xiaomi 17 Pro + val modelDir = + "sherpa-onnx-qnn-SM8850-binary-x-asr-streaming-zipformer-transducer-zh-en-2026-06-05-chunk-size-480ms" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/encoder.bin,$modelDir/decoder.bin,$modelDir/joiner.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + provider = "qnn", + modelType = "zipformer", + ) + } + + 9030 -> { + // for Xiaomi 17 Pro + val modelDir = + "sherpa-onnx-qnn-SM8850-binary-x-asr-streaming-zipformer-transducer-zh-en-punct-2026-06-05-chunk-size-480ms" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + qnnConfig = QnnConfig( + backendLib = "libQnnHtp.so", + systemLib = "libQnnSystem.so", + contextBinary = "$modelDir/encoder.bin,$modelDir/decoder.bin,$modelDir/joiner.bin", + ), + ), + tokens = "$modelDir/tokens.txt", + provider = "qnn", + modelType = "zipformer", + ) + } + + 1000 -> { + val modelDir = "sherpa-onnx-rk3588-streaming-zipformer-bilingual-zh-en-2023-02-20" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.rknn", + decoder = "$modelDir/decoder.rknn", + joiner = "$modelDir/joiner.rknn", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer", + provider = "rknn", + ) + } + + 1001 -> { + val modelDir = "sherpa-onnx-rk3588-streaming-zipformer-small-bilingual-zh-en-2023-02-16" + return OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = "$modelDir/encoder.rknn", + decoder = "$modelDir/decoder.rknn", + joiner = "$modelDir/joiner.rknn", + ), + tokens = "$modelDir/tokens.txt", + modelType = "zipformer", + provider = "rknn", + ) + } + + } + return null +} + +/* +Please see +https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html +for a list of pre-trained models. + +We only add a few here. Please change the following code +to add your own LM model. (It should be straightforward to train a new NN LM model +by following the code, https://github.com/k2-fsa/icefall/blob/master/icefall/rnn_lm/train.py) + +@param type +0 - sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 (Bilingual, Chinese + English) + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english + */ +fun getOnlineLMConfig(type: Int): OnlineLMConfig { + when (type) { + 0 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20" + return OnlineLMConfig( + model = "$modelDir/with-state-epoch-99-avg-1.int8.onnx", + scale = 0.5f, + ) + } + } + return OnlineLMConfig() +} + +fun getEndpointConfig(): EndpointConfig { + return EndpointConfig( + rule1 = EndpointRule(false, 2.4f, 0.0f), + rule2 = EndpointRule(true, 1.4f, 0.0f), + rule3 = EndpointRule(false, 0.0f, 20.0f) + ) +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt new file mode 100644 index 0000000..10497eb --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt @@ -0,0 +1,46 @@ +package com.k2fsa.sherpa.onnx + +class OnlineStream(var ptr: Long = 0) { + init { + require(ptr != 0L) { "Failed to create native OnlineStream" } + } + + fun acceptWaveform(samples: FloatArray, sampleRate: Int) = + acceptWaveform(ptr, samples, sampleRate) + + fun inputFinished() = inputFinished(ptr) + + fun setOption(key: String, value: String) = setOption(ptr, key, value) + + fun getOption(key: String): String = getOption(ptr, key) + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun use(block: (OnlineStream) -> Unit) { + try { + block(this) + } finally { + release() + } + } + + private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int) + private external fun inputFinished(ptr: Long) + private external fun setOption(ptr: Long, key: String, value: String) + private external fun getOption(ptr: Long, key: String): String + private external fun delete(ptr: Long) + + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt new file mode 100644 index 0000000..ea7aa51 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt @@ -0,0 +1,7 @@ +package com.k2fsa.sherpa.onnx + +data class QnnConfig( + var backendLib: String = "", + var contextBinary: String = "", + var systemLib: String = "", +) diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt new file mode 100644 index 0000000..563c64d --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbedding.kt @@ -0,0 +1,235 @@ +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager +import android.util.Log +import kotlin.math.sqrt + +// --------------------------------------------------------------------------- +// Native speaker-embedding extractor (wraps libsherpa-onnx-jni). +// Ported from android/SherpaOnnxSpeakerIdentification so both apps stay +// compatible with the same JNI symbols. +// --------------------------------------------------------------------------- +class SpeakerEmbeddingExtractor( + assetManager: AssetManager? = null, + config: SpeakerEmbeddingExtractorConfig, +) { + private var ptr: Long + + init { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + require(ptr != 0L) { + "Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor" + } + } + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun createStream(): OnlineStream { + val p = createStream(ptr) + return OnlineStream(p) + } + + fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr) + fun compute(stream: OnlineStream) = compute(ptr, stream.ptr) + fun dim() = dim(ptr) + + private external fun newFromAsset( + assetManager: AssetManager, + config: SpeakerEmbeddingExtractorConfig, + ): Long + + private external fun newFromFile(config: SpeakerEmbeddingExtractorConfig): Long + private external fun delete(ptr: Long) + private external fun createStream(ptr: Long): Long + private external fun isReady(ptr: Long, streamPtr: Long): Boolean + private external fun compute(ptr: Long, streamPtr: Long): FloatArray + private external fun dim(ptr: Long): Int + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} + +// --------------------------------------------------------------------------- +// SpeakerRegistry: lightweight in-Kotlin speaker store + cosine matching. +// +// We avoid the native SpeakerEmbeddingManager on purpose: it cannot rename a +// speaker once enrolled, which we need for the "auto-label then assign a real +// name" flow. Matching here prefers named entries over auto (S0/S1/...) ones. +// --------------------------------------------------------------------------- +class SpeakerRegistry { + private val named = ArrayList>() + private val auto = ArrayList>() + + val namedNames: List + get() = named.map { it.first } + + // The rolling window checker runs on its own thread while the endpoint + // identification runs on the recording thread: serialize everything. + @Synchronized + fun clear() { + named.clear() + auto.clear() + } + + // Returns the best label for embedding `e`, or null if no speaker is close + // enough (below threshold). Registered names win ties against auto labels. + @Synchronized + fun search(e: FloatArray, threshold: Float): String? { + match(named, e, threshold)?.let { return it } + match(auto, e, threshold)?.let { return it } + return null + } + + @Synchronized + fun addAuto(name: String, e: FloatArray) { + auto.add(Pair(name, l2normalize(e))) + } + + // Enroll (or overwrite) a real name for a given embedding. + @Synchronized + fun addNamed(name: String, e: FloatArray) { + val v = l2normalize(e) + named.removeAll { it.first == name } + // Drop any auto entry whose embedding now equals this named one so the + // same voice is not also reported as an anonymous Sx id. + named.add(Pair(name, v)) + } + + /** + * Best cosine similarity of `e` against all enrolled speakers (0..1 scale, + * may be slightly noisy). Used by the rolling checker to tell "no idea + * whose voice this is" from "definitely the current speaker". + */ + @Synchronized + fun bestScore(e: FloatArray): Float { + val q = l2normalize(e) + var best = -1.0f + for ((_, emb) in named) best = maxOf(best, dot(q, emb)) + for ((_, emb) in auto) best = maxOf(best, dot(q, emb)) + return best + } + + private fun match( + list: List>, + e: FloatArray, + threshold: Float, + ): String? { + if (list.isEmpty()) return null + val q = l2normalize(e) + var bestName: String? = null + var bestScore = -1.0f + for ((name, emb) in list) { + val s = dot(q, emb) + if (s > bestScore) { + bestScore = s + bestName = name + } + } + return if (bestScore >= threshold) bestName else null + } + + private fun l2normalize(v: FloatArray): FloatArray { + var sum = 0.0f + for (x in v) sum += x * x + if (sum <= 0f) return v.copyOf() + val inv = 1.0f / sqrt(sum) + return FloatArray(v.size) { v[it] * inv } + } + + private fun dot(a: FloatArray, b: FloatArray): Float { + var s = 0.0f + val n = minOf(a.size, b.size) + for (i in 0 until n) s += a[i] * b[i] + return s + } +} + +// --------------------------------------------------------------------------- +// Singleton holder shared by the UI thread and the recording thread. +// --------------------------------------------------------------------------- +object SpeakerRecognition { + private var _extractor: SpeakerEmbeddingExtractor? = null + val extractor: SpeakerEmbeddingExtractor + get() = _extractor!! + + val registry = SpeakerRegistry() + var autoCounter: Int = 0 + private set + + // autoCounter is touched from the recording thread (identifySpeaker) while + // resetSpeakers runs on the UI thread — keep them serialized. + @Synchronized + fun nextAutoLabel(): String = "S${autoCounter++}" + + @Synchronized + fun resetSpeakers() { + registry.clear() + autoCounter = 0 + } + + // assetManager = null -> `model` is treated as an absolute file system path + fun initExtractor(assetManager: AssetManager?, model: String) { + synchronized(this) { + if (_extractor != null) return + Log.i(TAG, "Initializing speaker embedding extractor with $model") + _extractor = SpeakerEmbeddingExtractor( + assetManager = assetManager, + config = SpeakerEmbeddingExtractorConfig( + model = model, + numThreads = 2, + debug = false, + provider = "cpu", + ) + ) + } + } + + // Compute one embedding from a complete utterance (float mono @ 16 kHz). + // Returns null when the utterance is too short for the model to consume. + // Serialized: the native extractor holds one session shared by the + // recording thread and the rolling-window checker thread. + @Synchronized + fun computeEmbedding(samples: FloatArray, sampleRate: Int): FloatArray? { + val ext = _extractor ?: return null + if (samples.isEmpty()) return null + val stream = ext.createStream() + stream.acceptWaveform(samples = samples, sampleRate = sampleRate) + stream.inputFinished() + val emb = if (ext.isReady(stream)) ext.compute(stream) else null + stream.release() + return emb + } + + // Cosine similarity between two embeddings (L2-normalized internally), + // in [-1, 1]. Pure function, safe from any thread. + fun similarity(a: FloatArray, b: FloatArray): Float { + val n = minOf(a.size, b.size) + if (n == 0) return 0f + var dot = 0.0 + var na = 0.0 + var nb = 0.0 + for (i in 0 until n) { + dot += a[i].toDouble() * b[i] + na += a[i].toDouble() * a[i] + nb += b[i].toDouble() * b[i] + } + if (na <= 0.0 || nb <= 0.0) return 0f + return (dot / (kotlin.math.sqrt(na) * kotlin.math.sqrt(nb))).toFloat() + } + + private const val TAG = "sherpa-onnx-speaker" +} diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.kt new file mode 100644 index 0000000..0ae2bb9 --- /dev/null +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.kt @@ -0,0 +1,8 @@ +package com.k2fsa.sherpa.onnx + +data class SpeakerEmbeddingExtractorConfig( + val model: String = "", + var numThreads: Int = 1, + var debug: Boolean = false, + var provider: String = "cpu", +) diff --git a/app/src/main/jniLibs/.gitkeep b/app/src/main/jniLibs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/src/main/jniLibs/arm64-v8a/.gitkeep b/app/src/main/jniLibs/arm64-v8a/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/src/main/jniLibs/armeabi-v7a/.gitkeep b/app/src/main/jniLibs/armeabi-v7a/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/src/main/jniLibs/x86/.gitkeep b/app/src/main/jniLibs/x86/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/src/main/jniLibs/x86_64/.gitkeep b/app/src/main/jniLibs/x86_64/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 0000000..2b068d1 --- /dev/null +++ b/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/drawable/ic_launcher_background.xml b/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 0000000..07d5da9 --- /dev/null +++ b/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml new file mode 100644 index 0000000..29abe2d --- /dev/null +++ b/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,76 @@ + + + + + + + + + + + + + + + + +