feat:mp3 (#3)
--------- Co-authored-by: intelligence <intelligence@debian.local> Reviewed-on: #3
This commit was merged in pull request #3.
This commit is contained in:
1 parent
32e7f32e0a
commit
64b9d704fb
25 files changed
+488
-98
No files matched your search
@@ -13,6 +13,8 @@ jobs:
|
|||||||
releases: write
|
releases: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
submodules: recursive
|
||||||
- uses: actions/setup-java@v4
|
- uses: actions/setup-java@v4
|
||||||
with:
|
with:
|
||||||
distribution: temurin
|
distribution: temurin
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ jobs:
|
|||||||
CMAKE_BUILD_PARALLEL_LEVEL: 2
|
CMAKE_BUILD_PARALLEL_LEVEL: 2
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
submodules: recursive
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: |
|
run: |
|
||||||
apt-get update
|
apt-get update
|
||||||
@@ -58,6 +60,8 @@ jobs:
|
|||||||
CMAKE_BUILD_PARALLEL_LEVEL: 2
|
CMAKE_BUILD_PARALLEL_LEVEL: 2
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
submodules: recursive
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: |
|
run: |
|
||||||
apt-get update
|
apt-get update
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[submodule "third_party/shine"]
|
||||||
|
path = third_party/shine
|
||||||
|
url = https://github.com/toots/shine.git
|
||||||
@@ -3,7 +3,8 @@
|
|||||||
- Linux is a GTK3 desktop recorder. It supports hosted transcription with an optional local fallback and can type at the cursor or copy to the clipboard.
|
- Linux is a GTK3 desktop recorder. It supports hosted transcription with an optional local fallback and can type at the cursor or copy to the clipboard.
|
||||||
- macOS is a SwiftUI menu bar recorder targeting macOS 15. It supports the same hosted-first, local-fallback behavior as Linux, a global recording hotkey, launch at login, cursor typing, and clipboard output. When hosted-first is off, use local only. Its compact floating recorder uses teal level bars for recording feedback.
|
- macOS is a SwiftUI menu bar recorder targeting macOS 15. It supports the same hosted-first, local-fallback behavior as Linux, a global recording hotkey, launch at login, cursor typing, and clipboard output. When hosted-first is off, use local only. Its compact floating recorder uses teal level bars for recording feedback.
|
||||||
- Android is remote-only. It provides an Android `RecognitionService`, a voice-only input method, and a floating recognition activity. Do not add a local llama.cpp path to Android.
|
- Android is remote-only. It provides an Android `RecognitionService`, a voice-only input method, and a floating recognition activity. Do not add a local llama.cpp path to Android.
|
||||||
- All clients call OpenAI-compatible `/v1/audio/transcriptions` endpoints and clean only the Qwen3-ASR `language …<astext>` or `language …<asr_text>` prefix.
|
- All clients upload 16 kHz mono, 32 kbps MP3 to OpenAI-compatible `/v1/audio/transcriptions` endpoints and clean only the Qwen3-ASR `language …<astext>` or `language …<asr_text>` prefix.
|
||||||
|
- MP3 encoding uses the `third_party/shine` submodule pinned to `ab5e3526b64af1a2eaa43aa6f441a7312e013519` (`3.1.1-59-gab5e352`).
|
||||||
- Generate the macOS Xcode project from `macos/project.yml` with XcodeGen. Treat the spec as the source of truth and leave `macos/Transui.xcodeproj` untracked.
|
- Generate the macOS Xcode project from `macos/project.yml` with XcodeGen. Treat the spec as the source of truth and leave `macos/Transui.xcodeproj` untracked.
|
||||||
- The Android voice UI is an opaque, compact system surface. Teal is the recording signal. `PushToTalkButton` draws the Apps and Stop Circle glyphs with Canvas because the target device drops `ImageButton` image layers. Keep the glyphs drawn above the button background.
|
- The Android voice UI is an opaque, compact system surface. Teal is the recording signal. `PushToTalkButton` draws the Apps and Stop Circle glyphs with Canvas because the target device drops `ImageButton` image layers. Keep the glyphs drawn above the button background.
|
||||||
|
|
||||||
|
|||||||
+21
-1
@@ -1,5 +1,5 @@
|
|||||||
cmake_minimum_required(VERSION 3.16)
|
cmake_minimum_required(VERSION 3.16)
|
||||||
project(transui VERSION 0.1.1 LANGUAGES CXX)
|
project(transui VERSION 0.1.1 LANGUAGES C CXX)
|
||||||
|
|
||||||
set(CMAKE_CXX_STANDARD 17)
|
set(CMAKE_CXX_STANDARD 17)
|
||||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||||
@@ -31,6 +31,26 @@ if(BUILD_TESTING)
|
|||||||
)
|
)
|
||||||
target_include_directories(transcript_cleaner_test PRIVATE src)
|
target_include_directories(transcript_cleaner_test PRIVATE src)
|
||||||
add_test(NAME transcript_cleaner COMMAND transcript_cleaner_test)
|
add_test(NAME transcript_cleaner COMMAND transcript_cleaner_test)
|
||||||
|
|
||||||
|
add_executable(mp3_encoder_test
|
||||||
|
tests/mp3_encoder_test.c
|
||||||
|
third_party/transui_mp3/transui_mp3.c
|
||||||
|
third_party/shine/src/lib/bitstream.c
|
||||||
|
third_party/shine/src/lib/huffman.c
|
||||||
|
third_party/shine/src/lib/l3bitstream.c
|
||||||
|
third_party/shine/src/lib/l3loop.c
|
||||||
|
third_party/shine/src/lib/l3mdct.c
|
||||||
|
third_party/shine/src/lib/l3subband.c
|
||||||
|
third_party/shine/src/lib/layer3.c
|
||||||
|
third_party/shine/src/lib/reservoir.c
|
||||||
|
third_party/shine/src/lib/tables.c
|
||||||
|
)
|
||||||
|
target_include_directories(mp3_encoder_test PRIVATE
|
||||||
|
third_party/transui_mp3
|
||||||
|
third_party/shine/src/lib
|
||||||
|
)
|
||||||
|
target_link_libraries(mp3_encoder_test PRIVATE m)
|
||||||
|
add_test(NAME mp3_encoder COMMAND mp3_encoder_test)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
set(CPACK_GENERATOR "DEB")
|
set(CPACK_GENERATOR "DEB")
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ sudo apt install cmake g++ pkg-config libgtk-3-dev libcurl4-openssl-dev \
|
|||||||
|
|
||||||
On macOS, install Xcode 16 and XcodeGen.
|
On macOS, install Xcode 16 and XcodeGen.
|
||||||
|
|
||||||
|
For Android, install Android SDK 35, the Android NDK, and CMake.
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
### Linux
|
### Linux
|
||||||
@@ -39,7 +41,7 @@ Build the `Transui` scheme in Xcode.
|
|||||||
### Android
|
### Android
|
||||||
|
|
||||||
Android release APKs build in Gitea CI. To run local checks, configure the
|
Android release APKs build in Gitea CI. To run local checks, configure the
|
||||||
Android SDK and run:
|
Android SDK and NDK, then run:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
cd android
|
cd android
|
||||||
|
|||||||
@@ -14,6 +14,12 @@ android {
|
|||||||
versionName '0.1.1'
|
versionName '0.1.1'
|
||||||
|
|
||||||
testInstrumentationRunner 'android.test.InstrumentationTestRunner'
|
testInstrumentationRunner 'android.test.InstrumentationTestRunner'
|
||||||
|
|
||||||
|
externalNativeBuild {
|
||||||
|
cmake {
|
||||||
|
arguments '-DANDROID_STL=none'
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
signingConfigs {
|
signingConfigs {
|
||||||
@@ -46,6 +52,12 @@ android {
|
|||||||
checkReleaseBuilds true
|
checkReleaseBuilds true
|
||||||
warningsAsErrors true
|
warningsAsErrors true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
externalNativeBuild {
|
||||||
|
cmake {
|
||||||
|
path file('src/main/cpp/CMakeLists.txt')
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
cmake_minimum_required(VERSION 3.22.1)
|
||||||
|
project(transui_mp3 LANGUAGES C)
|
||||||
|
|
||||||
|
set(SHINE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../third_party/shine/src/lib")
|
||||||
|
set(TRANSUI_MP3_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../third_party/transui_mp3")
|
||||||
|
|
||||||
|
add_library(transui_mp3 SHARED
|
||||||
|
native_mp3_encoder.c
|
||||||
|
"${TRANSUI_MP3_DIR}/transui_mp3.c"
|
||||||
|
"${SHINE_DIR}/bitstream.c"
|
||||||
|
"${SHINE_DIR}/huffman.c"
|
||||||
|
"${SHINE_DIR}/l3bitstream.c"
|
||||||
|
"${SHINE_DIR}/l3loop.c"
|
||||||
|
"${SHINE_DIR}/l3mdct.c"
|
||||||
|
"${SHINE_DIR}/l3subband.c"
|
||||||
|
"${SHINE_DIR}/layer3.c"
|
||||||
|
"${SHINE_DIR}/reservoir.c"
|
||||||
|
"${SHINE_DIR}/tables.c"
|
||||||
|
)
|
||||||
|
|
||||||
|
target_compile_features(transui_mp3 PRIVATE c_std_11)
|
||||||
|
target_compile_options(transui_mp3 PRIVATE -Wall -Wextra)
|
||||||
|
target_include_directories(transui_mp3 PRIVATE "${TRANSUI_MP3_DIR}" "${SHINE_DIR}")
|
||||||
|
target_link_libraries(transui_mp3 PRIVATE m)
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
#include <jni.h>
|
||||||
|
|
||||||
|
#include "transui_mp3.h"
|
||||||
|
|
||||||
|
JNIEXPORT jbyteArray JNICALL
|
||||||
|
Java_io_github_transui_android_Mp3Encoder_encodeNative(JNIEnv *env,
|
||||||
|
jclass type,
|
||||||
|
jbyteArray pcm) {
|
||||||
|
(void)type;
|
||||||
|
if (!pcm) return NULL;
|
||||||
|
|
||||||
|
const jsize pcm_size = (*env)->GetArrayLength(env, pcm);
|
||||||
|
jbyte *pcm_bytes = (*env)->GetByteArrayElements(env, pcm, NULL);
|
||||||
|
if (!pcm_bytes) return NULL;
|
||||||
|
|
||||||
|
uint8_t *mp3 = NULL;
|
||||||
|
size_t mp3_size = 0;
|
||||||
|
const int result = transui_mp3_encode_mono_16k_32kbps(
|
||||||
|
(const uint8_t *)pcm_bytes, (size_t)pcm_size, &mp3, &mp3_size);
|
||||||
|
(*env)->ReleaseByteArrayElements(env, pcm, pcm_bytes, JNI_ABORT);
|
||||||
|
if (result != TRANSUI_MP3_OK || !mp3 || mp3_size > INT32_MAX) {
|
||||||
|
transui_mp3_free(mp3);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
jbyteArray output = (*env)->NewByteArray(env, (jsize)mp3_size);
|
||||||
|
if (output) {
|
||||||
|
(*env)->SetByteArrayRegion(env, output, 0, (jsize)mp3_size,
|
||||||
|
(const jbyte *)mp3);
|
||||||
|
}
|
||||||
|
transui_mp3_free(mp3);
|
||||||
|
return output;
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package io.github.transui.android;
|
||||||
|
|
||||||
|
final class Mp3Encoder {
|
||||||
|
static {
|
||||||
|
System.loadLibrary("transui_mp3");
|
||||||
|
}
|
||||||
|
|
||||||
|
private Mp3Encoder() {}
|
||||||
|
|
||||||
|
static byte[] encodePcm16Mono(byte[] pcm) {
|
||||||
|
if (pcm == null || pcm.length == 0 || pcm.length % 2 != 0) {
|
||||||
|
throw new IllegalArgumentException("PCM input must contain complete 16-bit samples");
|
||||||
|
}
|
||||||
|
byte[] mp3 = encodeNative(pcm);
|
||||||
|
if (mp3 == null || mp3.length == 0) {
|
||||||
|
throw new IllegalStateException("MP3 encoding failed");
|
||||||
|
}
|
||||||
|
return mp3;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static native byte[] encodeNative(byte[] pcm);
|
||||||
|
}
|
||||||
@@ -19,11 +19,11 @@ final class TranscriptionClient {
|
|||||||
private static final int MAX_RESPONSE_BYTES = 1024 * 1024;
|
private static final int MAX_RESPONSE_BYTES = 1024 * 1024;
|
||||||
private volatile HttpURLConnection activeConnection;
|
private volatile HttpURLConnection activeConnection;
|
||||||
|
|
||||||
Result transcribe(AppConfig config, byte[] wav) {
|
Result transcribe(AppConfig config, byte[] mp3) {
|
||||||
if (config.endpoint.isEmpty()) {
|
if (config.endpoint.isEmpty()) {
|
||||||
return Result.error("No transcription endpoint is configured", SpeechRecognizer.ERROR_CLIENT);
|
return Result.error("No transcription endpoint is configured", SpeechRecognizer.ERROR_CLIENT);
|
||||||
}
|
}
|
||||||
return request(config.endpoint, config.model, config.apiKey, config.prompt, wav);
|
return request(config.endpoint, config.model, config.apiKey, config.prompt, mp3);
|
||||||
}
|
}
|
||||||
|
|
||||||
void cancel() {
|
void cancel() {
|
||||||
@@ -36,11 +36,11 @@ final class TranscriptionClient {
|
|||||||
String model,
|
String model,
|
||||||
String key,
|
String key,
|
||||||
String prompt,
|
String prompt,
|
||||||
byte[] wav) {
|
byte[] mp3) {
|
||||||
HttpURLConnection connection = null;
|
HttpURLConnection connection = null;
|
||||||
try {
|
try {
|
||||||
String boundary = "transui-" + UUID.randomUUID();
|
String boundary = "transui-" + UUID.randomUUID();
|
||||||
byte[] requestBody = multipartBody(boundary, model, prompt, wav);
|
byte[] requestBody = multipartBody(boundary, model, prompt, mp3);
|
||||||
connection = (HttpURLConnection) new URL(endpointUrl(endpoint)).openConnection();
|
connection = (HttpURLConnection) new URL(endpointUrl(endpoint)).openConnection();
|
||||||
activeConnection = connection;
|
activeConnection = connection;
|
||||||
connection.setConnectTimeout(12_000);
|
connection.setConnectTimeout(12_000);
|
||||||
@@ -86,17 +86,17 @@ final class TranscriptionClient {
|
|||||||
return result + "/v1/audio/transcriptions";
|
return result + "/v1/audio/transcriptions";
|
||||||
}
|
}
|
||||||
|
|
||||||
private static byte[] multipartBody(String boundary, String model, String prompt, byte[] wav)
|
private static byte[] multipartBody(String boundary, String model, String prompt, byte[] mp3)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
ByteArrayOutputStream bytes = new ByteArrayOutputStream(wav.length + 1024);
|
ByteArrayOutputStream bytes = new ByteArrayOutputStream(mp3.length + 1024);
|
||||||
try (DataOutputStream output = new DataOutputStream(bytes)) {
|
try (DataOutputStream output = new DataOutputStream(bytes)) {
|
||||||
writeField(output, boundary, "response_format", "json");
|
writeField(output, boundary, "response_format", "json");
|
||||||
if (!model.isEmpty()) writeField(output, boundary, "model", model);
|
if (!model.isEmpty()) writeField(output, boundary, "model", model);
|
||||||
if (!prompt.isEmpty()) writeField(output, boundary, "prompt", prompt);
|
if (!prompt.isEmpty()) writeField(output, boundary, "prompt", prompt);
|
||||||
output.writeBytes("--" + boundary + "\r\n");
|
output.writeBytes("--" + boundary + "\r\n");
|
||||||
output.writeBytes("Content-Disposition: form-data; name=\"file\"; filename=\"recording.wav\"\r\n");
|
output.writeBytes("Content-Disposition: form-data; name=\"file\"; filename=\"recording.mp3\"\r\n");
|
||||||
output.writeBytes("Content-Type: audio/wav\r\n\r\n");
|
output.writeBytes("Content-Type: audio/mpeg\r\n\r\n");
|
||||||
output.write(wav);
|
output.write(mp3);
|
||||||
output.writeBytes("\r\n--" + boundary + "--\r\n");
|
output.writeBytes("\r\n--" + boundary + "--\r\n");
|
||||||
}
|
}
|
||||||
return bytes.toByteArray();
|
return bytes.toByteArray();
|
||||||
|
|||||||
@@ -124,8 +124,15 @@ public final class TransuiRecognitionService extends RecognitionService {
|
|||||||
if (pcm == null || cancelled.get()) return;
|
if (pcm == null || cancelled.get()) return;
|
||||||
sendEndOfSpeech();
|
sendEndOfSpeech();
|
||||||
if (cancelled.get()) return;
|
if (cancelled.get()) return;
|
||||||
TranscriptionClient.Result result = client.transcribe(
|
byte[] mp3;
|
||||||
config, WavEncoder.encodePcm16Mono(pcm, SAMPLE_RATE));
|
try {
|
||||||
|
mp3 = Mp3Encoder.encodePcm16Mono(pcm);
|
||||||
|
} catch (IllegalArgumentException | IllegalStateException error) {
|
||||||
|
Log.e(TAG, "MP3 encoding failed", error);
|
||||||
|
sendError(callback, SpeechRecognizer.ERROR_AUDIO);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
TranscriptionClient.Result result = client.transcribe(config, mp3);
|
||||||
if (cancelled.get()) return;
|
if (cancelled.get()) return;
|
||||||
if (result.text == null) {
|
if (result.text == null) {
|
||||||
Log.w(TAG, result.error);
|
Log.w(TAG, result.error);
|
||||||
|
|||||||
@@ -67,8 +67,14 @@ final class VoiceRecordingController {
|
|||||||
byte[] pcm = captureAudio();
|
byte[] pcm = captureAudio();
|
||||||
if (pcm == null || cancelled.get()) return;
|
if (pcm == null || cancelled.get()) return;
|
||||||
listener.onTranscribing();
|
listener.onTranscribing();
|
||||||
TranscriptionClient.Result result = client.transcribe(
|
byte[] mp3;
|
||||||
config, WavEncoder.encodePcm16Mono(pcm, SAMPLE_RATE));
|
try {
|
||||||
|
mp3 = Mp3Encoder.encodePcm16Mono(pcm);
|
||||||
|
} catch (IllegalArgumentException | IllegalStateException error) {
|
||||||
|
listener.onError(SpeechRecognizer.ERROR_AUDIO);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
TranscriptionClient.Result result = client.transcribe(config, mp3);
|
||||||
if (cancelled.get()) return;
|
if (cancelled.get()) return;
|
||||||
if (result.text != null) {
|
if (result.text != null) {
|
||||||
listener.onResult(result.text);
|
listener.onResult(result.text);
|
||||||
|
|||||||
@@ -1,44 +0,0 @@
|
|||||||
package io.github.transui.android;
|
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
|
||||||
|
|
||||||
final class WavEncoder {
|
|
||||||
private static final int HEADER_SIZE = 44;
|
|
||||||
|
|
||||||
private WavEncoder() {}
|
|
||||||
|
|
||||||
static byte[] encodePcm16Mono(byte[] pcm, int sampleRate) {
|
|
||||||
ByteArrayOutputStream output = new ByteArrayOutputStream(HEADER_SIZE + pcm.length);
|
|
||||||
writeAscii(output, "RIFF");
|
|
||||||
writeInt(output, 36 + pcm.length);
|
|
||||||
writeAscii(output, "WAVE");
|
|
||||||
writeAscii(output, "fmt ");
|
|
||||||
writeInt(output, 16);
|
|
||||||
writeShort(output, 1);
|
|
||||||
writeShort(output, 1);
|
|
||||||
writeInt(output, sampleRate);
|
|
||||||
writeInt(output, sampleRate * 2);
|
|
||||||
writeShort(output, 2);
|
|
||||||
writeShort(output, 16);
|
|
||||||
writeAscii(output, "data");
|
|
||||||
writeInt(output, pcm.length);
|
|
||||||
output.write(pcm, 0, pcm.length);
|
|
||||||
return output.toByteArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void writeAscii(ByteArrayOutputStream output, String value) {
|
|
||||||
for (int index = 0; index < value.length(); index++) output.write(value.charAt(index));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void writeShort(ByteArrayOutputStream output, int value) {
|
|
||||||
output.write(value & 0xff);
|
|
||||||
output.write((value >>> 8) & 0xff);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void writeInt(ByteArrayOutputStream output, int value) {
|
|
||||||
output.write(value & 0xff);
|
|
||||||
output.write((value >>> 8) & 0xff);
|
|
||||||
output.write((value >>> 16) & 0xff);
|
|
||||||
output.write((value >>> 24) & 0xff);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
package io.github.transui.android;
|
|
||||||
|
|
||||||
import static org.junit.Assert.assertArrayEquals;
|
|
||||||
import static org.junit.Assert.assertEquals;
|
|
||||||
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
|
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
public final class WavEncoderTest {
|
|
||||||
@Test
|
|
||||||
public void writesPcm16MonoHeaderAndPayload() {
|
|
||||||
byte[] pcm = new byte[]{1, 2, 3, 4};
|
|
||||||
byte[] wav = WavEncoder.encodePcm16Mono(pcm, 16_000);
|
|
||||||
|
|
||||||
assertEquals("RIFF", ascii(wav, 0, 4));
|
|
||||||
assertEquals(40, littleEndianInt(wav, 4));
|
|
||||||
assertEquals("WAVE", ascii(wav, 8, 4));
|
|
||||||
assertEquals(16_000, littleEndianInt(wav, 24));
|
|
||||||
assertEquals(32_000, littleEndianInt(wav, 28));
|
|
||||||
assertEquals(4, littleEndianInt(wav, 40));
|
|
||||||
assertArrayEquals(pcm, new byte[]{wav[44], wav[45], wav[46], wav[47]});
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String ascii(byte[] bytes, int offset, int length) {
|
|
||||||
return new String(bytes, offset, length, StandardCharsets.US_ASCII);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static int littleEndianInt(byte[] bytes, int offset) {
|
|
||||||
return (bytes[offset] & 0xff)
|
|
||||||
| ((bytes[offset + 1] & 0xff) << 8)
|
|
||||||
| ((bytes[offset + 2] & 0xff) << 16)
|
|
||||||
| ((bytes[offset + 3] & 0xff) << 24);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -145,8 +145,12 @@ final class AppModel: ObservableObject {
|
|||||||
Task {
|
Task {
|
||||||
defer { try? FileManager.default.removeItem(at: recording) }
|
defer { try? FileManager.default.removeItem(at: recording) }
|
||||||
do {
|
do {
|
||||||
|
let encoded = try await Task.detached(priority: .userInitiated) {
|
||||||
|
try MP3Encoder.encode(wav: recording)
|
||||||
|
}.value
|
||||||
|
defer { try? FileManager.default.removeItem(at: encoded) }
|
||||||
let text = try await transcriptionClient.transcribe(
|
let text = try await transcriptionClient.transcribe(
|
||||||
recording: recording,
|
recording: encoded,
|
||||||
plan: transcriptionPlan,
|
plan: transcriptionPlan,
|
||||||
prompt: settings.prompt
|
prompt: settings.prompt
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
import Foundation
|
||||||
|
|
||||||
|
enum MP3EncodingError: LocalizedError {
|
||||||
|
case invalidRecording
|
||||||
|
case encodingFailed
|
||||||
|
|
||||||
|
var errorDescription: String? {
|
||||||
|
switch self {
|
||||||
|
case .invalidRecording: "The recording could not be read."
|
||||||
|
case .encodingFailed: "The recording could not be compressed."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum MP3Encoder {
|
||||||
|
static func encode(wav url: URL) throws -> URL {
|
||||||
|
let wav = try Data(contentsOf: url, options: .mappedIfSafe)
|
||||||
|
let pcm = try pcmData(fromWAV: wav)
|
||||||
|
var encodedBytes: UnsafeMutablePointer<UInt8>?
|
||||||
|
var encodedSize = 0
|
||||||
|
let result = pcm.withUnsafeBytes { bytes in
|
||||||
|
transui_mp3_encode_mono_16k_32kbps(
|
||||||
|
bytes.bindMemory(to: UInt8.self).baseAddress,
|
||||||
|
bytes.count,
|
||||||
|
&encodedBytes,
|
||||||
|
&encodedSize
|
||||||
|
)
|
||||||
|
}
|
||||||
|
guard result == TRANSUI_MP3_OK, let encodedBytes, encodedSize > 0 else {
|
||||||
|
transui_mp3_free(encodedBytes)
|
||||||
|
throw MP3EncodingError.encodingFailed
|
||||||
|
}
|
||||||
|
defer { transui_mp3_free(encodedBytes) }
|
||||||
|
|
||||||
|
let output = FileManager.default.temporaryDirectory
|
||||||
|
.appendingPathComponent("transui-\(UUID().uuidString).mp3")
|
||||||
|
try Data(bytes: encodedBytes, count: encodedSize).write(to: output, options: .atomic)
|
||||||
|
return output
|
||||||
|
}
|
||||||
|
|
||||||
|
static func pcmData(fromWAV wav: Data) throws -> Data {
|
||||||
|
guard wav.count >= 12,
|
||||||
|
String(decoding: wav[0..<4], as: UTF8.self) == "RIFF",
|
||||||
|
String(decoding: wav[8..<12], as: UTF8.self) == "WAVE"
|
||||||
|
else { throw MP3EncodingError.invalidRecording }
|
||||||
|
|
||||||
|
var formatIsSupported = false
|
||||||
|
var audio: Data?
|
||||||
|
var offset = 12
|
||||||
|
while offset <= wav.count - 8 {
|
||||||
|
let chunkName = String(decoding: wav[offset..<(offset + 4)], as: UTF8.self)
|
||||||
|
let chunkSize = Int(readUInt32(wav, at: offset + 4))
|
||||||
|
let contentOffset = offset + 8
|
||||||
|
guard chunkSize >= 0, contentOffset <= wav.count,
|
||||||
|
chunkSize <= wav.count - contentOffset
|
||||||
|
else { throw MP3EncodingError.invalidRecording }
|
||||||
|
|
||||||
|
if chunkName == "fmt ", chunkSize >= 16 {
|
||||||
|
formatIsSupported = readUInt16(wav, at: contentOffset) == 1
|
||||||
|
&& readUInt16(wav, at: contentOffset + 2) == 1
|
||||||
|
&& readUInt32(wav, at: contentOffset + 4) == 16_000
|
||||||
|
&& readUInt16(wav, at: contentOffset + 14) == 16
|
||||||
|
} else if chunkName == "data" {
|
||||||
|
audio = wav.subdata(in: contentOffset..<(contentOffset + chunkSize))
|
||||||
|
}
|
||||||
|
|
||||||
|
let paddedSize = chunkSize + (chunkSize & 1)
|
||||||
|
guard paddedSize <= Int.max - contentOffset else {
|
||||||
|
throw MP3EncodingError.invalidRecording
|
||||||
|
}
|
||||||
|
offset = contentOffset + paddedSize
|
||||||
|
}
|
||||||
|
|
||||||
|
guard formatIsSupported, let audio, !audio.isEmpty, audio.count.isMultiple(of: 2) else {
|
||||||
|
throw MP3EncodingError.invalidRecording
|
||||||
|
}
|
||||||
|
return audio
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func readUInt16(_ data: Data, at offset: Int) -> UInt16 {
|
||||||
|
UInt16(data[offset]) | UInt16(data[offset + 1]) << 8
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func readUInt32(_ data: Data, at offset: Int) -> UInt32 {
|
||||||
|
UInt32(data[offset])
|
||||||
|
| UInt32(data[offset + 1]) << 8
|
||||||
|
| UInt32(data[offset + 2]) << 16
|
||||||
|
| UInt32(data[offset + 3]) << 24
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -146,8 +146,8 @@ struct TranscriptionClient {
|
|||||||
if !model.isEmpty { field("model", model) }
|
if !model.isEmpty { field("model", model) }
|
||||||
if !prompt.isEmpty { field("prompt", prompt) }
|
if !prompt.isEmpty { field("prompt", prompt) }
|
||||||
data.append("--\(boundary)\r\n")
|
data.append("--\(boundary)\r\n")
|
||||||
data.append("Content-Disposition: form-data; name=\"file\"; filename=\"recording.wav\"\r\n")
|
data.append("Content-Disposition: form-data; name=\"file\"; filename=\"recording.mp3\"\r\n")
|
||||||
data.append("Content-Type: audio/wav\r\n\r\n")
|
data.append("Content-Type: audio/mpeg\r\n\r\n")
|
||||||
data.append(try Data(contentsOf: recording))
|
data.append(try Data(contentsOf: recording))
|
||||||
data.append("\r\n--\(boundary)--\r\n")
|
data.append("\r\n--\(boundary)--\r\n")
|
||||||
return data
|
return data
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
#include "transui_mp3.h"
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import XCTest
|
||||||
|
|
||||||
|
@testable import Transui
|
||||||
|
|
||||||
|
final class MP3EncoderTests: XCTestCase {
|
||||||
|
func testExtractsSupportedPCMDataFromWAV() throws {
|
||||||
|
let samples = Data([0x01, 0x02, 0x03, 0x04])
|
||||||
|
let wav = makeWAV(samples: samples)
|
||||||
|
|
||||||
|
XCTAssertEqual(try MP3Encoder.pcmData(fromWAV: wav), samples)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRejectsWrongSampleRate() {
|
||||||
|
XCTAssertThrowsError(try MP3Encoder.pcmData(fromWAV: makeWAV(sampleRate: 44_100)))
|
||||||
|
}
|
||||||
|
|
||||||
|
private func makeWAV(sampleRate: UInt32 = 16_000, samples: Data = Data([0, 0])) -> Data {
|
||||||
|
var data = Data("RIFF".utf8)
|
||||||
|
append(UInt32(36 + samples.count), to: &data)
|
||||||
|
data.append(Data("WAVEfmt ".utf8))
|
||||||
|
append(UInt32(16), to: &data)
|
||||||
|
append(UInt16(1), to: &data)
|
||||||
|
append(UInt16(1), to: &data)
|
||||||
|
append(sampleRate, to: &data)
|
||||||
|
append(sampleRate * 2, to: &data)
|
||||||
|
append(UInt16(2), to: &data)
|
||||||
|
append(UInt16(16), to: &data)
|
||||||
|
data.append(Data("data".utf8))
|
||||||
|
append(UInt32(samples.count), to: &data)
|
||||||
|
data.append(samples)
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
private func append(_ value: UInt16, to data: inout Data) {
|
||||||
|
data.append(UInt8(value & 0xff))
|
||||||
|
data.append(UInt8(value >> 8))
|
||||||
|
}
|
||||||
|
|
||||||
|
private func append(_ value: UInt32, to data: inout Data) {
|
||||||
|
data.append(UInt8(value & 0xff))
|
||||||
|
data.append(UInt8((value >> 8) & 0xff))
|
||||||
|
data.append(UInt8((value >> 16) & 0xff))
|
||||||
|
data.append(UInt8((value >> 24) & 0xff))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,6 +24,8 @@ targets:
|
|||||||
deploymentTarget: "15.0"
|
deploymentTarget: "15.0"
|
||||||
sources:
|
sources:
|
||||||
- path: Sources/Transui
|
- path: Sources/Transui
|
||||||
|
- path: ../third_party/transui_mp3
|
||||||
|
- path: ../third_party/shine/src/lib
|
||||||
settings:
|
settings:
|
||||||
base:
|
base:
|
||||||
CODE_SIGN_ENTITLEMENTS: Transui.entitlements
|
CODE_SIGN_ENTITLEMENTS: Transui.entitlements
|
||||||
@@ -32,6 +34,11 @@ targets:
|
|||||||
INFOPLIST_FILE: Info.plist
|
INFOPLIST_FILE: Info.plist
|
||||||
PRODUCT_BUNDLE_IDENTIFIER: io.github.transui.macos
|
PRODUCT_BUNDLE_IDENTIFIER: io.github.transui.macos
|
||||||
PRODUCT_NAME: Transui
|
PRODUCT_NAME: Transui
|
||||||
|
GCC_C_LANGUAGE_STANDARD: c11
|
||||||
|
HEADER_SEARCH_PATHS:
|
||||||
|
- "$(SRCROOT)/../third_party/transui_mp3"
|
||||||
|
- "$(SRCROOT)/../third_party/shine/src/lib"
|
||||||
|
SWIFT_OBJC_BRIDGING_HEADER: Sources/Transui/Transui-Bridging-Header.h
|
||||||
|
|
||||||
TransuiTests:
|
TransuiTests:
|
||||||
type: bundle.unit-test
|
type: bundle.unit-test
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
#include "transui_mp3.h"
|
||||||
|
|
||||||
|
#include <assert.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
|
||||||
|
int main(int argc, char **argv) {
|
||||||
|
const size_t pcm_size = 16000 * 2;
|
||||||
|
uint8_t *pcm = calloc(pcm_size, 1);
|
||||||
|
assert(pcm != NULL);
|
||||||
|
|
||||||
|
uint8_t *mp3 = NULL;
|
||||||
|
size_t mp3_size = 0;
|
||||||
|
assert(transui_mp3_encode_mono_16k_32kbps(pcm, pcm_size, &mp3,
|
||||||
|
&mp3_size) == TRANSUI_MP3_OK);
|
||||||
|
assert(mp3 != NULL);
|
||||||
|
assert(mp3_size > 2);
|
||||||
|
assert(mp3[0] == 0xff);
|
||||||
|
assert((mp3[1] & 0xe0) == 0xe0);
|
||||||
|
if (argc == 2) {
|
||||||
|
FILE *output = fopen(argv[1], "wb");
|
||||||
|
assert(output != NULL);
|
||||||
|
assert(fwrite(mp3, 1, mp3_size, output) == mp3_size);
|
||||||
|
assert(fclose(output) == 0);
|
||||||
|
}
|
||||||
|
transui_mp3_free(mp3);
|
||||||
|
free(pcm);
|
||||||
|
|
||||||
|
assert(transui_mp3_encode_mono_16k_32kbps(NULL, 0, &mp3, &mp3_size) ==
|
||||||
|
TRANSUI_MP3_INVALID_INPUT);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
+1
Submodule third_party/shine added at ab5e3526b6.
+123
@@ -0,0 +1,123 @@
|
|||||||
|
#include "transui_mp3.h"
|
||||||
|
|
||||||
|
#include "layer3.h"
|
||||||
|
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
static int append_bytes(uint8_t **output, size_t *size, size_t *capacity,
|
||||||
|
const unsigned char *bytes, size_t byte_count) {
|
||||||
|
if (byte_count == 0) return TRANSUI_MP3_OK;
|
||||||
|
if (*size > SIZE_MAX - byte_count) return TRANSUI_MP3_OUT_OF_MEMORY;
|
||||||
|
const size_t required = *size + byte_count;
|
||||||
|
if (required > *capacity) {
|
||||||
|
size_t next_capacity = *capacity;
|
||||||
|
while (next_capacity < required) {
|
||||||
|
if (next_capacity > SIZE_MAX / 2) {
|
||||||
|
next_capacity = required;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
next_capacity *= 2;
|
||||||
|
}
|
||||||
|
uint8_t *resized = realloc(*output, next_capacity);
|
||||||
|
if (!resized) return TRANSUI_MP3_OUT_OF_MEMORY;
|
||||||
|
*output = resized;
|
||||||
|
*capacity = next_capacity;
|
||||||
|
}
|
||||||
|
memcpy(*output + *size, bytes, byte_count);
|
||||||
|
*size = required;
|
||||||
|
return TRANSUI_MP3_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
int transui_mp3_encode_mono_16k_32kbps(const uint8_t *pcm_le,
|
||||||
|
size_t pcm_byte_count,
|
||||||
|
uint8_t **output,
|
||||||
|
size_t *output_size) {
|
||||||
|
if (!output || !output_size) return TRANSUI_MP3_INVALID_INPUT;
|
||||||
|
*output = NULL;
|
||||||
|
*output_size = 0;
|
||||||
|
if (!pcm_le || pcm_byte_count < 2 || pcm_byte_count % 2 != 0) {
|
||||||
|
return TRANSUI_MP3_INVALID_INPUT;
|
||||||
|
}
|
||||||
|
if (pcm_byte_count / 8 > SIZE_MAX - 8192) {
|
||||||
|
return TRANSUI_MP3_OUT_OF_MEMORY;
|
||||||
|
}
|
||||||
|
|
||||||
|
shine_config_t config;
|
||||||
|
config.wave.channels = PCM_MONO;
|
||||||
|
config.wave.samplerate = 16000;
|
||||||
|
shine_set_config_mpeg_defaults(&config.mpeg);
|
||||||
|
config.mpeg.mode = MONO;
|
||||||
|
config.mpeg.bitr = 32;
|
||||||
|
|
||||||
|
shine_t encoder = shine_initialise(&config);
|
||||||
|
if (!encoder) return TRANSUI_MP3_ENCODER_ERROR;
|
||||||
|
|
||||||
|
const int samples_per_pass = shine_samples_per_pass(encoder);
|
||||||
|
if (samples_per_pass <= 0) {
|
||||||
|
shine_close(encoder);
|
||||||
|
return TRANSUI_MP3_ENCODER_ERROR;
|
||||||
|
}
|
||||||
|
int16_t *frame = calloc((size_t)samples_per_pass, sizeof(int16_t));
|
||||||
|
size_t capacity = pcm_byte_count / 8 + 8192;
|
||||||
|
uint8_t *encoded = malloc(capacity);
|
||||||
|
if (!frame || !encoded) {
|
||||||
|
free(frame);
|
||||||
|
free(encoded);
|
||||||
|
shine_close(encoder);
|
||||||
|
return TRANSUI_MP3_OUT_OF_MEMORY;
|
||||||
|
}
|
||||||
|
|
||||||
|
const size_t sample_count = pcm_byte_count / 2;
|
||||||
|
size_t sample_offset = 0;
|
||||||
|
size_t encoded_size = 0;
|
||||||
|
int result = TRANSUI_MP3_OK;
|
||||||
|
while (sample_offset < sample_count) {
|
||||||
|
const size_t remaining = sample_count - sample_offset;
|
||||||
|
const size_t frame_samples =
|
||||||
|
remaining < (size_t)samples_per_pass ? remaining
|
||||||
|
: (size_t)samples_per_pass;
|
||||||
|
memset(frame, 0, (size_t)samples_per_pass * sizeof(int16_t));
|
||||||
|
for (size_t index = 0; index < frame_samples; ++index) {
|
||||||
|
const size_t byte_offset = (sample_offset + index) * 2;
|
||||||
|
frame[index] = (int16_t)((uint16_t)pcm_le[byte_offset] |
|
||||||
|
((uint16_t)pcm_le[byte_offset + 1] << 8));
|
||||||
|
}
|
||||||
|
|
||||||
|
int written = 0;
|
||||||
|
unsigned char *bytes =
|
||||||
|
shine_encode_buffer_interleaved(encoder, frame, &written);
|
||||||
|
if (written < 0 || (!bytes && written > 0)) {
|
||||||
|
result = TRANSUI_MP3_ENCODER_ERROR;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
result = append_bytes(&encoded, &encoded_size, &capacity, bytes,
|
||||||
|
(size_t)written);
|
||||||
|
if (result != TRANSUI_MP3_OK) break;
|
||||||
|
sample_offset += frame_samples;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result == TRANSUI_MP3_OK) {
|
||||||
|
int written = 0;
|
||||||
|
unsigned char *bytes = shine_flush(encoder, &written);
|
||||||
|
if (written < 0 || (!bytes && written > 0)) {
|
||||||
|
result = TRANSUI_MP3_ENCODER_ERROR;
|
||||||
|
} else {
|
||||||
|
result = append_bytes(&encoded, &encoded_size, &capacity, bytes,
|
||||||
|
(size_t)written);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
free(frame);
|
||||||
|
shine_close(encoder);
|
||||||
|
if (result != TRANSUI_MP3_OK || encoded_size == 0) {
|
||||||
|
free(encoded);
|
||||||
|
return result == TRANSUI_MP3_OK ? TRANSUI_MP3_ENCODER_ERROR : result;
|
||||||
|
}
|
||||||
|
|
||||||
|
*output = encoded;
|
||||||
|
*output_size = encoded_size;
|
||||||
|
return TRANSUI_MP3_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
void transui_mp3_free(void *memory) { free(memory); }
|
||||||
+29
@@ -0,0 +1,29 @@
|
|||||||
|
#ifndef TRANSUI_MP3_H
|
||||||
|
#define TRANSUI_MP3_H
|
||||||
|
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
#ifdef __cplusplus
|
||||||
|
extern "C" {
|
||||||
|
#endif
|
||||||
|
|
||||||
|
enum {
|
||||||
|
TRANSUI_MP3_OK = 0,
|
||||||
|
TRANSUI_MP3_INVALID_INPUT = 1,
|
||||||
|
TRANSUI_MP3_ENCODER_ERROR = 2,
|
||||||
|
TRANSUI_MP3_OUT_OF_MEMORY = 3,
|
||||||
|
};
|
||||||
|
|
||||||
|
int transui_mp3_encode_mono_16k_32kbps(const uint8_t *pcm_le,
|
||||||
|
size_t pcm_byte_count,
|
||||||
|
uint8_t **output,
|
||||||
|
size_t *output_size);
|
||||||
|
|
||||||
|
void transui_mp3_free(void *memory);
|
||||||
|
|
||||||
|
#ifdef __cplusplus
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#endif
|
||||||
Reference in new issue
Block a user