t3-code-android-nightly/patches/@react-native-ai__apple@0.12.0.patch
Theo Browne 352710d497
feat(mobile): add offline iPhone voice input (#8614)
Co-authored-by: Julius Marminge <julius@mac.lan>
2026-08-30 18:08:36 -07:00

194 lines
7.6 KiB
Diff

diff --git a/ios/transcription/AppleTranscriptionImpl.swift b/ios/transcription/AppleTranscriptionImpl.swift
index 188371a5f55fadae19187108c4a688569d0223e3..3e1cdd77b0bb8ae6c97a40d29c1854200d931eee 100644
--- a/ios/transcription/AppleTranscriptionImpl.swift
+++ b/ios/transcription/AppleTranscriptionImpl.swift
@@ -12,6 +12,11 @@ import UniformTypeIdentifiers
@objc
public class AppleTranscriptionImpl: NSObject {
+ private struct CollectedSegment: Sendable {
+ let text: String
+ let startSecond: Double
+ let endSecond: Double
+ }
@available(iOS 26, *)
private func createTranscriber(for locale: Locale) -> SpeechTranscriber {
@@ -41,7 +46,7 @@ public class AppleTranscriptionImpl: NSObject {
let locale = Locale(identifier: language)
guard let supportedLocale = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else {
- reject("AppleTranscription", "Locale not supported: \(language)", nil)
+ reject("AppleTranscriptionUnsupportedLocale", "Locale not supported: \(language)", nil)
return
}
@@ -51,16 +56,14 @@ public class AppleTranscriptionImpl: NSObject {
switch status {
case .installed:
- resolve(nil)
+ resolve(supportedLocale.identifier)
case .supported, .downloading:
- if let request = try? await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
+ if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
try await request.downloadAndInstall()
- resolve(nil)
- } else {
- resolve(nil)
}
+ resolve(supportedLocale.identifier)
case .unsupported:
- reject("AppleTranscription", "Assets not supported for locale: \(supportedLocale.identifier)", nil)
+ reject("AppleTranscriptionUnsupportedLocale", "Assets not supported for locale: \(supportedLocale.identifier)", nil)
@unknown default:
reject ("AppleTranscription", "Unknown asset inventory status", nil)
}
@@ -83,57 +86,77 @@ public class AppleTranscriptionImpl: NSObject {
do {
try audioData.write(to: fileURL)
- guard let audioFile = try? AVAudioFile(forReading: fileURL) else {
- reject("AppleTranscription", "Invalid audio data", nil)
+ let audioFile: AVAudioFile
+ do {
+ audioFile = try AVAudioFile(forReading: fileURL)
+ } catch {
+ try? FileManager.default.removeItem(at: fileURL)
+ reject("AppleTranscription", "Invalid audio data", error)
return
}
Task {
+ defer {
+ try? FileManager.default.removeItem(at: fileURL)
+ }
+
do {
let transcriber = createTranscriber(for: Locale(identifier: language))
let analyzer = SpeechAnalyzer(modules: [transcriber])
-
- defer {
- try? FileManager.default.removeItem(at: fileURL)
- }
-
- var segments: [[String: Any]] = []
-
- Task {
+
+ let collectorTask = Task { () throws -> [CollectedSegment] in
+ var segments: [CollectedSegment] = []
+
for try await result in transcriber.results {
if result.isFinal {
- let segment: [String: Any] = [
- "text": String(result.text.characters),
- "startSecond": CMTimeGetSeconds(result.range.start),
- "endSecond": CMTimeGetSeconds(CMTimeRangeGetEnd(result.range))
- ]
- segments.append(segment)
+ segments.append(
+ CollectedSegment(
+ text: String(result.text.characters),
+ startSecond: CMTimeGetSeconds(result.range.start),
+ endSecond: CMTimeGetSeconds(CMTimeRangeGetEnd(result.range))
+ )
+ )
}
}
+
+ return segments
}
-
- let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile)
-
- if let lastSampleTime {
- try await analyzer.finalizeAndFinish(through: lastSampleTime)
- } else {
+
+ do {
+ let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile)
+
+ if let lastSampleTime {
+ try await analyzer.finalizeAndFinish(through: lastSampleTime)
+ } else {
+ await analyzer.cancelAndFinishNow()
+ }
+
+ let segments: [[String: Any]] = try await collectorTask.value.map { segment in
+ [
+ "text": segment.text,
+ "startSecond": segment.startSecond,
+ "endSecond": segment.endSecond
+ ]
+ }
+ let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 }
+
+ resolve([
+ "segments": segments,
+ "duration": totalDuration
+ ])
+ } catch {
+ collectorTask.cancel()
await analyzer.cancelAndFinishNow()
+ _ = try? await collectorTask.value
+ throw error
}
-
- let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 }
-
- let result: [String: Any] = [
- "segments": segments,
- "duration": totalDuration
- ]
-
- resolve(result)
} catch {
reject("AppleTranscription", "Transcription failed: \(error.localizedDescription)", error)
}
}
} catch {
+ try? FileManager.default.removeItem(at: fileURL)
reject("AppleTranscription", "Failed to write audio data: \(error.localizedDescription)", error)
}
} else {
@@ -141,4 +164,3 @@ public class AppleTranscriptionImpl: NSObject {
}
}
}
-
diff --git a/lib/typescript/NativeAppleTranscription.d.ts b/lib/typescript/NativeAppleTranscription.d.ts
index 985b6b3593a884c41d689346be9d73d86c55eae1..86e7936f23a8673d099f7fe18af9a31abae5a13a 100644
--- a/lib/typescript/NativeAppleTranscription.d.ts
+++ b/lib/typescript/NativeAppleTranscription.d.ts
@@ -10,14 +10,14 @@ export interface TranscriptionResult {
}
export interface Spec extends TurboModule {
isAvailable(language: string): boolean;
- prepare(language: string): Promise<void>;
+ prepare(language: string): Promise<string>;
}
declare global {
function __apple__llm__transcribe__(data: ArrayBufferLike, language: string): Promise<TranscriptionResult>;
}
declare const _default: {
transcribe: (data: ArrayBufferLike, language: string) => Promise<TranscriptionResult>;
- prepare: (language: string) => Promise<void>;
+ prepare: (language: string) => Promise<string>;
isAvailable: (language: string) => boolean;
};
export default _default;
diff --git a/src/NativeAppleTranscription.ts b/src/NativeAppleTranscription.ts
index 13332a0176000b60b6f6f043de681db55a7b4bef..5389fb4b3a53de48721c307d2145b6770624c852 100644
--- a/src/NativeAppleTranscription.ts
+++ b/src/NativeAppleTranscription.ts
@@ -14,7 +14,7 @@ export interface TranscriptionResult {
export interface Spec extends TurboModule {
isAvailable(language: string): boolean
- prepare(language: string): Promise<void>
+ prepare(language: string): Promise<string>
}
declare global {