[Flutter] How can i stream incoming pcm16 data in flutter/ iOS?

This is my flutter code for the incoming data, what I get is a uint8list data stream that kinda looks like this (if transcribed)

Hello Hello how Hello how are Hello how are you?

client.on(RealtimeEventType.conversationUpdated, (e) async {
final event = e as RealtimeEventConversationUpdated;
final item = event.result.item;

if (item?.formatted?.audio != null && item!.formatted!.audio.isNotEmpty) {
try {
await _trickleAudioStream?.playAudioStream(item.formatted!.audio, isFirstChunk);
if (isFirstChunk) {
isFirstChunk = false;
}
} catch (e) {
print("EXCEPTION: $e");
}
}
});


and this is my native code (for the plugin I'm writing)

import Flutter
import AVFoundation
import Foundation

struct CircularBuffer<Element> {
private var array: [Element?]
private var readIndex = 0
private var writeIndex = 0
private(set) var count = 0
private let capacity: Int


init(capacity: Int) {
self.capacity = capacity
self.array = Array(repeating: nil, count: capacity)

}

mutating func write(_ element: Element) {
array[writeIndex % capacity] = element
writeIndex += 1
if count < capacity {
count += 1
} else {
readIndex += 1 // Overwrite oldest if buffer is full
}
}

mutating func read() -> Element? {
guard count > 0 else { return nil }
let element = array[readIndex % capacity]
array[readIndex % capacity] = nil
readIndex += 1
count -= 1
return element
}
}



public class TrickleAudioStreamPlugin: NSObject, FlutterPlugin {
...

public static func register(with registrar: FlutterPluginRegistrar) {
let channel = FlutterMethodChannel(name: "trickle_audio_stream", binaryMessenger: registrar.messenger())
let instance = TrickleAudioStreamPlugin()
registrar.addMethodCallDelegate(instance, channel: channel)
}

public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) {
if call.method == "playAudioStream" {
guard let args = call.arguments as? [String: Any],
let pcmData = args["data"] as? FlutterStandardTypedData,
let isFirst = args["isFirst"] as? Bool else {
result(FlutterError(code: "INVALID_ARGUMENTS", message: "Invalid arguments", details: nil))
return
}

let data = pcmData.data

let timestamp = playAudioStream(convertPCM16ToFloat32(data), isFirst: isFirst)
result(timestamp)
} else if call.method == "stopAudioStream" {
stopAudioStream()
} else {
result(FlutterMethodNotImplemented)
}
}

func stopAudioStream() {
playerNode.stop()
isPlaying = false
}

func playAudioStream(_ floatPCMData: Data, isFirst: Bool) -> Double {
if isFirst {
resetBuffers()
stopAudioStream()
processedDataLength = 0

do {
try startAudioEngine()
playerNode.play()
isPlaying = true
} catch {
print("Error starting audio engine: \(error)")
return timeNow()
}
}

guard floatPCMData.count > processedDataLength else {
return timeNow()
}

let newData = floatPCMData.subdata(in: processedDataLength..<floatPCMData.count)
processedDataLength = floatPCMData.count

guard let buffer = createPCMBuffer(from: newData) else {
print("Failed to create PCM buffer")
return timeNow()
}

buffers.append(buffer)
currentSegmentDuration += Double(buffer.frameLength) / format.sampleRate

playerNode.scheduleBuffer(buffer) {
self.currentPlaybackPosition += Double(buffer.frameLength) / self.format.sampleRate
if let nextBuffer = self.buffers.first {
self.buffers.removeFirst()
self.playerNode.scheduleBuffer(nextBuffer) {
// Schedule next on completion
}
}
}

if isFirst {
playerNode.play()
isPlaying = true
}

return timeNow()
}



func startAudioEngine() throws {
if !audioEngine.isRunning {
try audioEngine.start()
}
}

private func createPCMBuffer(from floatPCMData: Data) -> AVAudioPCMBuffer? {
let frameCount = floatPCMData.count / MemoryLayout<Float>.size
guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frameCount)) else {
return nil
}
buffer.frameLength = buffer.frameCapacity

floatPCMData.withUnsafeBytes { floatBytes in
let floatBufferPointer = floatBytes.bindMemory(to: Float.self).baseAddress!
buffer.floatChannelData?.pointee.update(from: floatBufferPointer, count: frameCount)
}
return buffer
}

func resetBuffers() {
buffers.removeAll()
currentSegmentDuration = 0
currentPlaybackPosition = 0
playerNode.reset()
}

private func timeNow() -> Double {
return Date().timeIntervalSince1970 * 1000
}


private func convertPCM16ToFloat32(_ pcm16Data: Data) -> Data {
var floatData = Data(capacity: pcm16Data.count * MemoryLayout<Float>.size / MemoryLayout<Int16>.size)
pcm16Data.withUnsafeBytes { (int16Pointer: UnsafeRawBufferPointer) in
let int16Buffer = int16Pointer.bindMemory(to: Int16.self)
for sample in int16Buffer {
let floatSample = Float(sample) / Float(Int16.max)
floatData.append(contentsOf: withUnsafeBytes(of: floatSample) { Data($0) })
}
}
return floatData
}
}



But sometimes it starts playing older audio again or small parts of it.

I expect the data to be streamed seamlessly and play the audio without repeating previous chunks of data.

Continue reading...