Connection of varispeed with RemoteIO in iOS

问题

I am working with audio units to play and change the speed of playback. Since AudioGraph is deprecated.

What I have done, I have successfully played Audio coming from UDP via audio-units and made connections like:

converterUnit -> varispeed -> outConverterUnit -> RemoteIO (Out)

Our format for playing is int16(PCM), but varispeed requires float datatype, So we are using converters for varispeed.

Here is my code:

var ioFormat = CAStreamBasicDescription(
    sampleRate: 48000.0,
    numChannels: 1,
    pcmf: .int16,
    isInterleaved: false
    )

var varispeedFormat = CAStreamBasicDescription(
    sampleRate: 16000,
    numChannels: 1,
    pcmf: .float32,
    isInterleaved: false
)


init(_ client: UDPClient, _ tcpClient: TCPClient, _ opusHelper: OpusHelper, _ tvTemp: UILabel) {
    super.init()

let success = initCircularBuffer(&circularBuffer, 4096)
if success {
    print("Circular buffer init was successful")
} else {
    print("Circular buffer init not successful")
}

self.opusHelper = opusHelper

self.tvTemp = tvTemp
monotonicTimer = MonotonicTimer()


self.udpClient = client
self.tcpClient = tcpClient

//Creating Description for REMOTE IO
var outputDesc = AudioComponentDescription(
    componentType: OSType(kAudioUnitType_Output),
    componentSubType: OSType(kAudioUnitSubType_VoiceProcessingIO),
    componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
    componentFlags: 0,
    componentFlagsMask: 0
)

let inputComponent = AudioComponentFindNext(nil, &outputDesc)

check(error: AudioComponentInstanceNew(inputComponent!, &outputUnit), description: "Output unit instance new failed")

//Creating Description for converterUnit
var firstConverterDesc = AudioComponentDescription(
    componentType: OSType(kAudioUnitType_FormatConverter),
    componentSubType: OSType(kAudioUnitSubType_AUConverter),
    componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
    componentFlags: 0,
    componentFlagsMask: 0
)

let firstConverterComponent = AudioComponentFindNext(nil, &firstConverterDesc)
check(error: AudioComponentInstanceNew(firstConverterComponent!, &firstConverterUnit), description: "First converter unit instance new failed")

//Creating Description for Varispeed Unit
var variSpeedConverterDesc = AudioComponentDescription(
    componentType: OSType(kAudioUnitType_FormatConverter),
    componentSubType: OSType(kAudioUnitSubType_Varispeed),
    componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
    componentFlags: 0,
    componentFlagsMask: 0
)

let variSpeedConverterComponent = AudioComponentFindNext(nil, &variSpeedConverterDesc)
check(error: AudioComponentInstanceNew(variSpeedConverterComponent!, &varispeedUnit), description: "First converter unit instance new failed")


//Creating Description for outConverter
var secondConverterDesc = AudioComponentDescription(
    componentType: OSType(kAudioUnitType_FormatConverter),
    componentSubType: OSType(kAudioUnitSubType_AUConverter),
    componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
    componentFlags: 0,
    componentFlagsMask: 0
)

let secondConverterComponent = AudioComponentFindNext(nil, &secondConverterDesc)
check(error: AudioComponentInstanceNew(secondConverterComponent!, &secondConverterUnit), description: "Second converter unit instance new failed")


//Converting incoming bytes to AUConverter (Float 32 format)
check(error: AudioUnitSetProperty(
    firstConverterUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Output),
    0,
    &varispeedFormat,
    MemoryLayoutStride.SizeOf32(varispeedFormat)
    ),
      description: "Failed to set input of second converter to our temp format"
)

//Putting converted bytes in varispeed unit
check(error: AudioUnitSetProperty(
    varispeedUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &varispeedFormat,
    MemoryLayoutStride.SizeOf32(varispeedFormat)
    ),
      description: "Failed to set input format as varispeed format of the second converter unit"
)

//Getting converted bytes from varispeed unit
check(error: AudioUnitSetProperty(
    varispeedUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Output),
    0,
    &varispeedFormat,
    MemoryLayoutStride.SizeOf32(varispeedFormat)
    ),
      description: "Failed to set input format as varispeed format of the second converter unit"
)

//Putting converted bytes in outConverterUnit
check(error: AudioUnitSetProperty(
    secondConverterUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &varispeedFormat,
    MemoryLayoutStride.SizeOf32(varispeedFormat)
    ),
      description: "Failed to set input of second converter to our temp11 format"
)

//Getting converted bytes from outConverterUnit in int16(PCM)
check(error: AudioUnitSetProperty(
    secondConverterUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Output),
    0,
    &ioFormat,
    MemoryLayoutStride.SizeOf32(ioFormat)
    ),
      description: "Failed to set input of second converter to our temp11 format"
)


//Connecting firstConverter to varispeed
var tempConnection = AudioUnitConnection(
    sourceAudioUnit: firstConverterUnit!,
    sourceOutputNumber: 0,
    destInputNumber: 0
)

check(error: AudioUnitSetProperty(
    varispeedUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_MakeConnection),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &tempConnection,
    MemoryLayoutStride.SizeOf32(AudioUnitConnection())
    ),
      description: "Failed to connect second converter to output Unit"
)

//Connecting verispeedUnit to to outConverter
var temp1Connection = AudioUnitConnection(
    sourceAudioUnit: varispeedUnit!,
    sourceOutputNumber: 0,
    destInputNumber: 0
)

check(error: AudioUnitSetProperty(
    secondConverterUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_MakeConnection),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &temp1Connection,
    MemoryLayoutStride.SizeOf32(AudioUnitConnection())
    ),
      description: "Failed to connect second converter to output Unit"
)

//Connecting outConverter to outputUnit
var secondToOutputConnection = AudioUnitConnection(
    sourceAudioUnit: secondConverterUnit!,
    sourceOutputNumber: 0,
    destInputNumber: 0
)

check(error: AudioUnitSetProperty(
    outputUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_MakeConnection),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &secondToOutputConnection,
    MemoryLayoutStride.SizeOf32(AudioUnitConnection())
    ),
      description: "Failed to connect second converter to output Unit"
)

check(error: AudioUnitInitialize(outputUnit!), description: "Failed to init output unit")
check(error: AudioUnitInitialize(firstConverterUnit!), description: "Failed to init first converter unit")
check(error: AudioUnitInitialize(varispeedUnit!), description: "Failed to init varispeed unit")
check(error: AudioUnitInitialize(secondConverterUnit!), description: "Failed to init second converter unit")


var playbackCallback = AURenderCallbackStruct(
    inputProc: AudioController_PlaybackCallback,
    inputProcRefCon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
)

check(error: AudioUnitSetProperty(
    outputUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_SetRenderCallback),
    AudioUnitScope(kAudioUnitScope_Input),
    kOutputBus,
    &playbackCallback,
    MemoryLayout<AURenderCallbackStruct>.size.ui
    ),
      description: "Failed to set recording render callback"
)
}

The parameter for changing the playback speed is given here.

func performPlayback(
            _ ioActionFlags: UnsafeMutablePointer<AudioUnitRenderActionFlags>,
            inTimeStamp: UnsafePointer<AudioTimeStamp>,
            inBufNumber: UInt32,
            inNumberFrames: UInt32,
            ioData: UnsafeMutablePointer<AudioBufferList>
        ) -> OSStatus {

let buffer = ioData[0].mBuffers

let bytesToCopy = ioData[0].mBuffers.mDataByteSize
var bufferTail: UnsafeMutableRawPointer?
//        print("BYTES TO COPY: \(bytesToCopy)")
self.availableBytes = 0
bufferTail = TPCircularBufferTail(&self.circularBuffer, &self.availableBytes)
bytesToWrite = min(bytesToCopy, self.availableBytes)

check(error: AudioUnitSetParameter(
       varispeedUnit!,
       AudioUnitParameterID(kVarispeedParam_PlaybackRate),
       AudioUnitScope(kAudioUnitScope_Global),
       0,
       AudioUnitParameterValue(1.9999000082426955),
       0
    ),
      description: "Failed to set parameter rate for varispeed unit"
)


print("BYTES TO WRITE: \(bytesToWrite)")
if bytesToWrite >= 3840 {
    memcpy(buffer.mData, bufferTail, Int(bytesToWrite))
    TPCircularBufferConsume(&self.circularBuffer, bytesToWrite)
} else {
    let silence = [Int16](repeating: 0, count: Int(bytesToCopy))
    memcpy(buffer.mData, silence, Int(bytesToCopy))
}
return noErr
}

The problem is, I don't feel any difference in voice if I use varispeed or not. Can anyone point out the problem in my code?