I am trying to build some kind of VoIP application in iOS. So far I have been able to successfully send the microphone data as a buffer from microphone to the server using GCDAsyncSocket. Now I need to play back the data I receive, which I am really confused about. I have looked online, but all I see is either playing back the audio file from remote or audio stream from a URL. I am actually receiving NSData regularly and need to figure out how to use those NSData to fill the audio units buffer list. I am new to C and finding it hard to get through it. This is where I get NSData from server.
- (void)socket:(GCDAsyncSocket *)sender didReadData:(NSData *)data withTag:(long)tag
{
if (tag == 1 ){
//this is where I read password and stuff to authenticate
}
else{
[self setUpAQOutput:data];//this should somehow initialize AU and fill the buffer
}
and in my AudioUnitProcessor, this is how I set up AUnit using Stefan Popp's codes :
//
// AudioProcessor.m
// MicInput
//
// Created by Stefan Popp on 21.09.11.
//
#import "AudioProcessor.h"
#import "PTTClient.h"
#pragma mark Recording callback
static OSStatus recordingCallback(void *inRefCon,
AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *inTimeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList *ioData) {
// the data gets rendered here
AudioBuffer buffer;
// a variable where we check the status
OSStatus status;
/**
This is the reference to the object who owns the callback.
*/
AudioProcessor *audioProcessor = (AudioProcessor*) inRefCon;
/**
on this point we define the number of channels, which is mono
for the iphone. the number of frames is usally 512 or 1024.
*/
buffer.mDataByteSize = inNumberFrames * 2; // sample size
buffer.mNumberChannels = 1; // one channel
buffer.mData = malloc( inNumberFrames * 2 ); // buffer size
// we put our buffer into a bufferlist array for rendering
AudioBufferList bufferList;
bufferList.mNumberBuffers = 1;
bufferList.mBuffers[0] = buffer;
// render input and check for error
status = AudioUnitRender([audioProcessor audioUnit], ioActionFlags, inTimeStamp, inBusNumber, inNumberFrames, &bufferList);
// process the bufferlist in the audio processor
[audioProcessor processBuffer:&bufferList];
// clean up the buffer
free(bufferList.mBuffers[0].mData);
return noErr;
}
#pragma mark Playback callback
static OSStatus playbackCallback(void *inRefCon,
AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *inTimeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList *ioData) {
//does nothing
return noErr;
}
#pragma mark objective-c class
@implementation AudioProcessor
@synthesize audioUnit, inAudioBuffer;
-(AudioProcessor*)init
{
self = [super init];
if (self) {
[self initializeAudio];
}
return self;
}
+ (OSStatus) playBytes:(NSArray*) byteArray {
/**
This is the reference to the object who owns the callback.
*/
// NSArray * byteArray = nil;
AudioProcessor *audioProcessor = [[AudioProcessor alloc] init];
// iterate over incoming stream an copy to output stream
for (int i=0; i < [byteArray count]; i++) {
// AudioBuffer buffer = ioData->mBuffers[i];
// find minimum size
UInt32 size = [audioProcessor inAudioBuffer].mDataByteSize;
// copy buffer to audio buffer which gets played after function return
memcpy(byteArray[i], [audioProcessor inAudioBuffer].mData, size);
// set data size
//buffer.mDataByteSize = size;
}
return noErr;
}
-(void)initializeAudio
{
OSStatus status;
// We define the audio component
AudioComponentDescription desc;
desc.componentType = kAudioUnitType_Output; // we want to ouput
desc.componentSubType = kAudioUnitSubType_RemoteIO; // we want in and ouput
desc.componentFlags = 0; // must be zero
desc.componentFlagsMask = 0; // must be zero
desc.componentManufacturer = kAudioUnitManufacturer_Apple; // select provider
// find the AU component by description
AudioComponent inputComponent = AudioComponentFindNext(NULL, &desc);
// create audio unit by component
status = AudioComponentInstanceNew(inputComponent, &audioUnit);
// define that we want record io on the input bus
UInt32 flag = 1;
status = AudioUnitSetProperty(audioUnit,
kAudioOutputUnitProperty_EnableIO, // use io
kAudioUnitScope_Input, // scope to input
kInputBus, // select input bus (1)
&flag, // set flag
sizeof(flag));
// define that we want play on io on the output bus
UInt32 stopFlag = 0;//stop flag 0 because we dont want to play audio back in device
status = AudioUnitSetProperty(audioUnit,
kAudioOutputUnitProperty_EnableIO, // use io
kAudioUnitScope_Output, // scope to output
kOutputBus, // select output bus (0)
&stopFlag, // set flag
sizeof(stopFlag));
/*
We need to specify our format on which we want to work.
We use Linear PCM cause its uncompressed and we work on raw data.
for more informations check.
We want 16 bits, 2 bytes per packet/frames at 44khz
*/
AudioStreamBasicDescription audioFormat;
audioFormat.mSampleRate = SAMPLE_RATE;
audioFormat.mFormatID = kAudioFormatLinearPCM;
audioFormat.mFormatFlags = kAudioFormatFlagIsPacked | kAudioFormatFlagIsSignedInteger;
audioFormat.mFramesPerPacket = 1;
audioFormat.mChannelsPerFrame = 1;
audioFormat.mBitsPerChannel = 16;
audioFormat.mBytesPerPacket = audioFormat.mChannelsPerFrame * sizeof( SInt16);
audioFormat.mBytesPerFrame = audioFormat.mChannelsPerFrame * sizeof( SInt16);
// set the format on the output stream
status = AudioUnitSetProperty(audioUnit,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Output,
kInputBus,
&audioFormat,
sizeof(audioFormat));
// set the format on the input stream
status = AudioUnitSetProperty(audioUnit,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Input,
kOutputBus,
&audioFormat,
sizeof(audioFormat));
/**
We need to define a callback structure which holds
a pointer to the recordingCallback and a reference to
the audio processor object
*/
AURenderCallbackStruct callbackStruct;
// set recording callback
callbackStruct.inputProc = recordingCallback; // recordingCallback pointer
callbackStruct.inputProcRefCon = self;
// set input callback to recording callback on the input bus
status = AudioUnitSetProperty(audioUnit,
kAudioOutputUnitProperty_SetInputCallback,
kAudioUnitScope_Global,
kInputBus,
&callbackStruct,
sizeof(callbackStruct));
/*
We do the same on the output stream to hear what is coming
from the input stream
*/
callbackStruct.inputProc = playbackCallback;
callbackStruct.inputProcRefCon = self;
// set playbackCallback as callback on our renderer for the output bus
status = AudioUnitSetProperty(audioUnit,
kAudioUnitProperty_SetRenderCallback,
kAudioUnitScope_Global,
kOutputBus,
&callbackStruct,
sizeof(callbackStruct));
// reset flag to 0
flag = 0;
/*
we need to tell the audio unit to allocate the render buffer,
that we can directly write into it.
*/
status = AudioUnitSetProperty(audioUnit,
kAudioUnitProperty_ShouldAllocateBuffer,
kAudioUnitScope_Output,
kInputBus,
&flag,
sizeof(flag));
/*
we set the number of channels to mono and allocate our block size to
1024 bytes.
*/
inAudioBuffer.mNumberChannels = 1;
inAudioBuffer.mDataByteSize = 512 * 2;
inAudioBuffer.mData = malloc( 512 * 2 );
// Initialize the Audio Unit and cross fingers =)
status = AudioUnitInitialize(audioUnit);
NSLog(@"Started");
}
#pragma mark controll stream
-(void)start;
{
// start the audio unit. You should hear something, hopefully :)
OSStatus status = AudioOutputUnitStart(audioUnit);
}
-(void)stop;
{
// stop the audio unit
OSStatus status = AudioOutputUnitStop(audioUnit);
}
#pragma mark processing
-(void)processBuffer: (AudioBufferList*) audioBufferList
{
AudioBuffer sourceBuffer = audioBufferList->mBuffers[0];
// we check here if the input data byte size has changed
if (inAudioBuffer.mDataByteSize != sourceBuffer.mDataByteSize) {
// clear old buffer
free(inAudioBuffer.mData);
// assing new byte size and allocate them on mData
inAudioBuffer.mDataByteSize = sourceBuffer.mDataByteSize;
inAudioBuffer.mData = malloc(sourceBuffer.mDataByteSize);
}
int currentBuffer =0;
int maxBuf = 800;
NSMutableData *data=[[NSMutableData alloc] init];
// CMBlockBufferRef blockBuffer;
// CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(ref, NULL, &audioBufferList, sizeof(audioBufferList), NULL, NULL, 0, &blockBuffer);
// NSLog(@"%@",blockBuffer);
// audioBufferList->mBuffers[0].mData, audioBufferList->mBuffers[0].mDataByteSize
for( int y=0; y<audioBufferList->mNumberBuffers; y++ )
{
if (currentBuffer < maxBuf){
AudioBuffer audioBuff = audioBufferList->mBuffers[y];
Float32 *frame = (Float32*)audioBuff.mData;
[data appendBytes:frame length:inAudioBuffer.mDataByteSize];
currentBuffer += audioBuff.mDataByteSize;
}
else{
break;
}
}
[[PTTClient getDefaultInstance] setAudioBufferData: data];//This is call to send buffer data to the server
// copy incoming audio data to the audio buffer (no need since we are not using playback)
//memcpy(inAudioBuffer.mData, audioBufferList->mBuffers[0].mData, audioBufferList->mBuffers[0].mDataByteSize);
}
@end
And finally this is the method to send audio data to server
-(void) setAudioBufferData: (NSData*) data{
[gcdSocket writeData:data withTimeout:timeout tag:tag];
}
All of this work fine, and I can listen to the sound in my server that runs in Java. Now I need to figure out how to adjust this audio unit to play the NSData packets that I continuously receiving from the server (I have looked at some examples that play back remote file, which is not what I need. I need to play voice). The source is not the file but some one talking, so I am kind of confused.
1)Well presumably the audio data could become large so I would file-buffer it. Not sure the more elegant way but hey...the brute force methods in your code running at C++ level logic deter me a bit...
Is using
...helpful at all? Perhaps then use that file as the audio source passed to some more convenient Core framework?
2)The only other thing coming to mind is some form of local sockets. i.e. open a connection to yourself and supply that as a "remote source"
Sorry wish I knew more to be more helpful.