Given below is a React Hook File which implements some Audio Processing of the audio tracks of Local and Remote Participants in a Meeting Application using Twilio Video Services. Till now I have been able to obtain results in the CustomAudioProcessor with a single input, but it doesn't seem to be working with a dual input system. Is there anything I am doing wrong here.
// client\src\Hooks\useAudioProcessing.ts - React Hook File
import { LocalAudioTrack, RemoteAudioTrack } from "twilio-video";
import useTwilioVideoContext from "./useTwilioVideoContext"
const useAudioProcessing = () => {
const { state } = useTwilioVideoContext();
const audioContext = new AudioContext();
const singleInputAudioProcessing = async (audioTrack: LocalAudioTrack | RemoteAudioTrack) => {
// Load the AudioWorkletProcessor
await audioContext.audioWorklet.addModule('worklets/customAudioProcessor.js');
const customAudioProcessor = new AudioWorkletNode(audioContext, 'custom-audio-processor');
// Create MediaStreamAudioSourceNode
const mediaStream = new MediaStream();
mediaStream.addTrack(audioTrack.mediaStreamTrack);
const audioSource = audioContext.createMediaStreamSource(mediaStream);
// Connect Nodes
audioSource.connect(customAudioProcessor);
customAudioProcessor.connect(audioContext.destination);
if (audioContext.state === 'suspended') {
await audioContext.resume();
}
}
const dualInputAudioProcessing_version1 = async (localAudioTrack: LocalAudioTrack, remoteAudioTrack: RemoteAudioTrack) => {
// Load the AudioWorkletProcessor
await audioContext.audioWorklet.addModule('worklets/customAudioProcessor.js');
const customAudioProcessor = new AudioWorkletNode(audioContext, 'custom-audio-processor');
// Create MediaStreamAudioSourceNodes
const mediaStream = new MediaStream();
mediaStream.addTrack(localAudioTrack.mediaStreamTrack);
mediaStream.addTrack(remoteAudioTrack.mediaStreamTrack);
const audioSource = audioContext.createMediaStreamSource(mediaStream);
// Connect Nodes
audioSource.connect(customAudioProcessor);
customAudioProcessor.connect(audioContext.destination);
if (audioContext.state === 'suspended') {
await audioContext.resume();
}
}
async function dualInputAudioProcessing_version2(localAudioTrack: LocalAudioTrack, remoteAudioTrack: RemoteAudioTrack) {
// Load the AudioWorkletProcessor
await audioContext.audioWorklet.addModule('worklets/customAudioProcessor.js');
const customAudioProcessorInputNode = new AudioWorkletNode(audioContext, 'custom-audio-processor');
// Create local MediaStreamAudioSourceNode - Has one output node
const localMediaStream = new MediaStream();
localMediaStream.addTrack(localAudioTrack.mediaStreamTrack);
const localSource = audioContext.createMediaStreamSource(localMediaStream);
// Create remote MediaStreamAudioSourceNode - Has one output node
const remoteMediaStream = new MediaStream();
remoteMediaStream.addTrack(remoteAudioTrack.mediaStreamTrack);
const remoteSource = audioContext.createMediaStreamSource(remoteMediaStream);
// Handle varying sample rates
if (localSource.context.sampleRate !== remoteSource.context.sampleRate) {
// Resampling logic here, if needed
console.log(localSource.context.sampleRate, remoteSource.context.sampleRate);
} else {
console.log(localSource.context.sampleRate, remoteSource.context.sampleRate);
}
// Create a ChannelMergerNode to merge the local and remote tracks
const merger = audioContext.createChannelMerger(2);
// Connect nodes
localSource.connect(merger, 0, 0); // Connect local source node output0 to merger node input0
remoteSource.connect(merger, 0, 1); // Connect remote source node output0 to merger node input1
merger.connect(customAudioProcessorInputNode); // Connect merger node to merger node to customAudioProcessorInputNode
customAudioProcessorInputNode.connect(audioContext.destination); // Connect to destination (speakers)
}
async function startAudioProcessing() {
try {
if (!state.room) throw new Error("Can not start audio processing without joining room");
// Use flatMap to eliminate the need for Array.from().map() chaining
const localAudioTracks = Array.from(state.room.localParticipant.audioTracks.values()).flatMap(trackPub => trackPub.track ? [trackPub.track] : []);
const firstLocalAudioTrack = localAudioTracks[0] || null;
const firstRemoteParticipant = Array.from(state.room.participants.values())[0];
const remoteAudioTracks = firstRemoteParticipant ? Array.from(firstRemoteParticipant.audioTracks.values()).flatMap(trackPub => trackPub.track ? [trackPub.track] : []) : [];
const firstRemoteAudioTrack = remoteAudioTracks[0] || null;
if (firstLocalAudioTrack) {
// Test 0
singleInputAudioProcessing(firstLocalAudioTrack);
if (firstRemoteAudioTrack) {
// Test 1
singleInputAudioProcessing(firstRemoteAudioTrack);
// Test 2
dualInputAudioProcessing_version1(firstLocalAudioTrack, firstRemoteAudioTrack);
// Test 3
dualInputAudioProcessing_version2(firstLocalAudioTrack, firstRemoteAudioTrack);
}
}
} catch (e) {
console.log(e);
}
}
return { startAudioProcessing }
}
export default useAudioProcessing;
Also here is the file for the CustomAudioProcessor which is located in the client/public/worklets/
// client\public\worklets\customAudioProcessor.js
// Custom Audio Processor - for now, it just outputs the inputs to the log
// Will involve some audio comparision algorithms later
class CustomAudioProcessor extends AudioWorkletProcessor {
process(inputList, outputList, parameters) {
// Processing logic here
console.log("Input", inputList);
return true;
}
}
registerProcessor("custom-audio-processor", CustomAudioProcessor);
I've tried running the following tests independently
- Test 0 -
singleInputAudioProcessing(firstLocalAudioTrack);
- Test 1 -
singleInputAudioProcessing(firstRemoteAudioTrack);
- Test 2 -
dualInputAudioProcessing_version1(firstLocalAudioTrack, firstRemoteAudioTrack);
- Test 3 -
dualInputAudioProcessing_version2(firstLocalAudioTrack, firstRemoteAudioTrack);
Test 0 gave me a result as follows: Test 0 single log and Test 0 logs
Test 1 fetched me results when there was a remote user connected: Test 1 logs
Test 2 and Test 3 failed to output anything to the logs. No errors were displayed in the logs.