Given below is a React Hook File which implements some Audio Processing of the audio tracks of Local and Remote Participants in a Meeting Application using Twilio Video Services. Till now I have been able to obtain results in the CustomAudioProcessor with a single input, but it doesn't seem to be working with a dual input system. Is there anything I am doing wrong here.
// client\src\Hooks\useAudioProcessing.ts - React Hook File
import { LocalAudioTrack, RemoteAudioTrack } from "twilio-video";
import useTwilioVideoContext from "./useTwilioVideoContext"
const useAudioProcessing = () => {
    const { state } = useTwilioVideoContext();
    const audioContext = new AudioContext();
    const singleInputAudioProcessing = async (audioTrack: LocalAudioTrack | RemoteAudioTrack) => {
        // Load the AudioWorkletProcessor
        await audioContext.audioWorklet.addModule('worklets/customAudioProcessor.js');
        const customAudioProcessor = new AudioWorkletNode(audioContext, 'custom-audio-processor');
        // Create MediaStreamAudioSourceNode
        const mediaStream = new MediaStream();
        mediaStream.addTrack(audioTrack.mediaStreamTrack);
        const audioSource = audioContext.createMediaStreamSource(mediaStream);
        // Connect Nodes
        audioSource.connect(customAudioProcessor);
        customAudioProcessor.connect(audioContext.destination);
        if (audioContext.state === 'suspended') {
            await audioContext.resume();
        }
    }
    const dualInputAudioProcessing_version1 = async (localAudioTrack: LocalAudioTrack, remoteAudioTrack: RemoteAudioTrack) => {
        // Load the AudioWorkletProcessor
        await audioContext.audioWorklet.addModule('worklets/customAudioProcessor.js');
        const customAudioProcessor = new AudioWorkletNode(audioContext, 'custom-audio-processor');
        // Create MediaStreamAudioSourceNodes
        const mediaStream = new MediaStream();
        mediaStream.addTrack(localAudioTrack.mediaStreamTrack);
        mediaStream.addTrack(remoteAudioTrack.mediaStreamTrack);
        const audioSource = audioContext.createMediaStreamSource(mediaStream);
        // Connect Nodes
        audioSource.connect(customAudioProcessor);
        customAudioProcessor.connect(audioContext.destination);
        if (audioContext.state === 'suspended') {
            await audioContext.resume();
        }
    }
    async function dualInputAudioProcessing_version2(localAudioTrack: LocalAudioTrack, remoteAudioTrack: RemoteAudioTrack) {
        // Load the AudioWorkletProcessor
        await audioContext.audioWorklet.addModule('worklets/customAudioProcessor.js');
        const customAudioProcessorInputNode = new AudioWorkletNode(audioContext, 'custom-audio-processor');
        // Create local MediaStreamAudioSourceNode - Has one output node
        const localMediaStream = new MediaStream();
        localMediaStream.addTrack(localAudioTrack.mediaStreamTrack);
        const localSource = audioContext.createMediaStreamSource(localMediaStream);
        // Create remote MediaStreamAudioSourceNode - Has one output node
        const remoteMediaStream = new MediaStream();
        remoteMediaStream.addTrack(remoteAudioTrack.mediaStreamTrack);
        const remoteSource = audioContext.createMediaStreamSource(remoteMediaStream);
        // Handle varying sample rates
        if (localSource.context.sampleRate !== remoteSource.context.sampleRate) {
            // Resampling logic here, if needed
            console.log(localSource.context.sampleRate, remoteSource.context.sampleRate);
        } else {
            console.log(localSource.context.sampleRate, remoteSource.context.sampleRate);
        }
        // Create a ChannelMergerNode to merge the local and remote tracks
        const merger = audioContext.createChannelMerger(2);
        // Connect nodes
        localSource.connect(merger, 0, 0); // Connect local source node output0 to merger node input0
        remoteSource.connect(merger, 0, 1); // Connect remote source node output0 to merger node input1
        merger.connect(customAudioProcessorInputNode); // Connect merger node to merger node to customAudioProcessorInputNode
        customAudioProcessorInputNode.connect(audioContext.destination); // Connect to destination (speakers)
    }
    async function startAudioProcessing() {
        try {
            if (!state.room) throw new Error("Can not start audio processing without joining room");
            // Use flatMap to eliminate the need for Array.from().map() chaining
            const localAudioTracks = Array.from(state.room.localParticipant.audioTracks.values()).flatMap(trackPub => trackPub.track ? [trackPub.track] : []);
            const firstLocalAudioTrack = localAudioTracks[0] || null;
            const firstRemoteParticipant = Array.from(state.room.participants.values())[0];
            const remoteAudioTracks = firstRemoteParticipant ? Array.from(firstRemoteParticipant.audioTracks.values()).flatMap(trackPub => trackPub.track ? [trackPub.track] : []) : [];
            const firstRemoteAudioTrack = remoteAudioTracks[0] || null;
            if (firstLocalAudioTrack) {
                // Test 0
                singleInputAudioProcessing(firstLocalAudioTrack);
                if (firstRemoteAudioTrack) {
                    // Test 1
                    singleInputAudioProcessing(firstRemoteAudioTrack);
                    // Test 2
                    dualInputAudioProcessing_version1(firstLocalAudioTrack, firstRemoteAudioTrack); 
                    // Test 3
                    dualInputAudioProcessing_version2(firstLocalAudioTrack, firstRemoteAudioTrack);
                }
            }
        } catch (e) {
            console.log(e);
        }
    }
    return { startAudioProcessing }
}
export default useAudioProcessing;
Also here is the file for the CustomAudioProcessor which is located in the client/public/worklets/
// client\public\worklets\customAudioProcessor.js 
// Custom Audio Processor - for now, it just outputs the inputs to the log
// Will involve some audio comparision algorithms later
class CustomAudioProcessor extends AudioWorkletProcessor {
    process(inputList, outputList, parameters) {
        // Processing logic here
        console.log("Input", inputList);
        
        return true;
    }
}
registerProcessor("custom-audio-processor", CustomAudioProcessor);
I've tried running the following tests independently
- Test 0 - 
singleInputAudioProcessing(firstLocalAudioTrack); - Test 1 - 
singleInputAudioProcessing(firstRemoteAudioTrack); - Test 2 - 
dualInputAudioProcessing_version1(firstLocalAudioTrack, firstRemoteAudioTrack); - Test 3 - 
dualInputAudioProcessing_version2(firstLocalAudioTrack, firstRemoteAudioTrack); 
Test 0 gave me a result as follows: Test 0 single log and Test 0 logs
Test 1 fetched me results when there was a remote user connected: Test 1 logs
Test 2 and Test 3 failed to output anything to the logs. No errors were displayed in the logs.