Unable to pass Multiple Audio Streams to AudioWorkletProcessor as inputs. (WebAudio API) (Twilio)

100 Views Asked by At

Given below is a React Hook File which implements some Audio Processing of the audio tracks of Local and Remote Participants in a Meeting Application using Twilio Video Services. Till now I have been able to obtain results in the CustomAudioProcessor with a single input, but it doesn't seem to be working with a dual input system. Is there anything I am doing wrong here.

// client\src\Hooks\useAudioProcessing.ts - React Hook File
import { LocalAudioTrack, RemoteAudioTrack } from "twilio-video";
import useTwilioVideoContext from "./useTwilioVideoContext"

const useAudioProcessing = () => {
    const { state } = useTwilioVideoContext();
    const audioContext = new AudioContext();
    const singleInputAudioProcessing = async (audioTrack: LocalAudioTrack | RemoteAudioTrack) => {
        // Load the AudioWorkletProcessor
        await audioContext.audioWorklet.addModule('worklets/customAudioProcessor.js');
        const customAudioProcessor = new AudioWorkletNode(audioContext, 'custom-audio-processor');

        // Create MediaStreamAudioSourceNode
        const mediaStream = new MediaStream();
        mediaStream.addTrack(audioTrack.mediaStreamTrack);
        const audioSource = audioContext.createMediaStreamSource(mediaStream);

        // Connect Nodes
        audioSource.connect(customAudioProcessor);
        customAudioProcessor.connect(audioContext.destination);

        if (audioContext.state === 'suspended') {
            await audioContext.resume();
        }
    }

    const dualInputAudioProcessing_version1 = async (localAudioTrack: LocalAudioTrack, remoteAudioTrack: RemoteAudioTrack) => {
        // Load the AudioWorkletProcessor
        await audioContext.audioWorklet.addModule('worklets/customAudioProcessor.js');
        const customAudioProcessor = new AudioWorkletNode(audioContext, 'custom-audio-processor');

        // Create MediaStreamAudioSourceNodes
        const mediaStream = new MediaStream();
        mediaStream.addTrack(localAudioTrack.mediaStreamTrack);
        mediaStream.addTrack(remoteAudioTrack.mediaStreamTrack);
        const audioSource = audioContext.createMediaStreamSource(mediaStream);

        // Connect Nodes
        audioSource.connect(customAudioProcessor);
        customAudioProcessor.connect(audioContext.destination);

        if (audioContext.state === 'suspended') {
            await audioContext.resume();
        }
    }

    async function dualInputAudioProcessing_version2(localAudioTrack: LocalAudioTrack, remoteAudioTrack: RemoteAudioTrack) {
        // Load the AudioWorkletProcessor
        await audioContext.audioWorklet.addModule('worklets/customAudioProcessor.js');
        const customAudioProcessorInputNode = new AudioWorkletNode(audioContext, 'custom-audio-processor');

        // Create local MediaStreamAudioSourceNode - Has one output node
        const localMediaStream = new MediaStream();
        localMediaStream.addTrack(localAudioTrack.mediaStreamTrack);
        const localSource = audioContext.createMediaStreamSource(localMediaStream);

        // Create remote MediaStreamAudioSourceNode - Has one output node
        const remoteMediaStream = new MediaStream();
        remoteMediaStream.addTrack(remoteAudioTrack.mediaStreamTrack);
        const remoteSource = audioContext.createMediaStreamSource(remoteMediaStream);

        // Handle varying sample rates
        if (localSource.context.sampleRate !== remoteSource.context.sampleRate) {
            // Resampling logic here, if needed
            console.log(localSource.context.sampleRate, remoteSource.context.sampleRate);
        } else {
            console.log(localSource.context.sampleRate, remoteSource.context.sampleRate);
        }

        // Create a ChannelMergerNode to merge the local and remote tracks
        const merger = audioContext.createChannelMerger(2);
        // Connect nodes
        localSource.connect(merger, 0, 0); // Connect local source node output0 to merger node input0
        remoteSource.connect(merger, 0, 1); // Connect remote source node output0 to merger node input1
        merger.connect(customAudioProcessorInputNode); // Connect merger node to merger node to customAudioProcessorInputNode
        customAudioProcessorInputNode.connect(audioContext.destination); // Connect to destination (speakers)
    }

    async function startAudioProcessing() {
        try {
            if (!state.room) throw new Error("Can not start audio processing without joining room");

            // Use flatMap to eliminate the need for Array.from().map() chaining
            const localAudioTracks = Array.from(state.room.localParticipant.audioTracks.values()).flatMap(trackPub => trackPub.track ? [trackPub.track] : []);
            const firstLocalAudioTrack = localAudioTracks[0] || null;

            const firstRemoteParticipant = Array.from(state.room.participants.values())[0];
            const remoteAudioTracks = firstRemoteParticipant ? Array.from(firstRemoteParticipant.audioTracks.values()).flatMap(trackPub => trackPub.track ? [trackPub.track] : []) : [];
            const firstRemoteAudioTrack = remoteAudioTracks[0] || null;
            if (firstLocalAudioTrack) {
                // Test 0
                singleInputAudioProcessing(firstLocalAudioTrack);
                if (firstRemoteAudioTrack) {
                    // Test 1
                    singleInputAudioProcessing(firstRemoteAudioTrack);
                    // Test 2
                    dualInputAudioProcessing_version1(firstLocalAudioTrack, firstRemoteAudioTrack); 
                    // Test 3
                    dualInputAudioProcessing_version2(firstLocalAudioTrack, firstRemoteAudioTrack);
                }
            }
        } catch (e) {
            console.log(e);
        }
    }
    return { startAudioProcessing }
}
export default useAudioProcessing;

Also here is the file for the CustomAudioProcessor which is located in the client/public/worklets/

// client\public\worklets\customAudioProcessor.js 
// Custom Audio Processor - for now, it just outputs the inputs to the log
// Will involve some audio comparision algorithms later
class CustomAudioProcessor extends AudioWorkletProcessor {
    process(inputList, outputList, parameters) {
        // Processing logic here
        console.log("Input", inputList);
        
        return true;
    }
}
registerProcessor("custom-audio-processor", CustomAudioProcessor);

I've tried running the following tests independently

  • Test 0 - singleInputAudioProcessing(firstLocalAudioTrack);
  • Test 1 - singleInputAudioProcessing(firstRemoteAudioTrack);
  • Test 2 - dualInputAudioProcessing_version1(firstLocalAudioTrack, firstRemoteAudioTrack);
  • Test 3 - dualInputAudioProcessing_version2(firstLocalAudioTrack, firstRemoteAudioTrack);

Test 0 gave me a result as follows: Test 0 single log and Test 0 logs

Test 1 fetched me results when there was a remote user connected: Test 1 logs

Test 2 and Test 3 failed to output anything to the logs. No errors were displayed in the logs.

0

There are 0 best solutions below