I am trying to stream RTSP to web browser using ffmpeg through web socket relay written in node js taken from https://github.com/phoboslab/jsmpeg , and on the browser i am using JSMpeg to display the RTSP stream, the video is playing fine, but audio is not playing,
The ffmpeg command:
ffmpeg -rtsp_transport tcp -i rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mp4
-f mpegts -c:v mpeg1video -c:a mp2 http://127.0.0.1:8081/stream_from_ffmpeg/<my_secret>
The node js web socket relay:
// Use the websocket-relay to serve a raw MPEG-TS over WebSockets. You can use
// ffmpeg to feed the relay. ffmpeg -> websocket-relay -> browser
// Example:
// node websocket-relay yoursecret 8081 8082
// ffmpeg -i <some input> -f mpegts http://localhost:8081/yoursecret
var fs = require('fs'),
http = require('http'),
WebSocket = require('ws');
if (process.argv.length < 3) {
console.log(
'Usage: \n' +
'node websocket-relay.js <secret> [<stream-port> <websocket-port>]'
);
process.exit();
}
var STREAM_SECRET = process.argv[2],
STREAM_PORT = process.argv[3] || 8081,
WEBSOCKET_PORT = process.argv[4] || 8082,
RECORD_STREAM = false;
// Websocket Server
var socketServer = new WebSocket.Server({port: WEBSOCKET_PORT, perMessageDeflate: false});
socketServer.connectionCount = 0;
socketServer.on('connection', function(socket, upgradeReq) {
socketServer.connectionCount++;
console.log(
'New WebSocket Connection: ',
(upgradeReq || socket.upgradeReq).socket.remoteAddress,
(upgradeReq || socket.upgradeReq).headers['user-agent'],
'('+socketServer.connectionCount+' total)'
);
socket.on('close', function(code, message){
socketServer.connectionCount--;
console.log(
'Disconnected WebSocket ('+socketServer.connectionCount+' total)'
);
});
});
socketServer.broadcast = function(data) {
socketServer.clients.forEach(function each(client) {
if (client.readyState === WebSocket.OPEN) {
client.send(data);
}
});
};
// HTTP Server to accept incoming MPEG-TS Stream from ffmpeg
var streamServer = http.createServer( function(request, response) {
var params = request.url.substr(1).split('/');
if (params[0] !== STREAM_SECRET) {
console.log(
'Failed Stream Connection: '+ request.socket.remoteAddress + ':' +
request.socket.remotePort + ' - wrong secret.'
);
response.end();
}
response.connection.setTimeout(0);
console.log(
'Stream Connected: ' +
request.socket.remoteAddress + ':' +
request.socket.remotePort
);
request.on('data', function(data){
socketServer.broadcast(data);
if (request.socket.recording) {
request.socket.recording.write(data);
}
});
request.on('end',function(){
console.log('close');
if (request.socket.recording) {
request.socket.recording.close();
}
});
// Record the stream to a local file?
if (RECORD_STREAM) {
var path = 'recordings/' + Date.now() + '.ts';
request.socket.recording = fs.createWriteStream(path);
}
})
// Keep the socket open for streaming
streamServer.headersTimeout = 0;
streamServer.listen(STREAM_PORT);
console.log('Listening for incoming MPEG-TS Stream on http://127.0.0.1:'+STREAM_PORT+'/<secret>');
console.log('Awaiting WebSocket connections on ws://127.0.0.1:'+WEBSOCKET_PORT+'/');
The front end code
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<script src="./jsmpeg.min.js"></script>
<title>Document</title>
</head>
<body>
<canvas id="video-canvas"></canvas>
</body>
<script>
let url;
let player;
let canvas = document.getElementById("video-canvas");
let ipAddr = "127.0.0.1:8082";
window.onload = async() => {
url = `ws://${ipAddr}`;
player = new JSMpeg.Player(url, { canvas: canvas, });
};
</script>
</html>
The above code works fine and plays the video, but no audio is playing Things I tried:
Changed the audio context state inside the player object from suspended to running
player.audioOut.context.onstatechange = async () => {
console.log("Event triggered by audio");
if (player.audioOut.context === "suspended") {
await player.audioOut.context.resume();
}
}
RTSP-Relay has an implementation of this system: https://github.com/k-yle/rtsp-relay
It's a really good library, and if you're interested in rolling it yourself I recommend reviewing the source code. It loads a node module that holds the ffmpeg binary (so you don't need ffmpeg on your system), spawns it to handle the rtsp connection, and pipes the ffmpeg output to that connections websocket. Handles connections and retries very well. It also has a client-side version that provides a helper to connect websocket audio stream to client.
Through this library, I've gotten audio (and video) to play when the audio was mpeg4 aac audio. Note - I needed to provide a button click to begin the audio stream. I used mediamtx / rtsp-simple-server to take one stream and convert the pcm ulaw audio to aac.
It really comes down knowing what formats you're working with at different stages.