Handle stream as individual frames using streamlink

Question

I`m trying to handle stream as individual frames using streamlink

args = ['streamlink', stream_url, "best", "-O"]
process = subprocess.Popen(args, stdout=subprocess.PIPE)
while True:
        frame_size = width * height * 3
        in_frame = streamlink.stdout.read(frame_size)
        if in_frame is None:
            break
        #cv2.imwrite(f'frames/{i}.jpg', in_frame)
        #do anything with in_frame

But getting images that looks like white noise. I think it because stream also contain audio in bytes. Then i try to pipe it to ffmpeg but cant get decoded bytes out of ffmpeg

args = (
        ffmpeg.input('pipe:')
        .filter('fps', fps=1)
        .output('pipe:', vframes=1, format='image2', vcodec='mjpeg')
        .compile()
 )
 process2 = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=None)
 buff = process1.stdout.read(n)
 process2.stdin.write(buff)
 frame = process2.stdout.read(n)

When i try to smt like that all my script hang and waiting something. How to properly handle stream from streamlink as individual frames. To get frame as bytes or else? Thank you.

Answer 1

Instead of piping the data to FFmpeg, you may pass the URL as input argument to FFmpeg.

• Get the stream URL from the WEB site URL:

   def stream_to_url(url, quality='best'):
       session = Streamlink()
       streams = session.streams(url)
       return streams[quality].to_url()
  

• Use FFprobe for getting the video resolution (if required):

   p = ffmpeg.probe(stream_url, select_streams='v');
   width = p['streams'][0]['width']
   height = p['streams'][0]['height']
  

• Execute FFmpeg sub-process with URL as input and raw (BGR) output format:

   process = (
       ffmpeg
       .input(stream_url)
       .video
       .output('pipe:', format='rawvideo', pix_fmt='bgr24')
       .run_async(pipe_stdout=True) # In case ffmpeg in not in executable path, add cmd=fullpath like: .run_async(pipe_stdout=True, cmd=r'c:\FFmpeg\bin\ffmpeg.exe')
   )
  

• Read frames from PIPE, convert to NumPy array, reshape and display:

   ...
   in_bytes = process.stdout.read(width * height * 3)
   frame = np.frombuffer(in_bytes, np.uint8).reshape([height, width, 3])
   cv2.imshow('frame', frame)
   ...
  

---

Complete code sample:

from streamlink import Streamlink
import numpy as np
import cv2
import ffmpeg

def stream_to_url(url, quality='best'):
    """ Get URL, and return streamlink URL """
    session = Streamlink()
    streams = session.streams(url)

    if streams:
        return streams[quality].to_url()
    else:
        raise ValueError('Could not locate your stream.')


url = 'https://www.twitch.tv/riotgames'  # Login to twitch TV before starting (the URL is for a random live stream).
quality='best'

stream_url = stream_to_url(url, quality)

# Use FFprobe to get video frames resolution (required in case resolution is unknown).
###############################################
p = ffmpeg.probe(stream_url, select_streams='v');
width = p['streams'][0]['width']
height = p['streams'][0]['height']
###############################################

# Execute FFmpeg sub-process with URL as input and raw (BGR) output format.
process = (
    ffmpeg
    .input(stream_url)
    .video
    .output('pipe:', format='rawvideo', pix_fmt='bgr24')
    .run_async(pipe_stdout=True) # In case ffmpeg in not in executable path, add cmd=fullpath like: .run_async(pipe_stdout=True, cmd=r'c:\FFmpeg\bin\ffmpeg.exe')
)


# Read decoded video (frame by frame), and display each frame (using cv2.imshow)
while True:
    # Read raw video frame from stdout as bytes array.
    in_bytes = process.stdout.read(width * height * 3)

    if not in_bytes:
        break

    # Transform the byte read into a NumPy array
    frame = np.frombuffer(in_bytes, np.uint8).reshape([height, width, 3])

    # Display the frame
    cv2.imshow('frame', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

process.stdout.close()
process.wait()
cv2.destroyAllWindows()

---

There is a simpler solution using cv2.VideoCapture:

stream_url = stream_to_url(url, quality)

cap = cv2.VideoCapture(stream_url)

while True:
    success, frame = cap.read()

    if not success:
        break

    cv2.imshow('frame', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

---

Update:

Piping from Streamlink sub-process to FFmpeg sub-process:

Assume you have to read the stream from stdout pipe of Streamlink and write it to stdin pipe of FFmpeg:

• Start Streamlink sub-process (use -O argument for piping):

   streamlink_args = [r'c:\Program Files (x86)\Streamlink\bin\streamlink.exe', stream_url, "best", "-O"]  # Windows executable downloaded from: https://github.com/streamlink/streamlink/releases/tag/2.4.0
   streamlink_process = sp.Popen(streamlink_args, stdout=sp.PIPE)  # Execute Streamlink as sub-process
  

• Implement a thread that read chunks from stdout pipe of Streamlink and write to FFmpeg stdin pipe:

   def writer(streamlink_proc, ffmpeg_proc):
       while (not streamlink_proc.poll()) and (not ffmpeg_proc.poll()):
           try:
               chunk = streamlink_proc.stdout.read(1024)
               ffmpeg_proc.stdin.write(chunk)
           except (BrokenPipeError, OSError) as e:
               pass
  

• Execute FFmpeg sub-process with input pipe and output pipe:

   ffmpeg_process = (
       ffmpeg
       .input('pipe:')
       .video
       .output('pipe:', format='rawvideo', pix_fmt='bgr24')
       .run_async(pipe_stdin=True, pipe_stdout=True) # In case ffmpeg in not in executable path, add cmd=fullpath like: .run_async(pipe_stdout=True, cmd=r'c:\FFmpeg\bin\ffmpeg.exe')
   )
  

• Create and start the thread:

   thread = threading.Thread(target=writer, args=(streamlink_process, ffmpeg_process))
   thread.start()
  

---

Complete code sample:

import numpy as np
import subprocess as sp
import threading
import cv2
import ffmpeg

#stream_url = 'https://www.nimo.tv/v/v-1712291636586087045'
stream_url = 'https://www.twitch.tv/esl_csgo'

# Assume video resolution is known.
width, height = 1920, 1080


# Writer thread (read from streamlink and write to FFmpeg in chunks of 1024 bytes).
def writer(streamlink_proc, ffmpeg_proc):
    while (not streamlink_proc.poll()) and (not ffmpeg_proc.poll()):
        try:
            chunk = streamlink_proc.stdout.read(1024)
            ffmpeg_proc.stdin.write(chunk)
        except (BrokenPipeError, OSError) as e:
            pass


streamlink_args = [r'c:\Program Files (x86)\Streamlink\bin\streamlink.exe', stream_url, "best", "-O"]  # Windows executable downloaded from: https://github.com/streamlink/streamlink/releases/tag/2.4.0
streamlink_process = sp.Popen(streamlink_args, stdout=sp.PIPE)  # Execute streamlink as sub-process


# Execute FFmpeg sub-process with URL as input and raw (BGR) output format.
ffmpeg_process = (
    ffmpeg
    .input('pipe:')
    .video
    .output('pipe:', format='rawvideo', pix_fmt='bgr24')
    .run_async(pipe_stdin=True, pipe_stdout=True) # In case ffmpeg in not in executable path, add cmd=fullpath like: .run_async(pipe_stdout=True, cmd=r'c:\FFmpeg\bin\ffmpeg.exe')
)


thread = threading.Thread(target=writer, args=(streamlink_process, ffmpeg_process))
thread.start()


# Read decoded video (frame by frame), and display each frame (using cv2.imshow)
while True:
    # Read raw video frame from stdout as bytes array.
    in_bytes = ffmpeg_process.stdout.read(width * height * 3)

    if not in_bytes:
        break

    # Transform the byte read into a NumPy array
    frame = np.frombuffer(in_bytes, np.uint8).reshape([height, width, 3])

    # Display the frame
    cv2.imshow('frame', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

ffmpeg_process.stdout.close()
ffmpeg_process.wait()
#streamlink_process.stdin.close()
streamlink_process.kill()
cv2.destroyAllWindows()

---

Notes:

• The code sample uses a link to twitch.tv instead of nimo.tv because "Nimo has broken streamlink plugin".
• The sample assumes width and height are known from advance.
• The sample was tested with Windows 10 (Executes streamlink.exe: r'c:\Program Files (x86)\Streamlink\bin\streamlink.exe' after installing Streamlink).