How to capture a video stream from a raspberry pi on a remote computer in Python

Question

I have an desktop application in Python that needs to capture a video stream from a Raspberry Pi Zero, display it in a PyQt widget and store its last frame in a numpy array for ML purposes. The application can communicate with both Wi-Fi or Bluetooth and must work the same way in both cases. The Rpi acts a server waiting for a connection and the desktop application as a client reading the video stream. Using OpenCV, it works good enough in Wi-Fi, but I get huge lag using Bluetooth. However, the lag is only at start-up, once the stream is running, the video is smooth, just with a 5 seconds delay.

Here are some code to reproduce:

server side

import sys
import socket
import picamera


def main():
    with picamera.PiCamera() as camera:
        camera.resolution = (320, 240)
        camera.framerate = 10
        if len(sys.argv) > 1 and sys.argv[1] == "b":
            server_socket = socket.socket(
                socket.AF_BLUETOOTH,
                socket.SOCK_STREAM,
                socket.BTPROTO_RFCOMM
            )
            server_socket.bind(('b8:27:eb:8f:28:12', 7))  # use your own Rpi bt address
        else:
            server_socket = socket.socket(
                socket.AF_INET,
                socket.SOCK_STREAM
            )
            server_socket.bind(('', 8000))
        server_socket.listen(0)

        # Accept a single connection and make a file-like object out of it
        print("socket ready to serve")
        connection = server_socket.accept()[0].makefile('wb')
        try:
            print("connection accepted, serving...")
            camera.start_recording(connection, format='mjpeg')
            camera.wait_recording(60000)
            camera.stop_recording()
        except (KeyboardInterrupt, BrokenPipeError, ConnectionResetError):
            print("Stopping")
        finally:
            connection.close()
            server_socket.close()


if __name__ == '__main__':
    main()

and client side

import socket
import threading

import cv2


class StreamHandler:
    def __init__(self, src_sock):
        self.src_sock = src_sock  # type: socket.socket
        self.dest_sock = socket.socket()
        self.dest_sock.bind(("127.0.0.1", 9090))
        self.dest_sock.listen(1)

    def run(self):
        print("waiting for redirection")
        connection, _ = self.dest_sock.accept()
        print("Redirection accepted, commencing...")
        while True:
            try:
                buffer = self.src_sock.recv(4096)
                connection.send(buffer)
            except ConnectionResetError:
                print("Stopping redirection")
                break


def main():
    # sock = socket.socket()
    # sock.connect(("10.0.0.1", 8000))  # replace with your Rpi IP address
    sock = socket.socket(
        socket.AF_BLUETOOTH,
        socket.SOCK_STREAM,
        socket.BTPROTO_RFCOMM
    )
    sock.connect(('b8:27:eb:8f:28:12', 7))  # replace with your Rpi bt address
    stream_handler = StreamHandler(sock)
    stream_thread = threading.Thread(target=stream_handler.run)
    stream_thread.start()
    cap = cv2.VideoCapture("tcp://127.0.0.1:9090")
    if not cap.isOpened():
        print('VideoCapture not opened')
        exit(-1)

    while True:
        ret, frame = cap.read()

        if not ret:
            print('frame empty')
            break

        cv2.imshow('image', frame)

        if cv2.waitKey(1) & 0XFF == ord('q'):
            break

    cap.release()
    cv2.destroyAllWindows()


if __name__ == '__main__':
    main()

The latency might come from the port redirection I do because of the Bluetooth communication, but I have no certainty regarding that since it does not lag in Wi-Fi. At first I wanted to use cv2.VideoCapture() and use the connection from the socket.makefile() as an input, but that operation is not permitted on a file-like object. I do not have a stop on OpenCV, I just use it for convenience, I just want to convert my video stream to a Numpy Array. The solution has to work regardless of the OS.

EDIT

Using Numpy function np.frombuffer() and OpenCV cv2.imdecode() spits out an error I don't understand:

cv2.imshow('image', data)
cv2.error: OpenCV(4.5.5) D:\a\opencv-python\opencv-python\opencv\modules\imgproc\src\color.cpp:182: error: (-215:Assertion failed) !_src.empty() in function 'cv::cvtColor'

from the code

import socket
import cv2
import numpy as np


def main():
    sock = socket.socket()
    sock.connect(("10.0.0.1", 8000))
    # sock = socket.socket(
    #     socket.AF_BLUETOOTH,
    #     socket.SOCK_STREAM,
    #     socket.BTPROTO_RFCOMM
    # )
    # sock.connect(('b8:27:eb:8f:28:12', 7))
    connection = sock.makefile('rb')

    while True:
        data = cv2.imdecode(np.frombuffer(
            connection.read(4096), dtype=np.uint8), 1
        )

        cv2.imshow('image', data)

        if cv2.waitKey(1) & 0XFF == ord('q'):
            break

    cv2.destroyAllWindows()


if __name__ == '__main__':
    main()

What am I doing wrong here?

tl;dr:

Is there a way to capture the PiCamera stream directly into a numpy array as described in the doc but in a Python environment that does not have the picamera module?

Thank you for reading!