Error in Gesture Recognition System in Webcam Display

Question

I am trying to implement the gesture recognition system for webcam using the mediapipe and gesturerecognizer.task model which I have already downloaded. Following is the python code.

import cv2
import mediapipe as mp
from mediapipe.tasks import python
from mediapipe.tasks.python import vision

model_path = "gesture_recognizer.task"
base_options = python.BaseOptions(model_asset_path=model_path)
GestureRecognizer = mp.tasks.vision.GestureRecognizer
GestureRecognizerOptions = mp.tasks.vision.GestureRecognizerOptions
GestureRecognizerResult = mp.tasks.vision.GestureRecognizerResult
VisionRunningMode = mp.tasks.vision.RunningMode

def print_result(result: GestureRecognizerResult, output_image: mp.Image, timestamp_ms: int):
    print('gesture recognition result: {}'.format(result))

options = GestureRecognizerOptions(
    base_options=python.BaseOptions(model_asset_path=model_path),
    running_mode=VisionRunningMode.LIVE_STREAM,
    result_callback=print_result)
recognizer = GestureRecognizer.create_from_options(options)

mp_drawing = mp.solutions.drawing_utils
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
        static_image_mode=False,
        max_num_hands=2,
        min_detection_confidence=0.65,
        min_tracking_confidence=0.65)

cap = cv2.VideoCapture(0)

while True:
    ret, frame = cap.read()
    if not ret:
        break
        
    i = 1  # left or right hand
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame)
    frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
    np_array = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

    # cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    # mp_image 
    
    if results.multi_hand_landmarks:
        for hand_landmarks in results.multi_hand_landmarks:
            h, w, c = frame.shape
            mp_drawing.draw_landmarks(frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
            mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np_array)
            results = recognizer.recognize(mp_image)
    else: 
        mp_image = []
        # print("NOOO")
    
    # show the prediction on the frame
    cv2.putText(mp_image, results, (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2, cv2.LINE_AA)
    cv2.imshow('MediaPipe Hands', frame)

    if cv2.waitKey(1) & 0xFF == 27:
        break

cap.release()

That print("NOOO") is only for debugging. The error is when there is no any hand in webcam, it runs fine. But once I show my hand, the window close and this error is shown in terminal.

Traceback (most recent call last):
  File "c:\Users\Nishar Miya\Downloads\new\webcam2.py", line 52, in <module>
    results = recognizer.recognize(mp_image)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Python311\Lib\site-packages\mediapipe\tasks\python\vision\gesture_recognizer.py", line 377, in recognize
    output_packets = self._process_image_data({
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Python311\Lib\site-packages\mediapipe\tasks\python\vision\core\base_vision_task_api.py", line 91, in _process_image_data
    raise ValueError(
ValueError: Task is not initialized with the image mode. Current running mode:LIVE_STREAM

Please Help

Sometimes this error is shown on putText

cv2.putText(mp_image, results, (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2, cv2.LINE_AA)
cv2.error: OpenCV(4.7.0) :-1: error: (-5:Bad argument) in function 'putText'
> Overload resolution failed:
>  - img is not a numpy array, neither a scalar
>  - Expected Ptr<cv::UMat> for argument 'img'

Answer 1

Hello try use a try catch block to safely handle the absence of a hand on that problematic function. or instead run the method once mp_image is not empty