How to get the coordinates of the dots in hand image?

Question

My image,

I've tried to get quite accurate palm from the hand image and used mediapipe(mp) library to do it. But the result is:

mp calculates and draws green strokes and red dots, but also give the coordinates of red dots. So I can get the coordinates of blue circles using given coordinates of red ones. But it can't be enough to get "accurate" palm. I need to get the black dots' coordinates. following is my code:

import cv2
import mediapipe as mp
import utils

mp_drawing = mp.solutions.drawing_utils
mp_hands = mp.solutions.hands




def makeCircle(img,circle_y,circle_x,radius):
    image_height, image_width, _ = img.shape
    y = int(circle_y * image_height)
    x = int(circle_x * image_width)
    circle_coordinates = (x,y)
    color = (255, 0, 0)
    thickness = 2
    return cv2.circle(img,circle_coordinates,radius,color,thickness)



# For static images:
IMAGE_FILES = [
    "C:/Users/USER/workspace/palm/images/sample1.png",
    "C:/Users/USER/workspace/palm/images/sample2.png",
    "C:/Users/USER/workspace/palm/images/sample3.png",
    "C:/Users/USER/workspace/palm/images/sample4.png"
]

with mp_hands.Hands(
    static_image_mode=True,
    max_num_hands=2,
    min_detection_confidence=0.5) as hands:
  for idx, file in enumerate(IMAGE_FILES):
    # Read an image, flip it around y-axis for correct handedness output (see
    # above).
    image = cv2.flip(cv2.imread(file), 1)
    image = utils.remove_bground(image)


    # resize image's height 600 fixing the ratio
    image = utils.ResizeWithAspectRatio(image,height=600)

    # Convert the BGR image to RGB before processing.
    results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))

    print("results:",results)
    # Print handedness and draw hand landmarks on the image.
    print('Handedness:', results.multi_handedness)
    if not results.multi_hand_landmarks:
      continue
    image_height, image_width, _ = image.shape
    annotated_image = image.copy()
    for hand_landmarks in results.multi_hand_landmarks:
      print('hand_landmarks:', hand_landmarks)
      print(
          f'Index finger tip coordinates: (',
          f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].x * image_width}, '
          f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * image_height})'
      )


      circle_coordinates = [
        (
            (hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP].y + hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP].y)/2,
            (hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP].x + hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP].x)/2
        ),
        (
            (hand_landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_MCP].y + hand_landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP].y)/2,
            (hand_landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_MCP].x + hand_landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP].x)/2
        ),
        (
            (hand_landmarks.landmark[mp_hands.HandLandmark.RING_FINGER_MCP].y + hand_landmarks.landmark[mp_hands.HandLandmark.RING_FINGER_PIP].y)/2,
            (hand_landmarks.landmark[mp_hands.HandLandmark.RING_FINGER_MCP].x + hand_landmarks.landmark[mp_hands.HandLandmark.RING_FINGER_PIP].x)/2
        ),
        (
        (hand_landmarks.landmark[mp_hands.HandLandmark.PINKY_MCP].y + hand_landmarks.landmark[mp_hands.HandLandmark.PINKY_PIP].y)/2,
        (hand_landmarks.landmark[mp_hands.HandLandmark.PINKY_MCP].x + hand_landmarks.landmark[mp_hands.HandLandmark.PINKY_PIP].x)/2
        ),
        (
            hand_landmarks.landmark[mp_hands.HandLandmark.WRIST].y,
            hand_landmarks.landmark[mp_hands.HandLandmark.WRIST].x
        ),
        (
            hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_CMC].y,
            hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_CMC].x
        ),
        (
            hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_MCP].y,
            hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_MCP].x
        ),
      ]

      for (circle_y,circle_x) in circle_coordinates:
          annotated_image = makeCircle(annotated_image,circle_y,circle_x,10)

      palm_center_y = (hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_CMC].y +
        hand_landmarks.landmark[mp_hands.HandLandmark.PINKY_MCP].y)/2
      palm_center_x = (hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_CMC].x +
        hand_landmarks.landmark[mp_hands.HandLandmark.PINKY_MCP].x)/2
      #
      palm_range = 100

      # annotated_image = makeCircle(annotated_image, palm_center_y,palm_center_x,palm_range)


      mp_drawing.draw_landmarks(
          annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
      hand_landmark_x = hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].x * image_width
      hand_landmark_y = hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * image_height


    cv2.imwrite(
        '/tmp/annotated_image' + str(idx) + '.png', cv2.flip(annotated_image, 1))
    cv2.imshow("img" + str(idx),cv2.flip(annotated_image,1))
    cv2.waitKey(0)

How to get the coordinates of the dots in first hand image?

Answer 1

maybe you could use cv2.convexHull and cv2.convexityDefects to find that dots. I leave you a video tutorial that I think it could be usefull: ️ CONTANDO DEDOS ✌️ (Usando defectos de convexidad) | Python - OpenCV