How to convert a pygame image to Open CV image?

Question

I am currently getting a real time RGB video from a Kinect2 camera using Pygame and pykinect2. I want to convert it into an open cv image so that it would be helpful for me in my further Computations.

import pykinect2
import pygame
import cv2
import ctypes
from pykinect2 import PyKinectV2
from pykinect2 import PyKinectRuntime
kinectcam = PyKinectRuntime.PyKinectRuntime(PyKinectV2.FrameSourceTypes_Color)
def draw_color_frame(frame, target_surface):
    target_surface.lock()
    address = kinectcam.surface_as_array(target_surface.get_buffer())
    ctypes.memmove(address, frame.ctypes.data, frame.size)
    del address
    target_surface.unlock()

pygame.init()
frame_surface = pygame.Surface((kinectcam.color_frame_desc.Width, kinectcam.color_frame_desc.Height), 0, 32)
clock = pygame.time.Clock()
pygame.display.set_caption("Kinect View")
infoObject = pygame.display.Info()
screen = pygame.display.set_mode((infoObject.current_w >> 1, infoObject.current_h >> 1),
                            pygame.HWSURFACE|pygame.DOUBLEBUF|pygame.RESIZABLE, 32)
clock = pygame.time.Clock()

done = False
while not done:
    for event in pygame.event.get(): # User did something
        if event.type == pygame.QUIT: # If user clicked close
            done = True # Flag that we are done so we exit this loop

        elif event.type == pygame.VIDEORESIZE: # window resized
            screen = pygame.display.set_mode(event.dict['size'], 
                                               pygame.HWSURFACE|pygame.DOUBLEBUF|pygame.RESIZABLE, 32)

    if kinectcam.has_new_color_frame():
        frame = kinectcam.get_last_color_frame()
        draw_color_frame(frame, frame_surface)
        frame = None   
    h_to_w = float(frame_surface.get_height()) / frame_surface.get_width()
    target_height = int(h_to_w * screen.get_width())
    surface_to_draw = pygame.transform.scale(frame_surface, (screen.get_width(), target_height));
    screen.blit(surface_to_draw, (0,0))
    surface_to_draw = None
    pygame.display.update()
    pygame.display.flip()
    clock.tick(60)
pygame.quit()
kinectcam.close()

Answer 1

Using pyKinect2 library, you can create an acquisitionClass.py which defines different methods and properties required for Kinect frame processing. Then call the acquisitionClass.py which contains the get_color_frame() that does the conversion, from your main script (Run.py) to use and display the converted color frame to opencv frame as follows:

Note: I'm answering this assuming you do not want to use the pyGame library but OpenCV instead.

Run.py

import cv2
from pykinect2 import PyKinectV2
from pykinect2.PyKinectV2 import *
from pykinect2 import PyKinectRuntime
from acquisitionKinect import AcquisitionKinect
from frame import Frame

if __name__ == '__main__':

    kinect = AcquisitionKinect()
    frame = Frame()

    while True:
        kinect.get_frame(frame)
        kinect.get_color_frame()
        image = kinect._frameRGB
        #OpenCv uses RGB image, kinect returns type RGBA, remove extra dim.
        image = cv2.cvtColor(image, cv2.COLOR_RGBA2RGB) 

        if not image is None:
            cv2.imshow("Output-Keypoints",image) 

        cv2.waitKey(30)
        if cv2.waitKey(1) & 0xFF == ord('q'):
           break

acquisitionKinect.py

import ctypes
import _ctypes
import sys

if sys.hexversion >= 0x03000000:
    import _thread as thread
else:
    import thread

class AcquisitionKinect():
    #Create a constructor to initialize different types of array and frame objects
    def __init__(self, resolution_mode=1.0):
    self.resolution_mode = resolution_mode

    self._done = False

    # Kinect runtime object, we want only color and body frames
    self._kinect = PyKinectRuntime.PyKinectRuntime(PyKinectV2.FrameSourceTypes_Color | PyKinectV2.FrameSourceTypes_Body | PyKinectV2.FrameSourceTypes_Depth)

    # here we will store skeleton data
    self._bodies = None
    self.body_tracked = False
    self.joint_points = np.array([])
    self.joint_points3D = np.array([])
    self.joint_points_RGB = np.array([])
    self.joint_state = np.array([])

    self._frameRGB = None
    self._frameDepth = None
    self._frameDepthQuantized = None
    self._frameSkeleton = None
    self.frameNum = 0

    def get_frame(self, frame):
        self.acquireFrame()
        frame.ts = int(round(time.time() * 1000))

        self.frameNum += 1

        frame.frameRGB = self._frameRGB
        frame.frameDepth = self._frameDepth
        frame.frameDepthQuantized = self._frameDepthQuantized
        frame.frameSkeleton = self._frameSkeleton

    #Get a color frame object
    def get_color_frame(self):
       self._frameRGB = self._kinect.get_last_color_frame()
       self._frameRGB = self._frameRGB.reshape((1080, 1920,-1)).astype(np.uint8)
       self._frameRGB = cv2.resize(self._frameRGB, (0,0), fx=1/self.resolution_mode, fy=1/self.resolution_mode)

    #Acquire the type of frame required
    def acquireFrame(self):
        if self._kinect.has_new_color_frame():
            self.get_color_frame()

    def close(self):
        self._kinect.close()
        self._frameDepth = None
        self._frameRGB = None
        self._frameSkeleton = None

Frame.py

class Frame():
    frameRGB = None
    frameDepth = None
    frameDepthQuantized = None
    frameSkeleton = None
    frame_num = 0
    shoulder_orientation_euler = None
    shoulder_orientation_quat = None

Answer 2

I assume your are trying to convert the image you are blitting (surface_to_draw). To convert pygame.Surface object to opencv image:

#  create a copy of the surface
view = pygame.surfarray.array3d(surface_to_draw)

#  convert from (width, height, channel) to (height, width, channel)
view = view.transpose([1, 0, 2])

#  convert from rgb to bgr
img_bgr = cv2.cvtColor(view, cv2.COLOR_RGB2BGR)

Update: I also assumed your pygame image is color image.