Save Kinect RGB and Depth Images using Pykinect and Open CV

Question

I am using PyKinect with OpenCV in windows to save RGB and kinect Depth images. Which works fine to display both RGB and Depth images. Frame rates for both RGB and Depth images are 30FPS.

I want to save the RGB images in .jpg format and Depth images in .xml format. which also works fine.

But My problem is when I start saving the RGB and Depth images I am not getting the same frame rate. Like if I save the images for 10 seconds I get 300 XML files, but for the same time I get 100 JPG images.

But if I execute the same program to save the RGB images only (comment out the depth image part) for 10 seconds I get 300 JPG files. So I believe this is a performance issue which can be improved. Any suggestion to improve the performance is appreciated.

Here is my Approach:

from pykinect import nui
from pykinect.nui import JointId
import numpy
from numpy import * 
import cv2
import datetime


import os
import os.path
from os.path import join as pjoin
from pykinect.nui import SkeletonTrackingState

current_directory = os.getcwd()

kinect = nui.Runtime()
save_image = False


def getColorImage(frame):
    height,width = frame.image.height,frame.image.width #get width and height of the images 
    rgb = numpy.empty((height,width,4),numpy.uint8) 
    frame.image.copy_bits(rgb.ctypes.data) #copy the bit of the image to the aray   
    cv2.imshow('KINECT Video Stream', rgb) # display the image
    # save to Folder
    folder = 'RGB_images'+'\\s'+ subject_id +'_a' +activity_id
    if not os.path.exists(folder):
        os.makedirs(folder)
    path = current_directory+str('\\')+folder
    image_name = 'color_image_'+str(frame.frame_number)+'.jpg'
    if save_image:
        cv2.imwrite(os.path.join(path,image_name), rgb)


def getDepthImage(frame):
    height,width = frame.image.height,frame.image.width #get frame height and width
    depth_frame = saveDepthImageData(frame.frame_number)
    depth = numpy.empty((height,width,1),numpy.uint8)   
    arr2d = (depth >> 3) & 4095 
    arr2d >>= 4

    frame.image.copy_bits(arr2d.ctypes.data)

    cv2.imshow('KINECT depth Stream', arr2d)

    folder = 'Depth_data'+'\\s'+ subject_id +'_a' +activity_id
    if not os.path.exists(folder):
        os.makedirs(folder)     
    path = current_directory+str('\\')+folder
    file_name = 'depth_image_'+str(frame.frame_number)+'.xml'
    file_to_save = os.path.join(path,file_name)
    if save_image:      
        f = cv2.FileStorage(file_to_save,flags=1)
        f.write("depthImg",arr2d)
        f.release() #close the file


def main():
    global save_image
    global subject_id
    global activity_id


    subject_id = raw_input("Subject id : ")
    activity_id = raw_input("Activity id : ")
    print "Press t to start saving"
    print "Press Esc to quit"

    kinect.video_frame_ready += getColorImage
    kinect.video_stream.open(nui.ImageStreamType.Video, 2,nui.ImageResolution.Resolution640x480,nui.ImageType.Color)
    cv2.namedWindow('KINECT Video Stream', cv2.WINDOW_AUTOSIZE)

    kinect.depth_frame_ready += getDepthImage
    kinect.depth_stream.open(nui.ImageStreamType.Depth, 2, nui.ImageResolution.Resolution320x240, nui.ImageType.Depth)
    cv2.namedWindow('KINECT depth Stream', cv2.WINDOW_AUTOSIZE)

    while True:
        key = cv2.waitKey(0)        
        if key == 116:
            save_image = not save_image
        if key == 27:           
            cv2.destroyAllWindows()
            kinect.close()
            break   


if __name__ == '__main__':
    main()

Answer 1

Converting the depth image to XML will massively increase its size. The Kinect depth image is 640x480 IIRC, and instead of a single 11-bit value for each pixel, the XML will use more like 10 bytes per pixel on average, that means ~3.5 MB per frame, and then your code would have to write out close to 100 MB/s just in terms of disk I/O (not counting the whole XML conversion).

I would suggest using another format for saving the depth images, probably something simple like PGM would be an idea.