Skeleton tracking & playing a video in processing

Question

We used the following code in order to do skeleton tracking(head tracking) and playing a video simultaneously:

import processing.video.*;
import SimpleOpenNI.*;
import java.util.*;

SimpleOpenNI  kinect;
PImage kinectDepth;
int[] userID;

color[] userColor = new color[]{ color(255,0,0), color(0,255,0), color(0,0,255),
                                 color(255,255,0), color(255,0,255), color(0,255,255)};
 PVector headPosition = new PVector();
float headSize = 200;
float confidenceLevel = 0.5;
float confidence;
PVector confidenceVector = new PVector();
Movie movie1;

void setup()
{
  size(640, 480);
  movie1 = new Movie(this, "moon.mP4");
  kinect = new SimpleOpenNI(this);
   kinect.enableDepth();
  kinect.enableUser();
    movie1.play();
} 

void draw(){
  image(movie1, 0, 0, width, height);
  kinect.update();
  kinectDepth = kinect.depthImage();
  image(kinectDepth,0,0); 
  userID = kinect.getUsers();

  for(int i=0;i<userID.length;i++)
  {
    if(kinect.isTrackingSkeleton(userID[i]))
    {
      confidence = kinect.getJointPositionSkeleton(userID[i],SimpleOpenNI.SKEL_HEAD,confidenceVector);

      if(confidence > confidenceLevel)
      {
        // change draw color based on hand id#
        stroke(userColor[(i)]);
        // fill the ellipse with the same color
        fill(userColor[(i)]);
        // draw the rest of the body
        drawSkeleton(userID[i]);

      } 
    }
  } 
} 

/*---------------------------------------------------------------
Draw the skeleton of a tracked user.  Input is userID
----------------------------------------------------------------*/
void drawSkeleton(int userId){
  kinect.getJointPositionSkeleton(userId, SimpleOpenNI.SKEL_HEAD,headPosition);
  kinect.convertRealWorldToProjective(headPosition,headPosition);
  ellipse(headPosition.x,headPosition.y,30,30);
} 

void onNewUser(SimpleOpenNI curContext, int userId){
  println("New User Detected - userId: " + userId);
  curContext.startTrackingSkeleton(userId);
} 


void onLostUser(SimpleOpenNI curContext, int userId){
  println("User Lost - userId: " + userId);
} 

void onVisibleUser(SimpleOpenNI curContext, int userId){
} //void onVisibleUser(SimpleOpenNI curContext, int userId)


void movieEvent(Movie m) {
  m.read();
}

The following error has been generated in the log file when we tied to run the above code:

Java frames: (J=compiled Java code, j=interpreted, Vv=VM code) J 1472 SimpleOpenNI.SimpleOpenNIJNI.IntVector_size(JLSimpleOpenNI/IntVector;)J (0 bytes) @ 0x0000000002ebe695 [0x0000000002ebe640+0x55] J 1471 C1 SimpleOpenNI.IntVector.size()J (8 bytes) @ 0x0000000002ebe314 [0x0000000002ebe280+0x94] j SimpleOpenNI.SimpleOpenNI.getUsers()[I+15 J 1777 C1 skeleton_track_simpleopen_video.draw()V (159 bytes) @ 0x0000000003004ca4 [0x0000000003004600+0x6a4] j processing.core.PApplet.handleDraw()V+161 J 1769 C1 processing.awt.PSurfaceAWT$12.callDraw()V (18 bytes) @ 0x000000000300009c [0x0000000002ffff80+0x11c] j processing.core.PSurfaceNone$AnimationThread.run()V+30 v ~StubRoutines::call_stub

It is remarkable that the above code has run without any error in the absence of playing video (processing.video library).

Is it possible to assist us to find the problem in the above code?

Answer 1

It is a strange behaviour indeed, but to get to the bottom of this it might be necessary to compile the SimpleOpenNI library from source, debugging it's getUsers() method is making an invalid memory reference.

This may not be practical if you simply want to some tests and get things going. I suggest not using the getUsers() method. You could probably get away using the getNumberOfUsers():

import processing.video.*;
import SimpleOpenNI.*;
import java.util.*;

SimpleOpenNI  kinect;
PImage kinectDepth;
int[] userID;

color[] userColor = new color[]{ color(255,0,0), color(0,255,0), color(0,0,255),
                                 color(255,255,0), color(255,0,255), color(0,255,255)};
 PVector headPosition = new PVector();
float headSize = 200;
float confidenceLevel = 0.5;
float confidence;
PVector confidenceVector = new PVector();
Movie movie1;

void setup()
{
  size(640, 480);
  movie1 = new Movie(this, "moon.mP4");
  kinect = new SimpleOpenNI(this,"/Users/George/Downloads/gaf/as/CityWall/oni/test2.oni");
   kinect.enableDepth();
  kinect.enableUser();
    movie1.loop();
} 

void draw(){
  kinect.update();
  kinectDepth = kinect.depthImage();
  image(kinectDepth,0,0); 
  //userID = kinect.getUsers();

  for(int i=0;i<kinect.getNumberOfUsers();i++)
  {
    if(kinect.isTrackingSkeleton(i+1))
    {
      confidence = kinect.getJointPositionSkeleton(i+1,SimpleOpenNI.SKEL_HEAD,confidenceVector);

      if(confidence > confidenceLevel)
      {
        // change draw color based on hand id#
        stroke(userColor[(i)]);
        // fill the ellipse with the same color
        fill(userColor[(i)]);
        // draw the rest of the body
        drawSkeleton(i+1);

      } 
    }
  } 
  image(movie1, 0, 0, movie1.width/4, movie1.height/4);
} 

/*---------------------------------------------------------------
Draw the skeleton of a tracked user.  Input is userID
----------------------------------------------------------------*/
void drawSkeleton(int userId){
  kinect.getJointPositionSkeleton(userId, SimpleOpenNI.SKEL_HEAD,headPosition);
  kinect.convertRealWorldToProjective(headPosition,headPosition);
  ellipse(headPosition.x,headPosition.y,30,30);
} 

void onNewUser(SimpleOpenNI curContext, int userId){
  println("New User Detected - userId: " + userId);
  curContext.startTrackingSkeleton(userId);
} 


void onLostUser(SimpleOpenNI curContext, int userId){
  println("User Lost - userId: " + userId);
} 

void onVisibleUser(SimpleOpenNI curContext, int userId){
} //void onVisibleUser(SimpleOpenNI curContext, int userId)


void movieEvent(Movie m) {
  m.read();
}

Bare in mind this will tell you how many users are tracked, but not what their ids are. Instead of getNumberOfUsers() you could even use an int, let's 15, could the maximum number of users OpenNI supports. This would work because you're always checking if kinect.isTrackingSkeleton.