Error trying to predict in TensorFlow.js and react

31 Views Asked by At

I have an error when I try to predict with an LSTM model in React with TensorFlow.js. The purpose of the React project is to translate sign language and I use media pipe holistic to detect key points of the face, hands and pose. To extract the key point I create a function that uses the module "@d4c/numjs" to generate an Ndarray. With that, I generate an array that allows me to count frames and predict, but when I try to predict this error appears:

ERROR Error when checking model: the Array of Tensors that you are passing to your model is not the size the model expected. I expected to see 1 Tensor(s), but instead got 0 Tensor (s**).

ValueError@http://localhost:3000/static/js/bundle.js:126537:5 checkInputData@http://localhost:3000/static/js/bundle.js:124254:13 predict@http://localhost:3000/static/js/bundle.js:124844:19 predict@http://localhost:3000/static/js/bundle.js:138492:23 onResults@http://localhost:3000/static/js/bundle.js:152:27 ./node_modules/@mediapipe/holistic/holistic.js/</Vd/onResults/a.I<@http://localhost:3000/static/js/bundle.js:7788:15

I don't know if the problem is that I use the module "@d4c/numjs" for the project or if I missing a reshape of some sort. The LSTM input layers is : lstm_input (InputLayer) [[null,15,1662]]

The error occurs in this particular lines:

 kp_sequence.push(extractKeyPoints(results));

    if (kp_sequence.length > 15 && thereHand(results)) {
      count_frame++;
    } else {
        if (count_frame >= 5) {
             const kpArray = nj.stack(kp_sequence.slice(-15)).reshape([1, 15, -1]);
             const res = model.predict(kpArray).tolist()[0];

            if (res.indexOf(Math.max(...res)) > 0.7) {
              console.log("prediciendo");
              const sent = actions[res.indexOf(Math.max(...res))];
              sentence.unshift(sent);
              [sentence, repe_sent] = formatSentences(sent, sentence, repe_sent);
              } 
            count_frame = 0;
            kp_sequence = [];
        }
    }

I posted the full code below, I hope you can help me with this one, I'm new to JavaScript. Thanks

import * as tf from '@tensorflow/tfjs';
import './App.css';
import React, {useEffect, useRef} from 'react';
import Webcam from "react-webcam";
import * as cam from "@mediapipe/camera_utils";
import {FACEMESH_TESSELATION,HAND_CONNECTIONS,POSE_CONNECTIONS, Holistic} from '@mediapipe/holistic';
import './App.css';
import nj from "@d4c/numjs";  
import { log } from '@d4c/numjs/build/main/lib';

function App() {
  let model = undefined;
  var count_frame=0;
  var repe_sent=1;
  var kp_sequence=[];
  var sentence = [];

  const webcamRef = useRef(null);
  const canvasRef = useRef(null);
  const connect = window.drawConnectors;
  var camera = null;

  async function loadModel(){

    model = await tf.loadLayersModel('url');
    model.summary();
  }
  loadModel();


function formatSentences(sent, sentence, repe_sent) {
    if (sentence.length > 1) {
        if (sentence[1].includes(sent)) {
          repe_sent++;
            sentence.shift();
            sentence[0] = `${sent} (x${repe_sent})`;
        } else {
          repe_sent = 1;
        }
    }
    return [sentence, repe_sent];
}


  function thereHand(results) {
    return !!results.leftHandLandmarks || !!results.rightHandLandmarks;
}

  function extractKeyPoints(results) {
    let pose = [];

    if (results.poseLandmarks) {
        pose = nj.array(results.poseLandmarks.map(res => [res.x, res.y, res.z, res.visibility])).flatten().tolist();
    } else {
        pose = nj.zeros(33 * 4).tolist();
    }

    let face = [];
    if (results.faceLandmarks) {
        face = nj.array(results.faceLandmarks.map(res => [res.x, res.y, res.z])).flatten().tolist();
    } else {
        face = nj.zeros(468 * 3).tolist();
    }

    let lh = [];
    if (results.leftHandLandmarks) {
        lh = nj.array(results.leftHandLandmarks.map(res => [res.x, res.y, res.z])).flatten().tolist();
    } else {
        lh = nj.zeros(21 * 3).tolist();
    }

    let rh = [];
    if (results.rightHandLandmarks) {
        rh = nj.array(results.rightHandLandmarks.map(res => [res.x, res.y, res.z])).flatten().tolist();
    } else {
        rh = nj.zeros(21 * 3).tolist();
    }
   

    return nj.concatenate([face,pose, lh, rh]);
}

  function onResults(results){
 
    var actions = ['como estas','gusto en verte','hola','tanto tiempo'];

    const videoWidth = webcamRef.current.video.videoWidth;
    const videoHeight = webcamRef.current.video.videoHeight;

    // Set canvas width
    canvasRef.current.width = videoWidth;
    canvasRef.current.height = videoHeight;

    const canvasElement = canvasRef.current;
    const canvasCtx = canvasElement.getContext("2d");

    canvasCtx.save();
    canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
    canvasCtx.drawImage(results.image, 0, 0,
                        canvasElement.width, canvasElement.height);
  
    canvasCtx.globalCompositeOperation = 'source-over';
    connect(canvasCtx, results.poseLandmarks, POSE_CONNECTIONS,
                   {color: '#00FF00', lineWidth: 4});
    connect(canvasCtx, results.poseLandmarks,
                  {color: '#FF0000', lineWidth: 2});
    connect(canvasCtx, results.faceLandmarks,FACEMESH_TESSELATION,
                   {color: '#C0C0C070', lineWidth: 1});
    connect(canvasCtx, results.leftHandLandmarks, HAND_CONNECTIONS,
                   {color: '#CC0000', lineWidth: 5});
    connect(canvasCtx, results.leftHandLandmarks,
                  {color: '#00FF00', lineWidth: 2});
    connect(canvasCtx, results.rightHandLandmarks, HAND_CONNECTIONS,
                   {color: '#00CC00', lineWidth: 5});
    connect(canvasCtx, results.rightHandLandmarks,
                  {color: '#FF0000', lineWidth: 2});
    canvasCtx.restore();

    kp_sequence.push(extractKeyPoints(results));

    if (kp_sequence.length > 15 && thereHand(results)) {
      count_frame++;
    } else {
        if (count_frame >= 5) {
             const kpArray = nj.stack(kp_sequence.slice(-15)).reshape([1, 15, -1]);
             const res = model.predict(kpArray).tolist()[0];

            if (res.indexOf(Math.max(...res)) > 0.7) {
              console.log("prediciendo");
              const sent = actions[res.indexOf(Math.max(...res))];
              sentence.unshift(sent);
              [sentence, repe_sent] = formatSentences(sent, sentence, repe_sent);
              } 

            count_frame = 0;
            kp_sequence = [];
        }
    }

  }



  useEffect(()=>{
    const holistic = new Holistic({
      locateFile: (file) => {
      return `https://cdn.jsdelivr.net/npm/@mediapipe/holistic/${file}`;
    }
  })

  holistic.setOptions({
    modelComplexity: 0,
    smoothLandmarks: true,
    enableSegmentation: true,
    smoothSegmentation: true,
    refineFaceLandmarks: false,
    minDetectionConfidence: 0.5,
    minTrackingConfidence: 0.5
  })
holistic.onResults(onResults);

if(typeof webcamRef.current!=="undefined" && webcamRef.current!==null){
    camera = new cam.Camera(webcamRef.current.video,{
      onFrame:async()=>{
        await holistic.send({image:webcamRef.current.video})
      },
      width:640,
      height:480
    });
    camera.start() 
}
})
return (
    <div className="App">
        <Webcam 
        ref={webcamRef}
          style= {{
            position:'absolute',
            marginRight:'auto',
            marginLeft:'auto',
            left:0,
            right:0,
            textAlign:'center',
            zIndex:9,
            width:640,
            height:480,
          }}
        />
        <canvas 
        ref={canvasRef}
          style= {{
            position:'absolute',
            marginRight:'auto',
            marginLeft:'auto',
            left:0,
            right:0,
            textAlign:'center',
            zIndex:9,
            width:640,
            height:480,
          }}>

        </canvas>
    </div>
  );
}
export default App;
0

There are 0 best solutions below