Is it possible to move a 3D object using the position of hand obtained from camera UNITY?

Question

[![enter image description here][1]][1]I want to create a unity program where I'm able to provide a 3D model of the hand on the screen based on the input from a camera. I'm using HandPoseBarracuda for detecting and obtaining the position of the hand in real-time. Is there any way to access the skeletal points of the rigged model through the C# code in Unity?

How can I map the position to the rigged model? [1]: https://i.stack.imgur.com/IHa3X.png

Answer 1

I looked at the code now. joint balls and bones are updated in the Animator game object's attached script HandAnimator.cs in LateUpdate() function.

you can simply get a list of all ball positions in the loop like this.

using UnityEngine;
using UnityEngine.UI;
**using System.Collections.Generic;**

namespace MediaPipe.HandPose
{

    public sealed class HandAnimator : MonoBehaviour
    {
        #region Editable attributes

        [SerializeField] WebcamInput _webcam = null;
        [SerializeField] ResourceSet _resources = null;
        [SerializeField] bool _useAsyncReadback = true;
        [Space]
        [SerializeField] Mesh _jointMesh = null;
        [SerializeField] Mesh _boneMesh = null;
        [Space]
        [SerializeField] Material _jointMaterial = null;
        [SerializeField] Material _boneMaterial = null;
        [Space]
        [SerializeField] RawImage _monitorUI = null;

        public List<Vector3> positions;

        #endregion

        #region Private members

        HandPipeline _pipeline;

        static readonly (int, int)[] BonePairs =
        {
        (0, 1), (1, 2), (1, 2), (2, 3), (3, 4),     // Thumb
        (5, 6), (6, 7), (7, 8),                     // Index finger
        (9, 10), (10, 11), (11, 12),                // Middle finger
        (13, 14), (14, 15), (15, 16),               // Ring finger
        (17, 18), (18, 19), (19, 20),               // Pinky
        (0, 17), (2, 5), (5, 9), (9, 13), (13, 17)  // Palm
    };

        Matrix4x4 CalculateJointXform(Vector3 pos)
          => Matrix4x4.TRS(pos, Quaternion.identity, Vector3.one * 0.07f);

        Matrix4x4 CalculateBoneXform(Vector3 p1, Vector3 p2)
        {
            var length = Vector3.Distance(p1, p2) / 2;
            var radius = 0.03f;

            var center = (p1 + p2) / 2;
            var rotation = Quaternion.FromToRotation(Vector3.up, p2 - p1);
            var scale = new Vector3(radius, length, radius);

            return Matrix4x4.TRS(center, rotation, scale);
        }

        #endregion

        #region MonoBehaviour implementation

        void Start()
          => _pipeline = new HandPipeline(_resources);

        void OnDestroy()
          => _pipeline.Dispose();

        void LateUpdate()
        {
            // Feed the input image to the Hand pose pipeline.
            _pipeline.UseAsyncReadback = _useAsyncReadback;
            _pipeline.ProcessImage(_webcam.Texture);

            var layer = gameObject.layer;

            **positions = new List<Vector3>();**
            // Joint balls
        for (var i = 0; i < HandPipeline.KeyPointCount; i++)
            {
                **Vector3 pos = _pipeline.GetKeyPoint(i);
                var xform = CalculateJointXform(pos);
                Graphics.DrawMesh(_jointMesh, xform, _jointMaterial, layer);
                positions.Add(pos);**
            }

            // Bones
            foreach (var pair in BonePairs)
            {
                var p1 = _pipeline.GetKeyPoint(pair.Item1);
                var p2 = _pipeline.GetKeyPoint(pair.Item2);
                var xform = CalculateBoneXform(p1, p2);
                Graphics.DrawMesh(_boneMesh, xform, _boneMaterial, layer);
            }

            // UI update
            _monitorUI.texture = _webcam.Texture;
        }

        #endregion
    }

} // namespace MediaPipe.HandPose

Finally select each position you want in the list and update your special object's position with the selected position.