Moving a parent object by a child object VR Oculus Unity

Question

I am trying to move an object by moving a handle contained in the same prefab.

the below code seems to be working except the board keeps jumping to a location behind me and then back, seemingly every frame. This is the closest I have got to it working correctly any advice would be helpful

using Oculus.Interaction;
using Oculus.Interaction.HandGrab;
using System.Linq;
using UnityEngine;

public class MoveBoardWithHandle : MonoBehaviour
{
    private HandGrabInteractable interactor;
    private Rigidbody board;
    private Rigidbody handle;
    Quaternion initRoataion;

    void Start()
    {
        interactor = GetComponent<HandGrabInteractable>();
        var rigidBodies = GetComponentsInParent<Rigidbody>();
        handle = rigidBodies.FirstOrDefault(x => x.name == "Handle");
        board = rigidBodies.FirstOrDefault(x => x.name == "ChessBoard");
        initRoataion = handle.rotation;
    }

    bool reSetHandleRotation = false;
    
    void Update()
    {

        // if (board == null) Debug.Log($"{nameof(board)} is null");
        // if (interactor == null) Debug.Log($"{nameof(interactor)} is null");
        // if (handle == null) Debug.Log($"{nameof(handle)} is null");

        if(interactor.State == InteractableState.Select)
        {
            Vector3 boardOffset = board.transform.position - handle.transform.position;
            board.transform.position = handle.transform.position -= boardOffset;
            reSetHandleRotation = true;
        }
        else
        {
            if(reSetHandleRotation)
            {
                handle.transform.rotation = initRoataion;
                reSetHandleRotation = false;
            }
        }
        
    }

}

Parent inspector

Child inspector