create bounding box of detected object to the 3D point cloud

Question

I am trying now to create the bounding box of detected object to the 3D point cloud.

I could detect the object by using yolov5 and build the point cloud from RGB and Depth image by using the camera d435i, but I couldn't recreate the bounding box from 2d image to point cloud, the position of bounding box is always wrong!

How I can do that? anyone can give me suggestion?

here are the images of 2d with bounding boxes and point cloud

I did try to align the color and depth image and use the rs2_deproject_pixel_to_point to find the position of bounding box in 3d cloud point, but it always show me the wrong position!

def deproject_bounding_box(bounding_box, depth_image, intrinsics):
    # Unpack the bounding box corners
    xmin, xmax, ymin, ymax = bounding_box
    depth_scale = 0.001  # depth scale factor

    # Get the depth at each corner and apply the depth scale
    z1 = depth_image[int(ymin), int(xmin)] * depth_scale
    z2 = depth_image[int(ymax), int(xmin)] * depth_scale
    z3 = depth_image[int(ymin), int(xmax)] * depth_scale
    z4 = depth_image[int(ymax), int(xmax)] * depth_scale
    
    # Deproject each corner to 3D space
    x1, y1, _ = rs.rs2_deproject_pixel_to_point(intrinsics, [xmin, ymin], z1)
    x2, y2, _ = rs.rs2_deproject_pixel_to_point(intrinsics, [xmin, ymax], z2)
    x3, y3, _ = rs.rs2_deproject_pixel_to_point(intrinsics, [xmax, ymin], z3)
    x4, y4, _ = rs.rs2_deproject_pixel_to_point(intrinsics, [xmax, ymax], z4)

    # Return the 3D coordinates of the bounding box corners
    return [(x1, y1, z1), (x2, y2, z2), (x3, y3, z3), (x4, y4, z4)]

# Get the 3D coordinates of the bounding box corners
bounding_boxes_3d = []

# Loop over the bounding boxes
for bounding_box in bounding_boxes:
    # Deproject the bounding box to 3D space
    bounding_box_3d = deproject_bounding_box(bounding_box, depth_image, intrinsics)
    
    # Add the 3D bounding box to the list
    bounding_boxes_3d.append(bounding_box_3d)

# Create a 3D bounding box from the corners
lines = [
    [0, 1], [1, 3], [3, 2], [2, 0],  # front face
    [4, 5], [5, 7], [7, 6], [6, 4],  # back face
    [0, 4], [1, 5], [2, 6], [3, 7]   # connecting edges
]
colors = [[1, 0, 0] for _ in range(len(lines))]  # red color for all lines
line_set = o3d.geometry.LineSet(
    points=o3d.utility.Vector3dVector(bounding_box_3d),
    lines=o3d.utility.Vector2iVector(lines),
)
line_set.colors = o3d.utility.Vector3dVector(colors)

# Plot the point cloud and the bounding box
o3d.visualization.draw_geometries([pcd, line_set])