DSI-Bench — Gemma4-31B — sample reverse__row_000348

# Select a few frames that are actually in InputImages for verification
# Use indices from InputImages.frame_indices
sample_indices = [0, len(InputImages)//2, len(InputImages)-1]
vis_frames = []

for idx in sample_indices:
    img = InputImages[idx]
    fi = img.frame_index
    # Get the mask for this absolute frame index
    mask_vis = seg.visualize(fi)
    vis_frames.append(img)
    vis_frames.append(mask_vis)

show(vis_frames)

 [image: show_0001] 

# Reconstruct the 3D scene using the available InputImages
recon = tools.Reconstruct.Reconstruct(InputImages)

# Verify reconstruction by rendering a BEV map with the boat masks
# This will give a qualitative view of the boat's movement
bev_vis = recon.render_bev(masks=seg)
show(bev_vis)

 [image: show_0002] 

# 1. Calculate displacement vector D using frames present in recon
fi_start = recon.frame_indices[0]
fi_end = recon.frame_indices[-1]

c_start = seg.get_centroid_3d(recon, frame=fi_start, object='boat')
c_end = seg.get_centroid_3d(recon, frame=fi_end, object='boat')

print(f"Start centroid (frame {fi_start}): {c_start}")
print(f"End centroid (frame {fi_end}): {c_end}")

displacement = c_end - c_start
print(f"Displacement vector: {displacement}")

# 2. Establish initial orientation vector F (Forward)
img_start = InputImages[0]
W, H = img_start.width, img_start.height

# Locate bow and stern
bow_resp = vlm.locate(img_start, "Give the (x, y) center coordinates in 0-1000 normalized scale for the bow (front tip) of the boat. Reply with ONLY the numbers.")
stern_resp = vlm.locate(img_start, "Give the (x, y) center coordinates in 0-1000 normalized scale for the stern (back part) of the boat. Reply with ONLY the numbers.")

print(f"Bow response: {bow_resp}")
print(f"Stern response: {stern_resp}")

def parse_coords(resp):
    # Handle potential "Not visible" or other text
    try:
        return [float(x) for x in resp.replace(',', ' ').split()]
    except:
        return None

bow_norm = parse_coords(bow_resp)
stern_norm = parse_coords(stern_resp)

if bow_norm and stern_norm:
    bow_px = tools.Geometry.normalized_to_pixel(bow_norm, W, H)
    stern_px = tools.Geometry.normalized_to_pixel(stern_norm, W, H)
    
    # Get 3D points under these pixels for the first frame
    # Use get_masked_points and find the point closest to the pixel, 
    # or more simply, use the depth map if available.
    # Since we have a mask, we can find the 3D point at that pixel.
    points_3d = recon.points[fi_start]
    # bow_px is (x, y)
    p_bow = points_3d[int(bow_px[1]), int(bow_px[0])]
    p_stern = points_3d[int(stern_px[1]), int(stern_px[0])]
    
    print(f"Bow 3D: {p_bow}")
    print(f"Stern 3D: {p_stern}")
    
    # Forward vector F = Bow - Stern
    forward_vec = p_bow - p_stern
    forward_vec /= np.linalg.norm(forward_vec)
    print(f"Normalized Forward Vector: {forward_vec}")
    
    # Right vector R = Forward x Up (assuming Y is up)
    up_vec = np.array([0, 1, 0])
    right_vec = np.cross(forward_vec, up_vec)
    right_vec /= np.linalg.norm(right_vec)
    print(f"Normalized Right Vector: {right_vec}")
    
    # 3. Project displacement onto local axes
    dot_fwd = np.dot(displacement, forward_vec)
    dot_right = np.dot(displacement, right_vec)
    
    print(f"Dot product with Forward: {dot_fwd}")
    print(f"Dot product with Right: {dot_right}")
    
    if abs(dot_fwd) > abs(dot_right):
        direction = "Forward" if dot_fwd > 0 else "Backward"
    else:
        direction = "Right" if dot_right > 0 else "Left"
    print(f"Primary motion direction: {direction}")
else:
    print("Failed to locate bow or stern.")

Start centroid (frame 0): [ 0.52666974  0.72795874 -5.4799547 ]
End centroid (frame 101): [ 1.0063361   0.54890156 -4.863613  ]
Displacement vector: [ 0.47966635 -0.17905718  0.6163416 ]
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the (x, y) center coordinates in 0-1000 normalized scale for the bow (front tip) of the boat. Reply with ONLY the numbers.
[VLM A | locate] 575 550
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the (x, y) center coordinates in 0-1000 normalized scale for the stern (back part) of the boat. Reply with ONLY the numbers.
[VLM A | locate] 925 635
Bow response: 575 550
Stern response: 925 635
Bow 3D: [-0.06975297  0.9009471  -6.7074294 ]
Stern 3D: [ 0.5958265   0.84380686 -5.229158  ]
Normalized Forward Vector: [-0.4102932   0.03522382 -0.9112732 ]
Normalized Right Vector: [ 0.91183901  0.         -0.41054795]
Dot product with Forward: -0.7647665143013
Dot product with Right: 0.18434071431321558
Primary motion direction: Backward

ReturnAnswer("B")

[ReturnAnswer] Answer submitted: B