MMSI — Gemma4-31B — sample 840

# Locate the door and socket in both images to ensure we have the best masks
# In InputImages[0], the door is prominent.
# In InputImages[1], the socket (white object on the left wall) is more visible.

door_box_0 = vlm.locate(InputImages[0], "Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the door. Reply with ONLY the numbers.")
socket_box_0 = vlm.locate(InputImages[0], "Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the electrical socket on the wall. Reply with ONLY the numbers.")

door_box_1 = vlm.locate(InputImages[1], "Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the door. Reply with ONLY the numbers.")
socket_box_1 = vlm.locate(InputImages[1], "Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the electrical socket on the wall. Reply with ONLY the numbers.")

print(f"Door box 0: {door_box_0}")
print(f"Socket box 0: {socket_box_0}")
print(f"Door box 1: {door_box_1}")
print(f"Socket box 1: {socket_box_1}")

# Convert to pixels for InputImages[1] since the socket is clearer there
W, H = InputImages[1].width, InputImages[1].height
try:
    # Parse coordinates from the VLM strings
    def parse_box(box_str):
        return [int(x) for x in box_str.replace(',', ' ').split()]

    d_box_1 = tools.Geometry.normalized_to_pixel(parse_box(door_box_1), W, H)
    s_box_1 = tools.Geometry.normalized_to_pixel(parse_box(socket_box_1), W, H)

    # Segment
    seg_door = tools.SAM3.segment_image_by_box(InputImages[1], d_box_1, "door")
    seg_socket = tools.SAM3.segment_image_by_box(InputImages[1], s_box_1, "socket")

    # Combine masks into one PerFrameMask for convenience if possible, 
    # but SAM3 returns separate PerFrameMask objects. I'll just keep them.
    
    # Visual verification
    fi = InputImages[1].frame_index
    show([InputImages[1], seg_door.visualize(fi), seg_socket.visualize(fi)])

except Exception as e:
    print(f"Error during segmentation: {e}")

[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the door. Reply with ONLY the numbers.
[VLM A | locate] 440 0 1000 575
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the electrical socket on the wall. Reply with ONLY the numbers.
[VLM A | locate] Not visible
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the door. Reply with ONLY the numbers.
[VLM A | locate] Not visible
No door is present in the image.
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw ins

# Door is in InputImages[0], Socket is in InputImages[1]

# Process Door in InputImages[0]
W0, H0 = InputImages[0].width, InputImages[0].height
door_box_0_coords = [int(x) for x in door_box_0.replace(',', ' ').split()]
d_box_0_px = tools.Geometry.normalized_to_pixel(door_box_0_coords, W0, H0)
seg_door = tools.SAM3.segment_image_by_box(InputImages[0], d_box_0_px, "door")

# Process Socket in InputImages[1]
W1, H1 = InputImages[1].width, InputImages[1].height
socket_box_1_coords = [int(x) for x in socket_box_1.replace(',', ' ').split()]
s_box_1_px = tools.Geometry.normalized_to_pixel(socket_box_1_coords, W1, H1)
seg_socket = tools.SAM3.segment_image_by_box(InputImages[1], s_box_1_px, "socket")

# Visual verification
fi0 = InputImages[0].frame_index
fi1 = InputImages[1].frame_index
show([
    InputImages[0], seg_door.visualize(fi0),
    InputImages[1], seg_socket.visualize(fi1)
])

# Programmatic check for mask area
door_area = seg_door.get_mask(frame=fi0, object=0).sum()
socket_area = seg_socket.get_mask(frame=fi1, object=0).sum()
print(f"Door mask area: {door_area} pixels")
print(f"Socket mask area: {socket_area} pixels")

 [image: show_0001] 
Door mask area: 73478 pixels
Socket mask area: 3505 pixels

# Perform 3D reconstruction
recon = tools.Reconstruct.Reconstruct(InputImages)

# Get 3D centroids
# Door was segmented in fi0
door_centroid = seg_door.get_centroid_3d(recon, frame=fi0, object=0)
# Socket was segmented in fi1
socket_centroid = seg_socket.get_centroid_3d(recon, frame=fi1, object=0)

print(f"Door 3D centroid: {door_centroid}")
print(f"Socket 3D centroid: {socket_centroid}")

# Sanity check with BEV
# To render BEV, I need a combined mask or just pass a list. 
# Since they are separate PerFrameMask objects, I'll create a simple array for the BEV.
# Or I can just use the ego_trajectory to see the camera movement.
# Actually, I can just use the centroids directly for the math.
# Let's render a BEV with just the socket to see where the camera is.
bev_socket = recon.render_bev(masks=seg_socket, labels=["socket"])
show(bev_socket)

Door 3D centroid: [ 0.6744245  0.9994168 -1.5981216]
Socket 3D centroid: [-1.219678    0.21966341 -0.18279685]
 [image: show_0002] 

import numpy as np

# 1. Extract points for the socket to find the wall normal
socket_points = seg_socket.get_masked_points(recon, frame=fi1, object=0)

# Fit a plane to the socket points using SVD to find the normal
# Center the points
centroid_pts = np.median(socket_points, axis=0)
centered_pts = socket_points - centroid_pts

# SVD to find the normal (the singular vector corresponding to the smallest singular value)
_, _, vh = np.linalg.svd(centered_pts)
normal = vh[-1] # This is the normal to the plane

# Ensure the normal points "into the room"
# In InputImages[1], the socket is on the left wall.
# Looking at the BEV, the socket is at x ~ -1.2, camera at 0.
# The wall is to the left of the camera, so the normal should point roughly in the +X direction.
if normal[0] < 0:
    normal = -normal

print(f"Socket normal (Front vector): {normal}")

# 2. Define the local coordinate system for the socket
f_socket = normal / np.linalg.norm(normal)
up = np.array([0, 1, 0])
r_socket = np.cross(f_socket, up)
r_socket /= np.linalg.norm(r_socket)

# 3. Calculate the relative vector from socket to door
v_socket_to_door = door_centroid - socket_centroid
print(f"Vector socket -> door: {v_socket_to_door}")

# 4. Project the relative vector onto the local axes
depth = np.dot(v_socket_to_door, f_socket)
lateral = np.dot(v_socket_to_door, r_socket)

print(f"Depth (Front/Rear): {depth:.3f} (Positive = Front)")
print(f"Lateral (Right/Left): {lateral:.3f} (Positive = Right)")

# 5. Determine the quadrant
if depth > 0:
    direction_front_rear = "Front"
else:
    direction_front_rear = "Rear"

if lateral > 0:
    direction_left_right = "right"
else:
    direction_left_right = "left"

result = f"{direction_front_rear} {direction_left_right}"
print(f"Final Relative Direction: {result}")

# Map to options
# A: Front left, B: Front right, C: Rear left, D: Rear right
mapping = {
    "Front left": "A",
    "Front right": "B",
    "Rear left": "C",
    "Rear right": "D"
}
print(f"Option: {mapping.get(result)}")

Socket normal (Front vector): [0.9980708  0.04413939 0.04366319]
Vector socket -> door: [ 1.8941026   0.77975345 -1.4153248 ]
Depth (Front/Rear): 1.863 (Positive = Front)
Lateral (Right/Left): -1.497 (Positive = Right)
Final Relative Direction: Front left
Option: A

ReturnAnswer("A")

[ReturnAnswer] Answer submitted: A