MMSI — Gemma4-31B — sample 433

# Check for existence of objects first
imac_existence = tools.SAM3.is_object_exist(InputImages, "iMac screen")
blackboard_existence = tools.SAM3.is_object_exist(InputImages, "blackboard")

print(f"iMac existence: {imac_existence}")
print(f"Blackboard existence: {blackboard_existence}")

# Segment iMac screen in frames where it exists
imac_masks = []
for i, exists in enumerate(imac_existence["exists"]):
    if exists:
        mask = tools.SAM3.segment_image_by_text(InputImages[i], "iMac screen", label="iMac screen")
        imac_masks.append(mask)

# Segment blackboard in frames where it exists
blackboard_masks = []
for i, exists in enumerate(blackboard_existence["exists"]):
    if exists:
        mask = tools.SAM3.segment_image_by_text(InputImages[i], "blackboard", label="blackboard")
        blackboard_masks.append(mask)

# Visual verification of the masks
# Since we have separate PerFrameMask objects, let's visualize them on the frames they were found
vis_list = []
if imac_masks:
    fi = imac_masks[0].frame_indices[0]
    vis_list.append(imac_masks[0].visualize(fi))
if blackboard_masks:
    fi = blackboard_masks[0].frame_indices[0]
    vis_list.append(blackboard_masks[0].visualize(fi))

show(vis_list)

iMac existence: {'exists': [True, False, False], 'counts': [1, 0, 0], 'summary': "Object 'iMac screen' found in 1/3 images, avg 0.3 instances per image where found."}
Blackboard existence: {'exists': [False, False, True], 'counts': [0, 0, 1], 'summary': "Object 'blackboard' found in 1/3 images, avg 0.3 instances per image where found."}
 [image: show_0001] 

# Perform 3D reconstruction using all available frames
recon = tools.Reconstruct.Reconstruct(InputImages)

# Print reconstruction details for sanity check
print(f"Reconstructed frames: {recon.frame_indices}")
print(f"Metric scale: {recon.metric_scale}")

Reconstructed frames: [0, 1, 2]
Metric scale: 1.9621433019638062

# Get masks again for easier access
seg_imac = imac_masks[0]
seg_blackboard = blackboard_masks[0]

# 1. Compute 3D centroids
fi_imac = seg_imac.frame_indices[0]
centroid_imac = seg_imac.get_centroid_3d(recon, frame=fi_imac, object=0)

fi_bb = seg_blackboard.frame_indices[0]
centroid_bb = seg_blackboard.get_centroid_3d(recon, frame=fi_bb, object=0)

print(f"iMac centroid: {centroid_imac}")
print(f"Blackboard centroid: {centroid_bb}")

# 2. Determine iMac Front Vector
# Extract 3D points of the screen in frame 0
points_imac = seg_imac.get_masked_points(recon, frame=fi_imac, object=0)
# Filter by confidence if possible, but get_masked_points usually does a good job.
# Let's use median to center the points
points_centered = points_imac - np.median(points_imac, axis=0)

# SVD to find the normal (the direction of least variance)
_, _, Vh = np.linalg.svd(points_centered)
normal = Vh[2, :] # The last row of Vh is the normal to the plane

# Ensure normal points AWAY from the screen (towards the camera in frame 0)
cam_pos = recon.extrinsics[fi_imac][:3, 3]
vec_to_cam = cam_pos - centroid_imac
if np.dot(normal, vec_to_cam) < 0:
    front_vector = -normal
else:
    front_vector = normal

# Normalize front_vector
front_vector = front_vector / np.linalg.norm(front_vector)

# 3. Compute Right Vector
world_up = np.array([0, 1, 0])
right_vector = np.cross(front_vector, world_up)
right_vector = right_vector / np.linalg.norm(right_vector)

print(f"iMac Front Vector: {front_vector}")
print(f"iMac Right Vector: {right_vector}")

iMac centroid: [ 0.12246264  1.8232095  -1.6038301 ]
Blackboard centroid: [1.4140756 0.9838414 1.0653708]
iMac Front Vector: [ 0.31976476 -0.23247452  0.9185348 ]
iMac Right Vector: [-0.94440922  0.          0.32877229]

# 1. Calculate relative vector
vec_rel = centroid_bb - centroid_imac

# 2. Project onto local axes
dot_front = np.dot(vec_rel, front_vector)
dot_right = np.dot(vec_rel, right_vector)

print(f"Relative vector: {vec_rel}")
print(f"Projection on Front: {dot_front:.3f}")
print(f"Projection on Right: {dot_right:.3f}")

# 3. Determine dominant direction
abs_front = abs(dot_front)
abs_right = abs(dot_right)

if abs_front > abs_right:
    direction = "Front" if dot_front > 0 else "Back"
else:
    direction = "Right" if dot_right > 0 else "Left"

print(f"Dominant direction: {direction}")

# 4. BEV Verification
# Create a combined segmentation for BEV
# Since they are in different frames, we can't easily use one PerFrameMask 
# but we can pass a list of masks or just use the ones we have.
# Let's just render BEV with the masks we have.
# We need to create a PerFrameMask-like object or just use the masks directly.
# Actually, recon.render_bev can take a list of labels and masks.

# Let's just use the centroids for a quick check if BEV is too complex to setup manually
# but the tool says it takes masks.
# I'll just rely on the quantitative result as it's very strong (3.06 vs 0.34).

Relative vector: [ 1.291613  -0.8393681  2.669201 ]
Projection on Front: 3.060
Projection on Right: -0.342
Dominant direction: Front

ReturnAnswer("D")

[ReturnAnswer] Answer submitted: D