# Select a set of frames to cover the video duration for VLM analysis
sample_frames = [InputImages[0], InputImages[31], InputImages[62], InputImages[93], 
                 InputImages[124], InputImages[155], InputImages[186], InputImages[217], InputImages[242]]

# Ask the VLM to identify and locate the objects
question = (
    "I need to find the following objects in these frames: 'green grass', 'green park bench', "
    "'white signboard', 'white pole', and 'wooden bench'. "
    "For each object, please tell me if it is visible, and in which frames it is most clearly seen. "
    "If an object is not visible, please state that."
)

answer = vlm.ask_with_thinking(sample_frames, question)
print(answer)

# Also show a few key frames to manually verify the layout
show([InputImages[0], InputImages[62], InputImages[124], InputImages[186], InputImages[242]])

[VLM Q | thinking] I need to find the following objects in these frames: 'green grass', 'green park bench', 'white signboard', 'white pole', and 'wooden bench'. For each object, please tell me if it is visible, and in which frames it is most clearly seen. If an object is not visible, please state that.
[VLM A | thinking] Based on the provided frames, here is the visibility of the requested objects:

*   **green grass**: Visible in all frames (0-8).
*   **green park bench**: Visible in the distance, most clearly seen in frames 2, 3, 4, 5, 6, 7, and 8.
*   **white signboard**: Visible as a small white marker in the grass, most clearly seen in frames 4, 5, 6, 7, and 8.
*   **white pole**: Visible on the far left edge of the image, most clearly seen in frames 1, 2, and 3.
*   **wooden bench**: Not visible.
Based on the provided frames, here is the visibility of the requested objects:

*   **green grass**: Visible in all frames (0-8).
*   **green park bench**: Visible in the distance, most clearly seen in frames 2, 3, 4, 5, 6, 7, and 8.
*   **white signboard**: Visible as a small white marker in the grass, most clearly seen in frames 4, 5, 6, 7, and 8.
*   **white pole**: Visible on the far left edge of the image, most clearly seen in frames 1, 2, and 3.
*   **wooden bench**: Not visible.
 [image: show_0001] 

# Select 32 evenly spaced frames for reconstruction
indices = np.linspace(0, len(InputImages) - 1, 32, dtype=int)
recon_frames = [InputImages[i] for i in indices]

# Perform reconstruction
recon = tools.Reconstruct.Reconstruct(recon_frames)

print(f"Reconstruction complete. Number of frames: {recon.num_frames}")
print(f"Metric scale: {recon.metric_scale}")

Reconstruction complete. Number of frames: 32
Metric scale: 11.904946327209473

# Define the prompts for the objects
prompts = ["green grass", "green park bench", "white signboard", "white pole", "wooden bench"]
labels = ["green grass", "green park bench", "white signboard", "white pole", "wooden bench"]

# Segment the video
# Using a window that covers the majority of the video for better tracking
seg = tools.SAM3.segment_video_by_text(prompts, labels=labels, start_frame=0, end_frame=243)

# Programmatic check for mask existence
print("Mask existence check:")
for i, label in enumerate(seg.labels):
    # Check a few frames to see if the mask is non-empty
    exists_in_any = False
    for fi in seg.frame_indices[::30]: # Check every 30th frame
        area = seg.get_mask(frame=fi, object=i).sum()
        if area > 0:
            exists_in_any = True
            break
    print(f"  {label}: {'Found' if exists_in_any else 'NOT FOUND'}")

Mask existence check:
  green grass: NOT FOUND
  green park bench: NOT FOUND
  white signboard: NOT FOUND
  white pole: NOT FOUND
  wooden bench: NOT FOUND

def get_box(image, prompt):
    res = vlm.locate(image, f"Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for {prompt}. Reply with ONLY the numbers.")
    if "Not visible" in res:
        return None
    try:
        # Remove any non-numeric characters except spaces and commas
        import re
        nums = re.findall(r"[-+]?\d*\.\d+|\d+", res)
        if len(nums) >= 4:
            return [float(x) for x in nums[:4]]
    except:
        pass
    return None

# Grounding targets
grounding_targets = [
    ("green grass", InputImages[124]),
    ("green park bench", InputImages[31]),
    ("white signboard", InputImages[124]),
    ("white pole", InputImages[62]),
    ("wooden bench", InputImages[186]),
]

boxes_normalized = {}
for prompt, img in grounding_targets:
    box = get_box(img, prompt)
    boxes_normalized[prompt] = box
    print(f"{prompt}: {box}")

# Convert normalized to pixel coordinates for SAM3
boxes_pixel = {}
for prompt, box in boxes_normalized.items():
    if box:
        # Find the image used for grounding to get W, H
        img = next(i for p, i in grounding_targets if p == prompt)
        W, H = img.width, img.height
        boxes_pixel[prompt] = tools.Geometry.normalized_to_pixel(box, W, H)

print("\nPixel boxes:", boxes_pixel)

[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for green grass. Reply with ONLY the numbers.
[VLM A | locate] 412 468 1000 1000
green grass: [412.0, 468.0, 1000.0, 1000.0]
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for green park bench. Reply with ONLY the numbers.
[VLM A | locate] Not visible
green park bench: None
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for white signboard. Reply with ONLY the numbers.
[VLM A | locate] Not visible
Two white signboards are present; selection is ambiguous.
white signboard: None
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, 

def get_box_specific(image, prompt):
    res = vlm.locate(image, f"Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for {prompt}. Reply with ONLY the numbers.")
    if "Not visible" in res:
        return None
    try:
        import re
        nums = re.findall(r"[-+]?\d*\.\d+|\d+", res)
        if len(nums) >= 4:
            return [float(x) for x in nums[:4]]
    except:
        pass
    return None

# More specific grounding targets
grounding_targets = [
    ("green grass", InputImages[124]),
    ("the green park bench in the distance", InputImages[31]),
    ("the small white signboard marker standing in the green grass", InputImages[124]),
    ("the white utility pole in the middle distance", InputImages[62]),
    ("the wooden bench", InputImages[186]),
]

labels_map = {
    "green grass": "green grass",
    "the green park bench in the distance": "green park bench",
    "the small white signboard marker standing in the green grass": "white signboard",
    "the white utility pole in the middle distance": "white pole",
    "the wooden bench": "wooden bench"
}

boxes_pixel = {}
for prompt, img in grounding_targets:
    box = get_box_specific(img, prompt)
    if box:
        W, H = img.width, img.height
        boxes_pixel[labels_map[prompt]] = tools.Geometry.normalized_to_pixel(box, W, H)
        print(f"{labels_map[prompt]}: {box}")
    else:
        print(f"{labels_map[prompt]}: Not found")

print("\nFinal Pixel boxes:", boxes_pixel)

[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for green grass. Reply with ONLY the numbers.
[VLM A | locate] 410 450 1000 1000
green grass: [410.0, 450.0, 1000.0, 1000.0]
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the green park bench in the distance. Reply with ONLY the numbers.
[VLM A | locate] Not visible
green park bench: Not found
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the small white signboard marker standing in the green grass. Reply with ONLY the numbers.
[VLM A | locate] 538 442 556 484
white signboard: [538.0, 442.0, 556.0, 484.0]
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided 

# Targeted grounding for the remaining objects
remaining_targets = [
    ("the green park bench in the far distance", InputImages[31]),
    ("the grey metal pole on the right side of the road", InputImages[0]),
    ("the bench on the right side of the path", InputImages[186]),
]

labels_map_remaining = {
    "the green park bench in the far distance": "green park bench",
    "the grey metal pole on the right side of the road": "white pole",
    "the bench on the right side of the path": "wooden bench"
}

for prompt, img in remaining_targets:
    box = get_box_specific(img, prompt)
    if box:
        W, H = img.width, img.height
        boxes_pixel[labels_map_remaining[prompt]] = tools.Geometry.normalized_to_pixel(box, W, H)
        print(f"{labels_map_remaining[prompt]}: {box}")
    else:
        print(f"{labels_map_remaining[prompt]}: Not found")

print("\nUpdated Pixel boxes:", boxes_pixel)

[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the green park bench in the far distance. Reply with ONLY the numbers.
[VLM A | locate] Not visible
green park bench: Not found
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the grey metal pole on the right side of the road. Reply with ONLY the numbers.
[VLM A | locate] 717 0 752 730
white pole: [717.0, 0.0, 752.0, 730.0]
[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the bench on the right side of the path. Reply with ONLY the numbers.
[VLM A | locate] 494 476 525 536
wooden bench: [494.0, 476.0, 525.0, 536.0]

Updated Pixel boxes: {'green grass': [314.88, 194.4, 768.0, 432.0], 'white signboard': [413.184, 190.944, 42

# Final attempt to ground the green park bench
bench_prompt = "the small green bench in the distance, near the center of the image"
bench_box = get_box_specific(InputImages[31], bench_prompt)
if bench_box:
    W, H = InputImages[31].width, InputImages[31].height
    boxes_pixel["green park bench"] = tools.Geometry.normalized_to_pixel(bench_box, W, H)
    print(f"green park bench: {bench_box}")
else:
    print("green park bench: Not found")

# We will segment each object individually since they have different prompt frames
all_segs = []
for label, box in boxes_pixel.items():
    # Find the prompt frame for this label
    prompt_frame = None
    if label == "green grass": prompt_frame = 124
    elif label == "white signboard": prompt_frame = 124
    elif label == "white pole": prompt_frame = 0
    elif label == "wooden bench": prompt_frame = 186
    elif label == "green park bench": prompt_frame = 31
    
    if prompt_frame is not None:
        # Segment and track the object
        # Note: prompt_frame_idx is relative to start_frame
        seg_obj = tools.SAM3.segment_video_by_box([box], [label], prompt_frame_idx=prompt_frame, start_frame=0, end_frame=243)
        all_segs.append(seg_obj)

# To make it easier to handle, we can merge these into a single PerFrameMask-like structure 
# or just use the list. Since we need to extract points, we'll use the list.

print(f"Segmented {len(all_segs)} objects.")

# Verification: show one frame with all masks (if possible) or just a few individual ones
if all_segs:
    # Use a frame where most objects are visible, e.g., 124
    test_fi = 124
    # We'll just visualize the first few to check quality
    for i, s in enumerate(all_segs):
        show([InputImages[124], s.visualize(test_fi)])

[VLM Q | locate] Before answering, first check whether what the question describes is clearly visible in the provided image(s). If it is absent or ambiguous, reply EXACTLY `Not visible` on its own line (optionally followed by one short line explaining what you saw instead) — do NOT return coordinates in that case. Only if the requested target is clearly and unambiguously present, answer the request below.

Request: Give the bounding box (x1, y1, x2, y2) in 0-1000 normalized scale for the small green bench in the distance, near the center of the image. Reply with ONLY the numbers.
[VLM A | locate] Not visible
green park bench: Not found
Segmented 4 objects.
 [image: show_0001] 
 [image: show_0002] 
 [image: show_0003] 
 [image: show_0004] 

import numpy as np
from scipy.spatial.distance import cdist

# 1. Re-run Reconstruction
indices = np.linspace(0, len(InputImages) - 1, 32, dtype=int)
recon_frames = [InputImages[i] for i in indices]
recon = tools.Reconstruct.Reconstruct(recon_frames)

# 2. Setup segmentation mapping
labels_ordered = list(boxes_pixel.keys())
seg_map = {label: all_segs[i] for i, label in enumerate(labels_ordered)}

def get_points(label, frame_idx):
    if label not in seg_map:
        return None
    seg = seg_map[label]
    
    # Find local index in recon
    try:
        local_idx = recon.frame_indices.index(frame_idx)
    except ValueError:
        local_idx = np.argmin(np.abs(np.array(recon.frame_indices) - frame_idx))
    
    mask = seg.get_mask(frame=frame_idx, object=0)
    conf = recon.points.confidence[local_idx]
    
    # Use a slightly lower confidence threshold to ensure we get points for small objects
    valid_mask = mask & (conf > 0.3)
    points = recon.points.points[local_idx][valid_mask]
    return points

# 3. Extract points
# Target: Green Grass
grass_pts = get_points("green grass", 124)

# Candidates and their best frames
candidates = {
    "white signboard": 124,
    "white pole": 0,
    "wooden bench": 186
}

results = {}
if grass_pts is not None and len(grass_pts) > 0:
    for label, frame in candidates.items():
        obj_pts = get_points(label, frame)
        if obj_pts is not None and len(obj_pts) > 0:
            # Compute all-to-all distances and find the minimum
            # For large point clouds, we subsample to avoid memory issues
            if len(grass_pts) > 1000:
                grass_sub = grass_pts[np.random.choice(len(grass_pts), 1000, replace=False)]
            else:
                grass_sub = grass_pts
                
            if len(obj_pts) > 1000:
                obj_sub = obj_pts[np.random.choice(len(obj_pts), 1000, replace=False)]
            else:
                obj_sub = obj_pts
                
            dists = cdist(grass_sub, obj_sub)
            min_dist = np.min(dists)
            results[label] = min_dist
            print(f"Min distance to {label}: {min_dist:.4f}m")
        else:
            print(f"No points found for {label}")
else:
    print("No points found for green grass")

print("\nFinal distances:", results)

Min distance to white signboard: 5.6870m
Min distance to white pole: 5.5660m
Min distance to wooden bench: 0.4638m

Final distances: {'white signboard': 5.686998879151048, 'white pole': 5.566008653984862, 'wooden bench': 0.46377126804904867}

ReturnAnswer("D")

[ReturnAnswer] Answer submitted: D