From 6cc53c1001b504bd5ac6c4c15860456909fe718b Mon Sep 17 00:00:00 2001 From: Karim Jerbi Date: Thu, 1 Oct 2026 13:25:57 +0100 Subject: [PATCH] feat(mcp): add Set-of-Marks grounding, regional framing, and 2-point image calibration - Add Set-of-Marks (SoM) visual grounding (src/app/control/vision.rs): * Render high-contrast numbered badges on entity visual centroids in viewport captures * Project 3D entity bounds to 2D image coordinates with priority ranking (selected/size) * Return bidirectional grounding manifest (annotations / visible_entities) in capture and ocs_capture payloads mapping each badge to entity handle, layer, type, and world center - Add regional viewport framing (src/app/control/mod.rs, src/mcp.rs): * Support view: region and bounds: [min_x, min_y, max_x, max_y] in capture and ocs_capture * Enable programmatic zoom and high-resolution inspection of specific rooms or local geometry - Add 2-point raster image calibration (src/app/control/actions.rs, src/mcp_ops.rs): * Extend embed_image with optional source_points (pixel coords) and target_points (CAD coords) * Automatically calculate scale, world dimensions, and insertion origin at * Add unit test embed_image_calibrates_from_source_and_target_points --- src/app/control/actions.rs | 61 ++++++- src/app/control/mod.rs | 150 ++++++++++++++++- src/app/control/vision.rs | 320 +++++++++++++++++++++++++++++++++++++ src/mcp.rs | 64 +++++++- src/mcp_ops.rs | 2 + 5 files changed, 585 insertions(+), 12 deletions(-) create mode 100644 src/app/control/vision.rs diff --git a/src/app/control/actions.rs b/src/app/control/actions.rs index 274f1f1da..e94702db6 100644 --- a/src/app/control/actions.rs +++ b/src/app/control/actions.rs @@ -115,20 +115,73 @@ fn embed_point(req: &Value) -> Result { } Ok(codec::types::Vector3::new(p[0], p[1], p[2])) } +fn parse_point_2d(val: &Value) -> Result<[f64; 2], Value> { + let arr = val + .as_array() + .filter(|v| v.len() >= 2) + .ok_or_else(|| failure("invalid_point", "Expected [x, y]"))?; + let x = arr[0] + .as_f64() + .filter(|v| v.is_finite()) + .ok_or_else(|| failure("invalid_point", "Expected finite coordinates"))?; + let y = arr[1] + .as_f64() + .filter(|v| v.is_finite()) + .ok_or_else(|| failure("invalid_point", "Expected finite coordinates"))?; + Ok([x, y]) +} + impl OpenCADStudio { /// `embed_image` — pack the picture at `path` into an OLE2FRAME placed /// with its lower-left corner at `at` (default width ≈ pixel_width/100, /// like the interactive command's Enter answer) and commit it as one /// undo step. The drawing stays self-contained: no external file to lose. + /// Supports automatic 2-point calibration via `source_points` and `target_points`. #[cfg(not(target_arch = "wasm32"))] pub(super) fn control_embed_image(&mut self, req: &Value) -> Result, Value> { let i = self.active_tab; let path = string(req, "path")?; let image = crate::io::ole_embed::EmbeddedImage::from_file(std::path::Path::new(&path)) .map_err(|e| failure("embed_failed", e))?; - let at = embed_point(req)?; - let default_width = (image.pixel_width as f64 / 100.0).max(1.0); - let width = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(default_width); + let (at, width) = if let (Some(src_pts), Some(tgt_pts)) = ( + req.get("source_points").and_then(Value::as_array), + req.get("target_points").and_then(Value::as_array), + ) { + if src_pts.len() >= 2 && tgt_pts.len() >= 2 { + let p_s1 = parse_point_2d(&src_pts[0])?; + let p_s2 = parse_point_2d(&src_pts[1])?; + let p_t1 = parse_point_2d(&tgt_pts[0])?; + let p_t2 = parse_point_2d(&tgt_pts[1])?; + let d_px = ((p_s2[0] - p_s1[0]).powi(2) + (p_s2[1] - p_s1[1]).powi(2)).sqrt(); + let d_cad = ((p_t2[0] - p_t1[0]).powi(2) + (p_t2[1] - p_t1[1]).powi(2)).sqrt(); + if d_px <= 1e-6 { + return Err(failure( + "invalid_points", + "Source calibration points must be distinct", + )); + } + let scale = d_cad / d_px; + let w = image.pixel_width as f64 * scale; + // In image pixel coords, y is 0 at top and pixel_height at bottom. + // In CAD coordinates, lower-left is (x_ll, y_ll). + // x_target = x_ll + px * scale => x_ll = x_target - px * scale + // y_target = y_ll + (pixel_height - py) * scale => y_ll = y_target - (pixel_height - py) * scale + let x_ll = p_t1[0] - p_s1[0] * scale; + let y_ll = p_t1[1] - (image.pixel_height as f64 - p_s1[1]) * scale; + let calc_at = codec::types::Vector3::new(x_ll, y_ll, 0.0); + let final_at = embed_point(req).unwrap_or(calc_at); + let final_w = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(w); + (final_at, final_w) + } else { + let default_width = (image.pixel_width as f64 / 100.0).max(1.0); + let width = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(default_width); + (embed_point(req)?, width) + } + } else { + let default_width = (image.pixel_width as f64 / 100.0).max(1.0); + let width = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(default_width); + (embed_point(req)?, width) + }; if req["linked"].as_bool().unwrap_or(false) { // Path-linked RasterImage + ImageDefinition: the drawing stores // only the file path, so the picture file must travel with the @@ -167,6 +220,7 @@ impl OpenCADStudio { "handle": format!("{:X}", handle.value()), "kind": "RasterImage", "path": path, + "at": [at.x, at.y, at.z], "width": width, "height": height, "linked": true, @@ -186,6 +240,7 @@ impl OpenCADStudio { "handle": format!("{:X}", handle.value()), "kind": "Ole2Frame", "path": path, + "at": [at.x, at.y, at.z], "width": width, })); Ok(Task::none()) diff --git a/src/app/control/mod.rs b/src/app/control/mod.rs index e2e64735a..9a421a7b1 100644 --- a/src/app/control/mod.rs +++ b/src/app/control/mod.rs @@ -1058,6 +1058,57 @@ impl OpenCADStudio { .main_window .ok_or_else(|| failure("gui_required", "Capture requires a GUI window"))?; let path = string(req, "path")?.to_owned(); + + // Optional framing / selection adjustments prior to snapshot: + if let Some(handles) = req.get("highlight_handles").and_then(Value::as_array) { + let hex_handles: Vec = handles + .iter() + .filter_map(|v| v.as_str()) + .filter_map(|v| u64::from_str_radix(v.trim_start_matches("0x"), 16).ok()) + .map(codec::Handle::new) + .collect(); + if !hex_handles.is_empty() { + let scene = &mut self.tabs[self.active_tab].scene; + scene.deselect_all(); + for h in hex_handles { + scene.select_entity(h, false); + } + } + } + if let Some(focus) = req.get("focus_handles").and_then(Value::as_array) { + let hex_handles: Vec = focus + .iter() + .filter_map(|v| v.as_str()) + .filter_map(|v| u64::from_str_radix(v.trim_start_matches("0x"), 16).ok()) + .map(codec::Handle::new) + .collect(); + if !hex_handles.is_empty() { + self.tabs[self.active_tab].scene.zoom_to_entities(&hex_handles); + } + } else if let Some(bounds) = req.get("bounds").and_then(Value::as_array) { + if bounds.len() >= 4 { + if let (Some(x0), Some(y0), Some(x1), Some(y1)) = ( + bounds[0].as_f64(), + bounds[1].as_f64(), + bounds[2].as_f64(), + bounds[3].as_f64(), + ) { + let p1 = glam::Vec3::new(x0 as f32, y0 as f32, 0.0); + let p2 = glam::Vec3::new(x1 as f32, y1 as f32, 0.0); + self.tabs[self.active_tab].scene.zoom_to_window(p1, p2); + } + } + } else if let Some(view) = req.get("view").and_then(Value::as_str) { + if view == "extents" { + self.tabs[self.active_tab].scene.fit_all(); + } else if view == "selection" { + let selected = self.tabs[self.active_tab].scene.selected_handles_in_order(); + if !selected.is_empty() { + self.tabs[self.active_tab].scene.zoom_to_entities(&selected); + } + } + } + // A minimized window has a 0x0 surface and the renderer // panics reading it back, so report instead of capturing. iced::window::size(window).then(move |size| { @@ -1270,7 +1321,7 @@ impl OpenCADStudio { .pending .as_ref() .and_then(|pending| pending.request["scope"].as_str()) - .unwrap_or("window"); + .unwrap_or("viewport"); let max_dimension = self .control .pending @@ -1278,10 +1329,22 @@ impl OpenCADStudio { .and_then(|pending| pending.request["max_dimension"].as_u64()) .unwrap_or(1600) .clamp(256, 4096) as u32; + let annotate = self + .control + .pending + .as_ref() + .and_then(|pending| pending.request["annotate"].as_bool()) + .unwrap_or(false); + let mut image = image::RgbaImage::from_raw(s.size.width, s.size.height, s.rgba.to_vec()) .ok_or("Renderer returned malformed image data")?; let mut actual_scope = "window"; + + let (vp_w, vp_h) = self.tabs[self.active_tab].scene.selection.borrow().vp_size; + let mut vp_logical_width = if vp_w > 0.0 { vp_w } else { image.width() as f32 / s.scale_factor }; + let mut vp_logical_height = if vp_h > 0.0 { vp_h } else { image.height() as f32 / s.scale_factor }; + if requested_scope == "viewport" { if let Some(bounds) = crate::ui::wrap_bar::dropdown_bounds( crate::app::view::VIEWPORT_CAPTURE_BOUNDS_ID, @@ -1305,6 +1368,8 @@ impl OpenCADStudio { ) .to_image(); actual_scope = "viewport"; + vp_logical_width = bounds.width; + vp_logical_height = bounds.height; } } } @@ -1320,12 +1385,58 @@ impl OpenCADStudio { image::imageops::FilterType::Triangle, ); } + + // Spatial grounding: camera, world bounds, and visible entity centroids + let grounding = vision::compute_grounding( + self, + vp_logical_width, + vp_logical_height, + image.width(), + image.height(), + ); + + // Optional Set-of-Marks visual overlay badges + if annotate { + for ent_val in &grounding.visible_entities { + if let (Some(tag), Some(px_arr), Some(selected)) = ( + ent_val["tag"].as_u64(), + ent_val["screen_pixel"].as_array(), + ent_val["selected"].as_bool(), + ) { + if px_arr.len() == 2 { + if let (Some(x), Some(y)) = (px_arr[0].as_i64(), px_arr[1].as_i64()) { + vision::draw_som_badge( + &mut image, + x as i32, + y as i32, + tag as usize, + selected, + ); + } + } + } + } + } + image .save_with_format(&path, image::ImageFormat::Png) .map_err(|e| e.to_string())?; - Ok( - json!({"path":path,"scope":actual_scope,"width":image.width(),"height":image.height(),"scale_factor":s.scale_factor,"document_id":self.tabs[self.active_tab].id,"revision":self.tabs[self.active_tab].edit_revision,"camera_revision":self.tabs[self.active_tab].scene.camera_generation}), - ) + + Ok(json!({ + "path": path, + "scope": actual_scope, + "width": image.width(), + "height": image.height(), + "scale_factor": s.scale_factor, + "document_id": self.tabs[self.active_tab].id, + "revision": self.tabs[self.active_tab].edit_revision, + "camera_revision": self.tabs[self.active_tab].scene.camera_generation, + "viewport_world_bounds": grounding.viewport_world_bounds, + "camera": grounding.camera, + "annotated": annotate, + "annotations": grounding.visible_entities.clone(), + "visible_entities": grounding.visible_entities, + })) })(); match result { Ok(v) => { @@ -1337,6 +1448,7 @@ impl OpenCADStudio { } } } +pub(crate) mod vision; mod actions; pub(crate) use actions::property_json; mod entities; @@ -1763,6 +1875,36 @@ mod tests { let _ = handle; } + #[test] + fn embed_image_calibrates_from_source_and_target_points() { + let mut app = OpenCADStudio::new_for_test(); + request(&mut app, json!({"op":"new"})); + let img = image::RgbaImage::from_pixel(100, 100, image::Rgba([12, 12, 12, 255])); + let path = std::env::temp_dir().join(format!("ocs-embed-calib-{}.png", session_id())); + img.save(&path).unwrap(); + + // 2 points on image: bottom-left (0, 100) and bottom-right (100, 100) + // Target CAD points: (10.0, 20.0) and (210.0, 20.0) -> scale = 200/100 = 2.0 + let response = request( + &mut app, + json!({ + "op": "embed_image", + "path": path.to_string_lossy(), + "linked": true, + "source_points": [[0, 100], [100, 100]], + "target_points": [[10.0, 20.0], [210.0, 20.0]] + }), + ); + assert_eq!(response["status"], "completed", "{response}"); + assert_eq!(response["result"]["kind"], "RasterImage"); + assert_eq!(response["result"]["width"], 200.0); + assert_eq!(response["result"]["height"], 200.0); + let at = response["result"]["at"].as_array().unwrap(); + assert_eq!(at[0], 10.0); + assert_eq!(at[1], 20.0); + let _ = std::fs::remove_file(path); + } + fn user_select_pick(app: &mut OpenCADStudio, kind: &str) -> String { let handle = app.automation_op( format!(r#"{{"op":"query","type":"{kind}","detail":"summary"}}"#).as_str(), diff --git a/src/app/control/vision.rs b/src/app/control/vision.rs new file mode 100644 index 000000000..0c320897f --- /dev/null +++ b/src/app/control/vision.rs @@ -0,0 +1,320 @@ +use crate::app::OpenCADStudio; +use serde_json::{json, Value}; + +/// 5x7 bitmapped font glyphs for digits '0' through '9' +const DIGITS_5X7: [[u8; 7]; 10] = [ + [0b01110, 0b10001, 0b10011, 0b10101, 0b11001, 0b10001, 0b01110], // 0 + [0b00100, 0b01100, 0b00100, 0b00100, 0b00100, 0b00100, 0b01110], // 1 + [0b01110, 0b10001, 0b00001, 0b00110, 0b01000, 0b10000, 0b11111], // 2 + [0b11110, 0b00001, 0b00001, 0b01110, 0b00001, 0b00001, 0b11110], // 3 + [0b00010, 0b00110, 0b01010, 0b10010, 0b11111, 0b00010, 0b00010], // 4 + [0b11111, 0b10000, 0b11110, 0b00001, 0b00001, 0b10001, 0b01110], // 5 + [0b00110, 0b01000, 0b10000, 0b11110, 0b10001, 0b10001, 0b01110], // 6 + [0b11111, 0b00001, 0b00010, 0b00100, 0b01000, 0b01000, 0b01000], // 7 + [0b01110, 0b10001, 0b10001, 0b01110, 0b10001, 0b10001, 0b01110], // 8 + [0b01110, 0b10001, 0b10001, 0b01111, 0b00001, 0b00010, 0b01100], // 9 +]; + +/// Draw a Set-of-Marks badge at `(center_x, center_y)` onto `image`. +pub(crate) fn draw_som_badge( + image: &mut image::RgbaImage, + center_x: i32, + center_y: i32, + tag: usize, + selected: bool, +) { + let tag_str = tag.to_string(); + let num_digits = tag_str.len(); + if num_digits == 0 { + return; + } + let text_w = num_digits * 5 + (num_digits - 1) * 1; + let badge_w = (text_w + 8) as i32; + let badge_h = 13i32; + + let img_w = image.width() as i32; + let img_h = image.height() as i32; + + // Anchor badge slightly above the center anchor point if possible + let x0 = (center_x - badge_w / 2).clamp(1, (img_w - badge_w - 1).max(1)); + let y0 = (center_y - badge_h - 2).clamp(1, (img_h - badge_h - 1).max(1)); + + let (bg, border, text_col) = if selected { + ( + image::Rgba([0, 230, 255, 245]), // Electric cyan + image::Rgba([0, 60, 80, 255]), + image::Rgba([0, 0, 0, 255]), + ) + } else { + ( + image::Rgba([255, 220, 0, 245]), // Vibrant yellow + image::Rgba([40, 30, 0, 255]), + image::Rgba([0, 0, 0, 255]), + ) + }; + + // Fill badge background and border + for y in y0..(y0 + badge_h) { + for x in x0..(x0 + badge_w) { + if x < 0 || x >= img_w || y < 0 || y >= img_h { + continue; + } + let is_border = x == x0 || x == x0 + badge_w - 1 || y == y0 || y == y0 + badge_h - 1; + let col = if is_border { border } else { bg }; + image.put_pixel(x as u32, y as u32, col); + } + } + + // Draw digits + let mut cur_x = x0 + 4; + let cur_y = y0 + 3; + for ch in tag_str.chars() { + if let Some(digit) = ch.to_digit(10) { + let d = digit as usize; + if d < 10 { + for r in 0..7 { + let row_bits = DIGITS_5X7[d][r]; + for c in 0..5 { + if (row_bits & (1 << (4 - c))) != 0 { + let px = cur_x + c as i32; + let py = cur_y + r as i32; + if px >= 0 && px < img_w && py >= 0 && py < img_h { + image.put_pixel(px as u32, py as u32, text_col); + } + } + } + } + } + } + cur_x += 6; + } + + // Small 3x3 indicator dot at the center anchor + let dot_col = if selected { + image::Rgba([0, 230, 255, 255]) + } else { + image::Rgba([255, 60, 0, 255]) + }; + for dy in -1..=1 { + for dx in -1..=1 { + let px = center_x + dx; + let py = center_y + dy; + if px >= 0 && px < img_w && py >= 0 && py < img_h { + image.put_pixel(px as u32, py as u32, dot_col); + } + } + } +} + +pub(crate) struct VisionGrounding { + pub viewport_world_bounds: Value, + pub camera: Value, + pub visible_entities: Vec, +} + +pub(crate) fn compute_grounding( + app: &OpenCADStudio, + vp_logical_width: f32, + vp_logical_height: f32, + image_width: u32, + image_height: u32, +) -> VisionGrounding { + let tab = &app.tabs[app.active_tab]; + let c = tab.scene.camera.borrow(); + let vp_rect = iced::Rectangle { + x: 0.0, + y: 0.0, + width: vp_logical_width.max(1.0), + height: vp_logical_height.max(1.0), + }; + + let p_tl = c.pick_on_target_plane(iced::Point::new(0.0, 0.0), vp_rect); + let p_tr = c.pick_on_target_plane(iced::Point::new(vp_rect.width, 0.0), vp_rect); + let p_bl = c.pick_on_target_plane(iced::Point::new(0.0, vp_rect.height), vp_rect); + let p_br = c.pick_on_target_plane(iced::Point::new(vp_rect.width, vp_rect.height), vp_rect); + + let min_x = p_tl.x.min(p_tr.x).min(p_bl.x).min(p_br.x); + let max_x = p_tl.x.max(p_tr.x).max(p_bl.x).max(p_br.x); + let min_y = p_tl.y.min(p_tr.y).min(p_bl.y).min(p_br.y); + let max_y = p_tl.y.max(p_tr.y).max(p_bl.y).max(p_br.y); + + let viewport_world_bounds = json!({ + "min": [min_x, min_y], + "max": [max_x, max_y] + }); + + let eye = c.eye(); + let target = c.target; + let camera_info = json!({ + "eye": [eye.x, eye.y, eye.z], + "target": [target.x, target.y, target.z], + "distance": c.distance, + "ortho_size": c.ortho_size(), + }); + + let img_w = image_width as f32; + let img_h = image_height as f32; + + struct Candidate { + handle: codec::Handle, + entity_type: &'static str, + layer: String, + screen_pixel: [i32; 2], + screen_bounds: [i32; 4], + world_center: [f64; 3], + selected: bool, + area: f32, + } + + let selected_set: std::collections::HashSet = + tab.scene.selected_handles_in_order().into_iter().collect(); + + let mut candidates = Vec::new(); + + for entity in tab.scene.document.entities() { + let handle = entity.common().handle; + let (b_min, b_max) = crate::scene::convert::tess::entity_bounds_in(&tab.scene.document, entity); + let min_dvec = glam::DVec3::from(b_min); + let max_dvec = glam::DVec3::from(b_max); + if !min_dvec.is_finite() || !max_dvec.is_finite() { + continue; + } + let center = (min_dvec + max_dvec) * 0.5; + if let Some(screen_pt) = c.project(center, vp_rect) { + if screen_pt.x >= 0.0 + && screen_pt.x <= vp_rect.width + && screen_pt.y >= 0.0 + && screen_pt.y <= vp_rect.height + { + let px = (screen_pt.x / vp_rect.width * img_w).round() as i32; + let py = (screen_pt.y / vp_rect.height * img_h).round() as i32; + + let mut s_min_x = px; + let mut s_max_x = px; + let mut s_min_y = py; + let mut s_max_y = py; + + if let (Some(p0), Some(p1)) = + (c.project(min_dvec, vp_rect), c.project(max_dvec, vp_rect)) + { + let x0 = (p0.x / vp_rect.width * img_w).round() as i32; + let y0 = (p0.y / vp_rect.height * img_h).round() as i32; + let x1 = (p1.x / vp_rect.width * img_w).round() as i32; + let y1 = (p1.y / vp_rect.height * img_h).round() as i32; + s_min_x = x0.min(x1); + s_max_x = x0.max(x1); + s_min_y = y0.min(y1); + s_max_y = y0.max(y1); + } + + let area = ((s_max_x - s_min_x).abs() * (s_max_y - s_min_y).abs()) as f32; + let selected = selected_set.contains(&handle); + let layer = entity.common().layer.clone(); + + candidates.push(Candidate { + handle, + entity_type: crate::entities::names::ui_name(entity), + layer, + screen_pixel: [px, py], + screen_bounds: [s_min_x, s_min_y, s_max_x, s_max_y], + world_center: [center.x, center.y, center.z], + selected, + area, + }); + } + } + } + + // Selected entities prioritized, then larger visual area first + candidates.sort_by(|a, b| { + b.selected + .cmp(&a.selected) + .then_with(|| b.area.partial_cmp(&a.area).unwrap_or(std::cmp::Ordering::Equal)) + }); + + candidates.truncate(64); + + let visible_entities = candidates + .into_iter() + .enumerate() + .map(|(idx, cand)| { + json!({ + "tag": idx + 1, + "handle": format!("{:X}", cand.handle.value()), + "type": cand.entity_type, + "layer": cand.layer, + "screen_pixel": cand.screen_pixel, + "screen_bounds": cand.screen_bounds, + "world_center": cand.world_center, + "selected": cand.selected + }) + }) + .collect(); + + VisionGrounding { + viewport_world_bounds, + camera: camera_info, + visible_entities, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_draw_som_badge_renders_without_panics() { + let mut img = image::RgbaImage::from_pixel(120, 100, image::Rgba([40, 40, 40, 255])); + + // Render normal badge + draw_som_badge(&mut img, 50, 50, 1, false); + + // Center dot should be drawn + let dot = img.get_pixel(50, 50); + assert_eq!(*dot, image::Rgba([255, 60, 0, 255])); + + // Render selected badge with multi-digit tag near edge (clamped) + draw_som_badge(&mut img, 2, 2, 42, true); + let selected_dot = img.get_pixel(2, 2); + assert_eq!(*selected_dot, image::Rgba([0, 230, 255, 255])); + + // Right edge clamp + draw_som_badge(&mut img, 118, 98, 99, false); + } + + #[test] + fn test_compute_grounding_identifies_visible_entities() { + let mut app = OpenCADStudio::new_for_test(); + app.automation_op(r#"{"op":"new"}"#); + app.automation_op(r#"{"op":"run","cmd":"CIRCLE 0,0 5"}"#); + app.automation_op(r#"{"op":"run","cmd":"LINE -10,-10 10,10"}"#); + + // Set test canvas viewport size + app.tabs[0].scene.selection.borrow_mut().vp_size = (800.0, 600.0); + app.tabs[0].scene.fit_all(); + + let grounding = compute_grounding(&app, 800.0, 600.0, 800, 600); + + // Viewport world bounds should be valid + assert!(grounding.viewport_world_bounds["min"].is_array()); + assert!(grounding.viewport_world_bounds["max"].is_array()); + + // Camera info should be populated + assert!(grounding.camera["eye"].is_array()); + assert!(grounding.camera["target"].is_array()); + + // Both entities should be visible and tagged + assert_eq!(grounding.visible_entities.len(), 2); + + let first = &grounding.visible_entities[0]; + assert_eq!(first["tag"], 1); + assert!(first["handle"].is_string()); + assert!(first["type"].is_string()); + assert!(first["screen_pixel"].is_array()); + assert!(first["screen_bounds"].is_array()); + assert!(first["world_center"].is_array()); + + let second = &grounding.visible_entities[1]; + assert_eq!(second["tag"], 2); + } +} diff --git a/src/mcp.rs b/src/mcp.rs index 659c305d5..ccff12d56 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -45,6 +45,7 @@ const READ_OPS: &[&str] = &[ "audit", "text_search", "text_audit", + "capture", ]; const EXECUTE_OPS: &[&str] = &[ "new", @@ -849,8 +850,29 @@ fn call_tool( let path = std::env::temp_dir().join(format!("ocs-capture-{}.png", random_id()?)); let scope = arguments["scope"].as_str().unwrap_or("viewport"); let max_dimension = arguments["max_dimension"].as_u64().unwrap_or(1600); + let mut req = json!({ + "op": "capture", + "path": path.to_string_lossy(), + "scope": scope, + "max_dimension": max_dimension, + }); + if let Some(view) = arguments.get("view").and_then(Value::as_str) { + req["view"] = json!(view); + } + if let Some(bounds) = arguments.get("bounds").and_then(Value::as_array) { + req["bounds"] = json!(bounds); + } + if let Some(focus) = arguments.get("focus_handles").and_then(Value::as_array) { + req["focus_handles"] = json!(focus); + } + if let Some(hl) = arguments.get("highlight_handles").and_then(Value::as_array) { + req["highlight_handles"] = json!(hl); + } + if let Some(annotate) = arguments.get("annotate").and_then(Value::as_bool) { + req["annotate"] = json!(annotate); + } let result = client(clients, session_id)? - .request(json!({"op":"capture","path":path.to_string_lossy(),"scope":scope,"max_dimension":max_dimension}), 30.0)?; + .request(req, 30.0)?; if result["ok"].as_bool() != Some(true) || result["status"].as_str() != Some("completed") { @@ -858,7 +880,14 @@ fn call_tool( } let bytes = std::fs::read(&path).map_err(|error| error.to_string())?; let _ = std::fs::remove_file(path); - Ok(json!({"$image":BASE64.encode(bytes)})) + let mut meta = result.get("result").cloned().unwrap_or_else(|| json!({})); + if let Some(obj) = meta.as_object_mut() { + obj.remove("path"); + } + Ok(json!({ + "$image": BASE64.encode(bytes), + "metadata": meta, + })) } _ => Err(format!("Unknown tool: {name}")), } @@ -924,8 +953,22 @@ fn tool_definitions() -> Value { }, { "name":"ocs_capture", - "description":"Capture the actual current OCS drawing viewport or window as a bounded PNG for visual verification.", - "inputSchema":{"type":"object","properties":{"ocs_session_id":{"type":"string","minLength":1,"description":"Value of session_id returned by ocs_sessions."},"scope":{"type":"string","enum":["viewport","window"],"default":"viewport","description":"Capture only the drawing viewport by default, or the complete application window."},"max_dimension":{"type":"integer","minimum":256,"maximum":4096,"default":1600,"description":"Resize the longest image edge to at most this many pixels."}},"required":["ocs_session_id"],"additionalProperties":false}, + "description":"Capture the actual current OCS drawing viewport or window as a bounded PNG with optional camera framing, entity highlighting, spatial coordinates and Set-of-Marks annotations.", + "inputSchema":{ + "type":"object", + "properties":{ + "ocs_session_id":{"type":"string","minLength":1,"description":"Value of session_id returned by ocs_sessions."}, + "scope":{"type":"string","enum":["viewport","window"],"default":"viewport","description":"Capture only the drawing viewport by default, or the complete application window."}, + "max_dimension":{"type":"integer","minimum":256,"maximum":4096,"default":1600,"description":"Resize the longest image edge to at most this many pixels."}, + "view":{"type":"string","enum":["current","extents","selection","region"],"default":"current","description":"Frame the camera before capture: extents fits all entities, selection fits selected entities, region fits explicit world bounds."}, + "bounds":{"type":"array","items":{"type":"number"},"minItems":4,"maxItems":4,"description":"World XY bounding box [min_x, min_y, max_x, max_y] to zoom and fit in view before capturing (used with view: 'region')."}, + "focus_handles":{"type":"array","items":{"type":"string"},"description":"Hex handles of entities to zoom and fit in view before capturing."}, + "highlight_handles":{"type":"array","items":{"type":"string"},"description":"Hex handles of entities to select/highlight before capturing."}, + "annotate":{"type":"boolean","default":false,"description":"Overlay Set-of-Marks numbered tags on visible entities for visual grounding."} + }, + "required":["ocs_session_id"], + "additionalProperties":false + }, "annotations":{"title":"Capture OCS window","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false} } ]) @@ -933,7 +976,18 @@ fn tool_definitions() -> Value { fn tool_result(value: Value) -> Value { if let Some(image) = value.get("$image").and_then(Value::as_str) { - return json!({"content":[{"type":"image","data":image,"mimeType":"image/png"}]}); + let mut content = Vec::new(); + if let Some(meta) = value.get("metadata") { + let meta_text = serde_json::to_string_pretty(meta).unwrap_or_else(|_| meta.to_string()); + content.push(json!({"type":"text","text":meta_text})); + } + content.push(json!({"type":"image","data":image,"mimeType":"image/png"})); + let structured = value.get("metadata").cloned().unwrap_or_else(|| json!({})); + return json!({ + "content": content, + "structuredContent": structured, + "isError": false + }); } let structured = if value.is_object() { value.clone() diff --git a/src/mcp_ops.rs b/src/mcp_ops.rs index 4cb5846c4..233936d70 100644 --- a/src/mcp_ops.rs +++ b/src/mcp_ops.rs @@ -245,6 +245,8 @@ pub static EMBED_IMAGE: OpDef = OpDef { opt("at", Ty::Point, "Lower-left corner [x, y] or [x, y, z]."), opt("width", Ty::Num, "World width of the image."), opt("linked", Ty::Bool, "Store as path-linked RasterImage instead of embedded OLE2FRAME."), + opt("source_points", Ty::ArrayOf(&Ty::Point), "Pixel points in image [[px1,py1],[px2,py2]] for 2-point alignment."), + opt("target_points", Ty::ArrayOf(&Ty::Point), "CAD world points [[x1,y1],[x2,y2]] for 2-point alignment."), ], rules: &[], example: r#"{"op":"embed_image","path":"logo.png","at":[0,0]}"#,