feat(mcp): add Set-of-Marks grounding, regional framing, and 2-point image calibration

- Add Set-of-Marks (SoM) visual grounding (src/app/control/vision.rs):
  * Render high-contrast numbered badges on entity visual centroids in viewport captures
  * Project 3D entity bounds to 2D image coordinates with priority ranking (selected/size)
  * Return bidirectional grounding manifest (annotations / visible_entities) in capture and ocs_capture payloads mapping each badge to entity handle, layer, type, and world center

- Add regional viewport framing (src/app/control/mod.rs, src/mcp.rs):
  * Support view: region and bounds: [min_x, min_y, max_x, max_y] in capture and ocs_capture
  * Enable programmatic zoom and high-resolution inspection of specific rooms or local geometry

- Add 2-point raster image calibration (src/app/control/actions.rs, src/mcp_ops.rs):
  * Extend embed_image with optional source_points (pixel coords) and target_points (CAD coords)
  * Automatically calculate scale, world dimensions, and insertion origin at
  * Add unit test embed_image_calibrates_from_source_and_target_points
This commit is contained in:
Karim Jerbi committed 2026-10-01 13:27:22 +01:00
1 parent 9fc4e381b3
commit 6cc53c1001
5 files changed
+585 -12

No files matched your search

+58 -3
View File
@@ -115,20 +115,73 @@ fn embed_point(req: &Value) -> Result<codec::types::Vector3, Value> {
}
Ok(codec::types::Vector3::new(p[0], p[1], p[2]))
}
fn parse_point_2d(val: &Value) -> Result<[f64; 2], Value> {
let arr = val
.as_array()
.filter(|v| v.len() >= 2)
.ok_or_else(|| failure("invalid_point", "Expected [x, y]"))?;
let x = arr[0]
.as_f64()
.filter(|v| v.is_finite())
.ok_or_else(|| failure("invalid_point", "Expected finite coordinates"))?;
let y = arr[1]
.as_f64()
.filter(|v| v.is_finite())
.ok_or_else(|| failure("invalid_point", "Expected finite coordinates"))?;
Ok([x, y])
}
impl OpenCADStudio {
/// `embed_image` — pack the picture at `path` into an OLE2FRAME placed
/// with its lower-left corner at `at` (default width ≈ pixel_width/100,
/// like the interactive command's Enter answer) and commit it as one
/// undo step. The drawing stays self-contained: no external file to lose.
/// Supports automatic 2-point calibration via `source_points` and `target_points`.
#[cfg(not(target_arch = "wasm32"))]
pub(super) fn control_embed_image(&mut self, req: &Value) -> Result<Task<Message>, Value> {
let i = self.active_tab;
let path = string(req, "path")?;
let image = crate::io::ole_embed::EmbeddedImage::from_file(std::path::Path::new(&path))
.map_err(|e| failure("embed_failed", e))?;
let at = embed_point(req)?;
let default_width = (image.pixel_width as f64 / 100.0).max(1.0);
let width = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(default_width);
let (at, width) = if let (Some(src_pts), Some(tgt_pts)) = (
req.get("source_points").and_then(Value::as_array),
req.get("target_points").and_then(Value::as_array),
) {
if src_pts.len() >= 2 && tgt_pts.len() >= 2 {
let p_s1 = parse_point_2d(&src_pts[0])?;
let p_s2 = parse_point_2d(&src_pts[1])?;
let p_t1 = parse_point_2d(&tgt_pts[0])?;
let p_t2 = parse_point_2d(&tgt_pts[1])?;
let d_px = ((p_s2[0] - p_s1[0]).powi(2) + (p_s2[1] - p_s1[1]).powi(2)).sqrt();
let d_cad = ((p_t2[0] - p_t1[0]).powi(2) + (p_t2[1] - p_t1[1]).powi(2)).sqrt();
if d_px <= 1e-6 {
return Err(failure(
"invalid_points",
"Source calibration points must be distinct",
));
}
let scale = d_cad / d_px;
let w = image.pixel_width as f64 * scale;
// In image pixel coords, y is 0 at top and pixel_height at bottom.
// In CAD coordinates, lower-left is (x_ll, y_ll).
// x_target = x_ll + px * scale => x_ll = x_target - px * scale
// y_target = y_ll + (pixel_height - py) * scale => y_ll = y_target - (pixel_height - py) * scale
let x_ll = p_t1[0] - p_s1[0] * scale;
let y_ll = p_t1[1] - (image.pixel_height as f64 - p_s1[1]) * scale;
let calc_at = codec::types::Vector3::new(x_ll, y_ll, 0.0);
let final_at = embed_point(req).unwrap_or(calc_at);
let final_w = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(w);
(final_at, final_w)
} else {
let default_width = (image.pixel_width as f64 / 100.0).max(1.0);
let width = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(default_width);
(embed_point(req)?, width)
}
} else {
let default_width = (image.pixel_width as f64 / 100.0).max(1.0);
let width = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(default_width);
(embed_point(req)?, width)
};
if req["linked"].as_bool().unwrap_or(false) {
// Path-linked RasterImage + ImageDefinition: the drawing stores
// only the file path, so the picture file must travel with the
@@ -167,6 +220,7 @@ impl OpenCADStudio {
"handle": format!("{:X}", handle.value()),
"kind": "RasterImage",
"path": path,
"at": [at.x, at.y, at.z],
"width": width,
"height": height,
"linked": true,
@@ -186,6 +240,7 @@ impl OpenCADStudio {
"handle": format!("{:X}", handle.value()),
"kind": "Ole2Frame",
"path": path,
"at": [at.x, at.y, at.z],
"width": width,
}));
Ok(Task::none())
+146 -4
View File
@@ -1058,6 +1058,57 @@ impl OpenCADStudio {
.main_window
.ok_or_else(|| failure("gui_required", "Capture requires a GUI window"))?;
let path = string(req, "path")?.to_owned();
// Optional framing / selection adjustments prior to snapshot:
if let Some(handles) = req.get("highlight_handles").and_then(Value::as_array) {
let hex_handles: Vec<codec::Handle> = handles
.iter()
.filter_map(|v| v.as_str())
.filter_map(|v| u64::from_str_radix(v.trim_start_matches("0x"), 16).ok())
.map(codec::Handle::new)
.collect();
if !hex_handles.is_empty() {
let scene = &mut self.tabs[self.active_tab].scene;
scene.deselect_all();
for h in hex_handles {
scene.select_entity(h, false);
}
}
}
if let Some(focus) = req.get("focus_handles").and_then(Value::as_array) {
let hex_handles: Vec<codec::Handle> = focus
.iter()
.filter_map(|v| v.as_str())
.filter_map(|v| u64::from_str_radix(v.trim_start_matches("0x"), 16).ok())
.map(codec::Handle::new)
.collect();
if !hex_handles.is_empty() {
self.tabs[self.active_tab].scene.zoom_to_entities(&hex_handles);
}
} else if let Some(bounds) = req.get("bounds").and_then(Value::as_array) {
if bounds.len() >= 4 {
if let (Some(x0), Some(y0), Some(x1), Some(y1)) = (
bounds[0].as_f64(),
bounds[1].as_f64(),
bounds[2].as_f64(),
bounds[3].as_f64(),
) {
let p1 = glam::Vec3::new(x0 as f32, y0 as f32, 0.0);
let p2 = glam::Vec3::new(x1 as f32, y1 as f32, 0.0);
self.tabs[self.active_tab].scene.zoom_to_window(p1, p2);
}
}
} else if let Some(view) = req.get("view").and_then(Value::as_str) {
if view == "extents" {
self.tabs[self.active_tab].scene.fit_all();
} else if view == "selection" {
let selected = self.tabs[self.active_tab].scene.selected_handles_in_order();
if !selected.is_empty() {
self.tabs[self.active_tab].scene.zoom_to_entities(&selected);
}
}
}
// A minimized window has a 0x0 surface and the renderer
// panics reading it back, so report instead of capturing.
iced::window::size(window).then(move |size| {
@@ -1270,7 +1321,7 @@ impl OpenCADStudio {
.pending
.as_ref()
.and_then(|pending| pending.request["scope"].as_str())
.unwrap_or("window");
.unwrap_or("viewport");
let max_dimension = self
.control
.pending
@@ -1278,10 +1329,22 @@ impl OpenCADStudio {
.and_then(|pending| pending.request["max_dimension"].as_u64())
.unwrap_or(1600)
.clamp(256, 4096) as u32;
let annotate = self
.control
.pending
.as_ref()
.and_then(|pending| pending.request["annotate"].as_bool())
.unwrap_or(false);
let mut image =
image::RgbaImage::from_raw(s.size.width, s.size.height, s.rgba.to_vec())
.ok_or("Renderer returned malformed image data")?;
let mut actual_scope = "window";
let (vp_w, vp_h) = self.tabs[self.active_tab].scene.selection.borrow().vp_size;
let mut vp_logical_width = if vp_w > 0.0 { vp_w } else { image.width() as f32 / s.scale_factor };
let mut vp_logical_height = if vp_h > 0.0 { vp_h } else { image.height() as f32 / s.scale_factor };
if requested_scope == "viewport" {
if let Some(bounds) = crate::ui::wrap_bar::dropdown_bounds(
crate::app::view::VIEWPORT_CAPTURE_BOUNDS_ID,
@@ -1305,6 +1368,8 @@ impl OpenCADStudio {
)
.to_image();
actual_scope = "viewport";
vp_logical_width = bounds.width;
vp_logical_height = bounds.height;
}
}
}
@@ -1320,12 +1385,58 @@ impl OpenCADStudio {
image::imageops::FilterType::Triangle,
);
}
// Spatial grounding: camera, world bounds, and visible entity centroids
let grounding = vision::compute_grounding(
self,
vp_logical_width,
vp_logical_height,
image.width(),
image.height(),
);
// Optional Set-of-Marks visual overlay badges
if annotate {
for ent_val in &grounding.visible_entities {
if let (Some(tag), Some(px_arr), Some(selected)) = (
ent_val["tag"].as_u64(),
ent_val["screen_pixel"].as_array(),
ent_val["selected"].as_bool(),
) {
if px_arr.len() == 2 {
if let (Some(x), Some(y)) = (px_arr[0].as_i64(), px_arr[1].as_i64()) {
vision::draw_som_badge(
&mut image,
x as i32,
y as i32,
tag as usize,
selected,
);
}
}
}
}
}
image
.save_with_format(&path, image::ImageFormat::Png)
.map_err(|e| e.to_string())?;
Ok(
json!({"path":path,"scope":actual_scope,"width":image.width(),"height":image.height(),"scale_factor":s.scale_factor,"document_id":self.tabs[self.active_tab].id,"revision":self.tabs[self.active_tab].edit_revision,"camera_revision":self.tabs[self.active_tab].scene.camera_generation}),
)
Ok(json!({
"path": path,
"scope": actual_scope,
"width": image.width(),
"height": image.height(),
"scale_factor": s.scale_factor,
"document_id": self.tabs[self.active_tab].id,
"revision": self.tabs[self.active_tab].edit_revision,
"camera_revision": self.tabs[self.active_tab].scene.camera_generation,
"viewport_world_bounds": grounding.viewport_world_bounds,
"camera": grounding.camera,
"annotated": annotate,
"annotations": grounding.visible_entities.clone(),
"visible_entities": grounding.visible_entities,
}))
})();
match result {
Ok(v) => {
@@ -1337,6 +1448,7 @@ impl OpenCADStudio {
}
}
}
pub(crate) mod vision;
mod actions;
pub(crate) use actions::property_json;
mod entities;
@@ -1763,6 +1875,36 @@ mod tests {
let _ = handle;
}
#[test]
fn embed_image_calibrates_from_source_and_target_points() {
let mut app = OpenCADStudio::new_for_test();
request(&mut app, json!({"op":"new"}));
let img = image::RgbaImage::from_pixel(100, 100, image::Rgba([12, 12, 12, 255]));
let path = std::env::temp_dir().join(format!("ocs-embed-calib-{}.png", session_id()));
img.save(&path).unwrap();
// 2 points on image: bottom-left (0, 100) and bottom-right (100, 100)
// Target CAD points: (10.0, 20.0) and (210.0, 20.0) -> scale = 200/100 = 2.0
let response = request(
&mut app,
json!({
"op": "embed_image",
"path": path.to_string_lossy(),
"linked": true,
"source_points": [[0, 100], [100, 100]],
"target_points": [[10.0, 20.0], [210.0, 20.0]]
}),
);
assert_eq!(response["status"], "completed", "{response}");
assert_eq!(response["result"]["kind"], "RasterImage");
assert_eq!(response["result"]["width"], 200.0);
assert_eq!(response["result"]["height"], 200.0);
let at = response["result"]["at"].as_array().unwrap();
assert_eq!(at[0], 10.0);
assert_eq!(at[1], 20.0);
let _ = std::fs::remove_file(path);
}
fn user_select_pick(app: &mut OpenCADStudio, kind: &str) -> String {
let handle = app.automation_op(
format!(r#"{{"op":"query","type":"{kind}","detail":"summary"}}"#).as_str(),
+320
View File
@@ -0,0 +1,320 @@
use crate::app::OpenCADStudio;
use serde_json::{json, Value};
/// 5x7 bitmapped font glyphs for digits '0' through '9'
const DIGITS_5X7: [[u8; 7]; 10] = [
[0b01110, 0b10001, 0b10011, 0b10101, 0b11001, 0b10001, 0b01110], // 0
[0b00100, 0b01100, 0b00100, 0b00100, 0b00100, 0b00100, 0b01110], // 1
[0b01110, 0b10001, 0b00001, 0b00110, 0b01000, 0b10000, 0b11111], // 2
[0b11110, 0b00001, 0b00001, 0b01110, 0b00001, 0b00001, 0b11110], // 3
[0b00010, 0b00110, 0b01010, 0b10010, 0b11111, 0b00010, 0b00010], // 4
[0b11111, 0b10000, 0b11110, 0b00001, 0b00001, 0b10001, 0b01110], // 5
[0b00110, 0b01000, 0b10000, 0b11110, 0b10001, 0b10001, 0b01110], // 6
[0b11111, 0b00001, 0b00010, 0b00100, 0b01000, 0b01000, 0b01000], // 7
[0b01110, 0b10001, 0b10001, 0b01110, 0b10001, 0b10001, 0b01110], // 8
[0b01110, 0b10001, 0b10001, 0b01111, 0b00001, 0b00010, 0b01100], // 9
];
/// Draw a Set-of-Marks badge at `(center_x, center_y)` onto `image`.
pub(crate) fn draw_som_badge(
image: &mut image::RgbaImage,
center_x: i32,
center_y: i32,
tag: usize,
selected: bool,
) {
let tag_str = tag.to_string();
let num_digits = tag_str.len();
if num_digits == 0 {
return;
}
let text_w = num_digits * 5 + (num_digits - 1) * 1;
let badge_w = (text_w + 8) as i32;
let badge_h = 13i32;
let img_w = image.width() as i32;
let img_h = image.height() as i32;
// Anchor badge slightly above the center anchor point if possible
let x0 = (center_x - badge_w / 2).clamp(1, (img_w - badge_w - 1).max(1));
let y0 = (center_y - badge_h - 2).clamp(1, (img_h - badge_h - 1).max(1));
let (bg, border, text_col) = if selected {
(
image::Rgba([0, 230, 255, 245]), // Electric cyan
image::Rgba([0, 60, 80, 255]),
image::Rgba([0, 0, 0, 255]),
)
} else {
(
image::Rgba([255, 220, 0, 245]), // Vibrant yellow
image::Rgba([40, 30, 0, 255]),
image::Rgba([0, 0, 0, 255]),
)
};
// Fill badge background and border
for y in y0..(y0 + badge_h) {
for x in x0..(x0 + badge_w) {
if x < 0 || x >= img_w || y < 0 || y >= img_h {
continue;
}
let is_border = x == x0 || x == x0 + badge_w - 1 || y == y0 || y == y0 + badge_h - 1;
let col = if is_border { border } else { bg };
image.put_pixel(x as u32, y as u32, col);
}
}
// Draw digits
let mut cur_x = x0 + 4;
let cur_y = y0 + 3;
for ch in tag_str.chars() {
if let Some(digit) = ch.to_digit(10) {
let d = digit as usize;
if d < 10 {
for r in 0..7 {
let row_bits = DIGITS_5X7[d][r];
for c in 0..5 {
if (row_bits & (1 << (4 - c))) != 0 {
let px = cur_x + c as i32;
let py = cur_y + r as i32;
if px >= 0 && px < img_w && py >= 0 && py < img_h {
image.put_pixel(px as u32, py as u32, text_col);
}
}
}
}
}
}
cur_x += 6;
}
// Small 3x3 indicator dot at the center anchor
let dot_col = if selected {
image::Rgba([0, 230, 255, 255])
} else {
image::Rgba([255, 60, 0, 255])
};
for dy in -1..=1 {
for dx in -1..=1 {
let px = center_x + dx;
let py = center_y + dy;
if px >= 0 && px < img_w && py >= 0 && py < img_h {
image.put_pixel(px as u32, py as u32, dot_col);
}
}
}
}
pub(crate) struct VisionGrounding {
pub viewport_world_bounds: Value,
pub camera: Value,
pub visible_entities: Vec<Value>,
}
pub(crate) fn compute_grounding(
app: &OpenCADStudio,
vp_logical_width: f32,
vp_logical_height: f32,
image_width: u32,
image_height: u32,
) -> VisionGrounding {
let tab = &app.tabs[app.active_tab];
let c = tab.scene.camera.borrow();
let vp_rect = iced::Rectangle {
x: 0.0,
y: 0.0,
width: vp_logical_width.max(1.0),
height: vp_logical_height.max(1.0),
};
let p_tl = c.pick_on_target_plane(iced::Point::new(0.0, 0.0), vp_rect);
let p_tr = c.pick_on_target_plane(iced::Point::new(vp_rect.width, 0.0), vp_rect);
let p_bl = c.pick_on_target_plane(iced::Point::new(0.0, vp_rect.height), vp_rect);
let p_br = c.pick_on_target_plane(iced::Point::new(vp_rect.width, vp_rect.height), vp_rect);
let min_x = p_tl.x.min(p_tr.x).min(p_bl.x).min(p_br.x);
let max_x = p_tl.x.max(p_tr.x).max(p_bl.x).max(p_br.x);
let min_y = p_tl.y.min(p_tr.y).min(p_bl.y).min(p_br.y);
let max_y = p_tl.y.max(p_tr.y).max(p_bl.y).max(p_br.y);
let viewport_world_bounds = json!({
"min": [min_x, min_y],
"max": [max_x, max_y]
});
let eye = c.eye();
let target = c.target;
let camera_info = json!({
"eye": [eye.x, eye.y, eye.z],
"target": [target.x, target.y, target.z],
"distance": c.distance,
"ortho_size": c.ortho_size(),
});
let img_w = image_width as f32;
let img_h = image_height as f32;
struct Candidate {
handle: codec::Handle,
entity_type: &'static str,
layer: String,
screen_pixel: [i32; 2],
screen_bounds: [i32; 4],
world_center: [f64; 3],
selected: bool,
area: f32,
}
let selected_set: std::collections::HashSet<codec::Handle> =
tab.scene.selected_handles_in_order().into_iter().collect();
let mut candidates = Vec::new();
for entity in tab.scene.document.entities() {
let handle = entity.common().handle;
let (b_min, b_max) = crate::scene::convert::tess::entity_bounds_in(&tab.scene.document, entity);
let min_dvec = glam::DVec3::from(b_min);
let max_dvec = glam::DVec3::from(b_max);
if !min_dvec.is_finite() || !max_dvec.is_finite() {
continue;
}
let center = (min_dvec + max_dvec) * 0.5;
if let Some(screen_pt) = c.project(center, vp_rect) {
if screen_pt.x >= 0.0
&& screen_pt.x <= vp_rect.width
&& screen_pt.y >= 0.0
&& screen_pt.y <= vp_rect.height
{
let px = (screen_pt.x / vp_rect.width * img_w).round() as i32;
let py = (screen_pt.y / vp_rect.height * img_h).round() as i32;
let mut s_min_x = px;
let mut s_max_x = px;
let mut s_min_y = py;
let mut s_max_y = py;
if let (Some(p0), Some(p1)) =
(c.project(min_dvec, vp_rect), c.project(max_dvec, vp_rect))
{
let x0 = (p0.x / vp_rect.width * img_w).round() as i32;
let y0 = (p0.y / vp_rect.height * img_h).round() as i32;
let x1 = (p1.x / vp_rect.width * img_w).round() as i32;
let y1 = (p1.y / vp_rect.height * img_h).round() as i32;
s_min_x = x0.min(x1);
s_max_x = x0.max(x1);
s_min_y = y0.min(y1);
s_max_y = y0.max(y1);
}
let area = ((s_max_x - s_min_x).abs() * (s_max_y - s_min_y).abs()) as f32;
let selected = selected_set.contains(&handle);
let layer = entity.common().layer.clone();
candidates.push(Candidate {
handle,
entity_type: crate::entities::names::ui_name(entity),
layer,
screen_pixel: [px, py],
screen_bounds: [s_min_x, s_min_y, s_max_x, s_max_y],
world_center: [center.x, center.y, center.z],
selected,
area,
});
}
}
}
// Selected entities prioritized, then larger visual area first
candidates.sort_by(|a, b| {
b.selected
.cmp(&a.selected)
.then_with(|| b.area.partial_cmp(&a.area).unwrap_or(std::cmp::Ordering::Equal))
});
candidates.truncate(64);
let visible_entities = candidates
.into_iter()
.enumerate()
.map(|(idx, cand)| {
json!({
"tag": idx + 1,
"handle": format!("{:X}", cand.handle.value()),
"type": cand.entity_type,
"layer": cand.layer,
"screen_pixel": cand.screen_pixel,
"screen_bounds": cand.screen_bounds,
"world_center": cand.world_center,
"selected": cand.selected
})
})
.collect();
VisionGrounding {
viewport_world_bounds,
camera: camera_info,
visible_entities,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_draw_som_badge_renders_without_panics() {
let mut img = image::RgbaImage::from_pixel(120, 100, image::Rgba([40, 40, 40, 255]));
// Render normal badge
draw_som_badge(&mut img, 50, 50, 1, false);
// Center dot should be drawn
let dot = img.get_pixel(50, 50);
assert_eq!(*dot, image::Rgba([255, 60, 0, 255]));
// Render selected badge with multi-digit tag near edge (clamped)
draw_som_badge(&mut img, 2, 2, 42, true);
let selected_dot = img.get_pixel(2, 2);
assert_eq!(*selected_dot, image::Rgba([0, 230, 255, 255]));
// Right edge clamp
draw_som_badge(&mut img, 118, 98, 99, false);
}
#[test]
fn test_compute_grounding_identifies_visible_entities() {
let mut app = OpenCADStudio::new_for_test();
app.automation_op(r#"{"op":"new"}"#);
app.automation_op(r#"{"op":"run","cmd":"CIRCLE 0,0 5"}"#);
app.automation_op(r#"{"op":"run","cmd":"LINE -10,-10 10,10"}"#);
// Set test canvas viewport size
app.tabs[0].scene.selection.borrow_mut().vp_size = (800.0, 600.0);
app.tabs[0].scene.fit_all();
let grounding = compute_grounding(&app, 800.0, 600.0, 800, 600);
// Viewport world bounds should be valid
assert!(grounding.viewport_world_bounds["min"].is_array());
assert!(grounding.viewport_world_bounds["max"].is_array());
// Camera info should be populated
assert!(grounding.camera["eye"].is_array());
assert!(grounding.camera["target"].is_array());
// Both entities should be visible and tagged
assert_eq!(grounding.visible_entities.len(), 2);
let first = &grounding.visible_entities[0];
assert_eq!(first["tag"], 1);
assert!(first["handle"].is_string());
assert!(first["type"].is_string());
assert!(first["screen_pixel"].is_array());
assert!(first["screen_bounds"].is_array());
assert!(first["world_center"].is_array());
let second = &grounding.visible_entities[1];
assert_eq!(second["tag"], 2);
}
}
+59 -5
View File
@@ -45,6 +45,7 @@ const READ_OPS: &[&str] = &[
"audit",
"text_search",
"text_audit",
"capture",
];
const EXECUTE_OPS: &[&str] = &[
"new",
@@ -849,8 +850,29 @@ fn call_tool(
let path = std::env::temp_dir().join(format!("ocs-capture-{}.png", random_id()?));
let scope = arguments["scope"].as_str().unwrap_or("viewport");
let max_dimension = arguments["max_dimension"].as_u64().unwrap_or(1600);
let mut req = json!({
"op": "capture",
"path": path.to_string_lossy(),
"scope": scope,
"max_dimension": max_dimension,
});
if let Some(view) = arguments.get("view").and_then(Value::as_str) {
req["view"] = json!(view);
}
if let Some(bounds) = arguments.get("bounds").and_then(Value::as_array) {
req["bounds"] = json!(bounds);
}
if let Some(focus) = arguments.get("focus_handles").and_then(Value::as_array) {
req["focus_handles"] = json!(focus);
}
if let Some(hl) = arguments.get("highlight_handles").and_then(Value::as_array) {
req["highlight_handles"] = json!(hl);
}
if let Some(annotate) = arguments.get("annotate").and_then(Value::as_bool) {
req["annotate"] = json!(annotate);
}
let result = client(clients, session_id)?
.request(json!({"op":"capture","path":path.to_string_lossy(),"scope":scope,"max_dimension":max_dimension}), 30.0)?;
.request(req, 30.0)?;
if result["ok"].as_bool() != Some(true)
|| result["status"].as_str() != Some("completed")
{
@@ -858,7 +880,14 @@ fn call_tool(
}
let bytes = std::fs::read(&path).map_err(|error| error.to_string())?;
let _ = std::fs::remove_file(path);
Ok(json!({"$image":BASE64.encode(bytes)}))
let mut meta = result.get("result").cloned().unwrap_or_else(|| json!({}));
if let Some(obj) = meta.as_object_mut() {
obj.remove("path");
}
Ok(json!({
"$image": BASE64.encode(bytes),
"metadata": meta,
}))
}
_ => Err(format!("Unknown tool: {name}")),
}
@@ -924,8 +953,22 @@ fn tool_definitions() -> Value {
},
{
"name":"ocs_capture",
"description":"Capture the actual current OCS drawing viewport or window as a bounded PNG for visual verification.",
"inputSchema":{"type":"object","properties":{"ocs_session_id":{"type":"string","minLength":1,"description":"Value of session_id returned by ocs_sessions."},"scope":{"type":"string","enum":["viewport","window"],"default":"viewport","description":"Capture only the drawing viewport by default, or the complete application window."},"max_dimension":{"type":"integer","minimum":256,"maximum":4096,"default":1600,"description":"Resize the longest image edge to at most this many pixels."}},"required":["ocs_session_id"],"additionalProperties":false},
"description":"Capture the actual current OCS drawing viewport or window as a bounded PNG with optional camera framing, entity highlighting, spatial coordinates and Set-of-Marks annotations.",
"inputSchema":{
"type":"object",
"properties":{
"ocs_session_id":{"type":"string","minLength":1,"description":"Value of session_id returned by ocs_sessions."},
"scope":{"type":"string","enum":["viewport","window"],"default":"viewport","description":"Capture only the drawing viewport by default, or the complete application window."},
"max_dimension":{"type":"integer","minimum":256,"maximum":4096,"default":1600,"description":"Resize the longest image edge to at most this many pixels."},
"view":{"type":"string","enum":["current","extents","selection","region"],"default":"current","description":"Frame the camera before capture: extents fits all entities, selection fits selected entities, region fits explicit world bounds."},
"bounds":{"type":"array","items":{"type":"number"},"minItems":4,"maxItems":4,"description":"World XY bounding box [min_x, min_y, max_x, max_y] to zoom and fit in view before capturing (used with view: 'region')."},
"focus_handles":{"type":"array","items":{"type":"string"},"description":"Hex handles of entities to zoom and fit in view before capturing."},
"highlight_handles":{"type":"array","items":{"type":"string"},"description":"Hex handles of entities to select/highlight before capturing."},
"annotate":{"type":"boolean","default":false,"description":"Overlay Set-of-Marks numbered tags on visible entities for visual grounding."}
},
"required":["ocs_session_id"],
"additionalProperties":false
},
"annotations":{"title":"Capture OCS window","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false}
}
])
@@ -933,7 +976,18 @@ fn tool_definitions() -> Value {
fn tool_result(value: Value) -> Value {
if let Some(image) = value.get("$image").and_then(Value::as_str) {
return json!({"content":[{"type":"image","data":image,"mimeType":"image/png"}]});
let mut content = Vec::new();
if let Some(meta) = value.get("metadata") {
let meta_text = serde_json::to_string_pretty(meta).unwrap_or_else(|_| meta.to_string());
content.push(json!({"type":"text","text":meta_text}));
}
content.push(json!({"type":"image","data":image,"mimeType":"image/png"}));
let structured = value.get("metadata").cloned().unwrap_or_else(|| json!({}));
return json!({
"content": content,
"structuredContent": structured,
"isError": false
});
}
let structured = if value.is_object() {
value.clone()
+2
View File
@@ -245,6 +245,8 @@ pub static EMBED_IMAGE: OpDef = OpDef {
opt("at", Ty::Point, "Lower-left corner [x, y] or [x, y, z]."),
opt("width", Ty::Num, "World width of the image."),
opt("linked", Ty::Bool, "Store as path-linked RasterImage instead of embedded OLE2FRAME."),
opt("source_points", Ty::ArrayOf(&Ty::Point), "Pixel points in image [[px1,py1],[px2,py2]] for 2-point alignment."),
opt("target_points", Ty::ArrayOf(&Ty::Point), "CAD world points [[x1,y1],[x2,y2]] for 2-point alignment."),
],
rules: &[],
example: r#"{"op":"embed_image","path":"logo.png","at":[0,0]}"#,