forked from 1b-agent/opencadstudio-upstream
feat(mcp): add Set-of-Marks grounding, regional framing, and 2-point image calibration
- Add Set-of-Marks (SoM) visual grounding (src/app/control/vision.rs): * Render high-contrast numbered badges on entity visual centroids in viewport captures * Project 3D entity bounds to 2D image coordinates with priority ranking (selected/size) * Return bidirectional grounding manifest (annotations / visible_entities) in capture and ocs_capture payloads mapping each badge to entity handle, layer, type, and world center - Add regional viewport framing (src/app/control/mod.rs, src/mcp.rs): * Support view: region and bounds: [min_x, min_y, max_x, max_y] in capture and ocs_capture * Enable programmatic zoom and high-resolution inspection of specific rooms or local geometry - Add 2-point raster image calibration (src/app/control/actions.rs, src/mcp_ops.rs): * Extend embed_image with optional source_points (pixel coords) and target_points (CAD coords) * Automatically calculate scale, world dimensions, and insertion origin at * Add unit test embed_image_calibrates_from_source_and_target_points
This commit is contained in:
1 parent
9fc4e381b3
commit
6cc53c1001
5 files changed
+585
-12
No files matched your search
@@ -115,20 +115,73 @@ fn embed_point(req: &Value) -> Result<codec::types::Vector3, Value> {
|
||||
}
|
||||
Ok(codec::types::Vector3::new(p[0], p[1], p[2]))
|
||||
}
|
||||
fn parse_point_2d(val: &Value) -> Result<[f64; 2], Value> {
|
||||
let arr = val
|
||||
.as_array()
|
||||
.filter(|v| v.len() >= 2)
|
||||
.ok_or_else(|| failure("invalid_point", "Expected [x, y]"))?;
|
||||
let x = arr[0]
|
||||
.as_f64()
|
||||
.filter(|v| v.is_finite())
|
||||
.ok_or_else(|| failure("invalid_point", "Expected finite coordinates"))?;
|
||||
let y = arr[1]
|
||||
.as_f64()
|
||||
.filter(|v| v.is_finite())
|
||||
.ok_or_else(|| failure("invalid_point", "Expected finite coordinates"))?;
|
||||
Ok([x, y])
|
||||
}
|
||||
|
||||
impl OpenCADStudio {
|
||||
/// `embed_image` — pack the picture at `path` into an OLE2FRAME placed
|
||||
/// with its lower-left corner at `at` (default width ≈ pixel_width/100,
|
||||
/// like the interactive command's Enter answer) and commit it as one
|
||||
/// undo step. The drawing stays self-contained: no external file to lose.
|
||||
/// Supports automatic 2-point calibration via `source_points` and `target_points`.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub(super) fn control_embed_image(&mut self, req: &Value) -> Result<Task<Message>, Value> {
|
||||
let i = self.active_tab;
|
||||
let path = string(req, "path")?;
|
||||
let image = crate::io::ole_embed::EmbeddedImage::from_file(std::path::Path::new(&path))
|
||||
.map_err(|e| failure("embed_failed", e))?;
|
||||
let at = embed_point(req)?;
|
||||
let default_width = (image.pixel_width as f64 / 100.0).max(1.0);
|
||||
let width = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(default_width);
|
||||
let (at, width) = if let (Some(src_pts), Some(tgt_pts)) = (
|
||||
req.get("source_points").and_then(Value::as_array),
|
||||
req.get("target_points").and_then(Value::as_array),
|
||||
) {
|
||||
if src_pts.len() >= 2 && tgt_pts.len() >= 2 {
|
||||
let p_s1 = parse_point_2d(&src_pts[0])?;
|
||||
let p_s2 = parse_point_2d(&src_pts[1])?;
|
||||
let p_t1 = parse_point_2d(&tgt_pts[0])?;
|
||||
let p_t2 = parse_point_2d(&tgt_pts[1])?;
|
||||
let d_px = ((p_s2[0] - p_s1[0]).powi(2) + (p_s2[1] - p_s1[1]).powi(2)).sqrt();
|
||||
let d_cad = ((p_t2[0] - p_t1[0]).powi(2) + (p_t2[1] - p_t1[1]).powi(2)).sqrt();
|
||||
if d_px <= 1e-6 {
|
||||
return Err(failure(
|
||||
"invalid_points",
|
||||
"Source calibration points must be distinct",
|
||||
));
|
||||
}
|
||||
let scale = d_cad / d_px;
|
||||
let w = image.pixel_width as f64 * scale;
|
||||
// In image pixel coords, y is 0 at top and pixel_height at bottom.
|
||||
// In CAD coordinates, lower-left is (x_ll, y_ll).
|
||||
// x_target = x_ll + px * scale => x_ll = x_target - px * scale
|
||||
// y_target = y_ll + (pixel_height - py) * scale => y_ll = y_target - (pixel_height - py) * scale
|
||||
let x_ll = p_t1[0] - p_s1[0] * scale;
|
||||
let y_ll = p_t1[1] - (image.pixel_height as f64 - p_s1[1]) * scale;
|
||||
let calc_at = codec::types::Vector3::new(x_ll, y_ll, 0.0);
|
||||
let final_at = embed_point(req).unwrap_or(calc_at);
|
||||
let final_w = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(w);
|
||||
(final_at, final_w)
|
||||
} else {
|
||||
let default_width = (image.pixel_width as f64 / 100.0).max(1.0);
|
||||
let width = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(default_width);
|
||||
(embed_point(req)?, width)
|
||||
}
|
||||
} else {
|
||||
let default_width = (image.pixel_width as f64 / 100.0).max(1.0);
|
||||
let width = req["width"].as_f64().filter(|w| *w > 0.0).unwrap_or(default_width);
|
||||
(embed_point(req)?, width)
|
||||
};
|
||||
if req["linked"].as_bool().unwrap_or(false) {
|
||||
// Path-linked RasterImage + ImageDefinition: the drawing stores
|
||||
// only the file path, so the picture file must travel with the
|
||||
@@ -167,6 +220,7 @@ impl OpenCADStudio {
|
||||
"handle": format!("{:X}", handle.value()),
|
||||
"kind": "RasterImage",
|
||||
"path": path,
|
||||
"at": [at.x, at.y, at.z],
|
||||
"width": width,
|
||||
"height": height,
|
||||
"linked": true,
|
||||
@@ -186,6 +240,7 @@ impl OpenCADStudio {
|
||||
"handle": format!("{:X}", handle.value()),
|
||||
"kind": "Ole2Frame",
|
||||
"path": path,
|
||||
"at": [at.x, at.y, at.z],
|
||||
"width": width,
|
||||
}));
|
||||
Ok(Task::none())
|
||||
|
||||
+146
-4
@@ -1058,6 +1058,57 @@ impl OpenCADStudio {
|
||||
.main_window
|
||||
.ok_or_else(|| failure("gui_required", "Capture requires a GUI window"))?;
|
||||
let path = string(req, "path")?.to_owned();
|
||||
|
||||
// Optional framing / selection adjustments prior to snapshot:
|
||||
if let Some(handles) = req.get("highlight_handles").and_then(Value::as_array) {
|
||||
let hex_handles: Vec<codec::Handle> = handles
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.filter_map(|v| u64::from_str_radix(v.trim_start_matches("0x"), 16).ok())
|
||||
.map(codec::Handle::new)
|
||||
.collect();
|
||||
if !hex_handles.is_empty() {
|
||||
let scene = &mut self.tabs[self.active_tab].scene;
|
||||
scene.deselect_all();
|
||||
for h in hex_handles {
|
||||
scene.select_entity(h, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(focus) = req.get("focus_handles").and_then(Value::as_array) {
|
||||
let hex_handles: Vec<codec::Handle> = focus
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.filter_map(|v| u64::from_str_radix(v.trim_start_matches("0x"), 16).ok())
|
||||
.map(codec::Handle::new)
|
||||
.collect();
|
||||
if !hex_handles.is_empty() {
|
||||
self.tabs[self.active_tab].scene.zoom_to_entities(&hex_handles);
|
||||
}
|
||||
} else if let Some(bounds) = req.get("bounds").and_then(Value::as_array) {
|
||||
if bounds.len() >= 4 {
|
||||
if let (Some(x0), Some(y0), Some(x1), Some(y1)) = (
|
||||
bounds[0].as_f64(),
|
||||
bounds[1].as_f64(),
|
||||
bounds[2].as_f64(),
|
||||
bounds[3].as_f64(),
|
||||
) {
|
||||
let p1 = glam::Vec3::new(x0 as f32, y0 as f32, 0.0);
|
||||
let p2 = glam::Vec3::new(x1 as f32, y1 as f32, 0.0);
|
||||
self.tabs[self.active_tab].scene.zoom_to_window(p1, p2);
|
||||
}
|
||||
}
|
||||
} else if let Some(view) = req.get("view").and_then(Value::as_str) {
|
||||
if view == "extents" {
|
||||
self.tabs[self.active_tab].scene.fit_all();
|
||||
} else if view == "selection" {
|
||||
let selected = self.tabs[self.active_tab].scene.selected_handles_in_order();
|
||||
if !selected.is_empty() {
|
||||
self.tabs[self.active_tab].scene.zoom_to_entities(&selected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A minimized window has a 0x0 surface and the renderer
|
||||
// panics reading it back, so report instead of capturing.
|
||||
iced::window::size(window).then(move |size| {
|
||||
@@ -1270,7 +1321,7 @@ impl OpenCADStudio {
|
||||
.pending
|
||||
.as_ref()
|
||||
.and_then(|pending| pending.request["scope"].as_str())
|
||||
.unwrap_or("window");
|
||||
.unwrap_or("viewport");
|
||||
let max_dimension = self
|
||||
.control
|
||||
.pending
|
||||
@@ -1278,10 +1329,22 @@ impl OpenCADStudio {
|
||||
.and_then(|pending| pending.request["max_dimension"].as_u64())
|
||||
.unwrap_or(1600)
|
||||
.clamp(256, 4096) as u32;
|
||||
let annotate = self
|
||||
.control
|
||||
.pending
|
||||
.as_ref()
|
||||
.and_then(|pending| pending.request["annotate"].as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
let mut image =
|
||||
image::RgbaImage::from_raw(s.size.width, s.size.height, s.rgba.to_vec())
|
||||
.ok_or("Renderer returned malformed image data")?;
|
||||
let mut actual_scope = "window";
|
||||
|
||||
let (vp_w, vp_h) = self.tabs[self.active_tab].scene.selection.borrow().vp_size;
|
||||
let mut vp_logical_width = if vp_w > 0.0 { vp_w } else { image.width() as f32 / s.scale_factor };
|
||||
let mut vp_logical_height = if vp_h > 0.0 { vp_h } else { image.height() as f32 / s.scale_factor };
|
||||
|
||||
if requested_scope == "viewport" {
|
||||
if let Some(bounds) = crate::ui::wrap_bar::dropdown_bounds(
|
||||
crate::app::view::VIEWPORT_CAPTURE_BOUNDS_ID,
|
||||
@@ -1305,6 +1368,8 @@ impl OpenCADStudio {
|
||||
)
|
||||
.to_image();
|
||||
actual_scope = "viewport";
|
||||
vp_logical_width = bounds.width;
|
||||
vp_logical_height = bounds.height;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1320,12 +1385,58 @@ impl OpenCADStudio {
|
||||
image::imageops::FilterType::Triangle,
|
||||
);
|
||||
}
|
||||
|
||||
// Spatial grounding: camera, world bounds, and visible entity centroids
|
||||
let grounding = vision::compute_grounding(
|
||||
self,
|
||||
vp_logical_width,
|
||||
vp_logical_height,
|
||||
image.width(),
|
||||
image.height(),
|
||||
);
|
||||
|
||||
// Optional Set-of-Marks visual overlay badges
|
||||
if annotate {
|
||||
for ent_val in &grounding.visible_entities {
|
||||
if let (Some(tag), Some(px_arr), Some(selected)) = (
|
||||
ent_val["tag"].as_u64(),
|
||||
ent_val["screen_pixel"].as_array(),
|
||||
ent_val["selected"].as_bool(),
|
||||
) {
|
||||
if px_arr.len() == 2 {
|
||||
if let (Some(x), Some(y)) = (px_arr[0].as_i64(), px_arr[1].as_i64()) {
|
||||
vision::draw_som_badge(
|
||||
&mut image,
|
||||
x as i32,
|
||||
y as i32,
|
||||
tag as usize,
|
||||
selected,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
image
|
||||
.save_with_format(&path, image::ImageFormat::Png)
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(
|
||||
json!({"path":path,"scope":actual_scope,"width":image.width(),"height":image.height(),"scale_factor":s.scale_factor,"document_id":self.tabs[self.active_tab].id,"revision":self.tabs[self.active_tab].edit_revision,"camera_revision":self.tabs[self.active_tab].scene.camera_generation}),
|
||||
)
|
||||
|
||||
Ok(json!({
|
||||
"path": path,
|
||||
"scope": actual_scope,
|
||||
"width": image.width(),
|
||||
"height": image.height(),
|
||||
"scale_factor": s.scale_factor,
|
||||
"document_id": self.tabs[self.active_tab].id,
|
||||
"revision": self.tabs[self.active_tab].edit_revision,
|
||||
"camera_revision": self.tabs[self.active_tab].scene.camera_generation,
|
||||
"viewport_world_bounds": grounding.viewport_world_bounds,
|
||||
"camera": grounding.camera,
|
||||
"annotated": annotate,
|
||||
"annotations": grounding.visible_entities.clone(),
|
||||
"visible_entities": grounding.visible_entities,
|
||||
}))
|
||||
})();
|
||||
match result {
|
||||
Ok(v) => {
|
||||
@@ -1337,6 +1448,7 @@ impl OpenCADStudio {
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(crate) mod vision;
|
||||
mod actions;
|
||||
pub(crate) use actions::property_json;
|
||||
mod entities;
|
||||
@@ -1763,6 +1875,36 @@ mod tests {
|
||||
let _ = handle;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embed_image_calibrates_from_source_and_target_points() {
|
||||
let mut app = OpenCADStudio::new_for_test();
|
||||
request(&mut app, json!({"op":"new"}));
|
||||
let img = image::RgbaImage::from_pixel(100, 100, image::Rgba([12, 12, 12, 255]));
|
||||
let path = std::env::temp_dir().join(format!("ocs-embed-calib-{}.png", session_id()));
|
||||
img.save(&path).unwrap();
|
||||
|
||||
// 2 points on image: bottom-left (0, 100) and bottom-right (100, 100)
|
||||
// Target CAD points: (10.0, 20.0) and (210.0, 20.0) -> scale = 200/100 = 2.0
|
||||
let response = request(
|
||||
&mut app,
|
||||
json!({
|
||||
"op": "embed_image",
|
||||
"path": path.to_string_lossy(),
|
||||
"linked": true,
|
||||
"source_points": [[0, 100], [100, 100]],
|
||||
"target_points": [[10.0, 20.0], [210.0, 20.0]]
|
||||
}),
|
||||
);
|
||||
assert_eq!(response["status"], "completed", "{response}");
|
||||
assert_eq!(response["result"]["kind"], "RasterImage");
|
||||
assert_eq!(response["result"]["width"], 200.0);
|
||||
assert_eq!(response["result"]["height"], 200.0);
|
||||
let at = response["result"]["at"].as_array().unwrap();
|
||||
assert_eq!(at[0], 10.0);
|
||||
assert_eq!(at[1], 20.0);
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
fn user_select_pick(app: &mut OpenCADStudio, kind: &str) -> String {
|
||||
let handle = app.automation_op(
|
||||
format!(r#"{{"op":"query","type":"{kind}","detail":"summary"}}"#).as_str(),
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
use crate::app::OpenCADStudio;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
/// 5x7 bitmapped font glyphs for digits '0' through '9'
|
||||
const DIGITS_5X7: [[u8; 7]; 10] = [
|
||||
[0b01110, 0b10001, 0b10011, 0b10101, 0b11001, 0b10001, 0b01110], // 0
|
||||
[0b00100, 0b01100, 0b00100, 0b00100, 0b00100, 0b00100, 0b01110], // 1
|
||||
[0b01110, 0b10001, 0b00001, 0b00110, 0b01000, 0b10000, 0b11111], // 2
|
||||
[0b11110, 0b00001, 0b00001, 0b01110, 0b00001, 0b00001, 0b11110], // 3
|
||||
[0b00010, 0b00110, 0b01010, 0b10010, 0b11111, 0b00010, 0b00010], // 4
|
||||
[0b11111, 0b10000, 0b11110, 0b00001, 0b00001, 0b10001, 0b01110], // 5
|
||||
[0b00110, 0b01000, 0b10000, 0b11110, 0b10001, 0b10001, 0b01110], // 6
|
||||
[0b11111, 0b00001, 0b00010, 0b00100, 0b01000, 0b01000, 0b01000], // 7
|
||||
[0b01110, 0b10001, 0b10001, 0b01110, 0b10001, 0b10001, 0b01110], // 8
|
||||
[0b01110, 0b10001, 0b10001, 0b01111, 0b00001, 0b00010, 0b01100], // 9
|
||||
];
|
||||
|
||||
/// Draw a Set-of-Marks badge at `(center_x, center_y)` onto `image`.
|
||||
pub(crate) fn draw_som_badge(
|
||||
image: &mut image::RgbaImage,
|
||||
center_x: i32,
|
||||
center_y: i32,
|
||||
tag: usize,
|
||||
selected: bool,
|
||||
) {
|
||||
let tag_str = tag.to_string();
|
||||
let num_digits = tag_str.len();
|
||||
if num_digits == 0 {
|
||||
return;
|
||||
}
|
||||
let text_w = num_digits * 5 + (num_digits - 1) * 1;
|
||||
let badge_w = (text_w + 8) as i32;
|
||||
let badge_h = 13i32;
|
||||
|
||||
let img_w = image.width() as i32;
|
||||
let img_h = image.height() as i32;
|
||||
|
||||
// Anchor badge slightly above the center anchor point if possible
|
||||
let x0 = (center_x - badge_w / 2).clamp(1, (img_w - badge_w - 1).max(1));
|
||||
let y0 = (center_y - badge_h - 2).clamp(1, (img_h - badge_h - 1).max(1));
|
||||
|
||||
let (bg, border, text_col) = if selected {
|
||||
(
|
||||
image::Rgba([0, 230, 255, 245]), // Electric cyan
|
||||
image::Rgba([0, 60, 80, 255]),
|
||||
image::Rgba([0, 0, 0, 255]),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
image::Rgba([255, 220, 0, 245]), // Vibrant yellow
|
||||
image::Rgba([40, 30, 0, 255]),
|
||||
image::Rgba([0, 0, 0, 255]),
|
||||
)
|
||||
};
|
||||
|
||||
// Fill badge background and border
|
||||
for y in y0..(y0 + badge_h) {
|
||||
for x in x0..(x0 + badge_w) {
|
||||
if x < 0 || x >= img_w || y < 0 || y >= img_h {
|
||||
continue;
|
||||
}
|
||||
let is_border = x == x0 || x == x0 + badge_w - 1 || y == y0 || y == y0 + badge_h - 1;
|
||||
let col = if is_border { border } else { bg };
|
||||
image.put_pixel(x as u32, y as u32, col);
|
||||
}
|
||||
}
|
||||
|
||||
// Draw digits
|
||||
let mut cur_x = x0 + 4;
|
||||
let cur_y = y0 + 3;
|
||||
for ch in tag_str.chars() {
|
||||
if let Some(digit) = ch.to_digit(10) {
|
||||
let d = digit as usize;
|
||||
if d < 10 {
|
||||
for r in 0..7 {
|
||||
let row_bits = DIGITS_5X7[d][r];
|
||||
for c in 0..5 {
|
||||
if (row_bits & (1 << (4 - c))) != 0 {
|
||||
let px = cur_x + c as i32;
|
||||
let py = cur_y + r as i32;
|
||||
if px >= 0 && px < img_w && py >= 0 && py < img_h {
|
||||
image.put_pixel(px as u32, py as u32, text_col);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cur_x += 6;
|
||||
}
|
||||
|
||||
// Small 3x3 indicator dot at the center anchor
|
||||
let dot_col = if selected {
|
||||
image::Rgba([0, 230, 255, 255])
|
||||
} else {
|
||||
image::Rgba([255, 60, 0, 255])
|
||||
};
|
||||
for dy in -1..=1 {
|
||||
for dx in -1..=1 {
|
||||
let px = center_x + dx;
|
||||
let py = center_y + dy;
|
||||
if px >= 0 && px < img_w && py >= 0 && py < img_h {
|
||||
image.put_pixel(px as u32, py as u32, dot_col);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct VisionGrounding {
|
||||
pub viewport_world_bounds: Value,
|
||||
pub camera: Value,
|
||||
pub visible_entities: Vec<Value>,
|
||||
}
|
||||
|
||||
pub(crate) fn compute_grounding(
|
||||
app: &OpenCADStudio,
|
||||
vp_logical_width: f32,
|
||||
vp_logical_height: f32,
|
||||
image_width: u32,
|
||||
image_height: u32,
|
||||
) -> VisionGrounding {
|
||||
let tab = &app.tabs[app.active_tab];
|
||||
let c = tab.scene.camera.borrow();
|
||||
let vp_rect = iced::Rectangle {
|
||||
x: 0.0,
|
||||
y: 0.0,
|
||||
width: vp_logical_width.max(1.0),
|
||||
height: vp_logical_height.max(1.0),
|
||||
};
|
||||
|
||||
let p_tl = c.pick_on_target_plane(iced::Point::new(0.0, 0.0), vp_rect);
|
||||
let p_tr = c.pick_on_target_plane(iced::Point::new(vp_rect.width, 0.0), vp_rect);
|
||||
let p_bl = c.pick_on_target_plane(iced::Point::new(0.0, vp_rect.height), vp_rect);
|
||||
let p_br = c.pick_on_target_plane(iced::Point::new(vp_rect.width, vp_rect.height), vp_rect);
|
||||
|
||||
let min_x = p_tl.x.min(p_tr.x).min(p_bl.x).min(p_br.x);
|
||||
let max_x = p_tl.x.max(p_tr.x).max(p_bl.x).max(p_br.x);
|
||||
let min_y = p_tl.y.min(p_tr.y).min(p_bl.y).min(p_br.y);
|
||||
let max_y = p_tl.y.max(p_tr.y).max(p_bl.y).max(p_br.y);
|
||||
|
||||
let viewport_world_bounds = json!({
|
||||
"min": [min_x, min_y],
|
||||
"max": [max_x, max_y]
|
||||
});
|
||||
|
||||
let eye = c.eye();
|
||||
let target = c.target;
|
||||
let camera_info = json!({
|
||||
"eye": [eye.x, eye.y, eye.z],
|
||||
"target": [target.x, target.y, target.z],
|
||||
"distance": c.distance,
|
||||
"ortho_size": c.ortho_size(),
|
||||
});
|
||||
|
||||
let img_w = image_width as f32;
|
||||
let img_h = image_height as f32;
|
||||
|
||||
struct Candidate {
|
||||
handle: codec::Handle,
|
||||
entity_type: &'static str,
|
||||
layer: String,
|
||||
screen_pixel: [i32; 2],
|
||||
screen_bounds: [i32; 4],
|
||||
world_center: [f64; 3],
|
||||
selected: bool,
|
||||
area: f32,
|
||||
}
|
||||
|
||||
let selected_set: std::collections::HashSet<codec::Handle> =
|
||||
tab.scene.selected_handles_in_order().into_iter().collect();
|
||||
|
||||
let mut candidates = Vec::new();
|
||||
|
||||
for entity in tab.scene.document.entities() {
|
||||
let handle = entity.common().handle;
|
||||
let (b_min, b_max) = crate::scene::convert::tess::entity_bounds_in(&tab.scene.document, entity);
|
||||
let min_dvec = glam::DVec3::from(b_min);
|
||||
let max_dvec = glam::DVec3::from(b_max);
|
||||
if !min_dvec.is_finite() || !max_dvec.is_finite() {
|
||||
continue;
|
||||
}
|
||||
let center = (min_dvec + max_dvec) * 0.5;
|
||||
if let Some(screen_pt) = c.project(center, vp_rect) {
|
||||
if screen_pt.x >= 0.0
|
||||
&& screen_pt.x <= vp_rect.width
|
||||
&& screen_pt.y >= 0.0
|
||||
&& screen_pt.y <= vp_rect.height
|
||||
{
|
||||
let px = (screen_pt.x / vp_rect.width * img_w).round() as i32;
|
||||
let py = (screen_pt.y / vp_rect.height * img_h).round() as i32;
|
||||
|
||||
let mut s_min_x = px;
|
||||
let mut s_max_x = px;
|
||||
let mut s_min_y = py;
|
||||
let mut s_max_y = py;
|
||||
|
||||
if let (Some(p0), Some(p1)) =
|
||||
(c.project(min_dvec, vp_rect), c.project(max_dvec, vp_rect))
|
||||
{
|
||||
let x0 = (p0.x / vp_rect.width * img_w).round() as i32;
|
||||
let y0 = (p0.y / vp_rect.height * img_h).round() as i32;
|
||||
let x1 = (p1.x / vp_rect.width * img_w).round() as i32;
|
||||
let y1 = (p1.y / vp_rect.height * img_h).round() as i32;
|
||||
s_min_x = x0.min(x1);
|
||||
s_max_x = x0.max(x1);
|
||||
s_min_y = y0.min(y1);
|
||||
s_max_y = y0.max(y1);
|
||||
}
|
||||
|
||||
let area = ((s_max_x - s_min_x).abs() * (s_max_y - s_min_y).abs()) as f32;
|
||||
let selected = selected_set.contains(&handle);
|
||||
let layer = entity.common().layer.clone();
|
||||
|
||||
candidates.push(Candidate {
|
||||
handle,
|
||||
entity_type: crate::entities::names::ui_name(entity),
|
||||
layer,
|
||||
screen_pixel: [px, py],
|
||||
screen_bounds: [s_min_x, s_min_y, s_max_x, s_max_y],
|
||||
world_center: [center.x, center.y, center.z],
|
||||
selected,
|
||||
area,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Selected entities prioritized, then larger visual area first
|
||||
candidates.sort_by(|a, b| {
|
||||
b.selected
|
||||
.cmp(&a.selected)
|
||||
.then_with(|| b.area.partial_cmp(&a.area).unwrap_or(std::cmp::Ordering::Equal))
|
||||
});
|
||||
|
||||
candidates.truncate(64);
|
||||
|
||||
let visible_entities = candidates
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(idx, cand)| {
|
||||
json!({
|
||||
"tag": idx + 1,
|
||||
"handle": format!("{:X}", cand.handle.value()),
|
||||
"type": cand.entity_type,
|
||||
"layer": cand.layer,
|
||||
"screen_pixel": cand.screen_pixel,
|
||||
"screen_bounds": cand.screen_bounds,
|
||||
"world_center": cand.world_center,
|
||||
"selected": cand.selected
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
VisionGrounding {
|
||||
viewport_world_bounds,
|
||||
camera: camera_info,
|
||||
visible_entities,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_draw_som_badge_renders_without_panics() {
|
||||
let mut img = image::RgbaImage::from_pixel(120, 100, image::Rgba([40, 40, 40, 255]));
|
||||
|
||||
// Render normal badge
|
||||
draw_som_badge(&mut img, 50, 50, 1, false);
|
||||
|
||||
// Center dot should be drawn
|
||||
let dot = img.get_pixel(50, 50);
|
||||
assert_eq!(*dot, image::Rgba([255, 60, 0, 255]));
|
||||
|
||||
// Render selected badge with multi-digit tag near edge (clamped)
|
||||
draw_som_badge(&mut img, 2, 2, 42, true);
|
||||
let selected_dot = img.get_pixel(2, 2);
|
||||
assert_eq!(*selected_dot, image::Rgba([0, 230, 255, 255]));
|
||||
|
||||
// Right edge clamp
|
||||
draw_som_badge(&mut img, 118, 98, 99, false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compute_grounding_identifies_visible_entities() {
|
||||
let mut app = OpenCADStudio::new_for_test();
|
||||
app.automation_op(r#"{"op":"new"}"#);
|
||||
app.automation_op(r#"{"op":"run","cmd":"CIRCLE 0,0 5"}"#);
|
||||
app.automation_op(r#"{"op":"run","cmd":"LINE -10,-10 10,10"}"#);
|
||||
|
||||
// Set test canvas viewport size
|
||||
app.tabs[0].scene.selection.borrow_mut().vp_size = (800.0, 600.0);
|
||||
app.tabs[0].scene.fit_all();
|
||||
|
||||
let grounding = compute_grounding(&app, 800.0, 600.0, 800, 600);
|
||||
|
||||
// Viewport world bounds should be valid
|
||||
assert!(grounding.viewport_world_bounds["min"].is_array());
|
||||
assert!(grounding.viewport_world_bounds["max"].is_array());
|
||||
|
||||
// Camera info should be populated
|
||||
assert!(grounding.camera["eye"].is_array());
|
||||
assert!(grounding.camera["target"].is_array());
|
||||
|
||||
// Both entities should be visible and tagged
|
||||
assert_eq!(grounding.visible_entities.len(), 2);
|
||||
|
||||
let first = &grounding.visible_entities[0];
|
||||
assert_eq!(first["tag"], 1);
|
||||
assert!(first["handle"].is_string());
|
||||
assert!(first["type"].is_string());
|
||||
assert!(first["screen_pixel"].is_array());
|
||||
assert!(first["screen_bounds"].is_array());
|
||||
assert!(first["world_center"].is_array());
|
||||
|
||||
let second = &grounding.visible_entities[1];
|
||||
assert_eq!(second["tag"], 2);
|
||||
}
|
||||
}
|
||||
+59
-5
@@ -45,6 +45,7 @@ const READ_OPS: &[&str] = &[
|
||||
"audit",
|
||||
"text_search",
|
||||
"text_audit",
|
||||
"capture",
|
||||
];
|
||||
const EXECUTE_OPS: &[&str] = &[
|
||||
"new",
|
||||
@@ -849,8 +850,29 @@ fn call_tool(
|
||||
let path = std::env::temp_dir().join(format!("ocs-capture-{}.png", random_id()?));
|
||||
let scope = arguments["scope"].as_str().unwrap_or("viewport");
|
||||
let max_dimension = arguments["max_dimension"].as_u64().unwrap_or(1600);
|
||||
let mut req = json!({
|
||||
"op": "capture",
|
||||
"path": path.to_string_lossy(),
|
||||
"scope": scope,
|
||||
"max_dimension": max_dimension,
|
||||
});
|
||||
if let Some(view) = arguments.get("view").and_then(Value::as_str) {
|
||||
req["view"] = json!(view);
|
||||
}
|
||||
if let Some(bounds) = arguments.get("bounds").and_then(Value::as_array) {
|
||||
req["bounds"] = json!(bounds);
|
||||
}
|
||||
if let Some(focus) = arguments.get("focus_handles").and_then(Value::as_array) {
|
||||
req["focus_handles"] = json!(focus);
|
||||
}
|
||||
if let Some(hl) = arguments.get("highlight_handles").and_then(Value::as_array) {
|
||||
req["highlight_handles"] = json!(hl);
|
||||
}
|
||||
if let Some(annotate) = arguments.get("annotate").and_then(Value::as_bool) {
|
||||
req["annotate"] = json!(annotate);
|
||||
}
|
||||
let result = client(clients, session_id)?
|
||||
.request(json!({"op":"capture","path":path.to_string_lossy(),"scope":scope,"max_dimension":max_dimension}), 30.0)?;
|
||||
.request(req, 30.0)?;
|
||||
if result["ok"].as_bool() != Some(true)
|
||||
|| result["status"].as_str() != Some("completed")
|
||||
{
|
||||
@@ -858,7 +880,14 @@ fn call_tool(
|
||||
}
|
||||
let bytes = std::fs::read(&path).map_err(|error| error.to_string())?;
|
||||
let _ = std::fs::remove_file(path);
|
||||
Ok(json!({"$image":BASE64.encode(bytes)}))
|
||||
let mut meta = result.get("result").cloned().unwrap_or_else(|| json!({}));
|
||||
if let Some(obj) = meta.as_object_mut() {
|
||||
obj.remove("path");
|
||||
}
|
||||
Ok(json!({
|
||||
"$image": BASE64.encode(bytes),
|
||||
"metadata": meta,
|
||||
}))
|
||||
}
|
||||
_ => Err(format!("Unknown tool: {name}")),
|
||||
}
|
||||
@@ -924,8 +953,22 @@ fn tool_definitions() -> Value {
|
||||
},
|
||||
{
|
||||
"name":"ocs_capture",
|
||||
"description":"Capture the actual current OCS drawing viewport or window as a bounded PNG for visual verification.",
|
||||
"inputSchema":{"type":"object","properties":{"ocs_session_id":{"type":"string","minLength":1,"description":"Value of session_id returned by ocs_sessions."},"scope":{"type":"string","enum":["viewport","window"],"default":"viewport","description":"Capture only the drawing viewport by default, or the complete application window."},"max_dimension":{"type":"integer","minimum":256,"maximum":4096,"default":1600,"description":"Resize the longest image edge to at most this many pixels."}},"required":["ocs_session_id"],"additionalProperties":false},
|
||||
"description":"Capture the actual current OCS drawing viewport or window as a bounded PNG with optional camera framing, entity highlighting, spatial coordinates and Set-of-Marks annotations.",
|
||||
"inputSchema":{
|
||||
"type":"object",
|
||||
"properties":{
|
||||
"ocs_session_id":{"type":"string","minLength":1,"description":"Value of session_id returned by ocs_sessions."},
|
||||
"scope":{"type":"string","enum":["viewport","window"],"default":"viewport","description":"Capture only the drawing viewport by default, or the complete application window."},
|
||||
"max_dimension":{"type":"integer","minimum":256,"maximum":4096,"default":1600,"description":"Resize the longest image edge to at most this many pixels."},
|
||||
"view":{"type":"string","enum":["current","extents","selection","region"],"default":"current","description":"Frame the camera before capture: extents fits all entities, selection fits selected entities, region fits explicit world bounds."},
|
||||
"bounds":{"type":"array","items":{"type":"number"},"minItems":4,"maxItems":4,"description":"World XY bounding box [min_x, min_y, max_x, max_y] to zoom and fit in view before capturing (used with view: 'region')."},
|
||||
"focus_handles":{"type":"array","items":{"type":"string"},"description":"Hex handles of entities to zoom and fit in view before capturing."},
|
||||
"highlight_handles":{"type":"array","items":{"type":"string"},"description":"Hex handles of entities to select/highlight before capturing."},
|
||||
"annotate":{"type":"boolean","default":false,"description":"Overlay Set-of-Marks numbered tags on visible entities for visual grounding."}
|
||||
},
|
||||
"required":["ocs_session_id"],
|
||||
"additionalProperties":false
|
||||
},
|
||||
"annotations":{"title":"Capture OCS window","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false}
|
||||
}
|
||||
])
|
||||
@@ -933,7 +976,18 @@ fn tool_definitions() -> Value {
|
||||
|
||||
fn tool_result(value: Value) -> Value {
|
||||
if let Some(image) = value.get("$image").and_then(Value::as_str) {
|
||||
return json!({"content":[{"type":"image","data":image,"mimeType":"image/png"}]});
|
||||
let mut content = Vec::new();
|
||||
if let Some(meta) = value.get("metadata") {
|
||||
let meta_text = serde_json::to_string_pretty(meta).unwrap_or_else(|_| meta.to_string());
|
||||
content.push(json!({"type":"text","text":meta_text}));
|
||||
}
|
||||
content.push(json!({"type":"image","data":image,"mimeType":"image/png"}));
|
||||
let structured = value.get("metadata").cloned().unwrap_or_else(|| json!({}));
|
||||
return json!({
|
||||
"content": content,
|
||||
"structuredContent": structured,
|
||||
"isError": false
|
||||
});
|
||||
}
|
||||
let structured = if value.is_object() {
|
||||
value.clone()
|
||||
|
||||
@@ -245,6 +245,8 @@ pub static EMBED_IMAGE: OpDef = OpDef {
|
||||
opt("at", Ty::Point, "Lower-left corner [x, y] or [x, y, z]."),
|
||||
opt("width", Ty::Num, "World width of the image."),
|
||||
opt("linked", Ty::Bool, "Store as path-linked RasterImage instead of embedded OLE2FRAME."),
|
||||
opt("source_points", Ty::ArrayOf(&Ty::Point), "Pixel points in image [[px1,py1],[px2,py2]] for 2-point alignment."),
|
||||
opt("target_points", Ty::ArrayOf(&Ty::Point), "CAD world points [[x1,y1],[x2,y2]] for 2-point alignment."),
|
||||
],
|
||||
rules: &[],
|
||||
example: r#"{"op":"embed_image","path":"logo.png","at":[0,0]}"#,
|
||||
|
||||
Reference in new issue
Block a user