forked from 1b-agent/opencadstudio-upstream
feat(mcp): add Set-of-Marks grounding, regional framing, and 2-point image calibration
- Add Set-of-Marks (SoM) visual grounding (src/app/control/vision.rs): * Render high-contrast numbered badges on entity visual centroids in viewport captures * Project 3D entity bounds to 2D image coordinates with priority ranking (selected/size) * Return bidirectional grounding manifest (annotations / visible_entities) in capture and ocs_capture payloads mapping each badge to entity handle, layer, type, and world center - Add regional viewport framing (src/app/control/mod.rs, src/mcp.rs): * Support view: region and bounds: [min_x, min_y, max_x, max_y] in capture and ocs_capture * Enable programmatic zoom and high-resolution inspection of specific rooms or local geometry - Add 2-point raster image calibration (src/app/control/actions.rs, src/mcp_ops.rs): * Extend embed_image with optional source_points (pixel coords) and target_points (CAD coords) * Automatically calculate scale, world dimensions, and insertion origin at * Add unit test embed_image_calibrates_from_source_and_target_points
This commit is contained in:
1 parent
9fc4e381b3
commit
6cc53c1001
5 files changed
+585
-12
No files matched your search
+59
-5
@@ -45,6 +45,7 @@ const READ_OPS: &[&str] = &[
|
||||
"audit",
|
||||
"text_search",
|
||||
"text_audit",
|
||||
"capture",
|
||||
];
|
||||
const EXECUTE_OPS: &[&str] = &[
|
||||
"new",
|
||||
@@ -849,8 +850,29 @@ fn call_tool(
|
||||
let path = std::env::temp_dir().join(format!("ocs-capture-{}.png", random_id()?));
|
||||
let scope = arguments["scope"].as_str().unwrap_or("viewport");
|
||||
let max_dimension = arguments["max_dimension"].as_u64().unwrap_or(1600);
|
||||
let mut req = json!({
|
||||
"op": "capture",
|
||||
"path": path.to_string_lossy(),
|
||||
"scope": scope,
|
||||
"max_dimension": max_dimension,
|
||||
});
|
||||
if let Some(view) = arguments.get("view").and_then(Value::as_str) {
|
||||
req["view"] = json!(view);
|
||||
}
|
||||
if let Some(bounds) = arguments.get("bounds").and_then(Value::as_array) {
|
||||
req["bounds"] = json!(bounds);
|
||||
}
|
||||
if let Some(focus) = arguments.get("focus_handles").and_then(Value::as_array) {
|
||||
req["focus_handles"] = json!(focus);
|
||||
}
|
||||
if let Some(hl) = arguments.get("highlight_handles").and_then(Value::as_array) {
|
||||
req["highlight_handles"] = json!(hl);
|
||||
}
|
||||
if let Some(annotate) = arguments.get("annotate").and_then(Value::as_bool) {
|
||||
req["annotate"] = json!(annotate);
|
||||
}
|
||||
let result = client(clients, session_id)?
|
||||
.request(json!({"op":"capture","path":path.to_string_lossy(),"scope":scope,"max_dimension":max_dimension}), 30.0)?;
|
||||
.request(req, 30.0)?;
|
||||
if result["ok"].as_bool() != Some(true)
|
||||
|| result["status"].as_str() != Some("completed")
|
||||
{
|
||||
@@ -858,7 +880,14 @@ fn call_tool(
|
||||
}
|
||||
let bytes = std::fs::read(&path).map_err(|error| error.to_string())?;
|
||||
let _ = std::fs::remove_file(path);
|
||||
Ok(json!({"$image":BASE64.encode(bytes)}))
|
||||
let mut meta = result.get("result").cloned().unwrap_or_else(|| json!({}));
|
||||
if let Some(obj) = meta.as_object_mut() {
|
||||
obj.remove("path");
|
||||
}
|
||||
Ok(json!({
|
||||
"$image": BASE64.encode(bytes),
|
||||
"metadata": meta,
|
||||
}))
|
||||
}
|
||||
_ => Err(format!("Unknown tool: {name}")),
|
||||
}
|
||||
@@ -924,8 +953,22 @@ fn tool_definitions() -> Value {
|
||||
},
|
||||
{
|
||||
"name":"ocs_capture",
|
||||
"description":"Capture the actual current OCS drawing viewport or window as a bounded PNG for visual verification.",
|
||||
"inputSchema":{"type":"object","properties":{"ocs_session_id":{"type":"string","minLength":1,"description":"Value of session_id returned by ocs_sessions."},"scope":{"type":"string","enum":["viewport","window"],"default":"viewport","description":"Capture only the drawing viewport by default, or the complete application window."},"max_dimension":{"type":"integer","minimum":256,"maximum":4096,"default":1600,"description":"Resize the longest image edge to at most this many pixels."}},"required":["ocs_session_id"],"additionalProperties":false},
|
||||
"description":"Capture the actual current OCS drawing viewport or window as a bounded PNG with optional camera framing, entity highlighting, spatial coordinates and Set-of-Marks annotations.",
|
||||
"inputSchema":{
|
||||
"type":"object",
|
||||
"properties":{
|
||||
"ocs_session_id":{"type":"string","minLength":1,"description":"Value of session_id returned by ocs_sessions."},
|
||||
"scope":{"type":"string","enum":["viewport","window"],"default":"viewport","description":"Capture only the drawing viewport by default, or the complete application window."},
|
||||
"max_dimension":{"type":"integer","minimum":256,"maximum":4096,"default":1600,"description":"Resize the longest image edge to at most this many pixels."},
|
||||
"view":{"type":"string","enum":["current","extents","selection","region"],"default":"current","description":"Frame the camera before capture: extents fits all entities, selection fits selected entities, region fits explicit world bounds."},
|
||||
"bounds":{"type":"array","items":{"type":"number"},"minItems":4,"maxItems":4,"description":"World XY bounding box [min_x, min_y, max_x, max_y] to zoom and fit in view before capturing (used with view: 'region')."},
|
||||
"focus_handles":{"type":"array","items":{"type":"string"},"description":"Hex handles of entities to zoom and fit in view before capturing."},
|
||||
"highlight_handles":{"type":"array","items":{"type":"string"},"description":"Hex handles of entities to select/highlight before capturing."},
|
||||
"annotate":{"type":"boolean","default":false,"description":"Overlay Set-of-Marks numbered tags on visible entities for visual grounding."}
|
||||
},
|
||||
"required":["ocs_session_id"],
|
||||
"additionalProperties":false
|
||||
},
|
||||
"annotations":{"title":"Capture OCS window","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false}
|
||||
}
|
||||
])
|
||||
@@ -933,7 +976,18 @@ fn tool_definitions() -> Value {
|
||||
|
||||
fn tool_result(value: Value) -> Value {
|
||||
if let Some(image) = value.get("$image").and_then(Value::as_str) {
|
||||
return json!({"content":[{"type":"image","data":image,"mimeType":"image/png"}]});
|
||||
let mut content = Vec::new();
|
||||
if let Some(meta) = value.get("metadata") {
|
||||
let meta_text = serde_json::to_string_pretty(meta).unwrap_or_else(|_| meta.to_string());
|
||||
content.push(json!({"type":"text","text":meta_text}));
|
||||
}
|
||||
content.push(json!({"type":"image","data":image,"mimeType":"image/png"}));
|
||||
let structured = value.get("metadata").cloned().unwrap_or_else(|| json!({}));
|
||||
return json!({
|
||||
"content": content,
|
||||
"structuredContent": structured,
|
||||
"isError": false
|
||||
});
|
||||
}
|
||||
let structured = if value.is_object() {
|
||||
value.clone()
|
||||
|
||||
Reference in new issue
Block a user