forked from 1b-agent/fieldcraft-upstream
Camera orientation in degrees: ui.camera and the MCP screenshot's camera
`ui.view` and the MCP `screenshot` tool only turned to the named views, so
a picture sequence could only cut between them. `Orientation` (render) is a
camera orientation in degrees, {yaw, pitch, distance?}, checked as hostile
input: angles finite, distance finite and above 0, yaw wrapped, pitch
clamped to ±90, distance to 0.001–1e9. `modelcraft_engine::view::orientation`
parses it from JSON for both callers:
- control channel `ui.camera {yaw, pitch, distance?}` turns the app's camera;
- MCP `screenshot {camera}` turns it first (bridged through `ui.camera`,
headless in the CPU render, framed unless a distance is given). `view` and
`camera` are not given together.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm
This commit is contained in:
1 parent
d84dae074d
commit
89651b707c
9 files changed
+210
-19
No files matched your search
@@ -1,6 +1,6 @@
|
||||
//! Where tool calls end up: an in-process headless session, or a running app's control channel
|
||||
//! (docs/control-protocol.md: `engine.execute`, `engine.commands`, `ui.view`, `ui.render`,
|
||||
//! `ui.capture`).
|
||||
//! (docs/control-protocol.md: `engine.execute`, `engine.commands`, `ui.view`, `ui.camera`,
|
||||
//! `ui.render`, `ui.capture`).
|
||||
|
||||
use std::io::{BufRead, BufReader, Read, Write};
|
||||
use std::net::{TcpStream, ToSocketAddrs};
|
||||
@@ -11,7 +11,7 @@ use std::time::Duration;
|
||||
use fieldcraft_engine::Session;
|
||||
use fieldcraft_engine::view::{Palette, render_png};
|
||||
use fieldcraft_io::roots::Roots;
|
||||
use fieldcraft_render::StandardView;
|
||||
use fieldcraft_render::{Orientation, StandardView};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// What a picture shows.
|
||||
@@ -19,6 +19,8 @@ use serde_json::{Value, json};
|
||||
pub struct PictureRequest {
|
||||
/// A standard view to turn to first (`iso`, `front`…; `home` and `fit` frame the model).
|
||||
pub view: Option<String>,
|
||||
/// A camera orientation to turn to first (degrees; `view` and `camera` are not given together).
|
||||
pub camera: Option<Orientation>,
|
||||
/// The whole app window (`true`) or the model only.
|
||||
pub window: bool,
|
||||
/// Image size in pixels (a window capture: points); `None` keeps the default.
|
||||
@@ -67,8 +69,9 @@ fn standard_view(v: &str) -> Result<StandardView, String> {
|
||||
/// An in-process headless session; pictures come from the CPU rasterizer.
|
||||
pub struct Headless {
|
||||
pub session: Session,
|
||||
/// The orientation of pictures; each one frames everything drawn.
|
||||
pub view: StandardView,
|
||||
/// The orientation of pictures; each one frames everything drawn, unless the orientation
|
||||
/// has a distance.
|
||||
pub view: Orientation,
|
||||
roots: Roots,
|
||||
}
|
||||
|
||||
@@ -83,7 +86,7 @@ impl Headless {
|
||||
pub fn new(roots: Roots) -> Self {
|
||||
let mut session = Session::default();
|
||||
session.file_access = roots.clone();
|
||||
Headless { session, view: StandardView::Iso, roots }
|
||||
Headless { session, view: Orientation::of(StandardView::Iso), roots }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,11 +101,14 @@ impl Backend for Headless {
|
||||
|
||||
fn picture(&mut self, r: &PictureRequest) -> Result<Picture, String> {
|
||||
if let Some(v) = &r.view {
|
||||
self.view = standard_view(v)?;
|
||||
self.view = Orientation::of(standard_view(v)?);
|
||||
}
|
||||
if let Some(o) = r.camera {
|
||||
self.view = o;
|
||||
}
|
||||
let w = r.width.unwrap_or(DEFAULT_SIZE.0).clamp(16, MAX_SIDE);
|
||||
let h = r.height.unwrap_or(DEFAULT_SIZE.1).clamp(16, MAX_SIDE);
|
||||
let (png, shown) = render_png(&self.session, self.view, w as usize, h as usize, &Palette::LIGHT).map_err(|e| e.to_string())?;
|
||||
let (png, shown) = render_png(&self.session, &self.view, w as usize, h as usize, &Palette::LIGHT).map_err(|e| e.to_string())?;
|
||||
let contour = shown
|
||||
.contour
|
||||
.map(|(field, min, max)| json!({"field": field, "min": {"node": min.0, "value": min.1}, "max": {"node": max.0, "value": max.1}}));
|
||||
@@ -229,6 +235,9 @@ impl Backend for Remote {
|
||||
standard_view(v)?;
|
||||
self.call("ui.view", json!({"view": v}))?;
|
||||
}
|
||||
if let Some(o) = r.camera {
|
||||
self.call("ui.camera", json!(o))?;
|
||||
}
|
||||
let path = temp_png();
|
||||
let shown = path.to_string_lossy().into_owned();
|
||||
let mut params = json!({"path": shown});
|
||||
|
||||
@@ -173,6 +173,12 @@ fn hostile_tool_arguments_are_tool_errors() {
|
||||
("probe", json!({"nodes": [1, 2, 3]})),
|
||||
("reactions", json!({"sets": ["nope"], "about": [1e308, 0, 0]})),
|
||||
("screenshot", json!({"view": "iso", "path": "/no/such/dir/x.png"})),
|
||||
("screenshot", json!({"camera": {"yaw": "left", "pitch": 0}})),
|
||||
("screenshot", json!({"camera": {"yaw": 0}})),
|
||||
("screenshot", json!({"camera": {"yaw": 0, "pitch": 0, "distance": -1}})),
|
||||
("screenshot", json!({"camera": {"yaw": 0, "pitch": 0, "distance": 0}})),
|
||||
("screenshot", json!({"camera": {"yaw": 0, "pitch": 0, "zoom": 2}})),
|
||||
("screenshot", json!({"camera": {"yaw": 0, "pitch": 0}, "view": "top"})),
|
||||
("batch", json!({"commands": [{"command": "model.info"}, {"params": {}}, {"command": "model.info"}]})),
|
||||
("batch", json!({"commands": vec![json!({"command": "model.info"}); crate::tools::MAX_STEPS + 1]})),
|
||||
("undo", json!({})),
|
||||
@@ -192,6 +198,26 @@ fn hostile_tool_arguments_are_tool_errors() {
|
||||
assert_eq!(r["result"]["content"][0]["type"], "image", "{r}");
|
||||
}
|
||||
|
||||
/// `screenshot {camera}` turns a headless picture to any orientation: hostile angles are clamped,
|
||||
/// and the orientation is kept for the next picture.
|
||||
#[test]
|
||||
fn screenshot_takes_a_camera_orientation() {
|
||||
let mut s = server();
|
||||
ok(&mut s, "execute", json!({"command": "file.sample", "params": {"name": "cantilever"}}));
|
||||
let picture = |s: &mut Server, args: Value| {
|
||||
let r = send(s, json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "screenshot", "arguments": args}}));
|
||||
let content = r["result"]["content"].as_array().cloned().unwrap_or_default();
|
||||
let info: Value = serde_json::from_str(content[1]["text"].as_str().unwrap()).unwrap();
|
||||
(content[0]["data"].as_str().unwrap().to_string(), info)
|
||||
};
|
||||
let (iso, _) = picture(&mut s, json!({"view": "iso", "width": 64, "height": 48}));
|
||||
let (turned, info) = picture(&mut s, json!({"camera": {"yaw": 30, "pitch": 1e9, "distance": 1e300}, "width": 64, "height": 48}));
|
||||
assert_ne!(iso, turned);
|
||||
assert_eq!(info["view"], json!({"yaw": 30.0, "pitch": 90.0, "distance": 1e9}), "{info}");
|
||||
let (again, _) = picture(&mut s, json!({"width": 64, "height": 48}));
|
||||
assert_eq!(again, turned);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_batch_stops_at_the_first_error() {
|
||||
let mut s = server();
|
||||
|
||||
@@ -272,10 +272,13 @@ pub fn tool_definitions() -> Vec<Value> {
|
||||
"Look at the model",
|
||||
"A PNG of the model and the results display. Headless: a CPU render framing everything drawn. Connected to \
|
||||
the app: the whole window drawn offscreen (source \"window\", the default; ui.capture) or the model with the \
|
||||
app's camera (source \"model\"; ui.render). `view` turns the camera to a standard view first.",
|
||||
app's camera (source \"model\"; ui.render). `view` turns the camera to a standard view first, or `camera` to \
|
||||
any orientation: {yaw, pitch, distance?} in degrees (yaw 0 looks along +Y, -45 is the iso view's; pitch is \
|
||||
the elevation, clamped to ±90) and model units; a sequence of them orbits smoothly.",
|
||||
obj(
|
||||
json!({
|
||||
"view": {"type": "string", "enum": VIEWS, "description": "Standard view (default: keep the current one; headless starts at iso)"},
|
||||
"camera": {"type": "object", "description": "Camera orientation {yaw, pitch, distance?}: degrees, and the distance from the target in model units (default: framed headless, kept in the app)"},
|
||||
"source": {"type": "string", "enum": ["window", "model"], "description": "window = the app's window, model = a model render"},
|
||||
"width": integer("Width in pixels (window: points), 16 to 4096"),
|
||||
"height": integer("Height in pixels (window: points), 16 to 4096"),
|
||||
@@ -525,8 +528,13 @@ fn capture(b: &mut dyn Backend, a: &Map<String, Value>) -> Result<(Vec<u8>, Valu
|
||||
None => None,
|
||||
};
|
||||
let size = |k: &str| a.get(k).and_then(Value::as_u64).map(|v| u32::try_from(v).unwrap_or(u32::MAX));
|
||||
let camera = a.get("camera").filter(|v| !v.is_null()).map(fieldcraft_engine::view::orientation).transpose().map_err(|e| e.to_string())?;
|
||||
if camera.is_some() && a.get("view").is_some_and(|v| !v.is_null()) {
|
||||
return Err("screenshot: give `view` or `camera`, not both".into());
|
||||
}
|
||||
let request = PictureRequest {
|
||||
view: str_arg(a, "view").map(str::to_string),
|
||||
camera,
|
||||
window: str_arg(a, "source") != Some("model"),
|
||||
width: size("width"),
|
||||
height: size("height"),
|
||||
|
||||
Reference in new issue
Block a user