Viewport picking: a BVH per body mesh and a GPU pick buffer for visible edges (#1776)

Thanks @oursland.

crates/geom/src/mesh.rs

@@ -559,13 +559,39 @@ impl Mesh {
     /// two triangles that share an edge get exactly opposite values there. A ray through a
     /// shared edge or vertex hits at least one of them; it never slips between them.
     pub fn raycast(&self, o: Vec3, dir: Vec3) -> Option<(f64, usize)> {
-        let comp = |v: Vec3, k: usize| match k {
-            0 => v.x,
-            1 => v.y,
-            _ => v.z,
-        };
-        // The ray's dominant axis becomes z; x and y follow it cyclically, swapped to keep
-        // the frame right-handed when the ray runs towards −z.
+        let frame = RayFrame::new(o, dir)?;
+        let mut best: Option<(f64, usize)> = None;
+        for (i, t) in self.triangles.iter().enumerate() {
+            let Some(p) = self.tri(t) else { continue };
+            if let Some(t) = frame.hit(p)
+                && best.is_none_or(|(bt, _)| t < bt)
+            {
+                best = Some((t, i));
+            }
+        }
+        best
+    }
+}
+
+/// A ray in the frame of [`Mesh::raycast`]'s watertight test: its dominant axis becomes z; x and
+/// y follow it cyclically, swapped to keep the frame right-handed when the ray runs towards −z.
+pub(crate) struct RayFrame {
+    o: Vec3,
+    k: [usize; 3],
+    s: [f64; 3],
+}
+
+fn comp(v: Vec3, k: usize) -> f64 {
+    match k {
+        0 => v.x,
+        1 => v.y,
+        _ => v.z,
+    }
+}
+
+impl RayFrame {
+    /// `None` for a direction that is zero or not finite.
+    pub(crate) fn new(o: Vec3, dir: Vec3) -> Option<RayFrame> {
         let a = [dir.x.abs(), dir.y.abs(), dir.z.abs()];
         let kz = if a[0] >= a[1] && a[0] >= a[2] {
             0
@@ -582,33 +608,31 @@ impl Mesh {
         if dz < 0.0 {
             std::mem::swap(&mut kx, &mut ky);
         }
-        let (sx, sy, sz) = (comp(dir, kx) / dz, comp(dir, ky) / dz, 1.0 / dz);
-        // A vertex in the ray's frame: (x, y) across the ray, z scaled along it.
-        let frame = |p: Vec3| {
-            let q = p - o;
-            let z = comp(q, kz);
-            (comp(q, kx) - sx * z, comp(q, ky) - sy * z, sz * z)
-        };
-        let mut best: Option<(f64, usize)> = None;
-        for (i, t) in self.triangles.iter().enumerate() {
-            let Some([pa, pb, pc]) = self.tri(t) else { continue };
-            let ((ax, ay, az), (bx, by, bz), (cx, cy, cz)) = (frame(pa), frame(pb), frame(pc));
-            let u = cx * by - cy * bx;
-            let v = ax * cy - ay * cx;
-            let w = bx * ay - by * ax;
-            if (u < 0.0 || v < 0.0 || w < 0.0) && (u > 0.0 || v > 0.0 || w > 0.0) {
-                continue;
-            }
-            let det = u + v + w;
-            if det == 0.0 {
-                continue;
-            }
-            let t = (u * az + v * bz + w * cz) / det;
-            if t.is_finite() && t > 1e-9 && best.is_none_or(|(bt, _)| t < bt) {
-                best = Some((t, i));
-            }
+        Some(RayFrame { o, k: [kx, ky, kz], s: [comp(dir, kx) / dz, comp(dir, ky) / dz, 1.0 / dz] })
+    }
+
+    /// A vertex in the ray's frame: (x, y) across the ray, z scaled along it.
+    fn frame(&self, p: Vec3) -> (f64, f64, f64) {
+        let q = p - self.o;
+        let z = comp(q, self.k[2]);
+        (comp(q, self.k[0]) - self.s[0] * z, comp(q, self.k[1]) - self.s[1] * z, self.s[2] * z)
+    }
+
+    /// The distance along the ray to triangle `[pa, pb, pc]` (`None`: missed, or not in front).
+    pub(crate) fn hit(&self, [pa, pb, pc]: [Vec3; 3]) -> Option<f64> {
+        let ((ax, ay, az), (bx, by, bz), (cx, cy, cz)) = (self.frame(pa), self.frame(pb), self.frame(pc));
+        let u = cx * by - cy * bx;
+        let v = ax * cy - ay * cx;
+        let w = bx * ay - by * ax;
+        if (u < 0.0 || v < 0.0 || w < 0.0) && (u > 0.0 || v > 0.0 || w > 0.0) {
+            return None;
         }
-        best
+        let det = u + v + w;
+        if det == 0.0 {
+            return None;
+        }
+        let t = (u * az + v * bz + w * cz) / det;
+        (t.is_finite() && t > 1e-9).then_some(t)
     }
 }