Large STEP imports: no more multi-minute hang, shared STEP text, cancel and progress (#925)

crates/kernel/src/step_in/mod.rs

@@ -257,6 +257,12 @@ struct Reader<'a> {
     unmeshed_checked: HashSet<u64>,
     bodies: Vec<ImportedBody>,
     workers: usize,
+    /// Whoever waits for the import (progress, Cancel).
+    watch: Option<Arc<dyn crate::watch::Watch>>,
+    /// Solid items in the file, the measure of the conversion's progress.
+    solid_items: usize,
+    /// The watch cancelled the import: the walk stops and the import fails.
+    cancelled: bool,
 }
 
 impl<'a> Reader<'a> {
@@ -280,9 +286,27 @@ impl<'a> Reader<'a> {
             unmeshed_checked: HashSet::new(),
             bodies: Vec::new(),
             workers,
+            watch: None,
+            solid_items: 0,
+            cancelled: false,
         }
     }
 
+    /// Report progress; true once the import is cancelled (the walk then unwinds).
+    fn stop(&mut self) -> bool {
+        if self.cancelled {
+            return true;
+        }
+        if let Some(w) = &self.watch {
+            if w.cancelled() {
+                self.cancelled = true;
+                return true;
+            }
+            w.progress("converting", self.bodies.len().min(self.solid_items), self.solid_items);
+        }
+        false
+    }
+
     fn warn(&mut self, msg: impl Into<String>) {
         *self.warnings.entry(msg.into()).or_default() += 1;
     }
@@ -584,6 +608,9 @@ impl<'a> Reader<'a> {
         let u = self.units_of(rep);
         let mut items = self.reps.get(&rep).map(|r| r.0.clone()).unwrap_or_default().into_iter().peekable();
         while let Some(item) = items.next() {
+            if self.stop() {
+                return;
+            }
             let Some(e) = self.ex.get(item) else { continue };
             match e.name() {
                 "MANIFOLD_SOLID_BREP" | "BREP_WITH_VOIDS" | "FACETED_BREP" | "SHELL_BASED_SURFACE_MODEL" => {
@@ -749,6 +776,9 @@ impl<'a> Reader<'a> {
     fn node(&mut self, pd: u64, local: mt::Matrix4, world: mt::Matrix4, path: &[String], depth: usize, stack: &mut Vec<u64>) -> StepNode {
         let name = self.product_name(pd);
         let mut node = StepNode { name: name.clone(), transform: to_array(&local), bodies: Vec::new(), children: Vec::new() };
+        if self.stop() {
+            return node;
+        }
         if depth > MAX_ASSEMBLY_DEPTH || stack.contains(&pd) {
             self.warn("assembly nested too deeply or recursive: some components were left out");
             return node;
@@ -933,40 +963,73 @@ pub fn step_validate(text: &str) -> std::result::Result<usize, String> {
     Ok(n)
 }
 
+/// A recent import: held weakly, so it lives only as long as someone uses it (the command that
+/// read the file, until the timeline has evaluated it), and strongly only when it is small.
+struct Recent {
+    key: (u64, usize),
+    import: std::sync::Weak<StepImport>,
+    /// Keeps a small import alive (never read).
+    _small: Option<Arc<StepImport>>,
+}
+
 /// Recent imports, so a file read for a command and then evaluated in the timeline is only
 /// read once.
-static RECENT: std::sync::Mutex<Vec<(u64, usize, std::sync::Arc<StepImport>)>> = std::sync::Mutex::new(Vec::new());
+static RECENT: std::sync::Mutex<Vec<Recent>> = std::sync::Mutex::new(Vec::new());
 
-/// [`step_import`] through a small cache of recent results (keyed by the text).
+/// Files up to this size stay cached after their last user is gone (a few cheap re-reads
+/// saved); a larger import is freed with the design that holds it.
+const KEEP_SMALL_BYTES: usize = 1 << 20;
+
+/// [`step_import`] through a small cache of recent results (keyed by the text). Keep the result
+/// while the same text is still to be read elsewhere (in the timeline): that read reuses it.
 pub fn step_import_shared(text: &str) -> Result<std::sync::Arc<StepImport>> {
     use std::hash::{Hash, Hasher};
     let mut h = std::collections::hash_map::DefaultHasher::new();
     text.hash(&mut h);
     let key = (h.finish(), text.len());
     if let Ok(cache) = RECENT.lock()
-        && let Some((_, _, imp)) = cache.iter().find(|(k, n, _)| (*k, *n) == key)
+        && let Some(imp) = cache.iter().filter(|r| r.key == key).find_map(|r| r.import.upgrade())
     {
-        return Ok(imp.clone());
+        return Ok(imp);
     }
     let imp = std::sync::Arc::new(step_import(text)?);
     if let Ok(mut cache) = RECENT.lock() {
-        cache.retain(|(k, n, _)| (*k, *n) != key);
-        cache.push((key.0, key.1, imp.clone()));
-        if cache.len() > 4 {
+        cache.retain(|r| r.key != key && r.import.strong_count() > 0);
+        let small = (text.len() <= KEEP_SMALL_BYTES).then(|| imp.clone());
+        cache.push(Recent { key, import: Arc::downgrade(&imp), _small: small });
+        while cache.len() > 4 {
             cache.remove(0);
         }
     }
     Ok(imp)
 }
 
+/// Imports the cache still holds alive, and how many STEP bytes they came from (tests and
+/// diagnostics: a closed design must not leave its import behind).
+pub fn step_cache_alive() -> (usize, usize) {
+    RECENT.lock().map(|c| c.iter().filter(|r| r.import.strong_count() > 0).fold((0, 0), |(n, b), r| (n + 1, b + r.key.1))).unwrap_or((0, 0))
+}
+
 /// Read a STEP file: its solids (as bodies, placed by the assembly structure) and the tree.
 pub fn step_import(text: &str) -> Result<StepImport> {
     step_import_with_workers(text, parallel::workers())
 }
 
 fn step_import_with_workers(text: &str, workers: usize) -> Result<StepImport> {
-    let ex = p21::parse(text).map_err(KernelError::Invalid)?;
+    let watch = crate::watch::current();
+    let cancelled = || watch.as_ref().is_some_and(|w| w.cancelled());
+    let ex = match p21::parse(text) {
+        Ok(ex) => ex,
+        Err(_) if cancelled() => return Err(crate::watch::cancelled_error("STEP import")),
+        Err(e) => return Err(KernelError::Invalid(e)),
+    };
     let mut rd = Reader::new(&ex, workers);
+    rd.solid_items = ex
+        .entities
+        .values()
+        .filter(|e| matches!(e.name(), "MANIFOLD_SOLID_BREP" | "BREP_WITH_VOIDS" | "FACETED_BREP" | "SHELL_BASED_SURFACE_MODEL"))
+        .count();
+    rd.watch = watch.clone();
     rd.index();
     // Roots: product definitions that are nobody's component.
     let child_pds: HashSet<u64> =
@@ -1006,6 +1069,10 @@ fn step_import_with_workers(text: &str, workers: usize) -> Result<StepImport> {
             tree.push(StepNode { name: "Import".into(), transform: to_array(&identity), bodies, children: Vec::new() });
         }
     }
+    // Cancelled: what was read so far is dropped here, not cached.
+    if rd.cancelled || cancelled() {
+        return Err(crate::watch::cancelled_error("STEP import"));
+    }
     if let Some(first) = ex.duplicates.first() {
         rd.warn(format!("{} entity ids are defined twice (first #{first}); the first definitions were used", ex.duplicates.len()));
     }