Design format: a change is one new file, never an edit to FORMAT (#1563)

A PR that saves something new adds crates/doc/src/format/pending/<name>.rs (a note, and an upgrade step if older designs must be rewritten). The build writes the next format number while changes are pending and lists them in format_pending; scripts/fold-format-pending.sh fixes them as v<N>/ at release time. Formats 23 and 24 move into v23/ and v24/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/doc/src/format.rs

@@ -222,29 +222,111 @@
 //!   parameters); the source's `at`, `height` and `angle` then hold the numbers they last came
 //!   to. A text without `exprs` is set as it always was; 0.12.0 asks for an update instead of
 //!   dropping the expressions and keeping the text at its last numbers.
-//! - 23: a motion link can carry a `coupling`: a gear pair (`ra`, `rb`, `internal`), a rack and
-//!   pinion (`radius`) or a screw (`pitch`), each size a number in mm, an expression or a picked
-//!   round face, and `reverse`. The link's `ratio` is worked out from it whenever the design
-//!   changes and saved as well, so a link reads the same without it. Version 22 designs need no
-//!   transformation: a link without `coupling` is a plain motion link, as it always was; 0.13.0
-//!   (which reads up to 22) asks for an update instead of keeping a gear's ratio when its radius
-//!   changes.
-//!   Also under 23, an Import's `file` can name a FreeCAD `.FCStd` or Rhino `.3dm` file (#1435,
-//!   read through cadmpeg); its `step` is still the STEP text the file was restated as, so the
-//!   feature's fields are unchanged and version 22 designs need no transformation; 0.13.x asks
-//!   for an update before it meets these imports.
-//! - 24: an Edge Flange feature can choose its own corner relief (`corner_relief_shape`: notch,
-//!   round, rectangular or tear, and `corner_relief`, the size across), for the corners its
-//!   flanges make. Version 23 designs need no transformation: a feature without them follows the
-//!   sheet metal rule, as it always did, and saves as before; 0.13.0 (which reads up to 22) asks
-//!   for an update instead of silently building the rule's relief where a design asks for another.
+//!
+//! Later formats are one file per change, so changes made side by side never edit the same line
+//! (`src/format/pending/README.md`):
+//! - `src/format/v<N>/`: the changes format N added, one file each, its `//!` note saying what is
+//!   saved and what older designs get, and a `step` when older designs are rewritten as they open
+//!   (23: motion link couplings, FreeCAD and Rhino imports; 24: an Edge Flange's own corner relief).
+//! - `src/format/pending/`: changes no release has fixed a number for yet. While there are any,
+//!   this build writes the next number (which no release reads), with the names of these changes
+//!   in `format_pending`, so a later build applies just the steps a file is missing. At release
+//!   time `scripts/fold-format-pending.sh` moves them into the next `v<N>/`.
 
 use serde_json::Value;
 
 use crate::{DocError, Document, Result};
 
-/// The format this build writes.
-pub const FORMAT: u32 = 24;
+/// The format this build writes: the last folded one, or the next while changes are pending.
+pub const FORMAT: u32 = FORMATS.current();
+
+/// The last format whose upgrade step is written out in [`JSON_STEPS`]; later formats are made
+/// of [`Part`]s (`src/format/`, listed by `build.rs`).
+const HAND_WRITTEN: u32 = 22;
+
+/// One change to the file format: a file in `src/format/v<N>/` or `src/format/pending/`.
+pub(crate) struct Part {
+    /// The file's name, which a file written while the change was pending lists.
+    pub name: &'static str,
+    /// How a design from before the change is rewritten as it opens, if it is.
+    pub step: Option<JsonStep>,
+}
+
+include!(concat!(env!("OUT_DIR"), "/format_parts.rs"));
+
+/// Every format this build knows and how to upgrade a design through them.
+const FORMATS: Formats<'static> = Formats { hand: &JSON_STEPS, folded: FOLDED, pending: PENDING };
+
+/// The names of the pending changes this build writes into a design (`format_pending`).
+pub fn pending_names() -> Vec<String> {
+    FORMATS.pending.iter().map(|p| p.name.to_string()).collect()
+}
+
+/// The upgrade path: the hand-written steps up to [`HAND_WRITTEN`], the folded formats after it
+/// (each a list of changes) and the pending changes, which make the next format.
+struct Formats<'a> {
+    hand: &'a [JsonStep],
+    folded: &'a [(u32, &'a [Part])],
+    pending: &'a [Part],
+}
+
+impl Formats<'_> {
+    const fn current(&self) -> u32 {
+        self.hand.len() as u32 + 1 + self.folded.len() as u32 + if self.pending.is_empty() { 0 } else { 1 }
+    }
+
+    /// The changes format `version` is made of, if it is made of changes.
+    fn parts(&self, version: u32) -> Option<&[Part]> {
+        let after = version.checked_sub(self.hand.len() as u32 + 2)? as usize;
+        match self.folded.get(after) {
+            Some((_, parts)) => Some(parts),
+            None if after == self.folded.len() && !self.pending.is_empty() => Some(self.pending),
+            None => None,
+        }
+    }
+
+    /// Bring a design of format `version` up to [`Formats::current`]. `applied` is the file's
+    /// `format_pending`: the changes of its format it already has, when it was written while they
+    /// were pending; the others' steps run first. A pending change this build doesn't have is a
+    /// newer build's, and the design is refused.
+    fn upgrade(&self, v: &mut Value, version: u32, applied: Option<&[String]>) -> Result<()> {
+        if version == 0 || version > self.current() {
+            return Err(DocError::Invalid(format!("not a SolidCraft design (format {version})")));
+        }
+        // A list beside a format that isn't made of changes means nothing (a tool or a test set
+        // an older number on a newer file): it is ignored, like any field an upgrade drops.
+        if let (Some(applied), Some(parts)) = (applied, self.parts(version)) {
+            let pending = version == self.current() && !self.pending.is_empty();
+            if let Some(unknown) = applied.iter().find(|name| pending && !parts.iter().any(|p| p.name == name.as_str())) {
+                return Err(DocError::Invalid(format!(
+                    "this design was saved by a newer SolidCraft (file format {version} with `{unknown}`, which this version doesn't know): update SolidCraft to open it"
+                )));
+            }
+            for part in parts.iter().filter(|p| !applied.iter().any(|name| name == p.name)) {
+                part.step.map_or(Ok(()), |step| step(v))?;
+            }
+        }
+        for from in version..self.current() {
+            match self.hand.get(from as usize - 1) {
+                Some(step) => step(v)?,
+                None => {
+                    for part in self.parts(from + 1).unwrap_or_default() {
+                        part.step.map_or(Ok(()), |step| step(v))?;
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+/// Take a file's `format_pending` (the pending changes it has) out of its JSON.
+fn take_pending(v: &mut Value) -> Result<Option<Vec<String>>> {
+    let Some(list) = v.as_object_mut().and_then(|o| o.remove("format_pending")) else { return Ok(None) };
+    let invalid = || DocError::Invalid("damaged design: `format_pending` is not a short list of names".into());
+    let list = list.as_array().filter(|l| l.len() <= 1000).ok_or_else(invalid)?;
+    list.iter().map(|n| n.as_str().filter(|n| n.len() <= 128).map(str::to_string).ok_or_else(invalid)).collect::<Result<_>>().map(Some)
+}
 
 pub fn format_string(n: u32) -> String {
     format!("solidcraft/{n}")
@@ -259,9 +341,10 @@ pub fn version_of(s: &str) -> Option<u32> {
     s.strip_prefix("solidcraft/").or_else(|| s.strip_prefix(LEGACY_FORMAT_PREFIX))?.parse().ok()
 }
 
-/// JSON-level upgrades: entry `i` turns format `i + 1` into `i + 2`.
+/// JSON-level upgrades up to [`HAND_WRITTEN`]: entry `i` turns format `i + 1` into `i + 2`.
+/// Never add to it: a new format is a file in `src/format/pending/`.
 type JsonStep = fn(&mut Value) -> Result<()>;
-const JSON_STEPS: [JsonStep; 23] = [
+const JSON_STEPS: [JsonStep; HAND_WRITTEN as usize - 1] = [
     |_| Ok(()),
     v2_to_v3,
     v3_to_v4,
@@ -283,8 +366,6 @@ const JSON_STEPS: [JsonStep; 23] = [
     |_| Ok(()),
     |_| Ok(()),
     |_| Ok(()),
-    |_| Ok(()),
-    |_| Ok(()),
 ];
 
 /// Stored face and edge names were worked out from geometry before format 14; they get the
@@ -475,6 +556,7 @@ pub fn read(s: &str) -> Result<Document> {
         Some(Value::String(f)) => version_of(f).ok_or_else(|| DocError::Invalid(format!("not a SolidCraft design (format `{f}`)")))?,
         Some(_) => return Err(DocError::Invalid("not a SolidCraft design (format is not a string)".into())),
     };
+    let applied = take_pending(&mut v)?;
     if version == 0 {
         return Err(DocError::Invalid("not a SolidCraft design (format 0)".into()));
     }
@@ -494,9 +576,7 @@ pub fn read(s: &str) -> Result<Document> {
     crate::format_offsets::validate_raw(&v)?;
     let legacy_wires = validate_wires(&v, version)?;
     validate_body_transfers(&v)?;
-    for step in JSON_STEPS.iter().skip(version as usize - 1).take((FORMAT - version) as usize) {
-        step(&mut v)?;
-    }
+    FORMATS.upgrade(&mut v, version, applied.as_deref())?;
     let mut d: Document = serde_json::from_value(v).map_err(|e| DocError::Invalid(format!("design file: {e}")))?;
     let mut legacy_by_feature = std::collections::BTreeMap::<usize, Vec<usize>>::new();
     for (feature, wire) in legacy_wires {
@@ -595,6 +675,7 @@ pub fn read(s: &str) -> Result<Document> {
     let floor = largest.checked_add(1).ok_or_else(|| DocError::Invalid("damaged design: the id space is exhausted".into()))?;
     d.next_id = d.next_id.max(floor).max(1);
     d.format = format_string(FORMAT);
+    d.format_pending = pending_names();
     Ok(d)
 }
 
@@ -1017,11 +1098,136 @@ fn normalise(d: &mut Document) -> Result<()> {
 mod tests {
     use super::*;
 
+    /// A pending change that records that its step ran, in `trace`.
+    fn traced(name: &'static str) -> Part {
+        fn step_a(v: &mut Value) -> Result<()> {
+            trace(v, "a")
+        }
+        fn step_b(v: &mut Value) -> Result<()> {
+            trace(v, "b")
+        }
+        fn trace(v: &mut Value, s: &str) -> Result<()> {
+            let o = v.as_object_mut().ok_or_else(|| DocError::Invalid("no object".into()))?;
+            o.entry("trace").or_insert_with(|| Value::Array(Vec::new())).as_array_mut().into_iter().for_each(|t| t.push(Value::from(s)));
+            Ok(())
+        }
+        Part { name, step: Some(if name.starts_with('a') { step_a } else { step_b }) }
+    }
+
+    fn trace_of(v: &Value) -> Vec<&str> {
+        v["trace"].as_array().into_iter().flatten().filter_map(Value::as_str).collect()
+    }
+
+    /// Formats after the hand-written ones follow on from them, one folder each, and the build
+    /// writes the next number exactly while changes are pending, so the last release (which
+    /// reads up to 22) refuses what a development build saves.
+    #[test]
+    fn formats_follow_on_and_pending_changes_take_the_next_number() {
+        assert_eq!(JSON_STEPS.len() as u32, HAND_WRITTEN - 1);
+        for (i, (version, parts)) in FOLDED.iter().enumerate() {
+            assert_eq!(*version, HAND_WRITTEN + 1 + i as u32);
+            assert!(!parts.is_empty(), "format {version} has no changes");
+        }
+        let last_folded = HAND_WRITTEN + FOLDED.len() as u32;
+        assert_eq!(FORMAT, last_folded + u32::from(!PENDING.is_empty()));
+        const { assert!(FORMAT >= 24, "designs saved as 23 and 24 by nightlies still open") };
+        assert_eq!(pending_names(), PENDING.iter().map(|p| p.name).collect::<Vec<_>>());
+        assert_eq!(Document::new("A").format_pending, pending_names());
+    }
+
+    /// The build lists every change file there is, in file-name order: a change is a file, so
+    /// two branches that each add one merge cleanly and the merged build has both.
+    #[test]
+    fn every_change_file_is_listed_in_name_order() {
+        let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/format");
+        let names = |dir: &str| -> Vec<String> {
+            let mut names: Vec<String> = std::fs::read_dir(root.join(dir))
+                .map(|d| d.filter_map(|e| e.ok()?.file_name().to_str()?.strip_suffix(".rs").map(str::to_string)).collect())
+                .unwrap_or_default();
+            names.sort();
+            names
+        };
+        assert_eq!(names("pending"), pending_names());
+        for (version, parts) in FOLDED {
+            assert_eq!(names(&format!("v{version}")), parts.iter().map(|p| p.name).collect::<Vec<_>>());
+        }
+    }
+
+    /// Two changes pending side by side both run, in name order, after every older format's
+    /// steps: an old design upgrades through all of them, and one from the last folded format
+    /// gets just the pending ones.
+    #[test]
+    fn pending_changes_apply_in_order_after_older_formats() {
+        let pending = [traced("a-first"), traced("b-second")];
+        let formats = Formats { hand: &JSON_STEPS, folded: FOLDED, pending: &pending };
+        let next = HAND_WRITTEN + FOLDED.len() as u32 + 1;
+        assert_eq!(formats.current(), next);
+        for version in [1, HAND_WRITTEN, next - 1] {
+            let mut v = serde_json::json!({"features": [], "assembly": {"joints": [{"flip": false}]}});
+            formats.upgrade(&mut v, version, None).unwrap();
+            assert_eq!(trace_of(&v), ["a", "b"], "from format {version}");
+            // The hand-written steps ran too: format 3's joints were flipped on the way up.
+            assert_eq!(v["assembly"]["joints"][0]["flip"], Value::Bool(version < 4), "from format {version}");
+        }
+        // A design of the pending number written before `b-second` existed gets just that step.
+        let mut v = serde_json::json!({});
+        formats.upgrade(&mut v, next, Some(&["a-first".to_string()])).unwrap();
+        assert_eq!(trace_of(&v), ["b"]);
+        // One that has both, or that this build wrote, needs nothing.
+        let mut v = serde_json::json!({});
+        formats.upgrade(&mut v, next, Some(&["a-first".into(), "b-second".into()])).unwrap();
+        assert_eq!(trace_of(&v), Vec::<&str>::new());
+        // A change this build doesn't know is a newer build's: refused, not guessed at.
+        let err = formats.upgrade(&mut serde_json::json!({}), next, Some(&["c-later".into()])).unwrap_err();
+        assert!(err.to_string().contains("newer SolidCraft"), "{err}");
+        // A list on a format that isn't made of changes means nothing.
+        let mut v = serde_json::json!({});
+        formats.upgrade(&mut v, HAND_WRITTEN, Some(&["a-first".into()])).unwrap();
+        assert_eq!(trace_of(&v), ["a", "b"]);
+    }
+
+    /// Folding the pending changes into a number (the release script) keeps every design
+    /// opening the same: a file saved while they were pending gets the steps it missed.
+    #[test]
+    fn folded_changes_upgrade_like_pending_ones() {
+        let parts = [traced("a-first"), traced("b-second")];
+        let mut folded: Vec<(u32, &[Part])> = FOLDED.to_vec();
+        let next = HAND_WRITTEN + folded.len() as u32 + 1;
+        folded.push((next, &parts));
+        let formats = Formats { hand: &JSON_STEPS, folded: &folded, pending: &[] };
+        assert_eq!(formats.current(), next);
+        let mut old = serde_json::json!({});
+        formats.upgrade(&mut old, HAND_WRITTEN, None).unwrap();
+        assert_eq!(trace_of(&old), ["a", "b"]);
+        let mut nightly = serde_json::json!({});
+        formats.upgrade(&mut nightly, next, Some(&["a-first".into()])).unwrap();
+        assert_eq!(trace_of(&nightly), ["b"]);
+        let mut released = serde_json::json!({});
+        formats.upgrade(&mut released, next, None).unwrap();
+        assert_eq!(trace_of(&released), Vec::<&str>::new());
+    }
+
+    /// `format_pending` is read and checked: a damaged list is refused, a list of changes this
+    /// build doesn't have is a newer build's, and saving writes this build's own list.
+    #[test]
+    fn format_pending_in_a_file_is_checked() {
+        let mut file: Value = serde_json::from_str(&Document::new("A").to_json()).unwrap();
+        assert_eq!(file.get("format_pending").is_some(), !PENDING.is_empty());
+        for damaged in [serde_json::json!("x"), serde_json::json!([1]), serde_json::json!(["x".repeat(200)])] {
+            file["format_pending"] = damaged;
+            assert!(read(&file.to_string()).is_err());
+        }
+        file["format_pending"] = serde_json::json!(["not-a-change-anyone-made"]);
+        assert_eq!(read(&file.to_string()).is_err(), !PENDING.is_empty(), "only a pending format's list names changes");
+        file["format_pending"] = Value::from(pending_names());
+        let opened = read(&file.to_string()).unwrap();
+        assert_eq!(opened.format_pending, pending_names());
+    }
+
     /// 24 added the corner relief of an edge flange: a no-op step carries a 23 design up, a flange
     /// without a relief of its own saves as before, and one with it is saved under 24 and kept.
     #[test]
     fn version_twenty_three_designs_open_and_an_edge_flange_relief_is_saved_in_twenty_four() {
-        assert_eq!(JSON_STEPS.len(), FORMAT as usize - 1, "one upgrade step for each format");
         let flange = |shape: Option<&str>, size: Option<&str>| crate::FeatureKind::SheetFlange {
             edges: vec![crate::FlangeEdge::at(solidcraft_geom::Vec3::ZERO)],
             height: "10 mm".into(),