From f4abb555d3b675f8b9e76fed49f2ffa091608da0 Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Tue, 20 May 2025 15:19:34 +1000
Subject: [PATCH 01/15] Add doodle-rec crate for recursion model
 experimentation

---
 Cargo.lock                         |  12 +-
 Cargo.toml                         |   2 +-
 experiments/doodle-rec/Cargo.toml  |  12 +
 experiments/doodle-rec/src/lib.rs  | 497 +++++++++++++++++++++++++++++
 experiments/doodle-rec/src/main.rs |   3 +
 src/valuetype.rs                   |   4 +-
 6 files changed, 525 insertions(+), 5 deletions(-)
 create mode 100644 experiments/doodle-rec/Cargo.toml
 create mode 100644 experiments/doodle-rec/src/lib.rs
 create mode 100644 experiments/doodle-rec/src/main.rs

diff --git a/Cargo.lock b/Cargo.lock
index d34974e3..3d17e7fb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -59,9 +59,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.79"
+version = "1.0.98"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca"
+checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
 
 [[package]]
 name = "autocfg"
@@ -388,6 +388,14 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "doodle-rec"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "doodle",
+]
+
 [[package]]
 name = "doodle_gencode"
 version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index dc36b8c2..76067621 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,5 @@
 [workspace]
-members = [".", "generated/", "doodle-formats/"]
+members = [".", "generated/", "doodle-formats/", "experiments/doodle-rec"]
 
 [package]
 name = "doodle"
diff --git a/experiments/doodle-rec/Cargo.toml b/experiments/doodle-rec/Cargo.toml
new file mode 100644
index 00000000..1e98e2ae
--- /dev/null
+++ b/experiments/doodle-rec/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "doodle-rec"
+version = "0.1.0"
+edition = "2024"
+
+[lib]
+path = "src/lib.rs"
+bench = false
+
+[dependencies]
+anyhow = "1.0.98"
+doodle = { path = "../../" }
diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
new file mode 100644
index 00000000..227e1a70
--- /dev/null
+++ b/experiments/doodle-rec/src/lib.rs
@@ -0,0 +1,497 @@
+use std::{borrow::Cow, cell::OnceCell, collections::{BTreeMap, HashSet}, ops::Range, rc::Rc};
+use doodle::byte_set::ByteSet;
+use anyhow::{anyhow, Result as AResult};
+
+
+pub type Label = Cow<'static, str>;
+
+pub type FormatId = usize;
+
+pub type RecId = usize;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct FormatRef(FormatId);
+
+impl FormatRef {
+    pub const fn get_level(&self) -> usize {
+        self.0
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct Span<Idx> {
+    pub start: Idx,
+    pub end: Idx,
+}
+
+impl<Idx> Span<Idx> {
+    pub const fn new(start: Idx, end: Idx) -> Self {
+        Self { start, end }
+    }
+}
+
+impl<Idx> From<Range<Idx>> for Span<Idx> {
+    fn from(value: Range<Idx>) -> Self {
+        Self { start: value.start, end: value.end }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct FormatDecl {
+    format: Format,
+    fmt_id: FormatId,
+    f_type: Rc<OnceCell<FormatType>>,
+    batch: Option<Span<FormatId>>,
+}
+
+impl FormatDecl {
+    pub fn solve_type(&self, module: &FormatModule) -> AResult<&FormatType> {
+        let mut visited = HashSet::new();
+        self.solve_type_with(module, &mut visited)
+    }
+
+    pub(crate) fn solve_type_with(&self, module: &FormatModule, visited: &mut HashSet<FormatId>) -> AResult<&FormatType> {
+        match self.f_type.get() {
+            None => {
+                visited.insert(self.fmt_id);
+                let f_type = self.format.infer_type(visited, module, self.batch)?;
+                let Ok(_) = self.f_type.set(f_type) else { unreachable!("synchronous TOCTOU!?") };
+                Ok(self.f_type.get().unwrap())
+            }
+            Some(f_type) => Ok(f_type),
+        }
+    }
+}
+
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BaseType {
+    Bool,
+    U8,
+    U16,
+    U32,
+    U64,
+    Char,
+}
+
+impl BaseType {
+    pub fn is_numeric(&self) -> bool {
+        matches!(self, BaseType::U8 | BaseType::U16 | BaseType::U32 | BaseType::U64)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum FormatType {
+    Any,
+    Void,
+    Unit,
+    Base(BaseType),
+    Ref(FormatId),
+    Shape(TypeShape),
+}
+
+impl FormatType {
+    pub fn is_numeric(&self) -> bool {
+        match self {
+            FormatType::Base(base) => base.is_numeric(),
+            _ => false,
+        }
+    }
+
+    fn unify(&self, other: &FormatType) -> AResult<FormatType> {
+        match (self, other) {
+            (FormatType::Any, _) => Ok(other.clone()),
+            (_, FormatType::Any) => Ok(self.clone()),
+            (FormatType::Void, _) | (_, FormatType::Void) => Ok(FormatType::Void),
+            (FormatType::Unit, FormatType::Unit) => Ok(FormatType::Unit),
+            (FormatType::Unit, FormatType::Shape(TypeShape::Tuple(empty)))
+            | (FormatType::Shape(TypeShape::Tuple(empty)), FormatType::Unit) if empty.is_empty() => Ok(FormatType::Unit),
+            (FormatType::Base(b1), FormatType::Base(b2)) if b1 == b2 => Ok(FormatType::Base(*b1)),
+            (FormatType::Shape(s1), FormatType::Shape(s2)) => {
+                let s = s1.unify(s2)?;
+                Ok(FormatType::Shape(s))
+            }
+            _ => Err(anyhow!("cannot unify incompatible types: {self:?}, {other:?}")),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum TypeShape {
+    Tuple(Vec<FormatType>),
+    Seq(Box<FormatType>),
+    Option(Box<FormatType>),
+    Union(BTreeMap<Label, FormatType>),
+}
+
+impl TypeShape {
+    fn unify(&self, other: &Self) -> AResult<TypeShape> {
+        match (self, other) {
+            (TypeShape::Tuple(t1), TypeShape::Tuple(t2)) => {
+                if t1.len() != t2.len() {
+                    return Err(anyhow!("cannot unify tuples of different arity: {t1:?}, {t2:?}"));
+                }
+                let mut unified = Vec::with_capacity(t1.len());
+                for (t1, t2) in t1.iter().zip(t2.iter()) {
+                    unified.push(t1.unify(t2)?);
+                }
+                Ok(TypeShape::Tuple(unified))
+            }
+            (TypeShape::Seq(t1), TypeShape::Seq(t2)) => Ok(TypeShape::Seq(Box::new(t1.unify(t2)?))),
+            (TypeShape::Option(t1), TypeShape::Option(t2)) => Ok(TypeShape::Option(Box::new(t1.unify(t2)?))),
+            (TypeShape::Union(bs1), TypeShape::Union(bs2)) => {
+                let mut bs = BTreeMap::new();
+
+                let keys1 = bs1.keys().collect::<HashSet<_>>();
+                let keys2 = bs2.keys().collect::<HashSet<_>>();
+
+                let all_keys = HashSet::union(&keys1, &keys2).cloned();
+
+                for key in all_keys.into_iter() {
+                    match (bs1.get(key), bs2.get(key)) {
+                        (Some(t1), Some(t2)) => {
+                            let t = t1.unify(t2)?;
+                            bs.insert(key.clone(), t);
+                        }
+                        (Some(t), None) | (None, Some(t)) => {
+                            bs.insert(key.clone(), t.clone());
+                        }
+                        (None, None) => unreachable!("key must appear in at least one operand"),
+                    }
+                }
+                Ok(TypeShape::Union(bs))
+            }
+            _ => Err(anyhow!("cannot unify shapes: {self:?}, {other:?}")),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum Format {
+    // References to other formats
+    ItemVar(FormatId),
+    RecVar(RecId),
+
+    // Basic Primitives
+    FailWith(Label),
+    EndOfInput,
+    Byte(ByteSet),
+    Compute(Box<Expr>),
+
+    // Union-Based
+    Variant(Label, Box<Format>),
+    Union(Vec<Format>),
+
+    // Sequential
+    Repeat(Box<Format>),
+    Seq(Vec<Format>),
+
+    // Higher-Order
+    Tuple(Vec<Format>),
+    Maybe(Box<Expr>, Box<Format>),
+}
+
+impl Format {
+    fn infer_type<'ctx>(&'ctx self, visited: &mut HashSet<FormatId>, module: &'ctx FormatModule, batch: Option<Span<FormatId>>) -> AResult<FormatType> {
+        match self {
+            Format::ItemVar(level) => {
+                if visited.contains(level) {
+                    Ok(FormatType::Ref(*level))
+                } else {
+                    let decl = &module.decls[*level];
+                    Ok(decl.solve_type_with(module, visited)?.clone())
+                }
+            }
+            Format::RecVar(batch_ix) => {
+                match batch {
+                    None => Err(anyhow!("Recursion without a batch")),
+                    Some(range) => {
+                        let level = range.start + batch_ix;
+                        if level > range.end {
+                            return Err(anyhow!("batch index out of range"))
+                        }
+                        if visited.contains(&level) {
+                            Ok(FormatType::Ref(level))
+                        } else {
+                            let decl = &module.decls[level];
+                            Ok(decl.solve_type_with(module, visited)?.clone())
+                        }
+                    }
+                }
+            }
+            Format::FailWith(_msg) => Ok(FormatType::Void),
+            Format::EndOfInput => Ok(FormatType::Unit),
+            Format::Byte(bs) if bs.is_empty() => Ok(FormatType::Void),
+            Format::Byte(_) => Ok(FormatType::Base(BaseType::U8)),
+            Format::Compute(expr) => expr.as_ref().infer_type(),
+            Format::Variant(label, inner) => {
+                let inner_type = inner.infer_type(visited, module, batch)?;
+                Ok(FormatType::Shape(TypeShape::Union(BTreeMap::from([(label.clone(), inner_type)]))))
+            }
+            Format::Union(branches) => {
+                let mut t = FormatType::Any;
+                for f in branches {
+                    t = t.unify(&f.infer_type(visited, module, batch)?)?;
+                }
+                Ok(t)
+            }
+            Format::Repeat(inner) => {
+                let t = inner.infer_type(visited, module, batch)?;
+                Ok(FormatType::Shape(TypeShape::Seq(Box::new(t))))
+            }
+            Format::Seq(elts) => {
+                let mut elem_type = FormatType::Any;
+                for elt in elts {
+                    elem_type = elem_type.unify(&elt.infer_type(visited, module, batch)?)?;
+                }
+                Ok(FormatType::Shape(TypeShape::Seq(Box::new(elem_type))))
+            }
+            Format::Tuple(elts) => {
+                let mut types = Vec::with_capacity(elts.len());
+                for elt in elts {
+                    types.push(elt.infer_type(visited, module, batch)?);
+                }
+                Ok(FormatType::Shape(TypeShape::Tuple(types)))
+            }
+            Format::Maybe(expr, format) => match expr.infer_type()? {
+                FormatType::Base(BaseType::Bool) => {
+                    let t = format.infer_type(visited, module, batch)?;
+                    Ok(FormatType::Shape(TypeShape::Option(Box::new(t))))
+                }
+                other => Err(anyhow!("maybe expression type was inferred to be non-bool: {other:?}")),
+            }
+        }
+    }
+
+
+}
+
+#[derive(Debug, Clone)]
+pub enum Expr {
+    // Primitive Values
+    U8(u8),
+    U16(u16),
+    U32(u32),
+    U64(u64),
+    Bool(bool),
+
+    // Primitive Value Casts
+    AsChar(Box<Expr>),
+    AsU8(Box<Expr>),
+    AsU16(Box<Expr>),
+    AsU32(Box<Expr>),
+    AsU64(Box<Expr>),
+
+    // Higher-Order Exprs
+    Seq(Vec<Expr>),
+    Tuple(Vec<Expr>),
+    LiftMaybe(Option<Box<Expr>>),
+    Variant(Label, Box<Expr>),
+
+    // Operational
+    IntRel(IntRel, Box<Expr>, Box<Expr>),
+    Arith(Arith, Box<Expr>, Box<Expr>),
+    Unary(Unary, Box<Expr>),
+}
+
+impl Expr {
+    fn infer_type(&self) -> AResult<FormatType> {
+        match self {
+            Expr::U8(_) => Ok(FormatType::Base(BaseType::U8)),
+            Expr::U16(_) => Ok(FormatType::Base(BaseType::U16)),
+            Expr::U32(_) => Ok(FormatType::Base(BaseType::U32)),
+            Expr::U64(_) => Ok(FormatType::Base(BaseType::U64)),
+            Expr::Bool(_) => Ok(FormatType::Base(BaseType::Bool)),
+            Expr::AsChar(expr) => {
+                let expr_type = expr.infer_type()?;
+                if expr_type.is_numeric() {
+                    Ok(FormatType::Base(BaseType::Char))
+                } else {
+                    Err(anyhow!("invalid char type conversion from {expr_type:?}"))
+                }
+            }
+            Expr::AsU8(expr) => {
+                let expr_type = expr.infer_type()?;
+                if expr_type.is_numeric() {
+                    Ok(FormatType::Base(BaseType::U8))
+                } else {
+                    Err(anyhow!("invalid u8 type conversion from {expr_type:?}"))
+                }
+            }
+            Expr::AsU16(expr) => {
+                let expr_type = expr.infer_type()?;
+                if expr_type.is_numeric() {
+                    Ok(FormatType::Base(BaseType::U16))
+                } else {
+                    Err(anyhow!("invalid u16 type conversion from {expr_type:?}"))
+                }
+            }
+            Expr::AsU32(expr) => {
+                let expr_type = expr.infer_type()?;
+                if expr_type.is_numeric() {
+                    Ok(FormatType::Base(BaseType::U32))
+                } else {
+                    Err(anyhow!("invalid u32 type conversion from {expr_type:?}"))
+                }
+            }
+            Expr::AsU64(expr) => {
+                let expr_type = expr.infer_type()?;
+                if expr_type.is_numeric() {
+                    Ok(FormatType::Base(BaseType::U64))
+                } else {
+                    Err(anyhow!("invalid u64 type conversion from {expr_type:?}"))
+                }
+            }
+            Expr::Seq(exprs) => {
+                let mut elem_type = FormatType::Any;
+                for expr in exprs {
+                    elem_type = expr.infer_type()?.unify(&elem_type)?;
+                }
+                Ok(FormatType::Shape(TypeShape::Seq(Box::new(elem_type))))
+            }
+            Expr::Tuple(exprs) => {
+                let mut elem_types = Vec::with_capacity(exprs.len());
+                for expr in exprs {
+                    elem_types.push(expr.infer_type()?);
+                }
+                Ok(FormatType::Shape(TypeShape::Tuple(elem_types)))
+            }
+            Expr::LiftMaybe(None) => Ok(FormatType::Shape(TypeShape::Option(Box::new(FormatType::Any)))),
+            Expr::LiftMaybe(Some(expr)) => {
+                let expr_type = expr.infer_type()?;
+                Ok(FormatType::Shape(TypeShape::Option(Box::new(expr_type))))
+            }
+            Expr::Variant(lab, expr) => {
+                let expr_type = expr.infer_type()?;
+                Ok(FormatType::Shape(TypeShape::Union(BTreeMap::from([(lab.clone(), expr_type)]))))
+            }
+            Expr::IntRel(_rel, lhs, rhs) => {
+                let lhs_type = lhs.infer_type()?;
+                let rhs_type = rhs.infer_type()?;
+                match (lhs_type, rhs_type) {
+                    (FormatType::Base(b1), FormatType::Base(b2)) if b1 == b2 && b1.is_numeric() => Ok(FormatType::Base(BaseType::Bool)),
+                    (lhs_type, rhs_type) => Err(anyhow!("invalid integer relation between {lhs_type:?} and {rhs_type:?}")),
+                }
+            }
+            Expr::Arith(_arith, lhs, rhs) => {
+                let lhs_type = lhs.infer_type()?;
+                let rhs_type = rhs.infer_type()?;
+                match (lhs_type, rhs_type) {
+                    (FormatType::Base(b1), FormatType::Base(b2)) if b1 == b2 && b1.is_numeric() => Ok(FormatType::Base(b1)),
+                    (lhs_type, rhs_type) => Err(anyhow!("invalid arithmetic operation between {lhs_type:?} and {rhs_type:?}")),
+                }
+            }
+            Expr::Unary(Unary::BoolNot, expr) => {
+                let expr_type = expr.infer_type()?;
+                if matches!(expr_type, FormatType::Base(BaseType::Bool)) {
+                    Ok(FormatType::Base(BaseType::Bool))
+                } else {
+                    Err(anyhow!("invalid bool-not on {expr_type:?}"))
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum IntRel {
+    Eq, Neq,
+    Gt, Gte,
+    Lt, Lte,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum Arith {
+    Add, Sub,
+    Mul, Div, Rem,
+    Shl, Shr,
+    BitOr, BitAnd,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum Unary {
+    BoolNot,
+}
+
+pub struct FormatModule {
+    names: Vec<Label>,
+    decls: Vec<FormatDecl>,
+}
+
+impl FormatModule {
+    pub fn new() -> Self {
+        Self {
+            names: Vec::new(),
+            decls: Vec::new(),
+        }
+    }
+
+    pub fn declare_rec_formats(&mut self, formats: Vec<(Label, Format)>) -> Vec<FormatRef> {
+        let fmt_id = self.decls.len();
+        let batch_size = formats.len();
+        let batch_id = Span::from(fmt_id..fmt_id + batch_size);
+        for (ix, (name, format)) in formats.into_iter().enumerate() {
+            let decl = FormatDecl {
+                format,
+                fmt_id: fmt_id + ix,
+                f_type: Rc::new(OnceCell::new()),
+                batch: Some(batch_id.clone()),
+            };
+            self.names.push(name);
+            self.decls.push(decl);
+        }
+        let mut accum = Vec::with_capacity(batch_size);
+        for ix in batch_id.start..=batch_id.end {
+            let decl = &self.decls[ix];
+            match decl.solve_type_with(self, &mut HashSet::new()) {
+                Ok(_) => {
+                    accum.push(FormatRef(ix));
+                }
+                Err(e) => {
+                    panic!("Failed to solve type for {name}: {e}", name = &self.names[ix]);
+                }
+            }
+        }
+        accum
+    }
+
+    pub fn declare_format(&mut self, name: Label, format: Format) -> FormatRef {
+        let fmt_id = self.decls.len();
+        let f_type = Rc::new(OnceCell::new());
+        let decl = FormatDecl {
+            format,
+            fmt_id,
+            f_type,
+            batch: None,
+        };
+        match decl.solve_type(&self) {
+            Ok(_) => {
+                self.names.push(name);
+                self.decls.push(decl);
+                FormatRef(fmt_id)
+            }
+            Err(e) => {
+                panic!("Failed to solve type for {name}: {e}");
+            }
+        }
+    }
+
+    pub fn get_format_type(&self, level: usize) -> &FormatType {
+        &self.decls[level].solve_type(self).unwrap()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn simple_type_inference() -> AResult<()> {
+        let mut module = FormatModule::new();
+        let expr = Expr::IntRel(IntRel::Eq, Box::new(Expr::Arith(Arith::Add, Box::new(Expr::U8(1)), Box::new(Expr::U8(1)))), Box::new(Expr::U8(2)));
+        let f = Format::Compute(Box::new(expr));
+        let fref = module.declare_format(Label::Borrowed("static_math"), f);
+        assert!(matches!(module.get_format_type(fref.get_level()), FormatType::Base(BaseType::Bool)));
+        Ok(())
+    }
+}
diff --git a/experiments/doodle-rec/src/main.rs b/experiments/doodle-rec/src/main.rs
new file mode 100644
index 00000000..e7a11a96
--- /dev/null
+++ b/experiments/doodle-rec/src/main.rs
@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}
diff --git a/src/valuetype.rs b/src/valuetype.rs
index fec6ddc7..4f1faff8 100644
--- a/src/valuetype.rs
+++ b/src/valuetype.rs
@@ -127,9 +127,9 @@ impl ValueType {
                 let keys1 = bs1.keys().collect::<HashSet<_>>();
                 let keys2 = bs2.keys().collect::<HashSet<_>>();
 
-                let keys_common = HashSet::union(&keys1, &keys2).cloned();
+                let all_keys = HashSet::union(&keys1, &keys2).cloned();
 
-                for key in keys_common.into_iter() {
+                for key in all_keys.into_iter() {
                     match (bs1.get(key), bs2.get(key)) {
                         (Some(t1), Some(t2)) => {
                             let t = t1.unify(t2)?;

From 7b6c8b69e30430c3efc52c8c4445b92777ace7f9 Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Tue, 20 May 2025 15:42:13 +1000
Subject: [PATCH 02/15] Add auto-rec format test, fix Span implementation

---
 experiments/doodle-rec/src/lib.rs | 54 +++++++++++++++++++++++++------
 1 file changed, 44 insertions(+), 10 deletions(-)

diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index 227e1a70..3a45b500 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -1,4 +1,4 @@
-use std::{borrow::Cow, cell::OnceCell, collections::{BTreeMap, HashSet}, ops::Range, rc::Rc};
+use std::{borrow::Cow, cell::OnceCell, collections::{BTreeMap, HashSet}, ops::RangeInclusive, rc::Rc};
 use doodle::byte_set::ByteSet;
 use anyhow::{anyhow, Result as AResult};
 
@@ -30,9 +30,9 @@ impl<Idx> Span<Idx> {
     }
 }
 
-impl<Idx> From<Range<Idx>> for Span<Idx> {
-    fn from(value: Range<Idx>) -> Self {
-        Self { start: value.start, end: value.end }
+impl<Idx: Copy> From<RangeInclusive<Idx>> for Span<Idx> {
+    fn from(value: RangeInclusive<Idx>) -> Self {
+        Self { start: *value.start(), end: *value.end() }
     }
 }
 
@@ -84,13 +84,14 @@ impl BaseType {
 pub enum FormatType {
     Any,
     Void,
-    Unit,
     Base(BaseType),
     Ref(FormatId),
     Shape(TypeShape),
 }
 
 impl FormatType {
+    pub const UNIT : FormatType = FormatType::Shape(TypeShape::Tuple(Vec::new()));
+
     pub fn is_numeric(&self) -> bool {
         match self {
             FormatType::Base(base) => base.is_numeric(),
@@ -102,10 +103,14 @@ impl FormatType {
         match (self, other) {
             (FormatType::Any, _) => Ok(other.clone()),
             (_, FormatType::Any) => Ok(self.clone()),
+            (FormatType::Ref(id0), FormatType::Ref(id1)) => {
+                if id0 == id1 {
+                    Ok(FormatType::Ref(*id0))
+                } else {
+                    unimplemented!("cross-ref unification not implemented");
+                }
+            }
             (FormatType::Void, _) | (_, FormatType::Void) => Ok(FormatType::Void),
-            (FormatType::Unit, FormatType::Unit) => Ok(FormatType::Unit),
-            (FormatType::Unit, FormatType::Shape(TypeShape::Tuple(empty)))
-            | (FormatType::Shape(TypeShape::Tuple(empty)), FormatType::Unit) if empty.is_empty() => Ok(FormatType::Unit),
             (FormatType::Base(b1), FormatType::Base(b2)) if b1 == b2 => Ok(FormatType::Base(*b1)),
             (FormatType::Shape(s1), FormatType::Shape(s2)) => {
                 let s = s1.unify(s2)?;
@@ -220,7 +225,7 @@ impl Format {
                 }
             }
             Format::FailWith(_msg) => Ok(FormatType::Void),
-            Format::EndOfInput => Ok(FormatType::Unit),
+            Format::EndOfInput => Ok(FormatType::UNIT),
             Format::Byte(bs) if bs.is_empty() => Ok(FormatType::Void),
             Format::Byte(_) => Ok(FormatType::Base(BaseType::U8)),
             Format::Compute(expr) => expr.as_ref().infer_type(),
@@ -429,7 +434,7 @@ impl FormatModule {
     pub fn declare_rec_formats(&mut self, formats: Vec<(Label, Format)>) -> Vec<FormatRef> {
         let fmt_id = self.decls.len();
         let batch_size = formats.len();
-        let batch_id = Span::from(fmt_id..fmt_id + batch_size);
+        let batch_id = Span::from(fmt_id..=fmt_id + batch_size - 1);
         for (ix, (name, format)) in formats.into_iter().enumerate() {
             let decl = FormatDecl {
                 format,
@@ -494,4 +499,33 @@ mod tests {
         assert!(matches!(module.get_format_type(fref.get_level()), FormatType::Base(BaseType::Bool)));
         Ok(())
     }
+
+    #[test]
+    fn cons_list_any_byte() -> AResult<()> {
+        let mut module = FormatModule::new();
+        let format0 = Format::Union(vec![
+            Format::Variant(Label::Borrowed("Cons"), Box::new(Format::Tuple(vec![
+                Format::Byte(ByteSet::full()),
+                Format::RecVar(0),
+            ]))),
+            Format::Variant(Label::Borrowed("Nil"), Box::new(Format::Tuple(vec![]))),
+        ]);
+        let fref = module.declare_rec_formats(vec![(Label::Borrowed("list_any_byte"), format0)])[0];
+        let expected = FormatType::Shape(TypeShape::Union(BTreeMap::from([
+            (Label::Borrowed("Cons"), FormatType::Shape(TypeShape::Tuple(vec![
+                    FormatType::Base(BaseType::U8),
+                    FormatType::Ref(0),
+                ])
+            )),
+            (Label::Borrowed("Nil"), FormatType::UNIT),
+        ])));
+        let actual = module.get_format_type(fref.get_level());
+        match actual.unify(&expected) {
+            Ok(FormatType::Shape(TypeShape::Union(bs))) => assert_eq!(bs.len(), 2),
+            Err(e) => panic!("unification failed: {e}"),
+            other => panic!("unexpected type: {other:?}"),
+        }
+        eprintln!("cons_list_any_byte :: {actual:?}");
+        Ok(())
+    }
 }

From 49b2f0d94b8636e557ba0536be0179f2beb60a89 Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Thu, 22 May 2025 17:54:00 +1000
Subject: [PATCH 03/15] Implement rudimentary decoder-layer for doodle-rec

---
 experiments/doodle-rec/src/decoder.rs       | 524 ++++++++++++++++++++
 experiments/doodle-rec/src/decoder/error.rs |  82 +++
 experiments/doodle-rec/src/lib.rs           | 278 ++++++++---
 experiments/doodle-rec/src/main.rs          |  12 +
 experiments/doodle-rec/src/matchtree.rs     | 324 ++++++++++++
 src/lib.rs                                  |   2 +-
 6 files changed, 1159 insertions(+), 63 deletions(-)
 create mode 100644 experiments/doodle-rec/src/decoder.rs
 create mode 100644 experiments/doodle-rec/src/decoder/error.rs
 create mode 100644 experiments/doodle-rec/src/matchtree.rs

diff --git a/experiments/doodle-rec/src/decoder.rs b/experiments/doodle-rec/src/decoder.rs
new file mode 100644
index 00000000..99920f4e
--- /dev/null
+++ b/experiments/doodle-rec/src/decoder.rs
@@ -0,0 +1,524 @@
+use std::{
+    cmp::Ordering,
+    collections::{HashMap, HashSet},
+    rc::Rc,
+};
+
+use crate::{
+    Arith, Expr, Format, FormatDecl, FormatId, FormatModule, FormatType, IntRel, Label, RecId,
+    Span, Unary,
+    matchtree::{MatchTree, Next},
+};
+use anyhow::{Result as AResult, anyhow};
+use doodle::{IntWidth, byte_set::ByteSet, read::ReadCtxt};
+
+#[derive(Debug, Clone, Copy, Default)]
+enum RecurseCtx<'a> {
+    #[default]
+    NonRec,
+    Recurse {
+        entry_id: RecId,
+        span: Span<usize>,
+        batch: &'a [FormatDecl],
+    },
+}
+
+impl<'a> RecurseCtx<'a> {
+    const fn is_recursive(&self) -> bool {
+        matches!(self, RecurseCtx::Recurse { .. })
+    }
+
+    pub const fn as_span(&self) -> Option<Span<usize>> {
+        match self {
+            RecurseCtx::NonRec => None,
+            RecurseCtx::Recurse { span, .. } => Some(*span),
+        }
+    }
+
+    /// Returns `(new_ctx, is_auto)`
+    fn enter(&self, ix: RecId) -> (Self, Ordering) {
+        match self {
+            RecurseCtx::NonRec => panic!("cannot recurse into non-recursive context"),
+            RecurseCtx::Recurse {
+                batch,
+                span,
+                entry_id,
+            } => {
+                assert!(ix < batch.len(), "batch index out of range");
+                let ret = RecurseCtx::Recurse {
+                    entry_id: ix,
+                    span: *span,
+                    batch,
+                };
+                (ret, entry_id.cmp(&ix))
+            }
+        }
+    }
+
+    /// Returns the global format-level of the closest entry-point
+    fn get_level(&self) -> Option<usize> {
+        match self {
+            RecurseCtx::NonRec => None,
+            RecurseCtx::Recurse {
+                batch, entry_id, ..
+            } => Some(batch[*entry_id].fmt_id),
+        }
+    }
+
+    fn get_format(&self) -> Option<&'a Format> {
+        match self {
+            RecurseCtx::NonRec => None,
+            RecurseCtx::Recurse {
+                batch, entry_id, ..
+            } => Some(&batch[*entry_id].format),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum Value {
+    // Primitive values
+    U8(u8),
+    U16(u16),
+    U32(u32),
+    U64(u64),
+    Bool(bool),
+    Char(char),
+
+    // Shape-based values
+    Tuple(Vec<Value>),
+    Seq(Vec<Value>),
+    LiftOption(Option<Box<Value>>),
+    Variant(Label, Box<Value>),
+}
+
+impl Value {
+    fn get_usize_with_precision(&self) -> (usize, IntWidth) {
+        match self {
+            Value::U8(n) => (*n as usize, IntWidth::Bits8),
+            Value::U16(n) => (*n as usize, IntWidth::Bits16),
+            Value::U32(n) => (*n as usize, IntWidth::Bits32),
+            Value::U64(n) => (*n as usize, IntWidth::Bits64),
+            _ => panic!("value is not a number: {self:?}"),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Program {
+    pub decoders: Vec<(Decoder, FormatType)>,
+}
+
+impl Program {
+    fn new() -> Self {
+        let decoders = Vec::new();
+        Program { decoders }
+    }
+
+    pub fn run<'input>(&self, input: ReadCtxt<'input>) -> DecodeResult<(Value, ReadCtxt<'input>)> {
+        self.decoders[0].0.parse(self, input)
+    }
+}
+
+type Batch = Option<Span<usize>>;
+
+pub struct Compiler<'a> {
+    module: &'a FormatModule,
+    program: Program,
+    decoder_map: HashMap<(usize, Rc<Next<'a>>), usize>,
+    compile_queue: Vec<(&'a Format, Rc<Next<'a>>, usize, Batch)>,
+}
+
+impl<'a> Compiler<'a> {
+    fn new(module: &'a FormatModule) -> Self {
+        let program = Program::new();
+        let decoder_map = HashMap::new();
+        let compile_queue = Vec::new();
+        Compiler {
+            module,
+            program,
+            decoder_map,
+            compile_queue,
+        }
+    }
+
+    pub fn compile_program(
+        module: &FormatModule,
+        format: &Format,
+        ctx: RecurseCtx,
+    ) -> AResult<Program> {
+        let mut compiler = Compiler::new(module);
+
+        let mut visited = HashSet::new();
+
+        let batch = ctx.as_span();
+
+        let t = format.infer_type(&mut visited, module, batch)?;
+        compiler.queue_compile(t, format, Rc::new(Next::Empty), batch);
+        while let Some((f, next, n, batch)) = compiler.compile_queue.pop() {
+            let d = compiler.compile_format(f, next, ctx)?;
+            compiler.program.decoders[n].0 = d;
+        }
+        Ok(compiler.program)
+    }
+
+    fn queue_compile(
+        &mut self,
+        t: FormatType,
+        f: &'a Format,
+        next: Rc<Next<'a>>,
+        batch: Option<Span<usize>>,
+    ) -> usize {
+        let n = self.program.decoders.len();
+        self.program.decoders.push((Decoder::FAIL, t));
+        self.compile_queue.push((f, next, n, batch));
+        n
+    }
+
+    pub fn compile_one(format: &Format) -> AResult<Decoder> {
+        let module = FormatModule::new();
+        let mut compiler = Compiler::new(&module);
+        let ctx = RecurseCtx::NonRec;
+        compiler.compile_format(format, Rc::new(Next::Empty), ctx)
+    }
+
+    fn compile_format(
+        &mut self,
+        format: &'a Format,
+        next: Rc<Next<'a>>,
+        ctx: RecurseCtx<'a>,
+    ) -> AResult<Decoder> {
+        match format {
+            Format::ItemVar(level) => {
+                let f = self.module.get_format(*level);
+                let next = if f.depends_on_next(self.module) {
+                    next
+                } else {
+                    Rc::new(Next::Empty)
+                };
+                let n = if let Some(n) = self.decoder_map.get(&(*level, next.clone())) {
+                    *n
+                } else {
+                    let t = self.module.get_format_type(*level).clone();
+                    let ctx = match self.module.get_batch(*level) {
+                        Some(span) => RecurseCtx::Recurse {
+                            entry_id: level - span.start,
+                            span,
+                            batch: &self.module.decls[span.start..=span.end],
+                        },
+                        None => RecurseCtx::NonRec,
+                    };
+                    let n = self.queue_compile(t, f, next.clone(), ctx.as_span());
+                    self.decoder_map.insert((*level, next.clone()), n);
+                    n
+                };
+                Ok(Decoder::Call(n))
+            }
+            Format::RecVar(batch_ix) => {
+                let (new_ctx, cmp) = ctx.enter(*batch_ix);
+                let level = new_ctx.get_level().unwrap();
+                // FIXME - figure out the proper logic
+                let n = match cmp {
+                    Ordering::Equal => {
+                        if let Some(n) = self.decoder_map.get(&(level, next.clone())) {
+                            *n
+                        } else {
+                            let format = new_ctx.get_format().unwrap();
+                            let t = self.module.get_format_type(level).clone();
+                            let n = self.queue_compile(t, format, next.clone(), new_ctx.as_span());
+                            self.decoder_map.insert((level, next.clone()), n);
+                            n
+                        }
+                    }
+                    // ???
+                    Ordering::Less => todo!(), // recurse into earlier decoder (which should already be in the map)
+                    Ordering::Greater => todo!(), // recurse into later decoder (which may not yet be in the map)
+                };
+                Ok(Decoder::CallRec(n, *batch_ix))
+            }
+            Format::FailWith(msg) => Ok(Decoder::FailWith(msg.clone())),
+            Format::EndOfInput => Ok(Decoder::EndOfInput),
+            Format::Byte(bs) => Ok(Decoder::Byte(*bs)),
+            Format::Variant(label, f) => {
+                let d = self.compile_format(f, next.clone(), ctx)?;
+                Ok(Decoder::Variant(label.clone(), Box::new(d)))
+            }
+            Format::Compute(expr) => Ok(Decoder::Compute(expr.clone())),
+            Format::Union(branches) => {
+                let mut ds = Vec::with_capacity(branches.len());
+                for f in branches {
+                    ds.push(self.compile_format(f, next.clone(), ctx)?);
+                }
+                if let Some(tree) = MatchTree::build(self.module, branches, next) {
+                    Ok(Decoder::Branch(tree, ds))
+                } else {
+                    Err(anyhow!("cannot build match tree for {:?}", format))
+                }
+            }
+            Format::Tuple(elems) => {
+                let mut decs = Vec::with_capacity(elems.len());
+                let mut fields = elems.iter();
+                while let Some(f) = fields.next() {
+                    let next = Rc::new(Next::Sequence(fields.as_slice(), next.clone()));
+                    let df = self.compile_format(f, next, ctx)?;
+                    decs.push(df);
+                }
+                Ok(Decoder::Tuple(decs))
+            }
+            Format::Seq(elems) => {
+                let mut decs = Vec::with_capacity(elems.len());
+                let mut fields = elems.iter();
+                while let Some(f) = fields.next() {
+                    let next = Rc::new(Next::Sequence(fields.as_slice(), next.clone()));
+                    let df = self.compile_format(f, next, ctx)?;
+                    decs.push(df);
+                }
+                Ok(Decoder::Seq(decs))
+            }
+            Format::Repeat(a) => {
+                if a.is_nullable(self.module) {
+                    return Err(anyhow!("cannot repeat nullable format: {a:?}"));
+                }
+                let da = self.compile_format(a, Rc::new(Next::Repeat(a, next.clone())), ctx)?;
+                let astar = Format::Repeat(a.clone());
+                let fa = Format::Tuple(vec![(**a).clone(), astar]);
+                let fb = Format::EMPTY;
+                if let Some(tree) = MatchTree::build(self.module, &[fa, fb], next) {
+                    Ok(Decoder::While(tree, Box::new(da)))
+                } else {
+                    Err(anyhow!("cannot build match tree for {:?}", format))
+                }
+            }
+            Format::Maybe(x, a) => {
+                let da = Box::new(self.compile_format(a, Rc::new(Next::Empty), ctx)?);
+                Ok(Decoder::Maybe(x.clone(), da))
+            }
+        }
+    }
+}
+
+impl Expr {
+    pub fn eval(&self) -> Value {
+        match self {
+            Expr::U8(i) => Value::U8(*i),
+            Expr::U16(i) => Value::U16(*i),
+            Expr::U32(i) => Value::U32(*i),
+            Expr::U64(i) => Value::U64(*i),
+            Expr::Bool(b) => Value::Bool(*b),
+
+            Expr::AsChar(expr) => match expr.eval() {
+                Value::U8(x) => Value::Char(char::from(x)),
+                Value::U16(x) => {
+                    Value::Char(char::from_u32(x as u32).unwrap_or(char::REPLACEMENT_CHARACTER))
+                }
+                Value::U32(x) => {
+                    Value::Char(char::from_u32(x).unwrap_or(char::REPLACEMENT_CHARACTER))
+                }
+                Value::U64(x) => Value::Char(
+                    char::from_u32(u32::try_from(x).unwrap())
+                        .unwrap_or(char::REPLACEMENT_CHARACTER),
+                ),
+                _ => panic!("AsChar: expected U8, U16, U32, or U64"),
+            },
+            Expr::AsU8(x) => {
+                match x.eval() {
+                    Value::U8(x) => Value::U8(x),
+                    Value::U16(x) => Value::U8(u8::try_from(x).unwrap_or_else(|err| {
+                        panic!("cannot perform AsU8 cast on u16 {x}: {err}")
+                    })),
+                    Value::U32(x) => Value::U8(u8::try_from(x).unwrap_or_else(|err| {
+                        panic!("cannot perform AsU8 cast on u32 {x}: {err}")
+                    })),
+                    Value::U64(x) => Value::U8(u8::try_from(x).unwrap_or_else(|err| {
+                        panic!("cannot perform AsU8 cast on u64 {x}: {err}")
+                    })),
+                    x => panic!("cannot convert {x:?} to U8"),
+                }
+            }
+
+            Expr::AsU16(x) => match x.eval() {
+                Value::U8(x) => Value::U16(u16::from(x)),
+                Value::U16(x) => Value::U16(x),
+                Value::U32(x) => Value::U16(u16::try_from(x).unwrap()),
+                Value::U64(x) => Value::U16(u16::try_from(x).unwrap()),
+                x => panic!("cannot convert {x:?} to U16"),
+            },
+            Expr::AsU32(x) => match x.eval() {
+                Value::U8(x) => Value::U32(u32::from(x)),
+                Value::U16(x) => Value::U32(u32::from(x)),
+                Value::U32(x) => Value::U32(x),
+                Value::U64(x) => Value::U32(u32::try_from(x).unwrap()),
+                x => panic!("cannot convert {x:?} to U32"),
+            },
+            Expr::AsU64(x) => match x.eval() {
+                Value::U8(x) => Value::U64(u64::from(x)),
+                Value::U16(x) => Value::U64(u64::from(x)),
+                Value::U32(x) => Value::U64(u64::from(x)),
+                Value::U64(x) => Value::U64(x),
+                x => panic!("cannot convert {x:?} to U64"),
+            },
+            Expr::Seq(exprs) => Value::Seq(exprs.iter().map(Expr::eval).collect()),
+            Expr::Tuple(exprs) => Value::Tuple(exprs.iter().map(Expr::eval).collect()),
+            Expr::LiftOption(None) => Value::LiftOption(None),
+            Expr::LiftOption(Some(expr)) => Value::LiftOption(Some(Box::new(expr.eval()))),
+            Expr::Variant(lab, expr) => Value::Variant(lab.clone(), Box::new(expr.eval())),
+            Expr::IntRel(rel, lhs, rhs) => {
+                let lhs = lhs.eval();
+                let rhs = rhs.eval();
+                let (l, _lw) = lhs.get_usize_with_precision();
+                let (r, _rw) = rhs.get_usize_with_precision();
+                if _lw != _rw {
+                    panic!("cannot compare {lhs:?} with {rhs:?}");
+                }
+                match rel {
+                    IntRel::Eq => Value::Bool(l == r),
+                    IntRel::Lt => Value::Bool(l < r),
+                    IntRel::Gt => Value::Bool(l > r),
+                    IntRel::Neq => Value::Bool(l != r),
+                    IntRel::Lte => Value::Bool(l <= r),
+                    IntRel::Gte => Value::Bool(l >= r),
+                }
+            }
+            Expr::Arith(Arith::Add, lhs, rhs) => match (lhs.eval(), rhs.eval()) {
+                (Value::U8(l), Value::U8(r)) => Value::U8(l.checked_add(r).unwrap()),
+                (Value::U16(l), Value::U16(r)) => Value::U16(l.checked_add(r).unwrap()),
+                (Value::U32(l), Value::U32(r)) => Value::U32(l.checked_add(r).unwrap()),
+                (Value::U64(l), Value::U64(r)) => Value::U64(l.checked_add(r).unwrap()),
+                (l, r) => panic!("cannot add {l:?} and {r:?}"),
+            },
+            Expr::Arith(Arith::Sub, lhs, rhs) => match (lhs.eval(), rhs.eval()) {
+                (Value::U8(l), Value::U8(r)) => Value::U8(l.checked_sub(r).unwrap()),
+                (Value::U16(l), Value::U16(r)) => Value::U16(l.checked_sub(r).unwrap()),
+                (Value::U32(l), Value::U32(r)) => Value::U32(l.checked_sub(r).unwrap()),
+                (Value::U64(l), Value::U64(r)) => Value::U64(l.checked_sub(r).unwrap()),
+                (l, r) => panic!("cannot subtract {l:?} and {r:?}"),
+            },
+            Expr::Arith(Arith::Mul, lhs, rhs) => match (lhs.eval(), rhs.eval()) {
+                (Value::U8(l), Value::U8(r)) => Value::U8(l.checked_mul(r).unwrap()),
+                (Value::U16(l), Value::U16(r)) => Value::U16(l.checked_mul(r).unwrap()),
+                (Value::U32(l), Value::U32(r)) => Value::U32(l.checked_mul(r).unwrap()),
+                (Value::U64(l), Value::U64(r)) => Value::U64(l.checked_mul(r).unwrap()),
+                (l, r) => panic!("cannot multiply {l:?} and {r:?}"),
+            },
+            Expr::Arith(Arith::Div, lhs, rhs) => match (lhs.eval(), rhs.eval()) {
+                (Value::U8(l), Value::U8(r)) => Value::U8(l.checked_div(r).unwrap()),
+                (Value::U16(l), Value::U16(r)) => Value::U16(l.checked_div(r).unwrap()),
+                (Value::U32(l), Value::U32(r)) => Value::U32(l.checked_div(r).unwrap()),
+                (Value::U64(l), Value::U64(r)) => Value::U64(l.checked_div(r).unwrap()),
+                (l, r) => panic!("cannot divide {l:?} and {r:?}"),
+            },
+            Expr::Arith(Arith::Rem, lhs, rhs) => match (lhs.eval(), rhs.eval()) {
+                (Value::U8(l), Value::U8(r)) => Value::U8(l.checked_rem(r).unwrap()),
+                (Value::U16(l), Value::U16(r)) => Value::U16(l.checked_rem(r).unwrap()),
+                (Value::U32(l), Value::U32(r)) => Value::U32(l.checked_rem(r).unwrap()),
+                (Value::U64(l), Value::U64(r)) => Value::U64(l.checked_rem(r).unwrap()),
+                (l, r) => panic!("cannot compute remainder {l:?} and {r:?}"),
+            },
+            Expr::Arith(Arith::BitAnd, lhs, rhs) => match (lhs.eval(), rhs.eval()) {
+                (Value::U8(l), Value::U8(r)) => Value::U8(l & r),
+                (Value::U16(l), Value::U16(r)) => Value::U16(l & r),
+                (Value::U32(l), Value::U32(r)) => Value::U32(l & r),
+                (Value::U64(l), Value::U64(r)) => Value::U64(l & r),
+                (l, r) => panic!("cannot bitwise and {l:?} and {r:?}"),
+            },
+            Expr::Arith(Arith::BitOr, lhs, rhs) => match (lhs.eval(), rhs.eval()) {
+                (Value::U8(l), Value::U8(r)) => Value::U8(l | r),
+                (Value::U16(l), Value::U16(r)) => Value::U16(l | r),
+                (Value::U32(l), Value::U32(r)) => Value::U32(l | r),
+                (Value::U64(l), Value::U64(r)) => Value::U64(l | r),
+                (l, r) => panic!("cannot bitwise or {l:?} and {r:?}"),
+            },
+            Expr::Arith(Arith::Shl, lhs, rhs) => match (lhs.eval(), rhs.eval()) {
+                (Value::U8(l), Value::U8(r)) => Value::U8(l.checked_shl(r as u32).unwrap()),
+                (Value::U16(l), Value::U16(r)) => Value::U16(l.checked_shl(r as u32).unwrap()),
+                (Value::U32(l), Value::U32(r)) => Value::U32(l.checked_shl(r).unwrap()),
+                (Value::U64(l), Value::U64(r)) => {
+                    Value::U64(l.checked_shl(u32::try_from(r).unwrap()).unwrap())
+                }
+                (l, r) => panic!("cannot shift left {l:?} and {r:?}"),
+            },
+            Expr::Arith(Arith::Shr, lhs, rhs) => match (lhs.eval(), rhs.eval()) {
+                (Value::U8(l), Value::U8(r)) => Value::U8(l.checked_shr(r as u32).unwrap()),
+                (Value::U16(l), Value::U16(r)) => Value::U16(l.checked_shr(r as u32).unwrap()),
+                (Value::U32(l), Value::U32(r)) => Value::U32(l.checked_shr(r).unwrap()),
+                (Value::U64(l), Value::U64(r)) => {
+                    Value::U64(l.checked_shr(u32::try_from(r).unwrap()).unwrap())
+                }
+                (l, r) => panic!("cannot shift right {l:?} and {r:?}"),
+            },
+            Expr::Unary(Unary::BoolNot, expr) => match expr.eval() {
+                Value::Bool(x) => Value::Bool(!x),
+                x => panic!("cannot negate {x:?}"),
+            },
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum Decoder {
+    Call(FormatId),
+    CallRec(FormatId, RecId),
+
+    FailWith(Label),
+    EndOfInput,
+    Byte(ByteSet),
+    Compute(Box<Expr>),
+
+    Variant(Label, Box<Decoder>),
+    Branch(MatchTree, Vec<Decoder>),
+
+    While(MatchTree, Box<Decoder>), // Repeat decoder while input matches
+
+    Seq(Vec<Decoder>),
+    Tuple(Vec<Decoder>),
+    Maybe(Box<Expr>, Box<Decoder>),
+}
+
+pub(crate) mod error;
+use error::DecodeError;
+
+pub type DecodeResult<T> = Result<T, DecodeError>;
+
+impl Decoder {
+    pub(crate) const FAIL: Self = Decoder::FailWith(Label::Borrowed("FAIL_CONST"));
+
+    pub fn parse<'input>(
+        &self,
+        program: &Program,
+        input: ReadCtxt<'input>,
+    ) -> DecodeResult<(Value, ReadCtxt<'input>)> {
+        match self {
+            Decoder::FailWith(msg) => Err(DecodeError::fail(msg.clone(), input)),
+            Decoder::EndOfInput => match input.read_byte() {
+                None => Ok((Value::Tuple(vec![]), input)),
+                Some((b, _)) => Err(DecodeError::Trailing {
+                    byte: b,
+                    offset: input.offset,
+                }),
+            },
+            Decoder::Byte(bs) => {
+                let (b, input) = input.read_byte().ok_or(DecodeError::Overbyte {
+                    offset: input.offset,
+                })?;
+                if bs.contains(b) {
+                    Ok((Value::U8(b), input))
+                } else {
+                    Err(DecodeError::Unexpected {
+                        found: b,
+                        expected: *bs,
+                        offset: input.offset,
+                    })
+                }
+            }
+            Decoder::Call(ix) => program.decoders[*ix].0.parse(program, input),
+            Decoder::CallRec(_, _) => todo!(),
+            Decoder::Compute(expr) => todo!(),
+            Decoder::Variant(cow, decoder) => todo!(),
+            Decoder::Branch(match_tree, decoders) => todo!(),
+            Decoder::While(match_tree, decoder) => todo!(),
+            Decoder::Seq(decoders) => todo!(),
+            Decoder::Tuple(decoders) => todo!(),
+            Decoder::Maybe(expr, decoder) => todo!(),
+        }
+    }
+}
diff --git a/experiments/doodle-rec/src/decoder/error.rs b/experiments/doodle-rec/src/decoder/error.rs
new file mode 100644
index 00000000..817920ad
--- /dev/null
+++ b/experiments/doodle-rec/src/decoder/error.rs
@@ -0,0 +1,82 @@
+use crate::Label;
+use doodle::{prelude::ByteSet, read::ReadCtxt};
+
+#[derive(Debug)]
+pub enum DecodeError {
+    Fail {
+        message: Label,
+        offset: usize,
+        buffer: Vec<u8>,
+    },
+    Trailing {
+        byte: u8,
+        offset: usize,
+    },
+    Overbyte {
+        offset: usize,
+    },
+    Unexpected {
+        found: u8,
+        expected: ByteSet,
+        offset: usize,
+    },
+    NoValidBranch {
+        offset: usize,
+    },
+}
+
+impl std::fmt::Display for DecodeError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Fail {
+                message,
+                offset,
+                buffer,
+            } => {
+                write!(
+                    f,
+                    "failed with message \"{message}\" at Offset={offset}, Buffer={buffer:#?}"
+                )
+            }
+            Self::Trailing { byte, offset } => {
+                write!(
+                    f,
+                    "byte `{byte:02x}` found when end-of-input expected (offset = {offset})"
+                )
+            }
+            Self::Overbyte { offset } => {
+                write!(
+                    f,
+                    "attempted read of byte would overrun buffer (offset = {offset})"
+                )
+            }
+            Self::Unexpected {
+                found,
+                expected,
+                offset,
+            } => {
+                write!(
+                    f,
+                    "byte `{found:02x}` found when {expected:?} expected (offset = {offset})"
+                )
+            }
+            Self::NoValidBranch { offset } => {
+                write!(f, "no valid branch at offset {offset}")
+            }
+        }
+    }
+}
+
+impl std::error::Error for DecodeError {}
+
+impl DecodeError {
+    pub fn fail(message: Label, input: ReadCtxt<'_>) -> Self {
+        let offset = input.offset;
+        let buffer = input.input.to_owned();
+        Self::Fail {
+            message,
+            offset,
+            buffer,
+        }
+    }
+}
diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index 3a45b500..672c68be 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -1,21 +1,36 @@
-use std::{borrow::Cow, cell::OnceCell, collections::{BTreeMap, HashSet}, ops::RangeInclusive, rc::Rc};
-use doodle::byte_set::ByteSet;
-use anyhow::{anyhow, Result as AResult};
-
+pub mod decoder;
+pub(crate) mod matchtree;
+pub(crate) use matchtree::{MatchTree, Next};
+
+use anyhow::{Result as AResult, anyhow};
+use doodle::{bounds::Bounds, byte_set::ByteSet};
+use std::{
+    borrow::Cow,
+    cell::OnceCell,
+    collections::{BTreeMap, HashSet},
+    ops::{Add as _, RangeInclusive},
+    rc::Rc,
+};
 
 pub type Label = Cow<'static, str>;
 
+/// Global index into the total set of formats within a Module
 pub type FormatId = usize;
 
+/// Local index into a Batch of formats (e.g. 0 would be 'self' in a singleton-batch)
 pub type RecId = usize;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub struct FormatRef(FormatId);
 
 impl FormatRef {
-    pub const fn get_level(&self) -> usize {
+    pub const fn get_level(self) -> usize {
         self.0
     }
+
+    pub fn call(self) -> Format {
+        Format::ItemVar(self.0)
+    }
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
@@ -32,7 +47,10 @@ impl<Idx> Span<Idx> {
 
 impl<Idx: Copy> From<RangeInclusive<Idx>> for Span<Idx> {
     fn from(value: RangeInclusive<Idx>) -> Self {
-        Self { start: *value.start(), end: *value.end() }
+        Self {
+            start: *value.start(),
+            end: *value.end(),
+        }
     }
 }
 
@@ -50,12 +68,18 @@ impl FormatDecl {
         self.solve_type_with(module, &mut visited)
     }
 
-    pub(crate) fn solve_type_with(&self, module: &FormatModule, visited: &mut HashSet<FormatId>) -> AResult<&FormatType> {
+    pub(crate) fn solve_type_with(
+        &self,
+        module: &FormatModule,
+        visited: &mut HashSet<FormatId>,
+    ) -> AResult<&FormatType> {
         match self.f_type.get() {
             None => {
                 visited.insert(self.fmt_id);
                 let f_type = self.format.infer_type(visited, module, self.batch)?;
-                let Ok(_) = self.f_type.set(f_type) else { unreachable!("synchronous TOCTOU!?") };
+                let Ok(_) = self.f_type.set(f_type) else {
+                    unreachable!("synchronous TOCTOU!?")
+                };
                 Ok(self.f_type.get().unwrap())
             }
             Some(f_type) => Ok(f_type),
@@ -63,7 +87,6 @@ impl FormatDecl {
     }
 }
 
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum BaseType {
     Bool,
@@ -76,7 +99,10 @@ pub enum BaseType {
 
 impl BaseType {
     pub fn is_numeric(&self) -> bool {
-        matches!(self, BaseType::U8 | BaseType::U16 | BaseType::U32 | BaseType::U64)
+        matches!(
+            self,
+            BaseType::U8 | BaseType::U16 | BaseType::U32 | BaseType::U64
+        )
     }
 }
 
@@ -90,7 +116,7 @@ pub enum FormatType {
 }
 
 impl FormatType {
-    pub const UNIT : FormatType = FormatType::Shape(TypeShape::Tuple(Vec::new()));
+    pub const UNIT: FormatType = FormatType::Shape(TypeShape::Tuple(Vec::new()));
 
     pub fn is_numeric(&self) -> bool {
         match self {
@@ -116,7 +142,9 @@ impl FormatType {
                 let s = s1.unify(s2)?;
                 Ok(FormatType::Shape(s))
             }
-            _ => Err(anyhow!("cannot unify incompatible types: {self:?}, {other:?}")),
+            _ => Err(anyhow!(
+                "cannot unify incompatible types: {self:?}, {other:?}"
+            )),
         }
     }
 }
@@ -134,7 +162,9 @@ impl TypeShape {
         match (self, other) {
             (TypeShape::Tuple(t1), TypeShape::Tuple(t2)) => {
                 if t1.len() != t2.len() {
-                    return Err(anyhow!("cannot unify tuples of different arity: {t1:?}, {t2:?}"));
+                    return Err(anyhow!(
+                        "cannot unify tuples of different arity: {t1:?}, {t2:?}"
+                    ));
                 }
                 let mut unified = Vec::with_capacity(t1.len());
                 for (t1, t2) in t1.iter().zip(t2.iter()) {
@@ -143,7 +173,9 @@ impl TypeShape {
                 Ok(TypeShape::Tuple(unified))
             }
             (TypeShape::Seq(t1), TypeShape::Seq(t2)) => Ok(TypeShape::Seq(Box::new(t1.unify(t2)?))),
-            (TypeShape::Option(t1), TypeShape::Option(t2)) => Ok(TypeShape::Option(Box::new(t1.unify(t2)?))),
+            (TypeShape::Option(t1), TypeShape::Option(t2)) => {
+                Ok(TypeShape::Option(Box::new(t1.unify(t2)?)))
+            }
             (TypeShape::Union(bs1), TypeShape::Union(bs2)) => {
                 let mut bs = BTreeMap::new();
 
@@ -171,7 +203,7 @@ impl TypeShape {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum Format {
     // References to other formats
     ItemVar(FormatId),
@@ -197,7 +229,14 @@ pub enum Format {
 }
 
 impl Format {
-    fn infer_type<'ctx>(&'ctx self, visited: &mut HashSet<FormatId>, module: &'ctx FormatModule, batch: Option<Span<FormatId>>) -> AResult<FormatType> {
+    pub const EMPTY: Self = Format::Tuple(vec![]);
+
+    fn infer_type<'ctx>(
+        &'ctx self,
+        visited: &mut HashSet<FormatId>,
+        module: &'ctx FormatModule,
+        batch: Option<Span<FormatId>>,
+    ) -> AResult<FormatType> {
         match self {
             Format::ItemVar(level) => {
                 if visited.contains(level) {
@@ -207,23 +246,21 @@ impl Format {
                     Ok(decl.solve_type_with(module, visited)?.clone())
                 }
             }
-            Format::RecVar(batch_ix) => {
-                match batch {
-                    None => Err(anyhow!("Recursion without a batch")),
-                    Some(range) => {
-                        let level = range.start + batch_ix;
-                        if level > range.end {
-                            return Err(anyhow!("batch index out of range"))
-                        }
-                        if visited.contains(&level) {
-                            Ok(FormatType::Ref(level))
-                        } else {
-                            let decl = &module.decls[level];
-                            Ok(decl.solve_type_with(module, visited)?.clone())
-                        }
+            Format::RecVar(batch_ix) => match batch {
+                None => Err(anyhow!("Recursion without a batch")),
+                Some(range) => {
+                    let level = range.start + batch_ix;
+                    if level > range.end {
+                        return Err(anyhow!("batch index out of range"));
+                    }
+                    if visited.contains(&level) {
+                        Ok(FormatType::Ref(level))
+                    } else {
+                        let decl = &module.decls[level];
+                        Ok(decl.solve_type_with(module, visited)?.clone())
                     }
                 }
-            }
+            },
             Format::FailWith(_msg) => Ok(FormatType::Void),
             Format::EndOfInput => Ok(FormatType::UNIT),
             Format::Byte(bs) if bs.is_empty() => Ok(FormatType::Void),
@@ -231,7 +268,10 @@ impl Format {
             Format::Compute(expr) => expr.as_ref().infer_type(),
             Format::Variant(label, inner) => {
                 let inner_type = inner.infer_type(visited, module, batch)?;
-                Ok(FormatType::Shape(TypeShape::Union(BTreeMap::from([(label.clone(), inner_type)]))))
+                Ok(FormatType::Shape(TypeShape::Union(BTreeMap::from([(
+                    label.clone(),
+                    inner_type,
+                )]))))
             }
             Format::Union(branches) => {
                 let mut t = FormatType::Any;
@@ -263,15 +303,77 @@ impl Format {
                     let t = format.infer_type(visited, module, batch)?;
                     Ok(FormatType::Shape(TypeShape::Option(Box::new(t))))
                 }
-                other => Err(anyhow!("maybe expression type was inferred to be non-bool: {other:?}")),
+                other => Err(anyhow!(
+                    "maybe expression type was inferred to be non-bool: {other:?}"
+                )),
+            },
+        }
+    }
+
+    fn depends_on_next(&self, module: &FormatModule) -> bool {
+        match self {
+            Format::ItemVar(level) => module.get_format(*level).depends_on_next(module),
+            Format::FailWith(..) => false,
+            Format::EndOfInput => false,
+            Format::Byte(..) => false,
+            Format::Compute(..) => false,
+            Format::RecVar(..) => {
+                // REVIEW - are there any recursive formats that *don't* depend on next?
+                // FIXME[epic=hardcoded] - this is a placeholder for future improvements to classification logic
+                true
+            }
+            Format::Variant(_, f) => f.depends_on_next(module),
+            Format::Union(branches) => Format::union_depends_on_next(branches, module),
+            Format::Repeat(..) => true,
+            Format::Seq(formats) | Format::Tuple(formats) => {
+                formats.iter().any(|f| f.depends_on_next(module))
+            }
+            Format::Maybe(..) => true,
+        }
+    }
+
+    fn union_depends_on_next(branches: &[Format], module: &FormatModule) -> bool {
+        let mut fs = Vec::with_capacity(branches.len());
+        for f in branches {
+            if f.depends_on_next(module) {
+                return true;
             }
+            fs.push(f.clone());
         }
+        MatchTree::build(module, &fs, Rc::new(Next::Empty)).is_none()
     }
 
+    fn is_nullable(&self, module: &FormatModule) -> bool {
+        self.match_bounds(module).min() == 0
+    }
 
+    fn match_bounds(&self, module: &FormatModule) -> Bounds {
+        match self {
+            Format::ItemVar(level) => module.get_format(*level).match_bounds(module),
+            Format::FailWith(..) | Format::EndOfInput | Format::Compute(..) => Bounds::exact(0),
+            Format::Byte(_) => Bounds::exact(1),
+            Format::Variant(_, f) => f.match_bounds(module),
+            Format::Union(branches) => branches
+                .iter()
+                .map(|f| f.match_bounds(module))
+                .reduce(Bounds::union)
+                .unwrap(),
+            Format::Tuple(fields) | Format::Seq(fields) => fields
+                .iter()
+                .map(|f| f.match_bounds(module))
+                .reduce(Bounds::add)
+                .unwrap_or(Bounds::exact(0)),
+            Format::Repeat(_) => Bounds::any(),
+            Format::Maybe(_, f) => Bounds::union(Bounds::exact(0), f.match_bounds(module)),
+            Format::RecVar(..) => {
+                // REVIEW - we cannot get better than this without a complex model, and certainly not without adding more parameters
+                Bounds::any()
+            }
+        }
+    }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum Expr {
     // Primitive Values
     U8(u8),
@@ -290,7 +392,7 @@ pub enum Expr {
     // Higher-Order Exprs
     Seq(Vec<Expr>),
     Tuple(Vec<Expr>),
-    LiftMaybe(Option<Box<Expr>>),
+    LiftOption(Option<Box<Expr>>),
     Variant(Label, Box<Expr>),
 
     // Operational
@@ -361,29 +463,42 @@ impl Expr {
                 }
                 Ok(FormatType::Shape(TypeShape::Tuple(elem_types)))
             }
-            Expr::LiftMaybe(None) => Ok(FormatType::Shape(TypeShape::Option(Box::new(FormatType::Any)))),
-            Expr::LiftMaybe(Some(expr)) => {
+            Expr::LiftOption(None) => Ok(FormatType::Shape(TypeShape::Option(Box::new(
+                FormatType::Any,
+            )))),
+            Expr::LiftOption(Some(expr)) => {
                 let expr_type = expr.infer_type()?;
                 Ok(FormatType::Shape(TypeShape::Option(Box::new(expr_type))))
             }
             Expr::Variant(lab, expr) => {
                 let expr_type = expr.infer_type()?;
-                Ok(FormatType::Shape(TypeShape::Union(BTreeMap::from([(lab.clone(), expr_type)]))))
+                Ok(FormatType::Shape(TypeShape::Union(BTreeMap::from([(
+                    lab.clone(),
+                    expr_type,
+                )]))))
             }
             Expr::IntRel(_rel, lhs, rhs) => {
                 let lhs_type = lhs.infer_type()?;
                 let rhs_type = rhs.infer_type()?;
                 match (lhs_type, rhs_type) {
-                    (FormatType::Base(b1), FormatType::Base(b2)) if b1 == b2 && b1.is_numeric() => Ok(FormatType::Base(BaseType::Bool)),
-                    (lhs_type, rhs_type) => Err(anyhow!("invalid integer relation between {lhs_type:?} and {rhs_type:?}")),
+                    (FormatType::Base(b1), FormatType::Base(b2)) if b1 == b2 && b1.is_numeric() => {
+                        Ok(FormatType::Base(BaseType::Bool))
+                    }
+                    (lhs_type, rhs_type) => Err(anyhow!(
+                        "invalid integer relation between {lhs_type:?} and {rhs_type:?}"
+                    )),
                 }
             }
             Expr::Arith(_arith, lhs, rhs) => {
                 let lhs_type = lhs.infer_type()?;
                 let rhs_type = rhs.infer_type()?;
                 match (lhs_type, rhs_type) {
-                    (FormatType::Base(b1), FormatType::Base(b2)) if b1 == b2 && b1.is_numeric() => Ok(FormatType::Base(b1)),
-                    (lhs_type, rhs_type) => Err(anyhow!("invalid arithmetic operation between {lhs_type:?} and {rhs_type:?}")),
+                    (FormatType::Base(b1), FormatType::Base(b2)) if b1 == b2 && b1.is_numeric() => {
+                        Ok(FormatType::Base(b1))
+                    }
+                    (lhs_type, rhs_type) => Err(anyhow!(
+                        "invalid arithmetic operation between {lhs_type:?} and {rhs_type:?}"
+                    )),
                 }
             }
             Expr::Unary(Unary::BoolNot, expr) => {
@@ -398,22 +513,30 @@ impl Expr {
     }
 }
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum IntRel {
-    Eq, Neq,
-    Gt, Gte,
-    Lt, Lte,
+    Eq,
+    Neq,
+    Gt,
+    Gte,
+    Lt,
+    Lte,
 }
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum Arith {
-    Add, Sub,
-    Mul, Div, Rem,
-    Shl, Shr,
-    BitOr, BitAnd,
+    Add,
+    Sub,
+    Mul,
+    Div,
+    Rem,
+    Shl,
+    Shr,
+    BitOr,
+    BitAnd,
 }
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum Unary {
     BoolNot,
 }
@@ -453,7 +576,10 @@ impl FormatModule {
                     accum.push(FormatRef(ix));
                 }
                 Err(e) => {
-                    panic!("Failed to solve type for {name}: {e}", name = &self.names[ix]);
+                    panic!(
+                        "Failed to solve type for {name}: {e}",
+                        name = &self.names[ix]
+                    );
                 }
             }
         }
@@ -484,6 +610,18 @@ impl FormatModule {
     pub fn get_format_type(&self, level: usize) -> &FormatType {
         &self.decls[level].solve_type(self).unwrap()
     }
+
+    pub fn get_format(&self, level: usize) -> &Format {
+        &self.decls[level].format
+    }
+
+    fn get_decl(&self, level: usize) -> &FormatDecl {
+        &self.decls[level]
+    }
+
+    fn get_batch(&self, level: usize) -> Option<Span<FormatId>> {
+        self.decls[level].batch
+    }
 }
 
 #[cfg(test)]
@@ -493,10 +631,21 @@ mod tests {
     #[test]
     fn simple_type_inference() -> AResult<()> {
         let mut module = FormatModule::new();
-        let expr = Expr::IntRel(IntRel::Eq, Box::new(Expr::Arith(Arith::Add, Box::new(Expr::U8(1)), Box::new(Expr::U8(1)))), Box::new(Expr::U8(2)));
+        let expr = Expr::IntRel(
+            IntRel::Eq,
+            Box::new(Expr::Arith(
+                Arith::Add,
+                Box::new(Expr::U8(1)),
+                Box::new(Expr::U8(1)),
+            )),
+            Box::new(Expr::U8(2)),
+        );
         let f = Format::Compute(Box::new(expr));
         let fref = module.declare_format(Label::Borrowed("static_math"), f);
-        assert!(matches!(module.get_format_type(fref.get_level()), FormatType::Base(BaseType::Bool)));
+        assert!(matches!(
+            module.get_format_type(fref.get_level()),
+            FormatType::Base(BaseType::Bool)
+        ));
         Ok(())
     }
 
@@ -504,19 +653,24 @@ mod tests {
     fn cons_list_any_byte() -> AResult<()> {
         let mut module = FormatModule::new();
         let format0 = Format::Union(vec![
-            Format::Variant(Label::Borrowed("Cons"), Box::new(Format::Tuple(vec![
-                Format::Byte(ByteSet::full()),
-                Format::RecVar(0),
-            ]))),
+            Format::Variant(
+                Label::Borrowed("Cons"),
+                Box::new(Format::Tuple(vec![
+                    Format::Byte(ByteSet::full()),
+                    Format::RecVar(0),
+                ])),
+            ),
             Format::Variant(Label::Borrowed("Nil"), Box::new(Format::Tuple(vec![]))),
         ]);
         let fref = module.declare_rec_formats(vec![(Label::Borrowed("list_any_byte"), format0)])[0];
         let expected = FormatType::Shape(TypeShape::Union(BTreeMap::from([
-            (Label::Borrowed("Cons"), FormatType::Shape(TypeShape::Tuple(vec![
+            (
+                Label::Borrowed("Cons"),
+                FormatType::Shape(TypeShape::Tuple(vec![
                     FormatType::Base(BaseType::U8),
                     FormatType::Ref(0),
-                ])
-            )),
+                ])),
+            ),
             (Label::Borrowed("Nil"), FormatType::UNIT),
         ])));
         let actual = module.get_format_type(fref.get_level());
diff --git a/experiments/doodle-rec/src/main.rs b/experiments/doodle-rec/src/main.rs
index e7a11a96..f97f2469 100644
--- a/experiments/doodle-rec/src/main.rs
+++ b/experiments/doodle-rec/src/main.rs
@@ -1,3 +1,15 @@
+use doodle_rec::{Format, FormatModule, FormatRef, Label};
+
+fn bson(module: &mut FormatModule) -> FormatRef {
+    todo!();
+}
+
 fn main() {
+    let mut module = FormatModule::new();
+    let bson = bson(&mut module);
+    let main = module.declare_format(
+        Label::Borrowed("main"),
+        Format::Tuple(vec![bson.call(), Format::EndOfInput]),
+    );
     println!("Hello, world!");
 }
diff --git a/experiments/doodle-rec/src/matchtree.rs b/experiments/doodle-rec/src/matchtree.rs
new file mode 100644
index 00000000..0f830b8f
--- /dev/null
+++ b/experiments/doodle-rec/src/matchtree.rs
@@ -0,0 +1,324 @@
+use std::{collections::HashSet, rc::Rc, vec};
+
+use crate::{Format, FormatModule};
+use doodle::{byte_set::ByteSet, read::ReadCtxt};
+
+#[derive(Clone, Debug)]
+pub(crate) struct MatchTreeStep<'a> {
+    accept: bool,
+    branches: Vec<(ByteSet, Rc<Next<'a>>)>,
+}
+
+impl<'a> MatchTreeStep<'a> {
+    pub fn accept() -> MatchTreeStep<'a> {
+        MatchTreeStep {
+            accept: true,
+            branches: vec![],
+        }
+    }
+
+    pub fn reject() -> MatchTreeStep<'a> {
+        MatchTreeStep {
+            accept: false,
+            branches: vec![],
+        }
+    }
+
+    fn branch(bs: ByteSet, next: Rc<Next<'a>>) -> MatchTreeStep<'a> {
+        MatchTreeStep {
+            accept: false,
+            branches: vec![(bs, next)],
+        }
+    }
+
+    /// Modifies a `MatchTreeStep` in place, so that it will accept a new branch given by the argument values.
+    fn union_branch(&mut self, mut bs: ByteSet, next: Rc<Next<'a>>) {
+        let mut branches = Vec::new();
+        for (bs0, next0) in self.branches.iter_mut() {
+            let common = bs0.intersection(&bs);
+            if !common.is_empty() {
+                let orig = bs0.difference(&bs);
+                if !orig.is_empty() {
+                    branches.push((orig, next0.clone()));
+                }
+                *bs0 = common;
+                *next0 = Rc::new(Next::Union(next0.clone(), next.clone()));
+                bs = bs.difference(bs0);
+            }
+        }
+        if !bs.is_empty() {
+            self.branches.push((bs, next));
+        }
+        self.branches.append(&mut branches);
+    }
+
+    /// Combines two `MatchTreeSteps` into their logical union
+    fn union(mut self, other: MatchTreeStep<'a>) -> MatchTreeStep<'a> {
+        self.accept = self.accept || other.accept;
+        for (bs, next) in other.branches {
+            self.union_branch(bs, next);
+        }
+        self
+    }
+
+    /// Constructs a [MatchTreeStep] that accepts a given tuple of sequential formats, with a trailing sequence of partially-consumed formats ([`Next`]s).
+    fn from_sequential(
+        module: &'a FormatModule,
+        fields: &'a [Format],
+        next: Rc<Next<'a>>,
+    ) -> MatchTreeStep<'a> {
+        match fields.split_first() {
+            None => Self::from_next(module, next),
+            Some((f, fs)) => Self::from_format(module, f, Rc::new(Next::Sequence(fs, next))),
+        }
+    }
+
+    /// Constructs a [MatchTreeStep] from a [`Next`]
+    fn from_next(module: &'a FormatModule, next: Rc<Next<'a>>) -> MatchTreeStep<'a> {
+        match next.as_ref() {
+            Next::Empty => Self::accept(),
+            Next::Union(next1, next2) => {
+                let tree1 = Self::from_next(module, next1.clone());
+                let tree2 = Self::from_next(module, next2.clone());
+                tree1.union(tree2)
+            }
+            Next::Cat(f, next) => MatchTreeStep::<'a>::from_format(module, *f, next.clone()),
+            Next::Sequence(fields, next) => {
+                let next = next.clone();
+                match fields.split_first() {
+                    None => Self::from_next(module, next),
+                    Some((f, fs)) => {
+                        Self::from_format(module, f, Rc::new(Next::Sequence(fs, next)))
+                    }
+                }
+            }
+            Next::Repeat(a, next0) => {
+                let tree = MatchTreeStep::<'a>::from_next(module, next0.clone());
+                let next1 = next.clone();
+                tree.union(MatchTreeStep::<'a>::from_format(module, *a, next1))
+            }
+        }
+    }
+
+    fn from_format(
+        module: &'a FormatModule,
+        f: &'a Format,
+        next: Rc<Next<'a>>,
+    ) -> MatchTreeStep<'a> {
+        match f {
+            Format::ItemVar(level) => Self::from_format(module, module.get_format(*level), next),
+            Format::FailWith(_) => Self::reject(),
+            Format::EndOfInput => Self::accept(),
+            Format::Byte(bs) => Self::branch(*bs, next),
+            Format::Variant(_label, f) => Self::from_format(module, f, next.clone()),
+            Format::Union(branches) => {
+                let mut tree = Self::reject();
+                for f in branches {
+                    tree = tree.union(Self::from_format(module, f, next.clone()));
+                }
+                tree
+            }
+            Format::Seq(fields) | Format::Tuple(fields) => {
+                Self::from_sequential(module, fields, next)
+            }
+            Format::Repeat(a) => {
+                let tree = Self::from_next(module, next.clone());
+                tree.union(Self::from_format(
+                    module,
+                    a,
+                    Rc::new(Next::Repeat(a, next.clone())),
+                ))
+            }
+            Format::Maybe(_expr, a) => {
+                let tree_some = Self::from_format(module, a, next.clone());
+                let tree_none = Self::from_next(module, next);
+                tree_some.union(tree_none)
+            }
+            Format::Compute(_expr) => Self::from_next(module, next),
+            Format::RecVar(_) => {
+                // TODO - implement proper logic for mut-rec/auto-rec references
+                Self::accept()
+            }
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct MatchTreeLevel<'a> {
+    accept: Option<usize>,
+    branches: Vec<(ByteSet, LevelBranch<'a>)>,
+}
+
+impl<'a> MatchTreeLevel<'a> {
+    /// Constructs a `MatchTreeLevel` that unconditionally rejects all inputs without branching.
+    fn reject() -> MatchTreeLevel<'a> {
+        MatchTreeLevel {
+            accept: None,
+            branches: vec![],
+        }
+    }
+
+    /// Attempts to modify `self` such that `index` is marked as the unique index of the accepting format.
+    ///
+    /// Returns `Err(())` if a different index was already marked as accepting, and `Ok(())` otherwise.
+    fn merge_accept(&mut self, index: usize) -> Result<(), ()> {
+        match self.accept {
+            None => {
+                self.accept = Some(index);
+                Ok(())
+            }
+            Some(i) if i == index => Ok(()),
+            Some(_) => Err(()),
+        }
+    }
+
+    /// Adds a new branch to `self` using a predicate byte-set and its associated follow-set,
+    fn merge_branch(&mut self, index: usize, mut bs: ByteSet, next: Rc<Next<'a>>) {
+        let mut new_branches = Vec::new();
+        // For each bs0, nexts in the extant branches of `self`:
+        for (bs0, nexts) in self.branches.iter_mut() {
+            let common = bs0.intersection(&bs);
+            // If bs and bs0 are not disjoint:
+            if !common.is_empty() {
+                let orig = bs0.difference(&bs);
+                if !orig.is_empty() {
+                    // 1. Enqueue a branch predicated on `bs0 - bs` with an inherited follow-set
+                    new_branches.push((orig, nexts.clone()));
+                }
+                // 2. Leave behind a branch predicated on `bs0 & bs`
+                *bs0 = common;
+                // 2a. Add the `next` parameter to the follow-set of the existing branch we modified in-place
+                nexts.insert((index, next.clone()));
+                // 3. Remove all bytes from `bs` that are now covered by the branch we modified in-place
+                bs = bs.difference(bs0);
+            }
+        }
+        // If any bytes of bs were completely unique among all extant branches:
+        if !bs.is_empty() {
+            // 1. Create a novel branch with the follow-set implied by the `next` parameter
+            let mut nexts = HashSet::new();
+            nexts.insert((index, next.clone()));
+            self.branches.push((bs, nexts));
+        }
+        // Append all enqueued branches from the iteration above
+        self.branches.append(&mut new_branches);
+    }
+
+    /// Extends the set of choice-points and follow-sets of `self` with a provided [`MatchTreeStep`].
+    fn merge_step(
+        mut self,
+        index: usize,
+        step: MatchTreeStep<'a>,
+    ) -> Result<MatchTreeLevel<'a>, ()> {
+        if step.accept {
+            self.merge_accept(index)?;
+        }
+        for (bs, next) in step.branches {
+            self.merge_branch(index, bs, next);
+        }
+        Ok(self)
+    }
+
+    /// Attempt to construct and return a `MatchTree` that unconditionally accepts
+    /// the same, common format-index as all elements of the set `nexts`.
+    ///
+    /// If `nexts` is empty, the `MatchTree` returned will instead reject all input
+    ///
+    /// If `nexts` contains multiple associated indices, returns `None`
+    fn accepts(nexts: &LevelBranch<'a>) -> Option<MatchTree> {
+        let mut tree = Self::reject();
+        for (i, _next) in nexts.iter() {
+            tree.merge_accept(*i).ok()?;
+        }
+        Some(MatchTree {
+            accept: tree.accept,
+            branches: vec![],
+        })
+    }
+
+    /// Attempts to accumulate a `MatchTree` recursively up to an overall depth of `depth` layers,
+    /// with the immediate layer constructed based on a bundle of indexed choice-points ([`LevelBranch`]).
+    ///
+    /// If the depth limit has been reached without a decisive choice of which index to accept, returns None.
+    ///
+    /// Otherwise, returns a `MatchTree` that is guaranteed to decide on a unique branch for
+    /// all input within at most `depth` bytes of lookahead.
+    fn grow(module: &'a FormatModule, nexts: LevelBranch<'a>, depth: usize) -> Option<MatchTree> {
+        if let Some(tree) = Self::accepts(&nexts) {
+            Some(tree)
+        } else if depth > 0 {
+            let mut tree = Self::reject();
+            let mut tmp = Vec::from_iter(nexts);
+            tmp.sort_by_key(|(ix, _)| *ix);
+            for (i, next) in tmp.into_iter() {
+                let subtree = MatchTreeStep::from_next(module, next);
+                tree = tree.merge_step(i, subtree).ok()?;
+            }
+            let mut branches = Vec::new();
+            for (bs, nexts) in tree.branches {
+                let t = Self::grow(module, nexts, depth - 1)?;
+                branches.push((bs, t));
+            }
+            Some(MatchTree {
+                accept: tree.accept,
+                branches,
+            })
+        } else {
+            None
+        }
+    }
+}
+
+type LevelBranch<'a> = HashSet<(usize, Rc<Next<'a>>)>;
+
+#[derive(PartialEq, Eq, Hash, Debug)]
+pub(crate) enum Next<'a> {
+    Empty,
+    Union(Rc<Next<'a>>, Rc<Next<'a>>),
+    Cat(&'a Format, Rc<Next<'a>>),
+    Sequence(&'a [Format], Rc<Next<'a>>),
+    Repeat(&'a Format, Rc<Next<'a>>),
+}
+
+#[derive(Debug, Clone)]
+pub struct MatchTree {
+    accept: Option<usize>,
+    branches: Vec<(ByteSet, MatchTree)>,
+}
+
+impl MatchTree {
+    /// Returns the accepting index associated with the input-sequence starting from the current offset of `input`,
+    /// looking ahead as many bytes as necessary until a definitive index is found or the lookahead limit is reached.
+    ///
+    /// Returns `None` if not enough lookahead remains to disambiguate multiple candidate indices.
+    pub(crate) fn matches(&self, input: ReadCtxt<'_>) -> Option<usize> {
+        match input.read_byte() {
+            None => self.accept,
+            Some((b, input)) => {
+                for (bs, s) in &self.branches {
+                    if bs.contains(b) {
+                        return s.matches(input);
+                    }
+                }
+                self.accept
+            }
+        }
+    }
+
+    /// Constructs a new `MatchTreeLevel` from an alternation of branches and a follow-set of partially decomposed formats,
+    /// to within a fixed but externally opaque lookahead-depth.
+    ///
+    /// A `FormatModule` is also accepted to contextualize any contextually dependent formats, e.g. [`Format::ItemVar`]
+    pub(crate) fn build(
+        module: &FormatModule,
+        branches: &[Format],
+        next: Rc<Next<'_>>,
+    ) -> Option<MatchTree> {
+        let mut nexts = HashSet::new();
+        for (i, f) in branches.iter().enumerate() {
+            nexts.insert((i, Rc::new(Next::Cat(f, next.clone()))));
+        }
+        const MAX_DEPTH: usize = 80;
+        MatchTreeLevel::grow(module, nexts, MAX_DEPTH)
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 673e6c57..44d2ae35 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -30,7 +30,7 @@ pub mod prelude;
 pub mod read;
 
 mod typecheck;
-pub use typecheck::{typecheck, TCError, TCResult};
+pub use typecheck::{typecheck, IntWidth, TCError, TCResult};
 
 pub type Label = std::borrow::Cow<'static, str>;
 

From 711bdb6efb8d5ad3e33928722a81a12195958ed7 Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Thu, 22 May 2025 18:43:39 +1000
Subject: [PATCH 04/15] Add in missing Decoder::parse branch-logic for
 doodle-rec

---
 experiments/doodle-rec/src/decoder.rs | 81 +++++++++++++++++++++++----
 src/codegen/mod.rs                    |  5 +-
 src/decoder.rs                        |  4 +-
 3 files changed, 76 insertions(+), 14 deletions(-)

diff --git a/experiments/doodle-rec/src/decoder.rs b/experiments/doodle-rec/src/decoder.rs
index 99920f4e..32092195 100644
--- a/experiments/doodle-rec/src/decoder.rs
+++ b/experiments/doodle-rec/src/decoder.rs
@@ -88,7 +88,7 @@ pub enum Value {
     // Shape-based values
     Tuple(Vec<Value>),
     Seq(Vec<Value>),
-    LiftOption(Option<Box<Value>>),
+    Option(Option<Box<Value>>),
     Variant(Label, Box<Value>),
 }
 
@@ -102,6 +102,13 @@ impl Value {
             _ => panic!("value is not a number: {self:?}"),
         }
     }
+
+    fn unwrap_bool(&self) -> bool {
+        match self {
+            Value::Bool(b) => *b,
+            _ => panic!("value is not a bool"),
+        }
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -359,8 +366,8 @@ impl Expr {
             },
             Expr::Seq(exprs) => Value::Seq(exprs.iter().map(Expr::eval).collect()),
             Expr::Tuple(exprs) => Value::Tuple(exprs.iter().map(Expr::eval).collect()),
-            Expr::LiftOption(None) => Value::LiftOption(None),
-            Expr::LiftOption(Some(expr)) => Value::LiftOption(Some(Box::new(expr.eval()))),
+            Expr::LiftOption(None) => Value::Option(None),
+            Expr::LiftOption(Some(expr)) => Value::Option(Some(Box::new(expr.eval()))),
             Expr::Variant(lab, expr) => Value::Variant(lab.clone(), Box::new(expr.eval())),
             Expr::IntRel(rel, lhs, rhs) => {
                 let lhs = lhs.eval();
@@ -511,14 +518,66 @@ impl Decoder {
                 }
             }
             Decoder::Call(ix) => program.decoders[*ix].0.parse(program, input),
-            Decoder::CallRec(_, _) => todo!(),
-            Decoder::Compute(expr) => todo!(),
-            Decoder::Variant(cow, decoder) => todo!(),
-            Decoder::Branch(match_tree, decoders) => todo!(),
-            Decoder::While(match_tree, decoder) => todo!(),
-            Decoder::Seq(decoders) => todo!(),
-            Decoder::Tuple(decoders) => todo!(),
-            Decoder::Maybe(expr, decoder) => todo!(),
+            Decoder::CallRec(level, _) => program.decoders[*level].0.parse(program, input),
+            Decoder::Compute(expr) => {
+                let v = expr.eval();
+                Ok((v, input))
+            }
+            Decoder::Variant(lab, da) => {
+                let (v, input) = da.parse(program, input)?;
+                Ok((Value::Variant(lab.clone(), Box::new(v)), input))
+            }
+            Decoder::Branch(tree, branches) => {
+                let index = tree.matches(input).ok_or(DecodeError::NoValidBranch {
+                    offset: input.offset,
+                })?;
+                let d = &branches[index];
+                // let (v, input) = d.parse(program, input)?;
+                // Ok(Value::Branch(index, Box::new(v)), input))
+                d.parse(program, input)
+            }
+            Decoder::Seq(decs) => {
+                let mut input = input;
+                let mut v = Vec::with_capacity(decs.len());
+                for d in decs {
+                    let (va, next_input) = d.parse(program, input)?;
+                    input = next_input;
+                    v.push(va);
+                }
+                Ok((Value::Seq(v), input))
+            }
+            Decoder::Tuple(decs) => {
+                let mut input = input;
+                let mut v = Vec::with_capacity(decs.len());
+                for d in decs {
+                    let (va, next_input) = d.parse(program, input)?;
+                    input = next_input;
+                    v.push(va);
+                }
+                Ok((Value::Tuple(v), input))
+            }
+            Decoder::While(tree, a) => {
+                let mut input = input;
+                let mut v = Vec::new();
+                while tree.matches(input).ok_or(DecodeError::NoValidBranch {
+                    offset: input.offset,
+                })? == 0
+                {
+                    let (va, next_input) = a.parse(program, input)?;
+                    input = next_input;
+                    v.push(va);
+                }
+                Ok((Value::Seq(v), input))
+            }
+            Decoder::Maybe(expr, a) => {
+                let is_present = expr.eval().unwrap_bool();
+                if is_present {
+                    let (v, input) = a.parse(program, input)?;
+                    Ok((Value::Option(Some(Box::new(v))), input))
+                } else {
+                    Ok((Value::Option(None), input))
+                }
+            }
         }
     }
 }
diff --git a/src/codegen/mod.rs b/src/codegen/mod.rs
index a81c0c17..96857b78 100644
--- a/src/codegen/mod.rs
+++ b/src/codegen/mod.rs
@@ -5114,7 +5114,10 @@ mod tests {
                 "{}",
                 lambda
                     .apply_pair(
-                        RustExpr::Owned(OwnedRustExpr { expr: Box::new(RustExpr::local("acc")), kind: OwnedKind::Cloned }),
+                        RustExpr::Owned(OwnedRustExpr {
+                            expr: Box::new(RustExpr::local("acc")),
+                            kind: OwnedKind::Cloned
+                        }),
                         RustExpr::local("seq"),
                         ExprInfo::default()
                     )
diff --git a/src/decoder.rs b/src/decoder.rs
index 7a03541b..746e47cb 100644
--- a/src/decoder.rs
+++ b/src/decoder.rs
@@ -1460,7 +1460,7 @@ impl Decoder {
                 for f in fields {
                     let (vf, next_input) = f.parse(program, scope, input)?;
                     input = next_input;
-                    v.push(vf.clone());
+                    v.push(vf);
                 }
                 Ok((Value::Tuple(v), input))
             }
@@ -1470,7 +1470,7 @@ impl Decoder {
                 for d in decs {
                     let (vf, next_input) = d.parse(program, scope, input)?;
                     input = next_input;
-                    v.push(vf.clone());
+                    v.push(vf);
                 }
                 Ok((Value::Seq(SeqKind::Strict(v)), input))
             }

From d8de30e91addb95b4b1713f8316cc91f4a6a021d Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Thu, 22 May 2025 19:06:06 +1000
Subject: [PATCH 05/15] Implement placeholder logic for compiling CallRec, test

---
 experiments/doodle-rec/src/decoder.rs | 76 +++++++++++++++++++++++----
 1 file changed, 65 insertions(+), 11 deletions(-)

diff --git a/experiments/doodle-rec/src/decoder.rs b/experiments/doodle-rec/src/decoder.rs
index 32092195..1ddd046d 100644
--- a/experiments/doodle-rec/src/decoder.rs
+++ b/experiments/doodle-rec/src/decoder.rs
@@ -13,7 +13,7 @@ use anyhow::{Result as AResult, anyhow};
 use doodle::{IntWidth, byte_set::ByteSet, read::ReadCtxt};
 
 #[derive(Debug, Clone, Copy, Default)]
-enum RecurseCtx<'a> {
+pub enum RecurseCtx<'a> {
     #[default]
     NonRec,
     Recurse {
@@ -24,7 +24,7 @@ enum RecurseCtx<'a> {
 }
 
 impl<'a> RecurseCtx<'a> {
-    const fn is_recursive(&self) -> bool {
+    pub const fn is_recursive(&self) -> bool {
         matches!(self, RecurseCtx::Recurse { .. })
     }
 
@@ -36,7 +36,7 @@ impl<'a> RecurseCtx<'a> {
     }
 
     /// Returns `(new_ctx, is_auto)`
-    fn enter(&self, ix: RecId) -> (Self, Ordering) {
+    pub fn enter(&self, ix: RecId) -> (Self, Ordering) {
         match self {
             RecurseCtx::NonRec => panic!("cannot recurse into non-recursive context"),
             RecurseCtx::Recurse {
@@ -50,13 +50,13 @@ impl<'a> RecurseCtx<'a> {
                     span: *span,
                     batch,
                 };
-                (ret, entry_id.cmp(&ix))
+                (ret, ix.cmp(entry_id))
             }
         }
     }
 
     /// Returns the global format-level of the closest entry-point
-    fn get_level(&self) -> Option<usize> {
+    pub fn get_level(&self) -> Option<usize> {
         match self {
             RecurseCtx::NonRec => None,
             RecurseCtx::Recurse {
@@ -65,7 +65,7 @@ impl<'a> RecurseCtx<'a> {
         }
     }
 
-    fn get_format(&self) -> Option<&'a Format> {
+    pub fn get_format(&self) -> Option<&'a Format> {
         match self {
             RecurseCtx::NonRec => None,
             RecurseCtx::Recurse {
@@ -163,7 +163,17 @@ impl<'a> Compiler<'a> {
         let t = format.infer_type(&mut visited, module, batch)?;
         compiler.queue_compile(t, format, Rc::new(Next::Empty), batch);
         while let Some((f, next, n, batch)) = compiler.compile_queue.pop() {
-            let d = compiler.compile_format(f, next, ctx)?;
+            let f_ctx = match batch {
+                Some(span) => {
+                    RecurseCtx::Recurse {
+                        span,
+                        batch: &module.decls[span.start..=span.end],
+                        entry_id: n - span.start,
+                    }
+                }
+                None => RecurseCtx::NonRec,
+            };
+            let d = compiler.compile_format(f, next, f_ctx)?;
             compiler.program.decoders[n].0 = d;
         }
         Ok(compiler.program)
@@ -224,7 +234,7 @@ impl<'a> Compiler<'a> {
             Format::RecVar(batch_ix) => {
                 let (new_ctx, cmp) = ctx.enter(*batch_ix);
                 let level = new_ctx.get_level().unwrap();
-                // FIXME - figure out the proper logic
+                // FIXME - figure out the proper logic - so far we are cookie-cutter copying the same logic between each case
                 let n = match cmp {
                     Ordering::Equal => {
                         if let Some(n) = self.decoder_map.get(&(level, next.clone())) {
@@ -237,9 +247,28 @@ impl<'a> Compiler<'a> {
                             n
                         }
                     }
-                    // ???
-                    Ordering::Less => todo!(), // recurse into earlier decoder (which should already be in the map)
-                    Ordering::Greater => todo!(), // recurse into later decoder (which may not yet be in the map)
+                    Ordering::Less => {
+                        if let Some(n) = self.decoder_map.get(&(level, next.clone())) {
+                            *n
+                        } else {
+                            let format = new_ctx.get_format().unwrap();
+                            let t = self.module.get_format_type(level).clone();
+                            let n = self.queue_compile(t, format, next.clone(), new_ctx.as_span());
+                            self.decoder_map.insert((level, next.clone()), n);
+                            n
+                        }
+                    }
+                    Ordering::Greater => {
+                        if let Some(n) = self.decoder_map.get(&(level, next.clone())) {
+                            *n
+                        } else {
+                            let format = new_ctx.get_format().unwrap();
+                            let t = self.module.get_format_type(level).clone();
+                            let n = self.queue_compile(t, format, next.clone(), new_ctx.as_span());
+                            self.decoder_map.insert((level, next.clone()), n);
+                            n
+                        }
+                    }
                 };
                 Ok(Decoder::CallRec(n, *batch_ix))
             }
@@ -581,3 +610,28 @@ impl Decoder {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn not_actually_recursive() -> AResult<()> {
+        let dead_end = Format::Byte(ByteSet::from_bits([1, 0, 0, 0]));
+        let text = Format::Tuple(vec![
+            Format::Repeat(Box::new(Format::Byte(ByteSet::from(0x01..=0x7f)))),
+            Format::RecVar(0),
+        ]);
+        let mut module = FormatModule::new();
+        let frefs = module.declare_rec_formats(vec![
+            (Label::Borrowed("text.null"), dead_end),
+            (Label::Borrowed("text.cstring"), text),
+        ]);
+        let f = Format::ItemVar(frefs[1].get_level());
+        let program = Compiler::compile_program(&module, &f, RecurseCtx::NonRec)?;
+        let input = ReadCtxt::new(b"hello world\x00");
+        let (value, _) = program.run(input)?;
+        eprintln!("{value:?}");
+        Ok(())
+    }
+}

From c1f7b2575b4f65b004c674a4bbb56ef4a47b5cf9 Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Fri, 23 May 2025 13:48:47 +1000
Subject: [PATCH 06/15] Refactors and tweaks around RecurseCtx and
 compile_format, peano test

---
 experiments/doodle-rec/Cargo.toml       |   2 +
 experiments/doodle-rec/src/decoder.rs   | 175 ++++++++----------------
 experiments/doodle-rec/src/lib.rs       | 116 ++++++++++++++--
 experiments/doodle-rec/src/matchtree.rs | 110 ++++++++++-----
 4 files changed, 241 insertions(+), 162 deletions(-)

diff --git a/experiments/doodle-rec/Cargo.toml b/experiments/doodle-rec/Cargo.toml
index 1e98e2ae..a14a0082 100644
--- a/experiments/doodle-rec/Cargo.toml
+++ b/experiments/doodle-rec/Cargo.toml
@@ -10,3 +10,5 @@ bench = false
 [dependencies]
 anyhow = "1.0.98"
 doodle = { path = "../../" }
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = "1.0.140"
diff --git a/experiments/doodle-rec/src/decoder.rs b/experiments/doodle-rec/src/decoder.rs
index 1ddd046d..59a551b1 100644
--- a/experiments/doodle-rec/src/decoder.rs
+++ b/experiments/doodle-rec/src/decoder.rs
@@ -3,79 +3,15 @@ use std::{
     collections::{HashMap, HashSet},
     rc::Rc,
 };
+use serde::Serialize;
 
 use crate::{
-    Arith, Expr, Format, FormatDecl, FormatId, FormatModule, FormatType, IntRel, Label, RecId,
-    Span, Unary,
-    matchtree::{MatchTree, Next},
+    matchtree::{MatchTree, Next}, Arith, Expr, Format, FormatDecl, FormatId, FormatModule, FormatType, IntRel, Label, RecId, RecurseCtx, Span, Unary
 };
 use anyhow::{Result as AResult, anyhow};
 use doodle::{IntWidth, byte_set::ByteSet, read::ReadCtxt};
 
-#[derive(Debug, Clone, Copy, Default)]
-pub enum RecurseCtx<'a> {
-    #[default]
-    NonRec,
-    Recurse {
-        entry_id: RecId,
-        span: Span<usize>,
-        batch: &'a [FormatDecl],
-    },
-}
-
-impl<'a> RecurseCtx<'a> {
-    pub const fn is_recursive(&self) -> bool {
-        matches!(self, RecurseCtx::Recurse { .. })
-    }
-
-    pub const fn as_span(&self) -> Option<Span<usize>> {
-        match self {
-            RecurseCtx::NonRec => None,
-            RecurseCtx::Recurse { span, .. } => Some(*span),
-        }
-    }
-
-    /// Returns `(new_ctx, is_auto)`
-    pub fn enter(&self, ix: RecId) -> (Self, Ordering) {
-        match self {
-            RecurseCtx::NonRec => panic!("cannot recurse into non-recursive context"),
-            RecurseCtx::Recurse {
-                batch,
-                span,
-                entry_id,
-            } => {
-                assert!(ix < batch.len(), "batch index out of range");
-                let ret = RecurseCtx::Recurse {
-                    entry_id: ix,
-                    span: *span,
-                    batch,
-                };
-                (ret, ix.cmp(entry_id))
-            }
-        }
-    }
-
-    /// Returns the global format-level of the closest entry-point
-    pub fn get_level(&self) -> Option<usize> {
-        match self {
-            RecurseCtx::NonRec => None,
-            RecurseCtx::Recurse {
-                batch, entry_id, ..
-            } => Some(batch[*entry_id].fmt_id),
-        }
-    }
-
-    pub fn get_format(&self) -> Option<&'a Format> {
-        match self {
-            RecurseCtx::NonRec => None,
-            RecurseCtx::Recurse {
-                batch, entry_id, ..
-            } => Some(&batch[*entry_id].format),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize)]
 pub enum Value {
     // Primitive values
     U8(u8),
@@ -192,6 +128,27 @@ impl<'a> Compiler<'a> {
         n
     }
 
+    fn queue_compile_batch(
+        &mut self,
+        decls: &'a [FormatDecl],
+        which_next: RecId,
+        next: Rc<Next<'a>>,
+        span: Span<FormatId>,
+    ) -> usize {
+        let n = self.program.decoders.len();
+        for (ix, d) in decls.into_iter().enumerate() {
+            let t = d.solve_type(self.module).unwrap().clone();
+            self.program.decoders.push((Decoder::FAIL, t));
+            let next = if ix == which_next {
+                next.clone()
+            } else {
+                Rc::new(Next::Empty)
+            };
+            self.compile_queue.push((&d.format, next, n + ix, Some(span)));
+        }
+        n + which_next
+    }
+
     pub fn compile_one(format: &Format) -> AResult<Decoder> {
         let module = FormatModule::new();
         let mut compiler = Compiler::new(&module);
@@ -208,7 +165,7 @@ impl<'a> Compiler<'a> {
         match format {
             Format::ItemVar(level) => {
                 let f = self.module.get_format(*level);
-                let next = if f.depends_on_next(self.module) {
+                let next = if f.depends_on_next(self.module, ctx) {
                     next
                 } else {
                     Rc::new(Next::Empty)
@@ -217,60 +174,25 @@ impl<'a> Compiler<'a> {
                     *n
                 } else {
                     let t = self.module.get_format_type(*level).clone();
-                    let ctx = match self.module.get_batch(*level) {
-                        Some(span) => RecurseCtx::Recurse {
-                            entry_id: level - span.start,
-                            span,
-                            batch: &self.module.decls[span.start..=span.end],
-                        },
-                        None => RecurseCtx::NonRec,
+                    let n = match self.module.get_batch(*level) {
+                        Some(span) => {
+                            let batch = &self.module.decls[span.start..=span.end];
+                            self.queue_compile_batch(batch, level - span.start, next.clone(), span)
+                        }
+                        None => {
+                            self.queue_compile(t, f, next.clone(), None)
+                        }
                     };
-                    let n = self.queue_compile(t, f, next.clone(), ctx.as_span());
                     self.decoder_map.insert((*level, next.clone()), n);
                     n
                 };
                 Ok(Decoder::Call(n))
             }
             Format::RecVar(batch_ix) => {
-                let (new_ctx, cmp) = ctx.enter(*batch_ix);
+                let (new_ctx, _) = ctx.enter(*batch_ix);
                 let level = new_ctx.get_level().unwrap();
-                // FIXME - figure out the proper logic - so far we are cookie-cutter copying the same logic between each case
-                let n = match cmp {
-                    Ordering::Equal => {
-                        if let Some(n) = self.decoder_map.get(&(level, next.clone())) {
-                            *n
-                        } else {
-                            let format = new_ctx.get_format().unwrap();
-                            let t = self.module.get_format_type(level).clone();
-                            let n = self.queue_compile(t, format, next.clone(), new_ctx.as_span());
-                            self.decoder_map.insert((level, next.clone()), n);
-                            n
-                        }
-                    }
-                    Ordering::Less => {
-                        if let Some(n) = self.decoder_map.get(&(level, next.clone())) {
-                            *n
-                        } else {
-                            let format = new_ctx.get_format().unwrap();
-                            let t = self.module.get_format_type(level).clone();
-                            let n = self.queue_compile(t, format, next.clone(), new_ctx.as_span());
-                            self.decoder_map.insert((level, next.clone()), n);
-                            n
-                        }
-                    }
-                    Ordering::Greater => {
-                        if let Some(n) = self.decoder_map.get(&(level, next.clone())) {
-                            *n
-                        } else {
-                            let format = new_ctx.get_format().unwrap();
-                            let t = self.module.get_format_type(level).clone();
-                            let n = self.queue_compile(t, format, next.clone(), new_ctx.as_span());
-                            self.decoder_map.insert((level, next.clone()), n);
-                            n
-                        }
-                    }
-                };
-                Ok(Decoder::CallRec(n, *batch_ix))
+                // REVIEW - do we need to do any work here?
+                Ok(Decoder::CallRec(level, *batch_ix))
             }
             Format::FailWith(msg) => Ok(Decoder::FailWith(msg.clone())),
             Format::EndOfInput => Ok(Decoder::EndOfInput),
@@ -285,7 +207,7 @@ impl<'a> Compiler<'a> {
                 for f in branches {
                     ds.push(self.compile_format(f, next.clone(), ctx)?);
                 }
-                if let Some(tree) = MatchTree::build(self.module, branches, next) {
+                if let Some(tree) = MatchTree::build(self.module, branches, next, ctx) {
                     Ok(Decoder::Branch(tree, ds))
                 } else {
                     Err(anyhow!("cannot build match tree for {:?}", format))
@@ -319,7 +241,7 @@ impl<'a> Compiler<'a> {
                 let astar = Format::Repeat(a.clone());
                 let fa = Format::Tuple(vec![(**a).clone(), astar]);
                 let fb = Format::EMPTY;
-                if let Some(tree) = MatchTree::build(self.module, &[fa, fb], next) {
+                if let Some(tree) = MatchTree::build(self.module, &[fa, fb], next, ctx) {
                     Ok(Decoder::While(tree, Box::new(da)))
                 } else {
                     Err(anyhow!("cannot build match tree for {:?}", format))
@@ -614,6 +536,7 @@ impl Decoder {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use serde_json::ser::to_string_pretty;
 
     #[test]
     fn not_actually_recursive() -> AResult<()> {
@@ -627,11 +550,29 @@ mod tests {
             (Label::Borrowed("text.null"), dead_end),
             (Label::Borrowed("text.cstring"), text),
         ]);
-        let f = Format::ItemVar(frefs[1].get_level());
+        let f = frefs[1].call();
         let program = Compiler::compile_program(&module, &f, RecurseCtx::NonRec)?;
         let input = ReadCtxt::new(b"hello world\x00");
         let (value, _) = program.run(input)?;
         eprintln!("{value:?}");
         Ok(())
     }
+
+    #[test]
+    fn auto_recursive() -> AResult<()> {
+        let peano = Format::Union(vec![
+            Format::Variant(Label::Borrowed("peanoZ"), Box::new(Format::Byte(ByteSet::from([b'Z'])))),
+            Format::Variant(Label::Borrowed("peanoS"), Box::new(Format::Tuple(vec![Format::Byte(ByteSet::from([b'S'])), Format::RecVar(0)]))),
+        ]);
+        let mut module = FormatModule::new();
+        let frefs = module.declare_rec_formats(vec![
+            (Label::Borrowed("test.peano"), peano),
+        ]);
+        let f = Format::Tuple(vec![frefs[0].call(), Format::EndOfInput]);
+        let program = Compiler::compile_program(&module, &f, RecurseCtx::NonRec)?;
+        let input = ReadCtxt::new(b"SSSSZ");
+        let (value, _) = program.run(input)?;
+        eprintln!("{}", to_string_pretty(&value).unwrap());
+        Ok(())
+    }
 }
diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index 672c68be..d46f3113 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -5,11 +5,7 @@ pub(crate) use matchtree::{MatchTree, Next};
 use anyhow::{Result as AResult, anyhow};
 use doodle::{bounds::Bounds, byte_set::ByteSet};
 use std::{
-    borrow::Cow,
-    cell::OnceCell,
-    collections::{BTreeMap, HashSet},
-    ops::{Add as _, RangeInclusive},
-    rc::Rc,
+    borrow::Cow, cell::OnceCell, cmp::Ordering, collections::{BTreeMap, HashSet}, ops::{Add as _, RangeInclusive}, rc::Rc
 };
 
 pub type Label = Cow<'static, str>;
@@ -20,6 +16,74 @@ pub type FormatId = usize;
 /// Local index into a Batch of formats (e.g. 0 would be 'self' in a singleton-batch)
 pub type RecId = usize;
 
+
+#[derive(Debug, Clone, Copy, Default)]
+pub enum RecurseCtx<'a> {
+    #[default]
+    NonRec,
+    Recurse {
+        entry_id: RecId,
+        span: Span<usize>,
+        batch: &'a [FormatDecl],
+    },
+}
+
+impl<'a> RecurseCtx<'a> {
+    pub const fn is_recursive(&self) -> bool {
+        matches!(self, RecurseCtx::Recurse { .. })
+    }
+
+    pub const fn as_span(&self) -> Option<Span<usize>> {
+        match self {
+            RecurseCtx::NonRec => None,
+            RecurseCtx::Recurse { span, .. } => Some(*span),
+        }
+    }
+
+    /// Returns `(new_ctx, is_auto)`
+    pub fn enter(&self, ix: RecId) -> (Self, Ordering) {
+        match self {
+            RecurseCtx::NonRec => panic!("cannot recurse into non-recursive context"),
+            RecurseCtx::Recurse {
+                batch,
+                span,
+                entry_id,
+            } => {
+                assert!(ix < batch.len(), "batch index out of range");
+                let ret = RecurseCtx::Recurse {
+                    entry_id: ix,
+                    span: *span,
+                    batch,
+                };
+                (ret, ix.cmp(entry_id))
+            }
+        }
+    }
+
+    pub fn convert_rec_var(&self, ix: RecId) -> FormatId {
+        self.as_span().unwrap().index(ix)
+    }
+
+    /// Returns the global format-level of the closest entry-point
+    pub fn get_level(&self) -> Option<usize> {
+        match self {
+            RecurseCtx::NonRec => None,
+            RecurseCtx::Recurse {
+                span, entry_id, ..
+            } => Some(span.index(*entry_id)),
+        }
+    }
+
+    pub fn get_format(&self) -> Option<&'a Format> {
+        match self {
+            RecurseCtx::NonRec => None,
+            RecurseCtx::Recurse {
+                batch, entry_id, ..
+            } => Some(&batch[*entry_id].format),
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub struct FormatRef(FormatId);
 
@@ -45,6 +109,13 @@ impl<Idx> Span<Idx> {
     }
 }
 
+impl Span<usize> {
+    pub fn index(self, ix: usize) -> usize {
+        assert!(ix < (self.end - self.start + 1) as usize);
+        self.start + ix
+    }
+}
+
 impl<Idx: Copy> From<RangeInclusive<Idx>> for Span<Idx> {
     fn from(value: RangeInclusive<Idx>) -> Self {
         Self {
@@ -310,9 +381,12 @@ impl Format {
         }
     }
 
-    fn depends_on_next(&self, module: &FormatModule) -> bool {
+    fn depends_on_next<'a>(&self, module: &'a FormatModule, ctx: RecurseCtx<'a>) -> bool {
         match self {
-            Format::ItemVar(level) => module.get_format(*level).depends_on_next(module),
+            Format::ItemVar(level) => {
+                let ctx = module.get_ctx(*level);
+                module.get_format(*level).depends_on_next(module, ctx)
+            }
             Format::FailWith(..) => false,
             Format::EndOfInput => false,
             Format::Byte(..) => false,
@@ -320,27 +394,28 @@ impl Format {
             Format::RecVar(..) => {
                 // REVIEW - are there any recursive formats that *don't* depend on next?
                 // FIXME[epic=hardcoded] - this is a placeholder for future improvements to classification logic
-                true
+                false
+                // true
             }
-            Format::Variant(_, f) => f.depends_on_next(module),
-            Format::Union(branches) => Format::union_depends_on_next(branches, module),
+            Format::Variant(_, f) => f.depends_on_next(module, ctx),
+            Format::Union(branches) => Format::union_depends_on_next(branches, module, ctx),
             Format::Repeat(..) => true,
             Format::Seq(formats) | Format::Tuple(formats) => {
-                formats.iter().any(|f| f.depends_on_next(module))
+                formats.iter().any(|f| f.depends_on_next(module, ctx))
             }
             Format::Maybe(..) => true,
         }
     }
 
-    fn union_depends_on_next(branches: &[Format], module: &FormatModule) -> bool {
+    fn union_depends_on_next<'a>(branches: &'a [Format], module: &'a FormatModule, ctx: RecurseCtx<'a>) -> bool {
         let mut fs = Vec::with_capacity(branches.len());
         for f in branches {
-            if f.depends_on_next(module) {
+            if f.depends_on_next(module, ctx) {
                 return true;
             }
             fs.push(f.clone());
         }
-        MatchTree::build(module, &fs, Rc::new(Next::Empty)).is_none()
+        MatchTree::build(module, &fs, Rc::new(Next::Empty), ctx).is_none()
     }
 
     fn is_nullable(&self, module: &FormatModule) -> bool {
@@ -554,6 +629,19 @@ impl FormatModule {
         }
     }
 
+    pub fn get_ctx<'a>(&'a self, level: usize) -> RecurseCtx<'a> {
+        let decl = &self.decls[level];
+        match decl.batch {
+            None => RecurseCtx::NonRec,
+            Some(span) => RecurseCtx::Recurse {
+                span,
+                entry_id: decl.fmt_id,
+                batch: &self.decls[span.start..=span.end],
+            },
+        }
+    }
+
+
     pub fn declare_rec_formats(&mut self, formats: Vec<(Label, Format)>) -> Vec<FormatRef> {
         let fmt_id = self.decls.len();
         let batch_size = formats.len();
diff --git a/experiments/doodle-rec/src/matchtree.rs b/experiments/doodle-rec/src/matchtree.rs
index 0f830b8f..c5947b51 100644
--- a/experiments/doodle-rec/src/matchtree.rs
+++ b/experiments/doodle-rec/src/matchtree.rs
@@ -1,6 +1,6 @@
 use std::{collections::HashSet, rc::Rc, vec};
 
-use crate::{Format, FormatModule};
+use crate::{Format, FormatModule, RecurseCtx};
 use doodle::{byte_set::ByteSet, read::ReadCtxt};
 
 #[derive(Clone, Debug)]
@@ -66,36 +66,41 @@ impl<'a> MatchTreeStep<'a> {
         module: &'a FormatModule,
         fields: &'a [Format],
         next: Rc<Next<'a>>,
+        ctx: RecurseCtx<'a>
     ) -> MatchTreeStep<'a> {
         match fields.split_first() {
-            None => Self::from_next(module, next),
-            Some((f, fs)) => Self::from_format(module, f, Rc::new(Next::Sequence(fs, next))),
+            None => Self::from_next(module, next, ctx),
+            Some((f, fs)) => Self::from_format(module, f, Rc::new(Next::Sequence(fs, next)), ctx),
         }
     }
 
     /// Constructs a [MatchTreeStep] from a [`Next`]
-    fn from_next(module: &'a FormatModule, next: Rc<Next<'a>>) -> MatchTreeStep<'a> {
+    fn from_next(module: &'a FormatModule, next: Rc<Next<'a>>, ctx: RecurseCtx<'a>) -> MatchTreeStep<'a> {
         match next.as_ref() {
             Next::Empty => Self::accept(),
             Next::Union(next1, next2) => {
-                let tree1 = Self::from_next(module, next1.clone());
-                let tree2 = Self::from_next(module, next2.clone());
+                let tree1 = Self::from_next(module, next1.clone(), ctx);
+                let tree2 = Self::from_next(module, next2.clone(), ctx);
                 tree1.union(tree2)
             }
-            Next::Cat(f, next) => MatchTreeStep::<'a>::from_format(module, *f, next.clone()),
+            Next::Cat(f, next) => MatchTreeStep::<'a>::from_format(module, *f, next.clone(), ctx),
             Next::Sequence(fields, next) => {
                 let next = next.clone();
                 match fields.split_first() {
-                    None => Self::from_next(module, next),
+                    None => Self::from_next(module, next, ctx),
                     Some((f, fs)) => {
-                        Self::from_format(module, f, Rc::new(Next::Sequence(fs, next)))
+                        Self::from_format(module, f, Rc::new(Next::Sequence(fs, next)), ctx)
                     }
                 }
             }
             Next::Repeat(a, next0) => {
-                let tree = MatchTreeStep::<'a>::from_next(module, next0.clone());
+                let tree = MatchTreeStep::<'a>::from_next(module, next0.clone(), ctx);
                 let next1 = next.clone();
-                tree.union(MatchTreeStep::<'a>::from_format(module, *a, next1))
+                tree.union(MatchTreeStep::<'a>::from_format(module, *a, next1, ctx))
+            }
+            Next::DelayRef(_ix) => {
+                // FIXME - figure out how to resolve this
+                Self::accept()
             }
         }
     }
@@ -104,40 +109,46 @@ impl<'a> MatchTreeStep<'a> {
         module: &'a FormatModule,
         f: &'a Format,
         next: Rc<Next<'a>>,
+        ctx: RecurseCtx<'a>,
     ) -> MatchTreeStep<'a> {
         match f {
-            Format::ItemVar(level) => Self::from_format(module, module.get_format(*level), next),
+            Format::ItemVar(level) => {
+                let ctx = module.get_ctx(*level);
+                Self::from_format(module, module.get_format(*level), next, ctx)
+            }
             Format::FailWith(_) => Self::reject(),
             Format::EndOfInput => Self::accept(),
             Format::Byte(bs) => Self::branch(*bs, next),
-            Format::Variant(_label, f) => Self::from_format(module, f, next.clone()),
+            Format::Variant(_label, f) => Self::from_format(module, f, next.clone(), ctx),
             Format::Union(branches) => {
                 let mut tree = Self::reject();
                 for f in branches {
-                    tree = tree.union(Self::from_format(module, f, next.clone()));
+                    tree = tree.union(Self::from_format(module, f, next.clone(), ctx));
                 }
                 tree
             }
             Format::Seq(fields) | Format::Tuple(fields) => {
-                Self::from_sequential(module, fields, next)
+                Self::from_sequential(module, fields, next, ctx)
             }
             Format::Repeat(a) => {
-                let tree = Self::from_next(module, next.clone());
+                let tree = Self::from_next(module, next.clone(), ctx);
                 tree.union(Self::from_format(
                     module,
                     a,
                     Rc::new(Next::Repeat(a, next.clone())),
+                    ctx,
                 ))
             }
             Format::Maybe(_expr, a) => {
-                let tree_some = Self::from_format(module, a, next.clone());
-                let tree_none = Self::from_next(module, next);
+                let tree_some = Self::from_format(module, a, next.clone(), ctx);
+                let tree_none = Self::from_next(module, next, ctx);
                 tree_some.union(tree_none)
             }
-            Format::Compute(_expr) => Self::from_next(module, next),
-            Format::RecVar(_) => {
-                // TODO - implement proper logic for mut-rec/auto-rec references
-                Self::accept()
+            Format::Compute(_expr) => Self::from_next(module, next, ctx),
+            Format::RecVar(rec_ix) => {
+                // FIXME - we discard the original `next` argument here
+                let next = Rc::new(Next::DelayRef(ctx.convert_rec_var(*rec_ix)));
+                Self::from_next(module, next, ctx)
             }
         }
     }
@@ -243,7 +254,7 @@ impl<'a> MatchTreeLevel<'a> {
     ///
     /// Otherwise, returns a `MatchTree` that is guaranteed to decide on a unique branch for
     /// all input within at most `depth` bytes of lookahead.
-    fn grow(module: &'a FormatModule, nexts: LevelBranch<'a>, depth: usize) -> Option<MatchTree> {
+    fn grow(module: &'a FormatModule, nexts: LevelBranch<'a>, depth: usize, ctx: RecurseCtx<'a>) -> Option<MatchTree> {
         if let Some(tree) = Self::accepts(&nexts) {
             Some(tree)
         } else if depth > 0 {
@@ -251,12 +262,12 @@ impl<'a> MatchTreeLevel<'a> {
             let mut tmp = Vec::from_iter(nexts);
             tmp.sort_by_key(|(ix, _)| *ix);
             for (i, next) in tmp.into_iter() {
-                let subtree = MatchTreeStep::from_next(module, next);
+                let subtree = MatchTreeStep::from_next(module, next, ctx);
                 tree = tree.merge_step(i, subtree).ok()?;
             }
             let mut branches = Vec::new();
             for (bs, nexts) in tree.branches {
-                let t = Self::grow(module, nexts, depth - 1)?;
+                let t = Self::grow(module, nexts, depth - 1, ctx)?;
                 branches.push((bs, t));
             }
             Some(MatchTree {
@@ -271,15 +282,28 @@ impl<'a> MatchTreeLevel<'a> {
 
 type LevelBranch<'a> = HashSet<(usize, Rc<Next<'a>>)>;
 
-#[derive(PartialEq, Eq, Hash, Debug)]
+#[derive(Eq, Hash, Debug)]
 pub(crate) enum Next<'a> {
     Empty,
+    DelayRef(usize),
     Union(Rc<Next<'a>>, Rc<Next<'a>>),
     Cat(&'a Format, Rc<Next<'a>>),
     Sequence(&'a [Format], Rc<Next<'a>>),
     Repeat(&'a Format, Rc<Next<'a>>),
 }
 
+impl<'a> PartialEq for Next<'a> {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Self::DelayRef(l0), Self::DelayRef(r0)) => l0 == r0,
+            (Self::Union(l0, l1), Self::Union(r0, r1)) => l0 == r0 && l1 == r1,
+            (Self::Cat(l0, l1), Self::Cat(r0, r1)) => l0 == r0 && l1 == r1,
+            (Self::Sequence(l0, l1), Self::Sequence(r0, r1)) => l0 == r0 && l1 == r1,
+            (Self::Repeat(l0, l1), Self::Repeat(r0, r1)) => l0 == r0 && l1 == r1,
+            _ => false,
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct MatchTree {
     accept: Option<usize>,
@@ -309,16 +333,40 @@ impl MatchTree {
     /// to within a fixed but externally opaque lookahead-depth.
     ///
     /// A `FormatModule` is also accepted to contextualize any contextually dependent formats, e.g. [`Format::ItemVar`]
-    pub(crate) fn build(
-        module: &FormatModule,
-        branches: &[Format],
-        next: Rc<Next<'_>>,
+    pub(crate) fn build<'a>(
+        module: &'a FormatModule,
+        branches: &'a [Format],
+        next: Rc<Next<'a>>,
+        ctx: RecurseCtx<'a>,
     ) -> Option<MatchTree> {
         let mut nexts = HashSet::new();
         for (i, f) in branches.iter().enumerate() {
             nexts.insert((i, Rc::new(Next::Cat(f, next.clone()))));
         }
         const MAX_DEPTH: usize = 80;
-        MatchTreeLevel::grow(module, nexts, MAX_DEPTH)
+        MatchTreeLevel::grow(module, nexts, MAX_DEPTH, ctx)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::Label;
+
+    use super::*;
+
+    #[test]
+    fn construct_autorec_next() {
+        let peano = Format::Union(vec![
+            Format::Variant(Label::Borrowed("peanoZ"), Box::new(Format::Byte(ByteSet::from([b'Z'])))),
+            Format::Variant(Label::Borrowed("peanoS"), Box::new(Format::Tuple(vec![Format::Byte(ByteSet::from([b'S'])), Format::RecVar(0)]))),
+        ]);
+        let mut module = FormatModule::new();
+        let frefs = module.declare_rec_formats(vec![
+            (Label::Borrowed("test.peano"), peano),
+        ]);
+        let f = Format::Tuple(vec![Format::ItemVar(frefs[0].get_level()), Format::EndOfInput]);
+        let ctx = RecurseCtx::NonRec;
+        let tree = MatchTreeStep::from_format(&module, &f, Rc::new(Next::Empty), ctx);
+        eprintln!("{tree:?}")
     }
 }

From 137000b3ede9464d56dc74a1ff38d67263b59822 Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Fri, 23 May 2025 17:32:25 +1000
Subject: [PATCH 07/15] Attempt at json-lite parser (but it stack-overflows)

---
 Cargo.lock                              | 102 ++++++++++++------------
 experiments/doodle-rec/Cargo.toml       |   1 +
 experiments/doodle-rec/src/decoder.rs   |   1 -
 experiments/doodle-rec/src/helper.rs    |  30 +++++++
 experiments/doodle-rec/src/lib.rs       |   5 +-
 experiments/doodle-rec/src/main.rs      |  69 ++++++++++++++--
 experiments/doodle-rec/src/matchtree.rs |  20 +----
 experiments/doodle-rec/test1.json       |   1 +
 experiments/doodle-rec/test2.json       |   1 +
 9 files changed, 154 insertions(+), 76 deletions(-)
 create mode 100644 experiments/doodle-rec/src/helper.rs
 create mode 100644 experiments/doodle-rec/test1.json
 create mode 100644 experiments/doodle-rec/test2.json

diff --git a/Cargo.lock b/Cargo.lock
index 3d17e7fb..19254305 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -19,42 +19,51 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
 
 [[package]]
 name = "anstream"
-version = "0.2.6"
+version = "0.6.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
 dependencies = [
  "anstyle",
  "anstyle-parse",
+ "anstyle-query",
  "anstyle-wincon",
- "concolor-override",
- "concolor-query",
- "is-terminal",
+ "colorchoice",
+ "is_terminal_polyfill",
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle"
-version = "0.3.5"
+version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.1.1"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
 dependencies = [
  "utf8parse",
 ]
 
+[[package]]
+name = "anstyle-query"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "anstyle-wincon"
-version = "0.2.0"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
 dependencies = [
  "anstyle",
- "windows-sys 0.45.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -161,33 +170,31 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.2.1"
+version = "4.5.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3"
+checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000"
 dependencies = [
  "clap_builder",
  "clap_derive",
- "once_cell",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.2.1"
+version = "4.5.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f"
+checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120"
 dependencies = [
  "anstream",
  "anstyle",
- "bitflags",
  "clap_lex",
- "strsim 0.10.0",
+ "strsim",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "4.2.0"
+version = "4.5.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4"
+checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -197,24 +204,15 @@ dependencies = [
 
 [[package]]
 name = "clap_lex"
-version = "0.4.1"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
 
 [[package]]
-name = "concolor-override"
-version = "1.0.0"
+name = "colorchoice"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f"
-
-[[package]]
-name = "concolor-query"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf"
-dependencies = [
- "windows-sys 0.45.0",
-]
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
 
 [[package]]
 name = "cons-rs"
@@ -309,7 +307,7 @@ dependencies = [
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim 0.11.1",
+ "strsim",
  "syn",
 ]
 
@@ -393,7 +391,10 @@ name = "doodle-rec"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "clap",
  "doodle",
+ "serde",
+ "serde_json",
 ]
 
 [[package]]
@@ -560,9 +561,9 @@ dependencies = [
 
 [[package]]
 name = "heck"
-version = "0.4.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hermit-abi"
@@ -608,6 +609,12 @@ dependencies = [
  "windows-sys 0.45.0",
 ]
 
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -927,18 +934,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.159"
+version = "1.0.219"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
+checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.159"
+version = "1.0.219"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
+checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -947,21 +954,16 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.95"
+version = "1.0.140"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744"
+checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
 dependencies = [
  "itoa",
+ "memchr",
  "ryu",
  "serde",
 ]
 
-[[package]]
-name = "strsim"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
-
 [[package]]
 name = "strsim"
 version = "0.11.1"
diff --git a/experiments/doodle-rec/Cargo.toml b/experiments/doodle-rec/Cargo.toml
index a14a0082..c58523c6 100644
--- a/experiments/doodle-rec/Cargo.toml
+++ b/experiments/doodle-rec/Cargo.toml
@@ -9,6 +9,7 @@ bench = false
 
 [dependencies]
 anyhow = "1.0.98"
+clap = "4.5.38"
 doodle = { path = "../../" }
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = "1.0.140"
diff --git a/experiments/doodle-rec/src/decoder.rs b/experiments/doodle-rec/src/decoder.rs
index 59a551b1..3c371e3d 100644
--- a/experiments/doodle-rec/src/decoder.rs
+++ b/experiments/doodle-rec/src/decoder.rs
@@ -1,5 +1,4 @@
 use std::{
-    cmp::Ordering,
     collections::{HashMap, HashSet},
     rc::Rc,
 };
diff --git a/experiments/doodle-rec/src/helper.rs b/experiments/doodle-rec/src/helper.rs
new file mode 100644
index 00000000..0401744f
--- /dev/null
+++ b/experiments/doodle-rec/src/helper.rs
@@ -0,0 +1,30 @@
+use doodle::prelude::ByteSet;
+use crate::{Expr, Format, Label};
+
+pub fn tuple(formats: impl IntoIterator<Item = Format>) -> Format {
+    Format::Tuple(formats.into_iter().collect())
+}
+
+pub fn is_byte(x: u8) -> Format {
+    Format::Byte(ByteSet::from([x]))
+}
+
+pub fn alts<Name: Into<Label>>(branches: impl IntoIterator<Item = (Name, Format)>) -> Format {
+    Format::Union(branches.into_iter().map(|(name, f)| Format::Variant(name.into(), Box::new(f))).collect())
+}
+
+pub fn byte_seq(bytes: impl IntoIterator<Item = u8>) -> Format {
+    Format::Seq(bytes.into_iter().map(is_byte).collect())
+}
+
+pub fn repeat(format: Format) -> Format {
+    Format::Repeat(Box::new(format))
+}
+
+pub fn var(ix: usize) -> Format {
+    Format::RecVar(ix)
+}
+
+pub fn repeat1(format: Format) -> Format {
+    tuple([format.clone(), repeat(format.clone())])
+}
diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index d46f3113..018e9afb 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -1,5 +1,6 @@
 pub mod decoder;
 pub(crate) mod matchtree;
+pub mod helper;
 pub(crate) use matchtree::{MatchTree, Next};
 
 use anyhow::{Result as AResult, anyhow};
@@ -394,8 +395,8 @@ impl Format {
             Format::RecVar(..) => {
                 // REVIEW - are there any recursive formats that *don't* depend on next?
                 // FIXME[epic=hardcoded] - this is a placeholder for future improvements to classification logic
-                false
-                // true
+                // false
+                true
             }
             Format::Variant(_, f) => f.depends_on_next(module, ctx),
             Format::Union(branches) => Format::union_depends_on_next(branches, module, ctx),
diff --git a/experiments/doodle-rec/src/main.rs b/experiments/doodle-rec/src/main.rs
index f97f2469..504ff269 100644
--- a/experiments/doodle-rec/src/main.rs
+++ b/experiments/doodle-rec/src/main.rs
@@ -1,15 +1,70 @@
-use doodle_rec::{Format, FormatModule, FormatRef, Label};
+use std::{io::Read, path::PathBuf};
 
-fn bson(module: &mut FormatModule) -> FormatRef {
-    todo!();
+use doodle::{prelude::ByteSet, read::ReadCtxt};
+use doodle_rec::{decoder::{Compiler, Value}, helper::*, Format, FormatModule, FormatRef, Label, RecurseCtx};
+use clap::Parser;
+
+fn surround(before: u8, after: u8, f: Format) -> Format {
+    Format::Tuple(vec![is_byte(before), f, is_byte(after)])
+}
+
+fn alpha() -> Format {
+    repeat(Format::Byte(ByteSet::union(&ByteSet::from(b'a'..b'z'), &ByteSet::from(b'A'..b'Z'))))
+}
+
+/// Restricted JSON object
+///
+/// Rules
+/// - all keys consist of [A-Za-z]+ without spaces or special characters/numbers
+/// - values cannot be strings
+
+/// - all numbers are natural
+/// - trailing commas are mandatory
+fn json_lite(module: &mut FormatModule) -> FormatRef {
+    let key = surround(b'"', b'"', repeat1(alpha()));
+    let key_prefix = tuple([key, is_byte(b':')]);
+    let value0 = alts([
+        ("JNull", byte_seq(*b"null")),
+        ("JTrue", byte_seq(*b"true")),
+        ("JFalse", byte_seq(*b"false")),
+        ("JNum", repeat1(Format::Byte(ByteSet::from(b'0'..b'9')))),
+        ("JObj", Format::RecVar(2)),
+        ("JArr", Format::RecVar(3)),
+    ]);
+    let kv1 = tuple([key_prefix, var(0)]);
+    let obj2 = surround(b'{', b'}', repeat(tuple([var(1), is_byte(b',')])));
+    let arr3 = surround(b'[', b']', repeat(tuple([is_byte(b','), var(0)])));
+    let decls = module.declare_rec_formats(
+        [(Label::Borrowed("json.value"), value0),
+         (Label::Borrowed("json.key_value"), kv1),
+         (Label::Borrowed("json.object"), obj2),
+         (Label::Borrowed("json.array"), arr3),
+        ].to_vec()
+    );
+    decls[0]
 }
 
-fn main() {
+#[derive(Parser)]
+struct Args {
+    file: PathBuf,
+}
+
+fn main() -> Result<(), Box<dyn Send + Sync + std::error::Error + 'static>> {
+    // Parse arguments and open file-stream
+    let args = Args::parse();
+    let mut fh = std::fs::File::open(args.file)?;
+    let mut buffer = Vec::new();
+    fh.read_to_end(&mut buffer)?;
+
     let mut module = FormatModule::new();
-    let bson = bson(&mut module);
+    let json = json_lite(&mut module);
     let main = module.declare_format(
         Label::Borrowed("main"),
-        Format::Tuple(vec![bson.call(), Format::EndOfInput]),
+        Format::Tuple(vec![json.call(), Format::EndOfInput]),
     );
-    println!("Hello, world!");
+    let program = Compiler::compile_program(&module, &main.call(), RecurseCtx::NonRec)?;
+    let (v, rest) = program.run(ReadCtxt::new(&buffer))?;
+    assert!(rest.remaining().is_empty());
+    println!("{v:?}");
+    Ok(())
 }
diff --git a/experiments/doodle-rec/src/matchtree.rs b/experiments/doodle-rec/src/matchtree.rs
index c5947b51..96d5f1e0 100644
--- a/experiments/doodle-rec/src/matchtree.rs
+++ b/experiments/doodle-rec/src/matchtree.rs
@@ -98,7 +98,7 @@ impl<'a> MatchTreeStep<'a> {
                 let next1 = next.clone();
                 tree.union(MatchTreeStep::<'a>::from_format(module, *a, next1, ctx))
             }
-            Next::DelayRef(_ix) => {
+            Next::DelayRef(_ix /*, next0 */) => {
                 // FIXME - figure out how to resolve this
                 Self::accept()
             }
@@ -147,7 +147,7 @@ impl<'a> MatchTreeStep<'a> {
             Format::Compute(_expr) => Self::from_next(module, next, ctx),
             Format::RecVar(rec_ix) => {
                 // FIXME - we discard the original `next` argument here
-                let next = Rc::new(Next::DelayRef(ctx.convert_rec_var(*rec_ix)));
+                let next = Rc::new(Next::DelayRef(ctx.convert_rec_var(*rec_ix) /* , next.clone() */));
                 Self::from_next(module, next, ctx)
             }
         }
@@ -282,28 +282,16 @@ impl<'a> MatchTreeLevel<'a> {
 
 type LevelBranch<'a> = HashSet<(usize, Rc<Next<'a>>)>;
 
-#[derive(Eq, Hash, Debug)]
+#[derive(PartialEq, Eq, Hash, Debug)]
 pub(crate) enum Next<'a> {
     Empty,
-    DelayRef(usize),
+    DelayRef(usize /* , Rc<Next<'a>> */),
     Union(Rc<Next<'a>>, Rc<Next<'a>>),
     Cat(&'a Format, Rc<Next<'a>>),
     Sequence(&'a [Format], Rc<Next<'a>>),
     Repeat(&'a Format, Rc<Next<'a>>),
 }
 
-impl<'a> PartialEq for Next<'a> {
-    fn eq(&self, other: &Self) -> bool {
-        match (self, other) {
-            (Self::DelayRef(l0), Self::DelayRef(r0)) => l0 == r0,
-            (Self::Union(l0, l1), Self::Union(r0, r1)) => l0 == r0 && l1 == r1,
-            (Self::Cat(l0, l1), Self::Cat(r0, r1)) => l0 == r0 && l1 == r1,
-            (Self::Sequence(l0, l1), Self::Sequence(r0, r1)) => l0 == r0 && l1 == r1,
-            (Self::Repeat(l0, l1), Self::Repeat(r0, r1)) => l0 == r0 && l1 == r1,
-            _ => false,
-        }
-    }
-}
 #[derive(Debug, Clone)]
 pub struct MatchTree {
     accept: Option<usize>,
diff --git a/experiments/doodle-rec/test1.json b/experiments/doodle-rec/test1.json
new file mode 100644
index 00000000..0967ef42
--- /dev/null
+++ b/experiments/doodle-rec/test1.json
@@ -0,0 +1 @@
+{}
diff --git a/experiments/doodle-rec/test2.json b/experiments/doodle-rec/test2.json
new file mode 100644
index 00000000..5bdf5940
--- /dev/null
+++ b/experiments/doodle-rec/test2.json
@@ -0,0 +1 @@
+{"hello":true,"world":1}

From 1d2d1fb5a812320d0ef6bc53aee60d240b30972c Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Mon, 26 May 2025 15:26:51 +1000
Subject: [PATCH 08/15] Prototype MatchForest (WIP)

---
 experiments/doodle-rec/src/lib.rs             |   3 +-
 experiments/doodle-rec/src/matchtree.rs       |   7 +-
 .../doodle-rec/src/matchtree/forest.rs        | 272 ++++++++++++++++++
 3 files changed, 279 insertions(+), 3 deletions(-)
 create mode 100644 experiments/doodle-rec/src/matchtree/forest.rs

diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index 018e9afb..3336f1d2 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -112,7 +112,7 @@ impl<Idx> Span<Idx> {
 
 impl Span<usize> {
     pub fn index(self, ix: usize) -> usize {
-        assert!(ix < (self.end - self.start + 1) as usize);
+        assert!(self.start + ix <= self.end);
         self.start + ix
     }
 }
@@ -617,6 +617,7 @@ pub enum Unary {
     BoolNot,
 }
 
+#[derive(Debug)]
 pub struct FormatModule {
     names: Vec<Label>,
     decls: Vec<FormatDecl>,
diff --git a/experiments/doodle-rec/src/matchtree.rs b/experiments/doodle-rec/src/matchtree.rs
index 96d5f1e0..1251d803 100644
--- a/experiments/doodle-rec/src/matchtree.rs
+++ b/experiments/doodle-rec/src/matchtree.rs
@@ -3,6 +3,9 @@ use std::{collections::HashSet, rc::Rc, vec};
 use crate::{Format, FormatModule, RecurseCtx};
 use doodle::{byte_set::ByteSet, read::ReadCtxt};
 
+pub mod forest;
+pub use forest::MatchForest;
+
 #[derive(Clone, Debug)]
 pub(crate) struct MatchTreeStep<'a> {
     accept: bool,
@@ -119,7 +122,7 @@ impl<'a> MatchTreeStep<'a> {
             Format::FailWith(_) => Self::reject(),
             Format::EndOfInput => Self::accept(),
             Format::Byte(bs) => Self::branch(*bs, next),
-            Format::Variant(_label, f) => Self::from_format(module, f, next.clone(), ctx),
+            Format::Variant(_label, f) => Self::from_format(module, f, next, ctx),
             Format::Union(branches) => {
                 let mut tree = Self::reject();
                 for f in branches {
@@ -135,7 +138,7 @@ impl<'a> MatchTreeStep<'a> {
                 tree.union(Self::from_format(
                     module,
                     a,
-                    Rc::new(Next::Repeat(a, next.clone())),
+                    Rc::new(Next::Repeat(a, next)),
                     ctx,
                 ))
             }
diff --git a/experiments/doodle-rec/src/matchtree/forest.rs b/experiments/doodle-rec/src/matchtree/forest.rs
new file mode 100644
index 00000000..82ee0421
--- /dev/null
+++ b/experiments/doodle-rec/src/matchtree/forest.rs
@@ -0,0 +1,272 @@
+use std::{cell::OnceCell, rc::Rc};
+
+use doodle::prelude::ByteSet;
+
+use crate::{Format, FormatModule, RecurseCtx};
+
+/// A forest of [`FMatchTree`]s, one for each level of a [`FormatModule`] (with directly corresponding indices)
+#[derive(Debug)]
+pub struct MatchForest<'a> {
+    module: &'a FormatModule,
+    trees: Vec<Rc<OnceCell<FMatchTree>>>,
+}
+
+impl<'a> MatchForest<'a> {
+    /// Initializes a new [`MatchForest`] from a [`FormatModule`].
+    pub fn new(module: &'a FormatModule) -> Self {
+        let trees = (0..module.decls.len())
+            .map(|_| Rc::new(OnceCell::new()))
+            .collect();
+        Self { module, trees }
+    }
+
+    /// Returns a shared-pointer to the [`FMatchTree`] at `level`.
+    #[must_use]
+    pub(crate) fn get_level(&self, level: usize) -> Rc<OnceCell<FMatchTree>> {
+        assert!(level < self.module.decls.len());
+        self.trees[level].clone()
+    }
+
+    /// Populates (if it is not already populated) and returns a shared-pointer to the [`FMatchTree`] at `level`.
+    fn populate_level(&self, level: usize) -> Rc<OnceCell<FMatchTree>> {
+        assert!(level < self.module.decls.len());
+
+        let ret = self.trees[level].clone();
+
+        const LIMIT: usize = 40;
+
+        if self.trees[level].get().is_none() {
+            let ctx = self.module.get_ctx(level);
+            let tree = FMatchTree::grow(&self, level, LIMIT, ctx);
+            ret.set(tree);
+        }
+
+        ret
+    }
+}
+
+#[derive(Debug, Clone)]
+enum FollowSet<'a> {
+    Delayed(usize, Rc<PartialFormat<'a>>),
+    Branch(ByteSet, Rc<PartialFormat<'a>>),
+}
+
+/// Representation of the right-justified remainder of a [`Format`] we have already
+/// consumed some number (possibly 0) bytes of.
+#[derive(PartialEq, Eq, Hash, Debug)]
+enum PartialFormat<'a> {
+    /// `ε`
+    Empty,
+    /// A full-format followed by a remnant
+    Cat(&'a Format, Rc<PartialFormat<'a>>),
+    /// A sequence of full-formats followed by a remnant
+    Sequence(&'a [Format], Rc<PartialFormat<'a>>),
+    /// Repeat the specified format zero or more times before processing a remnant
+    Repeat(&'a Format, Rc<PartialFormat<'a>>),
+    /// Alternation over two partial-formats that shared a common prefix we already consumed
+    Union(Rc<PartialFormat<'a>>, Rc<PartialFormat<'a>>),
+}
+
+#[derive(Debug)]
+pub struct MatchBranch<'a> {
+    accept: bool,
+    branches: Vec<FollowSet<'a>>,
+}
+
+impl<'a> MatchBranch<'a> {
+    pub fn reject() -> MatchBranch<'a> {
+        Self {
+            accept: false,
+            branches: vec![],
+        }
+    }
+
+    pub fn accept() -> MatchBranch<'a> {
+        Self {
+            accept: true,
+            branches: vec![],
+        }
+    }
+
+    fn branch(bs: ByteSet, remnant: Rc<PartialFormat<'a>>) -> MatchBranch<'a> {
+        Self {
+            accept: false,
+            branches: vec![FollowSet::Branch(bs, remnant)],
+        }
+    }
+
+    fn delayed(forest: &'a MatchForest<'a>, level: usize, remnant: Rc<PartialFormat<'a>>) -> MatchBranch<'a> {
+        Self {
+            accept: false,
+            branches: vec![FollowSet::Delayed(level, remnant)]
+        }
+    }
+    pub fn from_level(forest: &'a MatchForest<'a>, level: usize) -> MatchBranch<'a> {
+        let decl = &forest.module.decls[level];
+        let ctx = forest.module.get_ctx(level);
+        MatchBranch::from_format(forest, &decl.format, Rc::new(PartialFormat::Empty), ctx)
+    }
+
+    fn from_partial_format(
+        forest: &'a MatchForest<'a>,
+        remnant: Rc<PartialFormat<'a>>,
+        ctx: RecurseCtx<'a>,
+    ) -> MatchBranch<'a> {
+        match remnant.as_ref() {
+            PartialFormat::Empty => Self::accept(),
+            PartialFormat::Cat(format, remnant) => Self::from_format(forest, *format, remnant.clone(), ctx),
+            PartialFormat::Sequence(formats, remnant) => {
+                let remnant = remnant.clone();
+                match formats.split_first() {
+                    None => Self::from_partial_format(forest, remnant, ctx),
+                    Some((f, fs)) => Self::from_format(forest, f, Rc::new(PartialFormat::Sequence(fs, remnant)), ctx),
+                }
+            }
+            PartialFormat::Repeat(format, remnant0) => {
+                let tree = Self::from_partial_format(forest, remnant0.clone(), ctx);
+                let remnant1 = remnant.clone();
+                tree.union(Self::from_format(forest, *format, remnant1, ctx))
+            }
+            PartialFormat::Union(remnant1, remnant2) => {
+                let tree1 = Self::from_partial_format(forest, remnant1.clone(), ctx);
+                let tree2 = Self::from_partial_format(forest, remnant2.clone(), ctx);
+                tree1.union(tree2)
+            }
+        }
+    }
+
+    fn union_branch(&mut self, mut bs: ByteSet, remnant: Rc<PartialFormat<'a>>) {
+        let mut branches = Vec::new();
+        for follow_set in self.branches.iter_mut() {
+            match follow_set {
+                FollowSet::Delayed(_level, _remnant) => {
+                    // FIXME - how do we model union with delayed FollowSets?
+                    todo!();
+                }
+                FollowSet::Branch(bs0, remnant0) => {
+                    let common = bs0.intersection(&bs);
+                    if !common.is_empty() {
+                        let orig = bs0.difference(&bs);
+                        if !orig.is_empty() {
+                            branches.push(FollowSet::Branch(orig, remnant0.clone()));
+                        }
+                        *bs0 = common;
+                        *remnant0 = Rc::new(PartialFormat::Union(remnant0.clone(), remnant.clone()));
+                        bs = bs.difference(bs0);
+                    }
+                }
+            }
+        }
+        if !bs.is_empty() {
+            self.branches.push(FollowSet::Branch(bs, remnant));
+        }
+        self.branches.append(&mut branches);
+    }
+
+    fn union_delayed(&mut self, level: usize, remnant: Rc<PartialFormat<'a>>) {
+        let fset = FollowSet::Delayed(level, remnant);
+        todo!()
+    }
+
+
+    fn union(mut self, other: MatchBranch<'a>) -> MatchBranch<'a> {
+        self.accept = self.accept || other.accept;
+        for follow_set in other.branches {
+            match &follow_set {
+                FollowSet::Delayed(level, remnant) => {
+                    self.union_delayed(*level, remnant.clone());
+                }
+                FollowSet::Branch(bs, remnant) => {
+                    self.union_branch(*bs, remnant.clone());
+                }
+            }
+        }
+        self
+    }
+
+    fn from_format(
+        forest: &'a MatchForest<'a>,
+        f: &'a Format,
+        remnant: Rc<PartialFormat<'a>>,
+        ctx: RecurseCtx<'a>,
+    ) -> MatchBranch<'a> {
+        match f {
+            Format::ItemVar(level) => {
+                let level = *level;
+                let format = forest.module.get_format(level);
+                let ctx = forest.module.get_ctx(level);
+                Self::from_format(forest, format, remnant, ctx)
+            }
+            Format::RecVar(rec_ix) => {
+                let RecurseCtx::Recurse { span, .. } = ctx else {
+                    unreachable!(
+                        "level {} is unexpectedly recursive (rec-var without ctx)",
+                        rec_ix
+                    )
+                };
+                let level = span.index(*rec_ix);
+                Self::delayed(forest, level, remnant)
+            }
+            Format::FailWith(..) => Self::reject(),
+            Format::EndOfInput => Self::accept(),
+            Format::Byte(bs) => Self::branch(*bs, remnant),
+            Format::Compute(..) => Self::from_partial_format(forest, remnant, ctx),
+            Format::Variant(_, format) => Self::from_format(forest, format, remnant, ctx),
+            Format::Union(branches) => {
+                let mut tree = Self::reject();
+                for f in branches {
+                    tree = tree.union(Self::from_format(forest, f, remnant.clone(), ctx));
+                }
+                tree
+            }
+            Format::Repeat(format) => todo!(),
+            Format::Seq(formats) => todo!(),
+            Format::Tuple(formats) => todo!(),
+            Format::Maybe(expr, format) => todo!(),
+        }
+    }
+
+}
+
+#[derive(Debug)]
+pub struct FMatchTree {
+    accept: Option<usize>,
+    branches: Vec<(ByteSet, FMatchTree)>,
+}
+
+impl FMatchTree {
+    fn grow<'a>(arg: &'a MatchForest<'a>, level: usize, limit: usize, ctx: RecurseCtx<'a>) -> Self {
+        todo!()
+    }
+}
+
+mod traversal {
+    use std::collections::BTreeSet;
+
+    pub(super) struct Traversal {
+        orig_level: usize,
+        seen_levels: BTreeSet<usize>,
+    }
+
+    impl Traversal {
+        pub(super) fn new(orig_level: usize) -> Self {
+            Self {
+                orig_level,
+                seen_levels: BTreeSet::new(),
+            }
+        }
+
+        /// Returns `true` if the level has not yet been seen (including the original level)
+        pub(super) fn is_novel(&self, level: usize) -> bool {
+            level != self.orig_level && !self.seen_levels.contains(&level)
+        }
+
+        /// Record a level as being seen, returning `true` if it is novel.
+        pub(super) fn insert(&mut self, level: usize) -> bool {
+            if level == self.orig_level {
+                return false;
+            }
+            self.seen_levels.insert(level)
+        }
+    }
+}

From 3c62e48df4eaf743d023dfc0fa459d2c773721ce Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Tue, 27 May 2025 18:06:08 +1000
Subject: [PATCH 09/15] Add first-set implementation for Formats in forest.rs

---
 experiments/doodle-rec/src/lib.rs             |   2 +-
 experiments/doodle-rec/src/main.rs            |   8 +-
 experiments/doodle-rec/src/matchtree.rs       |   2 +-
 .../doodle-rec/src/matchtree/forest.rs        | 224 ++++++++++++++----
 4 files changed, 192 insertions(+), 44 deletions(-)

diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index 3336f1d2..ea4086c0 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -705,7 +705,7 @@ impl FormatModule {
         &self.decls[level].format
     }
 
-    fn get_decl(&self, level: usize) -> &FormatDecl {
+    pub fn get_decl(&self, level: usize) -> &FormatDecl {
         &self.decls[level]
     }
 
diff --git a/experiments/doodle-rec/src/main.rs b/experiments/doodle-rec/src/main.rs
index 504ff269..1885a4fb 100644
--- a/experiments/doodle-rec/src/main.rs
+++ b/experiments/doodle-rec/src/main.rs
@@ -27,7 +27,7 @@ fn json_lite(module: &mut FormatModule) -> FormatRef {
         ("JNull", byte_seq(*b"null")),
         ("JTrue", byte_seq(*b"true")),
         ("JFalse", byte_seq(*b"false")),
-        ("JNum", repeat1(Format::Byte(ByteSet::from(b'0'..b'9')))),
+        ("JNum", repeat1(Format::Byte(ByteSet::from(b'0'..=b'9')))),
         ("JObj", Format::RecVar(2)),
         ("JArr", Format::RecVar(3)),
     ]);
@@ -62,9 +62,15 @@ fn main() -> Result<(), Box<dyn Send + Sync + std::error::Error + 'static>> {
         Label::Borrowed("main"),
         Format::Tuple(vec![json.call(), Format::EndOfInput]),
     );
+
+    let first_set = module.get_decl(main.get_level()).first_set(&module)?;
+    println!("jsonValue.first: {first_set:?}");
+
+    /*
     let program = Compiler::compile_program(&module, &main.call(), RecurseCtx::NonRec)?;
     let (v, rest) = program.run(ReadCtxt::new(&buffer))?;
     assert!(rest.remaining().is_empty());
     println!("{v:?}");
+    */
     Ok(())
 }
diff --git a/experiments/doodle-rec/src/matchtree.rs b/experiments/doodle-rec/src/matchtree.rs
index 1251d803..c653cb4e 100644
--- a/experiments/doodle-rec/src/matchtree.rs
+++ b/experiments/doodle-rec/src/matchtree.rs
@@ -4,7 +4,7 @@ use crate::{Format, FormatModule, RecurseCtx};
 use doodle::{byte_set::ByteSet, read::ReadCtxt};
 
 pub mod forest;
-pub use forest::MatchForest;
+// pub use forest::MatchForest;
 
 #[derive(Clone, Debug)]
 pub(crate) struct MatchTreeStep<'a> {
diff --git a/experiments/doodle-rec/src/matchtree/forest.rs b/experiments/doodle-rec/src/matchtree/forest.rs
index 82ee0421..ea7e1ba3 100644
--- a/experiments/doodle-rec/src/matchtree/forest.rs
+++ b/experiments/doodle-rec/src/matchtree/forest.rs
@@ -1,9 +1,171 @@
-use std::{cell::OnceCell, rc::Rc};
-
 use doodle::prelude::ByteSet;
 
-use crate::{Format, FormatModule, RecurseCtx};
+use crate::{Format, FormatDecl, FormatId, FormatModule, RecId, RecurseCtx};
+
+pub type FirstSet = ByteSet;
+
+#[derive(Debug)]
+pub enum GrammarError {
+    EpsilonLoop { top: FormatId, reentrant: RecId },
+    RepeatNullable { format: Format },
+    MultiNullUnion { format: Format },
+}
+
+impl std::fmt::Display for GrammarError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            GrammarError::EpsilonLoop { top, reentrant } => {
+                write!(
+                    f,
+                    "epsilon-move cycle found in format: #{} -*-> ~{} -*-> ~{}",
+                    top,
+                    reentrant,
+                    reentrant,
+                )
+            }
+            GrammarError::RepeatNullable { format  } => {
+                write!(f, "repeat of nullable format: {:?}", format)
+            }
+            GrammarError::MultiNullUnion { format } => {
+                write!(f, "multiple nullable formats in union: {:?}", format)
+            }
+        }
+    }
+}
+
+impl std::error::Error for GrammarError {}
+
+impl FormatDecl {
+    pub fn first_set(&self, module: &FormatModule) -> Result<FirstSet, GrammarError> {
+        let mut traversal = Traversal::new(self.fmt_id);
+        let ctx = module.get_ctx(self.fmt_id);
+        Ok(self.format.solve_first_set(module, &mut traversal, ctx)?.0)
+    }
+}
+
+impl Format {
+    /// Returns the first-set, along with `true` if the format is nullable and `false` otherwise
+    fn solve_first_set(
+        &self,
+        module: &FormatModule,
+        visited: &mut Traversal,
+        ctx: RecurseCtx<'_>,
+    ) -> Result<(FirstSet, bool), GrammarError> {
+        match self {
+            Format::ItemVar(level) => {
+                let level = *level;
+                let ctx = module.get_ctx(level);
+                let mut visited = Traversal::new(level);
+                module
+                    .get_format(level)
+                    .solve_first_set(module, &mut visited, ctx)
+            },
+            Format::RecVar(rec_ix) => {
+                let level = ctx.convert_rec_var(*rec_ix);
+                if visited.insert(level) {
+                    let ctx = ctx.enter(*rec_ix).0;
+                    ctx.get_format().unwrap().solve_first_set(module, visited, ctx)
+                } else {
+                    Err(GrammarError::EpsilonLoop {
+                        top: visited.orig_level,
+                        reentrant: *rec_ix,
+                    })
+                }
+            }
+            Format::Byte(set) => Ok((*set, false)),
+            Format::FailWith(..) => Ok((FirstSet::empty(), false)),
+            Format::EndOfInput => Ok((FirstSet::empty(), true)),
+            Format::Compute(..) => Ok((FirstSet::empty(), true)),
+            Format::Variant(.., format) => format.solve_first_set(module, visited, ctx),
+            Format::Union(formats) => {
+                let mut ret = ByteSet::empty();
+                let mut any_nullable = false;
+                for format in formats {
+                    let (first_set, is_nullable) = format.solve_first_set(module, visited, ctx)?;
+                    ret = ret.union(&first_set);
+                    if is_nullable {
+                        if any_nullable {
+                            return Err(GrammarError::MultiNullUnion { format: self.clone() });
+                        }
+                        any_nullable = true;
+                    }
+                }
+                Ok((ret, any_nullable))
+            }
+            Format::Repeat(format) => {
+                let (first_set, is_nullable) = format.solve_first_set(module, visited, ctx)?;
+                if is_nullable {
+                    return Err(GrammarError::RepeatNullable { format: format.as_ref().clone() });
+                }
+                Ok((first_set, true))
+            }
+            Format::Tuple(formats) | Format::Seq(formats) => {
+                let mut ret = ByteSet::empty();
+                let mut nullable = true;
+                for format in formats {
+                    if !ret.is_empty() { break; }
+                    let (first_set, is_nullable) = format.solve_first_set(module, visited, ctx)?;
+                    ret = ret.union(&first_set);
+                    nullable &= is_nullable;
+                    if !nullable {
+                        break;
+                    }
+                }
+                Ok((ret, nullable))
+            }
+            Format::Maybe(_cond, format) => {
+                let (first_set, _) = format.solve_first_set(module, visited, ctx)?;
+                Ok((first_set, true))
+            }
+        }
+    }
+}
+
+
+use traversal::Traversal;
+mod traversal {
+    use std::collections::BTreeSet;
+
+    pub(super) struct Traversal {
+        pub(super) orig_level: usize,
+        seen_levels: BTreeSet<usize>,
+    }
+
+    impl Traversal {
+        pub(super) fn new(orig_level: usize) -> Self {
+            Self {
+                orig_level,
+                seen_levels: BTreeSet::new(),
+            }
+        }
+
+        /// Returns `true` if the level has not yet been seen (including the original level)
+        pub(super) fn is_novel(&self, level: usize) -> bool {
+            level != self.orig_level && !self.seen_levels.contains(&level)
+        }
+
+        /// Record a level as being seen, returning `true` if it is novel.
+        pub(super) fn insert(&mut self, level: usize) -> bool {
+            if level == self.orig_level {
+                return false;
+            }
+            self.seen_levels.insert(level)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_follow_set() {
 
+    }
+}
+
+
+/*
 /// A forest of [`FMatchTree`]s, one for each level of a [`FormatModule`] (with directly corresponding indices)
 #[derive(Debug)]
 pub struct MatchForest<'a> {
@@ -95,10 +257,14 @@ impl<'a> MatchBranch<'a> {
         }
     }
 
-    fn delayed(forest: &'a MatchForest<'a>, level: usize, remnant: Rc<PartialFormat<'a>>) -> MatchBranch<'a> {
+    fn delayed(
+        forest: &'a MatchForest<'a>,
+        level: usize,
+        remnant: Rc<PartialFormat<'a>>,
+    ) -> MatchBranch<'a> {
         Self {
             accept: false,
-            branches: vec![FollowSet::Delayed(level, remnant)]
+            branches: vec![FollowSet::Delayed(level, remnant)],
         }
     }
     pub fn from_level(forest: &'a MatchForest<'a>, level: usize) -> MatchBranch<'a> {
@@ -114,12 +280,19 @@ impl<'a> MatchBranch<'a> {
     ) -> MatchBranch<'a> {
         match remnant.as_ref() {
             PartialFormat::Empty => Self::accept(),
-            PartialFormat::Cat(format, remnant) => Self::from_format(forest, *format, remnant.clone(), ctx),
+            PartialFormat::Cat(format, remnant) => {
+                Self::from_format(forest, *format, remnant.clone(), ctx)
+            }
             PartialFormat::Sequence(formats, remnant) => {
                 let remnant = remnant.clone();
                 match formats.split_first() {
                     None => Self::from_partial_format(forest, remnant, ctx),
-                    Some((f, fs)) => Self::from_format(forest, f, Rc::new(PartialFormat::Sequence(fs, remnant)), ctx),
+                    Some((f, fs)) => Self::from_format(
+                        forest,
+                        f,
+                        Rc::new(PartialFormat::Sequence(fs, remnant)),
+                        ctx,
+                    ),
                 }
             }
             PartialFormat::Repeat(format, remnant0) => {
@@ -151,7 +324,8 @@ impl<'a> MatchBranch<'a> {
                             branches.push(FollowSet::Branch(orig, remnant0.clone()));
                         }
                         *bs0 = common;
-                        *remnant0 = Rc::new(PartialFormat::Union(remnant0.clone(), remnant.clone()));
+                        *remnant0 =
+                            Rc::new(PartialFormat::Union(remnant0.clone(), remnant.clone()));
                         bs = bs.difference(bs0);
                     }
                 }
@@ -168,7 +342,6 @@ impl<'a> MatchBranch<'a> {
         todo!()
     }
 
-
     fn union(mut self, other: MatchBranch<'a>) -> MatchBranch<'a> {
         self.accept = self.accept || other.accept;
         for follow_set in other.branches {
@@ -225,7 +398,6 @@ impl<'a> MatchBranch<'a> {
             Format::Maybe(expr, format) => todo!(),
         }
     }
-
 }
 
 #[derive(Debug)]
@@ -239,34 +411,4 @@ impl FMatchTree {
         todo!()
     }
 }
-
-mod traversal {
-    use std::collections::BTreeSet;
-
-    pub(super) struct Traversal {
-        orig_level: usize,
-        seen_levels: BTreeSet<usize>,
-    }
-
-    impl Traversal {
-        pub(super) fn new(orig_level: usize) -> Self {
-            Self {
-                orig_level,
-                seen_levels: BTreeSet::new(),
-            }
-        }
-
-        /// Returns `true` if the level has not yet been seen (including the original level)
-        pub(super) fn is_novel(&self, level: usize) -> bool {
-            level != self.orig_level && !self.seen_levels.contains(&level)
-        }
-
-        /// Record a level as being seen, returning `true` if it is novel.
-        pub(super) fn insert(&mut self, level: usize) -> bool {
-            if level == self.orig_level {
-                return false;
-            }
-            self.seen_levels.insert(level)
-        }
-    }
-}
+*/

From 89a858f8cee5c8860ce3d32492a0fd9b7a9adeb4 Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Wed, 28 May 2025 21:34:27 +1000
Subject: [PATCH 10/15] LL1 acceptance-checker (not working yet)

---
 experiments/doodle-rec/Cargo.toml             |   1 +
 experiments/doodle-rec/src/decoder.rs         |   2 +-
 experiments/doodle-rec/src/lib.rs             |   1 +
 experiments/doodle-rec/src/main.rs            |  28 +-
 .../doodle-rec/src/matchtree/forest.rs        | 631 +++++++++++++++---
 5 files changed, 556 insertions(+), 107 deletions(-)

diff --git a/experiments/doodle-rec/Cargo.toml b/experiments/doodle-rec/Cargo.toml
index c58523c6..2f4401fe 100644
--- a/experiments/doodle-rec/Cargo.toml
+++ b/experiments/doodle-rec/Cargo.toml
@@ -11,5 +11,6 @@ bench = false
 anyhow = "1.0.98"
 clap = "4.5.38"
 doodle = { path = "../../" }
+linked_hash_set = "0.1.5"
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = "1.0.140"
diff --git a/experiments/doodle-rec/src/decoder.rs b/experiments/doodle-rec/src/decoder.rs
index 3c371e3d..6b4b9af7 100644
--- a/experiments/doodle-rec/src/decoder.rs
+++ b/experiments/doodle-rec/src/decoder.rs
@@ -38,7 +38,7 @@ impl Value {
         }
     }
 
-    fn unwrap_bool(&self) -> bool {
+    pub(crate) fn unwrap_bool(&self) -> bool {
         match self {
             Value::Bool(b) => *b,
             _ => panic!("value is not a bool"),
diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index ea4086c0..f9eafe13 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -2,6 +2,7 @@ pub mod decoder;
 pub(crate) mod matchtree;
 pub mod helper;
 pub(crate) use matchtree::{MatchTree, Next};
+pub use matchtree::forest::Interpreter;
 
 use anyhow::{Result as AResult, anyhow};
 use doodle::{bounds::Bounds, byte_set::ByteSet};
diff --git a/experiments/doodle-rec/src/main.rs b/experiments/doodle-rec/src/main.rs
index 1885a4fb..e15bb889 100644
--- a/experiments/doodle-rec/src/main.rs
+++ b/experiments/doodle-rec/src/main.rs
@@ -1,7 +1,7 @@
 use std::{io::Read, path::PathBuf};
 
 use doodle::{prelude::ByteSet, read::ReadCtxt};
-use doodle_rec::{decoder::{Compiler, Value}, helper::*, Format, FormatModule, FormatRef, Label, RecurseCtx};
+use doodle_rec::{helper::*, Format, FormatModule, FormatRef, Interpreter, Label};
 use clap::Parser;
 
 fn surround(before: u8, after: u8, f: Format) -> Format {
@@ -9,7 +9,7 @@ fn surround(before: u8, after: u8, f: Format) -> Format {
 }
 
 fn alpha() -> Format {
-    repeat(Format::Byte(ByteSet::union(&ByteSet::from(b'a'..b'z'), &ByteSet::from(b'A'..b'Z'))))
+    Format::Byte(ByteSet::union(&ByteSet::from(b'a'..b'z'), &ByteSet::from(b'A'..b'Z')))
 }
 
 /// Restricted JSON object
@@ -62,15 +62,21 @@ fn main() -> Result<(), Box<dyn Send + Sync + std::error::Error + 'static>> {
         Label::Borrowed("main"),
         Format::Tuple(vec![json.call(), Format::EndOfInput]),
     );
+    let interpreter = Interpreter::new(&module);
 
-    let first_set = module.get_decl(main.get_level()).first_set(&module)?;
-    println!("jsonValue.first: {first_set:?}");
-
-    /*
-    let program = Compiler::compile_program(&module, &main.call(), RecurseCtx::NonRec)?;
-    let (v, rest) = program.run(ReadCtxt::new(&buffer))?;
-    assert!(rest.remaining().is_empty());
-    println!("{v:?}");
-    */
+    let input = ReadCtxt::new(&buffer);
+    let (trace, rest, opt_err) = interpreter.run_level(main.get_level(), input);
+    match opt_err {
+        None => {
+            println!("input accepted!");
+            println!("trace: {trace:?}");
+            println!("remainder: {:?}", rest.remaining());
+        }
+        Some(err) => {
+            println!("input rejected: {err:?}");
+            println!("trace: {trace:?}");
+            println!("remainder: {:?}", rest.remaining());
+        }
+    }
     Ok(())
 }
diff --git a/experiments/doodle-rec/src/matchtree/forest.rs b/experiments/doodle-rec/src/matchtree/forest.rs
index ea7e1ba3..eb445e5c 100644
--- a/experiments/doodle-rec/src/matchtree/forest.rs
+++ b/experiments/doodle-rec/src/matchtree/forest.rs
@@ -1,33 +1,151 @@
-use doodle::prelude::ByteSet;
+use std::{collections::HashSet, rc::Rc};
 
-use crate::{Format, FormatDecl, FormatId, FormatModule, RecId, RecurseCtx};
+use doodle::{prelude::ByteSet, read::ReadCtxt};
 
-pub type FirstSet = ByteSet;
+use crate::{Format, FormatDecl, FormatId, FormatModule, RecId, RecurseCtx};
 
 #[derive(Debug)]
 pub enum GrammarError {
-    EpsilonLoop { top: FormatId, reentrant: RecId },
+    LeftRecursion { top: FormatId, cycle: Vec<FormatId> },
     RepeatNullable { format: Format },
-    MultiNullUnion { format: Format },
+    AmbiguousFirst { left: ByteSet, right: ByteSet },
+    MultiNullUnion,
+    AmbiguousFollow { left: ByteSet, right: ByteSet },
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub(crate) struct Determinations {
+    pub is_nullable: bool,
+    pub is_productive: bool,
+    pub first_set: ByteSet,
+    pub should_not_follow_set: ByteSet,
+}
+
+impl Determinations {
+    /// Additive (i.e. sequencing) identity element
+    pub const fn zero() -> Self {
+        Self {
+            first_set: ByteSet::empty(),
+            is_productive: true,
+            is_nullable: true,
+            should_not_follow_set: ByteSet::empty(),
+        }
+    }
+
+    /// Multiplicative (i.e. disjunction) identity element
+    pub const fn one() -> Self {
+        Self {
+            first_set: ByteSet::empty(),
+            is_productive: false,
+            is_nullable: false,
+            should_not_follow_set: ByteSet::empty(),
+        }
+    }
+
+    /// Solve the determination for `F >> G` (sequencing) given the determinations for F and G.
+    pub fn merge_seq(self, other: Self) -> Result<Self, GrammarError> {
+        if !self.should_not_follow_set.is_disjoint(&other.first_set) {
+            return Err(GrammarError::AmbiguousFollow {
+                left: self.should_not_follow_set,
+                right: other.first_set,
+            });
+        }
+        let is_nullable = self.is_nullable && other.is_nullable;
+        let is_productive = self.is_productive && other.is_productive;
+        let first_set = {
+            let left = if self.is_nullable {
+                other.first_set
+            } else {
+                ByteSet::empty()
+            };
+            let right = if other.is_productive {
+                self.first_set
+            } else {
+                ByteSet::empty()
+            };
+            left.union(&right)
+        };
+        let should_not_follow_set = {
+            let left = if other.is_nullable {
+                self.should_not_follow_set
+            } else {
+                ByteSet::empty()
+            };
+            let right = if self.is_productive {
+                other.should_not_follow_set
+            } else {
+                ByteSet::empty()
+            };
+            left.union(&right)
+        };
+        Ok(Self {
+            first_set,
+            is_nullable,
+            is_productive,
+            should_not_follow_set,
+        })
+    }
+
+    pub fn union(self, other: Self) -> Result<Self, GrammarError> {
+        if self.is_nullable && other.is_nullable {
+            return Err(GrammarError::MultiNullUnion);
+        }
+        if !self.first_set.is_disjoint(&other.first_set) {
+            return Err(GrammarError::AmbiguousFirst {
+                left: self.first_set,
+                right: other.first_set,
+            });
+        }
+        let first_set = self.first_set.union(&other.first_set);
+        let should_not_follow_set = {
+            let left = self.should_not_follow_set;
+            let right = other.should_not_follow_set;
+            let mut ret = left.union(&right);
+            if self.is_nullable {
+                ret = ret.union(&other.first_set);
+            } else if other.is_nullable {
+                ret = ret.union(&self.first_set);
+            }
+            ret
+        };
+        Ok(Self {
+            first_set,
+            is_productive: self.is_productive || other.is_productive,
+            is_nullable: self.is_nullable || other.is_nullable,
+            should_not_follow_set,
+        })
+    }
 }
 
 impl std::fmt::Display for GrammarError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            GrammarError::EpsilonLoop { top, reentrant } => {
+            GrammarError::LeftRecursion { top, cycle } => {
                 write!(
                     f,
-                    "epsilon-move cycle found in format: #{} -*-> ~{} -*-> ~{}",
-                    top,
-                    reentrant,
-                    reentrant,
+                    "left-recursion found in format: #{} -> {:#?}",
+                    top, cycle,
                 )
             }
-            GrammarError::RepeatNullable { format  } => {
+            GrammarError::RepeatNullable { format } => {
                 write!(f, "repeat of nullable format: {:?}", format)
             }
-            GrammarError::MultiNullUnion { format } => {
-                write!(f, "multiple nullable formats in union: {:?}", format)
+            GrammarError::MultiNullUnion => {
+                write!(f, "multiple nullable formats in union")
+            }
+            GrammarError::AmbiguousFirst { left, right } => {
+                write!(
+                    f,
+                    "ambiguity introduced by union of non-disjoint first sets: {:?} <|> {:?}",
+                    left, right
+                )
+            }
+            GrammarError::AmbiguousFollow { left, right } => {
+                write!(
+                    f,
+                    "follow set and first set conflict: {:?} & {:?} ",
+                    left, right
+                )
             }
         }
     }
@@ -36,121 +154,462 @@ impl std::fmt::Display for GrammarError {
 impl std::error::Error for GrammarError {}
 
 impl FormatDecl {
-    pub fn first_set(&self, module: &FormatModule) -> Result<FirstSet, GrammarError> {
-        let mut traversal = Traversal::new(self.fmt_id);
+    pub fn first_set(&self, module: &FormatModule) -> Result<ByteSet, GrammarError> {
+        Ok(self.determinations(module)?.first_set)
+    }
+
+    pub fn is_nullable(&self, module: &FormatModule) -> Result<bool, GrammarError> {
+        Ok(self.determinations(module)?.is_nullable)
+    }
+
+    pub(crate) fn determinations(
+        &self,
+        module: &FormatModule,
+    ) -> Result<Determinations, GrammarError> {
         let ctx = module.get_ctx(self.fmt_id);
-        Ok(self.format.solve_first_set(module, &mut traversal, ctx)?.0)
+        Ok(self
+            .format
+            .solve_determinations(module, ctx)?)
     }
 }
 
 impl Format {
+    fn solve_determinations_sequence(
+        formats: &[Format],
+        module: &FormatModule,
+        ctx: RecurseCtx,
+    ) -> Result<Determinations, GrammarError> {
+        let mut det_seq = Determinations::zero();
+        for format in formats {
+            let det_format = format.solve_determinations(module, ctx)?;
+            det_seq = det_seq.merge_seq(det_format)?;
+        }
+        Ok(det_seq)
+    }
+
     /// Returns the first-set, along with `true` if the format is nullable and `false` otherwise
-    fn solve_first_set(
+    fn solve_determinations(
         &self,
         module: &FormatModule,
-        visited: &mut Traversal,
         ctx: RecurseCtx<'_>,
-    ) -> Result<(FirstSet, bool), GrammarError> {
+    ) -> Result<Determinations, GrammarError> {
         match self {
             Format::ItemVar(level) => {
                 let level = *level;
                 let ctx = module.get_ctx(level);
-                let mut visited = Traversal::new(level);
                 module
                     .get_format(level)
-                    .solve_first_set(module, &mut visited, ctx)
-            },
+                    .solve_determinations(module,  ctx)
+            }
             Format::RecVar(rec_ix) => {
-                let level = ctx.convert_rec_var(*rec_ix);
-                if visited.insert(level) {
-                    let ctx = ctx.enter(*rec_ix).0;
-                    ctx.get_format().unwrap().solve_first_set(module, visited, ctx)
-                } else {
-                    Err(GrammarError::EpsilonLoop {
-                        top: visited.orig_level,
-                        reentrant: *rec_ix,
-                    })
-                }
+                let ctx = ctx.enter(*rec_ix).0;
+                let ret = ctx
+                    .get_format()
+                    .unwrap()
+                    .solve_determinations(module, ctx)?;
+                Ok(ret)
             }
-            Format::Byte(set) => Ok((*set, false)),
-            Format::FailWith(..) => Ok((FirstSet::empty(), false)),
-            Format::EndOfInput => Ok((FirstSet::empty(), true)),
-            Format::Compute(..) => Ok((FirstSet::empty(), true)),
-            Format::Variant(.., format) => format.solve_first_set(module, visited, ctx),
+            Format::Byte(set) => {
+                Ok(Determinations {
+                    first_set: *set,
+                    is_productive: true,
+                    is_nullable: false,
+                    should_not_follow_set: ByteSet::empty(),
+                })
+            },
+            Format::FailWith(..) => Ok(Determinations::one()),
+            Format::Compute(..) => Ok(Determinations::zero()),
+            // NOTE - EOI cannot be followed with other formats, but such cases are unlikely to occur...
+            // REVIEW - if we add a should-not-follow or similar, this might need some thinking...
+            Format::EndOfInput => Ok(Determinations::zero()),
+            Format::Variant(.., format) => format.solve_determinations(module, ctx),
             Format::Union(formats) => {
-                let mut ret = ByteSet::empty();
-                let mut any_nullable = false;
+                let mut det = Determinations::one();
                 for format in formats {
-                    let (first_set, is_nullable) = format.solve_first_set(module, visited, ctx)?;
-                    ret = ret.union(&first_set);
-                    if is_nullable {
-                        if any_nullable {
-                            return Err(GrammarError::MultiNullUnion { format: self.clone() });
-                        }
-                        any_nullable = true;
-                    }
+                    let det_format = format.solve_determinations(module,  ctx)?;
+                    det = det.union(det_format)?;
                 }
-                Ok((ret, any_nullable))
+                Ok(det)
             }
             Format::Repeat(format) => {
-                let (first_set, is_nullable) = format.solve_first_set(module, visited, ctx)?;
-                if is_nullable {
-                    return Err(GrammarError::RepeatNullable { format: format.as_ref().clone() });
+                let det_format = format.solve_determinations(module, ctx)?;
+                if det_format.is_nullable {
+                    return Err(GrammarError::RepeatNullable {
+                        format: format.as_ref().clone(),
+                    });
                 }
-                Ok((first_set, true))
+                Ok(Determinations {
+                    is_nullable: true,
+                    should_not_follow_set: det_format.first_set,
+                    ..det_format
+                })
             }
             Format::Tuple(formats) | Format::Seq(formats) => {
-                let mut ret = ByteSet::empty();
-                let mut nullable = true;
-                for format in formats {
-                    if !ret.is_empty() { break; }
-                    let (first_set, is_nullable) = format.solve_first_set(module, visited, ctx)?;
-                    ret = ret.union(&first_set);
-                    nullable &= is_nullable;
-                    if !nullable {
-                        break;
-                    }
-                }
-                Ok((ret, nullable))
+                Format::solve_determinations_sequence(formats, module, ctx)
             }
             Format::Maybe(_cond, format) => {
-                let (first_set, _) = format.solve_first_set(module, visited, ctx)?;
-                Ok((first_set, true))
+                let det_format = format.solve_determinations(module, ctx)?;
+                Ok(Determinations {
+                    is_nullable: true,
+                    ..det_format
+                })
             }
         }
     }
 }
 
-
 use traversal::Traversal;
 mod traversal {
-    use std::collections::BTreeSet;
-
-    pub(super) struct Traversal {
+    use linked_hash_set::LinkedHashSet;
+    /// Semi-mutable traversal state for tracking which format-levels have been seen before and which are novel
+    pub struct Traversal {
         pub(super) orig_level: usize,
-        seen_levels: BTreeSet<usize>,
+        pub(super) seen_levels: LinkedHashSet<usize>,
     }
 
     impl Traversal {
-        pub(super) fn new(orig_level: usize) -> Self {
+        pub fn new(orig_level: usize) -> Self {
             Self {
                 orig_level,
-                seen_levels: BTreeSet::new(),
+                seen_levels: LinkedHashSet::new(),
             }
         }
 
         /// Returns `true` if the level has not yet been seen (including the original level)
-        pub(super) fn is_novel(&self, level: usize) -> bool {
+        pub fn is_novel(&self, level: usize) -> bool {
             level != self.orig_level && !self.seen_levels.contains(&level)
         }
 
-        /// Record a level as being seen, returning `true` if it is novel.
-        pub(super) fn insert(&mut self, level: usize) -> bool {
+        /// Immutable insert that returns the updated state and whether the level was novel
+        pub fn insert(&mut self, level: usize) -> bool {
             if level == self.orig_level {
                 return false;
             }
             self.seen_levels.insert(level)
         }
+
+        /// Removes the most-recently inserted level, to avoid double-counting between branches rather than
+        /// merely witnessing true cycles on a singular path.
+        pub fn escape(&mut self) {
+            let Some(_) = self.seen_levels.pop_back() else {
+                panic!("traversal stack is empty");
+            };
+        }
+
+        pub fn reset(&mut self) {
+            self.seen_levels.clear();
+        }
+    }
+}
+
+/// Representation of the right-justified remainder of a [`Format`] we have already
+/// consumed some number (possibly 0) bytes of.
+#[derive(PartialEq, Eq, Hash, Debug)]
+enum PartialFormat<'a> {
+    /// `ε`
+    Empty,
+    /// A full-format followed by a remnant
+    Cat(&'a Format, Rc<PartialFormat<'a>>),
+    /// A sequence of full-formats followed by a remnant
+    Sequence(&'a [Format], Rc<PartialFormat<'a>>),
+    /// Repeat the specified format zero or more times before processing a remnant
+    Repeat(&'a Format, Rc<PartialFormat<'a>>),
+}
+
+impl<'a> PartialFormat<'a> {
+    fn solve_determinations(
+        &self,
+        module: &'a FormatModule,
+        ctx: RecurseCtx<'a>,
+    ) -> Result<Determinations, GrammarError> {
+        match self {
+            PartialFormat::Empty => Ok(Determinations::zero()),
+            PartialFormat::Cat(format, remnant) => {
+                let det_format = format.solve_determinations(module, ctx)?;
+                let det_remnant = remnant.solve_determinations(module, ctx)?;
+                det_format.merge_seq(det_remnant)
+            }
+            PartialFormat::Sequence(formats, remnant) => {
+                let det_formats =
+                    Format::solve_determinations_sequence(formats, module, ctx)?;
+                let det_remnant = remnant.solve_determinations(module, ctx)?;
+                det_formats.merge_seq(det_remnant)
+            }
+            PartialFormat::Repeat(format, remnant) => {
+                let det_format = format.solve_determinations(module, ctx)?;
+                if det_format.is_nullable {
+                    return Err(GrammarError::RepeatNullable {
+                        format: (*format).clone(),
+                    });
+                }
+                let det_remnant = remnant.solve_determinations(module, ctx)?;
+                det_format.merge_seq(det_remnant)
+            }
+        }
+    }
+}
+
+pub struct Interpreter<'a> {
+    module: &'a FormatModule,
+}
+
+pub type PathTrace = Vec<Option<usize>>;
+
+#[derive(Debug)]
+pub enum InterpError {
+    NoParse,
+    DeadEnd {
+        start: usize,
+        trace: PathTrace,
+        byte: u8,
+        expects: ByteSet,
+    },
+    BadEpsilon {
+        expects: ByteSet,
+    },
+    Fail {
+        message: crate::Label,
+    },
+}
+
+impl std::fmt::Display for InterpError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            InterpError::NoParse => write!(f, "no format to parse"),
+            InterpError::DeadEnd {
+                start,
+                trace,
+                byte,
+                expects,
+            } => {
+                write!(
+                    f,
+                    "no valid path for byte `{byte:#02x}` ∉ {expects:?} (#{start}: {:?})",
+                    trace
+                )
+            }
+            InterpError::BadEpsilon { expects } => {
+                write!(
+                    f,
+                    "no epsilon-move at end-of-input (allowed next-byte: {:?})",
+                    expects
+                )
+            }
+            InterpError::Fail { message } => {
+                write!(f, "fail: {message}")
+            }
+        }
+    }
+}
+
+impl<'a> Interpreter<'a> {
+    pub fn new(module: &'a FormatModule) -> Self {
+        Self { module }
+    }
+
+    pub fn run_level(
+        &self,
+        level: usize,
+        input: ReadCtxt<'a>,
+    ) -> (PathTrace, ReadCtxt<'a>, Option<InterpError>) {
+        let ctx = self.module.get_ctx(level);
+        let format = self.module.get_format(level);
+        let mut parse = Rc::new(PartialFormat::Cat(format, Rc::new(PartialFormat::Empty)));
+        let mut trace = Vec::new();
+        let mut input = input;
+        loop {
+            if let Some((byte, new_input)) = input.read_byte() {
+                let mut visited = Traversal::new(level);
+                let result =
+                    self.parse_byte_from_partial_format(parse, byte, &mut trace, &mut visited, ctx);
+                match result {
+                    Err(e) => return (trace, input, Some(e)),
+                    Ok(new_parse) => {
+                        input = new_input;
+                        parse = new_parse;
+                        continue;
+                    }
+                }
+            } else {
+                match parse.solve_determinations(self.module, ctx) {
+                    Err(_) => panic!("failed to solve determinations: {:?}", parse),
+                    Ok(dets) => {
+                        if dets.is_nullable {
+                            return (trace, input, None);
+                        } else {
+                            return (
+                                trace,
+                                input,
+                                Some(InterpError::BadEpsilon {
+                                    expects: dets.first_set,
+                                }),
+                            );
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn parse_byte_from_format(
+        &self,
+        f: &'a Format,
+        remnant: Rc<PartialFormat<'a>>,
+        byte: u8,
+        trace: &mut PathTrace,
+        visited: &mut Traversal,
+        ctx: RecurseCtx<'a>,
+    ) -> Result<Rc<PartialFormat<'a>>, InterpError> {
+        match f {
+            Format::ItemVar(level) => {
+                let new_ctx = self.module.get_ctx(*level);
+                let format = &self.module.decls[*level].format;
+                self.parse_byte_from_format(format, remnant, byte, trace, visited, new_ctx)
+            }
+            Format::RecVar(rec_ix) => {
+                let level = ctx.convert_rec_var(*rec_ix);
+                if visited.insert(level) {
+                    let format = &self.module.decls[level].format;
+                    let (new_ctx, _) = ctx.enter(*rec_ix);
+                    let ret = self
+                        .parse_byte_from_format(format, remnant, byte, trace, visited, new_ctx)?;
+                    visited.reset();
+                    Ok(ret)
+                } else {
+                    unreachable!("left-recursion")
+                }
+            }
+            Format::FailWith(msg) => {
+                return Err(InterpError::Fail {
+                    message: msg.clone(),
+                });
+            }
+            Format::EndOfInput => todo!(),
+            Format::Byte(bs) => {
+                if bs.contains(byte) {
+                    Ok(remnant)
+                } else {
+                    Err(InterpError::DeadEnd {
+                        start: visited.orig_level,
+                        trace: trace.clone(),
+                        byte,
+                        expects: *bs,
+                    })
+                }
+            }
+            Format::Compute(_) => {
+                self.parse_byte_from_partial_format(remnant, byte, trace, visited, ctx)
+            }
+            Format::Variant(_, format) => {
+                self.parse_byte_from_format(format, remnant, byte, trace, visited, ctx)
+            }
+            Format::Union(branches) => {
+                let mut accepts = ByteSet::empty();
+                for (ix, branch) in branches.iter().enumerate() {
+                    let dets = branch
+                        .solve_determinations(self.module, ctx)
+                        .unwrap();
+                    if dets.first_set.contains(byte) {
+                        trace.push(Some(ix));
+                        return self
+                            .parse_byte_from_format(branch, remnant, byte, trace, visited, ctx);
+                    } else {
+                        accepts = accepts.union(&dets.first_set);
+                    }
+                }
+                Err(InterpError::DeadEnd {
+                    start: visited.orig_level,
+                    trace: trace.clone(),
+                    byte,
+                    expects: accepts,
+                })
+            }
+            Format::Repeat(format) => {
+                let dets = format
+                    .solve_determinations(self.module, ctx)
+                    .unwrap();
+                if dets.first_set.contains(byte) {
+                    let new_remnant = self.parse_byte_from_format(
+                        format,
+                        Rc::new(PartialFormat::Repeat(format, remnant)),
+                        byte,
+                        trace,
+                        visited,
+                        ctx,
+                    )?;
+                    Ok(new_remnant)
+                } else {
+                    self.parse_byte_from_partial_format(remnant, byte, trace, visited, ctx)
+                }
+            }
+            Format::Seq(formats) | Format::Tuple(formats) => {
+                let (format, rest) = formats.split_first().unwrap();
+                self.parse_byte_from_format(
+                    format,
+                    Rc::new(PartialFormat::Sequence(rest, remnant)),
+                    byte,
+                    trace,
+                    visited,
+                    ctx,
+                )
+            }
+            Format::Maybe(expr, format) => {
+                let present = expr.eval().unwrap_bool();
+                if present {
+                    self.parse_byte_from_format(format, remnant, byte, trace, visited, ctx)
+                } else {
+                    self.parse_byte_from_partial_format(remnant, byte, trace, visited, ctx)
+                }
+            }
+        }
+    }
+
+    pub fn parse_byte_from_partial_format(
+        &self,
+        parse: Rc<PartialFormat<'a>>,
+        byte: u8,
+        trace: &mut PathTrace,
+        visited: &mut Traversal,
+        ctx: RecurseCtx<'a>,
+    ) -> Result<Rc<PartialFormat<'a>>, InterpError> {
+        match parse.as_ref() {
+            PartialFormat::Empty => return Err(InterpError::NoParse),
+            PartialFormat::Cat(f, remnant) => {
+                self.parse_byte_from_format(f, remnant.clone(), byte, trace, visited, ctx)
+            }
+            PartialFormat::Sequence(formats, remnant) => {
+                if formats.is_empty() {
+                    self.parse_byte_from_partial_format(remnant.clone(), byte, trace, visited, ctx)
+                } else {
+                    let (format, rest) = formats.split_first().unwrap();
+                    self.parse_byte_from_format(
+                        format,
+                        Rc::new(PartialFormat::Sequence(rest, remnant.clone())),
+                        byte,
+                        trace,
+                        visited,
+                        ctx,
+                    )
+                }
+            }
+            PartialFormat::Repeat(format, remnant) => {
+                let dets = format
+                    .solve_determinations(self.module, ctx)
+                    .unwrap();
+                if !dets.first_set.contains(byte) {
+                    self.parse_byte_from_partial_format(remnant.clone(), byte, trace, visited, ctx)
+                } else {
+                    self.parse_byte_from_format(
+                        format,
+                        Rc::new(PartialFormat::Repeat(format, remnant.clone())),
+                        byte,
+                        trace,
+                        visited,
+                        ctx,
+                    )
+                }
+            }
+        }
     }
 }
 
@@ -159,12 +618,9 @@ mod tests {
     use super::*;
 
     #[test]
-    fn test_follow_set() {
-
-    }
+    fn test_follow_set() {}
 }
 
-
 /*
 /// A forest of [`FMatchTree`]s, one for each level of a [`FormatModule`] (with directly corresponding indices)
 #[derive(Debug)]
@@ -213,21 +669,6 @@ enum FollowSet<'a> {
     Branch(ByteSet, Rc<PartialFormat<'a>>),
 }
 
-/// Representation of the right-justified remainder of a [`Format`] we have already
-/// consumed some number (possibly 0) bytes of.
-#[derive(PartialEq, Eq, Hash, Debug)]
-enum PartialFormat<'a> {
-    /// `ε`
-    Empty,
-    /// A full-format followed by a remnant
-    Cat(&'a Format, Rc<PartialFormat<'a>>),
-    /// A sequence of full-formats followed by a remnant
-    Sequence(&'a [Format], Rc<PartialFormat<'a>>),
-    /// Repeat the specified format zero or more times before processing a remnant
-    Repeat(&'a Format, Rc<PartialFormat<'a>>),
-    /// Alternation over two partial-formats that shared a common prefix we already consumed
-    Union(Rc<PartialFormat<'a>>, Rc<PartialFormat<'a>>),
-}
 
 #[derive(Debug)]
 pub struct MatchBranch<'a> {

From eb5f32ea119c70369d8f0fc83c170724b0b9079c Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Wed, 28 May 2025 22:06:37 +1000
Subject: [PATCH 11/15] Quasi-fix for acceptance check

---
 experiments/doodle-rec/src/lib.rs             |  4 +-
 experiments/doodle-rec/src/matchtree.rs       |  2 +-
 .../doodle-rec/src/matchtree/forest.rs        | 93 ++++++++++++-------
 experiments/doodle-rec/test1.json             |  2 +-
 experiments/doodle-rec/test2.json             |  2 +-
 5 files changed, 63 insertions(+), 40 deletions(-)

diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index f9eafe13..d65354c7 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -62,8 +62,8 @@ impl<'a> RecurseCtx<'a> {
         }
     }
 
-    pub fn convert_rec_var(&self, ix: RecId) -> FormatId {
-        self.as_span().unwrap().index(ix)
+    pub fn convert_rec_var(&self, ix: RecId) -> Option<FormatId> {
+        self.as_span().map(|span| span.index(ix))
     }
 
     /// Returns the global format-level of the closest entry-point
diff --git a/experiments/doodle-rec/src/matchtree.rs b/experiments/doodle-rec/src/matchtree.rs
index c653cb4e..7f54707f 100644
--- a/experiments/doodle-rec/src/matchtree.rs
+++ b/experiments/doodle-rec/src/matchtree.rs
@@ -150,7 +150,7 @@ impl<'a> MatchTreeStep<'a> {
             Format::Compute(_expr) => Self::from_next(module, next, ctx),
             Format::RecVar(rec_ix) => {
                 // FIXME - we discard the original `next` argument here
-                let next = Rc::new(Next::DelayRef(ctx.convert_rec_var(*rec_ix) /* , next.clone() */));
+                let next = Rc::new(Next::DelayRef(ctx.convert_rec_var(*rec_ix).unwrap() /* , next.clone() */));
                 Self::from_next(module, next, ctx)
             }
         }
diff --git a/experiments/doodle-rec/src/matchtree/forest.rs b/experiments/doodle-rec/src/matchtree/forest.rs
index eb445e5c..d55d192e 100644
--- a/experiments/doodle-rec/src/matchtree/forest.rs
+++ b/experiments/doodle-rec/src/matchtree/forest.rs
@@ -166,10 +166,11 @@ impl FormatDecl {
         &self,
         module: &FormatModule,
     ) -> Result<Determinations, GrammarError> {
+        let mut traversal = Traversal::new(self.fmt_id);
         let ctx = module.get_ctx(self.fmt_id);
         Ok(self
             .format
-            .solve_determinations(module, ctx)?)
+            .solve_determinations(module, &mut traversal, ctx)?)
     }
 }
 
@@ -177,11 +178,12 @@ impl Format {
     fn solve_determinations_sequence(
         formats: &[Format],
         module: &FormatModule,
+        visited: &mut Traversal,
         ctx: RecurseCtx,
     ) -> Result<Determinations, GrammarError> {
         let mut det_seq = Determinations::zero();
         for format in formats {
-            let det_format = format.solve_determinations(module, ctx)?;
+            let det_format = format.solve_determinations(module, visited, ctx)?;
             det_seq = det_seq.merge_seq(det_format)?;
         }
         Ok(det_seq)
@@ -191,23 +193,34 @@ impl Format {
     fn solve_determinations(
         &self,
         module: &FormatModule,
+        visited: &mut Traversal,
         ctx: RecurseCtx<'_>,
     ) -> Result<Determinations, GrammarError> {
         match self {
             Format::ItemVar(level) => {
                 let level = *level;
                 let ctx = module.get_ctx(level);
+                let mut visited = Traversal::new(level);
                 module
                     .get_format(level)
-                    .solve_determinations(module,  ctx)
+                    .solve_determinations(module, &mut visited, ctx)
             }
             Format::RecVar(rec_ix) => {
-                let ctx = ctx.enter(*rec_ix).0;
-                let ret = ctx
-                    .get_format()
-                    .unwrap()
-                    .solve_determinations(module, ctx)?;
-                Ok(ret)
+                let level = ctx.convert_rec_var(*rec_ix).unwrap_or_else(|| {
+                    unreachable!("solve_determinations: {ctx:?} has no recursive variable ~{rec_ix} (visited: {:#?})", visited.seen_levels.iter().copied().collect::<Vec<usize>>());
+                });
+                if visited.insert(level) {
+                    let ctx = ctx.enter(*rec_ix).0;
+                    let ret = ctx
+                        .get_format()
+                        .unwrap()
+                        .solve_determinations(module, visited, ctx)?;
+                    visited.escape();
+                    Ok(ret)
+                } else {
+                    // REVIEW - loop-breaker fall-back
+                    Ok(Determinations::zero())
+                }
             }
             Format::Byte(set) => {
                 Ok(Determinations {
@@ -222,17 +235,17 @@ impl Format {
             // NOTE - EOI cannot be followed with other formats, but such cases are unlikely to occur...
             // REVIEW - if we add a should-not-follow or similar, this might need some thinking...
             Format::EndOfInput => Ok(Determinations::zero()),
-            Format::Variant(.., format) => format.solve_determinations(module, ctx),
+            Format::Variant(.., format) => format.solve_determinations(module, visited, ctx),
             Format::Union(formats) => {
                 let mut det = Determinations::one();
                 for format in formats {
-                    let det_format = format.solve_determinations(module,  ctx)?;
+                    let det_format = format.solve_determinations(module, visited, ctx)?;
                     det = det.union(det_format)?;
                 }
                 Ok(det)
             }
             Format::Repeat(format) => {
-                let det_format = format.solve_determinations(module, ctx)?;
+                let det_format = format.solve_determinations(module, visited, ctx)?;
                 if det_format.is_nullable {
                     return Err(GrammarError::RepeatNullable {
                         format: format.as_ref().clone(),
@@ -245,10 +258,10 @@ impl Format {
                 })
             }
             Format::Tuple(formats) | Format::Seq(formats) => {
-                Format::solve_determinations_sequence(formats, module, ctx)
+                Format::solve_determinations_sequence(formats, module, visited, ctx)
             }
             Format::Maybe(_cond, format) => {
-                let det_format = format.solve_determinations(module, ctx)?;
+                let det_format = format.solve_determinations(module, visited, ctx)?;
                 Ok(Determinations {
                     is_nullable: true,
                     ..det_format
@@ -320,29 +333,30 @@ impl<'a> PartialFormat<'a> {
     fn solve_determinations(
         &self,
         module: &'a FormatModule,
+        visited: &mut Traversal,
         ctx: RecurseCtx<'a>,
     ) -> Result<Determinations, GrammarError> {
         match self {
             PartialFormat::Empty => Ok(Determinations::zero()),
             PartialFormat::Cat(format, remnant) => {
-                let det_format = format.solve_determinations(module, ctx)?;
-                let det_remnant = remnant.solve_determinations(module, ctx)?;
+                let det_format = format.solve_determinations(module, visited, ctx)?;
+                let det_remnant = remnant.solve_determinations(module, visited, ctx)?;
                 det_format.merge_seq(det_remnant)
             }
             PartialFormat::Sequence(formats, remnant) => {
                 let det_formats =
-                    Format::solve_determinations_sequence(formats, module, ctx)?;
-                let det_remnant = remnant.solve_determinations(module, ctx)?;
+                    Format::solve_determinations_sequence(formats, module, visited, ctx)?;
+                let det_remnant = remnant.solve_determinations(module, visited, ctx)?;
                 det_formats.merge_seq(det_remnant)
             }
             PartialFormat::Repeat(format, remnant) => {
-                let det_format = format.solve_determinations(module, ctx)?;
+                let det_format = format.solve_determinations(module, visited, ctx)?;
                 if det_format.is_nullable {
                     return Err(GrammarError::RepeatNullable {
                         format: (*format).clone(),
                     });
                 }
-                let det_remnant = remnant.solve_determinations(module, ctx)?;
+                let det_remnant = remnant.solve_determinations(module, visited, ctx)?;
                 det_format.merge_seq(det_remnant)
             }
         }
@@ -370,12 +384,14 @@ pub enum InterpError {
     Fail {
         message: crate::Label,
     },
+    ExpectsEnd,
 }
 
 impl std::fmt::Display for InterpError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             InterpError::NoParse => write!(f, "no format to parse"),
+            InterpError::ExpectsEnd => write!(f, "EOI parse failed"),
             InterpError::DeadEnd {
                 start,
                 trace,
@@ -412,7 +428,7 @@ impl<'a> Interpreter<'a> {
         level: usize,
         input: ReadCtxt<'a>,
     ) -> (PathTrace, ReadCtxt<'a>, Option<InterpError>) {
-        let ctx = self.module.get_ctx(level);
+        let mut ctx = self.module.get_ctx(level);
         let format = self.module.get_format(level);
         let mut parse = Rc::new(PartialFormat::Cat(format, Rc::new(PartialFormat::Empty)));
         let mut trace = Vec::new();
@@ -424,14 +440,17 @@ impl<'a> Interpreter<'a> {
                     self.parse_byte_from_partial_format(parse, byte, &mut trace, &mut visited, ctx);
                 match result {
                     Err(e) => return (trace, input, Some(e)),
-                    Ok(new_parse) => {
+                    Ok((new_ctx, new_parse)) => {
                         input = new_input;
                         parse = new_parse;
+                        // NOTE - this is bugged but it should work for our JSON-lite
+                        ctx = new_ctx;
                         continue;
                     }
                 }
             } else {
-                match parse.solve_determinations(self.module, ctx) {
+                let mut visited = Traversal::new(level);
+                match parse.solve_determinations(self.module, &mut visited, ctx) {
                     Err(_) => panic!("failed to solve determinations: {:?}", parse),
                     Ok(dets) => {
                         if dets.is_nullable {
@@ -451,7 +470,7 @@ impl<'a> Interpreter<'a> {
         }
     }
 
-    pub fn parse_byte_from_format(
+    fn parse_byte_from_format(
         &self,
         f: &'a Format,
         remnant: Rc<PartialFormat<'a>>,
@@ -459,7 +478,7 @@ impl<'a> Interpreter<'a> {
         trace: &mut PathTrace,
         visited: &mut Traversal,
         ctx: RecurseCtx<'a>,
-    ) -> Result<Rc<PartialFormat<'a>>, InterpError> {
+    ) -> Result<(RecurseCtx<'a>, Rc<PartialFormat<'a>>), InterpError> {
         match f {
             Format::ItemVar(level) => {
                 let new_ctx = self.module.get_ctx(*level);
@@ -467,14 +486,16 @@ impl<'a> Interpreter<'a> {
                 self.parse_byte_from_format(format, remnant, byte, trace, visited, new_ctx)
             }
             Format::RecVar(rec_ix) => {
-                let level = ctx.convert_rec_var(*rec_ix);
+                let level = ctx.convert_rec_var(*rec_ix).unwrap_or_else(|| {
+                    panic!("recursion variable not found in {ctx:?}: {rec_ix}")
+                });
                 if visited.insert(level) {
                     let format = &self.module.decls[level].format;
                     let (new_ctx, _) = ctx.enter(*rec_ix);
-                    let ret = self
+                    let (ctx, ret) = self
                         .parse_byte_from_format(format, remnant, byte, trace, visited, new_ctx)?;
                     visited.reset();
-                    Ok(ret)
+                    Ok((ctx, ret))
                 } else {
                     unreachable!("left-recursion")
                 }
@@ -484,10 +505,12 @@ impl<'a> Interpreter<'a> {
                     message: msg.clone(),
                 });
             }
-            Format::EndOfInput => todo!(),
+            Format::EndOfInput => {
+                return Err(InterpError::ExpectsEnd)
+            }
             Format::Byte(bs) => {
                 if bs.contains(byte) {
-                    Ok(remnant)
+                    Ok((ctx, remnant))
                 } else {
                     Err(InterpError::DeadEnd {
                         start: visited.orig_level,
@@ -507,7 +530,7 @@ impl<'a> Interpreter<'a> {
                 let mut accepts = ByteSet::empty();
                 for (ix, branch) in branches.iter().enumerate() {
                     let dets = branch
-                        .solve_determinations(self.module, ctx)
+                        .solve_determinations(self.module, visited, ctx)
                         .unwrap();
                     if dets.first_set.contains(byte) {
                         trace.push(Some(ix));
@@ -526,7 +549,7 @@ impl<'a> Interpreter<'a> {
             }
             Format::Repeat(format) => {
                 let dets = format
-                    .solve_determinations(self.module, ctx)
+                    .solve_determinations(self.module, visited, ctx)
                     .unwrap();
                 if dets.first_set.contains(byte) {
                     let new_remnant = self.parse_byte_from_format(
@@ -564,14 +587,14 @@ impl<'a> Interpreter<'a> {
         }
     }
 
-    pub fn parse_byte_from_partial_format(
+    fn parse_byte_from_partial_format(
         &self,
         parse: Rc<PartialFormat<'a>>,
         byte: u8,
         trace: &mut PathTrace,
         visited: &mut Traversal,
         ctx: RecurseCtx<'a>,
-    ) -> Result<Rc<PartialFormat<'a>>, InterpError> {
+    ) -> Result<(RecurseCtx<'a>, Rc<PartialFormat<'a>>), InterpError> {
         match parse.as_ref() {
             PartialFormat::Empty => return Err(InterpError::NoParse),
             PartialFormat::Cat(f, remnant) => {
@@ -594,7 +617,7 @@ impl<'a> Interpreter<'a> {
             }
             PartialFormat::Repeat(format, remnant) => {
                 let dets = format
-                    .solve_determinations(self.module, ctx)
+                    .solve_determinations(self.module, visited, ctx)
                     .unwrap();
                 if !dets.first_set.contains(byte) {
                     self.parse_byte_from_partial_format(remnant.clone(), byte, trace, visited, ctx)
diff --git a/experiments/doodle-rec/test1.json b/experiments/doodle-rec/test1.json
index 0967ef42..9e26dfee 100644
--- a/experiments/doodle-rec/test1.json
+++ b/experiments/doodle-rec/test1.json
@@ -1 +1 @@
-{}
+{}
\ No newline at end of file
diff --git a/experiments/doodle-rec/test2.json b/experiments/doodle-rec/test2.json
index 5bdf5940..1edba81d 100644
--- a/experiments/doodle-rec/test2.json
+++ b/experiments/doodle-rec/test2.json
@@ -1 +1 @@
-{"hello":true,"world":1}
+{"hello":true,"world":1,}
\ No newline at end of file

From 73f690bae7fe706c2f8c97b6b234f35be9c1fcf7 Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Thu, 29 May 2025 11:27:57 +1000
Subject: [PATCH 12/15] Enrich PathTrace to represent repeats

---
 experiments/doodle-rec/src/decoder.rs         | 42 +++++++++++--------
 experiments/doodle-rec/src/helper.rs          |  9 +++-
 experiments/doodle-rec/src/lib.rs             | 23 ++++++----
 experiments/doodle-rec/src/main.rs            | 21 ++++++----
 experiments/doodle-rec/src/matchtree.rs       | 39 +++++++++++++----
 .../doodle-rec/src/matchtree/forest.rs        | 39 ++++++++++-------
 6 files changed, 111 insertions(+), 62 deletions(-)

diff --git a/experiments/doodle-rec/src/decoder.rs b/experiments/doodle-rec/src/decoder.rs
index 6b4b9af7..b44ed60e 100644
--- a/experiments/doodle-rec/src/decoder.rs
+++ b/experiments/doodle-rec/src/decoder.rs
@@ -1,11 +1,13 @@
+use serde::Serialize;
 use std::{
     collections::{HashMap, HashSet},
     rc::Rc,
 };
-use serde::Serialize;
 
 use crate::{
-    matchtree::{MatchTree, Next}, Arith, Expr, Format, FormatDecl, FormatId, FormatModule, FormatType, IntRel, Label, RecId, RecurseCtx, Span, Unary
+    Arith, Expr, Format, FormatDecl, FormatId, FormatModule, FormatType, IntRel, Label, RecId,
+    RecurseCtx, Span, Unary,
+    matchtree::{MatchTree, Next},
 };
 use anyhow::{Result as AResult, anyhow};
 use doodle::{IntWidth, byte_set::ByteSet, read::ReadCtxt};
@@ -99,13 +101,11 @@ impl<'a> Compiler<'a> {
         compiler.queue_compile(t, format, Rc::new(Next::Empty), batch);
         while let Some((f, next, n, batch)) = compiler.compile_queue.pop() {
             let f_ctx = match batch {
-                Some(span) => {
-                    RecurseCtx::Recurse {
-                        span,
-                        batch: &module.decls[span.start..=span.end],
-                        entry_id: n - span.start,
-                    }
-                }
+                Some(span) => RecurseCtx::Recurse {
+                    span,
+                    batch: &module.decls[span.start..=span.end],
+                    entry_id: n - span.start,
+                },
                 None => RecurseCtx::NonRec,
             };
             let d = compiler.compile_format(f, next, f_ctx)?;
@@ -143,7 +143,8 @@ impl<'a> Compiler<'a> {
             } else {
                 Rc::new(Next::Empty)
             };
-            self.compile_queue.push((&d.format, next, n + ix, Some(span)));
+            self.compile_queue
+                .push((&d.format, next, n + ix, Some(span)));
         }
         n + which_next
     }
@@ -178,9 +179,7 @@ impl<'a> Compiler<'a> {
                             let batch = &self.module.decls[span.start..=span.end];
                             self.queue_compile_batch(batch, level - span.start, next.clone(), span)
                         }
-                        None => {
-                            self.queue_compile(t, f, next.clone(), None)
-                        }
+                        None => self.queue_compile(t, f, next.clone(), None),
                     };
                     self.decoder_map.insert((*level, next.clone()), n);
                     n
@@ -560,13 +559,20 @@ mod tests {
     #[test]
     fn auto_recursive() -> AResult<()> {
         let peano = Format::Union(vec![
-            Format::Variant(Label::Borrowed("peanoZ"), Box::new(Format::Byte(ByteSet::from([b'Z'])))),
-            Format::Variant(Label::Borrowed("peanoS"), Box::new(Format::Tuple(vec![Format::Byte(ByteSet::from([b'S'])), Format::RecVar(0)]))),
+            Format::Variant(
+                Label::Borrowed("peanoZ"),
+                Box::new(Format::Byte(ByteSet::from([b'Z']))),
+            ),
+            Format::Variant(
+                Label::Borrowed("peanoS"),
+                Box::new(Format::Tuple(vec![
+                    Format::Byte(ByteSet::from([b'S'])),
+                    Format::RecVar(0),
+                ])),
+            ),
         ]);
         let mut module = FormatModule::new();
-        let frefs = module.declare_rec_formats(vec![
-            (Label::Borrowed("test.peano"), peano),
-        ]);
+        let frefs = module.declare_rec_formats(vec![(Label::Borrowed("test.peano"), peano)]);
         let f = Format::Tuple(vec![frefs[0].call(), Format::EndOfInput]);
         let program = Compiler::compile_program(&module, &f, RecurseCtx::NonRec)?;
         let input = ReadCtxt::new(b"SSSSZ");
diff --git a/experiments/doodle-rec/src/helper.rs b/experiments/doodle-rec/src/helper.rs
index 0401744f..bb1f78b3 100644
--- a/experiments/doodle-rec/src/helper.rs
+++ b/experiments/doodle-rec/src/helper.rs
@@ -1,5 +1,5 @@
-use doodle::prelude::ByteSet;
 use crate::{Expr, Format, Label};
+use doodle::prelude::ByteSet;
 
 pub fn tuple(formats: impl IntoIterator<Item = Format>) -> Format {
     Format::Tuple(formats.into_iter().collect())
@@ -10,7 +10,12 @@ pub fn is_byte(x: u8) -> Format {
 }
 
 pub fn alts<Name: Into<Label>>(branches: impl IntoIterator<Item = (Name, Format)>) -> Format {
-    Format::Union(branches.into_iter().map(|(name, f)| Format::Variant(name.into(), Box::new(f))).collect())
+    Format::Union(
+        branches
+            .into_iter()
+            .map(|(name, f)| Format::Variant(name.into(), Box::new(f)))
+            .collect(),
+    )
 }
 
 pub fn byte_seq(bytes: impl IntoIterator<Item = u8>) -> Format {
diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index d65354c7..fc2fb53c 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -1,13 +1,18 @@
 pub mod decoder;
-pub(crate) mod matchtree;
 pub mod helper;
-pub(crate) use matchtree::{MatchTree, Next};
+pub(crate) mod matchtree;
 pub use matchtree::forest::Interpreter;
+pub(crate) use matchtree::{MatchTree, Next};
 
 use anyhow::{Result as AResult, anyhow};
 use doodle::{bounds::Bounds, byte_set::ByteSet};
 use std::{
-    borrow::Cow, cell::OnceCell, cmp::Ordering, collections::{BTreeMap, HashSet}, ops::{Add as _, RangeInclusive}, rc::Rc
+    borrow::Cow,
+    cell::OnceCell,
+    cmp::Ordering,
+    collections::{BTreeMap, HashSet},
+    ops::{Add as _, RangeInclusive},
+    rc::Rc,
 };
 
 pub type Label = Cow<'static, str>;
@@ -18,7 +23,6 @@ pub type FormatId = usize;
 /// Local index into a Batch of formats (e.g. 0 would be 'self' in a singleton-batch)
 pub type RecId = usize;
 
-
 #[derive(Debug, Clone, Copy, Default)]
 pub enum RecurseCtx<'a> {
     #[default]
@@ -70,9 +74,7 @@ impl<'a> RecurseCtx<'a> {
     pub fn get_level(&self) -> Option<usize> {
         match self {
             RecurseCtx::NonRec => None,
-            RecurseCtx::Recurse {
-                span, entry_id, ..
-            } => Some(span.index(*entry_id)),
+            RecurseCtx::Recurse { span, entry_id, .. } => Some(span.index(*entry_id)),
         }
     }
 
@@ -409,7 +411,11 @@ impl Format {
         }
     }
 
-    fn union_depends_on_next<'a>(branches: &'a [Format], module: &'a FormatModule, ctx: RecurseCtx<'a>) -> bool {
+    fn union_depends_on_next<'a>(
+        branches: &'a [Format],
+        module: &'a FormatModule,
+        ctx: RecurseCtx<'a>,
+    ) -> bool {
         let mut fs = Vec::with_capacity(branches.len());
         for f in branches {
             if f.depends_on_next(module, ctx) {
@@ -644,7 +650,6 @@ impl FormatModule {
         }
     }
 
-
     pub fn declare_rec_formats(&mut self, formats: Vec<(Label, Format)>) -> Vec<FormatRef> {
         let fmt_id = self.decls.len();
         let batch_size = formats.len();
diff --git a/experiments/doodle-rec/src/main.rs b/experiments/doodle-rec/src/main.rs
index e15bb889..9c91c0fa 100644
--- a/experiments/doodle-rec/src/main.rs
+++ b/experiments/doodle-rec/src/main.rs
@@ -1,15 +1,18 @@
 use std::{io::Read, path::PathBuf};
 
-use doodle::{prelude::ByteSet, read::ReadCtxt};
-use doodle_rec::{helper::*, Format, FormatModule, FormatRef, Interpreter, Label};
 use clap::Parser;
+use doodle::{prelude::ByteSet, read::ReadCtxt};
+use doodle_rec::{Format, FormatModule, FormatRef, Interpreter, Label, helper::*};
 
 fn surround(before: u8, after: u8, f: Format) -> Format {
     Format::Tuple(vec![is_byte(before), f, is_byte(after)])
 }
 
 fn alpha() -> Format {
-    Format::Byte(ByteSet::union(&ByteSet::from(b'a'..b'z'), &ByteSet::from(b'A'..b'Z')))
+    Format::Byte(ByteSet::union(
+        &ByteSet::from(b'a'..b'z'),
+        &ByteSet::from(b'A'..b'Z'),
+    ))
 }
 
 /// Restricted JSON object
@@ -35,11 +38,13 @@ fn json_lite(module: &mut FormatModule) -> FormatRef {
     let obj2 = surround(b'{', b'}', repeat(tuple([var(1), is_byte(b',')])));
     let arr3 = surround(b'[', b']', repeat(tuple([is_byte(b','), var(0)])));
     let decls = module.declare_rec_formats(
-        [(Label::Borrowed("json.value"), value0),
-         (Label::Borrowed("json.key_value"), kv1),
-         (Label::Borrowed("json.object"), obj2),
-         (Label::Borrowed("json.array"), arr3),
-        ].to_vec()
+        [
+            (Label::Borrowed("json.value"), value0),
+            (Label::Borrowed("json.key_value"), kv1),
+            (Label::Borrowed("json.object"), obj2),
+            (Label::Borrowed("json.array"), arr3),
+        ]
+        .to_vec(),
     );
     decls[0]
 }
diff --git a/experiments/doodle-rec/src/matchtree.rs b/experiments/doodle-rec/src/matchtree.rs
index 7f54707f..ed783530 100644
--- a/experiments/doodle-rec/src/matchtree.rs
+++ b/experiments/doodle-rec/src/matchtree.rs
@@ -69,7 +69,7 @@ impl<'a> MatchTreeStep<'a> {
         module: &'a FormatModule,
         fields: &'a [Format],
         next: Rc<Next<'a>>,
-        ctx: RecurseCtx<'a>
+        ctx: RecurseCtx<'a>,
     ) -> MatchTreeStep<'a> {
         match fields.split_first() {
             None => Self::from_next(module, next, ctx),
@@ -78,7 +78,11 @@ impl<'a> MatchTreeStep<'a> {
     }
 
     /// Constructs a [MatchTreeStep] from a [`Next`]
-    fn from_next(module: &'a FormatModule, next: Rc<Next<'a>>, ctx: RecurseCtx<'a>) -> MatchTreeStep<'a> {
+    fn from_next(
+        module: &'a FormatModule,
+        next: Rc<Next<'a>>,
+        ctx: RecurseCtx<'a>,
+    ) -> MatchTreeStep<'a> {
         match next.as_ref() {
             Next::Empty => Self::accept(),
             Next::Union(next1, next2) => {
@@ -150,7 +154,9 @@ impl<'a> MatchTreeStep<'a> {
             Format::Compute(_expr) => Self::from_next(module, next, ctx),
             Format::RecVar(rec_ix) => {
                 // FIXME - we discard the original `next` argument here
-                let next = Rc::new(Next::DelayRef(ctx.convert_rec_var(*rec_ix).unwrap() /* , next.clone() */));
+                let next = Rc::new(Next::DelayRef(
+                    ctx.convert_rec_var(*rec_ix).unwrap(), /* , next.clone() */
+                ));
                 Self::from_next(module, next, ctx)
             }
         }
@@ -257,7 +263,12 @@ impl<'a> MatchTreeLevel<'a> {
     ///
     /// Otherwise, returns a `MatchTree` that is guaranteed to decide on a unique branch for
     /// all input within at most `depth` bytes of lookahead.
-    fn grow(module: &'a FormatModule, nexts: LevelBranch<'a>, depth: usize, ctx: RecurseCtx<'a>) -> Option<MatchTree> {
+    fn grow(
+        module: &'a FormatModule,
+        nexts: LevelBranch<'a>,
+        depth: usize,
+        ctx: RecurseCtx<'a>,
+    ) -> Option<MatchTree> {
         if let Some(tree) = Self::accepts(&nexts) {
             Some(tree)
         } else if depth > 0 {
@@ -348,14 +359,24 @@ mod tests {
     #[test]
     fn construct_autorec_next() {
         let peano = Format::Union(vec![
-            Format::Variant(Label::Borrowed("peanoZ"), Box::new(Format::Byte(ByteSet::from([b'Z'])))),
-            Format::Variant(Label::Borrowed("peanoS"), Box::new(Format::Tuple(vec![Format::Byte(ByteSet::from([b'S'])), Format::RecVar(0)]))),
+            Format::Variant(
+                Label::Borrowed("peanoZ"),
+                Box::new(Format::Byte(ByteSet::from([b'Z']))),
+            ),
+            Format::Variant(
+                Label::Borrowed("peanoS"),
+                Box::new(Format::Tuple(vec![
+                    Format::Byte(ByteSet::from([b'S'])),
+                    Format::RecVar(0),
+                ])),
+            ),
         ]);
         let mut module = FormatModule::new();
-        let frefs = module.declare_rec_formats(vec![
-            (Label::Borrowed("test.peano"), peano),
+        let frefs = module.declare_rec_formats(vec![(Label::Borrowed("test.peano"), peano)]);
+        let f = Format::Tuple(vec![
+            Format::ItemVar(frefs[0].get_level()),
+            Format::EndOfInput,
         ]);
-        let f = Format::Tuple(vec![Format::ItemVar(frefs[0].get_level()), Format::EndOfInput]);
         let ctx = RecurseCtx::NonRec;
         let tree = MatchTreeStep::from_format(&module, &f, Rc::new(Next::Empty), ctx);
         eprintln!("{tree:?}")
diff --git a/experiments/doodle-rec/src/matchtree/forest.rs b/experiments/doodle-rec/src/matchtree/forest.rs
index d55d192e..bbdec047 100644
--- a/experiments/doodle-rec/src/matchtree/forest.rs
+++ b/experiments/doodle-rec/src/matchtree/forest.rs
@@ -222,14 +222,12 @@ impl Format {
                     Ok(Determinations::zero())
                 }
             }
-            Format::Byte(set) => {
-                Ok(Determinations {
-                    first_set: *set,
-                    is_productive: true,
-                    is_nullable: false,
-                    should_not_follow_set: ByteSet::empty(),
-                })
-            },
+            Format::Byte(set) => Ok(Determinations {
+                first_set: *set,
+                is_productive: true,
+                is_nullable: false,
+                should_not_follow_set: ByteSet::empty(),
+            }),
             Format::FailWith(..) => Ok(Determinations::one()),
             Format::Compute(..) => Ok(Determinations::zero()),
             // NOTE - EOI cannot be followed with other formats, but such cases are unlikely to occur...
@@ -367,7 +365,14 @@ pub struct Interpreter<'a> {
     module: &'a FormatModule,
 }
 
-pub type PathTrace = Vec<Option<usize>>;
+pub type PathTrace = Vec<Choice>;
+
+#[derive(Debug, Clone)]
+pub enum Choice {
+    UnionBranch(usize),
+    RepeatYes,
+    RepeatNo,
+}
 
 #[derive(Debug)]
 pub enum InterpError {
@@ -486,9 +491,9 @@ impl<'a> Interpreter<'a> {
                 self.parse_byte_from_format(format, remnant, byte, trace, visited, new_ctx)
             }
             Format::RecVar(rec_ix) => {
-                let level = ctx.convert_rec_var(*rec_ix).unwrap_or_else(|| {
-                    panic!("recursion variable not found in {ctx:?}: {rec_ix}")
-                });
+                let level = ctx
+                    .convert_rec_var(*rec_ix)
+                    .unwrap_or_else(|| panic!("recursion variable not found in {ctx:?}: {rec_ix}"));
                 if visited.insert(level) {
                     let format = &self.module.decls[level].format;
                     let (new_ctx, _) = ctx.enter(*rec_ix);
@@ -505,9 +510,7 @@ impl<'a> Interpreter<'a> {
                     message: msg.clone(),
                 });
             }
-            Format::EndOfInput => {
-                return Err(InterpError::ExpectsEnd)
-            }
+            Format::EndOfInput => return Err(InterpError::ExpectsEnd),
             Format::Byte(bs) => {
                 if bs.contains(byte) {
                     Ok((ctx, remnant))
@@ -533,7 +536,7 @@ impl<'a> Interpreter<'a> {
                         .solve_determinations(self.module, visited, ctx)
                         .unwrap();
                     if dets.first_set.contains(byte) {
-                        trace.push(Some(ix));
+                        trace.push(Choice::UnionBranch(ix));
                         return self
                             .parse_byte_from_format(branch, remnant, byte, trace, visited, ctx);
                     } else {
@@ -552,6 +555,7 @@ impl<'a> Interpreter<'a> {
                     .solve_determinations(self.module, visited, ctx)
                     .unwrap();
                 if dets.first_set.contains(byte) {
+                    trace.push(Choice::RepeatYes);
                     let new_remnant = self.parse_byte_from_format(
                         format,
                         Rc::new(PartialFormat::Repeat(format, remnant)),
@@ -562,6 +566,7 @@ impl<'a> Interpreter<'a> {
                     )?;
                     Ok(new_remnant)
                 } else {
+                    trace.push(Choice::RepeatNo);
                     self.parse_byte_from_partial_format(remnant, byte, trace, visited, ctx)
                 }
             }
@@ -620,8 +625,10 @@ impl<'a> Interpreter<'a> {
                     .solve_determinations(self.module, visited, ctx)
                     .unwrap();
                 if !dets.first_set.contains(byte) {
+                    trace.push(Choice::RepeatNo);
                     self.parse_byte_from_partial_format(remnant.clone(), byte, trace, visited, ctx)
                 } else {
+                    trace.push(Choice::RepeatYes);
                     self.parse_byte_from_format(
                         format,
                         Rc::new(PartialFormat::Repeat(format, remnant.clone())),

From 5338d4ea60c8c1c4a697cc9ca47b06e1d31d217a Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Thu, 29 May 2025 14:09:46 +1000
Subject: [PATCH 13/15] Add context to GrammarError, patch json-lite

---
 experiments/doodle-rec/src/helper.rs          |  13 +-
 experiments/doodle-rec/src/lib.rs             |   6 +-
 experiments/doodle-rec/src/main.rs            |  22 +-
 experiments/doodle-rec/src/matchtree.rs       |   2 +-
 .../{forest.rs => determinations.rs}          | 384 ++++++++++++++----
 experiments/doodle-rec/test2.json             |   2 +-
 6 files changed, 340 insertions(+), 89 deletions(-)
 rename experiments/doodle-rec/src/matchtree/{forest.rs => determinations.rs} (75%)

diff --git a/experiments/doodle-rec/src/helper.rs b/experiments/doodle-rec/src/helper.rs
index bb1f78b3..8934a4d0 100644
--- a/experiments/doodle-rec/src/helper.rs
+++ b/experiments/doodle-rec/src/helper.rs
@@ -1,4 +1,4 @@
-use crate::{Expr, Format, Label};
+use crate::{Format, Label};
 use doodle::prelude::ByteSet;
 
 pub fn tuple(formats: impl IntoIterator<Item = Format>) -> Format {
@@ -30,6 +30,13 @@ pub fn var(ix: usize) -> Format {
     Format::RecVar(ix)
 }
 
-pub fn repeat1(format: Format) -> Format {
-    tuple([format.clone(), repeat(format.clone())])
+pub fn fmt_variant<Name: Into<Label>>(name: Name, format: Format) -> Format {
+    Format::Variant(name.into(), Box::new(format))
+}
+
+pub fn optional(format: Format) -> Format {
+    Format::Union(vec![
+        fmt_variant("no", Format::EMPTY),
+        fmt_variant("yes", format),
+    ])
 }
diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index fc2fb53c..5fdc7226 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -1,7 +1,7 @@
 pub mod decoder;
-pub mod helper;
 pub(crate) mod matchtree;
-pub use matchtree::forest::Interpreter;
+pub use matchtree::determinations;
+pub mod helper;
 pub(crate) use matchtree::{MatchTree, Next};
 
 use anyhow::{Result as AResult, anyhow};
@@ -132,7 +132,7 @@ impl<Idx: Copy> From<RangeInclusive<Idx>> for Span<Idx> {
 #[derive(Debug, Clone)]
 pub struct FormatDecl {
     format: Format,
-    fmt_id: FormatId,
+    pub fmt_id: FormatId,
     f_type: Rc<OnceCell<FormatType>>,
     batch: Option<Span<FormatId>>,
 }
diff --git a/experiments/doodle-rec/src/main.rs b/experiments/doodle-rec/src/main.rs
index 9c91c0fa..69a48e66 100644
--- a/experiments/doodle-rec/src/main.rs
+++ b/experiments/doodle-rec/src/main.rs
@@ -2,12 +2,20 @@ use std::{io::Read, path::PathBuf};
 
 use clap::Parser;
 use doodle::{prelude::ByteSet, read::ReadCtxt};
-use doodle_rec::{Format, FormatModule, FormatRef, Interpreter, Label, helper::*};
+use doodle_rec::{Format, FormatModule, FormatRef, Label, determinations::Interpreter, helper::*};
 
 fn surround(before: u8, after: u8, f: Format) -> Format {
     Format::Tuple(vec![is_byte(before), f, is_byte(after)])
 }
 
+fn once_then_repeat(a: Format, b: Format) -> Format {
+    Format::Tuple(vec![a, Format::Repeat(Box::new(b))])
+}
+
+fn repeat1(a: Format) -> Format {
+    once_then_repeat(a.clone(), a)
+}
+
 fn alpha() -> Format {
     Format::Byte(ByteSet::union(
         &ByteSet::from(b'a'..b'z'),
@@ -35,8 +43,16 @@ fn json_lite(module: &mut FormatModule) -> FormatRef {
         ("JArr", Format::RecVar(3)),
     ]);
     let kv1 = tuple([key_prefix, var(0)]);
-    let obj2 = surround(b'{', b'}', repeat(tuple([var(1), is_byte(b',')])));
-    let arr3 = surround(b'[', b']', repeat(tuple([is_byte(b','), var(0)])));
+    let obj2 = surround(
+        b'{',
+        b'}',
+        optional(once_then_repeat(var(1), tuple([is_byte(b','), var(1)]))),
+    );
+    let arr3 = surround(
+        b'[',
+        b']',
+        optional(once_then_repeat(var(0), tuple([is_byte(b','), var(0)]))),
+    );
     let decls = module.declare_rec_formats(
         [
             (Label::Borrowed("json.value"), value0),
diff --git a/experiments/doodle-rec/src/matchtree.rs b/experiments/doodle-rec/src/matchtree.rs
index ed783530..55d686a1 100644
--- a/experiments/doodle-rec/src/matchtree.rs
+++ b/experiments/doodle-rec/src/matchtree.rs
@@ -3,7 +3,7 @@ use std::{collections::HashSet, rc::Rc, vec};
 use crate::{Format, FormatModule, RecurseCtx};
 use doodle::{byte_set::ByteSet, read::ReadCtxt};
 
-pub mod forest;
+pub mod determinations;
 // pub use forest::MatchForest;
 
 #[derive(Clone, Debug)]
diff --git a/experiments/doodle-rec/src/matchtree/forest.rs b/experiments/doodle-rec/src/matchtree/determinations.rs
similarity index 75%
rename from experiments/doodle-rec/src/matchtree/forest.rs
rename to experiments/doodle-rec/src/matchtree/determinations.rs
index bbdec047..c6ac04b8 100644
--- a/experiments/doodle-rec/src/matchtree/forest.rs
+++ b/experiments/doodle-rec/src/matchtree/determinations.rs
@@ -5,16 +5,264 @@ use doodle::{prelude::ByteSet, read::ReadCtxt};
 use crate::{Format, FormatDecl, FormatId, FormatModule, RecId, RecurseCtx};
 
 #[derive(Debug)]
-pub enum GrammarError {
-    LeftRecursion { top: FormatId, cycle: Vec<FormatId> },
-    RepeatNullable { format: Format },
-    AmbiguousFirst { left: ByteSet, right: ByteSet },
-    MultiNullUnion,
-    AmbiguousFollow { left: ByteSet, right: ByteSet },
+pub enum GrammarError<CtxValue: std::fmt::Debug + Sized = ()> {
+    LeftRecursion {
+        top: FormatId,
+        cycle: Vec<FormatId>,
+        context: CtxValue,
+    },
+    RepeatNullable {
+        format: Format,
+        context: CtxValue,
+    },
+    AmbiguousFirst {
+        left: ByteSet,
+        right: ByteSet,
+        context: CtxValue,
+    },
+    MultiNullUnion {
+        context: CtxValue,
+    },
+    AmbiguousFollow {
+        left: ByteSet,
+        right: ByteSet,
+        context: CtxValue,
+    },
+}
+
+mod private {
+    pub trait Sealed {}
+    impl Sealed for crate::Format {}
+    impl<'a> Sealed for std::rc::Rc<super::PartialFormat<'a>> {}
+    impl<'a> Sealed for super::FormatKind<'a> {}
+}
+
+trait IsFormat: private::Sealed + Clone + std::fmt::Debug + Sized {}
+
+impl IsFormat for Format {}
+impl<'a> IsFormat for Rc<PartialFormat<'a>> {}
+impl<'a> IsFormat for FormatKind<'a> {}
+
+#[derive(Clone)]
+enum FormatKind<'a> {
+    Total(Format),
+    Partial(Rc<PartialFormat<'a>>),
+}
+
+impl<'a> std::fmt::Debug for FormatKind<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FormatKind::Total(format) => write!(f, "{:?}", format),
+            FormatKind::Partial(partial_format) => write!(f, "{:?}", partial_format),
+        }
+    }
+}
+
+impl<'a> From<Format> for FormatKind<'a> {
+    fn from(format: Format) -> Self {
+        FormatKind::Total(format)
+    }
+}
+
+impl<'a> From<Rc<PartialFormat<'a>>> for FormatKind<'a> {
+    fn from(partial_format: Rc<PartialFormat<'a>>) -> Self {
+        FormatKind::Partial(partial_format)
+    }
+}
+
+impl<'a> From<GrammarError<Format>> for GrammarError<FormatKind<'a>> {
+    fn from(err: GrammarError<Format>) -> Self {
+        match err {
+            GrammarError::LeftRecursion {
+                top,
+                cycle,
+                context,
+            } => GrammarError::LeftRecursion {
+                top,
+                cycle,
+                context: context.into(),
+            },
+            GrammarError::RepeatNullable { format, context } => GrammarError::RepeatNullable {
+                format,
+                context: context.into(),
+            },
+            GrammarError::AmbiguousFirst {
+                left,
+                right,
+                context,
+            } => GrammarError::AmbiguousFirst {
+                left,
+                right,
+                context: context.into(),
+            },
+            GrammarError::MultiNullUnion { context } => GrammarError::MultiNullUnion {
+                context: context.into(),
+            },
+            GrammarError::AmbiguousFollow {
+                left,
+                right,
+                context,
+            } => GrammarError::AmbiguousFollow {
+                left,
+                right,
+                context: context.into(),
+            },
+        }
+    }
+}
+
+impl<'a> From<GrammarError<Rc<PartialFormat<'a>>>> for GrammarError<FormatKind<'a>> {
+    fn from(err: GrammarError<Rc<PartialFormat<'a>>>) -> Self {
+        match err {
+            GrammarError::LeftRecursion {
+                top,
+                cycle,
+                context,
+            } => GrammarError::LeftRecursion {
+                top,
+                cycle,
+                context: context.into(),
+            },
+            GrammarError::RepeatNullable { format, context } => GrammarError::RepeatNullable {
+                format,
+                context: context.into(),
+            },
+            GrammarError::AmbiguousFirst {
+                left,
+                right,
+                context,
+            } => GrammarError::AmbiguousFirst {
+                left,
+                right,
+                context: context.into(),
+            },
+            GrammarError::MultiNullUnion { context } => GrammarError::MultiNullUnion {
+                context: context.into(),
+            },
+            GrammarError::AmbiguousFollow {
+                left,
+                right,
+                context,
+            } => GrammarError::AmbiguousFollow {
+                left,
+                right,
+                context: context.into(),
+            },
+        }
+    }
+}
+
+impl<F: IsFormat> std::fmt::Display for GrammarError<F> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            GrammarError::LeftRecursion {
+                top,
+                cycle,
+                context,
+            } => {
+                write!(
+                    f,
+                    "left-recursion found in format ({context:?}): #{} -> {:#?}",
+                    top, cycle,
+                )
+            }
+            GrammarError::RepeatNullable { format, context } => {
+                write!(f, "repeat of nullable format ({context:?}): {:?}", format)
+            }
+            GrammarError::MultiNullUnion { context } => {
+                write!(f, "multiple nullable formats in union ({context:?}")
+            }
+            GrammarError::AmbiguousFirst {
+                left,
+                right,
+                context,
+            } => {
+                write!(
+                    f,
+                    "ambiguity introduced by union of non-disjoint first sets ({context:?}): {:?} <|> {:?}",
+                    left, right
+                )
+            }
+            GrammarError::AmbiguousFollow {
+                left,
+                right,
+                context,
+            } => {
+                write!(
+                    f,
+                    "follow set and first set conflict ({context:?}): {:?} & {:?} ",
+                    left, right
+                )
+            }
+        }
+    }
+}
+
+impl std::fmt::Display for GrammarError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            GrammarError::LeftRecursion { top, cycle, .. } => {
+                write!(
+                    f,
+                    "left-recursion found in format: #{} -> {:#?}",
+                    top, cycle,
+                )
+            }
+            GrammarError::RepeatNullable { format, .. } => {
+                write!(f, "repeat of nullable format: {:?}", format)
+            }
+            GrammarError::MultiNullUnion { .. } => {
+                write!(f, "multiple nullable formats in union")
+            }
+            GrammarError::AmbiguousFirst { left, right, .. } => {
+                write!(
+                    f,
+                    "ambiguity introduced by union of non-disjoint first sets: {:?} <|> {:?}",
+                    left, right
+                )
+            }
+            GrammarError::AmbiguousFollow { left, right, .. } => {
+                write!(
+                    f,
+                    "follow set and first set conflict: {:?} & {:?} ",
+                    left, right
+                )
+            }
+        }
+    }
+}
+
+impl std::error::Error for GrammarError {}
+impl<F: IsFormat> std::error::Error for GrammarError<F> {}
+
+impl GrammarError<()> {
+    fn add_context<F: IsFormat>(self, context: F) -> GrammarError<F> {
+        match self {
+            GrammarError::LeftRecursion { top, cycle, .. } => GrammarError::LeftRecursion {
+                top,
+                cycle,
+                context,
+            },
+            GrammarError::RepeatNullable { format, .. } => {
+                GrammarError::RepeatNullable { format, context }
+            }
+            GrammarError::AmbiguousFirst { left, right, .. } => GrammarError::AmbiguousFirst {
+                left,
+                right,
+                context,
+            },
+            GrammarError::MultiNullUnion { .. } => GrammarError::MultiNullUnion { context },
+            GrammarError::AmbiguousFollow { left, right, .. } => GrammarError::AmbiguousFollow {
+                left,
+                right,
+                context,
+            },
+        }
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub(crate) struct Determinations {
+pub struct Determinations {
     pub is_nullable: bool,
     pub is_productive: bool,
     pub first_set: ByteSet,
@@ -48,6 +296,7 @@ impl Determinations {
             return Err(GrammarError::AmbiguousFollow {
                 left: self.should_not_follow_set,
                 right: other.first_set,
+                context: (),
             });
         }
         let is_nullable = self.is_nullable && other.is_nullable;
@@ -88,12 +337,13 @@ impl Determinations {
 
     pub fn union(self, other: Self) -> Result<Self, GrammarError> {
         if self.is_nullable && other.is_nullable {
-            return Err(GrammarError::MultiNullUnion);
+            return Err(GrammarError::MultiNullUnion { context: () });
         }
         if !self.first_set.is_disjoint(&other.first_set) {
             return Err(GrammarError::AmbiguousFirst {
                 left: self.first_set,
                 right: other.first_set,
+                context: (),
             });
         }
         let first_set = self.first_set.union(&other.first_set);
@@ -117,55 +367,19 @@ impl Determinations {
     }
 }
 
-impl std::fmt::Display for GrammarError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            GrammarError::LeftRecursion { top, cycle } => {
-                write!(
-                    f,
-                    "left-recursion found in format: #{} -> {:#?}",
-                    top, cycle,
-                )
-            }
-            GrammarError::RepeatNullable { format } => {
-                write!(f, "repeat of nullable format: {:?}", format)
-            }
-            GrammarError::MultiNullUnion => {
-                write!(f, "multiple nullable formats in union")
-            }
-            GrammarError::AmbiguousFirst { left, right } => {
-                write!(
-                    f,
-                    "ambiguity introduced by union of non-disjoint first sets: {:?} <|> {:?}",
-                    left, right
-                )
-            }
-            GrammarError::AmbiguousFollow { left, right } => {
-                write!(
-                    f,
-                    "follow set and first set conflict: {:?} & {:?} ",
-                    left, right
-                )
-            }
-        }
-    }
-}
-
-impl std::error::Error for GrammarError {}
-
 impl FormatDecl {
-    pub fn first_set(&self, module: &FormatModule) -> Result<ByteSet, GrammarError> {
+    pub fn first_set(&self, module: &FormatModule) -> Result<ByteSet, GrammarError<Format>> {
         Ok(self.determinations(module)?.first_set)
     }
 
-    pub fn is_nullable(&self, module: &FormatModule) -> Result<bool, GrammarError> {
+    pub fn is_nullable(&self, module: &FormatModule) -> Result<bool, GrammarError<Format>> {
         Ok(self.determinations(module)?.is_nullable)
     }
 
-    pub(crate) fn determinations(
+    pub fn determinations(
         &self,
         module: &FormatModule,
-    ) -> Result<Determinations, GrammarError> {
+    ) -> Result<Determinations, GrammarError<Format>> {
         let mut traversal = Traversal::new(self.fmt_id);
         let ctx = module.get_ctx(self.fmt_id);
         Ok(self
@@ -175,27 +389,13 @@ impl FormatDecl {
 }
 
 impl Format {
-    fn solve_determinations_sequence(
-        formats: &[Format],
-        module: &FormatModule,
-        visited: &mut Traversal,
-        ctx: RecurseCtx,
-    ) -> Result<Determinations, GrammarError> {
-        let mut det_seq = Determinations::zero();
-        for format in formats {
-            let det_format = format.solve_determinations(module, visited, ctx)?;
-            det_seq = det_seq.merge_seq(det_format)?;
-        }
-        Ok(det_seq)
-    }
-
     /// Returns the first-set, along with `true` if the format is nullable and `false` otherwise
     fn solve_determinations(
         &self,
         module: &FormatModule,
         visited: &mut Traversal,
         ctx: RecurseCtx<'_>,
-    ) -> Result<Determinations, GrammarError> {
+    ) -> Result<Determinations, GrammarError<Format>> {
         match self {
             Format::ItemVar(level) => {
                 let level = *level;
@@ -219,7 +419,7 @@ impl Format {
                     Ok(ret)
                 } else {
                     // REVIEW - loop-breaker fall-back
-                    Ok(Determinations::zero())
+                    Ok(Determinations::one())
                 }
             }
             Format::Byte(set) => Ok(Determinations {
@@ -238,7 +438,9 @@ impl Format {
                 let mut det = Determinations::one();
                 for format in formats {
                     let det_format = format.solve_determinations(module, visited, ctx)?;
-                    det = det.union(det_format)?;
+                    det = det
+                        .union(det_format)
+                        .map_err(|e| e.add_context(self.clone()))?;
                 }
                 Ok(det)
             }
@@ -247,6 +449,7 @@ impl Format {
                 if det_format.is_nullable {
                     return Err(GrammarError::RepeatNullable {
                         format: format.as_ref().clone(),
+                        context: self.clone(),
                     });
                 }
                 Ok(Determinations {
@@ -256,7 +459,14 @@ impl Format {
                 })
             }
             Format::Tuple(formats) | Format::Seq(formats) => {
-                Format::solve_determinations_sequence(formats, module, visited, ctx)
+                let mut det_seq = Determinations::zero();
+                for format in formats {
+                    let det_format = format.solve_determinations(module, visited, ctx)?;
+                    det_seq = det_seq
+                        .merge_seq(det_format)
+                        .map_err(|e| e.add_context(self.clone()))?;
+                }
+                Ok(det_seq)
             }
             Format::Maybe(_cond, format) => {
                 let det_format = format.solve_determinations(module, visited, ctx)?;
@@ -329,33 +539,48 @@ enum PartialFormat<'a> {
 
 impl<'a> PartialFormat<'a> {
     fn solve_determinations(
-        &self,
+        self: Rc<Self>,
         module: &'a FormatModule,
         visited: &mut Traversal,
         ctx: RecurseCtx<'a>,
-    ) -> Result<Determinations, GrammarError> {
-        match self {
+    ) -> Result<Determinations, GrammarError<FormatKind<'a>>> {
+        match self.as_ref() {
             PartialFormat::Empty => Ok(Determinations::zero()),
             PartialFormat::Cat(format, remnant) => {
                 let det_format = format.solve_determinations(module, visited, ctx)?;
-                let det_remnant = remnant.solve_determinations(module, visited, ctx)?;
-                det_format.merge_seq(det_remnant)
+                let det_remnant = remnant.clone().solve_determinations(module, visited, ctx)?;
+                det_format
+                    .merge_seq(det_remnant)
+                    .map_err(|e| e.add_context(self.clone().into()))
             }
             PartialFormat::Sequence(formats, remnant) => {
-                let det_formats =
-                    Format::solve_determinations_sequence(formats, module, visited, ctx)?;
-                let det_remnant = remnant.solve_determinations(module, visited, ctx)?;
-                det_formats.merge_seq(det_remnant)
+                let det_formats = {
+                    let mut det_seq = Determinations::zero();
+                    for format in *formats {
+                        let det_format = format.solve_determinations(module, visited, ctx)?;
+                        det_seq = det_seq
+                            .merge_seq(det_format)
+                            .map_err(|e| e.add_context(self.clone()))?;
+                    }
+                    det_seq
+                };
+                let det_remnant = remnant.clone().solve_determinations(module, visited, ctx)?;
+                det_formats
+                    .merge_seq(det_remnant)
+                    .map_err(|e| e.add_context(self.clone().into()))
             }
             PartialFormat::Repeat(format, remnant) => {
                 let det_format = format.solve_determinations(module, visited, ctx)?;
                 if det_format.is_nullable {
                     return Err(GrammarError::RepeatNullable {
                         format: (*format).clone(),
+                        context: self.clone().into(),
                     });
                 }
-                let det_remnant = remnant.solve_determinations(module, visited, ctx)?;
-                det_format.merge_seq(det_remnant)
+                let det_remnant = remnant.clone().solve_determinations(module, visited, ctx)?;
+                det_format
+                    .merge_seq(det_remnant)
+                    .map_err(|e| e.add_context(self.clone().into()))
             }
         }
     }
@@ -455,7 +680,10 @@ impl<'a> Interpreter<'a> {
                 }
             } else {
                 let mut visited = Traversal::new(level);
-                match parse.solve_determinations(self.module, &mut visited, ctx) {
+                match parse
+                    .clone()
+                    .solve_determinations(self.module, &mut visited, ctx)
+                {
                     Err(_) => panic!("failed to solve determinations: {:?}", parse),
                     Ok(dets) => {
                         if dets.is_nullable {
diff --git a/experiments/doodle-rec/test2.json b/experiments/doodle-rec/test2.json
index 1edba81d..ca4cc6fd 100644
--- a/experiments/doodle-rec/test2.json
+++ b/experiments/doodle-rec/test2.json
@@ -1 +1 @@
-{"hello":true,"world":1,}
\ No newline at end of file
+{"hello":true,"world":1}
\ No newline at end of file

From f07a3b390c815ecbc8eae51a10da356126288681 Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Thu, 5 Jun 2025 17:05:42 +1000
Subject: [PATCH 14/15] Add parser, printer to doodle-rec

---
 experiments/doodle-rec/src/decoder.rs         |  64 +-
 experiments/doodle-rec/src/decoder/ll1.rs     | 264 +++++++
 experiments/doodle-rec/src/lib.rs             |  19 +-
 experiments/doodle-rec/src/main.rs            |  26 +-
 .../src/matchtree/determinations.rs           |  29 +-
 experiments/doodle-rec/src/output/mod.rs      | 720 ++++++++++++++++++
 src/lib.rs                                    |   2 +-
 src/output.rs                                 |  18 +-
 src/output/tree.rs                            |   4 +-
 src/precedence.rs                             |  68 +-
 10 files changed, 1127 insertions(+), 87 deletions(-)
 create mode 100644 experiments/doodle-rec/src/decoder/ll1.rs
 create mode 100644 experiments/doodle-rec/src/output/mod.rs

diff --git a/experiments/doodle-rec/src/decoder.rs b/experiments/doodle-rec/src/decoder.rs
index b44ed60e..df4442c5 100644
--- a/experiments/doodle-rec/src/decoder.rs
+++ b/experiments/doodle-rec/src/decoder.rs
@@ -12,6 +12,9 @@ use crate::{
 use anyhow::{Result as AResult, anyhow};
 use doodle::{IntWidth, byte_set::ByteSet, read::ReadCtxt};
 
+mod ll1;
+pub use ll1::LL1Interpreter;
+
 #[derive(Debug, Clone, Serialize)]
 pub enum Value {
     // Primitive values
@@ -27,9 +30,66 @@ pub enum Value {
     Seq(Vec<Value>),
     Option(Option<Box<Value>>),
     Variant(Label, Box<Value>),
+
+    Branch(usize, Box<Value>),
+}
+
+const MAX_SEQ_LEN: usize = 64;
+
+impl std::fmt::Display for Value {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Value::Bool(b) => write!(f, "{}", b),
+            Value::U8(i) => write!(f, "{}", i),
+            Value::U16(i) => write!(f, "{}", i),
+            Value::U32(i) => write!(f, "{}", i),
+            Value::U64(i) => write!(f, "{}", i),
+            Value::Char(c) => write!(f, "{:?}", c),
+
+            Value::Option(v) => match v {
+                None => write!(f, "None"),
+                Some(v) => write!(f, "Some({})", v),
+            },
+            Value::Tuple(vs) => {
+                write!(
+                    f,
+                    "({})",
+                    vs.iter()
+                        .map(|v| v.to_string())
+                        .collect::<Vec<_>>()
+                        .join(", ")
+                )
+            }
+            Value::Variant(label, value) => {
+                write!(f, "`{}({})", label, value)
+            }
+            Value::Seq(vs) => {
+                if vs.len() > MAX_SEQ_LEN {
+                    write!(f, "[...; {}]", vs.len())
+                } else {
+                    write!(
+                        f,
+                        "[{}]",
+                        vs.iter()
+                            .map(|v| v.to_string())
+                            .collect::<Vec<_>>()
+                            .join(", ")
+                    )
+                }
+            }
+            Value::Branch(_, v) => write!(f, "{}", v),
+        }
+    }
 }
 
 impl Value {
+    pub fn coerce_value(&self) -> &Value {
+        match self {
+            Value::Branch(_, v) => v.coerce_value(),
+            _ => self,
+        }
+    }
+
     fn get_usize_with_precision(&self) -> (usize, IntWidth) {
         match self {
             Value::U8(n) => (*n as usize, IntWidth::Bits8),
@@ -64,7 +124,7 @@ impl Program {
     }
 }
 
-type Batch = Option<Span<usize>>;
+pub(crate) type Batch = Option<Span<usize>>;
 
 pub struct Compiler<'a> {
     module: &'a FormatModule,
@@ -187,7 +247,7 @@ impl<'a> Compiler<'a> {
                 Ok(Decoder::Call(n))
             }
             Format::RecVar(batch_ix) => {
-                let (new_ctx, _) = ctx.enter(*batch_ix);
+                let new_ctx = ctx.enter(*batch_ix);
                 let level = new_ctx.get_level().unwrap();
                 // REVIEW - do we need to do any work here?
                 Ok(Decoder::CallRec(level, *batch_ix))
diff --git a/experiments/doodle-rec/src/decoder/ll1.rs b/experiments/doodle-rec/src/decoder/ll1.rs
new file mode 100644
index 00000000..e07e6428
--- /dev/null
+++ b/experiments/doodle-rec/src/decoder/ll1.rs
@@ -0,0 +1,264 @@
+use std::rc::Rc;
+
+use doodle::prelude::ByteSet;
+use doodle::read::ReadCtxt;
+
+use super::Value;
+use crate::{
+    Format, FormatModule, RecurseCtx,
+    determinations::{Choice, InterpError, PartialFormat, PathTrace, Traversal},
+};
+
+pub struct LL1Interpreter<'a> {
+    module: &'a FormatModule,
+}
+
+impl<'a> LL1Interpreter<'a> {
+    pub fn new(module: &'a FormatModule) -> Self {
+        Self { module }
+    }
+
+    pub fn parse_level(
+        &self,
+        level: usize,
+        input: ReadCtxt<'a>,
+    ) -> Result<(Value, ReadCtxt), InterpError> {
+        let ctx = self.module.get_ctx(level);
+        let format = self.module.get_format(level);
+        let mut trace = PathTrace::new();
+        let mut visited = Traversal::new(level);
+        self.parse_format(
+            format,
+            Rc::new(PartialFormat::Empty),
+            ctx,
+            input,
+            &mut trace,
+            &mut visited,
+        )
+    }
+
+    fn parse_format(
+        &self,
+        format: &'a Format,
+        remnant: Rc<PartialFormat<'a>>,
+        ctx: RecurseCtx<'a>,
+        input: ReadCtxt<'a>,
+        trace: &mut PathTrace,
+        visited: &mut Traversal,
+    ) -> Result<(Value, ReadCtxt<'a>), InterpError> {
+        match format {
+            Format::ItemVar(level) => {
+                let f = self.module.get_format(*level);
+                let ctx = self.module.get_ctx(*level);
+                self.parse_format(f, remnant, ctx, input, trace, visited)
+            }
+            Format::RecVar(rec_ix) => {
+                let level = ctx
+                    .convert_rec_var(*rec_ix)
+                    .unwrap_or_else(|| panic!("recursion variable not found in {ctx:?}: {rec_ix}"));
+                if visited.insert(level) {
+                    let new_ctx = ctx.enter(*rec_ix);
+                    let format = new_ctx.get_format().unwrap();
+                    let ret = self.parse_format(format, remnant, new_ctx, input, trace, visited)?;
+                    let _ = visited.escape();
+                    Ok(ret)
+                } else {
+                    unreachable!("left recursion")
+                }
+            }
+            Format::FailWith(msg) => {
+                return Err(InterpError::Fail {
+                    message: msg.clone(),
+                });
+            }
+            Format::EndOfInput => {
+                let b = input.read_byte();
+                if b.is_none() {
+                    Ok((Value::Tuple(vec![]), input))
+                } else {
+                    Err(InterpError::ExpectsEnd)
+                }
+            }
+            Format::Byte(bs) => {
+                let (b, input) = input
+                    .read_byte()
+                    .ok_or(InterpError::BadEpsilon { expects: *bs })?;
+                if bs.contains(b) {
+                    visited.reset();
+                    Ok((Value::U8(b), input))
+                } else {
+                    Err(InterpError::DeadEnd {
+                        start: visited.orig_level,
+                        trace: trace.clone(),
+                        byte: b,
+                        expects: *bs,
+                    })
+                }
+            }
+            Format::Compute(expr) => {
+                let val = expr.eval();
+                Ok((val, input))
+            }
+            Format::Variant(lab, format) => {
+                let (val, input) =
+                    self.parse_format(format, remnant, ctx, input, trace, visited)?;
+                Ok((Value::Variant(lab.clone(), Box::new(val)), input))
+            }
+            Format::Union(formats) => {
+                let mut branches: Vec<ByteSet> = Vec::with_capacity(formats.len());
+                let mut accept = None;
+                for (ix, branch) in formats.iter().enumerate() {
+                    let mut _visited = Traversal::new(visited.orig_level);
+                    let dets = branch
+                        .solve_determinations(self.module, &mut _visited, ctx)
+                        .unwrap();
+                    if dets.is_nullable {
+                        if let Some(ix0) = accept.replace(ix) {
+                            unreachable!("multiple nullable branches: {ix0}, {ix}");
+                        };
+                    }
+                    branches.push(dets.first_set);
+                }
+                match input.read_byte() {
+                    None => match accept {
+                        None => {
+                            let expects = branches
+                                .iter()
+                                .fold(ByteSet::empty(), |acc, bs| acc.union(bs));
+                            Err(InterpError::BadEpsilon { expects })
+                        }
+                        Some(ix) => {
+                            let (val, input) = self.parse_format(
+                                &formats[ix],
+                                remnant,
+                                ctx,
+                                input,
+                                trace,
+                                visited,
+                            )?;
+                            Ok((val, input))
+                        }
+                    },
+                    Some((byte, _input)) => {
+                        for (ix, bs) in branches.iter().enumerate() {
+                            if bs.contains(byte) {
+                                let (val, input) = self.parse_format(
+                                    &formats[ix],
+                                    remnant,
+                                    ctx,
+                                    input,
+                                    trace,
+                                    visited,
+                                )?;
+                                return Ok((Value::Branch(ix, Box::new(val)), input));
+                            }
+                        }
+                        match accept {
+                            None => {
+                                let expects = branches
+                                    .iter()
+                                    .fold(ByteSet::empty(), |acc, bs| acc.union(bs));
+                                Err(InterpError::BadEpsilon { expects })
+                            }
+                            Some(ix) => {
+                                let (val, input) = self.parse_format(
+                                    &formats[ix],
+                                    remnant,
+                                    ctx,
+                                    input,
+                                    trace,
+                                    visited,
+                                )?;
+                                Ok((Value::Branch(ix, Box::new(val)), input))
+                            }
+                        }
+                    }
+                }
+            }
+            Format::Repeat(format0) => {
+                let mut values = Vec::new();
+                let mut input = input;
+                let dets = format0
+                    .solve_determinations(self.module, visited, ctx)
+                    .unwrap();
+                if dets.is_nullable {
+                    unreachable!("bad repeat of nullable format: {format:?}");
+                }
+                let dets_next = {
+                    let mut visited = Traversal::new(visited.orig_level);
+                    remnant
+                        .clone()
+                        .solve_determinations(self.module, &mut visited, ctx)
+                        .unwrap()
+                };
+                loop {
+                    match input.read_byte() {
+                        None => {
+                            if dets_next.is_nullable {
+                                break;
+                            } else {
+                                return Err(InterpError::BadEpsilon {
+                                    expects: dets_next.first_set.union(&dets.first_set),
+                                });
+                            }
+                        }
+                        Some((byte, _)) => {
+                            if dets.first_set.contains(byte) {
+                                trace.push(Choice::RepeatYes);
+                                let remnant0 =
+                                    Rc::new(PartialFormat::Repeat(format0, remnant.clone()));
+                                let (val, new_input) = self
+                                    .parse_format(format0, remnant0, ctx, input, trace, visited)?;
+                                values.push(val);
+                                input = new_input;
+                                continue;
+                            } else {
+                                trace.push(Choice::RepeatNo);
+                                break;
+                            }
+                        }
+                    }
+                }
+                return Ok((Value::Seq(values), input));
+            }
+            Format::Seq(formats) => {
+                let mut values = Vec::with_capacity(formats.len());
+                let mut input = input;
+                let mut iter = formats.iter();
+                while let Some(format) = iter.next() {
+                    let remnant0 =
+                        Rc::new(PartialFormat::Sequence(iter.as_slice(), remnant.clone()));
+                    let (val, new_input) =
+                        self.parse_format(format, remnant0, ctx, input, trace, visited)?;
+                    values.push(val);
+                    input = new_input;
+                }
+                return Ok((Value::Seq(values), input));
+            }
+            Format::Tuple(formats) => {
+                let mut values = Vec::with_capacity(formats.len());
+                let mut input = input;
+                let mut iter = formats.iter();
+                while let Some(format) = iter.next() {
+                    let remnant0 =
+                        Rc::new(PartialFormat::Sequence(iter.as_slice(), remnant.clone()));
+                    let (val, new_input) =
+                        self.parse_format(format, remnant0, ctx, input, trace, visited)?;
+                    values.push(val);
+                    input = new_input;
+                }
+                return Ok((Value::Tuple(values), input));
+            }
+            Format::Maybe(expr, format) => {
+                let present = expr.eval().unwrap_bool();
+                if present {
+                    let (val, input) =
+                        self.parse_format(format, remnant, ctx, input, trace, visited)?;
+                    Ok((Value::Option(Some(Box::new(val))), input))
+                } else {
+                    Ok((Value::Option(None), input))
+                }
+            }
+        }
+    }
+}
diff --git a/experiments/doodle-rec/src/lib.rs b/experiments/doodle-rec/src/lib.rs
index 5fdc7226..5f32cacb 100644
--- a/experiments/doodle-rec/src/lib.rs
+++ b/experiments/doodle-rec/src/lib.rs
@@ -3,13 +3,13 @@ pub(crate) mod matchtree;
 pub use matchtree::determinations;
 pub mod helper;
 pub(crate) use matchtree::{MatchTree, Next};
+pub mod output;
 
 use anyhow::{Result as AResult, anyhow};
 use doodle::{bounds::Bounds, byte_set::ByteSet};
 use std::{
     borrow::Cow,
     cell::OnceCell,
-    cmp::Ordering,
     collections::{BTreeMap, HashSet},
     ops::{Add as _, RangeInclusive},
     rc::Rc,
@@ -47,21 +47,16 @@ impl<'a> RecurseCtx<'a> {
     }
 
     /// Returns `(new_ctx, is_auto)`
-    pub fn enter(&self, ix: RecId) -> (Self, Ordering) {
+    pub fn enter(&self, ix: RecId) -> Self {
         match self {
             RecurseCtx::NonRec => panic!("cannot recurse into non-recursive context"),
-            RecurseCtx::Recurse {
-                batch,
-                span,
-                entry_id,
-            } => {
+            RecurseCtx::Recurse { batch, span, .. } => {
                 assert!(ix < batch.len(), "batch index out of range");
-                let ret = RecurseCtx::Recurse {
+                RecurseCtx::Recurse {
                     entry_id: ix,
                     span: *span,
                     batch,
-                };
-                (ret, ix.cmp(entry_id))
+                }
             }
         }
     }
@@ -718,6 +713,10 @@ impl FormatModule {
     fn get_batch(&self, level: usize) -> Option<Span<FormatId>> {
         self.decls[level].batch
     }
+
+    fn get_name(&self, level: usize) -> &Label {
+        &self.names[level]
+    }
 }
 
 #[cfg(test)]
diff --git a/experiments/doodle-rec/src/main.rs b/experiments/doodle-rec/src/main.rs
index 69a48e66..1cb14370 100644
--- a/experiments/doodle-rec/src/main.rs
+++ b/experiments/doodle-rec/src/main.rs
@@ -2,7 +2,9 @@ use std::{io::Read, path::PathBuf};
 
 use clap::Parser;
 use doodle::{prelude::ByteSet, read::ReadCtxt};
-use doodle_rec::{Format, FormatModule, FormatRef, Label, determinations::Interpreter, helper::*};
+use doodle_rec::{
+    Format, FormatModule, FormatRef, Label, decoder::LL1Interpreter, helper::*, output::PPrinter,
+};
 
 fn surround(before: u8, after: u8, f: Format) -> Format {
     Format::Tuple(vec![is_byte(before), f, is_byte(after)])
@@ -32,7 +34,7 @@ fn alpha() -> Format {
 /// - all numbers are natural
 /// - trailing commas are mandatory
 fn json_lite(module: &mut FormatModule) -> FormatRef {
-    let key = surround(b'"', b'"', repeat1(alpha()));
+    let key = surround(b'"', b'"', repeat(alpha()));
     let key_prefix = tuple([key, is_byte(b':')]);
     let value0 = alts([
         ("JNull", byte_seq(*b"null")),
@@ -83,21 +85,13 @@ fn main() -> Result<(), Box<dyn Send + Sync + std::error::Error + 'static>> {
         Label::Borrowed("main"),
         Format::Tuple(vec![json.call(), Format::EndOfInput]),
     );
-    let interpreter = Interpreter::new(&module);
+    let interpreter = LL1Interpreter::new(&module);
 
     let input = ReadCtxt::new(&buffer);
-    let (trace, rest, opt_err) = interpreter.run_level(main.get_level(), input);
-    match opt_err {
-        None => {
-            println!("input accepted!");
-            println!("trace: {trace:?}");
-            println!("remainder: {:?}", rest.remaining());
-        }
-        Some(err) => {
-            println!("input rejected: {err:?}");
-            println!("trace: {trace:?}");
-            println!("remainder: {:?}", rest.remaining());
-        }
-    }
+    let (val, _) = interpreter.parse_level(main.get_level(), input)?;
+    let mut printer = PPrinter::new(&module);
+    let ctx = module.get_ctx(main.get_level());
+    let oput = printer.compile_decoded_value(&val, &main.call(), ctx);
+    println!("{}", oput);
     Ok(())
 }
diff --git a/experiments/doodle-rec/src/matchtree/determinations.rs b/experiments/doodle-rec/src/matchtree/determinations.rs
index c6ac04b8..ab73bcf0 100644
--- a/experiments/doodle-rec/src/matchtree/determinations.rs
+++ b/experiments/doodle-rec/src/matchtree/determinations.rs
@@ -1,8 +1,8 @@
-use std::{collections::HashSet, rc::Rc};
+use std::rc::Rc;
 
 use doodle::{prelude::ByteSet, read::ReadCtxt};
 
-use crate::{Format, FormatDecl, FormatId, FormatModule, RecId, RecurseCtx};
+use crate::{Format, FormatDecl, FormatId, FormatModule, RecurseCtx};
 
 #[derive(Debug)]
 pub enum GrammarError<CtxValue: std::fmt::Debug + Sized = ()> {
@@ -44,7 +44,7 @@ impl<'a> IsFormat for Rc<PartialFormat<'a>> {}
 impl<'a> IsFormat for FormatKind<'a> {}
 
 #[derive(Clone)]
-enum FormatKind<'a> {
+pub(crate) enum FormatKind<'a> {
     Total(Format),
     Partial(Rc<PartialFormat<'a>>),
 }
@@ -390,7 +390,7 @@ impl FormatDecl {
 
 impl Format {
     /// Returns the first-set, along with `true` if the format is nullable and `false` otherwise
-    fn solve_determinations(
+    pub(crate) fn solve_determinations(
         &self,
         module: &FormatModule,
         visited: &mut Traversal,
@@ -410,7 +410,7 @@ impl Format {
                     unreachable!("solve_determinations: {ctx:?} has no recursive variable ~{rec_ix} (visited: {:#?})", visited.seen_levels.iter().copied().collect::<Vec<usize>>());
                 });
                 if visited.insert(level) {
-                    let ctx = ctx.enter(*rec_ix).0;
+                    let ctx = ctx.enter(*rec_ix);
                     let ret = ctx
                         .get_format()
                         .unwrap()
@@ -479,12 +479,12 @@ impl Format {
     }
 }
 
-use traversal::Traversal;
+pub(crate) use traversal::Traversal;
 mod traversal {
     use linked_hash_set::LinkedHashSet;
     /// Semi-mutable traversal state for tracking which format-levels have been seen before and which are novel
     pub struct Traversal {
-        pub(super) orig_level: usize,
+        pub(crate) orig_level: usize,
         pub(super) seen_levels: LinkedHashSet<usize>,
     }
 
@@ -496,6 +496,7 @@ mod traversal {
             }
         }
 
+        #[expect(dead_code)]
         /// Returns `true` if the level has not yet been seen (including the original level)
         pub fn is_novel(&self, level: usize) -> bool {
             level != self.orig_level && !self.seen_levels.contains(&level)
@@ -511,10 +512,8 @@ mod traversal {
 
         /// Removes the most-recently inserted level, to avoid double-counting between branches rather than
         /// merely witnessing true cycles on a singular path.
-        pub fn escape(&mut self) {
-            let Some(_) = self.seen_levels.pop_back() else {
-                panic!("traversal stack is empty");
-            };
+        pub fn escape(&mut self) -> Option<usize> {
+            self.seen_levels.pop_back()
         }
 
         pub fn reset(&mut self) {
@@ -526,7 +525,7 @@ mod traversal {
 /// Representation of the right-justified remainder of a [`Format`] we have already
 /// consumed some number (possibly 0) bytes of.
 #[derive(PartialEq, Eq, Hash, Debug)]
-enum PartialFormat<'a> {
+pub(crate) enum PartialFormat<'a> {
     /// `ε`
     Empty,
     /// A full-format followed by a remnant
@@ -538,7 +537,7 @@ enum PartialFormat<'a> {
 }
 
 impl<'a> PartialFormat<'a> {
-    fn solve_determinations(
+    pub(crate) fn solve_determinations(
         self: Rc<Self>,
         module: &'a FormatModule,
         visited: &mut Traversal,
@@ -648,6 +647,8 @@ impl std::fmt::Display for InterpError {
     }
 }
 
+impl std::error::Error for InterpError {}
+
 impl<'a> Interpreter<'a> {
     pub fn new(module: &'a FormatModule) -> Self {
         Self { module }
@@ -724,7 +725,7 @@ impl<'a> Interpreter<'a> {
                     .unwrap_or_else(|| panic!("recursion variable not found in {ctx:?}: {rec_ix}"));
                 if visited.insert(level) {
                     let format = &self.module.decls[level].format;
-                    let (new_ctx, _) = ctx.enter(*rec_ix);
+                    let new_ctx = ctx.enter(*rec_ix);
                     let (ctx, ret) = self
                         .parse_byte_from_format(format, remnant, byte, trace, visited, new_ctx)?;
                     visited.reset();
diff --git a/experiments/doodle-rec/src/output/mod.rs b/experiments/doodle-rec/src/output/mod.rs
new file mode 100644
index 00000000..23234e29
--- /dev/null
+++ b/experiments/doodle-rec/src/output/mod.rs
@@ -0,0 +1,720 @@
+use std::rc::Rc;
+
+use crate::{Arith, Expr, Format, FormatModule, IntRel, RecurseCtx, Unary, decoder::Value};
+pub(crate) use doodle::output::Fragment;
+use doodle::{
+    output::{FragmentBuilder, Symbol},
+    precedence::{Precedence, cond_paren},
+};
+
+pub struct Flags {
+    omit_implied_values: bool,
+    pretty_quoted_strings: bool,
+    show_redundant_formats: bool,
+}
+
+enum Column {
+    Branch,
+    Space,
+}
+
+pub struct PPrinter<'module> {
+    gutter: Vec<Column>,
+    preview_len: Option<usize>,
+    flags: Flags,
+    module: &'module FormatModule,
+}
+
+impl<'module> PPrinter<'module> {
+    pub fn new(module: &'module FormatModule) -> Self {
+        let flags = Flags {
+            omit_implied_values: true,
+            pretty_quoted_strings: true,
+            show_redundant_formats: false,
+        };
+        Self {
+            gutter: Vec::new(),
+            preview_len: Some(10),
+            flags,
+            module,
+        }
+    }
+
+    pub fn compile_decoded_value(
+        &mut self,
+        value: &Value,
+        fmt: &Format,
+        ctx: RecurseCtx<'module>,
+    ) -> Fragment {
+        match fmt {
+            Format::ItemVar(level) => {
+                let ctx = self.module.get_ctx(*level);
+                self.compile_decoded_value(value, self.module.get_format(*level), ctx)
+            }
+            Format::RecVar(level) => {
+                let ctx = ctx.enter(*level);
+                self.compile_decoded_value(value, ctx.get_format().unwrap(), ctx)
+            }
+            Format::FailWith(..) => unreachable!("uninhabited format (value={value})"),
+            Format::EndOfInput => self.compile_value(value),
+            Format::Byte(_) => self.compile_value(value),
+            Format::Compute(_) => self.compile_value(value),
+            Format::Variant(label, format) => match value {
+                Value::Variant(label2, value) => {
+                    if label == label2 {
+                        self.compile_variant(label, value, Some((format, ctx)))
+                    } else {
+                        panic!("expected variant {label}, found {label2}")
+                    }
+                }
+                _ => panic!("expected variant, found {value}"),
+            },
+            Format::Union(formats) => match value {
+                Value::Branch(ix, value) => {
+                    let format = &formats[*ix];
+                    self.compile_decoded_value(value, format, ctx)
+                }
+                _ => panic!("expected branch, found {value:?}"),
+            },
+            Format::Repeat(format) => match value {
+                Value::Seq(values) => self.compile_seq(values, Some((format, ctx))),
+                _ => panic!("expected sequence, found {value}"),
+            },
+            Format::Seq(formats) => match value {
+                Value::Seq(values) => {
+                    self.compile_seq_formats(values, Some((formats.as_slice(), ctx)))
+                }
+                _ => panic!("expected sequence, found {value}"),
+            },
+            Format::Tuple(formats) => match value {
+                Value::Tuple(values) => match &values[..] {
+                    [Value::U8(b'"'), Value::Seq(str_contents), Value::U8(b'"')]
+                        if self.flags.pretty_quoted_strings
+                            && self.is_quoted_string(str_contents) =>
+                    {
+                        self.compile_quoted_string(str_contents)
+                    }
+                    _ => self.compile_tuple(values, Some((formats, ctx))),
+                },
+                _ => panic!("expected tuple, found {value}"),
+            },
+            Format::Maybe(_expr, format) => match value {
+                Value::Option(None) => Fragment::string("none"),
+                Value::Option(Some(value)) => {
+                    self.compile_variant("some", value, Some((format, ctx)))
+                }
+                _ => panic!("expected option, found {value}"),
+            },
+        }
+    }
+
+    fn compile_variant(
+        &mut self,
+        label: &str,
+        value: &Value,
+        format: Option<(&Format, RecurseCtx<'module>)>,
+    ) -> Fragment {
+        if self.flags.omit_implied_values
+            && format.is_some_and(|(format, ctx)| self.is_implied_value_format(format, ctx))
+        {
+            Fragment::string(label.to_string())
+        } else if self.is_atomic_value(value, format) {
+            let mut frag = Fragment::new();
+            frag.append(Fragment::String(format!("{{ {label} := ").into()));
+            if let Some((format, ctx)) = format {
+                frag.append(self.compile_decoded_value(value, format, ctx));
+            } else {
+                frag.append(self.compile_value(value));
+            }
+            frag.append(Fragment::String(" }".into()));
+            frag.enclose();
+            frag
+        } else {
+            self.compile_field_value_last(label, value, format, true)
+        }
+    }
+
+    fn compile_quoted_string(&mut self, str_contents: &[Value]) -> Fragment {
+        Fragment::seq(
+            str_contents.iter().map(|v| match v {
+                Value::U8(b) => Fragment::Char(*b as char),
+                Value::Char(c) => Fragment::Char(*c),
+                _ => panic!("expected U8 or Char value, found {v}"),
+            }),
+            None,
+        )
+        .delimit(Fragment::Char('\"'), Fragment::Char('\"'))
+    }
+
+    fn compile_tuple(
+        &mut self,
+        vals: &[Value],
+        formats: Option<(&[Format], RecurseCtx<'module>)>,
+    ) -> Fragment {
+        if vals.is_empty() {
+            Fragment::String("()".into())
+        } else {
+            let mut frag = Fragment::new();
+            let last_index = vals.len() - 1;
+            for index in 0..last_index {
+                frag.append(self.compile_field_value_continue(
+                    index,
+                    &vals[index],
+                    formats.map(|(fs, ctx)| (&fs[index], ctx)),
+                    true,
+                ));
+            }
+            frag.append(self.compile_field_value_last(
+                last_index,
+                &vals[last_index],
+                formats.map(|(fs, ctx)| (&fs[last_index], ctx)),
+                true,
+            ));
+            frag
+        }
+    }
+
+    fn compile_seq_formats(
+        &mut self,
+        vals: &[Value],
+        formats: Option<(&[Format], RecurseCtx<'module>)>,
+    ) -> Fragment {
+        if vals.is_empty() {
+            Fragment::string("[]")
+        } else {
+            let mut frag = Fragment::new();
+            let last_index = vals.len() - 1;
+            let (upper_bound, any_skipped) = match self.preview_len {
+                Some(preview_len) if vals.len() > preview_len => {
+                    (preview_len, preview_len != last_index)
+                }
+                Some(_) | None => (last_index, false),
+            };
+            for index in 0..upper_bound {
+                let val = &vals[index];
+                let format = formats.map(|(fs, ctx)| (&fs[index], ctx));
+                frag.append(self.compile_field_value_continue(index, val, format, false));
+            }
+            if any_skipped {
+                frag.append(self.compile_field_skipped());
+            }
+            let format = formats.map(|(fs, ctx)| (&fs[last_index], ctx));
+            frag.append(self.compile_field_value_last(
+                last_index,
+                &vals[last_index],
+                format,
+                false,
+            ));
+            frag
+        }
+    }
+
+    fn compile_seq(
+        &mut self,
+        vals: &[Value],
+        format: Option<(&Format, RecurseCtx<'module>)>,
+    ) -> Fragment {
+        if vals.is_empty() {
+            Fragment::String("[]".into())
+        } else {
+            let mut frag = Fragment::new();
+            let last_index = vals.len() - 1;
+            let (upper_bound, any_skipped) = match self.preview_len {
+                Some(preview_len) if vals.len() > preview_len => {
+                    (preview_len, preview_len != last_index)
+                }
+                Some(_) | None => (last_index, false),
+            };
+            for index in 0..upper_bound {
+                let val = &vals[index];
+                frag.append(self.compile_field_value_continue(index, val, format, false));
+            }
+            if any_skipped {
+                frag.append(self.compile_field_skipped());
+            }
+            frag.append(self.compile_field_value_last(
+                last_index,
+                &vals[last_index],
+                format,
+                false,
+            ));
+            frag
+        }
+    }
+
+    fn compile_value(&mut self, value: &Value) -> Fragment {
+        match value {
+            Value::Bool(true) => Fragment::string("true"),
+            Value::Bool(false) => Fragment::string("false"),
+            Value::U8(i) => Fragment::DisplayAtom(Rc::new(*i)),
+            Value::U16(i) => Fragment::DisplayAtom(Rc::new(*i)),
+            Value::U32(i) => Fragment::DisplayAtom(Rc::new(*i)),
+            Value::U64(i) => Fragment::DisplayAtom(Rc::new(*i)),
+            Value::Char(c) => Fragment::DebugAtom(Rc::new(*c)),
+            Value::Tuple(vals) => self.compile_tuple(vals, None),
+            Value::Seq(vals) => self.compile_seq(vals, None),
+            Value::Variant(label, value) => self.compile_variant(label, value, None),
+            Value::Branch(_n, value) => self.compile_value(value),
+            Value::Option(None) => Fragment::string("none"),
+            Value::Option(Some(value)) => self.compile_variant("some", value, None),
+        }
+    }
+
+    fn is_quoted_string(&self, str_contents: &[Value]) -> bool {
+        str_contents
+            .iter()
+            .all(|v| matches!(v, Value::U8(0..=127) | Value::Char(_)))
+    }
+
+    fn is_implied_value_format(&self, format: &Format, ctx: RecurseCtx<'module>) -> bool {
+        match format {
+            Format::ItemVar(level) => {
+                let format = self.module.get_format(*level);
+                let ctx = self.module.get_ctx(*level);
+                self.is_implied_value_format(format, ctx)
+            }
+            Format::RecVar(level) => {
+                let ctx = ctx.enter(*level);
+                self.is_implied_value_format(ctx.get_format().unwrap(), ctx)
+            }
+            Format::EndOfInput => true,
+            Format::Byte(bs) => bs.len() == 1,
+            Format::Repeat(format) => self.is_implied_value_format(format, ctx),
+            Format::Seq(formats) | Format::Tuple(formats) => {
+                formats.iter().all(|f| self.is_implied_value_format(f, ctx))
+            }
+            _ => false,
+        }
+    }
+
+    fn is_atomic_value(
+        &self,
+        value: &Value,
+        format: Option<(&Format, RecurseCtx<'module>)>,
+    ) -> bool {
+        match value {
+            Value::Char(_) => true,
+            Value::Bool(_) => true,
+            Value::U8(_) | Value::U16(_) | Value::U32(_) | Value::U64(_) => true,
+            Value::Tuple(values) => values.is_empty(),
+            Value::Seq(values) => values.is_empty(),
+            Value::Variant(label, value) => match format {
+                Some((Format::Variant(label2, format), ctx)) => {
+                    assert_eq!(label, label2);
+                    self.is_atomic_value(value.as_ref(), Some((format, ctx)))
+                }
+                _ => self.is_atomic_value(value.as_ref(), None),
+            },
+            Value::Branch(n, value) => {
+                match format.map(|(f, ctx)| (self.unwrap_itemvars(f, ctx), ctx)) {
+                    Some((Format::Union(branches), ctx)) => {
+                        let format = &branches[*n];
+                        self.is_atomic_value(value.as_ref(), Some((format, ctx)))
+                    }
+                    None => self.is_atomic_value(value.as_ref(), None),
+                    f => panic!("expected format suitable for branch: {f:?}"),
+                }
+            }
+            Value::Option(None) => true,
+            Value::Option(Some(value)) => self.is_atomic_value(value, None),
+        }
+    }
+
+    fn unwrap_itemvars<'a>(&'a self, format: &'a Format, ctx: RecurseCtx<'a>) -> &'a Format {
+        match format {
+            &Format::ItemVar(level, ..) => self.unwrap_itemvars(self.module.get_format(level), ctx),
+            &Format::RecVar(level, ..) => {
+                let ctx = ctx.enter(level);
+                self.unwrap_itemvars(ctx.get_format().unwrap(), ctx)
+            }
+            _ => format,
+        }
+    }
+
+    fn compile_field_value_continue(
+        &mut self,
+        label: impl std::fmt::Display,
+        value: &Value,
+        format: Option<(&Format, RecurseCtx<'module>)>,
+        format_needed: bool,
+    ) -> Fragment {
+        let mut frags = FragmentBuilder::new();
+        frags.push(self.compile_gutter());
+        frags.push(Fragment::cat(
+            Fragment::Symbol(Symbol::Junction),
+            Fragment::String(format!("{label}").into()),
+        ));
+
+        self.gutter.push(Column::Branch);
+        let frag_value = self.compile_field_value(value, format);
+        self.gutter.pop();
+
+        if let Some((format, ctx)) = format {
+            if format_needed
+                || self.flags.show_redundant_formats
+                || (self.is_indirect_format(format) && !frag_value.is_single_line(true))
+            {
+                frags.push(Fragment::String(" <- ".into()));
+                frags.push(self.compile_format(format, ctx, Precedence::FORMAT_COMPOUND));
+            }
+        }
+        frags.push(frag_value);
+        frags.finalize().group()
+    }
+
+    fn compile_field_value_last(
+        &mut self,
+        label: impl std::fmt::Display,
+        value: &Value,
+        format: Option<(&Format, RecurseCtx<'module>)>,
+        format_needed: bool,
+    ) -> Fragment {
+        let mut frags = FragmentBuilder::new();
+        frags.push(self.compile_gutter());
+        frags.push(Fragment::cat(
+            Fragment::Symbol(Symbol::Elbow),
+            Fragment::String(format!("{label}").into()),
+        ));
+
+        self.gutter.push(Column::Space);
+        let frag_value = self.compile_field_value(value, format);
+        self.gutter.pop();
+
+        if let Some((format, ctx)) = format {
+            if format_needed
+                || self.flags.show_redundant_formats
+                || (self.is_indirect_format(format) && !frag_value.is_single_line(true))
+            {
+                frags.push(Fragment::String(" <- ".into()));
+                frags.push(self.compile_format(format, ctx, Default::default()));
+            }
+        }
+        frags.push(frag_value);
+        frags.finalize().group()
+    }
+
+    fn compile_field_skipped(&self) -> Fragment {
+        self.compile_gutter()
+            .cat(Fragment::String("~\n".into()))
+            .group()
+    }
+
+    fn compile_gutter(&self) -> Fragment {
+        let mut frags = FragmentBuilder::new();
+        for column in &self.gutter {
+            match column {
+                Column::Branch => frags.push(Fragment::Symbol(Symbol::Pipe)),
+                Column::Space => frags.push(Fragment::Symbol(Symbol::Vacuum)),
+            }
+        }
+        frags.finalize()
+    }
+
+    fn compile_field_value(
+        &mut self,
+        value: &Value,
+        format: Option<(&Format, RecurseCtx<'module>)>,
+    ) -> Fragment {
+        match format {
+            Some((format, ctx)) => {
+                if self.flags.omit_implied_values && self.is_implied_value_format(format, ctx) {
+                    Fragment::Char('\n')
+                } else {
+                    Fragment::join_with_wsp(
+                        Fragment::String(" :=".into()),
+                        self.compile_decoded_value(value, format, ctx),
+                    )
+                    .group()
+                }
+            }
+            None => {
+                Fragment::join_with_wsp(Fragment::String(" :=".into()), self.compile_value(value))
+                    .group()
+            }
+        }
+    }
+
+    fn is_indirect_format(&self, format: &Format) -> bool {
+        matches!(format, Format::ItemVar(..) | Format::RecVar(..))
+    }
+
+    fn compile_format(
+        &mut self,
+        format: &Format,
+        ctx: RecurseCtx<'module>,
+        prec: Precedence,
+    ) -> Fragment {
+        match format {
+            Format::Variant(label, f) => cond_paren(
+                self.compile_nested_format(
+                    "variant",
+                    Some(&[Fragment::String(label.clone())]),
+                    f,
+                    ctx,
+                    prec,
+                ),
+                prec,
+                Precedence::FORMAT_COMPOUND,
+            ),
+            Format::Union(_) => cond_paren(
+                Fragment::String("_ |...| _".into()),
+                prec,
+                Precedence::FORMAT_COMPOUND,
+            ),
+            Format::Maybe(expr, f) => {
+                let frag_expr = self.compile_expr(expr, Precedence::ATOM);
+                cond_paren(
+                    self.compile_nested_format("maybe", Some(&[frag_expr]), f, ctx, prec),
+                    prec,
+                    Precedence::FORMAT_COMPOUND,
+                )
+            }
+            Format::Repeat(format) => cond_paren(
+                self.compile_nested_format("repeat", None, format, ctx, prec),
+                prec,
+                Precedence::FORMAT_COMPOUND,
+            ),
+            Format::Compute(expr) => cond_paren(
+                Fragment::cat(
+                    Fragment::String("compute ".into()),
+                    self.compile_expr(expr, Default::default()),
+                ),
+                prec,
+                Precedence::FORMAT_COMPOUND,
+            ),
+            Format::ItemVar(var) => Fragment::String(self.module.get_name(*var).to_string().into()),
+            Format::RecVar(var) => Fragment::String(
+                self.module
+                    .get_name(ctx.convert_rec_var(*var).unwrap())
+                    .to_string()
+                    .into(),
+            ),
+            Format::Byte(bs) => match bs.len() {
+                0 => unreachable!("matches against the empty byteset are unsatisfiable"),
+                1..=127 => {
+                    let mut frags = FragmentBuilder::new();
+                    frags.push(Fragment::String("[=".into()));
+                    for b in bs.iter() {
+                        frags.push(Fragment::String(format!(" {b}").into()));
+                    }
+                    frags.push(Fragment::Char(']'));
+                    frags.finalize()
+                }
+                128..=255 => {
+                    let mut frags = FragmentBuilder::new();
+                    frags.push(Fragment::String("[!=".into()));
+                    for b in (!bs).iter() {
+                        frags.push(Fragment::String(format!(" {b}").into()));
+                    }
+                    frags.push(Fragment::Char(']'));
+                    frags.finalize()
+                }
+                256 => Fragment::String("U8".into()),
+                _n => unreachable!("impossible ByteSet size {_n}"),
+            },
+            Format::FailWith(lab) => Fragment::string(format!("fail({})", lab)),
+            Format::EndOfInput => Fragment::string("end-of-input"),
+            Format::Tuple(formats) if formats.is_empty() => Fragment::String("()".into()),
+            Format::Tuple(_) => Fragment::String("(...)".into()),
+
+            Format::Seq(formats) if formats.is_empty() => Fragment::String("[]".into()),
+            Format::Seq(_) => Fragment::String("[ ... ]".into()),
+        }
+    }
+
+    fn compile_nested_format(
+        &mut self,
+        label: &'static str,
+        args: Option<&[Fragment]>,
+        inner: &Format,
+        ctx: RecurseCtx<'module>,
+        prec: Precedence,
+    ) -> Fragment {
+        let mut frags = FragmentBuilder::new();
+        frags.push(Fragment::String(label.into()));
+        if let Some(args) = args {
+            for arg in args.iter() {
+                frags.push(arg.clone());
+            }
+        }
+        frags.push(self.compile_format(inner, ctx, prec.bump_format()));
+        frags.finalize_with_sep(Fragment::Char(' '))
+    }
+
+    fn compile_expr(&self, expr: &Expr, prec: Precedence) -> Fragment {
+        match expr {
+            Expr::IntRel(IntRel::Eq, lhs, rhs) => cond_paren(
+                self.binary_op(" == ", lhs, rhs, Precedence::EQUALITY, Precedence::EQUALITY),
+                prec,
+                Precedence::COMPARE,
+            ),
+            Expr::IntRel(IntRel::Neq, lhs, rhs) => cond_paren(
+                self.binary_op(" != ", lhs, rhs, Precedence::EQUALITY, Precedence::EQUALITY),
+                prec,
+                Precedence::COMPARE,
+            ),
+            Expr::IntRel(IntRel::Lt, lhs, rhs) => cond_paren(
+                self.binary_op(" < ", lhs, rhs, Precedence::COMPARE, Precedence::COMPARE),
+                prec,
+                Precedence::COMPARE,
+            ),
+            Expr::IntRel(IntRel::Gt, lhs, rhs) => cond_paren(
+                self.binary_op(" > ", lhs, rhs, Precedence::COMPARE, Precedence::COMPARE),
+                prec,
+                Precedence::COMPARE,
+            ),
+            Expr::IntRel(IntRel::Lte, lhs, rhs) => cond_paren(
+                self.binary_op(" <= ", lhs, rhs, Precedence::COMPARE, Precedence::COMPARE),
+                prec,
+                Precedence::COMPARE,
+            ),
+            Expr::IntRel(IntRel::Gte, lhs, rhs) => cond_paren(
+                self.binary_op(" >= ", lhs, rhs, Precedence::COMPARE, Precedence::COMPARE),
+                prec,
+                Precedence::COMPARE,
+            ),
+            Expr::Arith(Arith::Add, lhs, rhs) => cond_paren(
+                self.binary_op(" + ", lhs, rhs, Precedence::ADD_SUB, Precedence::ADD_SUB),
+                prec,
+                Precedence::ADD_SUB,
+            ),
+            Expr::Arith(Arith::Sub, lhs, rhs) => cond_paren(
+                self.binary_op(" - ", lhs, rhs, Precedence::ADD_SUB, Precedence::ADD_SUB),
+                prec,
+                Precedence::ADD_SUB,
+            ),
+            Expr::Arith(Arith::Mul, lhs, rhs) => cond_paren(
+                self.binary_op(" * ", lhs, rhs, Precedence::MUL, Precedence::MUL),
+                prec,
+                Precedence::MUL,
+            ),
+            Expr::Arith(Arith::Div, lhs, rhs) => cond_paren(
+                self.binary_op(" / ", lhs, rhs, Precedence::DIV_REM, Precedence::DIV_REM),
+                prec,
+                Precedence::DIV_REM,
+            ),
+            Expr::Arith(Arith::Rem, lhs, rhs) => cond_paren(
+                self.binary_op(" % ", lhs, rhs, Precedence::DIV_REM, Precedence::DIV_REM),
+                prec,
+                Precedence::DIV_REM,
+            ),
+            Expr::Arith(Arith::BitAnd, lhs, rhs) => cond_paren(
+                self.binary_op(" & ", lhs, rhs, Precedence::BITAND, Precedence::BITAND),
+                prec,
+                Precedence::BITAND,
+            ),
+            Expr::Arith(Arith::BitOr, lhs, rhs) => cond_paren(
+                self.binary_op(" | ", lhs, rhs, Precedence::BITOR, Precedence::BITOR),
+                prec,
+                Precedence::BITOR,
+            ),
+            Expr::Arith(Arith::Shl, lhs, rhs) => cond_paren(
+                self.binary_op(
+                    " << ",
+                    lhs,
+                    rhs,
+                    Precedence::BIT_SHIFT,
+                    Precedence::BIT_SHIFT,
+                ),
+                prec,
+                Precedence::BIT_SHIFT,
+            ),
+            Expr::Arith(Arith::Shr, lhs, rhs) => cond_paren(
+                self.binary_op(
+                    " >> ",
+                    lhs,
+                    rhs,
+                    Precedence::BIT_SHIFT,
+                    Precedence::BIT_SHIFT,
+                ),
+                prec,
+                Precedence::BIT_SHIFT,
+            ),
+            Expr::Unary(Unary::BoolNot, expr) => cond_paren(
+                self.prefix_op("!", None, expr),
+                prec,
+                Precedence::LOGICAL_NEGATE,
+            ),
+            Expr::AsU8(expr) => cond_paren(
+                self.prefix_op("as-u8", None, expr),
+                prec,
+                Precedence::CAST_PREFIX,
+            ),
+            Expr::AsU16(expr) => cond_paren(
+                self.prefix_op("as-u16", None, expr),
+                prec,
+                Precedence::CAST_PREFIX,
+            ),
+            Expr::AsU32(expr) => cond_paren(
+                self.prefix_op("as-u32", None, expr),
+                prec,
+                Precedence::CAST_PREFIX,
+            ),
+            Expr::AsU64(expr) => cond_paren(
+                self.prefix_op("as-u64", None, expr),
+                prec,
+                Precedence::CAST_PREFIX,
+            ),
+            Expr::AsChar(expr) => cond_paren(
+                self.prefix_op("as-char", None, expr),
+                prec,
+                Precedence::CAST_PREFIX,
+            ),
+            Expr::LiftOption(Some(expr)) => cond_paren(
+                self.prefix_op("some", None, expr),
+                prec,
+                Precedence::FUN_APPLICATION,
+            ),
+            Expr::LiftOption(None) => Fragment::string("none"),
+            Expr::Bool(b) => Fragment::DisplayAtom(Rc::new(*b)),
+            Expr::U8(i) => Fragment::DisplayAtom(Rc::new(*i)),
+            Expr::U16(i) => Fragment::DisplayAtom(Rc::new(*i)),
+            Expr::U32(i) => Fragment::DisplayAtom(Rc::new(*i)),
+            Expr::U64(i) => Fragment::DisplayAtom(Rc::new(*i)),
+            Expr::Tuple(..) => Fragment::String("(...)".into()),
+            Expr::Variant(label, expr) => Fragment::String("{ ".into())
+                .cat(Fragment::String(label.clone()))
+                .cat(Fragment::String(" := ".into()))
+                .cat(self.compile_expr(expr, Default::default()))
+                .cat(Fragment::String(" }".into()))
+                .group(),
+            Expr::Seq(..) => Fragment::String("[..]".into()),
+        }
+    }
+
+    fn prefix_op(&self, op: &'static str, args: Option<&[&Expr]>, operand: &Expr) -> Fragment {
+        let mut frags = FragmentBuilder::new();
+
+        frags.push(Fragment::String(op.into()));
+        match args {
+            None => (),
+            Some(args) => {
+                let frag = frags.active_mut();
+                frag.append(Fragment::Char('('));
+                frag.append(Fragment::seq(
+                    args.iter()
+                        .map(|arg| self.compile_expr(arg, Precedence::default()))
+                        .collect::<Vec<_>>(),
+                    Some(Fragment::String(", ".into())),
+                ));
+                frag.append(Fragment::Char(')'));
+            }
+        }
+        frags.push(self.compile_expr(operand, Precedence::ATOM));
+        frags.finalize_with_sep(Fragment::Char(' '))
+    }
+
+    fn binary_op(
+        &self,
+        op: &'static str,
+        lhs: &Expr,
+        rhs: &Expr,
+        lhs_prec: Precedence,
+        rhs_prec: Precedence,
+    ) -> Fragment {
+        self.compile_expr(lhs, lhs_prec)
+            .cat(Fragment::String(op.into()))
+            .cat(self.compile_expr(rhs, rhs_prec))
+            .group()
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 44d2ae35..1acdbf8d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -25,7 +25,7 @@ pub mod loc_decoder;
 
 pub mod output;
 pub mod parser;
-mod precedence;
+pub mod precedence;
 pub mod prelude;
 pub mod read;
 
diff --git a/src/output.rs b/src/output.rs
index 43fc5039..d3900b22 100644
--- a/src/output.rs
+++ b/src/output.rs
@@ -230,12 +230,12 @@ impl Fragment {
     /// There are currently no display rules that differentiate [`Fragment::Group`](x) from `x` itself, but
     /// it is defined anyway and supported with this helper to allow for cleaner adoption of a more nuanced
     /// model in which logically-grouped fragments have their own display rules.
-    pub(crate) fn group(self) -> Self {
+    pub fn group(self) -> Self {
         Self::Group(Box::new(self))
     }
 
     /// Like [Fragment::group], except that it modifies a mutable reference in-place and passes it back to the caller
-    pub(crate) fn enclose(&mut self) -> &mut Self {
+    pub fn enclose(&mut self) -> &mut Self {
         let this = Box::new(std::mem::take(self));
         *self = Self::Group(this);
         self
@@ -251,12 +251,12 @@ impl Fragment {
     ///
     /// Returns the same mutable reference as was passed in, to allow chaining of similar operations.
     #[inline]
-    fn append_break(&mut self) -> &mut Self {
+    pub fn append_break(&mut self) -> &mut Self {
         self.append(Fragment::Char('\n'))
     }
 
     /// Returns an empty fragment
-    fn new() -> Self {
+    pub fn new() -> Self {
         Self::Empty
     }
 
@@ -266,7 +266,7 @@ impl Fragment {
     /// rather than whether inline concatenations are possible.
     ///
     /// Importantly, a single trailing newline character is permitted, and `Symbols` care allowed to appear anywhere
-    pub(crate) fn is_single_line(&self, is_final: bool) -> bool {
+    pub fn is_single_line(&self, is_final: bool) -> bool {
         match self {
             Fragment::Empty => true,
             Fragment::Char('\n') => is_final,
@@ -307,7 +307,7 @@ impl Fragment {
     /// Returns `false` if any of the following are true:
     ///   - The Display form of `self` contains any newline characters
     ///   - `self` contains any `Symbol` sub-fragments
-    fn fits_inline(&self) -> bool {
+    pub fn fits_inline(&self) -> bool {
         match self {
             Fragment::Empty => true,
             Fragment::Char(c) => *c != '\n',
@@ -333,7 +333,7 @@ impl Fragment {
     /// Joins two fragments with appropriate whitespace:
     ///   - If `other` fits on a single line with no trailing newline, joins with `' '`, with a newline at the very end
     ///   - Otherwise, joins with `'\n'`
-    pub(crate) fn join_with_wsp(self, other: Self) -> Self {
+    pub fn join_with_wsp(self, other: Self) -> Self {
         if other.fits_inline() {
             self.cat(Self::Char(' ')).cat(other).cat_break()
         } else {
@@ -344,7 +344,7 @@ impl Fragment {
     /// Joins two fragments with the appropriate whitespace and a conditional line-ending `trailer`:
     ///    - If `other` does not require more than one line to print, joins with `' '`, with a newline at the very end (and without `trailer`).
     ///    - Otherwise, joins `self` and `trailer` with no separation, followed by `other` on the following line
-    pub(crate) fn join_with_wsp_eol(self, other: Self, trailer: Self) -> Self {
+    pub fn join_with_wsp_eol(self, other: Self, trailer: Self) -> Self {
         if other.fits_inline() {
             self.cat(Self::Char(' ')).cat(other).cat_break()
         } else {
@@ -354,7 +354,7 @@ impl Fragment {
 }
 
 /// Builder pattern helper-struct for accumulating up longer sequences of [Fragment]s.
-pub(crate) struct FragmentBuilder {
+pub struct FragmentBuilder {
     frozen: Vec<Fragment>,
     active: Fragment,
 }
diff --git a/src/output/tree.rs b/src/output/tree.rs
index e26161dd..426a3ab6 100644
--- a/src/output/tree.rs
+++ b/src/output/tree.rs
@@ -107,7 +107,9 @@ impl<'module> TreePrinter<'module> {
             }
             Format::EndOfInput => true,
             Format::Byte(bs) => bs.len() == 1,
-            Format::Tuple(fields) => fields.iter().all(|f| self.is_implied_value_format(f)),
+            Format::Sequence(formats) | Format::Tuple(formats) => {
+                formats.iter().all(|f| self.is_implied_value_format(f))
+            }
             Format::Hint(StyleHint::Record { old_style }, inner) => {
                 if *old_style {
                     self.is_implied_value_format_old_style_record(inner)
diff --git a/src/precedence.rs b/src/precedence.rs
index 05728551..d319da92 100644
--- a/src/precedence.rs
+++ b/src/precedence.rs
@@ -4,7 +4,7 @@ use crate::output::Fragment;
 ///
 ///
 #[derive(Copy, Clone, Debug, Default)]
-pub(crate) enum Precedence {
+pub enum Precedence {
     /// Highest precedence, as if implicitly (if not actually) parenthesized
     Atomic,
     /// Post-fix projection such as method call, field access, or Try (`?`)
@@ -27,20 +27,20 @@ pub(crate) enum Precedence {
 }
 
 #[derive(Copy, Clone, Debug)]
-pub(crate) enum CalculusLevel {
+pub enum CalculusLevel {
     Invoke, // Highest calculus precedence
     Lambda,
     Match,
 }
 
 #[derive(Copy, Clone, Debug)]
-pub(crate) enum CompareLevel {
+pub enum CompareLevel {
     Comparison = 0, // Highest comparative precedence
     Equality,
 }
 
 #[derive(Copy, Clone, Debug)]
-pub(crate) enum ArithLevel {
+pub enum ArithLevel {
     DivRem = 0, // Highest arithmetic precedence
     Mul,
     AddSub,
@@ -48,14 +48,14 @@ pub(crate) enum ArithLevel {
 
 #[derive(Copy, Clone, Debug)]
 #[repr(u8)]
-pub(crate) enum BitwiseLevel {
+pub enum BitwiseLevel {
     Shift = 0, // Highest bitwise precedence
     And = 1,
     Or = 2,
 }
 
 #[derive(Copy, Clone, Debug)]
-pub(crate) enum LogicalLevel {
+pub enum LogicalLevel {
     And = 0,
     Or = 1,
 }
@@ -73,7 +73,7 @@ impl IntransitiveOrd for LogicalLevel {
 
 /// Intransitive partial relation over operator subclasses
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub(crate) enum Relation {
+pub enum Relation {
     /// `.<`
     Inferior,
     /// `.=`
@@ -84,7 +84,7 @@ pub(crate) enum Relation {
     Disjoint,
 }
 
-pub(crate) trait IntransitiveOrd {
+pub trait IntransitiveOrd {
     fn relate(&self, other: &Self) -> Relation;
 }
 
@@ -197,34 +197,34 @@ impl IntransitiveOrd for Precedence {
 }
 
 impl Precedence {
-    pub(crate) const TOP: Self = Precedence::Top;
-    pub(crate) const ARROW: Self = Precedence::Calculus(CalculusLevel::Lambda);
-    pub(crate) const MATCH: Self = Precedence::Calculus(CalculusLevel::Match);
-    pub(crate) const INVOKE: Self = Precedence::Calculus(CalculusLevel::Invoke);
-    pub(crate) const COMPARE: Self = Precedence::Comparison(CompareLevel::Comparison);
-    pub(crate) const EQUALITY: Self = Precedence::Comparison(CompareLevel::Equality);
-    pub(crate) const BITOR: Self = Precedence::BitwiseInfix(BitwiseLevel::Or);
-    pub(crate) const ADD_SUB: Self = Precedence::ArithInfix(ArithLevel::AddSub);
-    pub(crate) const DIV_REM: Self = Precedence::ArithInfix(ArithLevel::DivRem);
-    pub(crate) const BITAND: Self = Precedence::BitwiseInfix(BitwiseLevel::And);
-    pub(crate) const LOGICAL_AND: Self = Precedence::LogicalInfix(LogicalLevel::And);
-    pub(crate) const LOGICAL_OR: Self = Precedence::LogicalInfix(LogicalLevel::Or);
-    pub(crate) const LOGICAL_NEGATE: Self = Precedence::Prefix;
-    pub(crate) const NUMERIC_PREFIX: Self = Precedence::Prefix;
-    pub(crate) const MUL: Self = Precedence::ArithInfix(ArithLevel::Mul);
-    pub(crate) const BIT_SHIFT: Self = Precedence::BitwiseInfix(BitwiseLevel::Shift);
-    pub(crate) const FUN_APPLICATION: Self = Precedence::Prefix;
-    pub(crate) const CAST_INFIX: Self = Precedence::Calculus(CalculusLevel::Invoke);
-    pub(crate) const CAST_PREFIX: Self = Precedence::Prefix;
-    pub(crate) const PROJ: Self = Precedence::Projection;
-    pub(crate) const ATOM: Self = Precedence::Atomic;
-
-    pub(crate) const FORMAT_COMPOUND: Self = Self::Top;
+    pub const TOP: Self = Precedence::Top;
+    pub const ARROW: Self = Precedence::Calculus(CalculusLevel::Lambda);
+    pub const MATCH: Self = Precedence::Calculus(CalculusLevel::Match);
+    pub const INVOKE: Self = Precedence::Calculus(CalculusLevel::Invoke);
+    pub const COMPARE: Self = Precedence::Comparison(CompareLevel::Comparison);
+    pub const EQUALITY: Self = Precedence::Comparison(CompareLevel::Equality);
+    pub const BITOR: Self = Precedence::BitwiseInfix(BitwiseLevel::Or);
+    pub const ADD_SUB: Self = Precedence::ArithInfix(ArithLevel::AddSub);
+    pub const DIV_REM: Self = Precedence::ArithInfix(ArithLevel::DivRem);
+    pub const BITAND: Self = Precedence::BitwiseInfix(BitwiseLevel::And);
+    pub const LOGICAL_AND: Self = Precedence::LogicalInfix(LogicalLevel::And);
+    pub const LOGICAL_OR: Self = Precedence::LogicalInfix(LogicalLevel::Or);
+    pub const LOGICAL_NEGATE: Self = Precedence::Prefix;
+    pub const NUMERIC_PREFIX: Self = Precedence::Prefix;
+    pub const MUL: Self = Precedence::ArithInfix(ArithLevel::Mul);
+    pub const BIT_SHIFT: Self = Precedence::BitwiseInfix(BitwiseLevel::Shift);
+    pub const FUN_APPLICATION: Self = Precedence::Prefix;
+    pub const CAST_INFIX: Self = Precedence::Calculus(CalculusLevel::Invoke);
+    pub const CAST_PREFIX: Self = Precedence::Prefix;
+    pub const PROJ: Self = Precedence::Projection;
+    pub const ATOM: Self = Precedence::Atomic;
+
+    pub const FORMAT_COMPOUND: Self = Self::Top;
 
     // REVIEW - does list-append need its own precedence or is this good enough?
-    pub(crate) const APPEND: Self = Precedence::ArithInfix(ArithLevel::AddSub);
+    pub const APPEND: Self = Precedence::ArithInfix(ArithLevel::AddSub);
 
-    pub(crate) fn bump_format(&self) -> Self {
+    pub fn bump_format(&self) -> Self {
         match self {
             Precedence::Top => Precedence::Atomic,
             Precedence::Atomic => Precedence::Atomic,
@@ -233,7 +233,7 @@ impl Precedence {
     }
 }
 
-pub(crate) fn cond_paren(frag: Fragment, current: Precedence, cutoff: Precedence) -> Fragment {
+pub fn cond_paren(frag: Fragment, current: Precedence, cutoff: Precedence) -> Fragment {
     match current.relate(&cutoff) {
         Relation::Disjoint | Relation::Superior => {
             Fragment::Char('(').cat(frag).cat(Fragment::Char(')'))

From 6f0bca897e7ab2e10c4ac6a2c5288f97f38f6b3c Mon Sep 17 00:00:00 2001
From: Peter Duchovni <caufeminecraft+github@gmail.com>
Date: Fri, 13 Jun 2025 17:32:06 +1000
Subject: [PATCH 15/15] Update Cargo.lock

---
 Cargo.lock | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index 19254305..7e8f7fff 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -393,6 +393,7 @@ dependencies = [
  "anyhow",
  "clap",
  "doodle",
+ "linked_hash_set",
  "serde",
  "serde_json",
 ]
@@ -657,6 +658,21 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
+[[package]]
+name = "linked_hash_set"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bae85b5be22d9843c80e5fc80e9b64c8a3b1f98f867c709956eca3efff4e92e2"
+dependencies = [
+ "linked-hash-map",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.3.1"