M6: finish all plans — audit remediation, STEP tokenizer fix, docs site

.github/workflows/docs.yml

@@ -0,0 +1,101 @@
+name: Docs
+
+# Builds the LatticeLabs Toolkit documentation site (site/, Astro + Starlight).
+#  - Pull requests: build + type-check + internal-link validation (gate, no deploy).
+#  - Push to main:  same checks, then deploy to GitHub Pages.
+# See docs/specs/SPEC-2-documentation-site.md.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'site/**'
+      - 'cadling/**'
+      - 'll_stepnet/**'
+      - 'geotoken/**'
+      - 'll_ocadr/**'
+      - 'll_gen/**'
+      - 'll_clouds/**'
+      - '.github/workflows/docs.yml'
+  pull_request:
+    paths:
+      - 'site/**'
+      - 'cadling/**'
+      - 'll_stepnet/**'
+      - 'geotoken/**'
+      - 'll_ocadr/**'
+      - 'll_gen/**'
+      - 'll_clouds/**'
+      - '.github/workflows/docs.yml'
+  workflow_dispatch:
+
+# Least-privilege defaults; the deploy job needs pages + id-token.
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build & validate
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: site
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22' # Astro 6 requires Node >= 22.12
+          cache: npm
+          cache-dependency-path: site/package-lock.json
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+
+      - name: Install API-generator dependency (griffe — static parse, no package import)
+        run: pip install "griffe>=2,<3"
+
+      - name: Install site dependencies
+        run: npm ci
+
+      - name: Generate API reference from docstrings
+        # Defined in M5 (scripts/gen-api.py). --if-present keeps the pipeline valid
+        # before that script lands; once present it regenerates <pkg>/reference/*.
+        run: npm run gen:api --if-present
+
+      - name: Type-check content (astro check)
+        run: npm run check
+
+      - name: Build site (validates internal links, builds Pagefind search index)
+        run: npm run build
+
+      - name: Upload Pages artifact
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site/dist
+
+  deploy:
+    name: Deploy to GitHub Pages
+    needs: build
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -110,8 +110,6 @@ venv.bak/
 # Rope project settings
 .ropeproject
 
-# mkdocs documentation
-/site
 
 # mypy
 .mypy_cache/
@@ -169,4 +167,7 @@ resources
 test_data
 research
 results
-.ruff_cache
+.ruff_cache
+site/node_modules
+site/dist
+site/build

README.md

@@ -2,6 +2,8 @@
 
 A monorepo of Python packages for CAD document processing, neural networks for 3D geometry, geometric tokenization, optical CAD recognition, and generative CAD modeling.
 
+📚 **Documentation:** <https://latticelabsai.github.io/ll_toolkit/> — per-package guides, tutorials, concepts, and a generated API reference. Source lives in [`site/`](site/).
+
 ## Packages
 
 | Package | Path | Description |

cadling/cadling/backend/step/stepnet_integration.py

@@ -15,6 +15,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import logging
 import re
 from collections import defaultdict
@@ -807,10 +808,16 @@ def _build_topology_alternative(self, entities: list[dict[str, Any]]) -> dict[st
                         params_to_use, dtype=torch.float32
                     )
 
-                # Entity type hash (last dimension)
+                # Entity type hash (last dimension). Use a DETERMINISTIC hash
+                # (blake2b) rather than Python's builtin hash(), which is salted
+                # per process (PYTHONHASHSEED) and would make this feature — and
+                # thus the produced training data — non-reproducible across runs.
                 entity_type = entity.get("entity_type") or entity.get("type", "")
                 if entity_type:
-                    type_hash = (hash(entity_type) % 10000) / 10000.0
+                    digest = hashlib.blake2b(
+                        entity_type.encode("utf-8"), digest_size=8
+                    ).digest()
+                    type_hash = (int.from_bytes(digest, "big") % 10000) / 10000.0
                     node_features[idx, -1] = type_hash
 
             topology["node_features"] = node_features

cadling/cadling/backend/step/tokenizer.py

@@ -372,19 +372,26 @@ def _collect_multiline_entity(self, lines: List[str], start_idx: int) -> tuple:
             # Remove comments
             line = self._remove_comments(line)
 
-            # Process character by character to find semicolon outside strings
-            prev_char = ''
-            for char in line:
-                if char == "'" and prev_char == "'":
-                    # Doubled quote '' is an escape in STEP, not a string boundary toggle
-                    prev_char = ''
-                    continue
-                elif char == "'":
+            # Process character by character to find semicolon outside strings.
+            # A doubled quote ('') is an escaped literal quote ONLY when already
+            # inside a string; an opening quote immediately followed by a closing
+            # quote is an EMPTY string literal, not an escape. Distinguishing the
+            # two requires lookahead, not a prev-char check (which would treat the
+            # empty-string '' as an escape and leave in_string stuck open).
+            j = 0
+            line_len = len(line)
+            while j < line_len:
+                char = line[j]
+                if char == "'":
+                    if in_string and j + 1 < line_len and line[j + 1] == "'":
+                        # Escaped quote inside a string: consume both, stay in.
+                        j += 2
+                        continue
                     in_string = not in_string
                 elif char == ';' and not in_string:
                     found_semicolon = True
                     break
-                prev_char = char
+                j += 1
 
             entity_parts.append(line)
             current_idx += 1
@@ -411,27 +418,37 @@ def parse_step_file(self, content: str) -> Dict[str, Any]:
             "entities": {}
         }
 
-        # Normalize whitespace before processing
+        # Normalize whitespace before processing (STEP is whitespace-insensitive
+        # outside string literals, so newlines/tabs collapse to single spaces).
         content = self._normalize_whitespace(content)
 
-        # Split into sections
-        lines = content.split('\n')
+        # Split into STEP STATEMENTS (terminated by ';'), not by newline — after
+        # normalization there are no newlines, and STEP statements may also share
+        # a line. The split is string-literal-aware so a ';' inside a quoted
+        # value (e.g. FILE_DESCRIPTION(...,'2;1')) does not terminate a statement.
+        statements = self._split_statements(content)
+
         current_section = None
-        header_content = []
-        data_content = []
+        header_content: List[str] = []
+        data_content: List[str] = []
 
-        for line in lines:
-            line = line.strip()
-            if line.startswith('HEADER;'):
+        for stmt in statements:
+            stmt = stmt.strip()
+            if not stmt:
+                continue
+            keyword = stmt.upper()
+            if keyword == 'HEADER':
                 current_section = 'header'
-            elif line.startswith('DATA;'):
+            elif keyword == 'DATA':
                 current_section = 'data'
-            elif line.startswith('ENDSEC;') or line.startswith('END-ISO'):
+            elif keyword == 'ENDSEC':
+                current_section = None
+            elif keyword.startswith('ISO-10303') or keyword.startswith('END-ISO'):
                 current_section = None
             elif current_section == 'header':
-                header_content.append(line)
+                header_content.append(stmt + ';')
             elif current_section == 'data':
-                data_content.append(line)
+                data_content.append(stmt + ';')
 
         # Parse header
         result["header"] = self._parse_header(header_content)
@@ -441,6 +458,48 @@ def parse_step_file(self, content: str) -> Dict[str, Any]:
 
         return result
 
+    def _split_statements(self, text: str) -> List[str]:
+        """Split STEP text into statements terminated by ';'.
+
+        The split respects string literals: a ';' inside a quoted value does not
+        terminate a statement. STEP escapes a quote inside a string by doubling
+        it (``''``), which is handled here. Returns the statements WITHOUT their
+        trailing ';'.
+
+        Args:
+            text: STEP content (typically whitespace-normalized).
+
+        Returns:
+            List of statement strings (no trailing ';').
+        """
+        statements: List[str] = []
+        current: List[str] = []
+        in_string = False
+        i = 0
+        n = len(text)
+        while i < n:
+            ch = text[i]
+            if ch == "'":
+                # An escaped quote inside a string is a doubled '' — keep both
+                # and stay in the string.
+                if in_string and i + 1 < n and text[i + 1] == "'":
+                    current.append("''")
+                    i += 2
+                    continue
+                in_string = not in_string
+                current.append(ch)
+            elif ch == ';' and not in_string:
+                statements.append(''.join(current))
+                current = []
+            else:
+                current.append(ch)
+            i += 1
+
+        tail = ''.join(current)
+        if tail.strip():
+            statements.append(tail)
+        return statements
+
     def _parse_header(self, header_lines: List[str]) -> Dict[str, Any]:
         """Parse STEP header section.
 
@@ -593,13 +652,24 @@ def _parse_params(self, params_str: str) -> List[Any]:
         return params
 
     def _parse_single_param(self, param: str) -> Any:
-        """Parse a single parameter value.
+        """Parse a single STEP parameter token into its Python type.
+
+        Numeric tokens are coerced to ``int``/``float`` — this is the canonical
+        contract every downstream consumer relies on: feature extraction,
+        coordinate/geometry parsing, and ll_stepnet tokenization all treat
+        numeric params as numbers on their primary path (the string branches are
+        defensive fallbacks). Other tokens map as: ``$``/empty -> ``None``;
+        ``'…'`` -> the unquoted ``str``; ``#N`` -> the reference ``str`` (kept
+        verbatim); ``.X.`` -> the enum ``str``; ``(…)`` -> a parsed ``list``;
+        anything else -> the original ``str``.
 
         Args:
-            param: Parameter string
+            param: A single parameter token (re-stripped here defensively).
 
         Returns:
-            Parsed parameter (can be string, number, reference, list, etc.)
+            ``int``/``float`` for numbers, ``str`` for strings/references/enums/
+            unparseable tokens, ``list`` for nested parentheses, or ``None`` for
+            ``$``/empty.
         """
         param = param.strip()
 

cadling/cadling/chunker/mesh_chunker/mesh_chunker.py

@@ -460,11 +460,14 @@ def _kmeans(self, points: np.ndarray, k: int, max_iters: int = 100) -> np.ndarra
         return labels
 
     def _segment_by_graph(self, mesh: MeshData) -> List[List[int]]:
-        """Segment mesh using graph-based methods.
+        """Segment the mesh into connected face regions via graph traversal.
 
-        Uses normalized cuts approximation. Ensures no faces are dropped:
-        faces that remain in the BFS queue when a segment reaches
-        max_faces_per_chunk are revisited as seeds for subsequent segments.
+        Partitions faces into spatially-connected components by breadth-first
+        region growing over the face-adjacency graph, capped at
+        ``max_faces_per_chunk`` faces per segment. This is connected-components
+        segmentation with a size limit — NOT a spectral normalized-cuts
+        partition. No faces are dropped: faces still queued when a segment
+        reaches the cap are unmarked and become seeds for subsequent segments.
 
         Args:
             mesh: Mesh data
@@ -475,7 +478,7 @@ def _segment_by_graph(self, mesh: MeshData) -> List[List[int]]:
         # Build adjacency graph
         adjacency = self._build_face_adjacency(mesh)
 
-        # Simple connected components with size limit
+        # Connected components via BFS region growing, with a per-segment cap.
         visited = set()
         segments = []
 

cadling/cadling/chunker/tokenizer/tokenizer.py

@@ -108,16 +108,22 @@ def tokenize(self, text: str) -> List[str]:
         return self.pattern.findall(text)
 
     def encode(self, text: str) -> List[int]:
-        """Encode text (simple hash-based encoding).
+        """Encode text (deterministic hash-based encoding).
+
+        Uses a stable cross-process hash so the same text always maps to the
+        same token ids (the builtin ``hash()`` is salted per process via
+        PYTHONHASHSEED, which would make serialized token ids non-reproducible).
 
         Args:
             text: Input text
 
         Returns:
-            List of token hashes
+            List of token ids
         """
+        from cadling.lib.hashing import stable_hash
+
         tokens = self.tokenize(text)
-        return [hash(token) % 50000 for token in tokens]
+        return [stable_hash(token, 50000) for token in tokens]
 
     def decode(self, token_ids: List[int]) -> str:
         """Decode not supported for simple tokenizer.