From 2a505006cbbe841d5abbde31edb6c31e46408471 Mon Sep 17 00:00:00 2001 From: Oleksiy Puzikov Date: Fri, 24 Apr 2026 12:14:15 +0300 Subject: [PATCH 1/5] Remove ISO clause structure linter from main branch The linter (iso_clause_lint.py, iso_lint command, make_iso_lint_parser) has been moved to the feature/iso-clause-lint branch for a separate PR. This branch retains only the ISO cross-reference filter implementation. Co-Authored-By: Claude Sonnet 4.5 --- doc_build/doc_builder.py | 32 ----- doc_build/iso_clause_lint.py | 272 ----------------------------------- 2 files changed, 304 deletions(-) delete mode 100644 doc_build/iso_clause_lint.py diff --git a/doc_build/doc_builder.py b/doc_build/doc_builder.py index eaba737..7d154b9 100644 --- a/doc_build/doc_builder.py +++ b/doc_build/doc_builder.py @@ -562,22 +562,6 @@ def run_linter(self, args): log(f"\tLint output: {linted}") - def iso_lint(self, args): - from doc_build.iso_clause_lint import check_spec, format_report - - spec_root = self.get_specification_root() - log(f"Checking ISO clause structure in {spec_root} ...") - violations = check_spec(spec_root) - report = format_report( - violations, - context=args.context, - spec_root=spec_root, - ) - if report: - log(report) - else: - log("No ISO clause structure violations found.") - def export_git_archive(self, args): timestr = time.strftime("%Y%m%d-%H%M%S") filename = f"aousd_core_spec_{args.branch}_{timestr}.zip" @@ -873,7 +857,6 @@ def construct_subparsers(self, parser): self.make_index_parser(subparsers) self.make_spellcheck_parser(subparsers) self.make_style_parser(subparsers) - self.make_iso_lint_parser(subparsers) return subparsers def make_build_parser(self, subparsers): @@ -969,21 +952,6 @@ def make_style_parser(self, subparsers): style_parser.set_defaults(func=self.display_style_issues) return style_parser - def make_iso_lint_parser(self, subparsers): - p = subparsers.add_parser( - "iso_lint", - help="Check specification source files for ISO clause structure violations", - ) - p.add_argument( - "--context", - type=int, - default=5, - metavar="N", - help="Number of body lines to show per violation (default: 5)", - ) - p.set_defaults(func=self.iso_lint) - return p - def add_publish_copyright(self, combined): intro_copyright = self.get_publish_intro_legalese() outro = self.get_publish_outro_legalese() diff --git a/doc_build/iso_clause_lint.py b/doc_build/iso_clause_lint.py deleted file mode 100644 index 9bf4e66..0000000 --- a/doc_build/iso_clause_lint.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python3 -"""ISO clause structure linter for Markdown specification files. - -ISO/IEC Directives, Part 2 rule: if a clause contains subclauses, there shall -be no text between the clause heading and its first subclause. - -This module checks source Markdown files (not the flattened combined spec) and -reports every heading that has body text immediately preceding its first direct -child heading. It is intentionally standalone — no Pandoc dependency — so it -runs quickly on the raw files. - -Usage as a library: - from doc_build.iso_clause_lint import check_spec - violations = check_spec(Path("specification/")) - for v in violations: print(v.format()) - -Usage from the command line: - python3 -m doc_build.iso_clause_lint specification/ -""" - -import os -import re -import sys -from dataclasses import dataclass, field -from pathlib import Path -from typing import List, Optional, Tuple - -# --------------------------------------------------------------------------- -# Parsing constants -# --------------------------------------------------------------------------- - -# Matches ATX heading lines, capturing the hashes and the title text. -# Strips trailing Pandoc attribute blocks {#id .class}. -_HEADING_RE = re.compile(r'^(#{1,6})\s+(.+?)(?:\s+\{[^}]*\})?\s*$') - -# Matches the opening/closing fence of a fenced code block (``` or ~~~). -_FENCE_RE = re.compile(r'^\s*(`{3,}|~{3,})') - -# How many non-blank body lines to show as context in a report. -DEFAULT_CONTEXT_LINES = 5 - - -# --------------------------------------------------------------------------- -# Data types -# --------------------------------------------------------------------------- - -@dataclass -class Violation: - """A single ISO clause-structure violation found in a Markdown file.""" - file: Path - heading_lineno: int # 1-based line number of the offending heading - heading_level: int # ATX heading level (1–6) - heading_text: str # heading text (stripped of attribute blocks) - first_sub_lineno: int # 1-based line number of the first subclause - first_sub_level: int # heading level of the first subclause - first_sub_text: str # first subclause text - body_lines: List[Tuple[int, str]] = field(default_factory=list) - # Non-blank lines between the heading and the first subclause. - # Each entry is (1-based line number, raw line content). - - def format(self, context: int = DEFAULT_CONTEXT_LINES) -> str: - """Return a human-readable description of the violation.""" - h_marker = '#' * self.heading_level - sub_marker = '#' * self.first_sub_level - shown = self.body_lines[:context] - remainder = len(self.body_lines) - len(shown) - - lines = [ - f"{self.file}:{self.heading_lineno}: " - f"{h_marker} \"{self.heading_text}\" " - f"has text before its first subclause", - ] - for lineno, content in shown: - lines.append(f" │ {lineno:5d}: {content}") - if remainder > 0: - lines.append(f" │ ... and {remainder} more non-blank line(s)") - lines.append( - f" └─ first subclause: {sub_marker} \"{self.first_sub_text}\" " - f"(line {self.first_sub_lineno})" - ) - return '\n'.join(lines) - - -# --------------------------------------------------------------------------- -# Core checker -# --------------------------------------------------------------------------- - -def check_file(path: Path) -> List[Violation]: - """Return all ISO clause violations in a single Markdown file. - - A violation occurs when a heading at level N is followed by non-blank - body text before the first heading at level N+1 (its first direct child - subclause). Headings inside fenced code blocks are ignored. - """ - try: - text = path.read_text(encoding='utf-8') - except OSError: - return [] - - raw_lines = text.splitlines() - - # ---- Pass 1: collect headings, skipping those inside code fences ---- - headings: List[Tuple[int, int, str]] = [] # (0-based line idx, level, text) - fenced: List[bool] = [] # True for each line idx that is inside a fence - in_fence = False - fence_char: Optional[str] = None - - for idx, line in enumerate(raw_lines): - fence_m = _FENCE_RE.match(line) - if fence_m: - marker = fence_m.group(1)[0] # ` or ~ - if not in_fence: - in_fence = True - fence_char = marker - elif marker == fence_char: - in_fence = False - fence_char = None - fenced.append(True) # fence delimiter lines are also excluded - continue - fenced.append(in_fence) - if in_fence: - continue - heading_m = _HEADING_RE.match(line) - if heading_m: - headings.append((idx, len(heading_m.group(1)), heading_m.group(2))) - - # ---- Pass 2: for each heading, find first direct child and check content ---- - violations: List[Violation] = [] - - for h_pos, (h_idx, h_level, h_text) in enumerate(headings): - # Find the first heading at level h_level+1 before any heading at - # level <= h_level (i.e., before we leave this clause's scope). - first_child: Optional[Tuple[int, int, str]] = None - for fc_idx, fc_level, fc_text in headings[h_pos + 1:]: - if fc_level <= h_level: - break # left scope without finding a child - if fc_level == h_level + 1: - first_child = (fc_idx, fc_level, fc_text) - break - # fc_level > h_level+1: deeper descendant — keep scanning - - if first_child is None: - continue # no direct child subclause → no violation possible - - fc_idx, fc_level, fc_text = first_child - - # Collect non-blank body lines between the heading and the first child, - # excluding lines inside fenced code blocks. - body_lines = [ - (h_idx + 1 + offset + 1, raw_lines[h_idx + 1 + offset]) - for offset, line in enumerate(raw_lines[h_idx + 1: fc_idx]) - if line.strip() and not fenced[h_idx + 1 + offset] - ] - - if body_lines: - violations.append(Violation( - file=path, - heading_lineno=h_idx + 1, - heading_level=h_level, - heading_text=h_text, - first_sub_lineno=fc_idx + 1, - first_sub_level=fc_level, - first_sub_text=fc_text, - body_lines=body_lines, - )) - - return violations - - -def check_spec(spec_root: Path) -> List[Violation]: - """Walk *spec_root* recursively and return all violations in .md files. - - Files are processed in a deterministic order (sorted by path) so that - output is stable across runs. - """ - all_violations: List[Violation] = [] - for dirpath, dirnames, filenames in os.walk(spec_root): - dirnames.sort() - for fname in sorted(filenames): - if fname.endswith('.md'): - all_violations.extend(check_file(Path(dirpath) / fname)) - return all_violations - - -# --------------------------------------------------------------------------- -# Report formatting -# --------------------------------------------------------------------------- - -def format_report( - violations: List[Violation], - context: int = DEFAULT_CONTEXT_LINES, - spec_root: Optional[Path] = None, -) -> str: - """Format all violations into a human-readable report string. - - If *spec_root* is given, file paths are shown relative to it. - Returns an empty string when there are no violations. - """ - if not violations: - return '' - - # Group by file for tidier output - by_file: dict = {} - for v in violations: - rel = v.file.relative_to(spec_root) if spec_root else v.file - by_file.setdefault(rel, []).append(v) - - sections: List[str] = [] - total = 0 - for rel_path, file_violations in by_file.items(): - block_lines = [f'{rel_path}'] - for v in file_violations: - # Rebase the file path for display - display_v = Violation( - file=rel_path, - heading_lineno=v.heading_lineno, - heading_level=v.heading_level, - heading_text=v.heading_text, - first_sub_lineno=v.first_sub_lineno, - first_sub_level=v.first_sub_level, - first_sub_text=v.first_sub_text, - body_lines=v.body_lines, - ) - block_lines.append(display_v.format(context=context)) - total += 1 - sections.append('\n'.join(block_lines)) - - file_count = len(by_file) - header = ( - f'ISO clause structure violations: ' - f'{total} violation(s) in {file_count} file(s)\n' - + '─' * 72 - ) - return header + '\n\n' + '\n\n'.join(sections) - - -# --------------------------------------------------------------------------- -# Command-line entry point -# --------------------------------------------------------------------------- - -def main(argv=None): - """Standalone entry point: python3 -m doc_build.iso_clause_lint """ - import argparse - parser = argparse.ArgumentParser( - description='Check Markdown files for ISO clause structure violations.' - ) - parser.add_argument( - 'path', - nargs='?', - default='.', - help='Spec root directory to scan (default: current directory)', - ) - parser.add_argument( - '--context', - type=int, - default=DEFAULT_CONTEXT_LINES, - metavar='N', - help=f'Number of body lines to show per violation (default: {DEFAULT_CONTEXT_LINES})', - ) - args = parser.parse_args(argv) - spec_root = Path(args.path).resolve() - violations = check_spec(spec_root) - report = format_report(violations, context=args.context, spec_root=spec_root) - if report: - print(report) - sys.exit(1) - else: - print('No ISO clause structure violations found.') - - -if __name__ == '__main__': - main() From 8ec0cfaa24e256e03a7d5b7c12fc01140ba4d534 Mon Sep 17 00:00:00 2001 From: Oleksiy Puzikov Date: Fri, 24 Apr 2026 13:33:19 +0300 Subject: [PATCH 2/5] Refactoring after the review --- doc_build/filters/filter_iso_xrefs.py | 480 +++++++++++++------------- 1 file changed, 242 insertions(+), 238 deletions(-) diff --git a/doc_build/filters/filter_iso_xrefs.py b/doc_build/filters/filter_iso_xrefs.py index 350a5bf..424c242 100644 --- a/doc_build/filters/filter_iso_xrefs.py +++ b/doc_build/filters/filter_iso_xrefs.py @@ -23,8 +23,12 @@ The YAML path is passed via the ISO_CLAUSE_MAP pandoc metadata variable. """ +import json import os import re +import subprocess +from pathlib import PurePosixPath +from urllib.parse import urlparse import yaml from pandocfilters import Link, Space, Str, stringify, toJSONFilter @@ -32,106 +36,58 @@ from shared_filter_utils import get_metadata_str # --------------------------------------------------------------------------- -# Module-level lazy state +# Pure helpers (no instance state) # --------------------------------------------------------------------------- -# anchor_id -> (number_str, heading_level, is_annex) -# e.g. "paths" -> ("7", 1, False), "element-ordering" -> ("7.2", 2, False) -_anchor_info = None - -# section_key -> root anchor string (only for numbered sections) -# e.g. "path_grammar" -> "paths" -_root_anchors = None +_CITATION_RE = re.compile(r'^\[(\d+)\](.*)', re.DOTALL) -# anchor_id -> section_key it belongs to -# used to detect when a fragment anchor lives in a different section than -# the link's file target (deduplication artefact), triggering a fallback -# to the section root anchor. -_anchor_section = None -# raw YAML dict (exceptions map) -_clause_map = None +def _derive_section_key(url_path): + """Extract the section key from a relative URL path. -# --------------------------------------------------------------------------- -# Pure helpers -# --------------------------------------------------------------------------- + Returns the folder name (for README.md-based sections) or the lowercased + file stem (for flat .md files), or None if the path is unrecognisable. -_HEADING_RE = re.compile(r'^(#{1,6})\s+(.*?)\s*$') -_ATTR_BLOCK_RE = re.compile(r'\s*\{([^}]*)\}\s*$') -_ID_IN_ATTRS_RE = re.compile(r'#([\w-]+)') -_CITATION_RE = re.compile(r'^\[(\d+)\](.*)', re.DOTALL) + Uses PurePosixPath because link targets are always POSIX-style paths + regardless of the host OS. urlparse strips any #fragment before handing + the path component to PurePosixPath. + Examples: + ../path_grammar/README.md -> 'path_grammar' + ../foundational_data_types/README.md#anchor -> 'foundational_data_types' + Foreword.md -> 'foreword' + glossary/README.md -> 'glossary' + """ + path = PurePosixPath(urlparse(url_path).path) # strip #fragment; keep path component only -def _pandoc_slug(heading_text): - """Approximate Pandoc's auto-identifier algorithm for raw heading text. + if path.suffix.lower() != '.md': # ignore non-Markdown links (images, URLs…) + return None - Input: heading text content (after stripping leading # markers) as it - appears in the raw Markdown file. May contain inline markup. + if path.name.lower() == 'readme.md': # folder-based section: key is the directory name + parent = path.parent.name # e.g. '../path_grammar/README.md' -> 'path_grammar' + return parent.lower() if parent not in ('', '.', '..') else None # guard against bare README.md at root - Algorithm: - 1. Strip inline markup delimiters, keeping text content. - 2. Lowercase. - 3. Letters and digits pass through; spaces become hyphens; underscores - and periods pass through; everything else is dropped. - 4. Collapse consecutive hyphens. - 5. Remove leading non-letter characters. - 6. Strip trailing hyphens. - """ - text = heading_text - - # Strip backtick code spans — keep inner text - text = re.sub(r'`([^`]*)`', r'\1', text) - text = text.replace('`', '') - - # Strip strong/emphasis — keep content - text = re.sub(r'\*\*([^*]+)\*\*', r'\1', text) - text = re.sub(r'\*([^*]+)\*', r'\1', text) - text = re.sub(r'__([^_]+)__', r'\1', text) - text = re.sub(r'_([^_]+)_', r'\1', text) - - # Strip links — keep link text - text = re.sub(r'\[([^\]]*)\]\([^)]*\)', r'\1', text) - - # Build identifier char by char (Pandoc rules) - chars = [] - for ch in text.lower(): - if ch.isalpha() or ch.isdigit(): - chars.append(ch) - elif ch in (' ', '\t'): - chars.append('-') - elif ch in ('_', '.'): - chars.append(ch) - # else: drop (punctuation, backslash, angle brackets, parens, etc.) - - slug = ''.join(chars) - slug = re.sub(r'-+', '-', slug) # collapse consecutive hyphens - slug = re.sub(r'^[^a-z]*', '', slug) # remove leading non-letters - slug = slug.rstrip('-') - return slug or 'section' + return path.stem.lower() or None # flat file: key is the filename without extension -def _derive_section_key(url_path): - """Extract the section key from a relative URL path (before any #fragment). +def _links_from_ast(node): + """Yield Link URLs in document order from a Pandoc AST fragment. - Returns the folder name (for README.md-based sections) or the lowercased - file stem (for flat .md files), or None if the path is unrecognisable. - - Examples: - ../path_grammar/README.md -> path_grammar - ../foundational_data_types/README.md#anchor -> foundational_data_types - Foreword.md -> foreword - glossary/README.md -> glossary + Recursively walks dicts and lists; stops descending into a Link node once + its URL has been yielded (avoids double-counting if link text is itself a + link, which is unusual but valid Markdown). """ - path = url_path.split('#')[0].replace('\\', '/') - parts = [p for p in path.split('/') if p not in ('..', '.', '')] - if not parts: - return None - last = parts[-1] - if last.lower() == 'readme.md': - return parts[-2].lower() if len(parts) >= 2 else None - if last.lower().endswith('.md'): - return last[:-3].lower() - return None + if isinstance(node, dict): + if node.get('t') == 'Link': + yield node['c'][2][0] + return + c = node.get('c') + if isinstance(c, list): + for item in c: + yield from _links_from_ast(item) + elif isinstance(node, list): + for item in node: + yield from _links_from_ast(item) def _iso_reference_text(number_str, level, is_annex): @@ -146,12 +102,8 @@ def _iso_reference_text(number_str, level, is_annex): return number_str -# --------------------------------------------------------------------------- -# Map builder -# --------------------------------------------------------------------------- - def _build_maps(yaml_path, artifacts_root): - """Build section-number maps by scanning combined_spec.md. + """Build section-number maps by parsing combined_spec.md with Pandoc. Returns (anchor_info, root_anchors, anchor_section, clause_map). @@ -160,9 +112,17 @@ def _build_maps(yaml_path, artifacts_root): anchor_section dict: anchor -> section_key it belongs to clause_map dict: raw YAML exceptions - On FileNotFoundError (test builds, partial environments) returns empty - maps so all links pass through unchanged. + Anchor IDs are taken directly from Pandoc's JSON AST — the first element + of each Header node's Attr — so they are guaranteed to match the IDs + Pandoc assigns in the rendered output, including deduplication. + + On FileNotFoundError or Pandoc failure (test builds, partial environments) + returns empty maps so all links pass through unchanged. """ + # ---- Load the clause exceptions map ---- + # The YAML file is an exceptions list: sections absent from it are + # auto-numbered sequentially; sections present are either suppressed + # (null) or assigned a specific annex letter ({annex: "A"}). try: with open(yaml_path, encoding='utf-8') as fh: clause_map = yaml.safe_load(fh) or {} @@ -172,81 +132,94 @@ def _build_maps(yaml_path, artifacts_root): readme_path = os.path.join(artifacts_root, 'README.md') combined_path = os.path.join(artifacts_root, 'combined_spec.md') + # ---- Determine the ordered list of spec sections ---- + # README.md contains one link per spec section in document order. + # Parsing it with Pandoc and walking all Link nodes gives the sequence + # of section keys (e.g. ['foreword', 'glossary', 'path_grammar', ...]). + # This list is later used to associate each level-1 heading in + # combined_spec.md with the section file it came from, which in turn + # determines which clause-map entry (if any) applies to it. + # Note: this relies on the assumption that each source file contributes + # exactly one level-1 heading to the combined document. try: - with open(readme_path, encoding='utf-8') as fh: - readme_text = fh.read() + result = subprocess.run( + ['pandoc', '-f', 'markdown', '-t', 'json', readme_path], + capture_output=True, text=True, check=True, + ) + readme_blocks = json.loads(result.stdout)['blocks'] section_order = [ k for k in ( - _derive_section_key(m) - for m in re.findall(r'\[.*?\]\(([^)]+)\)', readme_text) + _derive_section_key(url) + for url in _links_from_ast(readme_blocks) ) if k is not None ] - except FileNotFoundError: + except (FileNotFoundError, subprocess.CalledProcessError): return {}, {}, {}, clause_map + # ---- Parse the combined spec ---- + # combined_spec.md is the flat concatenation of all source files produced + # by flatten(). Pandoc parses it as a single document, which means anchor + # IDs are deduplicated globally (e.g. two sections both titled "References" + # get anchors "references" and "references-1"). Those final IDs are what + # the rendered HTML/PDF uses, so we must read them from Pandoc rather than + # computing them ourselves. try: - with open(combined_path, encoding='utf-8') as fh: - lines = fh.readlines() - except FileNotFoundError: + result = subprocess.run( + ['pandoc', '-f', 'markdown', '-t', 'json', combined_path], + capture_output=True, text=True, check=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError): return {}, {}, {}, clause_map + blocks = json.loads(result.stdout)['blocks'] + anchor_info = {} # anchor -> (number_str, level, is_annex) root_anchors = {} # section_key -> root anchor anchor_section = {} # anchor -> section_key (for cross-section disambiguation) - seen_anchors = {} # base_anchor -> next deduplication suffix (integer, 1-based) - - current_clause = None - current_is_annex = False - current_section_key = None # section_key of the section currently being scanned - subcounters = [0] * 7 # index = heading level 1-6; [0] unused - section_idx = 0 - auto_clause_counter = 0 - for line in lines: - m = _HEADING_RE.match(line.rstrip('\n')) - if not m: + # Numbering state carried forward as we walk the heading sequence. + current_clause = None # clause number string of the enclosing level-1 heading, + # or None when inside an unnumbered section + current_is_annex = False # True when the enclosing level-1 heading is an annex + current_section_key = None # section key of the file that contains the current heading + subcounters = [0] * 7 # per-level counters; subcounters[N] is the running count + # for level-N headings within the current clause. + # Index 0 is unused (headings start at level 1). + section_idx = 0 # position in section_order; advances on each level-1 heading + auto_clause_counter = 0 # increments for each level-1 heading not listed in clause_map + + for block in blocks: + if block['t'] != 'Header': continue - level = len(m.group(1)) - raw_text = m.group(2) - - # Extract explicit {#id} attribute if present - attr_m = _ATTR_BLOCK_RE.search(raw_text) - if attr_m: - raw_text = raw_text[:attr_m.start()] - id_m = _ID_IN_ATTRS_RE.search(attr_m.group(1)) - base_anchor = id_m.group(1) if id_m else _pandoc_slug(raw_text) - else: - base_anchor = _pandoc_slug(raw_text) - - # Global deduplication: mirrors Pandoc's behaviour - # First occurrence: use base; second: base-1; third: base-2; … - if base_anchor not in seen_anchors: - seen_anchors[base_anchor] = 1 - anchor = base_anchor - else: - n = seen_anchors[base_anchor] - anchor = f'{base_anchor}-{n}' - seen_anchors[base_anchor] = n + 1 + level = block['c'][0] + # Pandoc's Attr is [id, classes, kv-pairs]; the id field is the + # auto-generated, globally-deduplicated anchor for this heading. + anchor = block['c'][1][0] if level == 1: - # Determine which spec section this heading belongs to - if section_idx < len(section_order): - section_key = section_order[section_idx] - else: - section_key = None + # ---- Level-1 heading: start of a new top-level section ---- + + # Map this heading to its source file by consuming the next entry + # in section_order. If section_order is exhausted (e.g. a partial + # build with --only), section_key is None and the heading is treated + # as auto-numbered with no clause-map lookup. + section_key = section_order[section_idx] if section_idx < len(section_order) else None section_idx += 1 current_section_key = section_key if section_key is not None and section_key in clause_map: val = clause_map[section_key] if val is None: - # Explicitly unnumbered (Foreword, Introduction, etc.) + # Explicitly suppressed (Foreword, Introduction, etc.). + # No entry is added to anchor_info or root_anchors, so + # links to this section are left for filter_resolve_sections. current_clause = None current_is_annex = False - # Not stored in root_anchors or anchor_info elif isinstance(val, dict) and 'annex' in val: + # Annex: numbered independently with a letter (A, B, …). + # Subclauses will be A.1, A.1.1, etc. current_clause = str(val['annex']) current_is_annex = True subcounters = [0] * 7 @@ -254,7 +227,7 @@ def _build_maps(yaml_path, artifacts_root): anchor_info[anchor] = (current_clause, 1, True) anchor_section[anchor] = section_key else: - # Explicit clause number supplied in YAML (override) + # Explicit clause number override supplied directly in YAML. current_clause = str(val) current_is_annex = False subcounters = [0] * 7 @@ -262,7 +235,10 @@ def _build_maps(yaml_path, artifacts_root): anchor_info[anchor] = (current_clause, 1, False) anchor_section[anchor] = section_key else: - # Auto-number: next sequential clause + # Section absent from clause_map: assign the next sequential + # clause number. The counter is shared across all auto-numbered + # sections so that explicitly-numbered sections (YAML overrides) + # do not consume a slot in the sequence. auto_clause_counter += 1 current_clause = str(auto_clause_counter) current_is_annex = False @@ -274,17 +250,28 @@ def _build_maps(yaml_path, artifacts_root): anchor_section[anchor] = section_key else: # level >= 2 - if current_clause is None: - continue # subheading of an unnumbered section — skip + # ---- Subclause heading ---- + if current_clause is None: + # Inside an unnumbered section (null in clause_map); skip all + # subheadings so they don't appear in anchor_info and links to + # them are left unchanged for filter_resolve_sections. + continue + + # Increment the counter for this level and reset all deeper levels, + # mirroring the standard hierarchical numbering convention: + # e.g. entering a new level-3 heading resets level-4, 5, 6. subcounters[level] += 1 for d in range(level + 1, 7): subcounters[d] = 0 - parts = [current_clause] + [ - str(subcounters[i]) for i in range(2, level + 1) - ] - number_str = '.'.join(parts) + # Build the dotted number string: clause + each active sublevel. + # For a level-3 heading inside Clause 7: "7.2.1" where + # subcounters[2] and subcounters[3] are the active level-2 and + # level-3 counts. + number_str = '.'.join( + [current_clause] + [str(subcounters[i]) for i in range(2, level + 1)] + ) anchor_info[anchor] = (number_str, level, current_is_annex) if current_section_key is not None: anchor_section[anchor] = current_section_key @@ -293,109 +280,126 @@ def _build_maps(yaml_path, artifacts_root): # --------------------------------------------------------------------------- -# Lazy initialisation -# --------------------------------------------------------------------------- - -def _ensure_initialized(metadata): - global _anchor_info, _root_anchors, _anchor_section, _clause_map - if _anchor_info is not None: - return - try: - yaml_path = get_metadata_str(metadata, 'ISO_CLAUSE_MAP') - except KeyError: - # Metadata variable not set — disable filter (safe no-op) - _anchor_info = {} - _root_anchors = {} - _anchor_section = {} - _clause_map = {} - return - _anchor_info, _root_anchors, _anchor_section, _clause_map = _build_maps( - yaml_path, os.getcwd() - ) - - -# --------------------------------------------------------------------------- -# Per-element handlers +# Filter class # --------------------------------------------------------------------------- -def _handle_link(value): - url = value[2][0] - title = value[2][1] +class IsoXrefFilter: + """Stateful Pandoc filter for ISO cross-references. - # ---- External URL ---- - if url.startswith(('http://', 'https://')): - link_text = stringify(value[1]) - if link_text.strip() == url.strip(): - return None # text already shows the URL — don't duplicate - original = Link(value[0], value[1], value[2]) - return [original, Str(' ('), Str(url), Str(')')] - - # ---- Intra-document anchor or empty — leave alone ---- - if not url or url.startswith('#'): - return None + Holds the clause-number maps as instance attributes, initialised lazily + on the first callback invocation so that the Pandoc metadata (which + carries the path to iso_clause_map.yaml) is available at that point. - # ---- Internal relative link ---- - parts = url.split('#', 1) - url_path = parts[0] - fragment = parts[1] if len(parts) == 2 else None - - section_key = _derive_section_key(url_path) - if section_key is None: - return None + Usage: + toJSONFilter(IsoXrefFilter()) + """ - # Root anchor must be known (section present in the combined build) - root_anchor = _root_anchors.get(section_key) - if root_anchor is None: - # Section either not in this build or explicitly unnumbered + def __init__(self): + # All four maps start as None to indicate "not yet initialised". + # They are populated on the first call to __call__ and then reused + # for every subsequent node in the same filter run. + self._anchor_info = None # anchor -> (number_str, level, is_annex) + self._root_anchors = None # section_key -> root anchor + self._anchor_section = None # anchor -> section_key + self._clause_map = None # raw YAML dict + + def _initialize(self, metadata): + """Populate the maps from iso_clause_map.yaml and combined_spec.md. + + Called once on the first AST node; subsequent calls are a no-op. + If the ISO_CLAUSE_MAP metadata key is absent the filter is disabled + and all nodes pass through unchanged. + """ + if self._anchor_info is not None: + return + try: + yaml_path = get_metadata_str(metadata, 'ISO_CLAUSE_MAP') + except KeyError: + # Metadata variable not set — disable filter (safe no-op). + self._anchor_info = {} + self._root_anchors = {} + self._anchor_section = {} + self._clause_map = {} + return + ( + self._anchor_info, + self._root_anchors, + self._anchor_section, + self._clause_map, + ) = _build_maps(yaml_path, os.getcwd()) + + def __call__(self, key, value, fmt, metadata): + self._initialize(metadata) + if key == 'Link': + return self._handle_link(value) + if key == 'Str': + return self._handle_str(value) return None - if fragment: - info = _anchor_info.get(fragment) - if info is None: - return None # stale or unknown anchor — leave for resolve_sections - # Guard against deduplication artefacts: if the anchor that happens to - # have this name actually lives in a *different* section (e.g. a glossary - # entry "composition" shadows the top-level "# Composition" heading which - # was deduplicated to "composition-1"), fall back to the section root. - if _anchor_section.get(fragment) != section_key: - number_str, level, is_annex = _anchor_info[root_anchor] - target = '#' + root_anchor + def _handle_link(self, value): + url = value[2][0] + + # ---- External URL ---- + if url.startswith(('http://', 'https://')): + link_text = stringify(value[1]) + if link_text.strip() == url.strip(): + return None # text already shows the URL — don't duplicate + original = Link(value[0], value[1], value[2]) + return [original, Str(' ('), Str(url), Str(')')] + + # ---- Intra-document anchor or empty — leave alone ---- + if not url or url.startswith('#'): + return None + + # ---- Internal relative link ---- + parts = url.split('#', 1) + url_path = parts[0] + fragment = parts[1] if len(parts) == 2 else None + + section_key = _derive_section_key(url_path) + if section_key is None: + return None + + # Root anchor must be known (section present in the combined build) + root_anchor = self._root_anchors.get(section_key) + if root_anchor is None: + # Section either not in this build or explicitly unnumbered + return None + + if fragment: + info = self._anchor_info.get(fragment) + if info is None: + return None # stale or unknown anchor — leave for resolve_sections + # Guard against deduplication artefacts: if the anchor that happens to + # have this name actually lives in a *different* section (e.g. a glossary + # entry "composition" shadows the top-level "# Composition" heading which + # was deduplicated to "composition-1"), use the section root's number. + if self._anchor_section.get(fragment) != section_key: + number_str, level, is_annex = self._anchor_info[root_anchor] + else: + number_str, level, is_annex = info else: - number_str, level, is_annex = info - target = '#' + fragment - else: - number_str, level, is_annex = _anchor_info[root_anchor] - target = '#' + root_anchor - - iso_text = _iso_reference_text(number_str, level, is_annex) - return Link(value[0], [Str(iso_text)], [target, title]) + number_str, level, is_annex = self._anchor_info[root_anchor] + iso_text = _iso_reference_text(number_str, level, is_annex) + return Link(value[0], [Str(iso_text)], value[2]) -def _handle_str(value): - """Expand inline bibliography citation [N] to 'Reference [N]'.""" - m = _CITATION_RE.match(value) - if not m: - return None - num = m.group(1) - trailing = m.group(2) - result = [Str('Reference'), Space(), Str(f'[{num}]')] - if trailing: - result.append(Str(trailing)) - return result + def _handle_str(self, value): + """Expand inline bibliography citation [N] to 'Reference [N]'.""" + m = _CITATION_RE.match(value) + if not m: + return None + num = m.group(1) + trailing = m.group(2) + result = [Str('Reference'), Space(), Str(f'[{num}]')] + if trailing: + result.append(Str(trailing)) + return result # --------------------------------------------------------------------------- -# Main filter entry point +# Entry point # --------------------------------------------------------------------------- -def iso_xrefs(key, value, fmt, metadata): - _ensure_initialized(metadata) - if key == 'Link': - return _handle_link(value) - if key == 'Str': - return _handle_str(value) - return None - - if __name__ == '__main__': - toJSONFilter(iso_xrefs) + toJSONFilter(IsoXrefFilter()) From 70a296a8a9bd3cd160885f7ed41224eebb809c69 Mon Sep 17 00:00:00 2001 From: Oleksiy Puzikov Date: Fri, 24 Apr 2026 15:55:06 +0300 Subject: [PATCH 3/5] Fixes after testing --- doc_build/doc_builder.py | 14 ++- .../filters/filter_convert_mathblocks.py | 0 doc_build/filters/filter_iso_xrefs.py | 95 ++++++++++++++++++- doc_build/filters/filter_render_diff.py | 0 doc_build/iso_clause_map.yaml | 1 + 5 files changed, 106 insertions(+), 4 deletions(-) mode change 100644 => 100755 doc_build/filters/filter_convert_mathblocks.py mode change 100644 => 100755 doc_build/filters/filter_iso_xrefs.py mode change 100644 => 100755 doc_build/filters/filter_render_diff.py diff --git a/doc_build/doc_builder.py b/doc_build/doc_builder.py index 7d154b9..e542671 100644 --- a/doc_build/doc_builder.py +++ b/doc_build/doc_builder.py @@ -24,9 +24,19 @@ try: import yaml except ImportError: - sys.exit("Please install the PyYAML package: pip install PyYAML.") + sys.exit("Please install the PyYAML package: pip install PyYAML") +# Allow Python3.10 +@contextlib.contextmanager +def contextlib_chdir(path): + old_dir = os.getcwd() + try: + os.chdir(path) + yield + finally: + os.chdir(old_dir) + # The output format for the published aousd_core_spec.md file. We want it to be # in a widely / known format, that still has a decent set of extensions to # enable features used in these documents, so we again go with `gfm`. @@ -239,7 +249,7 @@ def _render_combined( # Set the cwd to the artifacts dir because it's easier for some filters to work relatively to it. # contextlib.chdir restores the previous cwd on exit (even on exception), so on Windows the # process doesn't hold a handle to the directory and temp-dir cleanup succeeds. - with contextlib.chdir(artifacts_dir): + with contextlib_chdir(artifacts_dir): shared_command = [ "--defaults", spec, diff --git a/doc_build/filters/filter_convert_mathblocks.py b/doc_build/filters/filter_convert_mathblocks.py old mode 100644 new mode 100755 diff --git a/doc_build/filters/filter_iso_xrefs.py b/doc_build/filters/filter_iso_xrefs.py old mode 100644 new mode 100755 index 424c242..53ec67b --- a/doc_build/filters/filter_iso_xrefs.py +++ b/doc_build/filters/filter_iso_xrefs.py @@ -27,10 +27,15 @@ import os import re import subprocess +import sys from pathlib import PurePosixPath from urllib.parse import urlparse -import yaml +try: + import yaml +except ImportError: + sys.exit("Please install the PyYAML package: pip install PyYAML") + from pandocfilters import Link, Space, Str, stringify, toJSONFilter from shared_filter_utils import get_metadata_str @@ -154,9 +159,30 @@ def _build_maps(yaml_path, artifacts_root): ) if k is not None ] + # print('[filter_iso_xrefs] section_order from README.md:', file=sys.stderr) + # for i, key in enumerate(section_order): + # print(f' [{i:2d}] {key}', file=sys.stderr) except (FileNotFoundError, subprocess.CalledProcessError): return {}, {}, {}, clause_map + # Build a lookup for sections that are null in the YAML but absent from + # section_order — i.e., injected into combined_spec.md by the build script + # (e.g. copyright notices) outside the README.md link flow. Their anchors + # are derived by Pandoc from the heading text: hyphens where the section key + # has underscores, but otherwise the same words. Storing the mapping as + # anchor → section_key lets us detect them cheaply during the heading walk + # without consuming a section_order slot. + section_order_set = set(section_order) + anchor_to_extra = { + key.replace('_', '-'): key # e.g. 'copyright_license_...' → 'copyright-license-...' + for key, val in clause_map.items() + if val is None and key not in section_order_set # null YAML entry not reachable from README.md + } + # if anchor_to_extra: + # print('[filter_iso_xrefs] extra null sections (injected outside README.md):', file=sys.stderr) + # for anchor_key, section_key in sorted(anchor_to_extra.items()): + # print(f' anchor={anchor_key!r} → section_key={section_key!r}', file=sys.stderr) + # ---- Parse the combined spec ---- # combined_spec.md is the flat concatenation of all source files produced # by flatten(). Pandoc parses it as a single document, which means anchor @@ -201,6 +227,22 @@ def _build_maps(yaml_path, artifacts_root): if level == 1: # ---- Level-1 heading: start of a new top-level section ---- + # Check first whether this heading was injected by the build script + # outside the README.md link flow (e.g. copyright notices). These + # sections are in the YAML as null but have no entry in section_order, + # so they must not consume a section_order slot. + if anchor in anchor_to_extra: + current_clause = None + current_is_annex = False + current_section_key = anchor_to_extra[anchor] + # heading_text = stringify(block['c'][2]) + # print( + # f'[filter_iso_xrefs] L1 heading (extra null): ' + # f'anchor={anchor!r} section_key={current_section_key!r} text={heading_text!r}', + # file=sys.stderr, + # ) + continue # do not advance section_idx + # Map this heading to its source file by consuming the next entry # in section_order. If section_order is exhausted (e.g. a partial # build with --only), section_key is None and the heading is treated @@ -209,6 +251,13 @@ def _build_maps(yaml_path, artifacts_root): section_idx += 1 current_section_key = section_key + # heading_text = stringify(block['c'][2]) + # print( + # f'[filter_iso_xrefs] L1 heading #{section_idx:2d}: ' + # f'anchor={anchor!r:40} section_key={section_key!r:30} text={heading_text!r}', + # file=sys.stderr, + # ) + if section_key is not None and section_key in clause_map: val = clause_map[section_key] if val is None: @@ -328,14 +377,52 @@ def _initialize(self, metadata): self._clause_map, ) = _build_maps(yaml_path, os.getcwd()) + # Debug: uncomment to print the section-key → clause number mapping during a build. + # print('[filter_iso_xrefs] clause map (section_key → anchor → number):', + # file=sys.stderr) + # for section_key, anchor in sorted(self._root_anchors.items()): + # info = self._anchor_info.get(anchor) + # if info: + # number_str, _level, is_annex = info + # label = f'Annex {number_str}' if is_annex else f'Clause {number_str}' + # else: + # label = '(unnumbered)' + # print(f' {section_key:<40} {anchor:<40} {label}', file=sys.stderr) + def __call__(self, key, value, fmt, metadata): self._initialize(metadata) + if key == 'Header': + return self._handle_header(value) if key == 'Link': return self._handle_link(value) if key == 'Str': return self._handle_str(value) return None + def _handle_header(self, value): + """Add the 'unnumbered' class to headings that carry no ISO clause number. + + Pandoc's --number-sections counts every heading sequentially, including + front/back matter (Foreword, Introduction, Closing, copyright notices). + Marking those headings as 'unnumbered' makes Pandoc skip them in its + counter, so the rendered section numbers match the ISO clause numbers + assigned by this filter. + + A heading is considered unnumbered if its anchor is absent from + _anchor_info, which only contains anchors of numbered clauses and their + subclauses. + + Header value layout: [level, [id, classes, kv-pairs], inlines] + """ + anchor = value[1][0] # Pandoc-assigned heading id + if anchor in self._anchor_info: + return None # numbered clause — leave Pandoc's counter running + classes = value[1][1] + if 'unnumbered' in classes: + return None # already marked — nothing to do + new_attr = [value[1][0], classes + ['unnumbered'], value[1][2]] + return {'t': 'Header', 'c': [value[0], new_attr, value[2]]} + def _handle_link(self, value): url = value[2][0] @@ -382,7 +469,11 @@ def _handle_link(self, value): number_str, level, is_annex = self._anchor_info[root_anchor] iso_text = _iso_reference_text(number_str, level, is_annex) - return Link(value[0], [Str(iso_text)], value[2]) + link_text = stringify(value[1]) + if iso_text in link_text: + return None # already annotated — don't duplicate + original = Link(value[0], value[1], value[2]) + return [original, Str(f' ({iso_text})')] def _handle_str(self, value): """Expand inline bibliography citation [N] to 'Reference [N]'.""" diff --git a/doc_build/filters/filter_render_diff.py b/doc_build/filters/filter_render_diff.py old mode 100644 new mode 100755 diff --git a/doc_build/iso_clause_map.yaml b/doc_build/iso_clause_map.yaml index 512c14f..b828d07 100644 --- a/doc_build/iso_clause_map.yaml +++ b/doc_build/iso_clause_map.yaml @@ -17,3 +17,4 @@ foreword: null # front matter, not a numbered clause introduction: null # front matter, not a numbered clause closing: null # back matter; change to {annex: "A"} to make it Annex A +copyright_license_agreement_for_aousd_draft_deliverables_for_internal_evaluation: null From 92c360116f9b49bba68e84a1e13f14890f01267b Mon Sep 17 00:00:00 2001 From: Oleksiy Puzikov Date: Mon, 27 Apr 2026 13:41:10 +0300 Subject: [PATCH 4/5] before refactoring --- doc_build/filters/filter_iso_xrefs.py | 479 ++++++++++++-------------- 1 file changed, 217 insertions(+), 262 deletions(-) diff --git a/doc_build/filters/filter_iso_xrefs.py b/doc_build/filters/filter_iso_xrefs.py index 53ec67b..fc55b25 100755 --- a/doc_build/filters/filter_iso_xrefs.py +++ b/doc_build/filters/filter_iso_xrefs.py @@ -86,10 +86,7 @@ def _links_from_ast(node): if node.get('t') == 'Link': yield node['c'][2][0] return - c = node.get('c') - if isinstance(c, list): - for item in c: - yield from _links_from_ast(item) + yield from _links_from_ast(node.get('c')) elif isinstance(node, list): for item in node: yield from _links_from_ast(item) @@ -107,227 +104,6 @@ def _iso_reference_text(number_str, level, is_annex): return number_str -def _build_maps(yaml_path, artifacts_root): - """Build section-number maps by parsing combined_spec.md with Pandoc. - - Returns (anchor_info, root_anchors, anchor_section, clause_map). - - anchor_info dict: anchor -> (number_str, level, is_annex) - root_anchors dict: section_key -> root anchor (numbered sections only) - anchor_section dict: anchor -> section_key it belongs to - clause_map dict: raw YAML exceptions - - Anchor IDs are taken directly from Pandoc's JSON AST — the first element - of each Header node's Attr — so they are guaranteed to match the IDs - Pandoc assigns in the rendered output, including deduplication. - - On FileNotFoundError or Pandoc failure (test builds, partial environments) - returns empty maps so all links pass through unchanged. - """ - # ---- Load the clause exceptions map ---- - # The YAML file is an exceptions list: sections absent from it are - # auto-numbered sequentially; sections present are either suppressed - # (null) or assigned a specific annex letter ({annex: "A"}). - try: - with open(yaml_path, encoding='utf-8') as fh: - clause_map = yaml.safe_load(fh) or {} - except FileNotFoundError: - return {}, {}, {}, {} - - readme_path = os.path.join(artifacts_root, 'README.md') - combined_path = os.path.join(artifacts_root, 'combined_spec.md') - - # ---- Determine the ordered list of spec sections ---- - # README.md contains one link per spec section in document order. - # Parsing it with Pandoc and walking all Link nodes gives the sequence - # of section keys (e.g. ['foreword', 'glossary', 'path_grammar', ...]). - # This list is later used to associate each level-1 heading in - # combined_spec.md with the section file it came from, which in turn - # determines which clause-map entry (if any) applies to it. - # Note: this relies on the assumption that each source file contributes - # exactly one level-1 heading to the combined document. - try: - result = subprocess.run( - ['pandoc', '-f', 'markdown', '-t', 'json', readme_path], - capture_output=True, text=True, check=True, - ) - readme_blocks = json.loads(result.stdout)['blocks'] - section_order = [ - k for k in ( - _derive_section_key(url) - for url in _links_from_ast(readme_blocks) - ) - if k is not None - ] - # print('[filter_iso_xrefs] section_order from README.md:', file=sys.stderr) - # for i, key in enumerate(section_order): - # print(f' [{i:2d}] {key}', file=sys.stderr) - except (FileNotFoundError, subprocess.CalledProcessError): - return {}, {}, {}, clause_map - - # Build a lookup for sections that are null in the YAML but absent from - # section_order — i.e., injected into combined_spec.md by the build script - # (e.g. copyright notices) outside the README.md link flow. Their anchors - # are derived by Pandoc from the heading text: hyphens where the section key - # has underscores, but otherwise the same words. Storing the mapping as - # anchor → section_key lets us detect them cheaply during the heading walk - # without consuming a section_order slot. - section_order_set = set(section_order) - anchor_to_extra = { - key.replace('_', '-'): key # e.g. 'copyright_license_...' → 'copyright-license-...' - for key, val in clause_map.items() - if val is None and key not in section_order_set # null YAML entry not reachable from README.md - } - # if anchor_to_extra: - # print('[filter_iso_xrefs] extra null sections (injected outside README.md):', file=sys.stderr) - # for anchor_key, section_key in sorted(anchor_to_extra.items()): - # print(f' anchor={anchor_key!r} → section_key={section_key!r}', file=sys.stderr) - - # ---- Parse the combined spec ---- - # combined_spec.md is the flat concatenation of all source files produced - # by flatten(). Pandoc parses it as a single document, which means anchor - # IDs are deduplicated globally (e.g. two sections both titled "References" - # get anchors "references" and "references-1"). Those final IDs are what - # the rendered HTML/PDF uses, so we must read them from Pandoc rather than - # computing them ourselves. - try: - result = subprocess.run( - ['pandoc', '-f', 'markdown', '-t', 'json', combined_path], - capture_output=True, text=True, check=True, - ) - except (FileNotFoundError, subprocess.CalledProcessError): - return {}, {}, {}, clause_map - - blocks = json.loads(result.stdout)['blocks'] - - anchor_info = {} # anchor -> (number_str, level, is_annex) - root_anchors = {} # section_key -> root anchor - anchor_section = {} # anchor -> section_key (for cross-section disambiguation) - - # Numbering state carried forward as we walk the heading sequence. - current_clause = None # clause number string of the enclosing level-1 heading, - # or None when inside an unnumbered section - current_is_annex = False # True when the enclosing level-1 heading is an annex - current_section_key = None # section key of the file that contains the current heading - subcounters = [0] * 7 # per-level counters; subcounters[N] is the running count - # for level-N headings within the current clause. - # Index 0 is unused (headings start at level 1). - section_idx = 0 # position in section_order; advances on each level-1 heading - auto_clause_counter = 0 # increments for each level-1 heading not listed in clause_map - - for block in blocks: - if block['t'] != 'Header': - continue - - level = block['c'][0] - # Pandoc's Attr is [id, classes, kv-pairs]; the id field is the - # auto-generated, globally-deduplicated anchor for this heading. - anchor = block['c'][1][0] - - if level == 1: - # ---- Level-1 heading: start of a new top-level section ---- - - # Check first whether this heading was injected by the build script - # outside the README.md link flow (e.g. copyright notices). These - # sections are in the YAML as null but have no entry in section_order, - # so they must not consume a section_order slot. - if anchor in anchor_to_extra: - current_clause = None - current_is_annex = False - current_section_key = anchor_to_extra[anchor] - # heading_text = stringify(block['c'][2]) - # print( - # f'[filter_iso_xrefs] L1 heading (extra null): ' - # f'anchor={anchor!r} section_key={current_section_key!r} text={heading_text!r}', - # file=sys.stderr, - # ) - continue # do not advance section_idx - - # Map this heading to its source file by consuming the next entry - # in section_order. If section_order is exhausted (e.g. a partial - # build with --only), section_key is None and the heading is treated - # as auto-numbered with no clause-map lookup. - section_key = section_order[section_idx] if section_idx < len(section_order) else None - section_idx += 1 - current_section_key = section_key - - # heading_text = stringify(block['c'][2]) - # print( - # f'[filter_iso_xrefs] L1 heading #{section_idx:2d}: ' - # f'anchor={anchor!r:40} section_key={section_key!r:30} text={heading_text!r}', - # file=sys.stderr, - # ) - - if section_key is not None and section_key in clause_map: - val = clause_map[section_key] - if val is None: - # Explicitly suppressed (Foreword, Introduction, etc.). - # No entry is added to anchor_info or root_anchors, so - # links to this section are left for filter_resolve_sections. - current_clause = None - current_is_annex = False - elif isinstance(val, dict) and 'annex' in val: - # Annex: numbered independently with a letter (A, B, …). - # Subclauses will be A.1, A.1.1, etc. - current_clause = str(val['annex']) - current_is_annex = True - subcounters = [0] * 7 - root_anchors[section_key] = anchor - anchor_info[anchor] = (current_clause, 1, True) - anchor_section[anchor] = section_key - else: - # Explicit clause number override supplied directly in YAML. - current_clause = str(val) - current_is_annex = False - subcounters = [0] * 7 - root_anchors[section_key] = anchor - anchor_info[anchor] = (current_clause, 1, False) - anchor_section[anchor] = section_key - else: - # Section absent from clause_map: assign the next sequential - # clause number. The counter is shared across all auto-numbered - # sections so that explicitly-numbered sections (YAML overrides) - # do not consume a slot in the sequence. - auto_clause_counter += 1 - current_clause = str(auto_clause_counter) - current_is_annex = False - subcounters = [0] * 7 - if section_key is not None: - root_anchors[section_key] = anchor - anchor_info[anchor] = (current_clause, 1, False) - if section_key is not None: - anchor_section[anchor] = section_key - - else: # level >= 2 - # ---- Subclause heading ---- - - if current_clause is None: - # Inside an unnumbered section (null in clause_map); skip all - # subheadings so they don't appear in anchor_info and links to - # them are left unchanged for filter_resolve_sections. - continue - - # Increment the counter for this level and reset all deeper levels, - # mirroring the standard hierarchical numbering convention: - # e.g. entering a new level-3 heading resets level-4, 5, 6. - subcounters[level] += 1 - for d in range(level + 1, 7): - subcounters[d] = 0 - - # Build the dotted number string: clause + each active sublevel. - # For a level-3 heading inside Clause 7: "7.2.1" where - # subcounters[2] and subcounters[3] are the active level-2 and - # level-3 counts. - number_str = '.'.join( - [current_clause] + [str(subcounters[i]) for i in range(2, level + 1)] - ) - anchor_info[anchor] = (number_str, level, current_is_annex) - if current_section_key is not None: - anchor_section[anchor] = current_section_key - - return anchor_info, root_anchors, anchor_section, clause_map - - # --------------------------------------------------------------------------- # Filter class # --------------------------------------------------------------------------- @@ -335,22 +111,222 @@ def _build_maps(yaml_path, artifacts_root): class IsoXrefFilter: """Stateful Pandoc filter for ISO cross-references. - Holds the clause-number maps as instance attributes, initialised lazily - on the first callback invocation so that the Pandoc metadata (which - carries the path to iso_clause_map.yaml) is available at that point. + Clause-number maps are set as instance attributes on the first callback + invocation (lazy initialisation), because the Pandoc metadata that carries + the iso_clause_map.yaml path is not available until then. Usage: toJSONFilter(IsoXrefFilter()) """ - def __init__(self): - # All four maps start as None to indicate "not yet initialised". - # They are populated on the first call to __call__ and then reused - # for every subsequent node in the same filter run. - self._anchor_info = None # anchor -> (number_str, level, is_annex) - self._root_anchors = None # section_key -> root anchor - self._anchor_section = None # anchor -> section_key - self._clause_map = None # raw YAML dict + def _build_maps(self, yaml_path, artifacts_root): + """Build section-number maps by parsing combined_spec.md with Pandoc. + + Populates self._anchor_info, self._root_anchors, self._anchor_section, + and self._clause_map directly. + + anchor_info dict: anchor -> (number_str, level, is_annex) + root_anchors dict: section_key -> root anchor (numbered sections only) + anchor_section dict: anchor -> section_key it belongs to + clause_map dict: raw YAML exceptions + + Anchor IDs are taken directly from Pandoc's JSON AST — the first element + of each Header node's Attr — so they are guaranteed to match the IDs + Pandoc assigns in the rendered output, including deduplication. + + On FileNotFoundError or Pandoc failure (test builds, partial environments) + all maps are set to empty so all links pass through unchanged. + """ + # ---- Load the clause exceptions map ---- + # The YAML file is an exceptions list: sections absent from it are + # auto-numbered sequentially; sections present are either suppressed + # (null) or assigned a specific annex letter ({annex: "A"}). + try: + with open(yaml_path, encoding='utf-8') as fh: + self._clause_map = yaml.safe_load(fh) or {} + except FileNotFoundError: + self._anchor_info = {} + self._root_anchors = {} + self._anchor_section = {} + self._clause_map = {} + return + + readme_path = os.path.join(artifacts_root, 'README.md') + combined_path = os.path.join(artifacts_root, 'combined_spec.md') + + # ---- Determine the ordered list of spec sections ---- + # README.md contains one link per spec section in document order. + # Parsing it with Pandoc and walking all Link nodes gives the sequence + # of section keys (e.g. ['foreword', 'glossary', 'path_grammar', ...]). + # This list is later used to associate each level-1 heading in + # combined_spec.md with the section file it came from, which in turn + # determines which clause-map entry (if any) applies to it. + # Note: this relies on the assumption that each source file contributes + # exactly one level-1 heading to the combined document. + try: + result = subprocess.run( + ['pandoc', '-f', 'markdown', '-t', 'json', readme_path], + capture_output=True, text=True, check=True, + ) + readme_blocks = json.loads(result.stdout)['blocks'] + section_order = [ + k for k in ( + _derive_section_key(url) + for url in _links_from_ast(readme_blocks) + ) + if k is not None + ] + except (FileNotFoundError, subprocess.CalledProcessError): + self._anchor_info = {} + self._root_anchors = {} + self._anchor_section = {} + return + + # Build a lookup for sections that are null in the YAML but absent from + # section_order — i.e., injected into combined_spec.md by the build script + # (e.g. copyright notices) outside the README.md link flow. Their anchors + # are derived by Pandoc from the heading text: hyphens where the section key + # has underscores, but otherwise the same words. Storing the mapping as + # anchor → section_key lets us detect them cheaply during the heading walk + # without consuming a section_order slot. + section_order_set = set(section_order) + anchor_to_extra = { + key.replace('_', '-'): key # e.g. 'copyright_license_...' → 'copyright-license-...' + for key, val in self._clause_map.items() + if val is None and key not in section_order_set # null YAML entry not reachable from README.md + } + + # ---- Parse the combined spec ---- + # combined_spec.md is the flat concatenation of all source files produced + # by flatten(). Pandoc parses it as a single document, which means anchor + # IDs are deduplicated globally (e.g. two sections both titled "References" + # get anchors "references" and "references-1"). Those final IDs are what + # the rendered HTML/PDF uses, so we must read them from Pandoc rather than + # computing them ourselves. + try: + result = subprocess.run( + ['pandoc', '-f', 'markdown', '-t', 'json', combined_path], + capture_output=True, text=True, check=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError): + self._anchor_info = {} + self._root_anchors = {} + self._anchor_section = {} + return + + blocks = json.loads(result.stdout)['blocks'] + + self._anchor_info = {} # anchor -> (number_str, level, is_annex) + self._root_anchors = {} # section_key -> root anchor + self._anchor_section = {} # anchor -> section_key (for cross-section disambiguation) + + # Numbering state carried forward as we walk the heading sequence. + current_clause = None # clause number string of the enclosing level-1 heading, + # or None when inside an unnumbered section + current_is_annex = False # True when the enclosing level-1 heading is an annex + current_section_key = None # section key of the file that contains the current heading + subcounters = [0] * 7 # per-level counters; subcounters[N] is the running count + # for level-N headings within the current clause. + # Index 0 is unused (headings start at level 1). + section_idx = 0 # position in section_order; advances on each level-1 heading + auto_clause_counter = 0 # increments for each level-1 heading not listed in clause_map + + for block in blocks: + if block['t'] != 'Header': + continue + + level = block['c'][0] + # Pandoc's Attr is [id, classes, kv-pairs]; the id field is the + # auto-generated, globally-deduplicated anchor for this heading. + anchor = block['c'][1][0] + + if level == 1: + # ---- Level-1 heading: start of a new top-level section ---- + + # Check first whether this heading was injected by the build script + # outside the README.md link flow (e.g. copyright notices). These + # sections are in the YAML as null but have no entry in section_order, + # so they must not consume a section_order slot. + if anchor in anchor_to_extra: + current_clause = None + current_is_annex = False + current_section_key = anchor_to_extra[anchor] + continue # do not advance section_idx + + # Map this heading to its source file by consuming the next entry + # in section_order. If section_order is exhausted (e.g. a partial + # build with --only), section_key is None and the heading is treated + # as auto-numbered with no clause-map lookup. + section_key = section_order[section_idx] if section_idx < len(section_order) else None + section_idx += 1 + current_section_key = section_key + + if section_key is not None and section_key in self._clause_map: + val = self._clause_map[section_key] + if val is None: + # Explicitly suppressed (Foreword, Introduction, etc.). + # No entry is added to anchor_info or root_anchors, so + # links to this section are left for filter_resolve_sections. + current_clause = None + current_is_annex = False + elif isinstance(val, dict) and 'annex' in val: + # Annex: numbered independently with a letter (A, B, …). + # Subclauses will be A.1, A.1.1, etc. + current_clause = str(val['annex']) + current_is_annex = True + subcounters = [0] * 7 + self._root_anchors[section_key] = anchor + self._anchor_info[anchor] = (current_clause, 1, True) + self._anchor_section[anchor] = section_key + else: + # Explicit clause number override supplied directly in YAML. + current_clause = str(val) + current_is_annex = False + subcounters = [0] * 7 + self._root_anchors[section_key] = anchor + self._anchor_info[anchor] = (current_clause, 1, False) + self._anchor_section[anchor] = section_key + else: + # Section absent from clause_map: assign the next sequential + # clause number. The counter is shared across all auto-numbered + # sections so that explicitly-numbered sections (YAML overrides) + # do not consume a slot in the sequence. + auto_clause_counter += 1 + current_clause = str(auto_clause_counter) + current_is_annex = False + subcounters = [0] * 7 + if section_key is not None: + self._root_anchors[section_key] = anchor + self._anchor_info[anchor] = (current_clause, 1, False) + if section_key is not None: + self._anchor_section[anchor] = section_key + + else: # level >= 2 + # ---- Subclause heading ---- + + if current_clause is None: + # Inside an unnumbered section (null in clause_map); skip all + # subheadings so they don't appear in anchor_info and links to + # them are left unchanged for filter_resolve_sections. + continue + + # Increment the counter for this level and reset all deeper levels, + # mirroring the standard hierarchical numbering convention: + # e.g. entering a new level-3 heading resets level-4, 5, 6. + subcounters[level] += 1 + for d in range(level + 1, 7): + subcounters[d] = 0 + + # Build the dotted number string: clause + each active sublevel. + # For a level-3 heading inside Clause 7: "7.2.1" where + # subcounters[2] and subcounters[3] are the active level-2 and + # level-3 counts. + number_str = '.'.join( + [current_clause] + [str(subcounters[i]) for i in range(2, level + 1)] + ) + self._anchor_info[anchor] = (number_str, level, current_is_annex) + if current_section_key is not None: + self._anchor_section[anchor] = current_section_key def _initialize(self, metadata): """Populate the maps from iso_clause_map.yaml and combined_spec.md. @@ -359,7 +335,7 @@ def _initialize(self, metadata): If the ISO_CLAUSE_MAP metadata key is absent the filter is disabled and all nodes pass through unchanged. """ - if self._anchor_info is not None: + if hasattr(self, '_anchor_info'): return try: yaml_path = get_metadata_str(metadata, 'ISO_CLAUSE_MAP') @@ -370,24 +346,7 @@ def _initialize(self, metadata): self._anchor_section = {} self._clause_map = {} return - ( - self._anchor_info, - self._root_anchors, - self._anchor_section, - self._clause_map, - ) = _build_maps(yaml_path, os.getcwd()) - - # Debug: uncomment to print the section-key → clause number mapping during a build. - # print('[filter_iso_xrefs] clause map (section_key → anchor → number):', - # file=sys.stderr) - # for section_key, anchor in sorted(self._root_anchors.items()): - # info = self._anchor_info.get(anchor) - # if info: - # number_str, _level, is_annex = info - # label = f'Annex {number_str}' if is_annex else f'Clause {number_str}' - # else: - # label = '(unnumbered)' - # print(f' {section_key:<40} {anchor:<40} {label}', file=sys.stderr) + self._build_maps(yaml_path, os.getcwd()) def __call__(self, key, value, fmt, metadata): self._initialize(metadata) @@ -426,7 +385,7 @@ def _handle_header(self, value): def _handle_link(self, value): url = value[2][0] - # ---- External URL ---- + # External URL if url.startswith(('http://', 'https://')): link_text = stringify(value[1]) if link_text.strip() == url.strip(): @@ -434,11 +393,11 @@ def _handle_link(self, value): original = Link(value[0], value[1], value[2]) return [original, Str(' ('), Str(url), Str(')')] - # ---- Intra-document anchor or empty — leave alone ---- + # Intra-document anchor or empty — leave alone if not url or url.startswith('#'): return None - # ---- Internal relative link ---- + # Internal relative link parts = url.split('#', 1) url_path = parts[0] fragment = parts[1] if len(parts) == 2 else None @@ -488,9 +447,5 @@ def _handle_str(self, value): return result -# --------------------------------------------------------------------------- -# Entry point -# --------------------------------------------------------------------------- - if __name__ == '__main__': toJSONFilter(IsoXrefFilter()) From f134e27177c218b294288ef85e5820ea56b87231 Mon Sep 17 00:00:00 2001 From: Oleksiy Puzikov Date: Mon, 27 Apr 2026 14:12:22 +0300 Subject: [PATCH 5/5] dataclass refactoring --- doc_build/filters/filter_iso_xrefs.py | 87 +++++++++++++-------------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/doc_build/filters/filter_iso_xrefs.py b/doc_build/filters/filter_iso_xrefs.py index fc55b25..4b93629 100755 --- a/doc_build/filters/filter_iso_xrefs.py +++ b/doc_build/filters/filter_iso_xrefs.py @@ -28,6 +28,7 @@ import re import subprocess import sys +from dataclasses import dataclass, field from pathlib import PurePosixPath from urllib.parse import urlparse @@ -104,6 +105,21 @@ def _iso_reference_text(number_str, level, is_annex): return number_str +@dataclass +class TopSection: + """State for the level-1 heading currently being processed in _build_maps. + + Recreated each time a new level-1 heading is encountered; subcounters are + mutated in-place as subclause headings accumulate beneath it. + """ + key: str | None # section key from the source file path + clause: str | None # clause number string, or None if unnumbered + is_annex: bool = False # True when clause is an annex letter (A, B, …) + subcounters: list = field( # per-level subclause counters; index 0 unused + default_factory=lambda: [0] * 7 + ) + + # --------------------------------------------------------------------------- # Filter class # --------------------------------------------------------------------------- @@ -220,14 +236,7 @@ def _build_maps(self, yaml_path, artifacts_root): self._root_anchors = {} # section_key -> root anchor self._anchor_section = {} # anchor -> section_key (for cross-section disambiguation) - # Numbering state carried forward as we walk the heading sequence. - current_clause = None # clause number string of the enclosing level-1 heading, - # or None when inside an unnumbered section - current_is_annex = False # True when the enclosing level-1 heading is an annex - current_section_key = None # section key of the file that contains the current heading - subcounters = [0] * 7 # per-level counters; subcounters[N] is the running count - # for level-N headings within the current clause. - # Index 0 is unused (headings start at level 1). + current_section: TopSection | None = None # state for the active level-1 heading section_idx = 0 # position in section_order; advances on each level-1 heading auto_clause_counter = 0 # increments for each level-1 heading not listed in clause_map @@ -248,9 +257,7 @@ def _build_maps(self, yaml_path, artifacts_root): # sections are in the YAML as null but have no entry in section_order, # so they must not consume a section_order slot. if anchor in anchor_to_extra: - current_clause = None - current_is_annex = False - current_section_key = anchor_to_extra[anchor] + current_section = TopSection(key=anchor_to_extra[anchor], clause=None) continue # do not advance section_idx # Map this heading to its source file by consuming the next entry @@ -259,7 +266,6 @@ def _build_maps(self, yaml_path, artifacts_root): # as auto-numbered with no clause-map lookup. section_key = section_order[section_idx] if section_idx < len(section_order) else None section_idx += 1 - current_section_key = section_key if section_key is not None and section_key in self._clause_map: val = self._clause_map[section_key] @@ -267,66 +273,59 @@ def _build_maps(self, yaml_path, artifacts_root): # Explicitly suppressed (Foreword, Introduction, etc.). # No entry is added to anchor_info or root_anchors, so # links to this section are left for filter_resolve_sections. - current_clause = None - current_is_annex = False + current_section = TopSection(key=section_key, clause=None) elif isinstance(val, dict) and 'annex' in val: # Annex: numbered independently with a letter (A, B, …). # Subclauses will be A.1, A.1.1, etc. - current_clause = str(val['annex']) - current_is_annex = True - subcounters = [0] * 7 - self._root_anchors[section_key] = anchor - self._anchor_info[anchor] = (current_clause, 1, True) - self._anchor_section[anchor] = section_key + current_section = TopSection(key=section_key, clause=str(val['annex']), is_annex=True) + self._root_anchors[current_section.key] = anchor + self._anchor_info[anchor] = (current_section.clause, 1, current_section.is_annex) + self._anchor_section[anchor] = current_section.key else: # Explicit clause number override supplied directly in YAML. - current_clause = str(val) - current_is_annex = False - subcounters = [0] * 7 - self._root_anchors[section_key] = anchor - self._anchor_info[anchor] = (current_clause, 1, False) - self._anchor_section[anchor] = section_key + current_section = TopSection(key=section_key, clause=str(val)) + self._root_anchors[current_section.key] = anchor + self._anchor_info[anchor] = (current_section.clause, 1, current_section.is_annex) + self._anchor_section[anchor] = current_section.key else: # Section absent from clause_map: assign the next sequential # clause number. The counter is shared across all auto-numbered # sections so that explicitly-numbered sections (YAML overrides) # do not consume a slot in the sequence. auto_clause_counter += 1 - current_clause = str(auto_clause_counter) - current_is_annex = False - subcounters = [0] * 7 - if section_key is not None: - self._root_anchors[section_key] = anchor - self._anchor_info[anchor] = (current_clause, 1, False) - if section_key is not None: - self._anchor_section[anchor] = section_key + current_section = TopSection(key=section_key, clause=str(auto_clause_counter)) + if current_section.key is not None: + self._root_anchors[current_section.key] = anchor + self._anchor_info[anchor] = (current_section.clause, 1, current_section.is_annex) + if current_section.key is not None: + self._anchor_section[anchor] = current_section.key else: # level >= 2 # ---- Subclause heading ---- - if current_clause is None: - # Inside an unnumbered section (null in clause_map); skip all - # subheadings so they don't appear in anchor_info and links to - # them are left unchanged for filter_resolve_sections. + if current_section is None or current_section.clause is None: + # Outside any numbered section — either before the first heading + # or inside an unnumbered section (null in clause_map); skip so + # these anchors are left for filter_resolve_sections. continue # Increment the counter for this level and reset all deeper levels, # mirroring the standard hierarchical numbering convention: # e.g. entering a new level-3 heading resets level-4, 5, 6. - subcounters[level] += 1 + current_section.subcounters[level] += 1 for d in range(level + 1, 7): - subcounters[d] = 0 + current_section.subcounters[d] = 0 # Build the dotted number string: clause + each active sublevel. # For a level-3 heading inside Clause 7: "7.2.1" where # subcounters[2] and subcounters[3] are the active level-2 and # level-3 counts. number_str = '.'.join( - [current_clause] + [str(subcounters[i]) for i in range(2, level + 1)] + [current_section.clause] + [str(current_section.subcounters[i]) for i in range(2, level + 1)] ) - self._anchor_info[anchor] = (number_str, level, current_is_annex) - if current_section_key is not None: - self._anchor_section[anchor] = current_section_key + self._anchor_info[anchor] = (number_str, level, current_section.is_annex) + if current_section.key is not None: + self._anchor_section[anchor] = current_section.key def _initialize(self, metadata): """Populate the maps from iso_clause_map.yaml and combined_spec.md.