diff --git a/doc_build/ISO_LINTERS.md b/doc_build/ISO_LINTERS.md new file mode 100644 index 0000000..924ca6c --- /dev/null +++ b/doc_build/ISO_LINTERS.md @@ -0,0 +1,122 @@ +# ISO linters + +Three linters check Markdown specification sources for compliance with +ISO/IEC Directives, Part 2. Each can run standalone or as a DocBuilder +subcommand. All accept a directory (scanned recursively) or a single +`.md` file. + +## Run all linters at once + +```sh +# Lint only (exits non-zero on any violation) +pixi run python -m doc_build.doc_builder iso_lint_all + +# Fix what can be auto-fixed +pixi run python -m doc_build.doc_builder iso_fix_all +``` + +`iso_fix_all` auto-fixes heading case and bold table headers. Run +`iso_lint_all` afterwards to check for clause structure issues +(which require manual editing). + +## Heading sentence case + +ISO 11.4 requires clause titles in sentence case. + +```sh +# Check (exits non-zero on violations) +pixi run python -m doc_build.iso_heading_case_lint specification/ + +# Check a single file +pixi run python -m doc_build.iso_heading_case_lint specification/color/README.md + +# Fix in-place +pixi run python -m doc_build.iso_heading_case_lint --fix specification/ + +# With a custom proper-nouns allowlist +pixi run python -m doc_build.iso_heading_case_lint --fix \ + --proper-nouns iso_heading_proper_nouns.yaml specification/ +``` + +Proper nouns (OpenUSD, API, camelCase identifiers, etc.) are preserved +automatically. Add domain terms to `iso_heading_proper_nouns.yaml` if +they are lowercased incorrectly. + +## Bold table headers + +ISO requires table column headings to be bold. + +```sh +# Check +pixi run python -m doc_build.iso_bold_table_lint specification/ + +# Check a single file +pixi run python -m doc_build.iso_bold_table_lint specification/color/README.md + +# Fix in-place +pixi run python -m doc_build.iso_bold_table_lint --fix specification/ +``` + +Code spans in header cells (`` `value` ``) are left unwrapped. + +## Clause structure + +Checks that clauses with subclauses have no body text between the +heading and the first subclause. + +```sh +pixi run python -m doc_build.iso_clause_lint specification/ + +# Single file +pixi run python -m doc_build.iso_clause_lint specification/composition/README.md +``` + +This linter has no auto-fix; violations require manual editing. + +## DocBuilder subcommands + +All linters are also available as subcommands of the doc builder: + +```sh +pixi run python -m doc_build.doc_builder iso_lint_all +pixi run python -m doc_build.doc_builder iso_fix_all + +pixi run python -m doc_build.doc_builder heading_case_lint +pixi run python -m doc_build.doc_builder heading_case_fix +pixi run python -m doc_build.doc_builder bold_table_lint +pixi run python -m doc_build.doc_builder bold_table_fix +pixi run python -m doc_build.doc_builder iso_clause_lint +``` + +## GitHub Actions + +Add a lint job to your workflow file (`.github/workflows/*.yml`): + +```yaml + iso-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: prefix-dev/setup-pixi@v0.8.9 + with: + pixi-version: v0.59.0 + + - name: ISO compliance checks + run: pixi run python -m doc_build.doc_builder iso_lint_all +``` + +Or run the linters individually: + +```yaml + - name: ISO heading sentence case + run: pixi run python -m doc_build.iso_heading_case_lint specification/ + + - name: ISO bold table headers + run: pixi run python -m doc_build.iso_bold_table_lint specification/ + + - name: ISO clause structure + run: pixi run python -m doc_build.iso_clause_lint specification/ +``` + +Each linter exits with code 1 when violations are found, which fails the +CI step. diff --git a/doc_build/doc_builder.py b/doc_build/doc_builder.py index a857729..000ad06 100644 --- a/doc_build/doc_builder.py +++ b/doc_build/doc_builder.py @@ -254,6 +254,18 @@ def build_docs(self, args): ) args.output.mkdir(parents=True, exist_ok=True) + if args.heading_case_lint: + from doc_build.iso_heading_case_lint import check_spec, format_report + pn_path = args.heading_proper_nouns or self.get_heading_proper_nouns() + spec_root = self.get_specification_root() + log(f"\tChecking heading sentence case in {spec_root} ...") + violations = check_spec(spec_root, proper_nouns_path=pn_path) + report = format_report(violations, spec_root=spec_root) + if report: + log(report) + else: + log("\tNo heading case violations found.") + if args.diff: from_ref, to_ref = args.diff[0], args.diff[1] diff_md, from_short, to_short = self.generate_combined_diff( @@ -334,7 +346,7 @@ def _render_combined( dejavufontpath = Path(os.path.relpath(fonts_dir, artifacts_dir)).as_posix() + "/" all_filters = self.get_doc_build_filters() - if not getattr(args, 'iso_xrefs', False): + if not args.iso_xrefs: iso_filter = self.get_filter("iso_xrefs") all_filters = [f for f in all_filters if f != iso_filter] doc_build_filters = [] @@ -402,7 +414,7 @@ def _render_combined( log("\tAdding Draft Watermark...") shared_command.extend(["-V", "draft=true"]) - if getattr(args, 'iso_xrefs', False): + if args.iso_xrefs: shared_command.extend(["-M", f"ISO_CLAUSE_MAP={self.get_iso_clause_map()}"]) if from_pretty is not None and to_pretty is not None: shared_command.extend([ @@ -485,7 +497,7 @@ def stderr_processor(std_err): f"--template={latex_template}", ] + pdf_extra - if not getattr(args, "keep_pdf_latex", False): + if not args.keep_pdf_latex: # Standard path: pandoc pipes directly to tectonic via stdin. log(f"\tBuilding PDF to {pdf}...") pandoc( @@ -924,7 +936,7 @@ def run_linter(self, args): log(f"\tLint output: {linted}") - def iso_lint(self, args): + def iso_clause_lint(self, args): from doc_build.iso_clause_lint import check_spec, format_report spec_root = self.get_specification_root() @@ -940,6 +952,145 @@ def iso_lint(self, args): else: log("No ISO clause structure violations found.") + def heading_case_lint(self, args): + from doc_build.iso_heading_case_lint import check_spec, format_report + + spec_root = self.get_specification_root() + proper_nouns = args.proper_nouns or self.get_heading_proper_nouns() + log(f"Checking heading sentence case in {spec_root} ...") + violations = check_spec( + spec_root, + proper_nouns_path=proper_nouns, + ) + report = format_report(violations, spec_root=spec_root) + if report: + log(report) + else: + log("No heading case violations found.") + + def heading_case_fix(self, args): + from doc_build.iso_heading_case_lint import ( + check_spec, fix_spec, format_report, + ) + + spec_root = self.get_specification_root() + proper_nouns = args.proper_nouns or self.get_heading_proper_nouns() + log(f"Fixing heading sentence case in {spec_root} ...") + violations = check_spec(spec_root, proper_nouns_path=proper_nouns) + if not violations: + log("No heading case violations found.") + return + + report = format_report(violations, spec_root=spec_root) + log(report) + + files_fixed, headings_fixed = fix_spec( + spec_root, proper_nouns_path=proper_nouns, violations=violations, + ) + log(f"Fixed {headings_fixed} heading(s) in {files_fixed} file(s).") + + def bold_table_lint(self, args): + from doc_build.iso_bold_table_lint import check_spec, format_report + + spec_root = self.get_specification_root() + log(f"Checking bold table headers in {spec_root} ...") + violations = check_spec(spec_root) + report = format_report(violations, spec_root=spec_root) + if report: + log(report) + else: + log("No bold-table-header violations found.") + + def bold_table_fix(self, args): + from doc_build.iso_bold_table_lint import check_spec, fix_spec, format_report + + spec_root = self.get_specification_root() + log(f"Fixing bold table headers in {spec_root} ...") + violations = check_spec(spec_root) + if not violations: + log("No bold-table-header violations found.") + return + + report = format_report(violations, spec_root=spec_root) + log(report) + + files_fixed, rows_fixed = fix_spec(spec_root, violations=violations) + log(f"Fixed {rows_fixed} header row(s) in {files_fixed} file(s).") + + def iso_lint_all(self, args): + """Run all three ISO linters: clause structure, heading case, bold table headers.""" + from doc_build.iso_clause_lint import ( + check_spec as clause_check, format_report as clause_report, + ) + from doc_build.iso_heading_case_lint import ( + check_spec as heading_check, format_report as heading_report, + ) + from doc_build.iso_bold_table_lint import ( + check_spec as bold_check, format_report as bold_report, + ) + + spec_root = self.get_specification_root() + proper_nouns = args.proper_nouns or self.get_heading_proper_nouns() + + linters = [ + ("ISO clause structure", "No ISO clause structure violations found.", + lambda: clause_report(clause_check(spec_root), context=args.context, spec_root=spec_root)), + ("heading sentence case", "No heading case violations found.", + lambda: heading_report(heading_check(spec_root, proper_nouns_path=proper_nouns), spec_root=spec_root)), + ("bold table headers", "No bold-table-header violations found.", + lambda: bold_report(bold_check(spec_root), spec_root=spec_root)), + ] + + failed = False + for label, ok_msg, run in linters: + log(f"\nChecking {label} in {spec_root} ...") + report = run() + if report: + log(report) + failed = True + else: + log(ok_msg) + + if failed: + sys.exit(1) + + def iso_fix_all(self, args): + """Auto-fix heading case and bold table headers across the spec.""" + from doc_build.iso_heading_case_lint import ( + check_spec as heading_check, fix_spec as heading_fix, + format_report as heading_report, + ) + from doc_build.iso_bold_table_lint import ( + check_spec as bold_check, fix_spec as bold_fix, + format_report as bold_report, + ) + + spec_root = self.get_specification_root() + proper_nouns = args.proper_nouns or self.get_heading_proper_nouns() + + fixers = [ + ("heading sentence case", "No heading case violations found.", + lambda: heading_check(spec_root, proper_nouns_path=proper_nouns), + lambda vs: heading_report(vs, spec_root=spec_root), + lambda vs: heading_fix(spec_root, proper_nouns_path=proper_nouns, violations=vs), + lambda f, n: f"Fixed {n} heading(s) in {f} file(s)."), + ("bold table headers", "No bold-table-header violations found.", + lambda: bold_check(spec_root), + lambda vs: bold_report(vs, spec_root=spec_root), + lambda vs: bold_fix(spec_root, violations=vs), + lambda f, n: f"Fixed {n} header row(s) in {f} file(s)."), + ] + + for label, ok_msg, check, report_fn, fix, summary_fn in fixers: + log(f"\nFixing {label} in {spec_root} ...") + violations = check() + if violations: + log(report_fn(violations)) + files_fixed, items_fixed = fix(violations) + log(summary_fn(files_fixed, items_fixed)) + else: + log(ok_msg) + def export_git_archive(self, args): timestr = time.strftime("%Y%m%d-%H%M%S") filename = f"aousd_core_spec_{args.branch}_{timestr}.zip" @@ -1167,6 +1318,17 @@ def get_metadata_defaults_file(self) -> Path: return self.get_scripts_root() / "defaults.yaml" + def get_heading_proper_nouns(self) -> Path: + """Return the heading proper-nouns YAML path. + + Checks for a specification-specific file first, then falls back + to the builder-bundled default. + """ + spec_specific = self.get_specification_root() / "iso_heading_proper_nouns.yaml" + if spec_specific.exists(): + return spec_specific + return self.get_scripts_root() / "iso_heading_proper_nouns.yaml" + def get_iso_clause_map(self) -> Path: """Return the ISO clause map YAML path. @@ -1205,7 +1367,13 @@ def construct_subparsers(self, parser): self.make_index_parser(subparsers) self.make_spellcheck_parser(subparsers) self.make_style_parser(subparsers) - self.make_iso_lint_parser(subparsers) + self.make_iso_clause_lint_parser(subparsers) + self.make_heading_case_lint_parser(subparsers) + self.make_heading_case_fix_parser(subparsers) + self.make_bold_table_lint_parser(subparsers) + self.make_bold_table_fix_parser(subparsers) + self.make_iso_lint_all_parser(subparsers) + self.make_iso_fix_all_parser(subparsers) return subparsers def make_build_parser(self, subparsers): @@ -1249,6 +1417,21 @@ def make_build_parser(self, subparsers): "script to recreate the PDF from the .tex (implies tectonic wrapper)", action="store_true", ) + build_parser.add_argument( + "--heading-case-lint", + help="Run the ISO heading sentence-case linter before building and " + "print violations (does not block the build).", + action="store_true", + ) + build_parser.add_argument( + "--heading-proper-nouns", + type=Path, + default=None, + metavar="YAML", + help="Path to a YAML file listing additional proper nouns for " + "heading-case enforcement. Defaults to iso_heading_proper_nouns.yaml " + "in the specification or builder root.", + ) build_parser.add_argument( "--diff", nargs="*", @@ -1306,9 +1489,9 @@ def make_style_parser(self, subparsers): style_parser.set_defaults(func=self.display_style_issues) return style_parser - def make_iso_lint_parser(self, subparsers): + def make_iso_clause_lint_parser(self, subparsers): p = subparsers.add_parser( - "iso_lint", + "iso_clause_lint", help="Check specification source files for ISO clause structure violations", ) p.add_argument( @@ -1318,7 +1501,94 @@ def make_iso_lint_parser(self, subparsers): metavar="N", help="Number of body lines to show per violation (default: 5)", ) - p.set_defaults(func=self.iso_lint) + p.set_defaults(func=self.iso_clause_lint) + return p + + def make_heading_case_lint_parser(self, subparsers): + p = subparsers.add_parser( + "heading_case_lint", + help="Check specification headings for ISO sentence-case compliance", + ) + p.add_argument( + "--proper-nouns", + type=Path, + default=None, + metavar="YAML", + help="Path to a YAML file listing additional proper nouns " + "(default: iso_heading_proper_nouns.yaml in the builder or spec root)", + ) + p.set_defaults(func=self.heading_case_lint) + return p + + def make_heading_case_fix_parser(self, subparsers): + p = subparsers.add_parser( + "heading_case_fix", + help="Edit specification sources in-place to convert headings to sentence case", + ) + p.add_argument( + "--proper-nouns", + type=Path, + default=None, + metavar="YAML", + help="Path to a YAML file listing additional proper nouns " + "(default: iso_heading_proper_nouns.yaml in the builder or spec root)", + ) + p.set_defaults(func=self.heading_case_fix) + return p + + def make_bold_table_lint_parser(self, subparsers): + p = subparsers.add_parser( + "bold_table_lint", + help="Check that table header cells in specification sources are bold", + ) + p.set_defaults(func=self.bold_table_lint) + return p + + def make_bold_table_fix_parser(self, subparsers): + p = subparsers.add_parser( + "bold_table_fix", + help="Edit specification sources in-place to bold table header cells", + ) + p.set_defaults(func=self.bold_table_fix) + return p + + def make_iso_lint_all_parser(self, subparsers): + p = subparsers.add_parser( + "iso_lint_all", + help="Run all ISO linters (clause structure, heading case, bold table headers)", + ) + p.add_argument( + "--proper-nouns", + type=Path, + default=None, + metavar="YAML", + help="Path to a YAML file listing additional proper nouns " + "(default: iso_heading_proper_nouns.yaml in the builder or spec root)", + ) + p.add_argument( + "--context", + type=int, + default=5, + metavar="N", + help="Number of body lines to show per clause-structure violation (default: 5)", + ) + p.set_defaults(func=self.iso_lint_all) + return p + + def make_iso_fix_all_parser(self, subparsers): + p = subparsers.add_parser( + "iso_fix_all", + help="Auto-fix heading case and bold table headers across the spec", + ) + p.add_argument( + "--proper-nouns", + type=Path, + default=None, + metavar="YAML", + help="Path to a YAML file listing additional proper nouns " + "(default: iso_heading_proper_nouns.yaml in the builder or spec root)", + ) + p.set_defaults(func=self.iso_fix_all) return p def add_publish_copyright(self, combined): diff --git a/doc_build/filters/heading_case.py b/doc_build/filters/heading_case.py new file mode 100644 index 0000000..1964e89 --- /dev/null +++ b/doc_build/filters/heading_case.py @@ -0,0 +1,198 @@ +"""Shared logic for ISO heading sentence-case enforcement. + +ISO/IEC Directives, Part 2, 11.4: clause titles shall be in sentence case +(only the first word and proper nouns capitalised). + +This module provides: +- Proper-noun detection via heuristics and an optional YAML allowlist. +- A function to convert heading inline elements to sentence case. +- A function to check whether a heading is already in sentence case. +""" + +import copy +import re +from pathlib import Path +from typing import List, Optional, Set + +from pandocfilters import stringify + +try: + import yaml +except ImportError: + yaml = None + + +# --------------------------------------------------------------------------- +# Proper-noun heuristics +# --------------------------------------------------------------------------- + +# Words that look like identifiers: camelCase, PascalCase, or contain +# internal capitals (e.g. ObjectPath, iPhone, OpenUSD). +_MIXED_CASE_RE = re.compile( + r'^[A-Z]?[a-z]+[A-Z]' # camelCase or PascalCase with interior capital + r'|^[A-Z][a-z]+[A-Z]' # PascalCase like OpenUSD +) + +# Fully uppercase with optional underscores/digits (USD, AOUSD, API, UTF-8). +_ALL_CAPS_RE = re.compile(r'^[A-Z][A-Z0-9_-]+$') + +# Common technical acronyms that should always stay uppercase. +BUILTIN_ACRONYMS = frozenset({ + "API", "APIs", "ASCII", "AOUSD", "CPU", "CSS", "DPI", "GPU", "GFM", + "HTML", "HTTP", "HTTPS", "ID", "IEEE", "IO", "ISO", "JSON", "LHS", + "MIME", "OCR", "OS", "PDF", "PEG", "RAM", "RCS", "REST", "RFC", + "RHS", "SDK", "SHA", "SQL", "SSH", "SSL", "SVG", "TCP", "TLS", + "TSV", "UDP", "UI", "URI", "URL", "USD", "UTF", "UUID", "XML", + "YAML", +}) + + +def load_proper_nouns(yaml_path: Optional[Path] = None) -> Set[str]: + """Load proper nouns from a YAML allowlist file. + + The YAML file is expected to have a top-level list under the key + ``proper_nouns``. Returns an empty set if the file does not exist + or yaml is unavailable. + """ + if yaml_path is None or yaml is None: + return set() + try: + with open(yaml_path, encoding='utf-8') as fh: + data = yaml.safe_load(fh) or {} + return set(data.get('proper_nouns', [])) + except (FileNotFoundError, OSError): + return set() + + +def is_proper_noun(word: str, extra_nouns: Set[str] = frozenset()) -> bool: + """Determine whether *word* should retain its capitalisation. + + A word is considered a proper noun if any of these hold: + - It is in *extra_nouns* (the YAML allowlist). + - It is in the built-in acronym set. + - It matches the mixed-case heuristic (camelCase / PascalCase). + - It is fully uppercase with length >= 2. + - It contains digits mixed with letters (e.g. "UTF-8", "H264"). + """ + stripped = word.strip(".,;:!?()[]{}\"'`") + if not stripped: + return False + if stripped in extra_nouns: + return True + if stripped in BUILTIN_ACRONYMS: + return True + if _ALL_CAPS_RE.match(stripped) and len(stripped) >= 2: + return True + if _MIXED_CASE_RE.match(stripped): + return True + if re.search(r'[A-Za-z]', stripped) and re.search(r'\d', stripped): + return True + return False + + +# --------------------------------------------------------------------------- +# Sentence-case conversion for Pandoc inlines +# --------------------------------------------------------------------------- + +def _sentence_case_track( + inlines: list, + extra_nouns: Set[str], + first_word_seen: bool, +) -> tuple: + """Internal: convert inlines to sentence case, returning (new_inlines, first_word_seen).""" + result = copy.deepcopy(inlines) + + for node in result: + if not isinstance(node, dict): + continue + t = node.get('t') + if t == 'Str': + text = node['c'] + words = text.split(' ') + new_words = [] + for word in words: + if not word: + new_words.append(word) + continue + has_alpha = any(c.isalpha() for c in word) + if not first_word_seen and has_alpha: + first_word_seen = True + new_words.append(word) + elif is_proper_noun(word, extra_nouns): + new_words.append(word) + elif has_alpha: + new_words.append(word.lower()) + else: + new_words.append(word) + node['c'] = ' '.join(new_words) + elif t in ('Code', 'Math', 'RawInline'): + first_word_seen = True + elif t == 'Space': + pass + elif t == 'Span': + # Span: node['c'] = [attr, inlines_list] + children = node['c'][1] + node['c'][1], first_word_seen = _sentence_case_track( + children, extra_nouns, first_word_seen, + ) + elif t in ('Emph', 'Strong', 'Strikeout', 'Superscript', + 'Subscript', 'SmallCaps'): + children = node['c'] if isinstance(node['c'], list) else [node['c']] + node['c'], first_word_seen = _sentence_case_track( + children, extra_nouns, first_word_seen, + ) + elif t == 'Quoted': + node['c'][1], first_word_seen = _sentence_case_track( + node['c'][1], extra_nouns, first_word_seen, + ) + + return result, first_word_seen + + +def sentence_case_inlines( + inlines: list, + extra_nouns: Set[str] = frozenset(), +) -> list: + """Convert a list of Pandoc inline elements to sentence case. + + Only ``Str`` nodes are modified; ``Code``, ``Math``, ``RawInline`` + and content inside ``Link``/``Image`` alt text are left unchanged. + + The first alphabetic word in the heading is always capitalised. + Subsequent words are lowercased unless they are proper nouns. + """ + result, _ = _sentence_case_track(inlines, extra_nouns, False) + return result + + +# --------------------------------------------------------------------------- +# Violation detection +# --------------------------------------------------------------------------- + +def _extract_words(inlines: list) -> List[str]: + """Extract plain-text words from Pandoc inlines, skipping Code/Math.""" + text = stringify(inlines) + return [w for w in re.split(r'\s+', text) if w] + + +def heading_needs_conversion( + inlines: list, + extra_nouns: Set[str] = frozenset(), +) -> bool: + """Return True if the heading appears to be in title case. + + A heading is flagged if at least one non-first word is capitalised + and is not a proper noun. Single-word headings are never flagged. + """ + words = _extract_words(inlines) + if len(words) <= 1: + return False + + for word in words[1:]: + stripped = word.strip(".,;:!?()[]{}\"'`") + if not stripped or not stripped[0].isalpha(): + continue + if stripped[0].isupper() and not is_proper_noun(word, extra_nouns): + return True + + return False diff --git a/doc_build/iso_bold_table_lint.py b/doc_build/iso_bold_table_lint.py new file mode 100644 index 0000000..7a5f4f4 --- /dev/null +++ b/doc_build/iso_bold_table_lint.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Lint and fix bold table headers in Markdown specification files. + +ISO/IEC Directives, Part 2 requires table column headings to be in bold. +In Markdown pipe tables this means every header cell should be wrapped in +``**…**``. Code spans (``` `` ```) are considered visually distinct and +are left unwrapped. + +Two modes: + +Lint (default) + Parse each ``.md`` file with Pandoc, walk the AST to find Table nodes + whose header cells are not bold, and report violations with file and + line number. + +Fix (``--fix``) + For every file with violations, rewrite the header row in-place so that + non-bold, non-code cells are wrapped in ``**…**``. + +Usage from the command line:: + + python3 -m doc_build.iso_bold_table_lint specification/ + python3 -m doc_build.iso_bold_table_lint --fix specification/ + +Usage as a library:: + + from doc_build.iso_bold_table_lint import check_spec, fix_file + violations = check_spec(Path("specification/")) + for v in violations: + print(v.format()) + fix_file(Path("specification/some_file.md"), violations) +""" + +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple + +from doc_build.iso_lint_utils import ( + DEFAULT_WORKERS, + collect_md_files, + format_report as _format_report, + get_sourcepos, + run_parallel_check, + stringify, + unwrap_sourcepos_spans, +) + + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + +@dataclass +class Violation: + """A table whose header row has non-bold cells.""" + file: Path + lineno: int # 1-based line number of the header row + header_text: str # raw source text of the header row + non_bold_cells: List[str] # cell texts that are not bold + + def format(self, *, display_path: Optional[Path] = None) -> str: + path = display_path if display_path is not None else self.file + cells_str = ', '.join(f'"{c.strip()}"' for c in self.non_bold_cells) + return ( + f"{path}:{self.lineno}: {self.header_text.strip()}\n" + f" non-bold cells: {cells_str}" + ) + + +# --------------------------------------------------------------------------- +# AST-based detection (lint) +# --------------------------------------------------------------------------- + +def _cell_needs_bold(blocks: list) -> bool: + """Return True if a header cell's content is not bold. + + A cell is considered bold if its Para/Plain block contains a single + Strong wrapper around all content. A cell containing only a Code + span is considered exempt (visually distinct already). + Empty cells are exempt. + + Handles Pandoc's +sourcepos Span wrappers transparently. + """ + if not blocks: + return False + + for block in blocks: + bt = block.get("t") + if bt not in ("Para", "Plain"): + continue + inlines = unwrap_sourcepos_spans(block.get("c", [])) + if not inlines: + continue + + # Single Code span — exempt. + if len(inlines) == 1 and inlines[0].get("t") == "Code": + continue + + # Single Strong wrapping everything — already bold. + if len(inlines) == 1 and inlines[0].get("t") == "Strong": + continue + + # Anything else: not bold. + return True + + return False + + +def _cell_text(blocks: list) -> str: + """Extract plain text from a cell's blocks for reporting.""" + return stringify(blocks).strip() + + +def check_file(path: Path) -> List[Violation]: + """Return all bold-table-header violations in a single Markdown file.""" + try: + raw_lines = path.read_text(encoding='utf-8').splitlines() + except OSError: + return [] + + try: + result = subprocess.run( + ["pandoc", "-f", "commonmark_x+sourcepos+pipe_tables", "-t", "json", + str(path)], + capture_output=True, text=True, check=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError): + return [] + + doc = json.loads(result.stdout) + violations: List[Violation] = [] + + for block in doc["blocks"]: + if block["t"] != "Table": + continue + + # Table: [attr, caption, colspecs, head, bodies, foot] + table_attr = block["c"][0] + head = block["c"][3] + table_lineno = get_sourcepos(table_attr) + + # TableHead: [head_attr, rows] + _, rows = head + if not rows: + continue + + for row in rows: + _, cells = row + non_bold = [] + for cell in cells: + # Cell: [attr, alignment, rowspan, colspan, blocks] + cell_blocks = cell[4] + if _cell_needs_bold(cell_blocks): + non_bold.append(_cell_text(cell_blocks)) + + if non_bold and table_lineno is not None: + # The header row is at the table's start line. + header_line = raw_lines[table_lineno - 1] if table_lineno <= len(raw_lines) else "" + violations.append(Violation( + file=path, + lineno=table_lineno, + header_text=header_line, + non_bold_cells=non_bold, + )) + + return violations + + +def check_spec( + spec_root: Path, + workers: int = DEFAULT_WORKERS, +) -> List[Violation]: + """Walk *spec_root* recursively and return all violations in .md files.""" + md_files = collect_md_files(spec_root) + + return run_parallel_check( + md_files, + check_fn=check_file, + sort_key=lambda v: (str(v.file), v.lineno), + workers=workers, + ) + + +# --------------------------------------------------------------------------- +# Source-level fix +# --------------------------------------------------------------------------- + +def _bold_header_row(line: str) -> str: + """Wrap non-bold, non-code cells in a pipe-table header row with **…**. + + Splits the line by ``|``, processes each cell, and reassembles. + Cells that are already bold (``**…**``) or are code spans (``` `…` ```) + are left unchanged. Empty/whitespace-only cells are left unchanged. + + Note: partially bold cells (e.g. ``| **Name** extra |``) are treated + as not bold — the whole cell text gets wrapped. This is intentional: + partial bold in a header cell is almost certainly a formatting error, + and wrapping the full cell produces the correct result. + """ + # Split preserving the pipe delimiters. A typical header row: + # "| Name | Type | Description |" + # splits to: ['', ' Name ', ' Type ', ' Description ', ''] + if '|' not in line: + return line + + parts = line.split('|') + new_parts = [] + + for i, part in enumerate(parts): + stripped = part.strip() + + # First and last parts are outside the table pipes — leave as-is. + if i == 0 or i == len(parts) - 1: + new_parts.append(part) + continue + + # Empty cell. + if not stripped: + new_parts.append(part) + continue + + # Already bold. + if stripped.startswith('**') and stripped.endswith('**'): + new_parts.append(part) + continue + + # Code span — exempt. + if stripped.startswith('`') and stripped.endswith('`'): + new_parts.append(part) + continue + + # Wrap in bold, preserving surrounding whitespace. + leading = part[:len(part) - len(part.lstrip())] + trailing = part[len(part.rstrip()):] + new_parts.append(f"{leading}**{stripped}**{trailing}") + + return '|'.join(new_parts) + + +def fix_file(path: Path, violations: Optional[List[Violation]] = None) -> int: + """Edit *path* in-place, bolding table header rows. + + If *violations* is None, runs check_file() first. + Returns the number of header rows fixed. + """ + if violations is None: + violations = check_file(path) + file_violations = [v for v in violations if v.file == path] + if not file_violations: + return 0 + + lines = path.read_text(encoding='utf-8').splitlines(keepends=True) + violation_lines = {v.lineno for v in file_violations} + fixed = 0 + + for lineno in sorted(violation_lines): + idx = lineno - 1 + if idx < 0 or idx >= len(lines): + continue + original = lines[idx].rstrip('\n') + trailing = lines[idx][len(original):] + bolded = _bold_header_row(original) + if bolded != original: + lines[idx] = bolded + trailing + fixed += 1 + + if fixed: + path.write_text(''.join(lines), encoding='utf-8') + + return fixed + + +def fix_spec( + spec_root: Path, + workers: int = DEFAULT_WORKERS, + violations: Optional[List[Violation]] = None, +) -> Tuple[int, int]: + """Fix all violations under *spec_root*. Returns (files_fixed, rows_fixed). + + If *violations* is provided, skips the check pass and fixes those + directly — useful when the caller already ran ``check_spec()``. + """ + if violations is None: + violations = check_spec(spec_root, workers=workers) + if not violations: + return 0, 0 + + # Group by file. + by_file: dict = {} + for v in violations: + by_file.setdefault(v.file, []).append(v) + + files_fixed = 0 + rows_fixed = 0 + for file_path, file_violations in by_file.items(): + n = fix_file(file_path, file_violations) + if n: + files_fixed += 1 + rows_fixed += n + + return files_fixed, rows_fixed + + +# --------------------------------------------------------------------------- +# Report formatting +# --------------------------------------------------------------------------- + +def format_report( + violations: List[Violation], + spec_root: Optional[Path] = None, +) -> str: + return _format_report(violations, "bold-table-header", spec_root=spec_root) + + +# --------------------------------------------------------------------------- +# Command-line entry point +# --------------------------------------------------------------------------- + +def main(argv=None): + import argparse + parser = argparse.ArgumentParser( + description='Check (and optionally fix) Markdown table headers for bold formatting.' + ) + parser.add_argument( + 'path', + nargs='?', + default='.', + help='Spec root directory to scan (default: current directory)', + ) + parser.add_argument( + '--fix', + action='store_true', + help='Edit files in-place to bold non-bold table header cells.', + ) + parser.add_argument( + '--workers', + type=int, + default=DEFAULT_WORKERS, + metavar='N', + help=f'Parallel Pandoc workers (default: {DEFAULT_WORKERS})', + ) + args = parser.parse_args(argv) + spec_root = Path(args.path).resolve() + + if args.fix: + violations = check_spec(spec_root, workers=args.workers) + if not violations: + print('No bold-table-header violations found.') + return + report = format_report(violations, spec_root=spec_root) + print(report) + print() + + files_fixed, rows_fixed = fix_spec(spec_root, workers=args.workers, violations=violations) + print(f'Fixed {rows_fixed} header row(s) in {files_fixed} file(s).') + else: + violations = check_spec(spec_root, workers=args.workers) + report = format_report(violations, spec_root=spec_root) + if report: + print(report) + sys.exit(1) + else: + print('No bold-table-header violations found.') + + +if __name__ == '__main__': + main() diff --git a/doc_build/iso_clause_lint.py b/doc_build/iso_clause_lint.py index 195a979..d728f79 100644 --- a/doc_build/iso_clause_lint.py +++ b/doc_build/iso_clause_lint.py @@ -21,22 +21,24 @@ """ import json -import os import subprocess import sys -from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path from typing import List, Optional, Tuple -from filters.pandocfilters import stringify +from doc_build.iso_lint_utils import ( + DEFAULT_WORKERS, + collect_md_files, + format_report as _format_report, + get_sourcepos, + run_parallel_check, + stringify, +) # How many non-blank body lines to show as context in a report. DEFAULT_CONTEXT_LINES = 5 -# Maximum number of parallel Pandoc subprocesses used by check_spec(). -DEFAULT_WORKERS = 8 - # --------------------------------------------------------------------------- # Data types @@ -56,17 +58,17 @@ class Violation: # Non-blank source lines between the heading and the first subclause. # Each entry is (1-based line number, raw line content). - def format(self, context: int = DEFAULT_CONTEXT_LINES) -> str: + def format(self, *, context: int = DEFAULT_CONTEXT_LINES, display_path: Optional[Path] = None) -> str: """Return a human-readable description of the violation.""" + path = display_path if display_path is not None else self.file h_marker = '#' * self.heading_level sub_marker = '#' * self.first_sub_level shown = self.body_lines[:context] remainder = len(self.body_lines) - len(shown) lines = [ - f"{self.file}:{self.heading_lineno}: " - f"{h_marker} \"{self.heading_text}\" " - # f"has text before its first subclause", + f"{path}:{self.heading_lineno}: " + f"{h_marker} \"{self.heading_text}\"", ] for lineno, content in shown: lines.append(f" │ {lineno:5d}: {content}") @@ -79,30 +81,6 @@ def format(self, context: int = DEFAULT_CONTEXT_LINES) -> str: return '\n'.join(lines) -# --------------------------------------------------------------------------- -# Pandoc AST helpers -# --------------------------------------------------------------------------- - -def _get_sourcepos(attr: list) -> Optional[int]: - """Extract the start line number from a Pandoc sourcepos Attr. - - Attr layout: [id, [classes], [[key, value], ...]] - - When Pandoc reads from a file the data-pos value has the form - "filepath@startrow:startcol-endrow:endcol"; when reading from stdin it - omits the "filepath@" prefix. Both forms are handled here. - - Returns the start row as a 1-based integer, or None if the attribute is - absent. - """ - for key, val in attr[2]: - if key == "data-pos": - # Strip optional "filepath@" prefix before the row:col range. - pos = val.split("@")[-1] - return int(pos.split(":")[0]) - return None - - # --------------------------------------------------------------------------- # Core checker # --------------------------------------------------------------------------- @@ -152,7 +130,7 @@ def check_file(path: Path) -> List[Violation]: level: int = block["c"][0] attr: list = block["c"][1] inlines: list = block["c"][2] - lineno: Optional[int] = _get_sourcepos(attr) + lineno: Optional[int] = get_sourcepos(attr) text: str = stringify(inlines).strip() # Check for a violation: the previous heading has body content @@ -205,25 +183,19 @@ def check_spec( ) -> List[Violation]: """Walk *spec_root* recursively and return all violations in .md files. - Files are processed in parallel (up to *workers* simultaneous Pandoc + *spec_root* may be a single ``.md`` file or a directory. Files are + processed in parallel (up to *workers* simultaneous Pandoc subprocesses) for speed, then results are sorted by (file path, line number) so that output is stable across runs. """ - md_files: List[Path] = [] - for dirpath, dirnames, filenames in os.walk(spec_root): - dirnames.sort() - for fname in sorted(filenames): - if fname.endswith('.md'): - md_files.append(Path(dirpath) / fname) - - all_violations: List[Violation] = [] - with ThreadPoolExecutor(max_workers=workers) as pool: - futures = {pool.submit(check_file, p): p for p in md_files} - for future in as_completed(futures): - all_violations.extend(future.result()) + md_files = collect_md_files(spec_root) - all_violations.sort(key=lambda v: (str(v.file), v.heading_lineno)) - return all_violations + return run_parallel_check( + md_files, + check_fn=check_file, + sort_key=lambda v: (str(v.file), v.heading_lineno), + workers=workers, + ) # --------------------------------------------------------------------------- @@ -240,39 +212,9 @@ def format_report( If *spec_root* is given, file paths are shown relative to it. Returns an empty string when there are no violations. """ - if not violations: - return '' - - # Group by file for tidier output. - by_file: dict = {} - for v in violations: - rel = v.file.relative_to(spec_root) if spec_root else v.file - by_file.setdefault(rel, []).append(v) - - sections: List[str] = [] - total = 0 - for rel_path, file_violations in by_file.items(): - block_lines = [f'{rel_path}'] - for v in file_violations: - display_v = Violation( - file=rel_path, - heading_lineno=v.heading_lineno, - heading_level=v.heading_level, - heading_text=v.heading_text, - first_sub_lineno=v.first_sub_lineno, - first_sub_level=v.first_sub_level, - first_sub_text=v.first_sub_text, - body_lines=v.body_lines, - ) - block_lines.append(display_v.format(context=context)) - total += 1 - sections.append('\n'.join(block_lines)) - - file_count = len(by_file) - header = ( - f'{total} violation(s) in {file_count} file(s)\n' + return _format_report( + violations, "clause structure", spec_root=spec_root, context=context, ) - return header + '\n\n' + '\n\n'.join(sections) # --------------------------------------------------------------------------- diff --git a/doc_build/iso_heading_case_lint.py b/doc_build/iso_heading_case_lint.py new file mode 100644 index 0000000..598eae6 --- /dev/null +++ b/doc_build/iso_heading_case_lint.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +"""Lint and fix ISO heading sentence case in Markdown specification files. + +ISO/IEC Directives, Part 2, 11.4: clause titles shall use sentence case +(only the first word and proper nouns capitalised). + +Two modes: + +Lint (default) + Parse each ``.md`` file with Pandoc, walk the AST to find Header nodes + that appear to be in title case, and report violations with file and + line number. + +Fix (``--fix``) + For every file with violations, rewrite the heading line in-place so + that non-first, non-proper-noun words are lowercased. + +Proper-noun detection combines heuristics (camelCase, ALL_CAPS, mixed +alphanumeric) with an optional YAML allowlist of domain-specific terms. + +Usage from the command line:: + + python3 -m doc_build.iso_heading_case_lint specification/ + python3 -m doc_build.iso_heading_case_lint --fix specification/ + python3 -m doc_build.iso_heading_case_lint --proper-nouns custom.yaml spec/ + +Usage as a library:: + + from doc_build.iso_heading_case_lint import check_spec, fix_file + violations = check_spec(Path("specification/")) + for v in violations: + print(v.format()) + fix_file(Path("specification/some_file.md"), violations, extra_nouns) +""" + +import json +import re +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional, Set, Tuple + +from filters.heading_case import ( + heading_needs_conversion, + is_proper_noun, + load_proper_nouns, + sentence_case_inlines, +) + +from doc_build.iso_lint_utils import ( + DEFAULT_WORKERS, + collect_md_files, + format_report as _format_report, + get_sourcepos, + run_parallel_check, + stringify, +) + + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + +@dataclass +class Violation: + """A heading that is not in ISO sentence case.""" + file: Path + lineno: int + level: int + heading_text: str + suggested_text: str + non_sentence_words: List[str] = field(default_factory=list) + + def format(self, *, display_path: Optional[Path] = None) -> str: + path = display_path if display_path is not None else self.file + marker = '#' * self.level + words_str = ', '.join(f'"{w}"' for w in self.non_sentence_words) + lines = [ + f"{path}:{self.lineno}: " + f"{marker} \"{self.heading_text}\"", + f" suggested: {marker} \"{self.suggested_text}\"", + ] + if self.non_sentence_words: + lines.append(f" title-cased words: {words_str}") + return '\n'.join(lines) + + +# --------------------------------------------------------------------------- +# Pandoc AST helpers +# --------------------------------------------------------------------------- + +def _find_non_sentence_words(inlines: list, extra_nouns: Set[str]) -> List[str]: + """Return words that are capitalised but not proper nouns (skipping the first word).""" + words = [] + text = stringify(inlines) + all_words = [w for w in re.split(r'\s+', text) if w] + for word in all_words[1:]: + stripped = word.strip(".,;:!?()[]{}\"'`") + if not stripped or not stripped[0].isalpha(): + continue + if stripped[0].isupper() and not is_proper_noun(word, extra_nouns): + words.append(word) + return words + + +# --------------------------------------------------------------------------- +# Core checker +# --------------------------------------------------------------------------- + +def check_file(path: Path, extra_nouns: Set[str] = frozenset()) -> List[Violation]: + """Return all heading sentence-case violations in a single Markdown file.""" + try: + result = subprocess.run( + ["pandoc", "-f", "commonmark_x+sourcepos", "-t", "json", str(path)], + capture_output=True, text=True, check=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError): + return [] + + doc = json.loads(result.stdout) + violations: List[Violation] = [] + + for block in doc["blocks"]: + if block["t"] != "Header": + continue + + level = block["c"][0] + attr = block["c"][1] + inlines = block["c"][2] + lineno = get_sourcepos(attr) + + if not heading_needs_conversion(inlines, extra_nouns): + continue + + heading_text = stringify(inlines).strip() + converted = sentence_case_inlines(inlines, extra_nouns) + suggested_text = stringify(converted).strip() + non_sentence = _find_non_sentence_words(inlines, extra_nouns) + + violations.append(Violation( + file=path, + lineno=lineno or 0, + level=level, + heading_text=heading_text, + suggested_text=suggested_text, + non_sentence_words=non_sentence, + )) + + return violations + + +def check_spec( + spec_root: Path, + workers: int = DEFAULT_WORKERS, + proper_nouns_path: Optional[Path] = None, +) -> List[Violation]: + """Walk *spec_root* recursively and return all violations in .md files.""" + extra_nouns = load_proper_nouns(proper_nouns_path) + md_files = collect_md_files(spec_root) + + return run_parallel_check( + md_files, + check_fn=lambda p: check_file(p, extra_nouns), + sort_key=lambda v: (str(v.file), v.lineno), + workers=workers, + ) + + +# --------------------------------------------------------------------------- +# Source-level fix +# --------------------------------------------------------------------------- + +# Regex matching an ATX heading line: "## Some Heading Text" +_ATX_HEADING_RE = re.compile(r'^(#{1,6}\s+)(.*?)(\s*)$') + + +def _sentence_case_text(text: str, extra_nouns: Set[str]) -> str: + """Convert the text portion of a heading to sentence case. + + Preserves inline Markdown formatting (``*``, ``**``, ``_``, ``[``, + etc.) by only modifying alphabetic word sequences. Code spans + (``` `` ```) and link URLs (``](…)``) are left untouched. + """ + # Build a set of character positions that are inside code spans or + # link URLs — these must not be modified. + protected: Set[int] = set() + for m in re.finditer(r'`+.+?`+', text): + protected.update(range(m.start(), m.end())) + for m in re.finditer(r'\]\([^)]*\)', text): + protected.update(range(m.start(), m.end())) + + first_word_seen = False + result = list(text) + + for m in re.finditer(r"[A-Za-z][A-Za-z0-9'_-]*", text): + word_range = range(m.start(), m.end()) + # Skip words inside protected ranges (code spans, link URLs). + if not protected.isdisjoint(word_range): + first_word_seen = True + continue + + word = m.group() + + # First alphabetic word in the heading is always preserved. + if not first_word_seen: + first_word_seen = True + continue + + if is_proper_noun(word, extra_nouns): + continue + + # Lowercase this word in-place. + lowered = word.lower() + for i, ch in enumerate(lowered): + result[m.start() + i] = ch + + return ''.join(result) + + +def _sentence_case_heading_line(line: str, extra_nouns: Set[str]) -> str: + """Convert an ATX heading source line to sentence case. + + Splits the line into ``# `` prefix, text, and trailing whitespace. + Only the text portion is case-converted; the prefix and trailing + whitespace are preserved verbatim. + """ + m = _ATX_HEADING_RE.match(line) + if not m: + return line + prefix, text, trailing = m.groups() + converted = _sentence_case_text(text, extra_nouns) + return prefix + converted + trailing + + +def fix_file( + path: Path, + violations: Optional[List[Violation]] = None, + extra_nouns: Set[str] = frozenset(), +) -> int: + """Edit *path* in-place, converting heading lines to sentence case. + + If *violations* is None, runs ``check_file()`` first. + Returns the number of headings fixed. + """ + if violations is None: + violations = check_file(path, extra_nouns) + file_violations = [v for v in violations if v.file == path] + if not file_violations: + return 0 + + lines = path.read_text(encoding='utf-8').splitlines(keepends=True) + violation_lines = {v.lineno for v in file_violations} + fixed = 0 + + for lineno in sorted(violation_lines): + idx = lineno - 1 + if idx < 0 or idx >= len(lines): + continue + original = lines[idx].rstrip('\n') + trailing = lines[idx][len(original):] + converted = _sentence_case_heading_line(original, extra_nouns) + if converted != original: + lines[idx] = converted + trailing + fixed += 1 + + if fixed: + path.write_text(''.join(lines), encoding='utf-8') + + return fixed + + +def fix_spec( + spec_root: Path, + workers: int = DEFAULT_WORKERS, + proper_nouns_path: Optional[Path] = None, + violations: Optional[List[Violation]] = None, +) -> Tuple[int, int]: + """Fix all violations under *spec_root*. Returns (files_fixed, headings_fixed). + + If *violations* is provided, skips the check pass and fixes those + directly — useful when the caller already ran ``check_spec()``. + """ + extra_nouns = load_proper_nouns(proper_nouns_path) + if violations is None: + violations = check_spec(spec_root, workers=workers, proper_nouns_path=proper_nouns_path) + if not violations: + return 0, 0 + + # Group by file. + by_file: dict = {} + for v in violations: + by_file.setdefault(v.file, []).append(v) + + files_fixed = 0 + headings_fixed = 0 + for file_path, file_violations in by_file.items(): + n = fix_file(file_path, file_violations, extra_nouns) + if n: + files_fixed += 1 + headings_fixed += n + + return files_fixed, headings_fixed + + +# --------------------------------------------------------------------------- +# Report formatting +# --------------------------------------------------------------------------- + +def format_report( + violations: List[Violation], + spec_root: Optional[Path] = None, +) -> str: + return _format_report(violations, "heading case", spec_root=spec_root) + + +# --------------------------------------------------------------------------- +# Command-line entry point +# --------------------------------------------------------------------------- + +def main(argv=None): + import argparse + parser = argparse.ArgumentParser( + description='Check (and optionally fix) Markdown headings for ISO sentence-case compliance.' + ) + parser.add_argument( + 'path', + nargs='?', + default='.', + help='Spec root directory to scan (default: current directory)', + ) + parser.add_argument( + '--fix', + action='store_true', + help='Edit files in-place to convert title-case headings to sentence case.', + ) + parser.add_argument( + '--proper-nouns', + type=Path, + default=None, + metavar='YAML', + help='Path to a YAML file listing additional proper nouns', + ) + parser.add_argument( + '--workers', + type=int, + default=DEFAULT_WORKERS, + metavar='N', + help=f'Parallel Pandoc workers (default: {DEFAULT_WORKERS})', + ) + args = parser.parse_args(argv) + spec_root = Path(args.path).resolve() + + if args.fix: + violations = check_spec( + spec_root, + workers=args.workers, + proper_nouns_path=args.proper_nouns, + ) + if not violations: + print('No heading case violations found.') + return + report = format_report(violations, spec_root=spec_root) + print(report) + print() + + files_fixed, headings_fixed = fix_spec( + spec_root, + workers=args.workers, + proper_nouns_path=args.proper_nouns, + violations=violations, + ) + print(f'Fixed {headings_fixed} heading(s) in {files_fixed} file(s).') + else: + violations = check_spec( + spec_root, + workers=args.workers, + proper_nouns_path=args.proper_nouns, + ) + report = format_report(violations, spec_root=spec_root) + if report: + print(report) + sys.exit(1) + else: + print('No heading case violations found.') + + +if __name__ == '__main__': + main() diff --git a/doc_build/iso_heading_proper_nouns.yaml b/doc_build/iso_heading_proper_nouns.yaml new file mode 100644 index 0000000..7b5caae --- /dev/null +++ b/doc_build/iso_heading_proper_nouns.yaml @@ -0,0 +1,50 @@ +# Proper nouns and domain-specific terms that must retain their capitalisation +# in heading sentence-case conversion. +# +# Words listed here will NOT be lowercased when a heading is converted from +# title case to sentence case (ISO/IEC Directives, Part 2, 11.4). +# +# Heuristics already preserve: +# - camelCase / PascalCase identifiers (ObjectPath, UsdGeomMesh) +# - ALL_CAPS acronyms (USD, API, ISO, AOUSD, ...) +# - Words mixing letters and digits (UTF-8, H264) +# +# Add words here that are proper nouns but don't match those patterns. +# Each entry is case-sensitive and must match the heading text exactly. + +proper_nouns: + # USD ecosystem + - OpenUSD + - Pixar + - Hydra + - Presto + - Katana + - Houdini + - Omniverse + - Blender + - Maya + - Nuke + - Solaris + + # Standards bodies and organisations + - Unicode + - IEEE + - Khronos + - Autodesk + - Apple + - Adobe + - Google + - Microsoft + - Nvidia + - Linux + - Windows + - macOS + + # Technical proper nouns + - Pandoc + - LaTeX + - Python + - Tectonic + - GitHub + - Markdown + - CommonMark \ No newline at end of file diff --git a/doc_build/iso_lint_utils.py b/doc_build/iso_lint_utils.py new file mode 100644 index 0000000..c49b52b --- /dev/null +++ b/doc_build/iso_lint_utils.py @@ -0,0 +1,156 @@ +"""Shared utilities for ISO linters. + +Common helpers used by ``iso_heading_case_lint``, ``iso_bold_table_lint``, +and ``iso_clause_lint``. +""" + +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any, Callable, List, Optional + +try: + from doc_build.filters.pandocfilters import stringify +except ImportError: + from filters.pandocfilters import stringify + +# Re-export so linters can ``from doc_build.iso_lint_utils import stringify``. +__all__ = [ + "DEFAULT_WORKERS", + "collect_md_files", + "format_report", + "get_sourcepos", + "run_parallel_check", + "stringify", + "unwrap_sourcepos_spans", +] + +# Maximum number of parallel Pandoc subprocesses. +DEFAULT_WORKERS = 8 + + +def collect_md_files(path: Path) -> List[Path]: + """Return a list of ``.md`` files under *path*. + + *path* may be a single file or a directory (walked recursively). + """ + path = Path(path) + if path.is_file(): + return [path] if path.suffix == '.md' else [] + md_files: List[Path] = [] + for dirpath, dirnames, filenames in os.walk(path): + dirnames.sort() + for fname in sorted(filenames): + if fname.endswith('.md'): + md_files.append(Path(dirpath) / fname) + return md_files + + +def run_parallel_check( + md_files: List[Path], + check_fn: Callable, + sort_key: Callable, + workers: int = DEFAULT_WORKERS, +) -> list: + """Run *check_fn* on each file in parallel and return sorted results. + + *check_fn* is called with a single ``Path`` argument and must return + a list of violation objects. Results are concatenated, sorted by + *sort_key*, and returned. + """ + all_violations: list = [] + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(check_fn, p): p for p in md_files} + for future in as_completed(futures): + all_violations.extend(future.result()) + all_violations.sort(key=sort_key) + return all_violations + + +def get_sourcepos(attr: list) -> Optional[int]: + """Extract the start line number from a Pandoc sourcepos Attr. + + Attr layout: ``[id, [classes], [[key, value], ...]]`` + + When Pandoc reads from a file the ``data-pos`` value has the form + ``filepath@startrow:startcol-endrow:endcol``; when reading from stdin + it omits the ``filepath@`` prefix. Both forms are handled here. + + Pandoc's data-pos for pipe tables may contain multiple + semicolon-separated ranges. This function returns the minimum start + line across all ranges, which is the header row for tables and the + only range for headings and other blocks. + + Returns the start row as a 1-based integer, or ``None`` if the + attribute is absent. + """ + for key, val in attr[2]: + if key == "data-pos": + # Strip optional "filepath@" prefix. + pos = val.split("@")[-1] + # Multiple ranges separated by ";". + min_line = None + for span in pos.split(";"): + start = span.split("-")[0] + line = int(start.split(":")[0]) + if min_line is None or line < min_line: + min_line = line + return min_line + return None + + +def format_report( + violations: List[Any], + label: str, + spec_root: Optional[Path] = None, + **format_kwargs, +) -> str: + """Format a list of violations into a human-readable report. + + Each violation must have a ``.file`` attribute (a ``Path``) and a + ``.format(display_path=..., **kwargs)`` method. + + *label* is a short noun phrase used in the summary line, e.g. + ``"heading case"`` → ``"3 heading case violation(s) in 2 file(s)"``. + + Extra *format_kwargs* are forwarded to each violation's ``.format()`` + call (e.g. ``context=5`` for the clause linter). + """ + if not violations: + return '' + + by_file: dict = {} + for v in violations: + rel = v.file.relative_to(spec_root) if spec_root else v.file + by_file.setdefault(rel, []).append(v) + + sections: List[str] = [] + total = 0 + for rel_path, file_violations in by_file.items(): + block_lines = [str(rel_path)] + for v in file_violations: + block_lines.append(v.format(display_path=rel_path, **format_kwargs)) + total += 1 + sections.append('\n'.join(block_lines)) + + file_count = len(by_file) + header = f'{total} {label} violation(s) in {file_count} file(s)\n' + return header + '\n' + '\n\n'.join(sections) + + +def unwrap_sourcepos_spans(inlines: list) -> list: + """Unwrap Span nodes injected by Pandoc's +sourcepos extension. + + With +sourcepos, every inline is wrapped in a Span carrying a + ``data-pos`` attribute. This function peels those wrappers so that + the structural content (Strong, Code, Str, ...) is directly visible. + """ + result = [] + for node in inlines: + if (isinstance(node, dict) + and node.get("t") == "Span" + and any(k == "wrapper" for k, _ in node["c"][0][2])): + result.extend(unwrap_sourcepos_spans(node["c"][1])) + else: + result.append(node) + return result diff --git a/doc_build/template/after-header-includes.latex b/doc_build/template/after-header-includes.latex index cf173aa..5cd80ae 100644 --- a/doc_build/template/after-header-includes.latex +++ b/doc_build/template/after-header-includes.latex @@ -90,9 +90,26 @@ $endif$ {2.5ex plus .2ex minus .2ex} % Vertical space before the title {0.5em} % Vertical space after the title (or horizontal for run-in) -% Fix the potential TOC issue +% Dynamic TOC number widths: measure each section number's natural width +% and add a small gap, instead of relying on LaTeX's fixed-width columns +% that overflow with double-digit numbers (issue #97). +% \numberline normally typesets into a fixed-width \@tempdima box; this +% redefinition uses the number's natural width + 0.5em padding so "1" and +% "12.2.1" both get exactly the space they need. \makeatletter -\newcommand{\l@subsubparagraph}{\@dottedtocline{6}{7em}{4em}} +\renewcommand{\numberline}[1]{% + \def\@tempa{#1}% + \ifx\@tempa\empty\else + \settowidth{\@tempdima}{#1}% + \hb@xt@\dimexpr\@tempdima+0.5em\relax{#1\hfil}% + \fi} + +% TOC entries for the custom deep levels that LaTeX doesn't define. +% Indents are relative to the left margin; numwidth is ignored now that +% \numberline auto-sizes, but \@dottedtocline still needs a placeholder. +\renewcommand{\l@paragraph}{\@dottedtocline{4}{7.0em}{4.1em}} +\renewcommand{\l@subparagraph}{\@dottedtocline{5}{10em}{5em}} +\newcommand{\l@subsubparagraph}{\@dottedtocline{6}{12em}{5.5em}} % hyperref uses \toclevel@ to determine PDF bookmark depth. % \titleclass does not set this, so without it bookmarks default to level 0 % (top-level), causing H6 to appear as a root item and nesting all following @@ -103,6 +120,39 @@ $endif$ \setcounter{secnumdepth}{6} \setcounter{tocdepth}{6} +% Allow deep list nesting (up to 9 levels). Default LaTeX limits itemize +% and enumerate to 4 levels, which is insufficient for deeply nested +% Markdown lists. The enumitem package raises the limit and lets us define +% labels for the extra levels. +\usepackage{enumitem} +\setlistdepth{9} + +% Redefine itemize and enumerate to support 9 nesting levels. +% \renewlist recreates the environment with the new max depth, defining +% the internal counters and label hooks that LaTeX normally caps at 4. +\renewlist{itemize}{itemize}{9} +\setlist[itemize,1]{label=\textbullet} +\setlist[itemize,2]{label=\textendash} +\setlist[itemize,3]{label=\textasteriskcentered} +\setlist[itemize,4]{label=\textperiodcentered} +\setlist[itemize,5]{label=\textopenbullet} +\setlist[itemize,6]{label=\textendash} +\setlist[itemize,7]{label=\textasteriskcentered} +\setlist[itemize,8]{label=\textperiodcentered} +\setlist[itemize,9]{label=\textbullet} + +% Same for enumerate: extend to 9 levels with cycling labels. +\renewlist{enumerate}{enumerate}{9} +\setlist[enumerate,1]{label=\arabic*.} +\setlist[enumerate,2]{label=\alph*.} +\setlist[enumerate,3]{label=\roman*.} +\setlist[enumerate,4]{label=\Alph*.} +\setlist[enumerate,5]{label=\arabic*.} +\setlist[enumerate,6]{label=\alph*.} +\setlist[enumerate,7]{label=\roman*.} +\setlist[enumerate,8]{label=\Alph*.} +\setlist[enumerate,9]{label=\arabic*.} + % adjust the title page style \usepackage{fontspec} \usepackage{titling} diff --git a/tests/build_scripts/build_diff.py b/tests/build_scripts/build_diff.py index 09a328f..4a96976 100644 --- a/tests/build_scripts/build_diff.py +++ b/tests/build_scripts/build_diff.py @@ -133,6 +133,9 @@ def build_diff(build_html: bool, build_pdf: bool, keep_pdf_latex: bool = False) only=[], exclude=[], keep_pdf_latex=keep_pdf_latex, + heading_case_lint=False, + heading_proper_nouns=None, + iso_xrefs=False, ) print("Running DocBuilder.build_docs() with --diff...") diff --git a/tests/specification/README.md b/tests/specification/README.md index 1ac5f93..83feab9 100644 --- a/tests/specification/README.md +++ b/tests/specification/README.md @@ -10,6 +10,7 @@ It is also a quick testbed for functionality. 1. [Inlined](Inlined.md) 2. [Header Levels](headers_levels.md) +3. [TOC and Lists Test](toc_and_lists_test.md) # Math Tests diff --git a/tests/specification/toc_and_lists_test.md b/tests/specification/toc_and_lists_test.md new file mode 100644 index 0000000..41edb39 --- /dev/null +++ b/tests/specification/toc_and_lists_test.md @@ -0,0 +1,216 @@ +# TOC width and list depth test + +This file tests two LaTeX issues: + +1. TOC number column overflow with double-digit section numbers (issue #97) +2. Deep list nesting beyond 4 levels (issue #98) + +## TOC overflow test sections + +The following sections have double-digit numbers that will overflow the default +TOC number column width. + +# Section 1 + +Content for section 1. + +## Section 1.1 + +Content for section 1.1. + +## Section 1.2 + +Content for section 1.2. + +# Section 2 + +Content for section 2. + +## Section 2.1 + +Content for section 2.1. + +# Section 3 + +Content for section 3. + +## Section 3.1 + +Content for section 3.1. + +# Section 4 + +Content for section 4. + +## Section 4.1 + +Content for section 4.1. + +# Section 5 + +Content for section 5. + +## Section 5.1 + +Content for section 5.1. + +# Section 6 + +Content for section 6. + +## Section 6.1 + +Content for section 6.1. + +# Section 7 + +Content for section 7. + +## Section 7.1 + +Content for section 7.1. + +# Section 8 + +Content for section 8. + +## Section 8.1 + +Content for section 8.1. + +# Section 9 + +Content for section 9. + +## Section 9.1 + +Content for section 9.1. + +# Section 10 + +Content for section 10. + +## Section 10.1 + +Content for section 10.1. + +## Section 10.2 + +Content for section 10.2. + +### Section 10.2.1 + +Content for section 10.2.1. + +# Section 11 + +Content for section 11. + +## Section 11.1 + +Content for section 11.1. + +# Section 12 + +Content for section 12. + +## Section 12.1 + +Content for section 12.1. + +## Section 12.2 + +Content for section 12.2. + +### Section 12.2.1 + +Content for section 12.2.1. + +### Section 12.2.2 + +Content for section 12.2.2. + +# Section 13 + +Content for section 13. + +## Section 13.1 + +Content for section 13.1. + +# Section 14 + +Content for section 14. + +# Section 15 + +Content for section 15. + +## Section 15.1 + +Content for section 15.1. + +## Section 15.2 + +Content for section 15.2. + +# Section 16 + +Content for section 16. + +# Section 17 + +Content for section 17. + +## Section 17.1 + +Content for section 17.1. + +# Section 18 + +Content for section 18. + +# Section 19 + +Content for section 19. + +## Section 19.1 + +Content for section 19.1. + +# Section 20 + +Content for section 20. + +## Section 20.1 + +Content for section 20.1. + +### Section 20.1.1 + +Content for section 20.1.1. + +# Foreword {.unnumbered} + +This is a simulated ISO foreword that should appear in the TOC without a +section number and without extra left padding. + +# Deep list nesting test + +This section tests list nesting beyond 4 levels, which causes a "Too deeply +nested" error in default LaTeX. + +- Level 1 item + - Level 2 item + - Level 3 item + - Level 4 item + - Level 5 item (this should fail without enumitem) + - Level 6 item + - Level 7 item + +1. Ordered level 1 + 1. Ordered level 2 + 1. Ordered level 3 + 1. Ordered level 4 + 1. Ordered level 5 (this should also fail) + 1. Ordered level 6