Migrate examples to Jupyter notebooks

.githooks/pre-commit

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mapfile -d '' notebooks < <(git diff --cached --name-only -z --diff-filter=ACMR -- '*.ipynb')
+if [[ ${#notebooks[@]} -eq 0 ]]; then
+  exit 0
+fi
+
+python3 scripts/sync_notebook_badges.py "${notebooks[@]}"
+python3 scripts/strip_notebook_outputs.py "${notebooks[@]}"
+
+# Re-stage files in case the stripper changed them.
+git add -- "${notebooks[@]}"

.githooks/pre-push

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mapfile -d '' notebooks < <(git ls-files -z '*.ipynb')
+if [[ ${#notebooks[@]} -eq 0 ]]; then
+  exit 0
+fi
+
+python3 scripts/sync_notebook_badges.py "${notebooks[@]}"
+python3 scripts/strip_notebook_outputs.py "${notebooks[@]}"
+
+if ! git diff --quiet -- "${notebooks[@]}"; then
+  git add -- "${notebooks[@]}"
+  echo "Notebook badges/outputs were normalized during pre-push. Commit the updated notebooks and push again." >&2
+  exit 1
+fi

.github/workflows/CI.yml

@@ -12,6 +12,21 @@ concurrency:
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 
 jobs:
+  notebook-clean:
+    name: Notebook Output Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Verify notebooks are stripped
+        run: |
+          notebooks=$(git ls-files '*.ipynb')
+          if [ -z "$notebooks" ]; then
+            exit 0
+          fi
+          python3 scripts/sync_notebook_badges.py $notebooks
+          python3 scripts/strip_notebook_outputs.py $notebooks
+          git diff --exit-code
+
   test:
     name: ${{ matrix.pkg.name }} - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
@@ -108,4 +123,4 @@ jobs:
             using CUDA;
             using GeneralisedFilters;
             println("GeneralisedFilters with CUDA loaded successfully");
-            println("CUDA functional: ", CUDA.functional())'
+            println("CUDA functional: ", CUDA.functional())'

.github/workflows/Documentation.yml

@@ -36,7 +36,24 @@ jobs:
       - uses: julia-actions/setup-julia@v2
         with:
           version: '1'
-      - uses: julia-actions/cache@v1
+      - uses: julia-actions/cache@v2
+      - name: Cache notebook outputs
+        if: matrix.pkg.name == 'GeneralisedFilters'
+        uses: actions/cache@v4
+        with:
+          path: GeneralisedFilters/docs/src/examples
+          key: notebooks-${{ runner.os }}-${{ hashFiles('GeneralisedFilters/examples/**/*.ipynb') }}
+      - name: Cache pip packages
+        if: matrix.pkg.name == 'GeneralisedFilters'
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-jupyter-nbconvert
+      - name: Install notebook tooling
+        if: matrix.pkg.name == 'GeneralisedFilters'
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install jupyter nbconvert
       - name: Install dependencies
         run: |
           julia --project=${{ matrix.pkg.dir }}/docs/ --color=yes -e '

GeneralisedFilters/docs/Project.toml

@@ -1,3 +1,3 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
-Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a"

GeneralisedFilters/docs/make.jl

@@ -1,17 +1,20 @@
 push!(LOAD_PATH, "../src/")
+const REPO = "TuringLang/SSMProblems.jl"
+const PKG_SUBDIR = "GeneralisedFilters"
 
 #
 # With minor changes from https://github.com/JuliaGaussianProcesses/AbstractGPs.jl/docs
 #
 ### Process examples
-# Always rerun examples
+const EXAMPLES_ROOT = joinpath(@__DIR__, "..", "examples")
 const EXAMPLES_OUT = joinpath(@__DIR__, "src", "examples")
-ispath(EXAMPLES_OUT) && rm(EXAMPLES_OUT; recursive=true)
 mkpath(EXAMPLES_OUT)
+const EXAMPLE_ASSETS_OUT = joinpath(@__DIR__, "src", "assets", "examples")
+mkpath(EXAMPLE_ASSETS_OUT)
 
 # Install and precompile all packages
 # Workaround for https://github.com/JuliaLang/Pkg.jl/issues/2219
-examples = filter!(isdir, readdir(joinpath(@__DIR__, "..", "examples"); join=true))
+examples = sort(filter!(isdir, readdir(EXAMPLES_ROOT; join=true)))
 above = joinpath(@__DIR__, "..")
 ssmproblems_path = joinpath(above, "..", "SSMProblems")
 let script = "using Pkg; Pkg.activate(ARGS[1]); Pkg.develop(path=\"$(above)\"); Pkg.develop(path=\"$(ssmproblems_path)\"); Pkg.instantiate()"
@@ -26,11 +29,13 @@ let script = "using Pkg; Pkg.activate(ARGS[1]); Pkg.develop(path=\"$(above)\");
     end
 end
 # Run examples asynchronously
-processes = let literatejl = joinpath(@__DIR__, "literate.jl")
+processes = let
+    notebookjl = joinpath(@__DIR__, "notebook.jl")
+    docs_project = abspath(@__DIR__)
     map(examples) do example
         return run(
             pipeline(
-                `$(Base.julia_cmd()) $literatejl $(basename(example)) $EXAMPLES_OUT`;
+                `$(Base.julia_cmd()) --project=$(docs_project) $notebookjl $(basename(example)) $EXAMPLES_OUT`;
                 stdin=devnull,
                 stdout=devnull,
                 stderr=stderr,
@@ -43,6 +48,108 @@ end
 # Check that all examples were run successfully
 isempty(processes) || success(processes) || error("some examples were not run successfully")
 
+example_slug(markdown_filename::AbstractString) = splitext(markdown_filename)[1]
+
+function example_title(markdown_path::AbstractString, slug::AbstractString)
+    for line in eachline(markdown_path)
+        stripped = strip(line)
+        startswith(stripped, "# ") && return strip(stripped[3:end])
+    end
+    return replace(slug, "-" => " ")
+end
+
+function example_summary(markdown_path::AbstractString)
+    in_fence = false
+    for line in eachline(markdown_path)
+        stripped = strip(line)
+
+        if startswith(stripped, "```")
+            in_fence = !in_fence
+            continue
+        end
+
+        if in_fence || isempty(stripped)
+            continue
+        end
+
+        if startswith(stripped, "# ")
+            continue
+        end
+
+        if occursin("Open in Colab", stripped) ||
+            occursin("Source notebook", stripped) ||
+            startswith(stripped, "*This page was generated")
+            continue
+        end
+
+        return replace(stripped, "|" => "\\|")
+    end
+
+    return "Runnable example with executable source."
+end
+
+function example_thumbnail(slug::AbstractString)
+    for ext in ("svg", "png", "jpg", "jpeg", "webp")
+        thumb = joinpath(EXAMPLE_ASSETS_OUT, string(slug, ".", ext))
+        if isfile(thumb)
+            return "../assets/examples/$(slug).$(ext)"
+        end
+    end
+    return "../assets/examples/default.svg"
+end
+
+function links_for_example(slug::AbstractString)
+    example_dir = joinpath(EXAMPLES_ROOT, slug)
+    notebook_name = string(slug, ".ipynb")
+    isfile(joinpath(example_dir, notebook_name)) ||
+        error("example $(slug) must include $(notebook_name)")
+
+    return join(
+        [
+            "[Colab](https://colab.research.google.com/github/$(REPO)/blob/main/$(PKG_SUBDIR)/examples/$(slug)/$(notebook_name))",
+            "[Notebook](https://github.com/$(REPO)/blob/main/$(PKG_SUBDIR)/examples/$(slug)/$(notebook_name))",
+        ],
+        " · ",
+    )
+end
+
+function write_examples_index(example_markdowns::Vector{String})
+    index_path = joinpath(EXAMPLES_OUT, "index.md")
+    open(index_path, "w") do io
+        println(io, "# Examples")
+        println(io)
+        println(
+            io,
+            "Executable examples for `GeneralisedFilters` with links to notebooks and source files.",
+        )
+        println(io)
+        println(io, "| Example | Preview |")
+        println(io, "| :-- | :-- |")
+        for markdown in example_markdowns
+            slug = example_slug(markdown)
+            markdown_path = joinpath(EXAMPLES_OUT, markdown)
+            title = example_title(markdown_path, slug)
+            summary = example_summary(markdown_path)
+            links = links_for_example(slug)
+            thumbnail = example_thumbnail(slug)
+
+            page_link = "[$(title)]($(markdown))"
+            left = string(page_link, "<br>", summary, "<br>", links)
+            right = "[![$(title)]($(thumbnail))]($(markdown))"
+            println(io, "| $(left) | $(right) |")
+        end
+    end
+    return nothing
+end
+
+const EXAMPLE_MARKDOWNS = sort(
+    filter(
+        filename -> endswith(filename, ".md") && filename != "index.md",
+        readdir(EXAMPLES_OUT),
+    ),
+)
+write_examples_index(EXAMPLE_MARKDOWNS)
+
 # Building Documenter
 using Documenter
 using GeneralisedFilters
@@ -56,11 +163,9 @@ makedocs(;
     format=Documenter.HTML(; size_threshold=1000 * 2^11), # 1Mb per page
     pages=[
         "Home" => "index.md",
-        "Examples" => [
-            map(
-                (x) -> joinpath("examples", x),
-                filter!(filename -> endswith(filename, ".md"), readdir(EXAMPLES_OUT)),
-            )...,
+        "Examples" => Any[
+            "examples/index.md",
+            map((x) -> joinpath("examples", x), EXAMPLE_MARKDOWNS)...,
         ],
     ],
     #strict=true,