Adds Qiskit addon prose content pipeline

README.md

@@ -140,6 +140,13 @@ API docs authors can preview their changes to one of the APIs by using the `-a`
 1. Run `npm run gen-api -- -p <pkg-name> -v <version> -a <path/to/docs/_build/html>`.
 2. Execute `./start` and open up `http://localhost:3000`, as explained in the prior section.
 
+### Addon docs authors: How to preview your changes
+
+Addon docs authors can preview guide and how-to changes by pointing the addon pipeline at a local Sphinx build:
+
+1. Run `npm run gen-addon -- -p <pkg-name> --sphinx-artifact-folder <path/to/docs/_build/html>`.
+2. Execute `./start` and navigate to `http://localhost:3000/docs/addons/<pkg-name>`.
+
 ## Run quality checks
 
 We use multiple tools to ensure that documentation meets high standards. These tools will run automatically in your PR through CI, but it is much faster to run the checks locally.
@@ -321,6 +328,43 @@ Additionally, If you are regenerating a dev version, then you can add `--dev` as
 
 In this case, no commit will be automatically created.
 
+## Generate or update addon docs
+
+Addon docs for packages like are generated by a separate pipeline from the API pipeline. Content is published to `docs/addons/{pkg}/` rather than `docs/api/{pkg}/`.
+
+### Generate addon docs for a new release
+
+3. Run:
+
+   ```sh
+   npm run gen-addon -- -p <pkg-name>
+   ```
+
+   To regenerate all addon packages at once:
+
+   ```sh
+   npm run gen-addon -- --all
+   ```
+
+   Or if you don't need to download but need to re-run the pipeline you can pass "--ship-download" to speed things up
+
+   ```sh
+   npm run gen-addon -- --all --skip-download
+   ```
+
+4. Open a pull request with the generated changes.
+
+### Add a new addon package
+
+1. Download the Sphinx HTML artifact from the package's GitHub Actions workflow (see the [CI artifact links](#initial-steps) in the API docs section).
+2. Unzip the artifact and make sure there is no subfolder before the content. Name the zip to its minor version, e.g. `0.5.zip`, and upload it to the Box folder.
+   - Unzipping should produce a flat structure of the content: 0.5.zip -> index.html. NOT: 0.5.zip -> sphinx-output/index.html.
+3. Add the package's Box artifact URL to `scripts/config/api-html-artifacts.json` (see how to get the shareable link [here](#Final-steps-for-the-rc1-release))
+4. Add the package name to `Pkg.ADDON_NAMES` in `scripts/js/lib/api/Pkg.ts` and add a `new Pkg(...)` with the correct `title`, `githubSlug`, and `language` in that file as well.
+5. Run the pipeline for the new addon (`gen-api`/`gen-addon`)
+6. Verify content with `npm run check:internal-links -- --current-apis`, `npm run check:spelling`, and `npm run check:markdown -- --apis`
+7. Verify rendering with `./start --apis`
+
 ## Generate new API docs
 
 Use this process when we want to publish new API docs, such as when we release a new version of a package like Qiskit SDK.
@@ -354,21 +398,25 @@ All release types start with the following steps:
 1. In Git, check out the main branch of `Qiskit/documentation` and pull any updates. Then, create a new Git branch.
 2. Determine which documentation you want to generate (e.g., `qiskit` or `qiskit-ibm-runtime`) and its full version, e.g., `0.45.2` or `1.2.0rc1`.
 3. Download a CI artifact with the project's documentation. To find this:
+
    1. Find the relevant GitHub Actions workflow for the project:
+
       - Qiskit SDK: https://github.com/Qiskit/qiskit/actions/workflows/docs_deploy.yml
       - Runtime: https://github.com/Qiskit/qiskit-ibm-runtime/actions/workflows/docs.yml
       - Transpiler Service: https://github.com/Qiskit/qiskit-ibm-transpiler/actions/workflows/upload-docs.yml
-      - qiskit-addon-acq-tensor: https://github.com/Qiskit/qiskit-addon-aqc-tensor/actions/workflows/docs.yml
+      - qiskit-addon-aqc-tensor: https://github.com/Qiskit/qiskit-addon-aqc-tensor/actions/workflows/docs.yml
       - qiskit-addon-obp: https://github.com/Qiskit/qiskit-addon-obp/actions/workflows/docs.yml
       - qiskit-addon-mpf: https://github.com/Qiskit/qiskit-addon-mpf/actions/workflows/docs.yml
       - qiskit-addon-sqd: https://github.com/Qiskit/qiskit-addon-sqd/actions/workflows/docs.yml
       - qiskit-addon-cutting: https://github.com/Qiskit/qiskit-addon-cutting/actions/workflows/docs.yml
       - qiskit-addon-utils: https://github.com/Qiskit/qiskit-addon-utils/actions/workflows/docs.yml
+
    2. Find the run for the release by looking at the middle column with the blue text; look for the version number, like `0.45.2`. For the `rc1` release, look for `main` in blue text and a run description like "Prepare 2.3.0rc1".
    3. Click the CI run name. (Not the middle column with the blue link!)
    4. In the left navbar, it should show as selected the "Summary" page with the house.
    5. Scroll down to "Artifacts" and look for the artifact related to documentation, such as `html_docs`.
    6. Download the artifact by clicking on its name.
+
 4. On some operating systems, the downloaded zip file will be auto-expanded rather than staying a zip file. If this happens, compress it back to a zip file. On macOS, secondary-click on the folder in Finder and use the "Compress" option.
 5. Rename the downloaded zip file with its minor-version number. For example, for the release `0.45.2`, rename `html_docs.zip` to `0.45.zip`. For release candidates (rc), use a value like `2.3-rc.zip`.
 6. Upload the renamed zip file to https://ibm.ent.box.com/folder/246867452622. If this is a patch release, this step will overwrite the prior file; if it's the `rc1` release or a new minor version like `2.3.0`, it will be a new file.

package.json

@@ -32,6 +32,8 @@
     "check:qiskit-versions": "tsx scripts/js/commands/checkQiskitApiVersions.ts",
     "regen-apis": "tsx scripts/js/commands/api/regenerateApiDocs.ts",
     "gen-api": "tsx scripts/js/commands/api/updateApiDocs.ts",
+    "gen-addon": "tsx scripts/js/commands/api/updateAddonDocs.ts",
+    "postgen-addon": "./fix",
     "generate-historical-redirects": "tsx scripts/js/commands/api/generateHistoricalRedirects.ts",
     "save-internal-links": "tsx scripts/js/commands/saveInternalLinks.ts"
   },

scripts/ci/check-all-notebooks-are-tested.py

@@ -15,22 +15,29 @@
 they don't slip through our CI tests.
 """
 
+import fnmatch
 from pathlib import Path
 import tomllib
 import sys
 
 config_path = Path("scripts/config/notebook-testing.toml")
 config = tomllib.loads(config_path.read_text())
 
-categorized_notebooks = set()
+categorized_notebooks: set[Path] = set()
+categorized_globs: list[str] = []
 for group in config["groups"].values():
     for path in group.get("notebooks", []):
-        categorized_notebooks.add(Path(path))
+        if "*" in path or "?" in path:
+            categorized_globs.append(path)
+        else:
+            categorized_notebooks.add(Path(path))
 
 uncategorized = [
     path
     for path in Path(".").glob("[!.]*/**/*.ipynb")
-    if not path.match("**/.ipynb_checkpoints/**") and path not in categorized_notebooks
+    if not path.match("**/.ipynb_checkpoints/**")
+    and path not in categorized_notebooks
+    and not any(fnmatch.fnmatch(str(path), g) for g in categorized_globs)
 ]
 
 if uncategorized:

scripts/config/allowLists.ts

@@ -21,6 +21,8 @@ export function ignoreTitleMismatch(filepath: string): boolean {
   return IGNORE_TITLE_MISMATCHES.includes(filepath);
 }
 
+export const IMAGE_ALLOWLIST: Set<string> = new Set([]);
+
 const IGNORE_TITLE_MISMATCHES: string[] = [
   "docs/guides/directed-execution-model.mdx", // ok
   "docs/guides/estimator-examples.ipynb", // ok

scripts/config/notebook-testing.toml

@@ -137,6 +137,12 @@ notebooks = [
 # Don't ever test the following notebooks
 [groups.exclude]
 notebooks = [
+  # Addon notebooks require the addon packages and are not run in standard CI
+  "docs/addons/**/*.ipynb",
+
+  # Test fixture — not real content
+  "scripts/js/lib/api/testdata/**/*.ipynb",
+
   # This notebook contains undefined variables so can't run at all.
   "docs/guides/function-template-hamiltonian-simulation.ipynb",
   "docs/guides/function-template-chemistry-workflow.ipynb",

scripts/js/commands/api/updateAddonDocs.ts

@@ -0,0 +1,106 @@
+// This code is a Qiskit project.
+//
+// (C) Copyright IBM 2024.
+//
+// This code is licensed under the Apache License, Version 2.0. You may
+// obtain a copy of this license in the LICENSE file in the root directory
+// of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+//
+// Any modifications or derivative works of this code must retain this
+// copyright notice, and modified files need to carry a notice indicating
+// that they have been altered from the originals.
+
+import { readdir } from "fs/promises";
+
+import yargs from "yargs/yargs";
+import { hideBin } from "yargs/helpers";
+
+import { Pkg } from "../../lib/api/Pkg.js";
+import { runAddonDocsPipeline } from "../../lib/api/addonDocsPipeline.js";
+import { zxMain } from "../../lib/zx.js";
+import { parseMinorVersion } from "../../lib/apiVersions.js";
+import { deleteOutputDirs, prepareSphinxFolder } from "./updateDocsShared.js";
+
+type Args = {
+  package?: string;
+  all: boolean;
+  skipDownload: boolean;
+  sphinxArtifactFolder?: string;
+};
+
+const readArgs = (): Args => {
+  return yargs(hideBin(process.argv))
+    .version(false)
+    .option("package", {
+      alias: "p",
+      type: "string",
+      choices: Pkg.ADDON_NAMES,
+      description: "Which addon package to update",
+    })
+    .option("all", {
+      type: "boolean",
+      default: false,
+      description: "Update all addon packages in parallel",
+    })
+    .option("skip-download", {
+      alias: "s",
+      type: "boolean",
+      default: false,
+      description: "Skip downloading the Sphinx artifact",
+    })
+    .option("sphinx-artifact-folder", {
+      type: "string",
+      description: "Use a local artifact folder instead of downloading",
+    })
+    .check((argv) => {
+      if (!argv.all && !argv.package) {
+        throw new Error("Either --package or --all is required");
+      }
+      return true;
+    })
+    .parseSync() as unknown as Args;
+};
+
+async function resolveVersion(pkgName: string): Promise<string> {
+  const entries = await readdir(`.sphinx-artifacts/${pkgName}`);
+  const versions = entries.filter((e) => /^\d/.test(e)).sort();
+  if (versions.length === 0) {
+    throw new Error(
+      `No artifact versions found for ${pkgName}. Run without --skip-download first.`,
+    );
+  }
+  return versions.at(-1)!;
+}
+
+async function generateVersion(pkg: Pkg, args: Args): Promise<void> {
+  const sphinxArtifactFolder = await prepareSphinxFolder(pkg, args);
+  await deleteOutputDirs(pkg, {
+    markdownDir: pkg.outputDir("docs/addons"),
+    imagesDir: pkg.outputDir("public/docs/images/addons"),
+    recursive: true,
+  });
+
+  console.log(`Run pipeline for ${pkg.name}:${pkg.versionWithoutPatch}`);
+  await runAddonDocsPipeline(
+    sphinxArtifactFolder,
+    "docs/addons",
+    "public/docs",
+    pkg,
+  );
+}
+
+async function generatePackage(pkgName: string, args: Args): Promise<void> {
+  const version = await resolveVersion(pkgName);
+  const minorVersion = parseMinorVersion(version);
+  if (minorVersion === null) {
+    throw new Error(`Could not parse version ${version} for ${pkgName}`);
+  }
+  const pkg = await Pkg.fromArgs(pkgName, version, minorVersion, "latest");
+  await generateVersion(pkg, args);
+}
+
+zxMain(async () => {
+  const args = readArgs();
+  const packages = args.all ? Pkg.ADDON_NAMES : [args.package!];
+  await Promise.all(packages.map((pkgName) => generatePackage(pkgName, args)));
+});

scripts/js/commands/api/updateApiDocs.ts

@@ -15,11 +15,10 @@ import { hideBin } from "yargs/helpers";
 
 import { Pkg } from "../../lib/api/Pkg.js";
 import { zxMain } from "../../lib/zx.js";
-import { parseMinorVersion, isValidVersion } from "../../lib/apiVersions.js";
-import { pathExists, rmFilesInFolder } from "../../lib/fs.js";
-import { downloadSphinxArtifact } from "../../lib/api/sphinxArtifacts.js";
-import { runConversionPipeline } from "../../lib/api/conversionPipeline.js";
+import { isValidVersion, parseMinorVersion } from "../../lib/apiVersions.js";
+import { runApiDocsPipeline } from "../../lib/api/apiDocsPipeline.js";
 import { generateHistoricalRedirects } from "./generateHistoricalRedirects.js";
+import { deleteOutputDirs, prepareSphinxFolder } from "./updateDocsShared.js";
 
 export interface Arguments {
   [x: string]: unknown;
@@ -87,10 +86,14 @@ export async function generateVersion(
   args: Arguments,
 ): Promise<void> {
   const sphinxArtifactFolder = await prepareSphinxFolder(pkg, args);
-  await deleteExistingFiles(pkg);
+  await deleteOutputDirs(pkg, {
+    markdownDir: pkg.apiOutputDir("docs"),
+    imagesDir: pkg.apiOutputDir("public/docs/images"),
+    recursive: false,
+  });
 
   console.log(`Run pipeline for ${pkg.name}:${pkg.versionWithoutPatch}`);
-  await runConversionPipeline(sphinxArtifactFolder, "docs", "public/docs", pkg);
+  await runApiDocsPipeline(sphinxArtifactFolder, "docs", "public/docs", pkg);
   await generateHistoricalRedirects();
 }
 
@@ -112,43 +115,6 @@ export function determineMinorVersion(args: Arguments): string {
   return minorVersion;
 }
 
-async function prepareSphinxFolder(pkg: Pkg, args: Arguments): Promise<string> {
-  if (args.sphinxArtifactFolder) {
-    if (!(await pathExists(args.sphinxArtifactFolder))) {
-      throw new Error(
-        `Explicit artifact path '${args.sphinxArtifactFolder}' does not exist.`,
-      );
-    }
-    return args.sphinxArtifactFolder;
-  }
-  const sphinxArtifactFolder = pkg.sphinxArtifactFolder();
-  if (
-    args.skipDownload &&
-    (await pathExists(`${sphinxArtifactFolder}/artifact`))
-  ) {
-    console.log(
-      `Skip downloading sources for ${pkg.name}:${pkg.versionWithoutPatch}`,
-    );
-  } else {
-    await downloadSphinxArtifact(pkg, sphinxArtifactFolder);
-  }
-  return `${sphinxArtifactFolder}/artifact`;
-}
-
-async function deleteExistingFiles(pkg: Pkg): Promise<void> {
-  const markdownDir = pkg.outputDir("docs");
-  if (await pathExists(markdownDir)) {
-    await rmFilesInFolder(markdownDir);
-  }
-  const imagesDir = pkg.outputDir("public/docs/images");
-  if (await pathExists(imagesDir)) {
-    await rmFilesInFolder(imagesDir);
-  }
-  console.log(
-    `Deleted existing markdown & images for ${pkg.name}:${pkg.versionWithoutPatch}`,
-  );
-}
-
 if (import.meta.url === `file://${process.argv[1]}`) {
   zxMain(async () => {
     const args = readArgs();