feat(opa-proto): support proto-format plan bundles (opa build --format=proto)

.github/workflows/pull-request.yml

@@ -248,6 +248,36 @@ jobs:
       - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0
       - run: ./gradlew :opa-slf4j:test
 
+  verify-proto-vendor:
+    name: Verify vendored proto schemas match go.mod
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
+        with:
+          distribution: temurin
+          java-version: 21
+      - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: tools/generate-compliance-tests/go.mod
+      # The vendored .proto files in opa-proto must always match the OPA version
+      # pinned in tools/generate-compliance-tests/go.mod (the source of truth).
+      # Re-vendor via the Gradle task and fail if it produces any change — i.e. the
+      # OPA version was bumped, or a .proto was hand-edited, without re-vendoring.
+      - name: Re-vendor and check for drift
+        run: |
+          set -e
+          ./gradlew --rerun-tasks :opa-proto:vendorProtoSchemas
+          if ! git diff --quiet -- opa-proto/src/main/proto; then
+            echo "::error::Vendored proto schemas are out of sync with the OPA version in tools/generate-compliance-tests/go.mod. Run ./gradlew :opa-proto:vendorProtoSchemas and commit the result."
+            git --no-pager diff -- opa-proto/src/main/proto
+            exit 1
+          fi
+          echo "Vendored proto schemas are in sync with the pinned OPA version."
+
   pr-check-summary:
     name: PR Check Summary
     runs-on: ubuntu-latest
@@ -256,6 +286,7 @@ jobs:
       - gh-actions-lint
       - lint
       - validate-pom
+      - verify-proto-vendor
       - build
       - test-opa-evaluator
       - test-opa-jackson

.gitignore

@@ -13,6 +13,9 @@
 *.tar.gz
 *.rar
 
+# …but keep bundle fixtures checked in as test resources
+!**/src/test/resources/**/*.tar.gz
+
 # Build directories
 build/
 target/
@@ -60,3 +63,6 @@ nb-configuration.xml
 .DS_Store
 Thumbs.db
 
+# Go — compiled helper tool binary (built by `go build` in the compliance-test tool)
+/tools/generate-compliance-tests/generate-compliance-tests
+

cli/build.gradle.kts

@@ -15,6 +15,9 @@ dependencies {
 
     runtimeOnly(project(":opa-builtins"))
     runtimeOnly(project(":opa-jackson"))
+    // Enables the CLI to read proto-format bundles (opa build --format=proto). opa-jackson still
+    // handles data.json and JSON-format plans/manifests; opa-proto decodes plan.pb/.manifest.pb.
+    runtimeOnly(project(":opa-proto"))
 
     testImplementation("org.junit.jupiter:junit-jupiter:6.1.1")
     testRuntimeOnly("org.junit.platform:junit-platform-launcher:6.1.1")

opa-evaluator/src/main/java/io/github/open_policy_agent/opa/bundle/BundleAssembler.java

@@ -3,13 +3,11 @@
 import io.github.open_policy_agent.opa.ast.types.RegoObject;
 import io.github.open_policy_agent.opa.ast.types.RegoString;
 import io.github.open_policy_agent.opa.ir.PolicyReader;
+import io.github.open_policy_agent.opa.spi.Services;
 import io.github.open_policy_agent.opa.storage.Store;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.ServiceLoader;
 
 /**
  * Processes individual bundle files and assembles them into a {@link Bundle}.
@@ -38,39 +36,53 @@ public class BundleAssembler {
 
   static final BundleParser BUNDLE_PARSER = loadSingleton(BundleParser.class);
 
+  /**
+   * Holds the optional proto decoder, resolved lazily on first proto-format load.
+   *
+   * <p>Deferring resolution (rather than a {@code static final} field on {@link BundleAssembler})
+   * keeps a proto-specific misconfiguration — no {@code opa-proto} module, two providers, or a
+   * provider that fails to instantiate — from running in {@code BundleAssembler}'s own class
+   * initializer, where it would break <em>all</em> bundle loading, including JSON-only bundles that
+   * never touch proto. The holder class initializes only when {@link #requireProtoDecoder} first
+   * references it, so any such failure surfaces only when a proto bundle is actually read.
+   */
+  private static final class ProtoDecoderHolder {
+    static final ProtoBundleDecoder INSTANCE = loadOptional(ProtoBundleDecoder.class);
+  }
+
   /**
    * Loads exactly one implementation of the given SPI from the classpath. Throws if zero or more
    * than one implementation is registered, since either case produces an ambiguous runtime.
    */
   private static <T> T loadSingleton(Class<T> spi) {
-    List<T> impls = new ArrayList<>();
-    for (T impl : ServiceLoader.load(spi)) {
-      impls.add(impl);
-    }
-    if (impls.isEmpty()) {
-      throw new IllegalStateException(
-          "No "
-              + spi.getSimpleName()
-              + " implementation found on the classpath. Add a module that provides "
-              + spi.getSimpleName()
-              + " (e.g. opa-jackson).");
-    }
-    if (impls.size() > 1) {
-      StringBuilder names = new StringBuilder();
-      for (int i = 0; i < impls.size(); i++) {
-          if (i > 0) {
-              names.append(", ");
-          }
-        names.append(impls.get(i).getClass().getName());
-      }
+    return Services.loadAtMostOne(spi)
+        .orElseThrow(
+            () ->
+                new IllegalStateException(
+                    "No "
+                        + spi.getSimpleName()
+                        + " implementation found on the classpath. Add a module that provides "
+                        + spi.getSimpleName()
+                        + " (e.g. opa-jackson)."));
+  }
+
+  /**
+   * Loads an optional SPI: returns {@code null} when no implementation is registered, the single
+   * implementation when exactly one is, and throws when more than one is registered.
+   */
+  private static <T> T loadOptional(Class<T> spi) {
+    return Services.loadAtMostOne(spi).orElse(null);
+  }
+
+  private static ProtoBundleDecoder requireProtoDecoder() {
+    ProtoBundleDecoder decoder = ProtoDecoderHolder.INSTANCE;
+    if (decoder == null) {
       throw new IllegalStateException(
-          "Multiple "
-              + spi.getSimpleName()
-              + " implementations found on the classpath: "
-              + names
-              + ". Only one provider may be registered.");
+          "No ProtoBundleDecoder implementation found on the classpath, but the bundle is in proto"
+              + " format (plan.pb / .manifest.pb). Add the opa-proto module to read proto-format"
+              + " bundles.");
     }
-    return impls.get(0);
+    return decoder;
   }
 
   private final Bundle.Builder builder = new Bundle.Builder();
@@ -83,6 +95,16 @@ public void loadPlan(InputStream in) throws IOException {
     hasContent = true;
   }
 
+  /**
+   * Load a compiled IR policy from a protobuf {@code plan.pb} stream.
+   *
+   * @throws IllegalStateException if no {@link ProtoBundleDecoder} is registered (add opa-proto)
+   */
+  public void loadPlanProto(InputStream in) throws IOException {
+    builder.withIrPolicy(requireProtoDecoder().decodePlan(in));
+    hasContent = true;
+  }
+
   /**
    * Load data from a {@code data.json} at the given path within the bundle.
    *
@@ -132,6 +154,63 @@ public void loadManifest(InputStream in) throws IOException {
     builder.withManifest(BUNDLE_PARSER.parseManifest(in));
   }
 
+  /**
+   * Load bundle metadata from a protobuf {@code .manifest.pb} stream.
+   *
+   * @throws IllegalStateException if no {@link ProtoBundleDecoder} is registered (add opa-proto)
+   */
+  public void loadManifestProto(InputStream in) throws IOException {
+    builder.withManifest(requireProtoDecoder().decodeManifest(in));
+  }
+
+  /**
+   * Detect the bundle's wire format from which plan/manifest artifacts are present, reject
+   * mixed-format bundles, and load the plan and manifest in the correct format.
+   *
+   * <p>This centralizes the format-detection and routing policy so {@link BundleLoader}
+   * implementations only need to locate the four possible artifacts and expose each as an {@link
+   * InputStreamSource} (or {@code null} when absent), regardless of whether the bytes come from a
+   * directory, a tarball, or elsewhere. Proto takes precedence over JSON for each artifact, matching
+   * OPA; a proto plan cannot be paired with a JSON manifest and vice versa (see {@link
+   * BundleFormat#validate}).
+   *
+   * @param planJson source for {@code plan.json}, or {@code null} if absent
+   * @param planProto source for {@code plan.pb}, or {@code null} if absent
+   * @param manifestJson source for {@code .manifest}, or {@code null} if absent
+   * @param manifestProto source for {@code .manifest.pb}, or {@code null} if absent
+   * @throws IllegalArgumentException if the bundle mixes wire formats
+   * @throws IOException if a source cannot be opened or its contents cannot be parsed
+   */
+  public void loadPlanAndManifest(
+      InputStreamSource planJson,
+      InputStreamSource planProto,
+      InputStreamSource manifestJson,
+      InputStreamSource manifestProto)
+      throws IOException {
+    BundleFormat.validate(
+        planJson != null, planProto != null, manifestJson != null, manifestProto != null);
+
+    if (planProto != null) {
+      try (InputStream in = planProto.open()) {
+        loadPlanProto(in);
+      }
+    } else if (planJson != null) {
+      try (InputStream in = planJson.open()) {
+        loadPlan(in);
+      }
+    }
+
+    if (manifestProto != null) {
+      try (InputStream in = manifestProto.open()) {
+        loadManifestProto(in);
+      }
+    } else if (manifestJson != null) {
+      try (InputStream in = manifestJson.open()) {
+        loadManifest(in);
+      }
+    }
+  }
+
   /** Add a Rego source file by its relative path. */
   public void addRego(String path, String content) {
     builder.withRego(path, content);

opa-evaluator/src/main/java/io/github/open_policy_agent/opa/bundle/BundleFormat.java

@@ -0,0 +1,86 @@
+package io.github.open_policy_agent.opa.bundle;
+
+/**
+ * Constants and validation for the two on-disk bundle wire formats.
+ *
+ * <p>OPA's {@code opa build} emits plan bundles in either JSON (the default) or protobuf form. The
+ * two forms use distinct filenames for the plan and manifest:
+ *
+ * <table border="1">
+ *   <caption>Format-specific filenames</caption>
+ *   <tr><th>Artifact</th><th>JSON</th><th>Proto</th></tr>
+ *   <tr><td>Plan</td><td>{@code plan.json}</td><td>{@code plan.pb}</td></tr>
+ *   <tr><td>Manifest</td><td>{@code .manifest}</td><td>{@code .manifest.pb}</td></tr>
+ * </table>
+ *
+ * <p>Data files ({@code data.json}) are JSON in both forms; only the plan and manifest change.
+ *
+ * <p>OPA rejects bundles whose plan and manifest formats disagree (e.g. {@code plan.pb} paired with
+ * a JSON {@code .manifest}). {@link BundleLoader} implementations call {@link #validate} to enforce
+ * the same rule and to reject bundles that ambiguously contain both formats of the same artifact.
+ */
+public final class BundleFormat {
+
+  /** Filename of a JSON-format IR plan. */
+  public static final String PLAN_JSON = "plan.json";
+
+  /** Filename of a protobuf-format IR plan. */
+  public static final String PLAN_PROTO = "plan.pb";
+
+  /** Filename of a JSON-format bundle manifest. */
+  public static final String MANIFEST_JSON = ".manifest";
+
+  /** Filename of a protobuf-format bundle manifest. */
+  public static final String MANIFEST_PROTO = ".manifest.pb";
+
+  private BundleFormat() {}
+
+  /**
+   * Reject bundles that mix the two wire formats.
+   *
+   * <p>Fails if a bundle contains both formats of the same artifact (e.g. both {@code plan.json} and
+   * {@code plan.pb}), or if a plan and a manifest are present in disagreeing formats (e.g. a proto
+   * plan with a JSON manifest), matching OPA's own auto-detection semantics. A bundle carrying only
+   * a plan or only a manifest is always accepted.
+   *
+   * @param hasPlanJson whether a {@code plan.json} is present
+   * @param hasPlanProto whether a {@code plan.pb} is present
+   * @param hasManifestJson whether a {@code .manifest} is present
+   * @param hasManifestProto whether a {@code .manifest.pb} is present
+   * @throws IllegalArgumentException if the bundle mixes formats
+   */
+  public static void validate(
+      boolean hasPlanJson,
+      boolean hasPlanProto,
+      boolean hasManifestJson,
+      boolean hasManifestProto) {
+    if (hasPlanJson && hasPlanProto) {
+      throw new IllegalArgumentException(
+          "Bundle contains both " + PLAN_JSON + " and " + PLAN_PROTO + "; plan format is ambiguous");
+    }
+    if (hasManifestJson && hasManifestProto) {
+      throw new IllegalArgumentException(
+          "Bundle contains both "
+              + MANIFEST_JSON
+              + " and "
+              + MANIFEST_PROTO
+              + "; manifest format is ambiguous");
+    }
+    if (hasPlanProto && hasManifestJson) {
+      throw new IllegalArgumentException(
+          "Bundle mixes plan and manifest formats: proto plan ("
+              + PLAN_PROTO
+              + ") with JSON manifest ("
+              + MANIFEST_JSON
+              + ")");
+    }
+    if (hasPlanJson && hasManifestProto) {
+      throw new IllegalArgumentException(
+          "Bundle mixes plan and manifest formats: JSON plan ("
+              + PLAN_JSON
+              + ") with proto manifest ("
+              + MANIFEST_PROTO
+              + ")");
+    }
+  }
+}

opa-evaluator/src/main/java/io/github/open_policy_agent/opa/bundle/FileSystemBundleLoader.java

@@ -31,19 +31,14 @@ public Bundle load(Store store) {
     BundleAssembler assembler = new BundleAssembler();
 
     try {
-      Path planPath = directory.resolve("plan.json");
-      if (Files.isRegularFile(planPath)) {
-        try (InputStream in = Files.newInputStream(planPath)) {
-          assembler.loadPlan(in);
-        }
-      }
-
-      Path manifestPath = directory.resolve(".manifest");
-      if (Files.isRegularFile(manifestPath)) {
-        try (InputStream in = Files.newInputStream(manifestPath)) {
-          assembler.loadManifest(in);
-        }
-      }
+      // OPA emits plan/manifest in either JSON (plan.json/.manifest) or proto
+      // (plan.pb/.manifest.pb). BundleAssembler detects the format, rejects mixed-format bundles,
+      // and loads each artifact; we only supply a stream for the files that exist.
+      assembler.loadPlanAndManifest(
+          fileSource(directory.resolve(BundleFormat.PLAN_JSON)),
+          fileSource(directory.resolve(BundleFormat.PLAN_PROTO)),
+          fileSource(directory.resolve(BundleFormat.MANIFEST_JSON)),
+          fileSource(directory.resolve(BundleFormat.MANIFEST_PROTO)));
 
       try (Stream<Path> walk = Files.walk(directory)) {
         walk.filter(Files::isRegularFile)
@@ -81,4 +76,9 @@ public Bundle load(Store store) {
           "Error reading bundle directory: " + e.getMessage(), e);
     }
   }
+
+  /** An {@link InputStreamSource} that opens {@code path}, or {@code null} if it is not a file. */
+  private static InputStreamSource fileSource(Path path) {
+    return Files.isRegularFile(path) ? () -> Files.newInputStream(path) : null;
+  }
 }

opa-evaluator/src/main/java/io/github/open_policy_agent/opa/bundle/InputStreamSource.java

@@ -0,0 +1,23 @@
+package io.github.open_policy_agent.opa.bundle;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Supplies a fresh {@link InputStream} for a single bundle artifact.
+ *
+ * <p>Lets {@link BundleLoader} implementations hand {@link BundleAssembler} a lazily-opened stream
+ * (e.g. a file opened on demand, or an in-memory buffer) without exposing how the bytes are
+ * sourced. Unlike {@link java.util.function.Supplier}, {@link #open()} may throw {@link IOException}.
+ */
+@FunctionalInterface
+public interface InputStreamSource {
+
+  /**
+   * Opens a new stream over the artifact's bytes. The caller closes it.
+   *
+   * @return the opened stream
+   * @throws IOException if the stream cannot be opened
+   */
+  InputStream open() throws IOException;
+}