[GLUTEN-12597][CORE] Migrate Substrait extension referencing from URI to URN

cpp/velox/substrait/SubstraitExtensionCollector.cc

@@ -39,15 +39,18 @@ bool SubstraitExtensionCollector::BiDirectionHashMap<T>::putIfAbsent(const int&
 }
 
 void SubstraitExtensionCollector::addExtensionsToPlan(::substrait::Plan* plan) const {
-  using SimpleExtensionURI = ::substrait::extensions::SimpleExtensionURI;
-  // Currently we don't introduce any substrait extension YAML files, so always
-  // only have one URI.
-  SimpleExtensionURI* extensionUri = plan->add_extension_uris();
-  extensionUri->set_extension_uri_anchor(1);
+  using SimpleExtensionURN = ::substrait::extensions::SimpleExtensionURN;
+  // Currently we don't map functions to their individual Substrait extension
+  // YAML files, so we emit a single catch-all URN and resolve functions by
+  // name. The URN follows the required extension:<OWNER>:<ID> format; consuming
+  // the upstream io.substrait function extensions is left to a follow-up.
+  SimpleExtensionURN* extensionUrn = plan->add_extension_urns();
+  extensionUrn->set_extension_urn_anchor(1);
+  extensionUrn->set_urn("extension:org.apache.gluten:functions");
 
   for (const auto& [referenceNum, functionId] : extensionFunctions_->forwardMap()) {
     auto extensionFunction = plan->add_extensions()->mutable_extension_function();
-    extensionFunction->set_extension_uri_reference(extensionUri->extension_uri_anchor());
+    extensionFunction->set_extension_urn_reference(extensionUrn->extension_urn_anchor());
     extensionFunction->set_function_anchor(referenceNum);
     extensionFunction->set_name(functionId.signature);
   }

cpp/velox/substrait/VeloxToSubstraitPlan.cc

@@ -103,7 +103,7 @@ ::substrait::Plan& VeloxToSubstraitPlanConvertor::toSubstrait(
   // Add unknown type in extension.
   auto unknownType = substraitPlan->add_extensions()->mutable_extension_type();
 
-  unknownType->set_extension_uri_reference(0);
+  unknownType->set_extension_urn_reference(0);
   unknownType->set_type_anchor(0);
   unknownType->set_name("UNKNOWN");
 

cpp/velox/tests/data/q1_first_stage.json

@@ -1,70 +1,70 @@
 {
-    "extension_uris": [
+    "extension_urns": [
         {
-            "extension_uri_anchor": 1,
-            "uri": "/functions_datetime.yaml"
+            "extension_urn_anchor": 1,
+            "urn": "extension:io.substrait:functions_datetime"
         }
     ],
     "extensions": [
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 1,
                 "name": "lte:fp64_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 6,
                 "name": "sum:opt_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 3,
                 "name": "subtract:opt_fp64_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 9,
                 "name": "is_not_null:fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 2,
                 "name": "and:bool_bool"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 5,
                 "name": "add:opt_fp64_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 7,
                 "name": "count:opt_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 4,
                 "name": "multiply:opt_fp64_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 8,
                 "name": "count:opt_i32"
             }

cpp/velox/tests/data/q6_first_stage.json

@@ -1,56 +1,56 @@
 {
-    "extension_uris": [
+    "extension_urns": [
         {
-            "extension_uri_anchor": 1,
-            "uri": "/functions_boolean.yaml"
+            "extension_urn_anchor": 1,
+            "urn": "extension:io.substrait:functions_boolean"
         }
     ],
     "extensions": [
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 4,
                 "name": "lte:fp64_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 6,
                 "name": "sum:opt_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 3,
                 "name": "lt:fp64_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 7,
                 "name": "is_not_null:fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 1,
                 "name": "and:bool_bool"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 2,
                 "name": "gte:fp64_fp64"
             }
         },
         {
             "extension_function": {
-                "extension_uri_reference": 1,
+                "extension_urn_reference": 1,
                 "function_anchor": 5,
                 "name": "multiply:opt_fp64_fp64"
             }

cpp/velox/tests/data/substrait_virtualTable.json

@@ -1,5 +1,5 @@
 {
- "extension_uris": [],
+ "extension_urns": [],
  "extensions": [],
  "relations": [
   {

gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto

@@ -9,7 +9,7 @@ import "substrait/plan.proto";
 import "substrait/type.proto";
 
 option csharp_namespace = "Substrait.Protobuf";
-option go_package = "github.com/substrait-io/substrait-go/proto";
+option go_package = "github.com/substrait-io/substrait-protobuf/go/substraitpb";
 option java_multiple_files = true;
 option java_package = "io.substrait.proto";
 
@@ -25,12 +25,15 @@ message ExpressionReference {
 // Describe a set of operations to complete.
 // For compactness sake, identifiers are normalized at the plan level.
 message ExtendedExpression {
+  reserved 1;
+
   // Substrait version of the expression. Optional up to 0.17.0, required for later
   // versions.
   Version version = 7;
 
-  // a list of yaml specifications this expression may depend on
-  repeated substrait.extensions.SimpleExtensionURI extension_uris = 1;
+  // a list of extension specifications this expression may depend on,
+  // referenced by Extension URN
+  repeated substrait.extensions.SimpleExtensionURN extension_urns = 8;
 
   // a list of extensions this expression may depend on
   repeated substrait.extensions.SimpleExtensionDeclaration extensions = 2;

gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto

@@ -6,21 +6,22 @@ package substrait.extensions;
 import "google/protobuf/any.proto";
 
 option csharp_namespace = "Substrait.Protobuf";
-option go_package = "github.com/substrait-io/substrait-go/proto/extensions";
+option go_package = "github.com/substrait-io/substrait-protobuf/go/substraitpb/extensions";
 option java_multiple_files = true;
 option java_package = "io.substrait.proto";
 
-message SimpleExtensionURI {
+message SimpleExtensionURN {
   // A surrogate key used in the context of a single plan used to reference the
-  // URI associated with an extension.
-  uint32 extension_uri_anchor = 1;
+  // URN associated with an extension.
+  // 0 is a valid anchor/reference, but prefer non-zero values for ergonomics.
+  uint32 extension_urn_anchor = 1;
 
-  // The URI where this extension YAML can be retrieved. This is the "namespace"
-  // of this extension.
-  string uri = 2;
+  // The extension URN that uniquely identifies this extension. This must follow the
+  // format extension:<OWNER>:<ID> and serves as the "namespace" of this extension.
+  string urn = 2;
 }
 
-// Describes a mapping between a specific extension entity and the uri where
+// Describes a mapping between a specific extension entity and the URN where
 // that extension can be found.
 message SimpleExtensionDeclaration {
   oneof mapping_type {
@@ -31,40 +32,47 @@ message SimpleExtensionDeclaration {
 
   // Describes a Type
   message ExtensionType {
-    // references the extension_uri_anchor defined for a specific extension URI.
-    uint32 extension_uri_reference = 1;
+    reserved 1;
+
+    // references the extension_urn_anchor defined for a specific extension URN.
+    uint32 extension_urn_reference = 4;
 
     // A surrogate key used in the context of a single plan to reference a
-    // specific extension type
+    // specific extension type.
+    // 0 is a valid anchor/reference, but prefer non-zero values for ergonomics.
     uint32 type_anchor = 2;
 
     // the name of the type in the defined extension YAML.
     string name = 3;
   }
 
   message ExtensionTypeVariation {
-    // references the extension_uri_anchor defined for a specific extension URI.
-    uint32 extension_uri_reference = 1;
+    reserved 1;
+
+    // references the extension_urn_anchor defined for a specific extension URN.
+    uint32 extension_urn_reference = 4;
 
     // A surrogate key used in the context of a single plan to reference a
-    // specific type variation
+    // specific type variation.
+    // Use non-zero values; 0 is reserved for the system-preferred variation.
     uint32 type_variation_anchor = 2;
 
     // the name of the type in the defined extension YAML.
     string name = 3;
   }
 
   message ExtensionFunction {
-    // references the extension_uri_anchor defined for a specific extension URI.
-    uint32 extension_uri_reference = 1;
+    reserved 1;
+
+    // references the extension_urn_anchor defined for a specific extension URN.
+    uint32 extension_urn_reference = 4;
 
     // A surrogate key used in the context of a single plan to reference a
-    // specific function
+    // specific function.
+    // 0 is a valid anchor/reference, but prefer non-zero values for ergonomics.
     uint32 function_anchor = 2;
 
-    // A simple name if there is only one impl for the function within the YAML.
-    // A compound name, referencing that includes type short names if there is
-    // more than one impl per name in the YAML.
+    // A function signature
     string name = 3;
   }
 }

gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto

@@ -7,7 +7,7 @@ import "substrait/algebra.proto";
 import "substrait/extensions/extensions.proto";
 
 option csharp_namespace = "Substrait.Protobuf";
-option go_package = "github.com/substrait-io/substrait-go/proto";
+option go_package = "github.com/substrait-io/substrait-protobuf/go/substraitpb";
 option java_multiple_files = true;
 option java_package = "io.substrait.proto";
 
@@ -24,12 +24,14 @@ message PlanRel {
 // Describe a set of operations to complete.
 // For compactness sake, identifiers are normalized at the plan level.
 message Plan {
+  reserved 1;
+
   // Substrait version of the plan. Optional up to 0.17.0, required for later
   // versions.
   Version version = 6;
 
-  // a list of yaml specifications this plan may depend on
-  repeated substrait.extensions.SimpleExtensionURI extension_uris = 1;
+  // a list of extension URNs this plan may depend on
+  repeated substrait.extensions.SimpleExtensionURN extension_urns = 8;
 
   // a list of extensions this plan may depend on
   repeated substrait.extensions.SimpleExtensionDeclaration extensions = 2;