diff --git a/cpp/velox/substrait/SubstraitExtensionCollector.cc b/cpp/velox/substrait/SubstraitExtensionCollector.cc index 472ef04e3fc..51695437c47 100644 --- a/cpp/velox/substrait/SubstraitExtensionCollector.cc +++ b/cpp/velox/substrait/SubstraitExtensionCollector.cc @@ -39,15 +39,18 @@ bool SubstraitExtensionCollector::BiDirectionHashMap::putIfAbsent(const int& } void SubstraitExtensionCollector::addExtensionsToPlan(::substrait::Plan* plan) const { - using SimpleExtensionURI = ::substrait::extensions::SimpleExtensionURI; - // Currently we don't introduce any substrait extension YAML files, so always - // only have one URI. - SimpleExtensionURI* extensionUri = plan->add_extension_uris(); - extensionUri->set_extension_uri_anchor(1); + using SimpleExtensionURN = ::substrait::extensions::SimpleExtensionURN; + // Currently we don't map functions to their individual Substrait extension + // YAML files, so we emit a single catch-all URN and resolve functions by + // name. The URN follows the required extension:: format; consuming + // the upstream io.substrait function extensions is left to a follow-up. + SimpleExtensionURN* extensionUrn = plan->add_extension_urns(); + extensionUrn->set_extension_urn_anchor(1); + extensionUrn->set_urn("extension:org.apache.gluten:functions"); for (const auto& [referenceNum, functionId] : extensionFunctions_->forwardMap()) { auto extensionFunction = plan->add_extensions()->mutable_extension_function(); - extensionFunction->set_extension_uri_reference(extensionUri->extension_uri_anchor()); + extensionFunction->set_extension_urn_reference(extensionUrn->extension_urn_anchor()); extensionFunction->set_function_anchor(referenceNum); extensionFunction->set_name(functionId.signature); } diff --git a/cpp/velox/substrait/VeloxToSubstraitPlan.cc b/cpp/velox/substrait/VeloxToSubstraitPlan.cc index bc9132a67b8..39d9d2e1528 100644 --- a/cpp/velox/substrait/VeloxToSubstraitPlan.cc +++ b/cpp/velox/substrait/VeloxToSubstraitPlan.cc @@ -103,7 +103,7 @@ ::substrait::Plan& VeloxToSubstraitPlanConvertor::toSubstrait( // Add unknown type in extension. auto unknownType = substraitPlan->add_extensions()->mutable_extension_type(); - unknownType->set_extension_uri_reference(0); + unknownType->set_extension_urn_reference(0); unknownType->set_type_anchor(0); unknownType->set_name("UNKNOWN"); diff --git a/cpp/velox/tests/data/q1_first_stage.json b/cpp/velox/tests/data/q1_first_stage.json index 1b9ba06231d..1413ffbd25e 100644 --- a/cpp/velox/tests/data/q1_first_stage.json +++ b/cpp/velox/tests/data/q1_first_stage.json @@ -1,70 +1,70 @@ { - "extension_uris": [ + "extension_urns": [ { - "extension_uri_anchor": 1, - "uri": "/functions_datetime.yaml" + "extension_urn_anchor": 1, + "urn": "extension:io.substrait:functions_datetime" } ], "extensions": [ { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 1, "name": "lte:fp64_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 6, "name": "sum:opt_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 3, "name": "subtract:opt_fp64_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 9, "name": "is_not_null:fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 2, "name": "and:bool_bool" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 5, "name": "add:opt_fp64_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 7, "name": "count:opt_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 4, "name": "multiply:opt_fp64_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 8, "name": "count:opt_i32" } diff --git a/cpp/velox/tests/data/q6_first_stage.json b/cpp/velox/tests/data/q6_first_stage.json index 031793a2b18..7c4d12f1c9c 100644 --- a/cpp/velox/tests/data/q6_first_stage.json +++ b/cpp/velox/tests/data/q6_first_stage.json @@ -1,56 +1,56 @@ { - "extension_uris": [ + "extension_urns": [ { - "extension_uri_anchor": 1, - "uri": "/functions_boolean.yaml" + "extension_urn_anchor": 1, + "urn": "extension:io.substrait:functions_boolean" } ], "extensions": [ { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 4, "name": "lte:fp64_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 6, "name": "sum:opt_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 3, "name": "lt:fp64_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 7, "name": "is_not_null:fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 1, "name": "and:bool_bool" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 2, "name": "gte:fp64_fp64" } }, { "extension_function": { - "extension_uri_reference": 1, + "extension_urn_reference": 1, "function_anchor": 5, "name": "multiply:opt_fp64_fp64" } diff --git a/cpp/velox/tests/data/substrait_virtualTable.json b/cpp/velox/tests/data/substrait_virtualTable.json index 06536fadd7e..5888df257ec 100644 --- a/cpp/velox/tests/data/substrait_virtualTable.json +++ b/cpp/velox/tests/data/substrait_virtualTable.json @@ -1,5 +1,5 @@ { - "extension_uris": [], + "extension_urns": [], "extensions": [], "relations": [ { diff --git a/gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto b/gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto index 5d115205593..60b7d5208df 100755 --- a/gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto +++ b/gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto @@ -9,7 +9,7 @@ import "substrait/plan.proto"; import "substrait/type.proto"; option csharp_namespace = "Substrait.Protobuf"; -option go_package = "github.com/substrait-io/substrait-go/proto"; +option go_package = "github.com/substrait-io/substrait-protobuf/go/substraitpb"; option java_multiple_files = true; option java_package = "io.substrait.proto"; @@ -25,12 +25,15 @@ message ExpressionReference { // Describe a set of operations to complete. // For compactness sake, identifiers are normalized at the plan level. message ExtendedExpression { + reserved 1; + // Substrait version of the expression. Optional up to 0.17.0, required for later // versions. Version version = 7; - // a list of yaml specifications this expression may depend on - repeated substrait.extensions.SimpleExtensionURI extension_uris = 1; + // a list of extension specifications this expression may depend on, + // referenced by Extension URN + repeated substrait.extensions.SimpleExtensionURN extension_urns = 8; // a list of extensions this expression may depend on repeated substrait.extensions.SimpleExtensionDeclaration extensions = 2; diff --git a/gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto b/gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto index d3af5f3d9ed..1e24ace044d 100644 --- a/gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto +++ b/gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto @@ -6,21 +6,22 @@ package substrait.extensions; import "google/protobuf/any.proto"; option csharp_namespace = "Substrait.Protobuf"; -option go_package = "github.com/substrait-io/substrait-go/proto/extensions"; +option go_package = "github.com/substrait-io/substrait-protobuf/go/substraitpb/extensions"; option java_multiple_files = true; option java_package = "io.substrait.proto"; -message SimpleExtensionURI { +message SimpleExtensionURN { // A surrogate key used in the context of a single plan used to reference the - // URI associated with an extension. - uint32 extension_uri_anchor = 1; + // URN associated with an extension. + // 0 is a valid anchor/reference, but prefer non-zero values for ergonomics. + uint32 extension_urn_anchor = 1; - // The URI where this extension YAML can be retrieved. This is the "namespace" - // of this extension. - string uri = 2; + // The extension URN that uniquely identifies this extension. This must follow the + // format extension:: and serves as the "namespace" of this extension. + string urn = 2; } -// Describes a mapping between a specific extension entity and the uri where +// Describes a mapping between a specific extension entity and the URN where // that extension can be found. message SimpleExtensionDeclaration { oneof mapping_type { @@ -31,11 +32,14 @@ message SimpleExtensionDeclaration { // Describes a Type message ExtensionType { - // references the extension_uri_anchor defined for a specific extension URI. - uint32 extension_uri_reference = 1; + reserved 1; + + // references the extension_urn_anchor defined for a specific extension URN. + uint32 extension_urn_reference = 4; // A surrogate key used in the context of a single plan to reference a - // specific extension type + // specific extension type. + // 0 is a valid anchor/reference, but prefer non-zero values for ergonomics. uint32 type_anchor = 2; // the name of the type in the defined extension YAML. @@ -43,11 +47,14 @@ message SimpleExtensionDeclaration { } message ExtensionTypeVariation { - // references the extension_uri_anchor defined for a specific extension URI. - uint32 extension_uri_reference = 1; + reserved 1; + + // references the extension_urn_anchor defined for a specific extension URN. + uint32 extension_urn_reference = 4; // A surrogate key used in the context of a single plan to reference a - // specific type variation + // specific type variation. + // Use non-zero values; 0 is reserved for the system-preferred variation. uint32 type_variation_anchor = 2; // the name of the type in the defined extension YAML. @@ -55,16 +62,17 @@ message SimpleExtensionDeclaration { } message ExtensionFunction { - // references the extension_uri_anchor defined for a specific extension URI. - uint32 extension_uri_reference = 1; + reserved 1; + + // references the extension_urn_anchor defined for a specific extension URN. + uint32 extension_urn_reference = 4; // A surrogate key used in the context of a single plan to reference a - // specific function + // specific function. + // 0 is a valid anchor/reference, but prefer non-zero values for ergonomics. uint32 function_anchor = 2; - // A simple name if there is only one impl for the function within the YAML. - // A compound name, referencing that includes type short names if there is - // more than one impl per name in the YAML. + // A function signature string name = 3; } } diff --git a/gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto b/gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto index e5657fb8f1e..2fddc1ddec6 100644 --- a/gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto +++ b/gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto @@ -7,7 +7,7 @@ import "substrait/algebra.proto"; import "substrait/extensions/extensions.proto"; option csharp_namespace = "Substrait.Protobuf"; -option go_package = "github.com/substrait-io/substrait-go/proto"; +option go_package = "github.com/substrait-io/substrait-protobuf/go/substraitpb"; option java_multiple_files = true; option java_package = "io.substrait.proto"; @@ -24,12 +24,14 @@ message PlanRel { // Describe a set of operations to complete. // For compactness sake, identifiers are normalized at the plan level. message Plan { + reserved 1; + // Substrait version of the plan. Optional up to 0.17.0, required for later // versions. Version version = 6; - // a list of yaml specifications this plan may depend on - repeated substrait.extensions.SimpleExtensionURI extension_uris = 1; + // a list of extension URNs this plan may depend on + repeated substrait.extensions.SimpleExtensionURN extension_urns = 8; // a list of extensions this plan may depend on repeated substrait.extensions.SimpleExtensionDeclaration extensions = 2;