diff --git a/CHANGELOG.md b/CHANGELOG.md index dc85470..63d4472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## Added + +- Install and manage the OpenLineageConnection custom resource definition ([#435]). + +[#435]: https://github.com/stackabletech/commons-operator/pull/435 + ## [26.7.0] - 2026-07-21 ## [26.7.0-rc1] - 2026-07-16 diff --git a/Cargo.lock b/Cargo.lock index 169b240..9aa0bcf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1597,7 +1597,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "darling", "regex", @@ -2939,9 +2939,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spki" @@ -2962,7 +2962,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "const-oid", "ecdsa", @@ -3005,8 +3005,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.113.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +version = "0.113.4" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "base64", "clap", @@ -3051,7 +3051,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "darling", "proc-macro2", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "jiff", "k8s-openapi", @@ -3079,7 +3079,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "axum", "clap", @@ -3103,7 +3103,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "kube", "schemars", @@ -3117,7 +3117,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "convert_case", "convert_case_extras", @@ -3135,7 +3135,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 1dd6915..14142ee 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5095,8 +5095,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2"; }; libName = "k8s_version"; authors = [ @@ -9684,9 +9684,9 @@ rec { }; "spin" = rec { crateName = "spin"; - version = "0.9.8"; + version = "0.9.9"; edition = "2015"; - sha256 = "0rvam5r0p3a6qhc18scqpvpgb3ckzyqxpgdfyjnghh8ja7byi039"; + sha256 = "03psal0vh1xdxp7agphw09p7kf50v3bj1zshijq1s5bkdd7jcqrp"; authors = [ "Mathijs van de Nes " "John Ericson " @@ -9759,8 +9759,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2"; }; libName = "stackable_certs"; authors = [ @@ -9940,13 +9940,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.113.3"; + version = "0.113.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2"; }; libName = "stackable_operator"; authors = [ @@ -10144,8 +10144,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10179,8 +10179,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2"; }; libName = "stackable_shared"; authors = [ @@ -10260,8 +10260,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2"; }; libName = "stackable_telemetry"; authors = [ @@ -10370,8 +10370,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2"; }; libName = "stackable_versioned"; authors = [ @@ -10420,8 +10420,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10488,8 +10488,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2"; }; libName = "stackable_webhook"; authors = [ diff --git a/Cargo.toml b/Cargo.toml index 6e4f386..2ba5c9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,10 @@ edition = "2024" repository = "https://github.com/stackabletech/commons-operator" [workspace.dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.113.3", features = ["crds", "webhook"] } +# TODO: Temporary branch reference to pull in the OpenLineageConnection CRD from the +# unmerged draft PR https://github.com/stackabletech/operator-rs/pull/1250. Revert to a +# version tag once the PR is merged and a new stackable-operator release is tagged. +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/openlineage-crd", features = ["crds", "webhook"] } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } diff --git a/crate-hashes.json b/crate-hashes.json index 77adf52..3c952a7 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#k8s-version@0.1.3": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-certs@0.4.1": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-operator-derive@0.3.1": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-operator@0.113.3": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-shared@0.1.2": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-telemetry@0.6.5": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-versioned-macros@0.11.1": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-versioned@0.11.1": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-webhook@0.9.2": "1ab5qmawljwbll6b44gw3md57s9kjiy5p16akz6rv52j6fdxjpgr", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#k8s-version@0.1.3": "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-certs@0.4.1": "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-operator-derive@0.3.1": "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-operator@0.113.4": "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-shared@0.1.2": "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-telemetry@0.6.5": "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-versioned-macros@0.11.1": "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-versioned@0.11.1": "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-webhook@0.9.2": "1irgyw99cp8vyw14l414aargkdvh5s43vxxldxlf3jr4q7fxv8h2", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/extra/crds.yaml b/extra/crds.yaml index aae13d5..1af5994 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -709,3 +709,102 @@ spec: served: true storage: true subresources: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: openlineageconnections.lineage.stackable.tech +spec: + group: lineage.stackable.tech + names: + categories: [] + kind: OpenLineageConnection + plural: openlineageconnections + shortNames: [] + singular: openlineageconnection + scope: Namespaced + versions: + - additionalPrinterColumns: [] + name: v1alpha1 + schema: + openAPIV3Schema: + description: A reusable definition of a connection to an OpenLineage backend. + properties: + spec: + description: |- + OpenLineage connection definition as a resource. + Learn more about [OpenLineage](https://openlineage.io/). + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the API key used to authenticate against the OpenLineage + backend. The API key must be stored under the key `apiKey`. The Secret must be located in + the same namespace as the workload using this connection. If not specified, no + authentication is used. + nullable: true + type: string + host: + description: 'Host of the OpenLineage backend without any protocol or port. For example: `marquez`.' + type: string + port: + description: 'Port the OpenLineage backend listens on. For example: `5000`.' + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + - port + type: object + required: + - spec + title: OpenLineageConnection + type: object + served: true + storage: true + subresources: {} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index a9fb028..c3248f9 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -11,6 +11,7 @@ use stackable_operator::{ cli::{Command, RunArguments}, crd::{ authentication::core::{AuthenticationClass, AuthenticationClassVersion}, + openlineage::{OpenLineageConnection, OpenLineageConnectionVersion}, s3::{S3Bucket, S3BucketVersion, S3Connection, S3ConnectionVersion}, }, eos::EndOfSupportChecker, @@ -62,6 +63,8 @@ async fn main() -> anyhow::Result<()> { .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; S3Bucket::merged_crd(S3BucketVersion::V1Alpha1)? .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; + OpenLineageConnection::merged_crd(OpenLineageConnectionVersion::V1Alpha1)? + .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; } Command::Run(CommonsOperatorRunArguments { common: diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index 7722ab5..559777e 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -1,6 +1,7 @@ use stackable_operator::{ crd::{ authentication::core::{AuthenticationClass, AuthenticationClassVersion}, + openlineage::{OpenLineageConnection, OpenLineageConnectionVersion}, s3::{S3Bucket, S3BucketVersion, S3Connection, S3ConnectionVersion}, }, kube::Client, @@ -23,6 +24,10 @@ pub fn create_webhook(disable_crd_maintenance: bool, client: Client) -> Box _, ), + ( + OpenLineageConnection::merged_crd(OpenLineageConnectionVersion::V1Alpha1).unwrap(), + OpenLineageConnection::try_convert as fn(_) -> _, + ), ]; let conversion_webhook_options = ConversionWebhookOptions {