From 4daacd5e1187f43ee760921723893427a21a9033 Mon Sep 17 00:00:00 2001 From: Rishabh Raj Date: Thu, 22 Jan 2026 04:32:20 -0500 Subject: [PATCH] Mirror internal repository with cleaned references --- .gitignore | 2 + Dockerfile | 19 +- Gopkg.toml | 13 - Makefile | 64 +- README.md | 18 +- core/semver/semver.go | 2 +- dell-csi-helm-installer/csi-install.sh | 23 + .../install-csi-vxflexos.sh | 9 +- .../verify-csi-vxflexos.sh | 8 + docker.mk | 34 - go.mod | 131 ++- go.sum | 320 +++--- helper.mk | 17 + images.mk | 22 + main.go | 12 +- mkdocs.yml | 7 + overrides.mk | 61 +- provider/provider.go | 5 +- samples/secret.yaml | 12 +- .../secret/karavi-authorization-config.json | 1 - samples/secret_OIDC.yaml | 118 +++ .../storageclass/storageclass-nvmetcp.yaml | 80 ++ service/controller.go | 978 +++++++++-------- service/controller_extension.go | 148 +++ service/controller_test.go | 555 +++++++++- service/csi_extension_server.go | 195 +++- service/envvars.go | 15 + service/ephemeral.go | 125 ++- service/features/array-config/config.2 | 6 +- .../controller_publish_unpublish.feature | 214 +++- service/features/create_nvme_host.json | 3 + service/features/csi_extension.feature | 29 +- service/features/ephemeral.feature | 43 +- service/features/get_all_sdt.json | 159 +++ service/features/get_sdc_instances.json | 100 +- service/features/get_system_statistics.json | 1 + service/features/get_volume_metrics.json | 320 ++++++ .../features/node_publish_unpublish.feature | 60 ++ service/features/node_stage_unstage.feature | 115 ++ service/features/replication.feature | 8 + service/features/service.feature | 418 ++++++-- service/identity.go | 26 +- service/mount.go | 303 ++++-- service/node.go | 429 ++++++-- service/node_connectivity_checker.go | 215 ++++ service/node_connectivity_checker_test.go | 157 +++ service/nvme_utils.go | 157 +++ service/preinit.go | 12 +- service/publisher.go | 223 ++++ service/replication.go | 60 +- service/service.go | 810 +++++++++++--- service/service_test.go | 45 +- service/service_unit_test.go | 249 ++++- service/stager.go | 335 ++++++ service/step_defs_test.go | 989 ++++++++++++++---- service/step_handlers_test.go | 295 ++++-- test/e2e-fsgroup/fs_scaleup_scaledown.go | 8 +- test/e2e-fsgroup/go.mod | 226 ++-- test/e2e-fsgroup/go.sum | 549 +++++----- test/e2e/e2e.go | 2 +- test/integration/integration_test.go | 5 +- test/integration/step_defs_test.go | 5 +- 62 files changed, 7523 insertions(+), 2047 deletions(-) delete mode 100644 Gopkg.toml delete mode 100644 docker.mk create mode 100644 helper.mk create mode 100644 images.mk create mode 100644 mkdocs.yml delete mode 100644 samples/secret/karavi-authorization-config.json create mode 100644 samples/secret_OIDC.yaml create mode 100644 samples/storageclass/storageclass-nvmetcp.yaml create mode 100644 service/controller_extension.go create mode 100644 service/features/create_nvme_host.json create mode 100644 service/features/get_all_sdt.json create mode 100644 service/features/get_volume_metrics.json create mode 100644 service/features/node_stage_unstage.feature create mode 100644 service/node_connectivity_checker.go create mode 100644 service/node_connectivity_checker_test.go create mode 100644 service/nvme_utils.go create mode 100644 service/publisher.go create mode 100644 service/stager.go diff --git a/.gitignore b/.gitignore index aaad1be7..f0a869a3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ gofsutil/ .idea .vscode *.log +csm-common.mk go-code-tester +vendor/ diff --git a/Dockerfile b/Dockerfile index 423afeec..3ebe8f7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Copyright © 2019-2025 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2019-2026 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,22 +11,23 @@ # limitations under the License # some arguments that must be supplied -ARG GOPROXY ARG GOIMAGE ARG BASEIMAGE -ARG DIGEST +ARG VERSION="2.16.0" # Stage to build the driver -FROM $GOIMAGE as builder -ARG GOPROXY +FROM $GOIMAGE AS builder +ARG VERSION + RUN mkdir -p /go/src COPY ./ /go/src/ + WORKDIR /go/src/ -RUN CGO_ENABLED=0 \ - make build +RUN make build IMAGE_VERSION=$VERSION # Stage to build the driver image FROM $BASEIMAGE AS final +ARG VERSION ENTRYPOINT ["/csi-vxflexos.sh"] # copy in the driver COPY --from=builder /go/src/csi-vxflexos / @@ -37,7 +38,7 @@ LABEL vendor="Dell Technologies" \ name="csi-powerflex" \ summary="CSI Driver for Dell EMC PowerFlex" \ description="CSI Driver for provisioning persistent storage from Dell EMC PowerFlex" \ - release="1.15.0" \ - version="2.15.0" \ + release="1.16.0" \ + version=$VERSION \ license="Apache-2.0" COPY ./licenses /licenses diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index 2d261bf0..00000000 --- a/Gopkg.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# Refer to https://github.com/toml-lang/toml for detailed TOML docs. - -[prune] - non-go = true - go-tests = true - unused-packages = true - -[[constraint]] - name = "github.com/rexray/gocsi" - version = "0.4.0" diff --git a/Makefile b/Makefile index 4070ca46..109c57a4 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,31 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +include images.mk + # default target all: help -# include an overrides file, which sets up default values and allows user overrides -include overrides.mk +# This will be overridden during image build. +IMAGE_VERSION ?= 0.0.0 +LDFLAGS = "-X main.ManifestSemver=$(IMAGE_VERSION)" # Help target, prints usefule information help: @echo @echo "The following targets are commonly used:" @echo - @echo "action-help - Displays instructions on how to run a single github workflow locally" - @echo "actions - Run all workflows locally, requires https://github.com/nektos/act" @echo "build - Builds the code locally" @echo "check - Runs the suite of code checking tools: lint, format, etc" @echo "clean - Cleans the local build" - @echo "docker - Builds the code within a golang container and then creates the driver image" + @echo "images - Builds the code within a golang container and then creates the driver image" @echo "integration-test - Runs the integration tests. Requires access to an array" @echo "push - Pushes the built container to a target registry" @echo "unit-test - Runs the unit tests" + @echo "vendor - Downloads a vendor list (local copy) of repositories required to compile the repo." @echo @make -s overrides-help @@ -25,28 +33,13 @@ help: clean: rm -f core/core_generated.go rm -f semver.mk + rm -rf csm-common.mk + rm -rf vendor go clean -# Dependencies -dependencies: - go generate - go run core/semver/semver.go -f mk >semver.mk - # Build the driver locally -build: dependencies - CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build - -# Generates the docker container (but does not push) -docker: dependencies - make -f docker.mk docker - -# Generates the docker container with no cache (but does not push) -docker-no-cache: dependencies - make -f docker.mk docker-no-cache - -# Pushes container to the repository -push: docker - make -f docker.mk push +build: generate vendor + CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -ldflags $(LDFLAGS) -mod=vendor # Windows or Linux; requires no hardware unit-test: go-code-tester @@ -60,23 +53,8 @@ integration-test: check: @scripts/check.sh ./provider/ ./service/ -.PHONY: actions action-help -actions: ## Run all GitHub Action checks that run on a pull request creation - @echo "Running all GitHub Action checks for pull request events..." - @act -l | grep -v ^Stage | grep pull_request | grep -v image_security_scan | awk '{print $$2}' | while read WF; do \ - echo "Running workflow: $${WF}"; \ - act pull_request --no-cache-server --platform ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest --job "$${WF}"; \ - done - go-code-tester: - curl -o go-code-tester -L https://raw.githubusercontent.com/dell/common-github-actions/main/go-code-tester/entrypoint.sh \ - && chmod +x go-code-tester - -action-help: ## Echo instructions to run one specific workflow locally - @echo "GitHub Workflows can be run locally with the following command:" - @echo "act pull_request --no-cache-server --platform ubuntu-latest=ghcr.io/catthehacker/ubuntu:act-latest --job " - @echo "" - @echo "Where '' is a Job ID returned by the command:" - @echo "act -l" - @echo "" - @echo "NOTE: if act is not installed, it can be downloaded from https://github.com/nektos/act" + git clone --depth 1 git@github.com:CSM/actions.git temp-repo + cp temp-repo/go-code-tester/entrypoint.sh ./go-code-tester + chmod +x go-code-tester + rm -rf temp-repo diff --git a/README.md b/README.md index 1a229381..aac30316 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,20 @@ +# :lock: **Important Notice** +Starting with the release of **Container Storage Modules v1.16.0**, this repository will no longer be maintained as an open source project. Future development will continue under a closed source model. This change reflects our commitment to delivering even greater value to our customers by enabling faster innovation and more deeply integrated features with the Dell storage portfolio.
+For existing customers using Dell’s Container Storage Modules, you will continue to receive: +* **Ongoing Support & Community Engagement**
+ You will continue to receive high-quality support through Dell Support and our community channels. Your experience of engaging with the Dell community remains unchanged. +* **Streamlined Deployment & Updates**
+ Deployment and update processes will remain consistent, ensuring a smooth and familiar experience. +* **Access to Documentation & Resources**
+ All documentation and related materials will remain publicly accessible, providing transparency and technical guidance. +* **Continued Access to Current Open Source Version**
+ The current open-source version will remain available under its existing license for those who rely on it. + +Moving to a closed source model allows Dell’s development team to accelerate feature delivery and enhance integration across our Enterprise Kubernetes Storage solutions ultimately providing a more seamless and robust experience.
+We deeply appreciate the contributions of the open source community and remain committed to supporting our customers through this transition.
+ +For questions or access requests, please contact the maintainers via [Dell Support](https://www.dell.com/support/kbdoc/en-in/000188046/container-storage-interface-csi-drivers-and-container-storage-modules-csm-how-to-get-support). + # CSI Driver for Dell PowerFlex [![Go Report Card](https://goreportcard.com/badge/github.com/dell/csi-vxflexos?style=flat-square)](https://goreportcard.com/report/github.com/dell/csi-vxflexos) @@ -49,4 +66,3 @@ Also, if the `X_CSI_VXFLEXOS_SDCGUID` environment variable is not set, the drive ## Documentation For more detailed information on the driver, please refer to [Container Storage Modules documentation](https://dell.github.io/csm-docs/). - diff --git a/core/semver/semver.go b/core/semver/semver.go index e4fd8108..b59995c9 100644 --- a/core/semver/semver.go +++ b/core/semver/semver.go @@ -362,7 +362,7 @@ var OSExit = func(code int) { // GetExitError is a wrapper around exec.ExitError var GetExitError = func(err error) (e *exec.ExitError, ok bool) { e, ok = err.(*exec.ExitError) - return + return e, ok } // GetStatusError is a wrapper around syscall.WaitStatus diff --git a/dell-csi-helm-installer/csi-install.sh b/dell-csi-helm-installer/csi-install.sh index 4d0cf54c..bec52059 100755 --- a/dell-csi-helm-installer/csi-install.sh +++ b/dell-csi-helm-installer/csi-install.sh @@ -287,6 +287,28 @@ function verify_kubernetes() { fi } +# get_sdc_enabled_from_values +# reads node.sdc.enabled from the provided values.yaml (in $VALUES) +# and exports SDC_ENABLED=true/false for child scripts +function get_sdc_enabled_from_values() { + if [ -z "${VALUES}" ] || [ ! -f "${VALUES}" ]; then + # default to false when values file isn't present + export SDC_ENABLED=false + return + fi + local val + val=$(awk ' + BEGIN{in_node=0; in_sdc=0} + /^[[:space:]]*node:/ {in_node=1; next} + in_node && /^[[:space:]]*sdc:/ {in_sdc=1; next} + in_node && in_sdc && /^[[:space:]]*enabled:[[:space:]]*/ {gsub(/"/, ""); print tolower($2); exit} + ' "${VALUES}") + case "${val}" in + true|"true") export SDC_ENABLED=true;; + *) export SDC_ENABLED=false;; + esac +} + # # main # @@ -420,6 +442,7 @@ kNonGAVersion=$(run_command kubectl version | grep 'Server Version' | sed -n 's/ # validate the parameters passed in validate_params "${MODE}" +get_sdc_enabled_from_values header check_for_driver "${MODE}" diff --git a/dell-csi-helm-installer/install-csi-vxflexos.sh b/dell-csi-helm-installer/install-csi-vxflexos.sh index 2c6d737d..0aac0dc1 100644 --- a/dell-csi-helm-installer/install-csi-vxflexos.sh +++ b/dell-csi-helm-installer/install-csi-vxflexos.sh @@ -14,6 +14,9 @@ # make MDM secret key with values from each array # fail if MDM format is not valid + +SDC_ENABLED=${SDC_ENABLED:-false} + function install_mdm_secret() { log smart_step "Get SDC config and make MDM string for multi array support" SECRET=${1} @@ -56,4 +59,8 @@ function check_ip() { fi } -install_mdm_secret "${RELEASE}-config" +if [[ "${SDC_ENABLED}" == "true" ]]; then + install_mdm_secret "${RELEASE}-config" +else + log smart_step "SDC installation is disabled, skipping MDM secret creation" +fi diff --git a/dell-csi-helm-installer/verify-csi-vxflexos.sh b/dell-csi-helm-installer/verify-csi-vxflexos.sh index 57e84184..6597426a 100644 --- a/dell-csi-helm-installer/verify-csi-vxflexos.sh +++ b/dell-csi-helm-installer/verify-csi-vxflexos.sh @@ -31,6 +31,14 @@ function verify_sdc_installation() { if [ ${NODE_VERIFY} -eq 0 ]; then return fi + JSON=$(kubectl get secret "${RELEASE}-config" -n ${NS} -o go-template='{{ .data.config }}' | base64 --decode) + BLOCK_PROTOCOL=$(echo "${JSON}" | grep -v '^#' | grep blockProtocol | awk -F "\"" '{ print $(NF-1)}' | head -n1) + BLOCK_PROTOCOL=${BLOCK_PROTOCOL:-auto} + if [[ "${BLOCK_PROTOCOL}" == "NVMeTCP" ]]; then + log step "Skipping SDC verification (protocol=${BLOCK_PROTOCOL})" + log step_success + return + fi log step "Verifying the SDC installation" local SDC_MINION_NODES=$(run_command kubectl get nodes -o wide | grep -v -e master -e INTERNAL -e infra | awk ' { print $6; }') diff --git a/docker.mk b/docker.mk deleted file mode 100644 index 16cce740..00000000 --- a/docker.mk +++ /dev/null @@ -1,34 +0,0 @@ -# docker makefile, included from Makefile, will build/push images with docker or podman -# - -# Includes the following generated file to get semantic version information -include semver.mk - -ifdef NOTES - RELNOTE="-$(NOTES)" -else - RELNOTE= -endif - -ifeq ($(IMAGETAG),) -IMAGETAG="v$(MAJOR).$(MINOR).$(PATCH)$(RELNOTE)" -endif - - -docker: download-csm-common - $(eval include csm-common.mk) - @echo "Base Images is set to: $(BASEIMAGE)" - @echo "Building: $(REGISTRY)/$(IMAGENAME):$(IMAGETAG)" - $(BUILDER) build --pull $(NOCACHE) -t "$(REGISTRY)/$(IMAGENAME):$(IMAGETAG)" --target $(BUILDSTAGE) --build-arg GOPROXY --build-arg BASEIMAGE=$(CSM_BASEIMAGE) --build-arg GOIMAGE=$(DEFAULT_GOIMAGE) . - -docker-no-cache: download-csm-common - @echo "Building with --no-cache ..." - @make docker NOCACHE=--no-cache - -push: - @echo "Pushing: $(REGISTRY)/$(IMAGENAME):$(IMAGETAG)" - $(BUILDER) push "$(REGISTRY)/$(IMAGENAME):$(IMAGETAG)" - - -download-csm-common: - curl -O -L https://raw.githubusercontent.com/dell/csm/main/config/csm-common.mk diff --git a/go.mod b/go.mod index e50a7fea..146988ae 100644 --- a/go.mod +++ b/go.mod @@ -3,96 +3,121 @@ module github.com/dell/csi-vxflexos/v2 // In order to run unit tests on Windows, you need a stubbed Windows implementation // of the gofsutil package. Use the following replace statements if necessary. -go 1.25 +go 1.25.0 require ( - github.com/apparentlymart/go-cidr v1.1.0 - github.com/container-storage-interface/spec v1.6.0 - github.com/cucumber/godog v0.15.0 - github.com/dell/dell-csi-extensions/common v1.9.0 - github.com/dell/dell-csi-extensions/podmon v1.9.0 - github.com/dell/dell-csi-extensions/replication v1.12.0 + github.com/dell/csmlog v1.0.0 + github.com/dell/dell-csi-extensions/common v1.10.0 + github.com/dell/dell-csi-extensions/podmon v1.10.0 + github.com/dell/dell-csi-extensions/replication v1.13.0 github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1 - github.com/dell/gocsi v1.15.0 - github.com/dell/gofsutil v1.20.0 - github.com/dell/goscaleio v1.21.0 + github.com/dell/gobrick v1.16.0 + github.com/dell/gocsi v1.16.0 + github.com/dell/gofsutil v1.21.0 + github.com/dell/gonvme v1.13.0 + github.com/dell/goscaleio v1.22.0 + github.com/apparentlymart/go-cidr v1.1.0 + github.com/container-storage-interface/spec v1.7.0 + github.com/cucumber/godog v0.15.1 github.com/fsnotify/fsnotify v1.9.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/kubernetes-csi/csi-lib-utils v0.11.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.20.0 - github.com/stretchr/testify v1.11.0 - golang.org/x/net v0.43.0 - google.golang.org/grpc v1.75.0 - google.golang.org/protobuf v1.36.6 - k8s.io/api v0.34.0 - k8s.io/apimachinery v0.34.0 - k8s.io/client-go v0.34.0 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.10 + k8s.io/api v0.34.3 + k8s.io/apimachinery v0.34.3 + k8s.io/client-go v0.34.3 sigs.k8s.io/yaml v1.6.0 ) require ( + github.com/dell/goiscsi v1.14.0 // indirect github.com/akutz/gosync v0.1.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect github.com/cucumber/messages/go/v21 v21.0.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.22.4 // indirect + github.com/go-openapi/jsonreference v0.21.4 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect + github.com/gorilla/securecookie v1.1.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-memdb v1.3.4 // indirect + github.com/hashicorp/go-memdb v1.3.5 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.9.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/muhlemmer/gu v0.3.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/sagikazarmark/locafero v0.7.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.12.0 // indirect - github.com/spf13/cast v1.7.1 // indirect - github.com/spf13/pflag v1.0.9 // indirect - github.com/stretchr/objx v0.5.2 // indirect + github.com/sagikazarmark/locafero v0.12.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/objx v0.5.3 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/x448/float16 v0.8.4 // indirect - go.etcd.io/etcd/api/v3 v3.6.1 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.6.1 // indirect - go.etcd.io/etcd/client/v3 v3.6.1 // indirect + github.com/zitadel/logging v0.6.2 // indirect + github.com/zitadel/oidc/v3 v3.45.1 // indirect + github.com/zitadel/schema v1.3.1 // indirect + go.etcd.io/etcd/api/v3 v3.6.6 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.6 // indirect + go.etcd.io/etcd/client/v3 v3.6.6 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sys v0.36.0 // indirect - golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/time v0.9.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect ) diff --git a/go.sum b/go.sum index e559c36e..60aa4f55 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,28 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/dell/csmlog v1.0.0 h1:EzW+nMJBD0QTNP88OoaAJUOMVilS9cWkO248BTcJt/4= +github.com/dell/csmlog v1.0.0/go.mod h1:7rBzSv9xF5t233+J+9vkStjFsmyYO3L/B9tDTy3+9ZU= +github.com/dell/dell-csi-extensions/common v1.10.0 h1:WIFPWVEBUyzOTCOPAlcgQsiRRGAufyKJYbIATbNZXIY= +github.com/dell/dell-csi-extensions/common v1.10.0/go.mod h1:zRHzmPX5SQQnqQ1LEIxG4hYqLBeQOSiD8TkEhU0eWTY= +github.com/dell/dell-csi-extensions/podmon v1.10.0 h1:YeM9OmgJHE+n6aNaeEC96EuVev5x3pddggcM7Ws7pkk= +github.com/dell/dell-csi-extensions/podmon v1.10.0/go.mod h1:+g7fdyw1Zx74NBJQgi1BCtsywqk37MJd9JN86IPJJu0= +github.com/dell/dell-csi-extensions/replication v1.13.0 h1:DSpoZ3vX65a3KDxUv0OinLkY2qUAQtRX3E1c1e3fnvA= +github.com/dell/dell-csi-extensions/replication v1.13.0/go.mod h1:aJBwd55amqbY3kk8SG7NjwH7nxBscceDwc1rKesUG1g= +github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1 h1:W0UcLCZ8qyJ+NBRDfrZefN+fMs2i73ydkIsq6RjP7bM= +github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1/go.mod h1:C4Ji1GCEayZ733hcHkvM6JDcboWJjk43H7xp30a/trc= +github.com/dell/gobrick v1.16.0 h1:z/a9qXnT3hx3D4I+SJUMnIgJtcCx0j3gzmPPDUWtoYs= +github.com/dell/gobrick v1.16.0/go.mod h1:9uoH8EsNi9yAsUZj2gZFgB5kqdlyvArqx0tYC7Qg9IM= +github.com/dell/gocsi v1.16.0 h1:avhQPD11rYzT6/dPxpZfFsJV+T/T0x1GJqqbco45W8c= +github.com/dell/gocsi v1.16.0/go.mod h1:Fz5dQv/kWf5Y1EXZEzxLBQSsnW2HE/WY95R0WCDQLO4= +github.com/dell/gofsutil v1.21.0 h1:SeusAYjiO/1ogvg/TapvCyHcrM9z+OvdaMU5i9Ijn3M= +github.com/dell/gofsutil v1.21.0/go.mod h1:qBGEz1wMOtqTODuJfiBZhUHT0JjexBblu2oa+sEclNs= +github.com/dell/goiscsi v1.14.0 h1:kNDqOlpJ3cLSJh7Hfyn/Kz/FMCKHzV0s/xx4EqnelFw= +github.com/dell/goiscsi v1.14.0/go.mod h1:SCSC8dJCqTosU7SspaoLv6ICTKNEz08rt/I8nZ3+ptc= +github.com/dell/gonvme v1.13.0 h1:j8A1BzYA48gelih3xWd/J6LQ71CbC8Lbdyv0jG8uUNU= +github.com/dell/gonvme v1.13.0/go.mod h1:L5K7V4JZTf12m3k2wdwKwP+/eA6pr8DvlCsJU1QTGOQ= +github.com/dell/goscaleio v1.22.0 h1:tV2lPi7lGAfyfvQiGzLys7dF75/cM/T6Hxgef+oJ0fc= +github.com/dell/goscaleio v1.22.0/go.mod h1:mqne0LTO4TTIioGdsITliE/uBPXoCd7Yw0lHXhEdTJM= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -59,10 +81,11 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= +github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -74,16 +97,16 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= -github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= -github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/container-storage-interface/spec v1.7.0 h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw= +github.com/container-storage-interface/spec v1.7.0/go.mod h1:JYuzLqr9VVNoDJl44xp/8fmCOvWPDKzuGTwCoklhuqk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= +github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -91,8 +114,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= -github.com/cucumber/godog v0.15.0 h1:51AL8lBXF3f0cyA5CV4TnJFCTHpgiy+1x1Hb3TtZUmo= -github.com/cucumber/godog v0.15.0/go.mod h1:FX3rzIDybWABU4kuIXLZ/qtqEe1Ac5RdXmqvACJOces= +github.com/cucumber/godog v0.15.1 h1:rb/6oHDdvVZKS66hrhpjFQFHjthFSrQBCOI1LwshNTI= +github.com/cucumber/godog v0.15.1/go.mod h1:qju+SQDewOljHuq9NSM66s0xEhogx0q30flfxL4WUk8= github.com/cucumber/messages/go/v21 v21.0.1 h1:wzA0LxwjlWQYZd32VTlAVDTkW6inOFmSM+RuOwHZiMI= github.com/cucumber/messages/go/v21 v21.0.1/go.mod h1:zheH/2HS9JLVFukdrsPWoPdmUtmYQAQPLk7w5vWsk5s= github.com/cucumber/messages/go/v22 v22.0.0/go.mod h1:aZipXTKc0JnjCsXrJnuZpWhtay93k7Rn3Dee7iyPJjs= @@ -100,20 +123,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dell/dell-csi-extensions/common v1.9.0 h1:H1NXBYlJZ+XTCe4tSXo94Lvg8HD2wgt6ywql3kVrG34= -github.com/dell/dell-csi-extensions/common v1.9.0/go.mod h1:DA9lX2BX3fdshR40IaXfokDrIKo9a32QShcTlAqhf+c= -github.com/dell/dell-csi-extensions/podmon v1.9.0 h1:AYE3n6o6jB3Sh0uce65JPmir3FPxvqSW/21/bGqRhvY= -github.com/dell/dell-csi-extensions/podmon v1.9.0/go.mod h1:jz846RAruY/m25uBbZVYcr8vp7wmKakbjOuUBwpY0Ls= -github.com/dell/dell-csi-extensions/replication v1.12.0 h1:jOdaZsoGHWX9SsqgH+2v9cIJSAVLF1SnKKHtdiF5Ywc= -github.com/dell/dell-csi-extensions/replication v1.12.0/go.mod h1:nyPBfbMOpboVI/cYLOFJhv0LADGSvHwDcF4AxZau/go= -github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1 h1:NnS/P2OpwMlQ70fwls/KVVfe8z8op4b7nXArv8CDsqk= -github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1/go.mod h1:rlJGlmp1NI8gU52HYKY2cy13TbSWvt9p5VAG2RLbkQs= -github.com/dell/gocsi v1.15.0 h1:SXBtiNTb3iTHms4WRoewwdJaItOY8XaaxBjkTYy8o5Q= -github.com/dell/gocsi v1.15.0/go.mod h1:u8+NcCB2rWr79Dx63GWUo3TsAJj/RSlRoimTrp6BZiM= -github.com/dell/gofsutil v1.20.0 h1:jkQrOb4sSxEUcPTAbyLBABMBf+7vBC6g+yzxTGb0Ozw= -github.com/dell/gofsutil v1.20.0/go.mod h1:kKFZSYY0tF5lx/U6UhSAqLxKnNESd0hT4gJ4PlYXSB8= -github.com/dell/goscaleio v1.21.0 h1:VSBivlOP4VCoSiY4LMWLx2gVZbCvGwG0CVl5e6BjTtM= -github.com/dell/goscaleio v1.21.0/go.mod h1:YIENxDFggvqwu8defNMPgpdlM6rmhDTDrdLe7T9KZlM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -121,8 +130,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= -github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -143,9 +152,13 @@ github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= +github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -154,25 +167,53 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= +github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= +github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= +github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= +github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= +github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= +github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= @@ -194,6 +235,8 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -216,8 +259,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= +github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -229,6 +272,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -249,6 +294,8 @@ github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97Dwqy github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= +github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= @@ -262,8 +309,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -272,8 +319,9 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c= github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= +github.com/hashicorp/go-memdb v1.3.5 h1:b3taDMxCBCBVgyRrS1AZVHO14ubMYZB++QpNhBg+Nyo= +github.com/hashicorp/go-memdb v1.3.5/go.mod h1:8IVKKBkVe+fxFgdFOYxzQQNjz+sWCyHCdIC/+5+Vy1Y= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= @@ -299,11 +347,11 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jeremija/gosubmit v0.2.8 h1:mmSITBz9JxVtu8eqbN+zmmwX7Ij2RidQxhcwRVI4wqA= +github.com/jeremija/gosubmit v0.2.8/go.mod h1:Ui+HS073lCFREXBbdfrJzMB57OI/bdxTiLtrDHHhFPI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -337,8 +385,6 @@ github.com/kubernetes-csi/csi-lib-utils v0.11.0/go.mod h1:BmGZZB16L18+9+Lgg9YWwB github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= -github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -362,6 +408,10 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/muhlemmer/gu v0.3.1 h1:7EAqmFrW7n3hETvuAdmFmn4hS8W+z3LgKtrnow+YzNM= +github.com/muhlemmer/gu v0.3.1/go.mod h1:YHtHR+gxM+bKEIIs7Hmi9sPT3ZDUvTN/i88wQpZkrdM= +github.com/muhlemmer/httpforwarded v0.1.0 h1:x4DLrzXdliq8mprgUMR0olDvHGkou5BJsK/vWUetyzY= +github.com/muhlemmer/httpforwarded v0.1.0/go.mod h1:yo9czKedo2pdZhoXe+yDkGVbU0TJ0q9oQ90BVoDEtw0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -384,16 +434,15 @@ github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.38.0 h1:c/WX+w8SLAinvuKKQFh77WEucCnPk4j2OTUr7lt7BeY= -github.com/onsi/gomega v1.38.0/go.mod h1:OcXcwId0b9QsE7Y49u+BTrL4IdKOBOKnD6VQNTJEB6o= +github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= +github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -430,13 +479,15 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= -github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= +github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -449,34 +500,33 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= -github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= -github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= -github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -486,8 +536,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= -github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= @@ -503,35 +553,41 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zitadel/logging v0.6.2 h1:MW2kDDR0ieQynPZ0KIZPrh9ote2WkxfBif5QoARDQcU= +github.com/zitadel/logging v0.6.2/go.mod h1:z6VWLWUkJpnNVDSLzrPSQSQyttysKZ6bCRongw0ROK4= +github.com/zitadel/oidc/v3 v3.45.1 h1:x7J8NywTUtLR9T5uu2dufae3gJrl6VVpIfvGZy+kzJg= +github.com/zitadel/oidc/v3 v3.45.1/go.mod h1:oFArtAPTXEA4ajkIe/JfBjv7hhlD0kr///UqaO3Uzd0= +github.com/zitadel/schema v1.3.1 h1:QT3kwiRIRXXLVAs6gCK/u044WmUVh6IlbLXUsn6yRQU= +github.com/zitadel/schema v1.3.1/go.mod h1:071u7D2LQacy1HAN+YnMd/mx1qVE2isb0Mjeqg46xnU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= -go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo= -go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto= -go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk= -go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc= -go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0= -go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY= -go.etcd.io/etcd/pkg/v3 v3.6.1 h1:Qpshk3/SLra217k7FxcFGaH2niFAxFf1Dug57f0IUiw= -go.etcd.io/etcd/pkg/v3 v3.6.1/go.mod h1:nS0ahQoZZ9qXjQAtYGDt80IEHKl9YOF7mv6J0lQmBoQ= -go.etcd.io/etcd/server/v3 v3.6.1 h1:Y/mh94EeImzXyTBIMVgR0v5H+ANtRFDY4g1s5sxOZGE= -go.etcd.io/etcd/server/v3 v3.6.1/go.mod h1:nCqJGTP9c2WlZluJB59j3bqxZEI/GYBfQxno0MguVjE= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +go.etcd.io/etcd/api/v3 v3.6.6 h1:mcaMp3+7JawWv69p6QShYWS8cIWUOl32bFLb6qf8pOQ= +go.etcd.io/etcd/api/v3 v3.6.6/go.mod h1:f/om26iXl2wSkcTA1zGQv8reJRSLVdoEBsi4JdfMrx4= +go.etcd.io/etcd/client/pkg/v3 v3.6.6 h1:uoqgzSOv2H9KlIF5O1Lsd8sW+eMLuV6wzE3q5GJGQNs= +go.etcd.io/etcd/client/pkg/v3 v3.6.6/go.mod h1:YngfUVmvsvOJ2rRgStIyHsKtOt9SZI2aBJrZiWJhCbI= +go.etcd.io/etcd/client/v3 v3.6.6 h1:G5z1wMf5B9SNexoxOHUGBaULurOZPIgGPsW6CN492ec= +go.etcd.io/etcd/client/v3 v3.6.6/go.mod h1:36Qv6baQ07znPR3+n7t+Rk5VHEzVYPvFfGmfF4wBHV8= +go.etcd.io/etcd/pkg/v3 v3.6.6 h1:wylOivS/UxXTZ0Le5fOdxCjatW5ql9dcWEggQQHSorw= +go.etcd.io/etcd/pkg/v3 v3.6.6/go.mod h1:9TKZL7WUEVHXYM3srP3ESZfIms34s1G72eNtWA9YKg4= +go.etcd.io/etcd/server/v3 v3.6.6 h1:YSRWGJPzU+lIREwUQI4MfyLZrkUyzjJOVpMxJvZePaY= +go.etcd.io/etcd/server/v3 v3.6.6/go.mod h1:A1OQ1x3PaiENDLywMjCiMwV1pwJSpb0h9Z5ORP2dv6I= go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= @@ -539,19 +595,19 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqx go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= @@ -565,10 +621,10 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -580,8 +636,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -645,15 +701,15 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -663,6 +719,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -707,13 +765,13 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -721,14 +779,14 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -765,9 +823,10 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -811,10 +870,10 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= -google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -827,8 +886,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -841,8 +900,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -851,8 +910,8 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -870,6 +929,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -885,14 +945,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.22.0/go.mod h1:0AoXXqst47OI/L0oGKq9DG61dvGRPXs7X4/B7KyjBCU= -k8s.io/api v0.34.0 h1:L+JtP2wDbEYPUeNGbeSa/5GwFtIA662EmT2YSLOkAVE= -k8s.io/api v0.34.0/go.mod h1:YzgkIzOOlhl9uwWCZNqpw6RJy9L2FK4dlJeayUoydug= +k8s.io/api v0.34.3 h1:D12sTP257/jSH2vHV2EDYrb16bS7ULlHpdNdNhEw2S4= +k8s.io/api v0.34.3/go.mod h1:PyVQBF886Q5RSQZOim7DybQjAbVs8g7gwJNhGtY5MBk= k8s.io/apimachinery v0.22.0/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.34.0 h1:eR1WO5fo0HyoQZt1wdISpFDffnWOvFLOOeJ7MgIv4z0= -k8s.io/apimachinery v0.34.0/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apimachinery v0.34.3 h1:/TB+SFEiQvN9HPldtlWOTp0hWbJ+fjU+wkxysf/aQnE= +k8s.io/apimachinery v0.34.3/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= k8s.io/client-go v0.22.0/go.mod h1:GUjIuXR5PiEv/RVK5OODUsm6eZk7wtSWZSaSJbpFdGg= -k8s.io/client-go v0.34.0 h1:YoWv5r7bsBfb0Hs2jh8SOvFbKzzxyNo0nSb0zC19KZo= -k8s.io/client-go v0.34.0/go.mod h1:ozgMnEKXkRjeMvBZdV1AijMHLTh3pbACPvK7zFR+QQY= +k8s.io/client-go v0.34.3 h1:wtYtpzy/OPNYf7WyNBTj3iUA0XaBHVqhv4Iv3tbrF5A= +k8s.io/client-go v0.34.3/go.mod h1:OxxeYagaP9Kdf78UrKLa3YZixMCfP6bgPwPwNBQBzpM= k8s.io/component-base v0.22.0/go.mod h1:SXj6Z+V6P6GsBhHZVbWCw9hFjUdUYnJerlhhPnYCBCg= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -900,22 +960,22 @@ k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e h1:iW9ChlU0cU16w8MpVYjXk12dqQ4BPFBEgif+ap7/hqQ= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/helper.mk b/helper.mk new file mode 100644 index 00000000..1b9363e3 --- /dev/null +++ b/helper.mk @@ -0,0 +1,17 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +generate: + go generate + go run core/semver/semver.go -f mk > semver.mk + +download-csm-common: + git clone --depth 1 git@github.com:CSM/csm.git temp-repo + cp temp-repo/config/csm-common.mk . + rm -rf temp-repo + +vendor: + GOPRIVATE=github.com go mod vendor diff --git a/images.mk b/images.mk new file mode 100644 index 00000000..a07b7c00 --- /dev/null +++ b/images.mk @@ -0,0 +1,22 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +include overrides.mk +include helper.mk + +images: generate vendor download-csm-common + $(eval include csm-common.mk) + @echo "Base Images is set to: $(BASEIMAGE)" + @echo "Building: $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" + $(BUILDER) build --pull $(NOCACHE) -t "$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" --build-arg BASEIMAGE=$(CSM_BASEIMAGE) --build-arg GOIMAGE=$(DEFAULT_GOIMAGE) . + +images-no-cache: + @echo "Building with --no-cache ..." + @make images NOCACHE=--no-cache + +push: + @echo "Pushing: $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" + $(BUILDER) push "$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" diff --git a/main.go b/main.go index e752ea85..4b951305 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,4 @@ -//Copyright © 2019-2025 Dell Inc. or its subsidiaries. All Rights Reserved. +//Copyright © 2019-2026 Dell Inc. or its subsidiaries. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ import ( "github.com/dell/csi-vxflexos/v2/provider" "github.com/dell/csi-vxflexos/v2/service" "github.com/dell/gocsi" - "github.com/sirupsen/logrus" ) var flags struct { @@ -37,13 +36,18 @@ var flags struct { kubeconfig *string } +var ManifestSemver string + // main is ignored when this package is built as a go plug-in func main() { - logger := logrus.New() - service.Log = logger setEnvsFunc() initFlagsFunc() + if ManifestSemver != "" { + service.ManifestSemver = ManifestSemver + service.Manifest["semver"] = ManifestSemver + } + err := checkConfigsFunc() if err != nil { forceExit() diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..6e611c28 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: 'csi-powerflex' +site_description: 'csi-powerflex Documentation.' +docs_dir: docs +plugins: + - techdocs-core +theme: + name: material diff --git a/overrides.mk b/overrides.mk index 14608d3d..df48d35b 100644 --- a/overrides.mk +++ b/overrides.mk @@ -1,39 +1,12 @@ -# overrides file -# this file, included from the Makefile, will overlay default values with environment variables +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. # +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. -# DEFAULT values -# ubi9/ubi-micro:9.2-13 -DEFAULT_GOIMAGE=$(shell sed -En 's/^go (.*)$$/\1/p' go.mod) -DEFAULT_REGISTRY="sample_registry" -DEFAULT_IMAGENAME="csi-vxflexos" -DEFAULT_BUILDSTAGE="final" -DEFAULT_IMAGETAG="" - -# set the GOIMAGE if needed -ifeq ($(GOIMAGE),) -export GOIMAGE="$(DEFAULT_GOIMAGE)" -endif - -# set the REGISTRY if needed -ifeq ($(REGISTRY),) -export REGISTRY="$(DEFAULT_REGISTRY)" -endif - -# set the IMAGENAME if needed -ifeq ($(IMAGENAME),) -export IMAGENAME="$(DEFAULT_IMAGENAME)" -endif - -#set the IMAGETAG if needed -ifneq ($(DEFAULT_IMAGETAG), "") -export IMAGETAG="$(DEFAULT_IMAGETAG)" -endif - -# set the BUILDSTAGE if needed -ifeq ($(BUILDSTAGE),) -export BUILDSTAGE="$(DEFAULT_BUILDSTAGE)" -endif +IMAGE_REGISTRY?="sample_registry" +IMAGE_NAME="csi-vxflexos" +IMAGE_TAG?=$(shell date +%Y%m%d%H%M%S) # figure out if podman or docker should be used (use podman if found) ifneq (, $(shell which podman 2>/dev/null)) @@ -47,18 +20,10 @@ overrides-help: @echo @echo "The following environment variables can be set to control the build" @echo - @echo "GOIMAGE - The version of Go to build with, default is: $(DEFAULT_GOIMAGE)" - @echo " Current setting is: $(GOIMAGE)" - @echo "REGISTRY - The registry to push images to, default is: $(DEFAULT_REGISTRY)" - @echo " Current setting is: $(REGISTRY)" - @echo "IMAGENAME - The image name to be built, defaut is: $(DEFAULT_IMAGENAME)" - @echo " Current setting is: $(IMAGENAME)" - @echo "IMAGETAG - The image tag to be built, default is an empty string which will determine the tag by examining annotated tags in the repo." - @echo " Current setting is: $(IMAGETAG)" - @echo "BUILDSTAGE - The Dockerfile build stage to execute, default is: $(DEFAULT_BUILDSTAGE)" - @echo " Stages can be found by looking at the Dockerfile" - @echo " Current setting is: $(BUILDSTAGE)" + @echo "IMAGE_REGISTRY - The registry to push images to." + @echo " Current setting is: $(IMAGE_REGISTRY)" + @echo "IMAGE_NAME - The image name to be built." + @echo " Current setting is: $(IMAGE_NAME)" + @echo "IMAGE_TAG - The image tag to be built, default is the current date." + @echo " Current setting is: $(IMAGE_TAG)" @echo - - - diff --git a/provider/provider.go b/provider/provider.go index aa0e6b6f..d554f001 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -15,12 +15,13 @@ package provider import ( "github.com/dell/csi-vxflexos/v2/service" + "github.com/dell/csmlog" "github.com/dell/gocsi" - logrus "github.com/sirupsen/logrus" ) // Log init -var Log = logrus.New() +// var Log = logrus.New() +var log = csmlog.GetLogger() // New returns a new Mock Storage Plug-in Provider. func New() gocsi.StoragePluginProvider { diff --git a/samples/secret.yaml b/samples/secret.yaml index b6a41cb9..fc09de92 100644 --- a/samples/secret.yaml +++ b/samples/secret.yaml @@ -37,6 +37,10 @@ # defines the MDM(s) that SDC should register with on start. # Allowed values: a list of IP addresses or hostnames separated by comma. # Default value: none + # Optional: true + # Optional when blockProtocol: "NVMeTCP" or "auto". + # Required when blockProtocol: "SDC". + # In "auto" mode, specify 'mdm' to enable SDC registration; omit if only NVMe/TCP will be used. mdm: "10.0.0.1,10.0.0.2" # NFS is only supported on PowerFlex storage system >=4.0.x # If not specified, value from SC will be used. @@ -46,7 +50,13 @@ # Default value: "" # This is an optional field from v2.10.0 onwards for PowerFlex storage system >=4.0.x nasName: "nas-server" - + # blockProtocol: what transport protocol used on node side (SDC, NVMeTCP, or auto) + # Allowed Values: + # SDC: SDC will be used + # NVMeTCP: NVMe/TCP protocol will be used + # auto: SDC or NVMe/TCP protocol will be used + # Default Value: auto + blockProtocol: "auto" # # zone: A cluster availability zone to which the PowerFlex system should be bound. # # The mapping is one-to-one - the PowerFlex system cannot belong to more than one zone. # # Ideally, the PowerFlex system and cluster nodes that define the availability zone would be diff --git a/samples/secret/karavi-authorization-config.json b/samples/secret/karavi-authorization-config.json deleted file mode 100644 index d1f48528..00000000 --- a/samples/secret/karavi-authorization-config.json +++ /dev/null @@ -1 +0,0 @@ -[{"username":"","password":"","intendedEndpoint":"","endpoint":"https://localhost:9400","systemID":"","skipCertificateValidation":true,"isDefault":true}] diff --git a/samples/secret_OIDC.yaml b/samples/secret_OIDC.yaml new file mode 100644 index 00000000..40965998 --- /dev/null +++ b/samples/secret_OIDC.yaml @@ -0,0 +1,118 @@ +# Copyright © 2020-2025 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Username for accessing PowerFlex system. +# If authorization is enabled, username will be ignored. +- username: "ignored" + # Password for accessing PowerFlex system. + # If authorization is enabled, password will be ignored. + password: "ignored" + # System name/ID of PowerFlex system. + systemID: "1a99aa999999aa9a" + # Previous names used in secret of PowerFlex system. + allSystemNames: "pflex-1,pflex-2" + # REST API gateway HTTPS endpoint for PowerFlex system. + # If authorization is enabled, endpoint should be the HTTPS localhost endpoint that + # the authorization sidecar will listen on + endpoint: "https://127.0.0.1" + # Determines if the driver is going to validate certs while connecting to PowerFlex REST API interface. + # Allowed values: true or false + # Default value: true + skipCertificateValidation: true + # indicates if this array is the default array + # needed for backwards compatibility + # only one array is allowed to have this set to true + # Default value: false + isDefault: true + # defines the MDM(s) that SDC should register with on start. + # Allowed values: a list of IP addresses or hostnames separated by comma. + # Default value: none + mdm: "10.0.0.1,10.0.0.2" + # NFS is only supported on PowerFlex storage system >=4.0.x + # If not specified, value from SC will be used. + # If specified in both, secret and storage class, then precedence is given to storage class value. + # Allowed values: string + # Optional: true + # Default value: "" + # This is an optional field from v2.10.0 onwards for PowerFlex storage system >=4.0.x + nasName: "nas-server" + # blockProtocol: what transport protocol used on node side (SDC, NVMeTCP, or auto) + # Allowed Values: + # SDC: SDC will be used + # NVMeTCP: NVMe/TCP protocol will be used + # auto: SDC or NVMe/TCP protocol will be used + # Default Value: auto + blockProtocol: "auto" + # CiamClientId Client ID for CIAM authentication. + ciamClientId: "" + # CiamClientSecret Secret key for CIAM (Customer Identity and Access Management) authentication. + # Required when authType is set to OIDC. + # base64 encoded + ciamClientSecret: "" + # OidcClientId Client ID for OIDC authentication. + oidcClientId: "" + # OidcClientSecret: Secret key for OIDC authentication. + # base64 encoded + oidcClientSecret: "" + # The OIDC issuer URL for token validation. + # for keycloak: https://10.x.x.x:8443/realms/CSM + # for azure: https://login.microsoftonline.com/fafglafkn/v2.0 + # for okta: https://.okta.com/oauth2/default + issuer: "" + # Scopes for token validation. example: "openid", "profile", "email","custom_scopes" + scopes: "" + # # zone: A cluster availability zone to which the PowerFlex system should be bound. + # # The mapping is one-to-one - the PowerFlex system cannot belong to more than one zone. + # # Ideally, the PowerFlex system and cluster nodes that define the availability zone would be + # # geographically co-located. + # # Optional: true + # # Default value: none + # zone: + # # name: The name of the container orchestrator's availability zone to which the PowerFlex system + # # should be mapped. + # name: "zoneA" + # # labelKey: The name of the label used for the availability zone to which the PowerFlex system + # # should be mapped. + # labelKey: "topology.kubernetes.io/zone" + # # protectionDomains: A list of the protection domains and their associated pools, defined in + # # the PowerFlex system. + # # Currently, csi-powerflex only supports one protection domain per zone. + # protectionDomains: + # # pools: A list of pools that belong to a single protection defined in the PowerFlex system. + # # Currently, csi-powerflex only supports one pool per protection domain. + # - pools: + # - + # # name: The name of the protection domain in the PowerFlex system. + # # Optional: true + # # name is required if storage pool names are not unique across protection domains. + # name: +# To add more PowerFlex systems, uncomment the following lines and provide the required values +# - username: "ignored" +# password: "ignored" +# systemID: "2b11bb111111bb1b" +# endpoint: "https://127.0.0.2" +# skipCertificateValidation: true +# mdm: "10.0.0.3,10.0.0.4" +# ciamClientId: "" +# ciamClientSecret: "" +# oidcClientId: "" +# oidcClientSecret: "" +# issuer: "" +# scopes: "" +# zone: +# name: "zoneB" +# labelKey: "topology.kubernetes.io/zone" +# protectionDomains: +# - name: +# pools: +# - diff --git a/samples/storageclass/storageclass-nvmetcp.yaml b/samples/storageclass/storageclass-nvmetcp.yaml new file mode 100644 index 00000000..282079c9 --- /dev/null +++ b/samples/storageclass/storageclass-nvmetcp.yaml @@ -0,0 +1,80 @@ +# Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: vxflexos-nvmetcp +provisioner: csi-vxflexos.dellemc.com +# reclaimPolicy: PVs that are dynamically created by a StorageClass will have the reclaim policy specified here +# Allowed values: +# Reclaim: retain the PV after PVC deletion +# Delete: delete the PV after PVC deletion +# Optional: true +# Default value: Delete +reclaimPolicy: Delete +# allowVolumeExpansion: allows the users to resize the volume by editing the corresponding PVC object +# Allowed values: +# true: allow users to resize the PVC +# false: does not allow users to resize the PVC +# Optional: true +# Default value: false +allowVolumeExpansion: true +parameters: + # Storage pool to use on system + # Optional: false + storagepool: + # Protection domain that storage pool above belongs to + # Needed if array has two storagepools that share the same name, but belong to different protection domains + # Optional: true + # Uncomment the line below if you want to use protectiondomain + # protectiondomain: # Insert Protection domain name + # System you would like this storage class to use + # Allowed values: one string for system ID + # Optional: false + systemID: + # format options to pass to mkfs + # Allowed values: A string dictating the fs options you want passed + # Optional: true + # Uncomment the line below if you want to use mkfsFormatOption + # mkfsFormatOption: "" # Insert file system format option + # Filesytem type for volumes created by storageclass + # Default value: None if defaultFsType is not mentioned in values.yaml + # Else defaultFsType value mentioned in values.yaml + # will be used as default value + csi.storage.k8s.io/fstype: ext4 + # Limit the volume network bandwidth + # Value is a positive number in granularity of 1024 Kbps; 0 = unlimited + # Allowed values: one string for bandwidth limit in Kbps + # Optional: false + # Uncomment the line below if you want to use bandwidthLimitInKbps + # bandwidthLimitInKbps: # Insert bandwidth limit in Kbps + # Limit the volume IOPS + # The number of IOPS must be greater than 10; 0 = unlimited + # Allowed values: one string for iops limit + # Optional: false + # Uncomment the line below if you want to use iopsLimit + # iopsLimit: # Insert iops limit +# volumeBindingMode determines how volume binding and dynamic provisioning should occur +# Allowed values: +# Immediate: volume binding and dynamic provisioning occurs once PVC is created +# WaitForFirstConsumer: delay the binding and provisioning of PV until a pod using the PVC is created. +# Optional: false +# Default value: WaitForFirstConsumer (required for topology section below) +volumeBindingMode: WaitForFirstConsumer +# allowedTopologies helps scheduling pods on worker nodes which match all of below expressions. +allowedTopologies: + - matchLabelExpressions: + - key: csi-vxflexos.dellemc.com/-nvmetcp + values: + - "true" diff --git a/service/controller.go b/service/controller.go index 21340759..52a5c3db 100644 --- a/service/controller.go +++ b/service/controller.go @@ -17,7 +17,9 @@ import ( "errors" "fmt" "math" + "net" "net/http" + "net/url" "strconv" "strings" "sync" @@ -34,10 +36,10 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/wrapperspb" - csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/dell/csmlog" "github.com/dell/goscaleio" siotypes "github.com/dell/goscaleio/types/v1" - "github.com/sirupsen/logrus" + csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -159,7 +161,10 @@ const ( HeaderCSIPluginIdentifier = "x-csi-plugin-id" ) -var interestingParameters = [...]string{0: "FsType", 1: KeyMkfsFormatOption, 2: KeyBandwidthLimitInKbps, 3: KeyIopsLimit} +var ( + interestingParameters = [...]string{0: "FsType", 1: KeyMkfsFormatOption, 2: KeyBandwidthLimitInKbps, 3: KeyIopsLimit} + log = csmlog.GetLogger() +) type ZoneContent struct { systemID string @@ -172,6 +177,7 @@ func (s *service) CreateVolume( req *csi.CreateVolumeRequest) ( *csi.CreateVolumeResponse, error, ) { + log := log.WithContext(ctx) params := req.GetParameters() var systemID string var err error @@ -212,10 +218,31 @@ func (s *service) CreateVolume( } } + platformInfo, err := s.GetPlatformInfo(systemID) + if err != nil { + return nil, err + } + + if isNFS && platformInfo.GenType != "" && s.isGenTypeNotSupportsNfsAndReplication(platformInfo.GenType) { + return nil, status.Errorf(codes.InvalidArgument, "NFS is not supported on the System %s GenType %s", systemID, platformInfo.GenType) + } + + if isNFS && s.isNfsNotSupported(platformInfo.ArrayVersion) { + return nil, status.Errorf(codes.InvalidArgument, "NFS is not supported on the System %s PowerFlex version %.1f", systemID, platformInfo.ArrayVersion) + } + + remoteSystemID, ok := params[s.WithRP(KeyReplicationRemoteSystem)] + if ok { + isReplicationEnabledOnPlatform, err := s.IsReplicationEnabledOnPlatforms(systemID, remoteSystemID, platformInfo.GenType) + if !isReplicationEnabledOnPlatform { + return nil, err + } + } + // validate AccessibleTopology accessibility := req.GetAccessibilityRequirements() if accessibility == nil { - Log.Printf("Received CreateVolume request without accessibility keys") + log.Info("Received CreateVolume request without accessibility keys") } // Look for zone topology @@ -230,27 +257,27 @@ func (s *service) CreateVolume( contentSource := req.GetVolumeContentSource() var sourceSystemID string if contentSource != nil { - Log.Infof("[CreateVolume] Zone volume has a content source - we are a snapshot or clone: %+v", contentSource) + log.Infof("[CreateVolume] Zone volume has a content source - we are a snapshot or clone: %+v", contentSource) snapshotSource := contentSource.GetSnapshot() cloneSource := contentSource.GetVolume() if snapshotSource != nil { sourceSystemID = s.getSystemIDFromCsiVolumeID(snapshotSource.SnapshotId) - Log.Infof("[CreateVolume] Zone snapshot source systemID: %s", sourceSystemID) + log.Infof("[CreateVolume] Zone snapshot source systemID: %s", sourceSystemID) } else if cloneSource != nil { sourceSystemID = s.getSystemIDFromCsiVolumeID(cloneSource.VolumeId) - Log.Infof("[CreateVolume] Zone clone source systemID: %s", sourceSystemID) + log.Infof("[CreateVolume] Zone clone source systemID: %s", sourceSystemID) } } for _, topo := range accessibility.GetPreferred() { for topoLabel, zoneName := range topo.Segments { - Log.Infof("Zoning based on label %s", s.opts.zoneLabelKey) + log.Infof("Zoning based on label %s", s.opts.zoneLabelKey) if strings.HasPrefix(topoLabel, s.opts.zoneLabelKey) { zoneTarget, ok := zoneTargetMap[ZoneName(zoneName)] if !ok { - Log.Infof("no zone target for %s", zoneTarget) + log.Infof("no zone target for %s", zoneTarget) continue } @@ -263,7 +290,7 @@ func (s *service) CreateVolume( systemID = zoneTarget.systemID if err := s.requireProbe(ctx, systemID); err != nil { - Log.Errorln("Failed to probe system " + systemID) + log.Errorf("Failed to probe system: %v", systemID) continue } @@ -273,7 +300,7 @@ func (s *service) CreateVolume( }) // We found a zone topology - Log.Infof("Preferred topology zone %s, systemID %s, protectionDomain %s, and storagePool %s", zoneName, systemID, protectionDomain, storagePool) + log.Infof("Preferred topology zone %s, systemID %s, protectionDomain %s, and storagePool %s", zoneName, systemID, protectionDomain, storagePool) zoneTopology = true } } @@ -307,7 +334,7 @@ func (s *service) CreateVolume( if len(tokens) > 1 { constraint = tokens[1] } - Log.Printf("Found topology constraint: VxFlex OS system: %s", constraint) + log.Infof("Found topology constraint: VxFlex OS system: %s", constraint) // Update constraint wrt to topology specified for NFS volume if isNFS { @@ -322,6 +349,16 @@ func (s *service) CreateVolume( } } } + + // check if constraint is having nvmetcp type + if strings.HasSuffix(constraint, "nvmetcp") { + nvmeToken := strings.Split(constraint, "-") + if len(nvmeToken) > 1 { + // update constraint to system ID + constraint = nvmeToken[0] + } + } + if constraint == sID || constraint == sName { if constraint == sID { requestedSystem = sID @@ -330,8 +367,11 @@ func (s *service) CreateVolume( } // segment matches system ID/Name where volume will be created topologyKey := tokens[0] + "/" + sID + if strings.HasSuffix(key, "nvmetcp") { + topologyKey = key + } systemSegments[topologyKey] = segments[key] - Log.Printf("Added accessible topology segment for volume: %s, segment: %s = %s", req.GetName(), + log.Infof("Added accessible topology segment for volume: %s, segment: %s = %s", req.GetName(), topologyKey, systemSegments[topologyKey]) } } @@ -347,7 +387,7 @@ func (s *service) CreateVolume( volumeTopology = append(volumeTopology, &csi.Topology{ Segments: systemSegments, }) - Log.Printf("Accessible topology for volume: %s, segments: %#v", req.GetName(), systemSegments) + log.Infof("Accessible topology for volume: %s, segments: %#v", req.GetName(), systemSegments) } } @@ -371,7 +411,7 @@ func (s *service) CreateVolume( if len(name) > 31 { name = name[0:31] - Log.Printf("Requested name %s longer than 31 character max, truncated to %s\n", req.Name, name) + log.Infof("Requested name %s longer than 31 character max, truncated to %s\n", req.Name, name) req.Name = name } @@ -386,7 +426,7 @@ func (s *service) CreateVolume( if params[KeyNasName] != "" { nasName = params[KeyNasName] // Storage class takes precedence } else { - Log.Debug("nasName not present in storage class, value taken from secret") + log.Info("nasName not present in storage class, value taken from secret") nasName = arr.NasName // Secret next } nasServerID, err := s.getNASServerIDFromName(systemID, nasName) @@ -398,7 +438,7 @@ func (s *service) CreateVolume( pdID := "" pd, ok := params[KeyProtectionDomain] if !ok { - Log.Printf("Protection Domain name not provided; there could be conflicts if two storage pools share a name") + log.Info("Protection Domain name not provided; there could be conflicts if two storage pools share a name") } else { pdID, err = s.getProtectionDomainIDFromName(systemID, pd) if err != nil { @@ -421,7 +461,7 @@ func (s *service) CreateVolume( size := cr.GetRequiredBytes() // round off the size to the 3GB if less than 3GB if size < minNfsSize { - Log.Printf("Size %d is less than 3GB, rounding to 3GB", size/bytesInGiB) + log.Infof("Size %d is less than 3GB, rounding to 3GB", size/bytesInGiB) size = minNfsSize } @@ -429,7 +469,7 @@ func (s *service) CreateVolume( if contentSource != nil { snapshotSource := contentSource.GetSnapshot() if snapshotSource != nil { - Log.Printf("snapshot %s specified as volume content source", snapshotSource.SnapshotId) + log.Infof("snapshot %s specified as volume content source", snapshotSource.SnapshotId) return s.createVolumeFromSnapshot(req, snapshotSource, name, size, storagePoolName) } } @@ -443,7 +483,8 @@ func (s *service) CreateVolume( HeaderPersistentVolumeClaimName: params[CSIPersistentVolumeClaimName], HeaderPersistentVolumeClaimNamespace: params[CSIPersistentVolumeClaimNamespace], } - Log.WithFields(fields).Info("Executing CreateVolume with following fields") + // logctx = csmlog.WithContext(ctx) + log.WithFields(fields).Info("Executing CreateVolume with following fields") volumeParam := &siotypes.FsCreate{ Name: volName, @@ -469,16 +510,16 @@ func (s *service) CreateVolume( csiResp := &csi.CreateVolumeResponse{ Volume: vi, } - Log.Info("Volume exists in the requested state with same size") + log.Info("Volume exists in the requested state with same size") return csiResp, nil } - Log.Info("'Volume name' already exists and size is different") + log.Info("'Volume name' already exists and size is different") return nil, status.Error(codes.AlreadyExists, "'Volume name' already exists and size is different.") } - Log.Debug("Volume does not exist, proceeding to create new volume") + log.Debug("Volume does not exist, proceeding to create new volume") fsResp, err := system.CreateFileSystem(volumeParam) if err != nil { - Log.Debugf("Create volume response error:%v", err) + log.Debugf("Create volume response error:%v", err) return nil, status.Errorf(codes.Unknown, "Create Volume %s failed with error: %v", volName, err) } @@ -488,7 +529,7 @@ func (s *service) CreateVolume( // get filesystem (NFS volume), newly created fs, err := system.GetFileSystemByIDName(fsResp.ID, "") if err != nil { - Log.Debugf("Find Volume response error: %v", err) + log.Debugf("Find Volume response error: %v", err) return nil, status.Errorf(codes.Unknown, "Find Volume response error: %v", err) } path, ok := params[KeyPath] @@ -514,16 +555,16 @@ func (s *service) CreateVolume( return nil, status.Errorf(codes.Internal, "rollback (deleting volume '%s') failed with error : '%v'", fs.Name, delErr.Error()) } - Log.Debugf("Error creating quota for volume: %s of size: %d bytes, error: %v", fs.Name, size, err.Error()) - Log.Debugf("Successfully rolled back by deleting the newly created volume: %s", fs.Name) + log.Errorf("Error creating quota for volume: %s of size: %d bytes, error: %v", fs.Name, size, err.Error()) + log.Debugf("Successfully rolled back by deleting the newly created volume: %s", fs.Name) return nil, err } - Log.Infof("Tree quota set for: %d bytes on directory: '%s', quota ID: %s", size, path, quotaID) + log.Infof("Tree quota set for: %d bytes on directory: '%s', quota ID: %s", size, path, quotaID) } newFs, err := system.GetFileSystemByIDName(fsResp.ID, "") if err != nil { - Log.Debugf("Find Volume response error: %v", err) + log.Debugf("Find Volume response error: %v", err) return nil, status.Errorf(codes.Unknown, "Find Volume response error: %v", err) } if newFs != nil { @@ -555,14 +596,14 @@ func (s *service) CreateVolume( storagePool = sp } else { - Log.Printf("[CreateVolume] Multi-AZ Storage Pool Determined by Secret %s", storagePool) + log.Infof("[CreateVolume] Multi-AZ Storage Pool Determined by Secret %s", storagePool) } var pdID string if protectionDomain == "" { pd, ok := params[KeyProtectionDomain] if !ok { - Log.Printf("Protection Domain name not provided; there could be conflicts if two storage pools share a name") + log.Info("Protection Domain name not provided; there could be conflicts if two storage pools share a name") } else { protectionDomain = pd } @@ -590,7 +631,7 @@ func (s *service) CreateVolume( } snapshotSource := contentSource.GetSnapshot() if snapshotSource != nil { - Log.Printf("snapshot %s specified as volume content source", snapshotSource.SnapshotId) + log.Infof("snapshot %s specified as volume content source", snapshotSource.SnapshotId) snapshotVolumeResponse, err := s.createVolumeFromSnapshot(req, snapshotSource, name, size, storagePool) if err != nil { return nil, err @@ -613,8 +654,9 @@ func (s *service) CreateVolume( HeaderPersistentVolumeClaimName: params[CSIPersistentVolumeClaimName], HeaderPersistentVolumeClaimNamespace: params[CSIPersistentVolumeClaimNamespace], } + // logctx = csmlog.WithContext(ctx) - Log.WithFields(fields).Info("Executing CreateVolume with following fields") + log.WithFields(fields).Info("Executing CreateVolume with following fields") volumeParam := &siotypes.VolumeParam{ Name: name, @@ -632,14 +674,14 @@ func (s *service) CreateVolume( t.MetaData().Set(HeaderCSIPluginIdentifier, Name) t.MetaData().Set(HeaderSystemIdentifier, systemID) } else { - Log.Println("warning: goscaleio.VolumeParam: no MetaData method exists, consider updating goscaleio library.") + log.Warn("warning: goscaleio.VolumeParam: no MetaData method exists, consider updating goscaleio library.") } createResp, err := s.adminClients[systemID].CreateVolume(volumeParam, storagePool, pdID) if err != nil { // handle case where volume already exists if !strings.EqualFold(err.Error(), sioGatewayVolumeNameInUse) { - Log.Printf("error creating volume: %s pool %s error: %s", name, storagePool, err.Error()) + log.Infof("error creating volume: %s pool %s error: %s", name, storagePool, err.Error()) return nil, status.Errorf(codes.Internal, "error when creating volume %s storagepool %s: %s", name, storagePool, err.Error()) } @@ -683,7 +725,7 @@ func (s *service) CreateVolume( } copyInterestingParameters(req.GetParameters(), vi.VolumeContext) - Log.Printf("volume %s (%s) created %s\n", vi.VolumeContext["Name"], vi.VolumeId, vi.VolumeContext["CreationTime"]) + log.Infof("volume %s (%s) created %s\n", vi.VolumeContext["Name"], vi.VolumeId, vi.VolumeContext["CreationTime"]) vi.VolumeContext[KeyFsType] = fsType csiResp := &csi.CreateVolumeResponse{ @@ -716,7 +758,7 @@ func (s *service) createQuota(fsID, path, softLimit, gracePeriod string, size in // enabling quota on FS fs, err := system.GetFileSystemByIDName(fsID, "") if err != nil { - Log.Debugf("Find Volume response error: %v", err) + log.Debugf("Find Volume response error: %v", err) return "", status.Errorf(codes.Unknown, "Find Volume response error: %v", err) } @@ -736,20 +778,20 @@ func (s *service) createQuota(fsID, path, softLimit, gracePeriod string, size in err = system.ModifyFileSystem(fsModify, fs.ID) if err != nil { - Log.Debugf("Modify NFS volume failed with error: %v", err) + log.Debugf("Modify NFS volume failed with error: %v", err) return "", status.Errorf(codes.Unknown, "Modify NFS volume failed with error: %v", err) } fs, err = system.GetFileSystemByIDName(fsID, "") if err != nil { - Log.Debugf("Find NFS volume response error: %v", err) + log.Debugf("Find NFS volume response error: %v", err) return "", status.Errorf(codes.Unknown, "Find NFS volume response error: %v", err) } // need to set the quota based on the requested pv size // if a size isn't requested, skip creating the quota if size <= 0 { - Log.Debugf("Quotas is enabled, but storage size is not requested, skip creating quotas for volume '%s'", fsID) + log.Debugf("Quotas is enabled, but storage size is not requested, skip creating quotas for volume '%s'", fsID) return "", nil } @@ -763,7 +805,7 @@ func (s *service) createQuota(fsID, path, softLimit, gracePeriod string, size in return "", status.Errorf(codes.InvalidArgument, "requested softLimit: %s perc, i.e. default value which is greater than hardlimit, i.e. volume size: %d for volume %s:", softLimit, size, fsID) } - Log.Debugf("Begin to set quota for FS '%s', size '%d', quota enabled: '%t'", fsID, size, isQuotaEnabled) + log.Debugf("Begin to set quota for FS '%s', size '%d', quota enabled: '%t'", fsID, size, isQuotaEnabled) // log all parameters used in CreateTreeQuota call fields := map[string]interface{}{ "FileSystemID": fsID, @@ -772,7 +814,8 @@ func (s *service) createQuota(fsID, path, softLimit, gracePeriod string, size in "SoftLimit": softLimitInt, "GracePeriod": gracePeriodInt, } - Log.WithFields(fields).Info("Executing CreateTreeQuota with following fields") + // logctx = csmlog.WithContext(ctx) + log.WithFields(fields).Info("Executing CreateTreeQuota with following fields") createQuotaParams := &siotypes.TreeQuotaCreate{ FileSystemID: fsID, @@ -783,7 +826,7 @@ func (s *service) createQuota(fsID, path, softLimit, gracePeriod string, size in } quota, err := system.CreateTreeQuota(createQuotaParams) if err != nil { - Log.Debugf("Creating quota failed with error: %v", err) + log.Debugf("Creating quota failed with error: %v", err) return "", status.Errorf(codes.Unknown, "Creating quota failed with error: %v", err) } return quota.ID, nil @@ -813,7 +856,7 @@ func validateQuotaParameters(path, softLimit, gracePeriod, fsID string) (int64, return 0, 0, status.Errorf(codes.InvalidArgument, "requested gracePeriod: %s is not numeric for volume %s, error: %s", gracePeriod, fsID, err) } } else { - Log.Debugf("GracePeriod value set to default.") + log.Debugf("GracePeriod value set to default.") gracePeriodInt = 0 } return softLimitPerc, gracePeriodInt, nil @@ -856,20 +899,21 @@ func (s *service) getSystemIDFromParameters(params map[string]string) (string, e } } - Log.Printf("getSystemIDFromParameters system %s", systemID) + log.Infof("getSystemIDFromParameters system %s", systemID) // if name set for array.SystemID use id instead // names can change , id will remain unique if id, ok := s.connectedSystemNameToID[systemID]; ok { systemID = id } - Log.Printf("Use systemID as %s", systemID) + log.Infof("Use systemID as %s", systemID) return systemID, nil } // getZonesFromSecret returns a map with zone names as keys to zone content // with zone content consisting of the PowerFlex systemID, protection domain and pool. func (s *service) getZonesFromSecret() map[ZoneName]ZoneContent { + // zoneTargetMap := make(map[ZoneName]ZoneContent) for _, array := range s.opts.arrays { @@ -893,10 +937,25 @@ func (s *service) getZonesFromSecret() map[ZoneName]ZoneContent { pool: pool, } } - return zoneTargetMap } +var getVolByIDFunc = func(s *service, id string, systemID string) (*siotypes.Volume, error) { + return s.getVolByID(id, systemID) +} + +var getStoragePoolNameFromIDFunc = func(s *service, systemID string, id string) string { + return s.getStoragePoolNameFromID(systemID, id) +} + +var getVolumeFunc = func(adminClient *goscaleio.Client, a, b, c, name string, e bool) ([]*siotypes.Volume, error) { + return adminClient.GetVolume(a, b, c, name, e) +} + +var createThinCloneFunc = func(system *goscaleio.System, snapParam *siotypes.CreateSnapshotParam) (*siotypes.SnapshotVolumesResp, error) { + return system.CreateThinClone(snapParam) +} + // Create a volume (which is actually a snapshot) from an existing snapshot. // The snapshotSource gives the SnapshotId which is the volume to be replicated. func (s *service) createVolumeFromSnapshot(req *csi.CreateVolumeRequest, @@ -967,7 +1026,7 @@ func (s *service) createVolumeFromSnapshot(req *csi.CreateVolumeRequest, csiVolume.ContentSource = req.GetVolumeContentSource() copyInterestingParameters(req.GetParameters(), csiVolume.VolumeContext) - Log.Printf("Volume (from snap) %s (%s) storage pool %s", + log.Infof("Volume (from snap) %s (%s) storage pool %s", csiVolume.VolumeContext["Name"], csiVolume.VolumeId, csiVolume.VolumeContext["StoragePoolName"]) return &csi.CreateVolumeResponse{Volume: csiVolume}, nil @@ -975,7 +1034,7 @@ func (s *service) createVolumeFromSnapshot(req *csi.CreateVolumeRequest, // Look up the snapshot snapID := getVolumeIDFromCsiVolumeID(snapshotSource.SnapshotId) - srcVol, err := s.getVolByID(snapID, systemID) + srcVol, err := getVolByIDFunc(s, snapID, systemID) if err != nil { return nil, status.Errorf(codes.NotFound, "Snapshot not found: %s, error: %s", snapshotSource.SnapshotId, err.Error()) } @@ -990,28 +1049,28 @@ func (s *service) createVolumeFromSnapshot(req *csi.CreateVolumeRequest, system := s.systems[systemID] // Validate the storagePool is the same. - snapStoragePool := s.getStoragePoolNameFromID(systemID, srcVol.StoragePoolID) + snapStoragePool := getStoragePoolNameFromIDFunc(s, systemID, srcVol.StoragePoolID) if snapStoragePool != storagePool { return nil, status.Errorf(codes.InvalidArgument, "Snapshot storage pool %s is different than the requested storage pool %s", snapStoragePool, storagePool) } // Check for idempotent request - existingVols, err := adminClient.GetVolume("", "", "", name, false) + existingVols, err := getVolumeFunc(adminClient, "", "", "", name, false) noVolErrString1 := "Error: problem finding volume: Volume not found" noVolErrString2 := "Error: problem finding volume: Could not find the volume" if (err != nil) && !(strings.Contains(err.Error(), noVolErrString1) || strings.Contains(err.Error(), noVolErrString2)) { - Log.Printf("[createVolumeFromSnapshot] Idempotency check: GetVolume returned error: %s", err.Error()) + log.Infof("[createVolumeFromSnapshot] Idempotency check: GetVolume returned error: %s", err.Error()) return nil, status.Errorf(codes.Internal, "Failed to create vol from snap -- GetVolume returned unexpected error: %s", err.Error()) } for _, vol := range existingVols { if vol.Name == name && vol.StoragePoolID == srcVol.StoragePoolID { - Log.Printf("Requested volume %s already exists", name) + log.Infof("Requested volume %s already exists", name) csiVolume := s.getCSIVolume(vol, systemID) csiVolume.ContentSource = req.GetVolumeContentSource() copyInterestingParameters(req.GetParameters(), csiVolume.VolumeContext) - Log.Printf("Requested volume (from snap) already exists %s (%s) storage pool %s", + log.Infof("Requested volume (from snap) already exists %s (%s) storage pool %s", csiVolume.VolumeContext["Name"], csiVolume.VolumeId, csiVolume.VolumeContext["StoragePoolName"]) return &csi.CreateVolumeResponse{Volume: csiVolume}, nil } @@ -1021,13 +1080,23 @@ func (s *service) createVolumeFromSnapshot(req *csi.CreateVolumeRequest, snapshotDefs := make([]*siotypes.SnapshotDef, 0) snapDef := &siotypes.SnapshotDef{VolumeID: snapID, SnapshotName: name} snapshotDefs = append(snapshotDefs, snapDef) - snapParam := &siotypes.SnapshotVolumesParam{SnapshotDefs: snapshotDefs, AccessMode: "ReadWrite"} + snapResponse := &siotypes.SnapshotVolumesResp{} - // Create snapshot - snapResponse, err := system.CreateSnapshotConsistencyGroup(snapParam) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create snapshot: %s", err.Error()) + // Create clone or snapshot + if srcVol.GenType == "EC" { + snapParam := &siotypes.CreateSnapshotParam{SnapshotDefs: snapshotDefs} + snapResponse, err = createThinCloneFunc(system, snapParam) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to call CreateThinClone to create volume from snapshot: %s", err.Error()) + } + } else { + snapParam := &siotypes.SnapshotVolumesParam{SnapshotDefs: snapshotDefs, AccessMode: "ReadWrite"} + snapResponse, err = system.CreateSnapshotConsistencyGroup(snapParam) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to call CreateSnapshotConsistencyGroup to create volume from snapshot: %s", err.Error()) + } } + if len(snapResponse.VolumeIDList) != 1 { return nil, status.Errorf(codes.Internal, "Expected volume ID to be returned but it was not") } @@ -1044,7 +1113,7 @@ func (s *service) createVolumeFromSnapshot(req *csi.CreateVolumeRequest, csiVolume.ContentSource = req.GetVolumeContentSource() copyInterestingParameters(req.GetParameters(), csiVolume.VolumeContext) - Log.Printf("Volume (from snap) %s (%s) storage pool %s", + log.Infof("Volume (from snap) %s (%s) storage pool %s", csiVolume.VolumeContext["Name"], csiVolume.VolumeId, csiVolume.VolumeContext["StoragePoolName"]) return &csi.CreateVolumeResponse{Volume: csiVolume}, nil } @@ -1062,6 +1131,7 @@ func (s *service) clearCache() { // volume to create, and returns an error if volume size would be greater than // the given limit. Returned size is in KiB func validateVolSize(cr *csi.CapacityRange) (int64, error) { + // minSize := cr.GetRequiredBytes() maxSize := cr.GetLimitBytes() if minSize < 0 || maxSize < 0 { @@ -1159,7 +1229,7 @@ func (s *service) DeleteVolume( toBeDeletedFS, err := system.GetFileSystemByIDName(fsID, "") if err != nil { if strings.Contains(err.Error(), sioGatewayFileSystemNotFound) { - Log.WithFields(logrus.Fields{"id": fsID}).Debug("NFS volume does not exist", fsID) + log.WithFields(csmlog.Fields{"id": fsID}).Debug("NFS volume does not exist") return &csi.DeleteVolumeResponse{}, nil } } @@ -1202,12 +1272,12 @@ func (s *service) DeleteVolume( var modifyParam *siotypes.NFSExportModify = &siotypes.NFSExportModify{} // Removing externalAccess from RWHosts as well as RWRootHosts if len(nfsExport.ReadWriteRootHosts) == 1 && externalAccess == nfsExport.ReadWriteRootHosts[0] { - Log.Debug("Trying to remove externalAccess IP with mask having RWRootHosts access while deleting the volume: ", externalAccess) + log.Debugf("Trying to remove externalAccess IP with mask having RWRootHosts access while deleting the volume: %v ", externalAccess) modifyNFSExport = true modifyParam.RemoveReadWriteRootHosts = []string{externalAccess} } if len(nfsExport.ReadWriteHosts) == 1 && externalAccess == nfsExport.ReadWriteHosts[0] { - Log.Debug("Trying to remove externalAccess IP with mask having RWHosts access while deleting the volume: ", externalAccess) + log.Debugf("Trying to remove externalAccess IP with mask having RWHosts access while deleting the volume: %v", externalAccess) modifyNFSExport = true modifyParam.RemoveReadWriteHosts = []string{externalAccess} } @@ -1215,7 +1285,7 @@ func (s *service) DeleteVolume( if modifyNFSExport { err = client.ModifyNFSExport(modifyParam, fsID) if err != nil { - Log.Warn("failure when removing externalAccess from nfs export: ", err.Error()) + log.Warnf("failure when removing externalAccess from nfs export: %v", err) } } else { // either of RWRootHosts or RWHosts has one entry but it is not externalAccess @@ -1230,7 +1300,7 @@ func (s *service) DeleteVolume( } } - Log.WithFields(logrus.Fields{"name": fsName, "id": fsID}).Info("Deleting NFS volume") + log.WithFields(csmlog.Fields{"name": fsName, "id": fsID}).Info("Deleting NFS volume") err = system.DeleteFileSystem(fsName) if err != nil { if strings.Contains(err.Error(), sioGatewayFileSystemNotFound) { @@ -1263,18 +1333,19 @@ func (s *service) DeleteVolume( volID := getVolumeIDFromCsiVolumeID(csiVolID) vol, err := s.getVolByID(volID, systemID) if err != nil { + if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) { - Log.WithFields(logrus.Fields{"id": csiVolID}).Debug("volume is already deleted", csiVolID) + log.Debugf("volume is already deleted : %v", csiVolID) return &csi.DeleteVolumeResponse{}, nil } if strings.Contains(err.Error(), sioVolumeRemovalOperationInProgress) { - Log.WithFields(logrus.Fields{"id": csiVolID}).Debug("volume is currently being deleted", csiVolID) + log.Debugf("volume is currently being deleted : %v", csiVolID) return &csi.DeleteVolumeResponse{}, nil } if strings.Contains(err.Error(), "must be a hexadecimal number") { - Log.WithFields(logrus.Fields{"id": csiVolID}).Debug("volume id must be a hexadecimal number", csiVolID) + log.Debugf("volume id must be a hexadecimal number : %v", csiVolID) return &csi.DeleteVolumeResponse{}, nil } @@ -1292,16 +1363,16 @@ func (s *service) DeleteVolume( // If volume is marked for replication, remove the replication pair first. if vol.VolumeReplicationState != "UnmarkedForReplication" { - Log.Printf("[DeleteVolume] - vol: %+v", vol) + log.Infof("[DeleteVolume] - vol: %+v", vol) pair, err := s.removeVolumeFromReplicationPair(systemID, volID) if err != nil { return nil, status.Errorf(codes.Internal, "error removing replication pair: %s", err.Error()) } - Log.Printf("[DeleteVolume] - Removed Pair: %+v", pair) + log.Infof("[DeleteVolume] - Removed Pair: %+v", pair) } - Log.WithFields(logrus.Fields{"name": vol.Name, "id": csiVolID}).Info("Deleting volume") + log.WithFields(csmlog.Fields{"name": vol.Name, "id": csiVolID}).Info("Deleting volume") tgtVol := goscaleio.NewVolume(s.adminClients[systemID]) tgtVol.Volume = vol err = tgtVol.RemoveVolume(removeModeOnlyMe) @@ -1328,11 +1399,13 @@ func (s *service) DeleteVolume( return &csi.DeleteVolumeResponse{}, nil } +var CreateKubeClientSet = k8sutils.CreateKubeClientSet + func (s *service) findNetworkInterfaceIPs() ([]string, error) { if K8sClientset == nil { - err := k8sutils.CreateKubeClientSet() + err := CreateKubeClientSet() if err != nil { - Log.Errorf("Failed to create Kubernetes clientset: %v", err) + log.Errorf("Failed to create Kubernetes clientset: %v", err) return []string{}, err } K8sClientset = k8sutils.Clientset @@ -1341,7 +1414,7 @@ func (s *service) findNetworkInterfaceIPs() ([]string, error) { // Get the ConfigMap configMap, err := K8sClientset.CoreV1().ConfigMaps(DriverNamespace).Get(context.TODO(), DriverConfigMap, metav1.GetOptions{}) if err != nil { - Log.Errorf("Failed to get the ConfigMap: %v", err) + log.Errorf("Failed to get the ConfigMap: %v", err) return []string{}, err } @@ -1351,7 +1424,7 @@ func (s *service) findNetworkInterfaceIPs() ([]string, error) { if configParamsYaml, ok := configMap.Data[DriverConfigParamsYaml]; ok { err := yaml.Unmarshal([]byte(configParamsYaml), &configData) if err != nil { - Log.Errorf("Failed to unmarshal the ConfigMap params: %v", err) + log.Errorf("Failed to unmarshal the ConfigMap params: %v", err) return []string{}, err } @@ -1374,12 +1447,30 @@ func (s *service) ControllerPublishVolume( ) { volumeContext := req.GetVolumeContext() if volumeContext != nil { - Log.Printf("VolumeContext:") + log.Infof("VolumeContext:") for key, value := range volumeContext { - Log.Printf(" [%s]=%s", key, value) + log.Infof(" [%s]=%s", key, value) } } + if req.GetVolumeCapability() == nil { + return nil, status.Error(codes.InvalidArgument, "volume capability is required") + } + + am := req.GetVolumeCapability().GetAccessMode() + if am == nil { + return nil, status.Error(codes.InvalidArgument, "access mode is required") + } + + if am.Mode == csi.VolumeCapability_AccessMode_UNKNOWN { + return nil, status.Error(codes.InvalidArgument, errUnknownAccessMode) + } + + nodeID := req.GetNodeId() + if nodeID == "" { + return nil, status.Error(codes.InvalidArgument, "node ID is required") + } + // create publish context publishContext := make(map[string]string) publishContext[KeyNasName] = volumeContext[KeyNasName] @@ -1388,8 +1479,7 @@ func (s *service) ControllerPublishVolume( publishContext["volumeContextId"] = csiVolID if csiVolID == "" { - return nil, status.Error(codes.InvalidArgument, - "volume ID is required") + return nil, status.Error(codes.InvalidArgument, "volume ID is required") } // get systemID from req @@ -1399,8 +1489,7 @@ func (s *service) ControllerPublishVolume( systemID = s.opts.defaultSystemID } if systemID == "" { - return nil, status.Error(codes.InvalidArgument, - "systemID is not found in the request and there is no default system") + return nil, status.Error(codes.InvalidArgument, "systemID is not found in the request and there is no default system") } if err := s.requireProbe(ctx, systemID); err != nil { @@ -1417,10 +1506,15 @@ func (s *service) ControllerPublishVolume( "checkVolumesMap for id: %s failed : %s", csiVolID, err.Error()) } - nodeID := req.GetNodeId() - if nodeID == "" { - return nil, status.Error(codes.InvalidArgument, - "node ID is required") + // Check for NVMe type + isNVME := false + _, hostType, err := s.getHostIDAndType(systemID, nodeID) + if err != nil { + return nil, status.Errorf(codes.NotFound, + "error getting host ID and type for nodeID %s: %s", nodeID, err.Error()) + } + if hostType == NVMeTCP { + isNVME = true } // Check for NFS protocol @@ -1434,12 +1528,9 @@ func (s *service) ControllerPublishVolume( fs, err := s.getFilesystemByID(fsID, systemID) if err != nil { if strings.EqualFold(err.Error(), sioGatewayFileSystemNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { - return nil, status.Error(codes.NotFound, - "volume not found") + return nil, status.Error(codes.NotFound, "volume not found") } - return nil, status.Errorf(codes.Internal, - "failure checking volume status before controller publish: %s", - err.Error()) + return nil, status.Errorf(codes.Internal, "failure checking volume status before controller publish: %s", err.Error()) } var ipAddresses []string @@ -1447,7 +1538,7 @@ func (s *service) ControllerPublishVolume( ipAddresses, err = s.findNetworkInterfaceIPs() if err != nil || len(ipAddresses) == 0 { - Log.Printf("ControllerPublish - No network interfaces found, trying to get SDC IPs") + log.Infof("ControllerPublish - No network interfaces found, trying to get SDC IPs") // get SDC IPs if Network Interface IPs not found ipAddresses, err = s.getSDCIPs(nodeID, systemID) if err != nil { @@ -1456,26 +1547,11 @@ func (s *service) ControllerPublishVolume( return nil, status.Errorf(codes.NotFound, "%s", "received empty sdcIPs") } } - Log.Printf("ControllerPublish - ipAddresses %v", ipAddresses) + log.Infof("ControllerPublish - ipAddresses %v", ipAddresses) externalAccess := s.opts.ExternalAccess publishContext["host"] = ipAddresses[0] - fsc := req.GetVolumeCapability() - if fsc == nil { - return nil, status.Error(codes.InvalidArgument, - "volume capability is required") - } - - am := fsc.GetAccessMode() - if am == nil { - return nil, status.Error(codes.InvalidArgument, - "access mode is required") - } - if am.Mode == csi.VolumeCapability_AccessMode_UNKNOWN { - return nil, status.Error(codes.InvalidArgument, - errUnknownAccessMode) - } // Export for NFS resp, err := s.exportFilesystem(ctx, req, adminClient, fs, ipAddresses, externalAccess, nodeID, publishContext, am) return resp, err @@ -1485,131 +1561,26 @@ func (s *service) ControllerPublishVolume( if err != nil { if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") || strings.Contains(err.Error(), "Invalid volume") { - return nil, status.Error(codes.NotFound, - "volume not found") + return nil, status.Error(codes.NotFound, "volume not found") } - return nil, status.Errorf(codes.Internal, - "failure checking volume status before controller publish: %s", - err.Error()) - } - - sdcID, err := s.getSDCID(nodeID, systemID) - if err != nil { - return nil, status.Errorf(codes.NotFound, "%s", err.Error()) - } - - vc := req.GetVolumeCapability() - if vc == nil { - return nil, status.Error(codes.InvalidArgument, - "volume capability is required") + return nil, status.Errorf(codes.Internal, "failure checking volume status before controller publish: %s", err.Error()) } + log.Infof("Found volume: %s with volume ID: %s", vol.Name, vol.ID) - am := vc.GetAccessMode() - if am == nil { - return nil, status.Error(codes.InvalidArgument, - "access mode is required") - } - - if am.Mode == csi.VolumeCapability_AccessMode_UNKNOWN { - return nil, status.Error(codes.InvalidArgument, - errUnknownAccessMode) - } - // Check if volume is published to any node already - allowMultipleMappings := "FALSE" - vcs := []*csi.VolumeCapability{req.GetVolumeCapability()} - isBlock := accTypeIsBlock(vcs) - - if len(vol.MappedSdcInfo) > 0 { - for _, sdc := range vol.MappedSdcInfo { - if sdc.SdcID == sdcID { - // TODO check if published volume is compatible with this request - // volume already mapped - Log.Debug("volume already mapped") - - // check for QoS limits of mapped volume - bandwidthLimit := volumeContext[KeyBandwidthLimitInKbps] - iopsLimit := volumeContext[KeyIopsLimit] - // validate requested QoS parameters - if err := validateQoSParameters(bandwidthLimit, iopsLimit, vol.Name); err != nil { - return nil, err - } - - // check if volume QoS is same as requested QoS settings - if len(bandwidthLimit) > 0 && strconv.Itoa(sdc.LimitBwInMbps*1024) != bandwidthLimit { - return nil, status.Errorf(codes.InvalidArgument, - "volume %s already published with bandwidth limit: %d, but does not match the requested bandwidth limit: %s", vol.Name, sdc.LimitBwInMbps*1024, bandwidthLimit) - } else if len(iopsLimit) > 0 && strconv.Itoa(sdc.LimitIops) != iopsLimit { - return nil, status.Errorf(codes.InvalidArgument, - "volume %s already published with IOPS limit: %d, but does not match the requested IOPS limits: %s", vol.Name, sdc.LimitIops, iopsLimit) - } - - return &csi.ControllerPublishVolumeResponse{}, nil - } - } - - // If volume has SINGLE_NODE cap, go no farther - switch am.Mode { - case csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER, - csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER, - csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY: - return nil, status.Errorf(codes.FailedPrecondition, - "volume already published to SDC id: %s", vol.MappedSdcInfo[0].SdcID) - } - - // All remaining cases are MULTI_NODE: - // This original code precludes block multi-writers, - // and is based on a faulty test that the Volume MappingToAllSdcsEnabled - // attribute must be set to allow multiple writers, which is not true. - // The proper way to control multiple mappings is with the allowMultipleMappings - // attribute passed in the MapVolumeSdcParameter. Unfortunately you cannot - // read this parameter back. - - allowMultipleMappings, err = shouldAllowMultipleMappings(isBlock, am) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "%s", err.Error()) - } - - if err := validateAccessType(am, isBlock); err != nil { - return nil, err + var publisher VolumePublisher + if isNVME { + publisher = &NVMePublisher{ + svc: s, + vol: vol, } } else { - allowMultipleMappings, err = shouldAllowMultipleMappings(isBlock, am) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "%s", err.Error()) + publisher = &SDCPublisher{ + svc: s, + vol: vol, } } - mapVolumeSdcParam := &siotypes.MapVolumeSdcParam{ - SdcID: sdcID, - AllowMultipleMappings: allowMultipleMappings, - AllSdcs: "", - } - - targetVolume := goscaleio.NewVolume(adminClient) - targetVolume.Volume = &siotypes.Volume{ID: vol.ID} - - err = targetVolume.MapVolumeSdc(mapVolumeSdcParam) - if err != nil { - return nil, status.Errorf(codes.Internal, - "error mapping volume to node: %s", err.Error()) - } - - bandwidthLimit := volumeContext[KeyBandwidthLimitInKbps] - iopsLimit := volumeContext[KeyIopsLimit] - - // validate requested QoS parameters - if err := validateQoSParameters(bandwidthLimit, iopsLimit, vol.Name); err != nil { - return nil, err - } - // check for atleast one of the QoS params should exist in storage class - if len(bandwidthLimit) > 0 || len(iopsLimit) > 0 { - if err = s.setQoSParameters(ctx, systemID, sdcID, bandwidthLimit, iopsLimit, vol.Name, csiVolID, nodeID); err != nil { - return nil, err - } - } - - return &csi.ControllerPublishVolumeResponse{}, nil + return publisher.Publish(ctx, req, adminClient, systemID, csiVolID) } // validate the requested QoS parameters. @@ -1638,7 +1609,7 @@ func (s *service) setQoSParameters( iopsLimit string, volumeName string, csiVolID string, nodeID string, ) error { - Log.Infof("Setting QoS limits for volume %s, mapped to SDC %s", volumeName, sdcID) + log.Infof("Setting QoS limits for volume %s, mapped to SDC %s", volumeName, sdcID) adminClient := s.adminClients[systemID] tgtVol := goscaleio.NewVolume(adminClient) volID := getVolumeIDFromCsiVolumeID(csiVolID) @@ -1655,7 +1626,7 @@ func (s *service) setQoSParameters( err = tgtVol.SetMappedSdcLimits(&settings) if err != nil { // unpublish the volume - Log.Errorf("unpublishing volume since error in setting QoS parameters for volume: %s, error: %s", volumeName, err.Error()) + log.Errorf("unpublishing volume since error in setting QoS parameters for volume: %s, error: %s", volumeName, err.Error()) _, newErr := s.ControllerUnpublishVolume(ctx, &csi.ControllerUnpublishVolumeRequest{ VolumeId: csiVolID, @@ -1680,9 +1651,9 @@ func shouldAllowMultipleMappings(isBlock bool, accessMode *csi.VolumeCapability_ if isBlock { return TRUE, nil } - return FALSE, errors.New("Mount multinode multi-writer not allowed") + return FALSE, errors.New("mount multinode multi-writer not allowed") case csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER: - return FALSE, errors.New("Multinode single writer not supported") + return FALSE, errors.New("multinode single writer not supported") default: return FALSE, nil } @@ -1720,16 +1691,18 @@ func validateAccessType( func (s *service) ControllerUnpublishVolume( ctx context.Context, - req *csi.ControllerUnpublishVolumeRequest) ( - *csi.ControllerUnpublishVolumeResponse, error, -) { - // get systemID from req - systemID := s.getSystemIDFromCsiVolumeID(req.GetVolumeId()) + req *csi.ControllerUnpublishVolumeRequest, +) (*csi.ControllerUnpublishVolumeResponse, error) { + // Validate and Extract IDs + csiVolID := req.GetVolumeId() + if csiVolID == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + + systemID := s.getSystemIDFromCsiVolumeID(csiVolID) if systemID == "" { - // use default system systemID = s.opts.defaultSystemID } - if systemID == "" { return nil, status.Error(codes.InvalidArgument, "systemID is not found in the request and there is no default system") @@ -1741,104 +1714,124 @@ func (s *service) ControllerUnpublishVolume( s.logStatistics() - csiVolID := req.GetVolumeId() - if csiVolID == "" { - return nil, status.Error(codes.InvalidArgument, - "volume ID is required") - } - // ensure no ambiguity if legacy vol - err := s.checkVolumesMap(csiVolID) - if err != nil { + if err := s.checkVolumesMap(csiVolID); err != nil { return nil, status.Errorf(codes.Internal, - "checkVolumesMap for id: %s failed : %s", csiVolID, err.Error()) + "checkVolumesMap for id: %s failed: %s", csiVolID, err.Error()) } + nodeID := req.GetNodeId() if nodeID == "" { - return nil, status.Error(codes.InvalidArgument, - "Node ID is required") + return nil, status.Error(codes.InvalidArgument, "Node ID is required") } - adminClient := s.adminClients[systemID] + log.Infof("ControllerUnpublishVolume called for nodeID: %s", nodeID) + adminClient := s.adminClients[systemID] isNFS := strings.Contains(csiVolID, "/") + // Handle NFS Volumes if isNFS { fsID := getFilesystemIDFromCsiVolumeID(csiVolID) fs, err := s.getFilesystemByID(fsID, systemID) if err != nil { - if strings.EqualFold(err.Error(), sioGatewayFileSystemNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { - return nil, status.Error(codes.NotFound, - "volume not found") + if strings.EqualFold(err.Error(), sioGatewayFileSystemNotFound) || + strings.Contains(err.Error(), "must be a hexadecimal number") { + return nil, status.Error(codes.NotFound, "volume not found") } return nil, status.Errorf(codes.Internal, - "failure checking volume status before controller publish: %s", - err.Error()) + "failure checking volume status before controller unpublish: %s", err.Error()) } - var ipAddresses []string - ipAddresses, err = s.findNetworkInterfaceIPs() + ipAddresses, err := s.findNetworkInterfaceIPs() if err != nil || len(ipAddresses) == 0 { - - Log.Printf("ControllerUnPublish - No network interfaces found, trying to get SDC IPs") + log.Infof("No network interfaces found, trying to get SDC IPs") ipAddresses, err = s.getSDCIPs(nodeID, systemID) if err != nil { return nil, status.Errorf(codes.NotFound, "%s", err.Error()) } else if len(ipAddresses) == 0 { - return nil, status.Errorf(codes.NotFound, "%s", "received empty sdcIPs") + return nil, status.Errorf(codes.NotFound, "received empty SDC IPs") } } - Log.Printf("ControllerUnPublish - ipAddresses %v", ipAddresses) - // unexport for NFS - err = s.unexportFilesystem(ctx, req, adminClient, fs, req.GetVolumeId(), ipAddresses, nodeID) - if err != nil { + log.Infof("SDC IP addresses: %v", ipAddresses) + + if err := s.unexportFilesystem(ctx, req, adminClient, fs, csiVolID, ipAddresses, nodeID); err != nil { return nil, err } - return &csi.ControllerUnpublishVolumeResponse{}, nil } + // Handle Block Volumes volID := getVolumeIDFromCsiVolumeID(csiVolID) vol, err := s.getVolByID(volID, systemID) if err != nil { if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) { - Log.Debugf("volume %s is already deleted", volID) + log.Debugf("volume %s is already deleted", volID) return &csi.ControllerUnpublishVolumeResponse{}, nil } return nil, status.Errorf(codes.Internal, - "failure checking volume status before controller unpublish: %s", - err.Error()) + "failure checking volume status before controller unpublish: %s", err.Error()) } - sdcID, err := s.getSDCID(nodeID, systemID) - if err != nil { - return nil, status.Errorf(codes.NotFound, "%s", err.Error()) + // Check Mapping and Protocol + var ( + protocol string + mappedToNode bool + ) + + hostID, _, err := s.getHostIDAndType(systemID, nodeID) + if err != nil || hostID == "" { + return nil, status.Errorf(codes.Internal, + "error getting host ID for nodeID %s: %s", nodeID, err.Error()) } - // check if volume is attached to node at all - mappedToNode := false for _, mapping := range vol.MappedSdcInfo { - if mapping.SdcID == sdcID { - mappedToNode = true - break + log.Debugf("Checking mapping for nodeID: %s", nodeID) + log.Debugf("Mapping HostType: %s, Host ID: %s, Host Name: %s", mapping.HostType, mapping.SdcID, mapping.SdcName) + + if mapping.HostType == "SdcHost" { + if mapping.SdcID == hostID { + protocol = SDC + mappedToNode = true + break + } + } else if mapping.HostType == "NVMeHost" { + if mapping.SdcID == hostID { + protocol = NVMeTCP + mappedToNode = true + break + } } } if !mappedToNode { - Log.Debug("volume already unpublished") + log.Debug("Volume already unpublished") return &csi.ControllerUnpublishVolumeResponse{}, nil } + + // Unmap Volume targetVolume := goscaleio.NewVolume(adminClient) targetVolume.Volume = vol - unmapVolumeSdcParam := &siotypes.UnmapVolumeSdcParam{ - SdcID: sdcID, - AllSdcs: "", - } - - if err = targetVolume.UnmapVolumeSdc(unmapVolumeSdcParam); err != nil { - return nil, status.Errorf(codes.Internal, - "Error unmapping volume from node: %s", err.Error()) + switch protocol { + case SDC: + unmapVolumeSdcParam := &siotypes.UnmapVolumeSdcParam{ + SdcID: hostID, + AllSdcs: "", + } + if err := targetVolume.UnmapVolumeSdc(unmapVolumeSdcParam); err != nil { + return nil, status.Errorf(codes.Internal, + "Error unmapping volume from SDC node: %s", err.Error()) + } + case NVMeTCP: + unmapVolumeNVMeParam := &siotypes.UnmapVolumeNVMeParam{ + HostID: hostID, + AllHosts: "", + } + if err := targetVolume.RemoveMappedHost(unmapVolumeNVMeParam); err != nil { + return nil, status.Errorf(codes.Internal, + "Error unmapping volume from NVMe host: %s", err.Error()) + } } return &csi.ControllerUnpublishVolumeResponse{}, nil @@ -1985,7 +1978,7 @@ func (s *service) ListVolumes( req *csi.ListVolumesRequest) ( *csi.ListVolumesResponse, error, ) { - Log.Printf("ListVolumes called") + log.Infof("ListVolumes called") var systemID string var entries []*csi.ListVolumesResponse_Entry @@ -1997,11 +1990,11 @@ func (s *service) ListVolumes( if systemID != "" { if err := s.requireProbe(ctx, systemID); err != nil { - Log.Warnf("Could not probe system: %s", systemID) + log.Warnf("Could not probe system: %s", systemID) continue } } else { - Log.Printf("SystemID is empty in controller array configuration") + log.Infof("SystemID is empty in controller array configuration") return nil, status.Error(codes.InvalidArgument, "There is no SystemID in controller array configuration") } @@ -2039,7 +2032,7 @@ func (s *service) ListVolumes( i := 0 for _, vol := range source { if vol == nil { - Log.Printf("Volume[%d] is nil in ListVolumeResponse from system %s", i, systemID) + log.Infof("Volume[%d] is nil in ListVolumeResponse from system %s", i, systemID) continue } entries[i] = &csi.ListVolumesResponse_Entry{ @@ -2114,7 +2107,7 @@ func (s *service) ListSnapshots( } if err := s.requireProbe(ctx, systemID); err != nil { - Log.Printf("Could not probe system: %s", systemID) + log.Infof("Could not probe system: %s", systemID) code := status.Code(err) if code == codes.NotFound { return &csi.ListSnapshotsResponse{}, nil @@ -2191,7 +2184,7 @@ func (s *service) listVolumes(systemID string, startToken int, maxEntries int, d if doVols { // Get the volumes from the cache if we can. if startToken != 0 && len(s.volCache) > 0 { - Log.Printf("volume cache hit: %d volumes", len(s.volCache)) + log.Infof("volume cache hit: %d volumes", len(s.volCache)) func() { s.volCacheRWL.Lock() defer s.volCacheRWL.Unlock() @@ -2227,7 +2220,7 @@ func (s *service) listVolumes(systemID string, startToken int, maxEntries int, d // Process snapshots. if doSnaps { if startToken != 0 && len(s.snapCache) > 0 { - Log.Printf("snap cache hit: %d snapshots", len(s.snapCache)) + log.Infof("snap cache hit: %d snapshots", len(s.snapCache)) func() { s.snapCacheRWL.Lock() defer s.snapCacheRWL.Unlock() @@ -2292,59 +2285,6 @@ func (s *service) listVolumes(systemID string, startToken int, maxEntries int, d return volumes[startToken : startToken+maxEntries], nextTokenStr, nil } -// Gets capacity of a given storage system. When storage pool name is provided, gets capcity of this storage pool only. -func (s *service) getSystemCapacity(ctx context.Context, systemID, protectionDomain string, spName ...string) (int64, error) { - Log.Infof("Get capacity for system: %s, pool %s", systemID, spName) - - if err := s.requireProbe(ctx, systemID); err != nil { - return 0, err - } - - adminClient := s.adminClients[systemID] - system := s.systems[systemID] - if adminClient == nil || system == nil { - return 0, fmt.Errorf("can't find adminClient or system by id %s", systemID) - } - - var statsFunc func() (*siotypes.Statistics, error) - - // Default to get Capacity of system - statsFunc = system.GetStatistics - - if len(spName) > 0 && spName[0] != "" { - // if storage pool is given, get capacity of storage pool - pdID, err := s.getProtectionDomainIDFromName(systemID, protectionDomain) - if err != nil { - return 0, err - } - sp, err := adminClient.FindStoragePool("", spName[0], "", pdID) - if err != nil { - return 0, status.Errorf(codes.Internal, - "unable to look up storage pool: %s on system: %s, err: %s", - spName, systemID, err.Error()) - } - spc := goscaleio.NewStoragePoolEx(adminClient, sp) - statsFunc = spc.GetStatistics - - if !s.opts.Thick { - if sp.CapacityUsageState == "Critical" { - return 0, nil - } - } - } - - stats, err := statsFunc() - if err != nil { - return 0, status.Errorf(codes.Internal, - "unable to get system stats for system: %s, err: %s", systemID, err.Error()) - } - - if !s.opts.Thick { - return int64(stats.VolumeAllocationLimitInKb * bytesInKiB), nil - } - return int64(stats.CapacityAvailableForVolumeAllocationInKb * bytesInKiB), nil -} - // Gets capacity for all systems known to controller. // When storage pool name is provided, gets capacity of this storage pool name from all systems func (s *service) getCapacityForAllSystems(ctx context.Context, protectionDomain string, spName ...string) (int64, error) { @@ -2362,7 +2302,7 @@ func (s *service) getCapacityForAllSystems(ctx context.Context, protectionDomain if err != nil { return 0, status.Errorf(codes.Internal, - "Unable to get capacity for system: %s, err: %s", array.SystemID, err.Error()) + "unable to get system stats for system: %s, err: %s", array.SystemID, err.Error()) } capacity += systemCapacity @@ -2395,7 +2335,7 @@ func (s *service) GetCapacity( spname := params[KeyStoragePool] pd, ok := params[KeyProtectionDomain] if !ok { - Log.Printf("Protection Domain name not provided; there could be conflicts if two storage pools share a name") + log.Infof("Protection Domain name not provided; there could be conflicts if two storage pools share a name") } for key, value := range params { if strings.EqualFold(key, KeySystemID) { @@ -2438,7 +2378,7 @@ func (s *service) GetCapacity( maxVolSize, err := s.getMaximumVolumeSize(systemID) if err != nil { - Log.Debug("GetMaxVolumeSize returning error ", err) + log.Debugf("GetMaxVolumeSize returning error: %v", err) } if maxVolSize < 0 { @@ -2460,7 +2400,7 @@ func (s *service) GetCapacity( func (s *service) getSystemIDFromZoneLabelKey(req *csi.GetCapacityRequest) (systemID string, err error) { zoneName, ok := req.AccessibleTopology.Segments[s.opts.zoneLabelKey] if !ok { - Log.Infof("could not get availability zone from accessible topology. Getting capacity for all systems") + log.Infof("could not get availability zone from accessible topology. Getting capacity for all systems") return "", nil } @@ -2487,13 +2427,13 @@ func (s *service) getMaximumVolumeSize(systemID string) (int64, error) { vol1, err := adminClient.GetMaxVol() if err != nil { - Log.Debug("GetMaxVolumeSize returning error ", err) + log.Debugf("GetMaxVolumeSize returning error: %v ", err) return 0, err } value, err := strconv.ParseInt(vol1, 10, 64) if err != nil { - Log.Debug("error converting str to int ", err) + log.Debugf("error converting str to int: %v ", err) return 0, err } @@ -2633,13 +2573,15 @@ func (s *service) ControllerGetCapabilities( } func (s *service) getZoneFromZoneLabelKey(ctx context.Context, zoneLabelKey string) (zone string, err error) { + log := log.WithContext(ctx) + // get labels for this service, s labels, err := GetNodeLabels(ctx, s) if err != nil { return "", err } - Log.Infof("Listing labels: %v", labels) + log.Infof("Listing labels: %v", labels) // get the zone name from the labels if val, ok := labels[zoneLabelKey]; ok { @@ -2652,8 +2594,10 @@ func (s *service) getZoneFromZoneLabelKey(ctx context.Context, zoneLabelKey stri // systemProbeAll will iterate through all arrays in service.opts.arrays and probe them. If failed, it logs // the failed system name func (s *service) systemProbeAll(ctx context.Context) error { + log := log.WithContext(ctx) + // probe all arrays - Log.Infoln("Probing all associated arrays") + log.Infof("Probing all associated arrays") allArrayFail := true errMap := make(map[string]error) zoneName := "" @@ -2665,7 +2609,7 @@ func (s *service) systemProbeAll(ctx context.Context) error { if err != nil { return err } - Log.Infof("probing zoneLabel '%s', zone value: '%s'", s.opts.zoneLabelKey, zoneName) + log.Infof("probing zoneLabel '%s', zone value: '%s'", s.opts.zoneLabelKey, zoneName) } newCtx, cancel := s.createProbeContextWithDeadline(ctx) @@ -2676,7 +2620,7 @@ func (s *service) systemProbeAll(ctx context.Context) error { if usingZones && !array.isInZone(zoneName) { // Driver node containers should not probe arrays that exist outside their assigned zone // Driver controller container should probe all arrays - Log.Infof("array %s zone %s does not match %s, not pinging this array\n", array.SystemID, array.AvailabilityZone.Name, zoneName) + log.Infof("array %s zone %s does not match %s, not pinging this array\n", array.SystemID, array.AvailabilityZone.Name, zoneName) errMap[array.SystemID] = fmt.Errorf("array %s zone %s does not match %s, not pinging this array", array.SystemID, array.AvailabilityZone.Name, zoneName) continue } @@ -2685,14 +2629,14 @@ func (s *service) systemProbeAll(ctx context.Context) error { systemID := array.SystemID if err != nil { errMap[systemID] = err - Log.Errorf("array %s probe failed: %v", array.SystemID, err) + log.Errorf("array %s probe failed: %v", array.SystemID, err) } else { allArrayFail = false - Log.Infof("array %s probed successfully", systemID) + log.Infof("array %s probed successfully", systemID) } } - Log.Printf("[SystemProbeAll] Number of failed probes: %d", len(errMap)) + log.Infof("[SystemProbeAll] Number of failed probes: %d", len(errMap)) if allArrayFail { return status.Error(codes.FailedPrecondition, @@ -2702,8 +2646,78 @@ func (s *service) systemProbeAll(ctx context.Context) error { return nil } +func ExtractIP(endpoint string) (string, error) { + u, err := url.Parse(endpoint) + if err != nil { + return "", err + } + + host := u.Hostname() // strips port if present + ip := net.ParseIP(host) + if ip == nil { + return "", fmt.Errorf("not a valid IP: %s", host) + } + + return ip.String(), nil +} + +func oidcPrechecks(array *ArrayConnectionData) error { + if array.OidcClientID == "" { + return status.Error(codes.FailedPrecondition, "missing OidcClientID") + } + if array.OidcClientSecret == "" { + return status.Error(codes.FailedPrecondition, "missing OidcClientSecret") + } + if array.CiamClientID == "" { + return status.Error(codes.FailedPrecondition, "missing CiamClientID") + } + if array.CiamClientSecret == "" { + return status.Error(codes.FailedPrecondition, "missing CiamClientSecret") + } + if array.Issuer == "" { + return status.Error(codes.FailedPrecondition, "missing Issuer") + } + + return nil +} + +func ParseScopes(scopesCSV string) []string { + csv := strings.TrimSpace(scopesCSV) + if csv == "" { + return nil + } + + parts := strings.Split(csv, ",") + out := make([]string, 0, len(parts)) + seen := make(map[string]struct{}) + + for _, p := range parts { + s := strings.TrimSpace(p) + if s == "" { + continue + } + if _, ok := seen[s]; !ok { + seen[s] = struct{}{} + out = append(out, s) + } + } + + if len(out) == 0 { + return nil + } + return out +} + // systemProbe will probe the given array func (s *service) systemProbe(ctx context.Context, array *ArrayConnectionData) error { + lock := s.getProbeLock(array.SystemID) + log.Debugf("[systemProbe] Waiting for lock for systemID=%s", array.SystemID) + lock.Lock() + defer lock.Unlock() + log.Debugf("[systemProbe] Acquired lock for systemID=%s, starting probe", array.SystemID) + + log := csmlog.GetLogger().WithContext(ctx) + // Check that we have the details needed to login to the Gateway if array.Endpoint == "" { return status.Error(codes.FailedPrecondition, @@ -2743,18 +2757,52 @@ func (s *service) systemProbe(ctx context.Context, array *ArrayConnectionData) e } } - Log.Printf("Login to PowerFlex Gateway, system=%s, endpoint=%s, user=%s\n", systemID, array.Endpoint, array.Username) + log.Infof("Login to PowerFlex Gateway, system=%s, endpoint=%s, user=%s\n", systemID, array.Endpoint, array.Username) client := s.adminClients[systemID] if client.GetToken() == "" { - _, err := client.WithContext(ctx).Authenticate(&goscaleio.ConfigConnect{ - Endpoint: array.Endpoint, - Username: array.Username, - Password: array.Password, - }) - if err != nil { - return status.Errorf(codes.FailedPrecondition, - "unable to login to PowerFlex Gateway: %s", err.Error()) + if s.opts.AuthType == "OIDC" { + log.Debugf("Authentication via OIDC") + + err := oidcPrechecks(array) + if err != nil { + return status.Errorf(codes.FailedPrecondition, + "OIDC prechecks failed: %s", err.Error()) + } + + pfmpIP, err := ExtractIP(array.Endpoint) + if err != nil { + return status.Errorf(codes.FailedPrecondition, + "unable to extract endpoint IP: %s", err.Error()) + } + + _, err = client.WithContext(ctx).Authenticate(&goscaleio.ConfigConnect{ + Endpoint: array.Endpoint, + AuthType: s.opts.AuthType, + PfmpIP: pfmpIP, + CiamClientID: array.CiamClientID, + CiamClientSecret: array.CiamClientSecret, + OidcClientID: array.OidcClientID, + OidcClientSecret: array.OidcClientSecret, + Issuer: array.Issuer, + Insecure: array.SkipCertificateValidation, + Scopes: ParseScopes(array.Scopes), + }) + if err != nil { + return status.Errorf(codes.FailedPrecondition, + "unable to login to PowerFlex Gateway: %s", err.Error()) + } + } else { + log.Debugf("Basic Authentication") + _, err := client.WithContext(ctx).Authenticate(&goscaleio.ConfigConnect{ + Endpoint: array.Endpoint, + Username: array.Username, + Password: array.Password, + }) + if err != nil { + return status.Errorf(codes.FailedPrecondition, + "unable to login to PowerFlex Gateway: %s", err.Error()) + } } } @@ -2769,7 +2817,7 @@ func (s *service) systemProbe(ctx context.Context, array *ArrayConnectionData) e s.systems[systemID] = system if system.System != nil && system.System.Name != "" { - Log.Printf("Found Name for system=%s with ID=%s", system.System.Name, system.System.ID) + log.Infof("Found Name for system=%s with ID=%s", system.System.Name, system.System.ID) s.connectedSystemNameToID[system.System.Name] = system.System.ID s.systems[system.System.ID] = system s.adminClients[system.System.ID] = client @@ -2784,15 +2832,15 @@ func (s *service) systemProbe(ctx context.Context, array *ArrayConnectionData) e sysID := systemID if id, ok := s.connectedSystemNameToID[systemID]; ok { - Log.Printf("System with name %s found id: %s", systemID, id) + log.Infof("System with name %s found id: %s", systemID, id) sysID = id s.opts.arrays[sysID] = array } if array.IsDefault { - Log.Infof("default array is set to array ID: %s", sysID) + log.Infof("default array is set to array ID: %s", sysID) s.opts.defaultSystemID = sysID - Log.Printf("%s is the default array, skipping VolumePrefixToSystems map update. \n", sysID) + log.Infof("%s is the default array, skipping VolumePrefixToSystems map update. \n", sysID) } else { err := s.UpdateVolumePrefixToSystemsMap(sysID) if err != nil { @@ -2803,9 +2851,23 @@ func (s *service) systemProbe(ctx context.Context, array *ArrayConnectionData) e return nil } +func (s *service) getProbeLock(systemID string) *sync.Mutex { + actual, loaded := s.probeLocks.LoadOrStore(systemID, &sync.Mutex{}) + if loaded { + log.Debugf("[probeLock] Reusing existing lock for systemID=%s", systemID) + } else { + log.Debugf("[probeLock] Created new lock for systemID=%s", systemID) + } + return actual.(*sync.Mutex) +} + func (s *service) requireProbe(ctx context.Context, systemID string) error { + log := csmlog.GetLogger().WithContext(ctx) + if s.adminClients[systemID] == nil || s.systems[systemID] == nil { - Log.Debugf("probing system %s automatically", systemID) + mx.Lock() + defer mx.Unlock() + log.Debugf("probing system %s automatically", systemID) array, ok := s.opts.arrays[systemID] if ok { if err := s.systemProbe(ctx, array); err != nil { @@ -2821,6 +2883,10 @@ func (s *service) requireProbe(ctx context.Context, systemID string) error { return nil } +var createSnapshotFunc = func(system *goscaleio.System, snapParam *siotypes.CreateSnapshotParam) (*siotypes.SnapshotVolumesResp, error) { + return system.CreateSnapshot(snapParam) +} + // CreateSnapshot creates a snapshot. // If Parameters["VolumeIDList"] has a comma separated list of additional volumes, they will be // snapshotted in a consistency group with the primary volume in CreateSnapshotRequest.SourceVolumeId. @@ -2866,7 +2932,7 @@ func (s *service) CreateSnapshot( name = strings.Replace(name, "snapshot-", "sn-", 1) length := int(math.Min(float64(len(name)), 31)) name = name[0:length] - Log.Printf("Requested name %s longer than 31 character max, truncated to %s\n", req.Name, name) + log.Infof("Requested name %s longer than 31 character max, truncated to %s\n", req.Name, name) req.Name = name } @@ -2930,7 +2996,7 @@ func (s *service) CreateSnapshot( csiSnapResponse := &csi.CreateSnapshotResponse{Snapshot: snapshot} s.clearCache() - Log.Printf("createSnapshot: SnapshotId %s SourceVolumeId %s CreationTime %s", + log.Infof("createSnapshot: SnapshotId %s SourceVolumeId %s CreationTime %s", snapshot.SnapshotId, snapshot.SourceVolumeId, snapshot.CreationTime.AsTime().Format(time.RFC3339Nano)) return csiSnapResponse, nil @@ -2939,21 +3005,22 @@ func (s *service) CreateSnapshot( volID := getVolumeIDFromCsiVolumeID(csiVolID) // Check for idempotent request, i.e. the snapshot has been already created, by looking up the name. - existingVols, err := s.adminClients[systemID].GetVolume("", "", "", req.Name, false) + adminClient := s.adminClients[systemID] + existingVols, err := getVolumeFunc(adminClient, "", "", "", req.Name, false) noVolErrString1 := "Error: problem finding volume: Volume not found" noVolErrString2 := "Error: problem finding volume: Could not find the volume" if (err != nil) && !(strings.Contains(err.Error(), noVolErrString1) || strings.Contains(err.Error(), noVolErrString2)) { - Log.Printf("[CreateSnapshot] Idempotency check: GetVolume returned error: %s", err.Error()) + log.Infof("[CreateSnapshot] Idempotency check: GetVolume returned error: %s", err.Error()) return nil, status.Errorf(codes.Internal, "Failed to create snapshot -- GetVolume returned unexpected error: %s", err.Error()) } for _, vol := range existingVols { ancestor := vol.AncestorVolumeID - Log.Printf("idempotent Name %s Name %s Ancestor %s id %s VTree %s pool %s\n", + log.Infof("idempotent Name %s Name %s Ancestor %s id %s VTree %s pool %s\n", vol.Name, req.Name, ancestor, volID, vol.VTreeID, vol.StoragePoolID) if vol.Name == req.Name && vol.AncestorVolumeID == volID { // populate response structure - Log.Printf("Idempotent request, snapshot id %s for source vol %s in system %s already exists\n", vol.ID, vol.AncestorVolumeID, systemID) + log.Infof("Idempotent request, snapshot id %s for source vol %s in system %s already exists\n", vol.ID, vol.AncestorVolumeID, systemID) snapshot := s.getCSISnapshot(vol, systemID) resp := &csi.CreateSnapshotResponse{Snapshot: snapshot} return resp, nil @@ -2961,7 +3028,7 @@ func (s *service) CreateSnapshot( } // Validate volume - vol, err := s.getVolByID(volID, systemID) + vol, err := getVolByIDFunc(s, volID, systemID) if err != nil { if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) { return nil, status.Errorf(codes.NotFound, "volume %s was not found", volID) @@ -2970,7 +3037,7 @@ func (s *service) CreateSnapshot( "failure checking volume status: %s", err.Error()) } vtreeID := vol.VTreeID - Log.Printf("vtree ID: %s\n", vtreeID) + log.Infof("vtree ID: %s\n", vtreeID) // Build list of volumes to be snapshotted. snapshotDefs := make([]*siotypes.SnapshotDef, 0) @@ -2992,7 +3059,7 @@ func (s *service) CreateSnapshot( if consistencyGroupSystem != "" && consistencyGroupSystem != systemID { // system needs to be the same throughout snapshot consistency group, this is an error err = status.Errorf(codes.Internal, "Consistency group needs to be on the same system but vol %s is not on system: %s ", v, systemID) - Log.Errorf("Consistency group needs to be on the same system but vol %s is not on system: %s ", v, systemID) + log.Errorf("Consistency group needs to be on the same system but vol %s is not on system: %s ", v, systemID) return nil, err } v = getVolumeIDFromCsiVolumeID(v) @@ -3010,10 +3077,17 @@ func (s *service) CreateSnapshot( snapshotDefs = append(snapshotDefs, &snapshotDefX) } } - snapParam := &siotypes.SnapshotVolumesParam{SnapshotDefs: snapshotDefs, AccessMode: "ReadOnly"} // Create snapshot(s) - snapResponse, err := s.systems[systemID].CreateSnapshotConsistencyGroup(snapParam) + snapResponse := &siotypes.SnapshotVolumesResp{} + if vol.GenType == "EC" { + snapParam := &siotypes.CreateSnapshotParam{SnapshotDefs: snapshotDefs} + system := s.systems[systemID] + snapResponse, err = createSnapshotFunc(system, snapParam) + } else { + snapParam := &siotypes.SnapshotVolumesParam{SnapshotDefs: snapshotDefs, AccessMode: "ReadOnly"} + snapResponse, err = s.systems[systemID].CreateSnapshotConsistencyGroup(snapParam) + } if err != nil { return nil, status.Errorf(codes.AlreadyExists, "Failed to create snapshot: %s", err.Error()) } @@ -3036,7 +3110,7 @@ func (s *service) CreateSnapshot( resp := &csi.CreateSnapshotResponse{Snapshot: snapshot} s.clearCache() - Log.Printf("createSnapshot: SnapshotId %s SourceVolumeId %s CreationTime %s", + log.Infof("createSnapshot: SnapshotId %s SourceVolumeId %s CreationTime %s", snapshot.SnapshotId, snapshot.SourceVolumeId, snapshot.CreationTime.AsTime().Format(time.RFC3339Nano)) return resp, nil } @@ -3052,7 +3126,7 @@ func generateSnapName(volumeName string) string { namebytes := []byte(name) if len(namebytes) > 31 { name = string(namebytes[0:31]) - Log.Printf("Requested name %s longer than 31 character max, truncated to %s\n", string(namebytes), name) + log.Infof("Requested name %s longer than 31 character max, truncated to %s\n", string(namebytes), name) } return name } @@ -3065,7 +3139,7 @@ func (s *service) DeleteSnapshot( // Display any secrets passed in secrets := req.GetSecrets() for k, v := range secrets { - Log.Printf("secret: %s = %s", k, v) + log.Infof("secret: %s = %s", k, v) } // Validate snapshot volume @@ -3107,7 +3181,7 @@ func (s *service) DeleteSnapshot( } if err != nil { if strings.Contains(err.Error(), sioGatewayFileSystemNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { - Log.Printf("Snapshot %s already deleted on system %s \n", snapID, systemID) + log.Infof("Snapshot %s already deleted on system %s \n", snapID, systemID) return &csi.DeleteSnapshotResponse{}, nil } return nil, err @@ -3115,7 +3189,7 @@ func (s *service) DeleteSnapshot( } if err != nil { if strings.Contains(err.Error(), sioGatewayFileSystemNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { - Log.Printf("Snapshot %s already deleted on system %s \n", snapID, systemID) + log.Infof("Snapshot %s already deleted on system %s \n", snapID, systemID) return &csi.DeleteSnapshotResponse{}, nil } return nil, err @@ -3126,7 +3200,7 @@ func (s *service) DeleteSnapshot( vol, err := s.getVolByID(snapID, systemID) if err != nil { if strings.Contains(err.Error(), "Could not find the volume") || strings.Contains(err.Error(), "must be a hexadecimal number") { - Log.Printf("Snapshot %s already deleted on system %s \n", snapID, systemID) + log.Infof("Snapshot %s already deleted on system %s \n", snapID, systemID) return &csi.DeleteSnapshotResponse{}, nil } return nil, status.Errorf(codes.Internal, "Failed to retrieve snapshot: %s", err.Error()) @@ -3174,7 +3248,8 @@ func (s *service) DeleteSnapshotConsistencyGroup( cgVols := make([]*siotypes.Volume, 0) exposedVols := make([]string, 0) cgID := snapVol.ConsistencyGroupID - Log.Printf("Called DeleteSnapshotConsistencyGroup id: cg %s\n", cgID) + // + log.Infof("Called DeleteSnapshotConsistencyGroup id: cg %s\n", cgID) // make call to cluster to get all volumes // Collect a list of the volumes in the same consistency group (cgVols) @@ -3182,7 +3257,7 @@ func (s *service) DeleteSnapshotConsistencyGroup( sioVols, err := adminClient.GetVolume("", "", "", "", true) for _, vol := range sioVols { if vol.ConsistencyGroupID == cgID { - Log.Printf("Name %s CG %s ID %s", vol.Name, vol.ConsistencyGroupID, vol.ID) + log.Infof("Name %s CG %s ID %s", vol.Name, vol.ConsistencyGroupID, vol.ID) cgVols = append(cgVols, vol) if len(vol.MappedSdcInfo) > 0 { exposedVols = append(exposedVols, fmt.Sprintf("%s (%s) ", vol.Name, vol.ID)) @@ -3197,10 +3272,10 @@ func (s *service) DeleteSnapshotConsistencyGroup( } // If there are no volumes, at least add the original one passed in. if len(cgVols) == 0 { - Log.Printf("Name %s CG %s ID %s", snapVol.Name, snapVol.ConsistencyGroupID, snapVol.ID) + log.Infof("Name %s CG %s ID %s", snapVol.Name, snapVol.ConsistencyGroupID, snapVol.ID) cgVols = append(cgVols, snapVol) } - Log.Printf("CG Snapshots to be deleted: %v\n", cgVols) + log.Infof("CG Snapshots to be deleted: %v\n", cgVols) // Otherwise let's delete them all. If there is an error we fail immediately. s.clearCache() @@ -3219,7 +3294,8 @@ func (s *service) DeleteSnapshotConsistencyGroup( } func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { - Log.Printf("[ControllerExpandVolume] req: %+v", req) + log := log.WithContext(ctx) + log.Infof("[ControllerExpandVolume] req: %+v", req) var reqID string var err error @@ -3271,28 +3347,23 @@ func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle fsName := fs.Name cr := req.GetCapacityRange() - Log.Printf("cr:%d", cr) + log.Infof("cr:%d", cr) requestedSize := int(cr.GetRequiredBytes()) + log.Infof("req.size:%d", requestedSize) + log.Infof("Executing ExpandVolume: reqID=%s, fsName=%s, requestedSize=%d", reqID, fsName, requestedSize) - Log.Printf("req.size:%d", requestedSize) - fields := map[string]interface{}{ - "RequestID": reqID, - "fileSystemName": fsName, - "RequestedSize": requestedSize, - } - Log.WithFields(fields).Info("Executing ExpandVolume with following fields") allocatedSize := fs.SizeTotal - Log.Printf("allocatedsize:%d", allocatedSize) + log.Infof("allocatedsize:%d", allocatedSize) // nil response returned if volume shrink operation is tried if requestedSize < allocatedSize { - Log.Printf("volume shrink tried") + log.Infof("volume shrink tried") return &csi.ControllerExpandVolumeResponse{}, nil } // idempotency check if requestedSize == allocatedSize { - Log.Infof("Idempotent call detected for volume (%s) with requested size (%d) SizeInKb and allocated size (%d) SizeInKb", + log.Infof("Idempotent call detected for volume (%s) with requested size (%d) SizeInKb and allocated size (%d) SizeInKb", fsName, requestedSize, allocatedSize) return &csi.ControllerExpandVolumeResponse{ CapacityBytes: int64(requestedSize), @@ -3306,7 +3377,7 @@ func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle } if err := system.ModifyFileSystem(&siotypes.FSModify{Size: requestedSize}, fsID); err != nil { - Log.Errorf("NFS volume expansion failed with error: %s", err.Error()) + log.Errorf("NFS volume expansion failed with error: %s", err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -3316,14 +3387,14 @@ func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle if isQuotaEnabled && fs.IsQuotaEnabled { treeQuota, err := system.GetTreeQuotaByFSID(fsID) if err != nil { - Log.Errorf("Fetching tree quota for NFS volume failed, error: %s", err.Error()) + log.Errorf("Fetching tree quota for NFS volume failed, error: %s", err.Error()) return nil, status.Error(codes.Internal, err.Error()) } // Modify Tree Quota updatedSoftLimit := treeQuota.SoftLimit * (requestedSize / treeQuota.HardLimit) treeQuotaID := treeQuota.ID - Log.Infof("Modifying tree quota ID %s for NFS volume ID: %s", treeQuotaID, fsID) + log.Infof("Modifying tree quota ID %s for NFS volume ID: %s", treeQuotaID, fsID) quotaModify := &siotypes.TreeQuotaModify{ HardLimit: requestedSize, SoftLimit: updatedSoftLimit, @@ -3331,10 +3402,10 @@ func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle err = system.ModifyTreeQuota(quotaModify, treeQuotaID) if err != nil { - Log.Errorf("Modifying tree quota for NFS volume failed, error: %s", err.Error()) + log.Errorf("Modifying tree quota for NFS volume failed, error: %s", err.Error()) return nil, status.Error(codes.Internal, err.Error()) } - Log.Infof("Tree quota modified successfully.") + log.Infof("Tree quota modified successfully.") } csiResp := &csi.ControllerExpandVolumeResponse{ @@ -3370,27 +3441,27 @@ func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle volName := vol.Name cr := req.GetCapacityRange() - Log.Printf("cr:%d", cr) + log.Infof("cr:%d", cr) requestedSize, err := validateVolSize(cr) if err != nil { return nil, err } - Log.Printf("req.size:%d", requestedSize) + log.Infof("req.size:%d", requestedSize) fields := map[string]interface{}{ "RequestID": reqID, "VolumeName": volName, "RequestedSize": requestedSize, } - Log.WithFields(fields).Info("Executing ExpandVolume with following fields") + log.WithFields(fields).Info("Executing ExpandVolume with following fields") allocatedSize := int64(vol.SizeInKb) - Log.Printf("allocatedsize:%d", allocatedSize) + log.Infof("allocatedsize:%d", allocatedSize) if requestedSize < allocatedSize { return &csi.ControllerExpandVolumeResponse{}, nil } if requestedSize == allocatedSize { - Log.Infof("Idempotent call detected for volume (%s) with requested size (%d) SizeInKb and allocated size (%d) SizeInKb", + log.Infof("Idempotent call detected for volume (%s) with requested size (%d) SizeInKb and allocated size (%d) SizeInKb", volName, requestedSize, allocatedSize) return &csi.ControllerExpandVolumeResponse{ CapacityBytes: requestedSize * bytesInKiB, @@ -3403,13 +3474,13 @@ func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle tgtVol.Volume = vol err = tgtVol.SetVolumeSize(strconv.Itoa(int(reqSize))) if err != nil { - Log.Errorf("Failed to execute ExpandVolume() with error (%s)", err.Error()) + log.Errorf("Failed to execute ExpandVolume() with error (%s)", err.Error()) return nil, status.Error(codes.Internal, err.Error()) } // If volume is marked for replication, remove the replication pair first. if vol.VolumeReplicationState != "UnmarkedForReplication" { - Log.Printf("[ControllerExpandVolume] - vol: %+v", vol) + log.Infof("[ControllerExpandVolume] - vol: %+v", vol) err := s.expandReplicationPair(ctx, req, systemID, volID) if err != nil { return nil, status.Errorf(codes.Internal, @@ -3483,17 +3554,17 @@ func (s *service) Clone(req *csi.CreateVolumeRequest, noVolErrString1 := "Error: problem finding volume: Volume not found" noVolErrString2 := "Error: problem finding volume: Could not find the volume" if (err != nil) && !(strings.Contains(err.Error(), noVolErrString1) || strings.Contains(err.Error(), noVolErrString2)) { - Log.Printf("[Clone] Idempotency check: GetVolume returned error: %s", err.Error()) + log.Infof("[Clone] Idempotency check: GetVolume returned error: %s", err.Error()) return nil, status.Errorf(codes.Internal, "Failed to create clone -- GetVolume returned unexpected error: %s", err.Error()) } for _, vol := range existingVols { if vol.Name == name && vol.StoragePoolID == srcVol.StoragePoolID { - Log.Printf("Requested volume %s already exists", name) + log.Infof("Requested volume %s already exists", name) csiVolume := s.getCSIVolume(vol, systemID) csiVolume.ContentSource = req.GetVolumeContentSource() copyInterestingParameters(req.GetParameters(), csiVolume.VolumeContext) - Log.Printf("Requested volume (from clone) already exists %s (%s) storage pool %s", + log.Infof("Requested volume (from clone) already exists %s (%s) storage pool %s", csiVolume.VolumeContext["Name"], csiVolume.VolumeId, csiVolume.VolumeContext["StoragePoolName"]) return &csi.CreateVolumeResponse{Volume: csiVolume}, nil @@ -3504,13 +3575,22 @@ func (s *service) Clone(req *csi.CreateVolumeRequest, snapshotDefs := make([]*siotypes.SnapshotDef, 0) snapDef := &siotypes.SnapshotDef{VolumeID: sourceVolID, SnapshotName: name} snapshotDefs = append(snapshotDefs, snapDef) - snapParam := &siotypes.SnapshotVolumesParam{SnapshotDefs: snapshotDefs, AccessMode: "ReadWrite"} + snapResponse := &siotypes.SnapshotVolumesResp{} // Create snapshot system := s.systems[systemID] - snapResponse, err := system.CreateSnapshotConsistencyGroup(snapParam) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to call CreateSnapshotConsistencyGroup to clone volume: %s", err.Error()) + if srcVol.GenType == "EC" { + snapParam := &siotypes.CreateSnapshotParam{SnapshotDefs: snapshotDefs} + snapResponse, err = system.CreateThinClone(snapParam) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to call CreateThinClone to clone volume: %s", err.Error()) + } + } else { + snapParam := &siotypes.SnapshotVolumesParam{SnapshotDefs: snapshotDefs, AccessMode: "ReadWrite"} + snapResponse, err = system.CreateSnapshotConsistencyGroup(snapParam) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to call CreateSnapshotConsistencyGroup to clone volume: %s", err.Error()) + } } if len(snapResponse.VolumeIDList) != 1 { @@ -3530,7 +3610,7 @@ func (s *service) Clone(req *csi.CreateVolumeRequest, csiVolume.ContentSource = req.GetVolumeContentSource() copyInterestingParameters(req.GetParameters(), csiVolume.VolumeContext) - Log.Printf("Volume (from volume clone) %s (%s) storage pool %s", + log.Infof("Volume (from volume clone) %s (%s) storage pool %s", csiVolume.VolumeContext["Name"], csiVolume.VolumeId, csiVolume.VolumeContext["storagePoolName"]) return &csi.CreateVolumeResponse{Volume: csiVolume}, nil @@ -3614,7 +3694,7 @@ func (s *service) CreateReplicationConsistencyGroup(systemID string, name string if err != nil { // Handle the case where it already exists. if !strings.EqualFold(err.Error(), sioReplicationGroupExists) { - Log.Printf("Replication Creation Error: %s", err.Error()) + log.Infof("Replication Creation Error: %s", err.Error()) return nil, err } } @@ -3629,7 +3709,7 @@ func (s *service) CreateReplicationConsistencyGroup(systemID string, name string // RCG already exists, find it on the array. for _, rcg := range rcgs { if rcg.Name == name && rcg.ProtectionDomainID == locatProtectionDomain && rcg.RemoteProtectionDomainID == remoteProtectionDomain { - Log.Printf("Replication Group Found: %s, %s", rcg.ID, rcg.RemoteID) + log.Infof("Replication Group Found: %s, %s", rcg.ID, rcg.RemoteID) id = rcg.ID break } @@ -3667,7 +3747,7 @@ func (s *service) CreateReplicationPair(systemID string, name string, if err != nil { // Handle the case where it already exists. if !strings.EqualFold(err.Error(), sioReplicationPairExists) { - Log.Printf("Replication Pair Creation Error: %s", err.Error()) + log.Infof("Replication Pair Creation Error: %s", err.Error()) return nil, err } } @@ -3680,7 +3760,7 @@ func (s *service) CreateReplicationPair(systemID string, name string, for _, pair := range pairs { if pair.Name == name { - Log.Printf("Replication Pair Found: %+v", pair) + log.Infof("Replication Pair Found: %+v", pair) response = pair break } @@ -3706,7 +3786,7 @@ func (s *service) DeleteReplicationConsistencyGroup(systemID string, groupID str group, err := adminClient.GetReplicationConsistencyGroupByID(groupID) if err != nil { - Log.Printf("Replication Deletion Error: %s", err.Error()) + log.Infof("Replication Deletion Error: %s", err.Error()) return err } @@ -3734,7 +3814,7 @@ func (s *service) ExecuteFailoverOnReplicationGroup(client *goscaleio.Client, gr rcg := goscaleio.NewReplicationConsistencyGroup(client) rcg.ReplicationConsistencyGroup = group - Log.Printf("[ExecuteFailoverOnReplicationGroup]: Executing Failover command") + log.Infof("[ExecuteFailoverOnReplicationGroup]: Executing Failover command") return rcg.ExecuteFailoverOnReplicationGroup() } @@ -3743,7 +3823,7 @@ func (s *service) ExecuteSwitchoverOnReplicationGroup(client *goscaleio.Client, rcg := goscaleio.NewReplicationConsistencyGroup(client) rcg.ReplicationConsistencyGroup = group - Log.Printf("[ExecuteSwitchoverOnReplicationGroup]: Executing Switchover (Unplanned Failover)") + log.Infof("[ExecuteSwitchoverOnReplicationGroup]: Executing Switchover (Unplanned Failover)") return rcg.ExecuteSwitchoverOnReplicationGroup(false) } @@ -3752,7 +3832,7 @@ func (s *service) ExecuteReverseOnReplicationGroup(client *goscaleio.Client, gro rcg := goscaleio.NewReplicationConsistencyGroup(client) rcg.ReplicationConsistencyGroup = group - Log.Printf("[ExecuteReverseOnReplicationGroup]: Executing Reverse (Reprotect Local)") + log.Infof("[ExecuteReverseOnReplicationGroup]: Executing Reverse (Reprotect Local)") return rcg.ExecuteReverseOnReplicationGroup() } @@ -3761,10 +3841,10 @@ func (s *service) ExecuteResumeOnReplicationGroup(client *goscaleio.Client, grou rcg := goscaleio.NewReplicationConsistencyGroup(client) rcg.ReplicationConsistencyGroup = group - Log.Printf("[ExecuteReverseOnReplicationGroup]: Resuming Replication Group") + log.Infof("[ExecuteReverseOnReplicationGroup]: Resuming Replication Group") if failover { - Log.Printf("[ExecuteReverseOnReplicationGroup]: In Failover, Restoring...") + log.Infof("[ExecuteReverseOnReplicationGroup]: In Failover, Restoring...") return rcg.ExecuteRestoreOnReplicationGroup() } @@ -3775,7 +3855,7 @@ func (s *service) ExecutePauseOnReplicationGroup(client *goscaleio.Client, group rcg := goscaleio.NewReplicationConsistencyGroup(client) rcg.ReplicationConsistencyGroup = group - Log.Printf("[ExecutePauseOnReplicationGroup]: Pause Replication Group") + log.Infof("[ExecutePauseOnReplicationGroup]: Pause Replication Group") return rcg.ExecutePauseOnReplicationGroup() } @@ -3784,7 +3864,7 @@ func (s *service) ExecuteSyncOnReplicationGroup(client *goscaleio.Client, group rcg := goscaleio.NewReplicationConsistencyGroup(client) rcg.ReplicationConsistencyGroup = group - Log.Printf("[ExecuteSyncOnReplicationGroup]: Executing SyncNow") + log.Infof("[ExecuteSyncOnReplicationGroup]: Executing SyncNow") return rcg.ExecuteSyncOnReplicationGroup() } @@ -3802,16 +3882,42 @@ func (s *service) createProbeContextWithDeadline(ctx context.Context) (context.C defaultProbeDeadline := time.Now().Add(s.opts.probeTimeout) probeDeadline, ok := ctx.Deadline() if !ok { - Log.Println("Probe deadline not in context, using default") + log.Infof("Probe deadline not in context, using default") probeDeadline = time.Now().Add(s.opts.probeTimeout) } // Set the deadline to be the lowest of the two times. if probeDeadline.After(defaultProbeDeadline) { - Log.Printf("Original Probe Deadline %s is greater than defaultProbeDeadline %s, setting to default", probeDeadline, defaultProbeDeadline) + log.Infof("Original Probe Deadline %s is greater than defaultProbeDeadline %s, setting to default", probeDeadline, defaultProbeDeadline) probeDeadline = defaultProbeDeadline } // Calculate the new deadline by subtracting the desired duration return context.WithDeadline(ctx, probeDeadline) } + +func (s *service) IsReplicationEnabledOnPlatforms(sourceSystemID, remoteSystemID, sourceGenType string) (bool, error) { + // check replication supports on source and Target + if remoteSystemID != "" { + log.Infof("Checking if Replication is Enabled on the Platform level. SourceSystemId: %s RemoteSystemId: %s", sourceSystemID, remoteSystemID) + + if s.isReplicationNotSupported(sourceGenType) { + log.Infof("Replication is not supported on this System %s with GenType: %s", sourceSystemID, sourceGenType) + return false, status.Errorf(codes.InvalidArgument, "Replication is not supported on this System %s with GenType: %s", sourceSystemID, sourceGenType) + } + + platformInfo, err := s.GetPlatformInfo(remoteSystemID) + if err != nil { + return false, err + } + + if s.isReplicationNotSupported(platformInfo.GenType) { + log.Infof("Replication is not supported on this System %s with GenType: %s", remoteSystemID, platformInfo.GenType) + return false, status.Errorf(codes.InvalidArgument, "Replication is not supported on this System %s with GenType: %s", remoteSystemID, platformInfo.GenType) + } + + log.Infof("Checked: Replication is Enabled on the Platform level. SourceSystemId: %s RemoteSystemId: %s", sourceSystemID, remoteSystemID) + } + + return true, nil +} diff --git a/service/controller_extension.go b/service/controller_extension.go new file mode 100644 index 00000000..557982f1 --- /dev/null +++ b/service/controller_extension.go @@ -0,0 +1,148 @@ +package service + +import ( + "fmt" + + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + sio "github.com/dell/goscaleio" + siotypes "github.com/dell/goscaleio/types/v1" +) + +// SystemCapacityCalculator is an interface for calculating system capacity. +type SystemCapacityCalculator interface { + GetSystemCapacity(systemID string) (int64, error) + GetStoragePoolCapacity(systemID, protectionDomain, spName string) (int64, error) +} + +// PowerFlexGen1 implements SystemCapacityCalculator for Gen1 systems. +type PowerFlexGen1 struct { + client *sio.Client + system *sio.System + s *service +} + +func (p *PowerFlexGen1) GetSystemCapacity(systemID string) (int64, error) { + log.Debugf("Getting system capacity for system ID: %s", systemID) + stats, err := p.system.GetStatistics() + if err != nil { + return 0, err + } + if !p.s.opts.Thick { + return int64(stats.VolumeAllocationLimitInKb * bytesInKiB), nil + } + return int64(stats.CapacityAvailableForVolumeAllocationInKb * bytesInKiB), nil +} + +func (p *PowerFlexGen1) GetStoragePoolCapacity(systemID, protectionDomain, spName string) (int64, error) { + log.Debugf("Getting storage pool capacity for system ID: %s, storage pool: %s", systemID, spName) + pdID, err := p.s.getProtectionDomainIDFromName(systemID, protectionDomain) + if err != nil { + return 0, err + } + sp, err := p.client.FindStoragePool(systemID, spName, "", pdID) + if err != nil { + return 0, status.Errorf(codes.Internal, + "unable to look up storage pool: %s on system: %s, err: %s", + spName, systemID, err.Error()) + } + spc := sio.NewStoragePoolEx(p.client, sp) + stats, err := spc.GetStatistics() + if err != nil { + return 0, err + } + + if !p.s.opts.Thick { + return int64(stats.VolumeAllocationLimitInKb * bytesInKiB), nil + } + + return int64(stats.CapacityAvailableForVolumeAllocationInKb * bytesInKiB), nil +} + +// PowerFlexGen2 implements SystemCapacityCalculator for Gen2 (GenTypeEC) systems. +type PowerFlexGen2 struct { + client *sio.Client + system *sio.System + s *service +} + +func (p *PowerFlexGen2) GetSystemCapacity(systemID string) (int64, error) { + log.Debugf("Getting system capacity for system ID: %s", systemID) + metrics, err := p.client.GetMetrics("system", []string{systemID}) + if err != nil { + return 0, status.Errorf(codes.Internal, + "unable to get system stats on system: %s, err: %s", systemID, err.Error()) + } + + if len(metrics.Resources) == 0 { + return 0, fmt.Errorf("no metrics found for system %s", systemID) + } + return int64(getMetric(metrics.Resources[0].Metrics, "physical_free")), nil +} + +func (p *PowerFlexGen2) GetStoragePoolCapacity(systemID, protectionDomain, spName string) (int64, error) { + log.Debugf("Getting storage pool capacity for system ID: %s, storage pool: %s", systemID, spName) + pdID, err := p.s.getProtectionDomainIDFromName(systemID, protectionDomain) + if err != nil { + return 0, err + } + + sp, err := p.client.FindStoragePool(systemID, spName, "", pdID) + if err != nil { + log.Errorf("Error finding storage pool: %s", err) + return 0, status.Errorf(codes.Internal, + "unable to look up storage pool: %s on system: %s, err: %s", + spName, systemID, err.Error()) + } + + metrics, err := p.client.GetMetrics("storage_pool", []string{sp.ID}) + if err != nil { + return 0, err + } + + if len(metrics.Resources) == 0 { + return 0, fmt.Errorf("no metrics found for storage pool %s", spName) + } + + return int64(getMetric(metrics.Resources[0].Metrics, "physical_free")), nil +} + +// getSystemCapacityCalculator returns a SystemCapacityCalculator based on the system's generation type. +func (s *service) getSystemCapacityCalculator(systemID string, service *service) (SystemCapacityCalculator, error) { + adminClient := s.adminClients[systemID] + system := s.systems[systemID] + if adminClient == nil || system == nil { + return nil, fmt.Errorf("can't find adminClient or system by id %s", systemID) + } + + platformInfo, err := s.GetPlatformInfo(systemID) + if err != nil { + return nil, err + } + + if platformInfo.GenType == siotypes.GenTypeEC { + log.Infof("GetType: %s", siotypes.GenTypeEC) + return &PowerFlexGen2{client: adminClient, system: system, s: service}, nil + } + return &PowerFlexGen1{client: adminClient, system: system, s: service}, nil +} + +// Gets capacity of a given storage system. When storage pool name is provided, gets capcity of this storage pool only. +func (s *service) getSystemCapacity(ctx context.Context, systemID, protectionDomain string, spName ...string) (int64, error) { + log.Infof("Get capacity for system: %s, pool %s", systemID, spName) + + if err := s.requireProbe(ctx, systemID); err != nil { + return 0, err + } + + calculator, err := s.getSystemCapacityCalculator(systemID, s) + if err != nil { + return 0, err + } + if len(spName) > 0 && spName[0] != "" { + return calculator.GetStoragePoolCapacity(systemID, protectionDomain, spName[0]) + } + return calculator.GetSystemCapacity(systemID) +} diff --git a/service/controller_test.go b/service/controller_test.go index 3fa6f13d..0ddd2773 100644 --- a/service/controller_test.go +++ b/service/controller_test.go @@ -14,14 +14,27 @@ package service import ( + "encoding/json" "errors" + "fmt" + "math" + "net/http" + "net/http/httptest" + "reflect" + "strings" "sync" + "sync/atomic" "testing" + "time" - csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/dell/goscaleio" sio "github.com/dell/goscaleio" siotypes "github.com/dell/goscaleio/types/v1" + csi "github.com/container-storage-interface/spec/lib/go/csi" "golang.org/x/net/context" + "golang.org/x/oauth2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func Test_service_getZoneFromZoneLabelKey(t *testing.T) { @@ -264,3 +277,543 @@ func Test_service_getSystemIDFromZoneLabelKey(t *testing.T) { }) } } + +func Test_service_createVolumeFromSnapshot(t *testing.T) { + tests := []struct { + name string + req *csi.CreateVolumeRequest + snapshotSource *csi.VolumeContentSource_SnapshotSource + name1 string + sizeInKbytes int64 + storagePool string + want *csi.CreateVolumeResponse + wantErr bool + }{ + { + name: "create volume from snapshot", + req: &csi.CreateVolumeRequest{ + Name: "volume-1", + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1, + }, + }, + snapshotSource: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: "sys-1", + }, + name1: "volume-1", + sizeInKbytes: 1024, + storagePool: "pool123", + want: &csi.CreateVolumeResponse{}, + wantErr: true, + }, + } + for _, tt := range tests { + clients1 := make(map[string]*sio.Client) + + client1, _ := sio.NewClientWithArgs("10.1.1.1", "", math.MaxInt64, false, false) + clients1["sys-1"] = client1 + t.Run(tt.name, func(t *testing.T) { + s := &service{ + opts: Opts{}, + connectedSystemNameToID: map[string]string{ + "snapshot": "sys-1", + }, + + adminClients: map[string]*sio.Client{ + "sys": client1, + }, + } + + getVolByIDFunc = func(_ *service, id string, _ string) (*siotypes.Volume, error) { + return &siotypes.Volume{ + ID: id, + SizeInKb: 1024, + StoragePoolID: "pool123", + Name: "mock-volume", + GenType: "EC", + }, nil + } + + defer func() { + // Restore original function after test + getVolByIDFunc = func(s *service, id string, systemID string) (*siotypes.Volume, error) { + return s.getVolByID(id, systemID) + } + }() + + getStoragePoolNameFromIDFunc = func(_ *service, _ string, _ string) string { + return "pool123" + } + defer func() { + getStoragePoolNameFromIDFunc = func(s *service, systemID string, id string) string { + return s.getStoragePoolNameFromID(systemID, id) + } + }() + + getVolumeFunc = func(_ *goscaleio.Client, _, _, _, _ string, _ bool) ([]*siotypes.Volume, error) { + return []*siotypes.Volume{ + { + ID: "volume-1", + SizeInKb: 1024, + StoragePoolID: "pool123", + Name: "mock-volume", + }, + }, nil + } + defer func() { + getVolumeFunc = func(adminClient *goscaleio.Client, a, b, c, name string, e bool) ([]*siotypes.Volume, error) { + return adminClient.GetVolume(a, b, c, name, e) + } + }() + + createThinCloneFunc = func(_ *goscaleio.System, _ *siotypes.CreateSnapshotParam) (*siotypes.SnapshotVolumesResp, error) { + return &siotypes.SnapshotVolumesResp{ + VolumeIDList: []string{"volume-1"}, + SnapshotGroupID: "snap-1", + }, fmt.Errorf("error in createThinClone") + } + + defer func() { + createThinCloneFunc = func(system *goscaleio.System, snapParam *siotypes.CreateSnapshotParam) (*siotypes.SnapshotVolumesResp, error) { + return system.CreateThinClone(snapParam) + } + }() + + _, gotErr := s.createVolumeFromSnapshot(tt.req, tt.snapshotSource, tt.name, tt.sizeInKbytes, tt.storagePool) + + if tt.wantErr { + if gotErr == nil { + t.Fatalf("createVolumeFromSnapshot() succeeded unexpectedly, expected error") + } + expectedErrMsg := "Failed to call CreateThinClone to create volume from snapshot" + if !strings.Contains(gotErr.Error(), expectedErrMsg) { + t.Errorf("createVolumeFromSnapshot() error = %v, want error containing %q", gotErr, expectedErrMsg) + } + return + } + }) + } +} + +func Test_service_CreateSnapshot(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + ctx context.Context + req *csi.CreateSnapshotRequest + want *csi.CreateSnapshotResponse + wantErr bool + }{ + { + name: "create snapshot", + ctx: context.Background(), + req: &csi.CreateSnapshotRequest{ + SourceVolumeId: "volume-1", + Name: "snap-1", + Parameters: nil, + }, + want: &csi.CreateSnapshotResponse{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // TODO: construct the receiver type. + clients1 := make(map[string]*sio.Client) + + client1, _ := sio.NewClientWithArgs("10.1.1.1", "", math.MaxInt64, false, false) + clients1["volume"] = client1 + s := &service{ + opts: Opts{}, + connectedSystemNameToID: map[string]string{ + "snapshot": "sys-1", + }, + + adminClients: map[string]*sio.Client{ + "volume": client1, + }, + systems: map[string]*sio.System{ + "volume": {}, + }, + } + getVolumeFunc = func(_ *goscaleio.Client, _, _, _, _ string, _ bool) ([]*siotypes.Volume, error) { + return []*siotypes.Volume{}, nil + } + defer func() { + getVolumeFunc = func(adminClient *goscaleio.Client, a, b, c, name string, e bool) ([]*siotypes.Volume, error) { + return adminClient.GetVolume(a, b, c, name, e) + } + }() + + getVolByIDFunc = func(_ *service, id string, _ string) (*siotypes.Volume, error) { + return &siotypes.Volume{ + ID: id, + SizeInKb: 1024, + StoragePoolID: "pool123", + Name: "mock-volume", + GenType: "EC", + }, nil + } + + defer func() { + // Restore original function after test + getVolByIDFunc = func(s *service, id string, systemID string) (*siotypes.Volume, error) { + return s.getVolByID(id, systemID) + } + }() + + createSnapshotFunc = func(_ *goscaleio.System, _ *siotypes.CreateSnapshotParam) (*siotypes.SnapshotVolumesResp, error) { + return &siotypes.SnapshotVolumesResp{ + VolumeIDList: []string{"volume-1"}, + SnapshotGroupID: "snap-1", + }, fmt.Errorf("error in createThinClone") + } + + defer func() { + createSnapshotFunc = func(system *goscaleio.System, snapParam *siotypes.CreateSnapshotParam) (*siotypes.SnapshotVolumesResp, error) { + return system.CreateSnapshot(snapParam) + } + }() + got, gotErr := s.CreateSnapshot(tt.ctx, tt.req) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("CreateSnapshot() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("CreateSnapshot() succeeded unexpectedly") + } + if true { + t.Errorf("CreateSnapshot() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExtractIP(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + endpoint string + wantIP string + wantErr bool + errContains string + }{ + { + name: "IPv4 with scheme and port", + endpoint: "https://192.168.1.10:8443", + wantIP: "192.168.1.10", + }, + { + name: "Hostname not IP", + endpoint: "https://example.com", + wantErr: true, + errContains: "not a valid IP: example.com", + }, + { + name: "Malformed URL", + endpoint: "http://%", + wantErr: true, + errContains: "parse", // err originates from url.Parse + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := ExtractIP(tc.endpoint) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got nil (got=%q)", got) + } + if tc.errContains != "" && !strings.Contains(err.Error(), tc.errContains) { + t.Fatalf("expected error to contain %q, got %q", tc.errContains, err.Error()) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tc.wantIP { + t.Fatalf("ip mismatch: want %q, got %q", tc.wantIP, got) + } + }) + } +} + +type mockClient struct { + goscaleio.Client + refreshCalls int32 +} + +func (m *mockClient) RefreshPowerFlexToken(_ *goscaleio.ConfigConnect) (*oauth2.Token, error) { + atomic.AddInt32(&m.refreshCalls, 1) + return &oauth2.Token{ + AccessToken: "new-access", + RefreshToken: "new-refresh", + TokenType: "Bearer", + ExpiresIn: 5, // seconds + }, nil +} + +func TestRefreshPowerFlexTokenNew(t *testing.T) { + tests := []struct { + name string + client *mockClient + pfmpIP string + ciamClientID string + ciamClientSecret string + insecure bool + PowerFlexToken *oauth2.Token + checkInterval time.Duration + expectRefresh bool + mockHTTPResponse func(w http.ResponseWriter, r *http.Request) + }{ + { + name: "token refresh with invalid PowerFlexToken", + client: &mockClient{}, + pfmpIP: "http://example.com", + ciamClientID: "client-id", + ciamClientSecret: "client-secret", + insecure: false, + PowerFlexToken: nil, + checkInterval: 1 * time.Minute, + expectRefresh: true, + mockHTTPResponse: func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"access_token": "new-access", "token_type": "Bearer", "expires_in": 3600}`)) + }, + }, + { + name: "token refresh with expired PowerFlexToken", + client: &mockClient{}, + pfmpIP: "http://example.com", + ciamClientID: "client-id", + ciamClientSecret: "client-secret", + insecure: false, + PowerFlexToken: &oauth2.Token{ + AccessToken: "old-access", + RefreshToken: "old-refresh", + TokenType: "Bearer", + ExpiresIn: -1, // expired + }, + checkInterval: 1 * time.Minute, + expectRefresh: true, + mockHTTPResponse: func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"access_token": "new-access", "token_type": "Bearer", "expires_in": 3600}`)) + }, + }, + { + name: "token refresh with valid PowerFlexToken", + client: &mockClient{}, + pfmpIP: "http://example.com", + ciamClientID: "client-id", + ciamClientSecret: "client-secret", + insecure: false, + PowerFlexToken: &oauth2.Token{ + AccessToken: "valid-access", + RefreshToken: "valid-refresh", + TokenType: "Bearer", + ExpiresIn: 3600, // valid + }, + checkInterval: 1 * time.Minute, + expectRefresh: false, + mockHTTPResponse: func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"access_token": "new-access", "token_type": "Bearer", "expires_in": 3600}`)) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tt.mockHTTPResponse(w, r) + })) + defer ts.Close() + + client := &mockClient{} + pfmpIP := ts.URL + go func() { + ctx := context.Background() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, pfmpIP+"/auth/realms/powerflex/protocol/openid-connect/token", strings.NewReader(`grant_type=refresh_token&refresh_token=refresh-token&client_id=client-id&client_secret=client-secret`)) + if err != nil { + t.Errorf("error creating request: %v", err) + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Errorf("error sending request: %v", err) + return + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) + return + } + var token struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` + } + err = json.NewDecoder(resp.Body).Decode(&token) + if err != nil { + t.Errorf("error decoding response: %v", err) + return + } + atomic.AddInt32(&client.refreshCalls, 1) + }() + + // Let it run briefly to execute one or more iterations. + time.Sleep(200 * time.Millisecond) + + if tt.expectRefresh && atomic.LoadInt32(&client.refreshCalls) == 0 { + t.Errorf("expected refresh calls, got %d", atomic.LoadInt32(&client.refreshCalls)) + } + }) + } +} + +// helper to build a valid baseline and then override fields +func validArray() *ArrayConnectionData { + return &ArrayConnectionData{ + OidcClientID: "oidc-client-id", + OidcClientSecret: "oidc-client-secret", + CiamClientID: "ciam-client-id", + CiamClientSecret: "ciam-client-secret", + Issuer: "https://issuer.example.com", + } +} + +func TestOidcPrechecks(t *testing.T) { + tests := []struct { + name string + array *ArrayConnectionData + wantCode codes.Code // expected gRPC status code (codes.OK if no error) + wantSubstr string // substring expected in error message ("" if none) + }{ + { + name: "missing OidcClientID", + array: func() *ArrayConnectionData { a := validArray(); a.OidcClientID = ""; return a }(), + wantCode: codes.FailedPrecondition, + wantSubstr: "missing OidcClientID", + }, + { + name: "missing OidcClientSecret", + array: func() *ArrayConnectionData { a := validArray(); a.OidcClientSecret = ""; return a }(), + wantCode: codes.FailedPrecondition, + wantSubstr: "missing OidcClientSecret", + }, + { + name: "missing CiamClientID", + array: func() *ArrayConnectionData { a := validArray(); a.CiamClientID = ""; return a }(), + wantCode: codes.FailedPrecondition, + wantSubstr: "missing CiamClientID", + }, + { + name: "missing CiamClientSecret", + array: func() *ArrayConnectionData { a := validArray(); a.CiamClientSecret = ""; return a }(), + wantCode: codes.FailedPrecondition, + wantSubstr: "missing CiamClientSecret", + }, + { + name: "missing Issuer", + array: func() *ArrayConnectionData { a := validArray(); a.Issuer = ""; return a }(), + wantCode: codes.FailedPrecondition, + wantSubstr: "missing Issuer", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := oidcPrechecks(tt.array) + + if tt.wantCode == codes.OK { + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + return + } + + if err == nil { + t.Fatalf("expected error, got nil") + } + + gotCode := status.Code(err) + if gotCode != tt.wantCode { + t.Fatalf("expected gRPC code %v, got %v (err=%v)", tt.wantCode, gotCode, err) + } + + if tt.wantSubstr != "" && !strings.Contains(err.Error(), tt.wantSubstr) { + t.Fatalf("expected error to contain %q, got %q", tt.wantSubstr, err.Error()) + } + }) + } +} + +func TestParseScopes(t *testing.T) { + tests := []struct { + name string + input string + expected []string + }{ + { + name: "basic comma separated", + input: "openid,pflex", + expected: []string{"openid", "pflex"}, + }, + { + name: "with spaces and duplicates", + input: "openid, pflex ,openid", + expected: []string{"openid", "pflex"}, + }, + { + name: "empty input returns nil", + input: "", + expected: nil, + }, + { + name: "only delimiters/whitespace returns nil", + input: " , , ", + expected: nil, + }, + { + name: "commas only returns nil", + input: ",,", + expected: nil, + }, + { + name: "input with empty entries gets filtered", + input: "openid,,pflex, ", + expected: []string{"openid", "pflex"}, + }, + { + name: "dedupe repeated scope", + input: "openid,openid", + expected: []string{"openid"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ParseScopes(tt.input) + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("ParseScopes(%q) = %#v; want %#v", tt.input, result, tt.expected) + } + }) + } +} diff --git a/service/csi_extension_server.go b/service/csi_extension_server.go index 8c32fa78..90c3626d 100644 --- a/service/csi_extension_server.go +++ b/service/csi_extension_server.go @@ -19,11 +19,11 @@ import ( "strconv" "strings" - csi "github.com/container-storage-interface/spec/lib/go/csi" podmon "github.com/dell/dell-csi-extensions/podmon" volumeGroupSnapshot "github.com/dell/dell-csi-extensions/volumeGroupSnapshot" sio "github.com/dell/goscaleio" siotypes "github.com/dell/goscaleio/types/v1" + csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -31,10 +31,13 @@ import ( const ( // ExistingGroupID group id on powerflex array ExistingGroupID = "existingSnapshotGroupID" + + // ArrayStatus is the endPoint for polling to check array status + ArrayStatus = "/array-status" ) func (s *service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmon.ValidateVolumeHostConnectivityRequest) (*podmon.ValidateVolumeHostConnectivityResponse, error) { - Log.Infof("ValidateVolumeHostConnectivity called %+v", req) + log.Infof("ValidateVolumeHostConnectivity called %+v", req) rep := &podmon.ValidateVolumeHostConnectivityResponse{ Messages: make([]string, 0), } @@ -45,8 +48,8 @@ func (s *service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmo return rep, nil } - // The NodeID for the VxFlex OS is the SdcGUID field. - if req.GetNodeId() == "" { + nodeID := req.GetNodeId() + if nodeID == "" { return nil, status.Errorf(codes.InvalidArgument, "The NodeID is a required field") } @@ -65,15 +68,48 @@ func (s *service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmo return nil, err } - // First- check to see if the SDC is Connected or Disconnected. - // Then retrieve the SDC and seet the connection state - sdc, err := s.systems[systemID].FindSdc("SdcGUID", req.GetNodeId()) + // First- check to see if the host is Connected or Disconnected. + _, hostType, err := s.getHostIDAndType(systemID, nodeID) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "NodeID is invalid: %s - there is no corresponding SDC, error: %s", req.GetNodeId(), err.Error()) + return nil, status.Errorf(codes.Internal, + "error getting host ID and type for nodeID %s: %s", nodeID, err.Error()) + } + if hostType == NVMeTCP { + var message string + rep.Connected = false + nodeIP := s.GetNodeIPByCSINodeID(nodeID) + if len(nodeIP) == 0 { + log.Errorf("could not resolve IP address for nodeID=%s", nodeID) + return nil, fmt.Errorf("failed to resolve IP address for nodeID=%s", nodeID) + } + + // Query the podmon API on the node to get the connectivity status + url := "http://" + nodeIP + s.opts.PodmonPort + ArrayStatus + "/" + systemID + connected, err := s.QueryArrayStatus(ctx, url) + if err != nil { + message = fmt.Sprintf("connectivity unknown for array %s to node %s due to %s", systemID, nodeID, err) + log.Error(message) + rep.Messages = append(rep.Messages, message) + log.Errorf("%s", err.Error()) + } + + if connected { + rep.Connected = true + message = fmt.Sprintf("array %s is connected to node %s", systemID, nodeID) + } else { + message = fmt.Sprintf("array %s is not connected to node %s", systemID, nodeID) + } + log.Info(message) + rep.Messages = append(rep.Messages, message) + } else { + sdc, err := s.systems[systemID].FindSdc("SdcGUID", req.GetNodeId()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "NodeID is invalid: %s - there is no corresponding SDC, error: %s", req.GetNodeId(), err.Error()) + } + connectionState := sdc.Sdc.MdmConnectionState + rep.Messages = append(rep.Messages, fmt.Sprintf("SDC connection state: %s", connectionState)) + rep.Connected = (connectionState == "Connected") } - connectionState := sdc.Sdc.MdmConnectionState - rep.Messages = append(rep.Messages, fmt.Sprintf("SDC connection state: %s", connectionState)) - rep.Connected = (connectionState == "Connected") // Second- check to see if the Volumes have any I/O in the recent past. for _, volID := range req.GetVolumeIds() { @@ -95,41 +131,84 @@ func (s *service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmo rep.Messages = append(rep.Messages, fmt.Sprintf("Could not retrieve volume: %s, error: %s", volID, err.Error())) continue } + + adminClient := s.adminClients[systemID] + // Get the volume statistics - volume := sio.NewVolume(s.adminClients[systemID]) + volume := sio.NewVolume(adminClient) volume.Volume = vol - stats, err := volume.GetVolumeStatistics() + + platformInfo, err := s.GetPlatformInfo(systemID) if err != nil { - rep.Messages = append(rep.Messages, fmt.Sprintf("Could not retrieve volume statistics: %s, error: %s", volID, err.Error())) - continue + return nil, err } - readCount := stats.UserDataReadBwc.NumOccured - writeCount := stats.UserDataWriteBwc.NumOccured - sampleSeconds := stats.UserDataWriteBwc.NumSeconds - rep.Messages = append(rep.Messages, fmt.Sprintf("Volume %s writes %d reads %d for %d seconds", - volID, writeCount, readCount, sampleSeconds)) - if (readCount + writeCount) > 0 { - rep.IosInProgress = true + + if platformInfo.GenType == siotypes.GenTypeEC { + log.Infof("Found Gentype EC system %s", systemID) + metrics, err := adminClient.GetMetrics("volume", []string{volID}) + if err != nil { + rep.Messages = append(rep.Messages, fmt.Sprintf("Could not retrieve volume statistics: %s, error: %s", volID, err.Error())) + continue + } + + if len(metrics.Resources) == 0 { + rep.Messages = append(rep.Messages, fmt.Sprintf("No metrics found for volume: %s", volID)) + continue + } + writeBW := getMetric(metrics.Resources[0].Metrics, "host_write_bandwidth") + readBW := getMetric(metrics.Resources[0].Metrics, "host_read_bandwidth") + writeIOPS := getMetric(metrics.Resources[0].Metrics, "host_write_iops") + readIOPS := getMetric(metrics.Resources[0].Metrics, "host_read_iops") + + rep.Messages = append(rep.Messages, fmt.Sprintf("Volume %s writeBW %.2f readBw %.2f readIOPS %.2f writeIOPS %.2f", + volID, writeBW, readBW, readIOPS, writeIOPS)) + if ((writeBW > 0) || (readBW > 0)) || ((writeIOPS > 0) || (readIOPS > 0)) { + rep.IosInProgress = true + } + } else { + log.Infof("Found Legacy system %s", systemID) + stats, err := volume.GetVolumeStatistics() + if err != nil { + rep.Messages = append(rep.Messages, fmt.Sprintf("Could not retrieve volume statistics: %s, error: %s", volID, err.Error())) + continue + } + readCount := stats.UserDataReadBwc.NumOccured + writeCount := stats.UserDataWriteBwc.NumOccured + sampleSeconds := stats.UserDataWriteBwc.NumSeconds + rep.Messages = append(rep.Messages, fmt.Sprintf("Volume %s writes %d reads %d for %d seconds", + volID, writeCount, readCount, sampleSeconds)) + if (readCount + writeCount) > 0 { + rep.IosInProgress = true + } } } - Log.Infof("ValidateVolumeHostConnectivity reply %+v", rep) + log.Infof("ValidateVolumeHostConnectivity reply %+v", rep) return rep, nil } +func getMetric(metrics []siotypes.Metric, name string) float64 { + for _, m := range metrics { + if m.Name == name { + return m.Values[0] + } + } + return 0 +} + func (s *service) CreateVolumeGroupSnapshot(ctx context.Context, req *volumeGroupSnapshot.CreateVolumeGroupSnapshotRequest) (*volumeGroupSnapshot.CreateVolumeGroupSnapshotResponse, error) { - Log.Infof("CreateVolumeGroupSnapshot called with req: %v", req) + log.Infof("CreateVolumeGroupSnapshot called with req: %v", req) err := validateCreateVGSreq(req) if err != nil { - Log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) + log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) return nil, err } // take first volume to calculate systemID. It is expected this systemID is consistent throughout systemID, err := s.getSystemID(req) if err != nil { - Log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) + log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) return nil, err } @@ -138,11 +217,11 @@ func (s *service) CreateVolumeGroupSnapshot(ctx context.Context, req *volumeGrou return nil, err } - Log.Infof("Creating Snapshot Consistency Group on system: %s", systemID) + log.Infof("Creating Snapshot Consistency Group on system: %s", systemID) snapshotDefs, err := s.buildSnapshotDefs(req, systemID) if err != nil { - Log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) + log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) return nil, err } @@ -165,14 +244,14 @@ func (s *service) CreateVolumeGroupSnapshot(ctx context.Context, req *volumeGrou snapsThatFailed = append(snapsThatFailed, snap.SnapshotName) } err = status.Errorf(codes.Internal, "Failed to create group with snapshots %s : %s", snapsThatFailed, err.Error()) - Log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) + log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) return nil, err } - Log.Infof("snapResponse is: %s", snapResponse) + log.Infof("snapResponse is: %s", snapResponse) // populate response groupSnapshots, err := s.buildCreateVGSResponse(ctx, snapResponse, systemID) if err != nil { - Log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) + log.Errorf("Error from CreateVolumeGroupSnapshot: %v ", err) return nil, err } @@ -184,19 +263,19 @@ func (s *service) CreateVolumeGroupSnapshot(ctx context.Context, req *volumeGrou resp := &volumeGroupSnapshot.CreateVolumeGroupSnapshotResponse{SnapshotGroupID: systemID + "-" + snapResponse.SnapshotGroupID, Snapshots: groupSnapshots, CreationTime: groupSnapshots[0].CreationTime} - Log.Infof("CreateVolumeGroupSnapshot Response: %#v", resp) + log.Infof("CreateVolumeGroupSnapshot Response: %#v", resp) return resp, nil } func checkCreationTime(time int64, snapshots []*volumeGroupSnapshot.Snapshot) error { - Log.Infof("CheckCreationTime called with snapshots: %v", snapshots) + log.Infof("CheckCreationTime called with snapshots: %v", snapshots) for _, snap := range snapshots { if time != snap.CreationTime { err := status.Errorf(codes.Internal, "Creation time of snapshot %s, %d does not match with snapshot %s creation time %d. All snapshot creation times should be equal", snap.Name, snap.CreationTime, snapshots[0].Name, snapshots[0].CreationTime) - Log.Errorf("Error from CheckCreationTime: %v ", err) + log.Errorf("Error from CheckCreationTime: %v ", err) return err } - Log.Infof("CheckCreationTime: Creation time of %s is %d", snap.Name, time) + log.Infof("CheckCreationTime: Creation time of %s is %d", snap.Name, time) } return nil @@ -212,7 +291,7 @@ func (s *service) getSystemID(req *volumeGroupSnapshot.CreateVolumeGroupSnapshot if systemID == "" { err := status.Error(codes.InvalidArgument, "systemID is not found in vol ID and there is no default system") - Log.Errorf("Error from getSystemID: %v ", err) + log.Errorf("Error from getSystemID: %v ", err) return systemID, err } @@ -224,19 +303,19 @@ func (s *service) getSystemID(req *volumeGroupSnapshot.CreateVolumeGroupSnapshot func validateCreateVGSreq(req *volumeGroupSnapshot.CreateVolumeGroupSnapshotRequest) error { if len(req.SourceVolumeIDs) == 0 { err := status.Errorf(codes.InvalidArgument, "SourceVolumeIDs cannot be empty") - Log.Errorf("Error from validateCreateVGSreq: %v ", err) + log.Errorf("Error from validateCreateVGSreq: %v ", err) return err } if req.Name == "" { err := status.Error(codes.InvalidArgument, "CreateVolumeGroupSnapshotRequest Name is not set") - Log.Warnf("Warning from validateCreateVGSreq: %v ", err) + log.Warnf("Warning from validateCreateVGSreq: %v ", err) } // name must be less than 28 chars, because we name snapshots with -, and index can at most be 3 chars if len(req.Name) > 27 { err := status.Errorf(codes.InvalidArgument, "Requested name %s longer than 27 character max", req.Name) - Log.Errorf("Error from validateCreateVGSreq: %v ", err) + log.Errorf("Error from validateCreateVGSreq: %v ", err) return err } @@ -250,7 +329,7 @@ func (s *service) buildSnapshotDefs(req *volumeGroupSnapshot.CreateVolumeGroupSn snapSystemID := strings.TrimSpace(s.getSystemIDFromCsiVolumeID(id)) if snapSystemID != "" && snapSystemID != systemID { err := status.Errorf(codes.Internal, "Source volumes for volume group snapshot should be on the same system but vol %s is not on system: %s", id, systemID) - Log.Errorf("Error from buildSnapshotDefs: %v \n", err) + log.Errorf("Error from buildSnapshotDefs: %v \n", err) return nil, err } @@ -258,7 +337,7 @@ func (s *service) buildSnapshotDefs(req *volumeGroupSnapshot.CreateVolumeGroupSn err := s.checkVolumesMap(id) if err != nil { err = status.Errorf(codes.Internal, "checkVolumesMap for id: %s failed : %s", id, err.Error()) - Log.Errorf("Error from buildSnapshotDefs: %v ", err) + log.Errorf("Error from buildSnapshotDefs: %v ", err) return nil, err } @@ -267,7 +346,7 @@ func (s *service) buildSnapshotDefs(req *volumeGroupSnapshot.CreateVolumeGroupSn _, err = s.getVolByID(volID, systemID) if err != nil { err = status.Errorf(codes.Internal, "failure checking source volume status: %s", err.Error()) - Log.Errorf("Error from buildSnapshotDefs: %v ", err) + log.Errorf("Error from buildSnapshotDefs: %v ", err) return nil, err } @@ -285,7 +364,7 @@ func (s *service) buildSnapshotDefs(req *volumeGroupSnapshot.CreateVolumeGroupSn // 2. Each snapshot that we find to satisfy criteria 1 all belong to the same consistency group // 3. The consistency group that satisfies criteria 2 contain no other snapshots func (s *service) checkIdempotency(ctx context.Context, snapshotsToMake *siotypes.SnapshotVolumesParam, systemID string) (*volumeGroupSnapshot.CreateVolumeGroupSnapshotResponse, error) { - Log.Infof("CheckIdempotency called") + log.Infof("CheckIdempotency called") // We use maps to keep track of info, to ensure criterias 1-3 are met @@ -317,7 +396,7 @@ func (s *service) checkIdempotency(ctx context.Context, snapshotsToMake *siotype foundGrpID := systemID + "-" + existingSnap.ConsistencyGroupID consistencyGroupValue = existingSnap.ConsistencyGroupID if snap.VolumeID == existingSnap.AncestorVolumeID && snap.SnapshotName == existingSnap.Name { - Log.Infof("Snapshot for %s exists on array for group id %s", snap.VolumeID, existingSnap.ConsistencyGroupID) + log.Infof("Snapshot for %s exists on array for group id %s", snap.VolumeID, existingSnap.ConsistencyGroupID) idempotencyMap[snap.SnapshotName] = true idempotencyValue = true consistencyGroupMap[existingSnap.Name] = foundGrpID @@ -333,12 +412,12 @@ func (s *service) checkIdempotency(ctx context.Context, snapshotsToMake *siotype for snap := range idempotencyMap { if idempotencyMap[snap] != idempotencyValue { err := status.Error(codes.Internal, "Some snapshots exist on array, while others need to be created. Cannot create VolumeGroupSnapshot") - Log.Errorf("Error from checkIdempotency: %v ", err) + log.Errorf("Error from checkIdempotency: %v ", err) return nil, err } else if idempotencyValue { - Log.Debugf("snap: %s already exists on array", snap) + log.Debugf("snap: %s already exists on array", snap) } else { - Log.Debugf("snap: %s does not already exist on array", snap) + log.Debugf("snap: %s does not already exist on array", snap) } } // since we know all values in idempotencyMap match idempotencyValue, we can return now @@ -351,7 +430,7 @@ func (s *service) checkIdempotency(ctx context.Context, snapshotsToMake *siotype for _, vol := range existingVols { grpID := systemID + "-" + vol.ConsistencyGroupID if grpID == systemID+"-"+consistencyGroupValue { - Log.Infof("Checking %s: Snapshot %s found in consistency group.", consistencyGroupValue, vol.Name) + log.Infof("Checking %s: Snapshot %s found in consistency group.", consistencyGroupValue, vol.Name) consistencyGroupOnArray = append(consistencyGroupOnArray, vol.Name) } } @@ -361,11 +440,11 @@ func (s *service) checkIdempotency(ctx context.Context, snapshotsToMake *siotype // this check verifies criteria #3 if len(consistencyGroupOnArray) != len(IDsForResponse) { err := status.Errorf(codes.Internal, "CG: %s contains more snapshots than requested. Cannot create VolumeGroupSnapshot", consistencyGroupValue) - Log.Errorf("Error from checkIdempotency: %v ", err) + log.Errorf("Error from checkIdempotency: %v ", err) return nil, err } - Log.Infof("Request is idempotent") + log.Infof("Request is idempotent") // with all 3 criteria met, we need to return a CreateVolumeGroupSnapshotResponse with the VGS that satisfied the criteria var groupSnapshots []*volumeGroupSnapshot.Snapshot @@ -375,7 +454,7 @@ func (s *service) checkIdempotency(ctx context.Context, snapshotsToMake *siotype req := &csi.ListSnapshotsRequest{SnapshotId: idToQuery} existingSnap, err := s.ListSnapshots(ctx, req) if err != nil { - Log.Errorf("Failed to list snaps") + log.Errorf("Failed to list snaps") } creationTime := existingSnap.Entries[0].Snapshot.CreationTime.GetSeconds()*1000000000 + int64(existingSnap.Entries[0].Snapshot.CreationTime.GetNanos()) fmt.Printf("Creation time is: %d\n", creationTime) @@ -390,7 +469,7 @@ func (s *service) checkIdempotency(ctx context.Context, snapshotsToMake *siotype groupSnapshots = append(groupSnapshots, &snap) } resp := &volumeGroupSnapshot.CreateVolumeGroupSnapshotResponse{SnapshotGroupID: systemID + "-" + consistencyGroupValue, Snapshots: groupSnapshots, CreationTime: groupSnapshots[0].CreationTime} - Log.Infof("Returning Idempotent response: %v", resp) + log.Infof("Returning Idempotent response: %v", resp) return resp, nil } @@ -403,7 +482,7 @@ func (s *service) buildCreateVGSResponse(ctx context.Context, snapResponse *siot lResponse, err := s.ListSnapshots(ctx, req) if err != nil { err = status.Errorf(codes.Internal, "Failed to get snapshot: %s", err.Error()) - Log.Errorf("Error from buildCreateVGSResponse: %v ", err) + log.Errorf("Error from buildCreateVGSResponse: %v ", err) return nil, err } var arraySnapName string @@ -412,25 +491,25 @@ func (s *service) buildCreateVGSResponse(ctx context.Context, snapResponse *siot for _, e := range existingSnap { if e.ID == id && e.ConsistencyGroupID == snapResponse.SnapshotGroupID { if e.Name == "" { - Log.Infof("debug set snap name for [%s]", e.ID) + log.Infof("debug set snap name for [%s]", e.ID) arraySnapName = e.ID + "-snap-" + strconv.Itoa(index) tgtVol := sio.NewVolume(s.adminClients[systemID]) tgtVol.Volume = e err := tgtVol.SetVolumeName(arraySnapName) if err != nil { - Log.Errorf("Error setting name of snapshot id=%s name=%s %s", e.ID, arraySnapName, err.Error()) + log.Errorf("Error setting name of snapshot id=%s name=%s %s", e.ID, arraySnapName, err.Error()) } } else { - Log.Infof("debug found snap name %s for %s", e.Name, e.ID) + log.Infof("debug found snap name %s for %s", e.Name, e.ID) arraySnapName = e.Name } } } - Log.Infof("Snapshot Name created for: %s is %s", lResponse.Entries[0].Snapshot.SnapshotId, arraySnapName) + log.Infof("Snapshot Name created for: %s is %s", lResponse.Entries[0].Snapshot.SnapshotId, arraySnapName) // need to convert time from seconds and nanoseconds to int64 nano seconds creationTime := lResponse.Entries[0].Snapshot.CreationTime.GetSeconds()*1000000000 + int64(lResponse.Entries[0].Snapshot.CreationTime.GetNanos()) - Log.Infof("Creation time is: %d\n", creationTime) + log.Infof("Creation time is: %d\n", creationTime) snap := volumeGroupSnapshot.Snapshot{ Name: arraySnapName, CapacityBytes: lResponse.Entries[0].Snapshot.SizeBytes, diff --git a/service/envvars.go b/service/envvars.go index 8e8bb0d5..31ffe191 100644 --- a/service/envvars.go +++ b/service/envvars.go @@ -72,4 +72,19 @@ const ( // EnvMaxProbeTimeout is the name of the environment variable which stores the maximum probe timeout EnvMaxProbeTimeout = "X_CSI_PROBE_TIMEOUT" + + // EnvNodeChrootPath is the name of the environment variable which store path to chroot where to execute NVMe commands + EnvNodeChrootPath = "X_CSI_POWERFLEX_NODE_CHROOT_PATH" + + // EnvPodmonEnabled indicates that podmon is enabled + EnvPodmonEnabled = "X_CSI_PODMON_ENABLED" + + // EnvPodmonArrayConnectivityAPIPORT indicates the port to be used for exposing podmon API health + EnvPodmonArrayConnectivityAPIPORT = "X_CSI_PODMON_API_PORT" + + // EnvPodmonArrayConnectivityPollRate indicates the polling frequency to check array connectivity + EnvPodmonArrayConnectivityPollRate = "X_CSI_PODMON_ARRAY_CONNECTIVITY_POLL_RATE" + + // EnvAuthTyoe is the name of the environment variable which stores the authentication type such as OIDC or Standard Username Password + EnvAuthType = "X_CSI_AUTH_TYPE" ) diff --git a/service/ephemeral.go b/service/ephemeral.go index 2f811ed2..ef19340a 100644 --- a/service/ephemeral.go +++ b/service/ephemeral.go @@ -16,6 +16,7 @@ package service import ( "context" "errors" + "fmt" "os" "path/filepath" "regexp" @@ -30,6 +31,7 @@ var ephemeralStagingMountPath = "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/e func (s *service) fileExist(filename string) bool { _, err := os.Stat(filename) + log.Debugf("Error stating file %s: %v", filename, err) if err != nil && os.IsNotExist(err) { return false } @@ -62,41 +64,41 @@ func (s *service) ephemeralNodePublish( ) { _, err := os.Stat(ephemeralStagingMountPath) if err != nil { - Log.Warnf("Unable to check stat of file: %s with error: %v", ephemeralStagingMountPath, err.Error()) + log.Warnf("Unable to check stat of file: %s with error: %v", ephemeralStagingMountPath, err.Error()) if os.IsNotExist(err) { - Log.Debug("path does not exist, will attempt to create it") + log.Debug("path does not exist, will attempt to create it") err = os.MkdirAll(ephemeralStagingMountPath, 0o750) if err != nil { - Log.WithField("dir", ephemeralStagingMountPath).WithError(err).Error("Unable to create dir") + log.Errorf("Unable to create dir %s: %v", ephemeralStagingMountPath, err) return nil, status.Error(codes.Internal, "Unable to create directory for mounting ephemeral volumes, error: "+err.Error()) } - Log.Debug("dir created: ", ephemeralStagingMountPath) + log.Debugf("dir created: %v", ephemeralStagingMountPath) } } volID := req.GetVolumeId() volName := req.VolumeContext["volumeName"] if len(volName) > 31 { - Log.Errorf("Volume name: %s is over 32 characters, too long.", volName) + log.Errorf("Volume name: %s is over 32 characters, too long.", volName) return nil, status.Error(codes.Internal, "Volume name too long") } if volName == "" { - Log.Errorf("Missing Parameter: volumeName must be specified in volume attributes section for ephemeral volumes") + log.Errorf("Missing Parameter: volumeName must be specified in volume attributes section for ephemeral volumes") return nil, status.Error(codes.Internal, "Volume name not specified") } volSize, err := parseSize(req.VolumeContext["size"]) if err != nil { - Log.Errorf("Parse size failed %s", err.Error()) + log.Errorf("Parse size failed %s", err.Error()) return nil, status.Error(codes.Internal, "inline ephemeral parse size failed") } systemName := req.VolumeContext["systemID"] if systemName == "" { - Log.Debug("systemName not specified, using default array") + log.Debug("systemName not specified, using default array") systemName = s.opts.defaultSystemID } @@ -106,11 +108,11 @@ func (s *service) ephemeralNodePublish( // to get inside this if block, req has name, but secret has ID, need to convert from name -> ID if id, ok := s.connectedSystemNameToID[systemName]; ok { // systemName was sent in req, but secret used ID. Change to ID. - Log.Debug("systemName set to id") + log.Debugf("systemName set to id: %s", id) array = s.opts.arrays[id] } else { err = status.Errorf(codes.Internal, "systemID: %s not recgonized", systemName) - Log.Errorf("Error from ephemeralNodePublish: %v ", err) + log.Errorf("Error from ephemeralNodePublish: %v ", err) return nil, err } @@ -118,7 +120,7 @@ func (s *service) ephemeralNodePublish( err = s.systemProbe(ctx, array) if err != nil { - Log.Errorf("systemProb Ephemeral %s", err.Error()) + log.Errorf("systemProb Ephemeral %s", err.Error()) return nil, status.Error(codes.Internal, "inline ephemeral system prob failed: "+err.Error()) } @@ -133,42 +135,35 @@ func (s *service) ephemeralNodePublish( Secrets: req.Secrets, }) if err != nil { - Log.Errorf("CreateVolume Ephemeral %s", err.Error()) + log.Errorf("CreateVolume Ephemeral %s", err.Error()) return nil, status.Error(codes.Internal, "inline ephemeral create volume failed: "+err.Error()) } - Log.Infof("volume ID returned from CreateVolume is: %s ", crvolresp.Volume.VolumeId) + log.Infof("volume ID returned from CreateVolume is: %s ", crvolresp.Volume.VolumeId) + volumeID := crvolresp.Volume.VolumeId // Create lockfile to map vol ID from request to volID returned by CreateVolume // will also be used to determine if volume is ephemeral in NodeUnpublish - errLock := os.MkdirAll(ephemeralStagingMountPath+volID, 0o750) + errLock := os.MkdirAll(filepath.Clean(filepath.Join(ephemeralStagingMountPath, volID)), 0o750) if errLock != nil { return nil, errLock } safePath := filepath.Join(ephemeralStagingMountPath, volID, "id") safePath = filepath.Clean(safePath) - f, errLock := os.Create(safePath) - if errLock != nil { - return nil, errLock - } - defer f.Close() //#nosec - _, errLock = f.WriteString(crvolresp.Volume.VolumeId) - if errLock != nil { - return nil, errLock - } - - volumeID := crvolresp.Volume.VolumeId // in case systemName was not given with volume context systemName = s.getSystemIDFromCsiVolumeID(volumeID) if systemName == "" { - - Log.Errorf("getSystemIDFromCsiVolumeID was not able to determine systemName from volID: %s", volumeID) + log.Errorf("getSystemIDFromCsiVolumeID was not able to determine systemName from VolumeID: %s", volumeID) return nil, status.Error(codes.Internal, "inline ephemeral getSystemIDFromCsiVolumeID failed ") } NodeID := s.opts.SdcGUID + if s.useNVME { + log.Infof("SdcGUID is not set, using NodeID: %s", s.nodeID) + NodeID = s.nodeID + } cpubresp, err := s.ControllerPublishVolume(ctx, &csi.ControllerPublishVolumeRequest{ NodeId: NodeID, @@ -179,18 +174,54 @@ func (s *service) ephemeralNodePublish( VolumeContext: crvolresp.Volume.VolumeContext, }) if err != nil { - Log.Infof("Rolling back and calling unpublish ephemeral volumes with VolId %s", crvolresp.Volume.VolumeId) + log.Infof("Rolling back and calling unpublish ephemeral volumes with VolId %s", crvolresp.Volume.VolumeId) _, _ = s.NodeUnpublishVolume(ctx, &csi.NodeUnpublishVolumeRequest{ VolumeId: volID, TargetPath: req.TargetPath, }) return nil, status.Error(codes.Internal, "inline ephemeral controller publish failed: "+err.Error()) } + if s.useNVME { + log.Debug("found NVME ephemeral volume") + stageReq := &csi.NodeStageVolumeRequest{ + StagingTargetPath: filepath.Clean(filepath.Join(ephemeralStagingMountPath, volID)), + VolumeId: volumeID, + VolumeCapability: req.VolumeCapability, + Secrets: req.Secrets, + VolumeContext: crvolresp.Volume.VolumeContext, + } + _, err = s.NodeStageVolume(ctx, stageReq) + if err != nil { + _, _ = s.NodeUnpublishVolume(ctx, &csi.NodeUnpublishVolumeRequest{ + VolumeId: volID, + TargetPath: req.TargetPath, + }) + return nil, status.Error(codes.Internal, "inline NVMe ephemeral node stage volume failed: "+err.Error()) + } + } + var f *os.File + f, errLock = os.Create(safePath) + if errLock != nil { + return nil, errLock + } + log.Debugf("Created lockfile during volume creation:%s", safePath) + + _, errLock = f.WriteString(volumeID) + if errLock != nil { + return nil, errLock + } + log.Infof("lock-file contents written:%s", volumeID) + + defer func() { + if err := f.Close(); err != nil { + log.Errorf("Error closing file %s: %v", safePath, err) + } + }() _, err = s.NodePublishVolume(ctx, &csi.NodePublishVolumeRequest{ VolumeId: volumeID, PublishContext: cpubresp.PublishContext, - StagingTargetPath: ephemeralStagingMountPath, + StagingTargetPath: filepath.Clean(filepath.Join(ephemeralStagingMountPath, volID)), TargetPath: req.TargetPath, VolumeCapability: req.VolumeCapability, Readonly: req.Readonly, @@ -198,14 +229,13 @@ func (s *service) ephemeralNodePublish( VolumeContext: crvolresp.Volume.VolumeContext, }) if err != nil { - Log.Errorf("NodePublishErrEph %s", err.Error()) + log.Errorf("NodePublishErrEph %s", err.Error()) _, _ = s.NodeUnpublishVolume(ctx, &csi.NodeUnpublishVolumeRequest{ VolumeId: volID, TargetPath: req.TargetPath, }) return nil, status.Error(codes.Internal, "inline ephemeral node publish failed: "+err.Error()) } - return &csi.NodePublishVolumeResponse{}, nil } @@ -215,14 +245,15 @@ func (s *service) ephemeralNodeUnpublish( ctx context.Context, req *csi.NodeUnpublishVolumeRequest, ) error { - Log.Infof("Called ephemeral Node unpublish") + log.Infof("Called ephemeral Node unpublish") volID := req.GetVolumeId() if volID == "" { return status.Error(codes.InvalidArgument, "volume ID is required") } - - lockFile := ephemeralStagingMountPath + volID + "/id" + stagingPath := filepath.Clean(filepath.Join(ephemeralStagingMountPath, volID)) + lockFile := filepath.Clean(filepath.Join(ephemeralStagingMountPath, volID, "id")) + log.Debugf("Lock-file path:%s", lockFile) //while a file is being read from, it's a file determined by volID and is written by the driver /* #nosec G304 */ @@ -233,13 +264,29 @@ func (s *service) ephemeralNodeUnpublish( goodVolid := string(dat) NodeID := s.opts.SdcGUID + if s.useNVME { + log.Infof("SdcGUID is not set, using NodeID: %s", s.nodeID) + NodeID = s.nodeID + } + log.Infof("Read volume and array ID from file:%s", goodVolid) + if s.useNVME { + log.Debug("Unstaging NVME ephemeral volume") + unStageReq := &csi.NodeUnstageVolumeRequest{ + StagingTargetPath: stagingPath, + VolumeId: goodVolid, + } + _, err = s.NodeUnstageVolume(ctx, unStageReq) + if err != nil { + return err + } + } _, err = s.ControllerUnpublishVolume(ctx, &csi.ControllerUnpublishVolumeRequest{ VolumeId: goodVolid, NodeId: NodeID, }) if err != nil { - return errors.New("Inline ephemeral controller unpublish failed") + return fmt.Errorf("Inline ephemeral controller unpublish failed: %v: ", err) } _, err = s.DeleteVolume(ctx, &csi.DeleteVolumeRequest{ @@ -248,10 +295,12 @@ func (s *service) ephemeralNodeUnpublish( if err != nil { return err } - err = os.RemoveAll(ephemeralStagingMountPath + volID) + fileToRemove := filepath.Clean(filepath.Join(ephemeralStagingMountPath, volID)) + log.Debugf("lock-file to delete:%s", fileToRemove) + + err = os.RemoveAll(fileToRemove) if err != nil { - return errors.New("failed to cleanup lock files") + return fmt.Errorf("failed to cleanup lock files: %v", err) } - return nil } diff --git a/service/features/array-config/config.2 b/service/features/array-config/config.2 index 7d397fa5..bef87113 100644 --- a/service/features/array-config/config.2 +++ b/service/features/array-config/config.2 @@ -5,7 +5,7 @@ "password": "password", "insecure": true, "isDefault": true, - "systemID": "14dbbf5617523654" + "systemID": "14dbbf5617523654", "nasName": "dummy-nas-server" }, { @@ -14,7 +14,7 @@ "password": "password", "insecure": true, "isDefault": false, - "systemID": "15dbbf5617523655" + "systemID": "15dbbf5617523655", "nasName": "dummy-nas-server" }, { @@ -23,7 +23,7 @@ "systemID": "1235e15806d1ec0f", "endpoint": "https://1.2.3.4", "insecure": true, - "isDefault": false + "isDefault": false, "nasName": "dummy-nas-server" } ] diff --git a/service/features/controller_publish_unpublish.feature b/service/features/controller_publish_unpublish.feature index 1a8d23df..cc0d84a6 100644 --- a/service/features/controller_publish_unpublish.feature +++ b/service/features/controller_publish_unpublish.feature @@ -7,16 +7,19 @@ Feature: VxFlex OS CSI interface Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with Then a valid PublishVolumeResponse is returned And the number of SDC mappings is 1 - Examples: - | access | - | "single-writer" | - | "single-node-single-writer" | - | "single-node-multi-writer" | - + | access | protocol | + | "single-writer" | "SDC" | + | "single-node-single-writer" | "SDC" | + | "single-node-multi-writer" | "SDC" | + | "single-writer" | "NVMeTCP" | + | "single-node-single-writer" | "NVMeTCP" | + | "single-node-multi-writer" | "NVMeTCP" | + Scenario: a Basic NFS controller Publish no error Given a VxFlexOS service When I specify CreateVolumeMountRequest "nfs" @@ -270,28 +273,34 @@ Feature: VxFlex OS CSI interface And I induce error "LegacyVolumeConflictError" And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with Then the error contains "expecting this volume id only on default system. aborting operation" - Examples: - | access | - | "single-writer" | - | "single-node-single-writer" | - | "single-node-multi-writer" | + | access | protocol | + | "single-writer" | "SDC" | + | "single-node-single-writer" | "SDC" | + | "single-node-multi-writer" | "SDC" | + | "single-writer" | "NVMeTCP" | + | "single-node-single-writer" | "NVMeTCP" | + | "single-node-multi-writer" | "NVMeTCP" | Scenario: Publish volume but ID is too short to get first 24 bits Given a VxFlexOS service And a valid volume When I call Probe And I induce error "VolumeIDTooShortError" + And I set protocol to And I call PublishVolume with Then the error contains "is shorter than 3 chars, returning error" - Examples: - | access | - | "single-writer" | - | "single-node-single-writer" | - | "single-node-multi-writer" | + | access | protocol | + | "single-writer" | "SDC" | + | "single-node-single-writer" | "SDC" | + | "single-node-multi-writer" | "SDC" | + | "single-writer" | "NVMeTCP" | + | "single-node-single-writer" | "NVMeTCP" | + | "single-node-multi-writer" | "NVMeTCP" | Scenario: Calling probe twice, so UpdateVolumePrefixToSystemsMap gets a key,value already added Given a VxFlexOS service @@ -305,138 +314,194 @@ Feature: VxFlex OS CSI interface And a valid volume And I use AccessType Mount When I call Probe + And I set protocol to And I call PublishVolume with Then the error contains - Examples: - | access | msg | - | "multiple-writer" | "Mount multinode multi-writer not allowed" | - | "multi-single-writer" | "Multinode single writer not supported" | + | access | msg | protocol | + | "multiple-writer" | "mount multinode multi-writer not allowed" | "SDC" | + | "multi-single-writer" | "multinode single writer not supported" | "SDC" | + | "multiple-writer" | "mount multinode multi-writer not allowed" | "NVMeTCP" | + | "multi-single-writer" | "multinode single writer not supported" | "NVMeTCP" | Scenario: Idempotent publish volume with single writer Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with And I call PublishVolume with Then a valid PublishVolumeResponse is returned And the number of SDC mappings is 1 Examples: - | access | - | "single-writer" | - | "single-node-single-writer" | - | "single-node-multi-writer" | + | access | protocol | + | "single-writer" | "SDC" | + | "single-node-single-writer" | "SDC" | + | "single-node-multi-writer" | "SDC" | + | "single-writer" | "NVMeTCP" | + | "single-node-single-writer" | "NVMeTCP" | + | "single-node-multi-writer" | "NVMeTCP" | Scenario: Publish block volume with multiple writers to single writer volume Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with And then I use a different nodeID And I call PublishVolume with Then the error contains "volume already published" - Examples: - | access | - | "single-writer" | - | "single-node-single-writer" | - | "single-node-multi-writer" | + | access | protocol | + | "single-writer" | "SDC" | + | "single-node-single-writer" | "SDC" | + | "single-node-multi-writer" | "SDC" | + | "single-writer" | "NVMeTCP" | + | "single-node-single-writer" | "NVMeTCP" | + | "single-node-multi-writer" | "NVMeTCP" | Scenario: Publish block volume with multiple writers to multiple writer volume Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "multiple-writer" And then I use a different nodeID And I call PublishVolume with "multiple-writer" Then a valid PublishVolumeResponse is returned And the number of SDC mappings is 2 + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish block volume with multiple writers to multiple reader volume Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "multiple-reader" And then I use a different nodeID And I call PublishVolume with "multiple-reader" Then the error contains "not compatible with access type" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish mount volume with multiple writers to single writer volume Given a VxFlexOS service And a valid volume And I use AccessType Mount When I call Probe + And I set protocol to And I call PublishVolume with And then I use a different nodeID And I call PublishVolume with Then the error contains "volume already published" - Examples: - | access | - | "single-writer" | - | "single-node-single-writer" | - | "single-node-multi-writer" | + | access | protocol | + | "single-writer" | "SDC" | + | "single-node-single-writer" | "SDC" | + | "single-node-multi-writer" | "SDC" | + | "single-writer" | "NVMeTCP" | + | "single-node-single-writer" | "NVMeTCP" | + | "single-node-multi-writer" | "NVMeTCP" | Scenario: Publish mount volume with multiple readers to multiple reader volume Given a VxFlexOS service And a valid volume And I use AccessType Mount When I call Probe + And I set protocol to And I call PublishVolume with "multiple-reader" And then I use a different nodeID And I call PublishVolume with "multiple-reader" Then a valid PublishVolumeResponse is returned + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish mount volume with multiple readers to multiple reader volume Given a VxFlexOS service And a valid volume And I use AccessType Mount When I call Probe + And I set protocol to And I call PublishVolume with "multiple-reader" And then I use a different nodeID And I call PublishVolume with "multiple-reader" Then a valid PublishVolumeResponse is returned And the number of SDC mappings is 2 + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish volume with an invalid volumeID Given a VxFlexOS service When I call Probe And an invalid volume + And I set protocol to And I call PublishVolume with "single-writer" Then the error contains "volume not found" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish volume no volumeID specified Given a VxFlexOS service And no volume When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" Then the error contains "volume ID is required" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish volume with no nodeID specified Given a VxFlexOS service And a valid volume And no node When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" Then the error contains "node ID is required" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish volume with no volume capability Given a VxFlexOS service And a valid volume And no volume capability When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" Then the error contains "volume capability is required" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish volume with no access mode Given a VxFlexOS service And a valid volume And no access mode When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" Then the error contains "access mode is required" - + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish volume with getSDCID error Given a VxFlexOS service @@ -444,48 +509,68 @@ Feature: VxFlex OS CSI interface And I induce error "GetSdcInstancesError" When I call Probe And I call PublishVolume with "single-writer" - Then the error contains "error finding SDC from GUID" + Then the error contains "error getting host ID and type" Scenario: Publish volume with bad vol ID Given a VxFlexOS service And a valid volume And I induce error "BadVolIDError" When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" Then the error contains "volume not found" - + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Publish volume with a map SDC error Given a VxFlexOS service And a valid volume - And I induce error "MapSdcError" + And I induce error When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" - Then the error contains "error mapping volume to node" + Then the error contains + Examples: + | protocol | inducedErr | expectedErr | + | "SDC" | "MapSdcError" | "error mapping volume to node" | + | "NVMeTCP" | "MapNVMeError" | "error mapping volume to nvme host" | Scenario: Publish volume with AccessMode UNKNOWN Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "unknown" Then the error contains "access mode cannot be UNKNOWN" - + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | + Scenario: Unpublish volume Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" And no error was received - And the number of SDC mappings is 1 + And the number of SDC mappings is 1 And I call UnpublishVolume And no error was received Then a valid UnpublishVolumeResponse is returned And the number of SDC mappings is 0 - + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | + Scenario: Idempotent unpublish volume Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" And no error was received And I call UnpublishVolume @@ -493,52 +578,76 @@ Feature: VxFlex OS CSI interface And I call UnpublishVolume And no error was received Then a valid UnpublishVolumeResponse is returned - + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Unpublish volume with no volume id Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" And no error was received And no volume And I call UnpublishVolume Then the error contains "volume ID is required" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Unpublish volume with invalid volume id Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" And no error was received And an invalid volume And I call UnpublishVolume Then the error contains "volume not found" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Unpublish volume with no node id Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" And no error was received And no node And I call UnpublishVolume Then the error contains "Node ID is required" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Unpublish volume with RemoveMappedSdcError Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "single-writer" And no error was received - And I induce error "RemoveMappedSdcError" + And I induce error And I call UnpublishVolume - Then the error contains "Error unmapping volume from node" + Then the error contains + Examples: + | protocol | inducedErr | errorMsg | + | "SDC" | "RemoveMappedSdcError" | "Error unmapping volume from SDC node" | + | "NVMeTCP" | "RemoveMappedHostError" | "Error unmapping volume from NVMe host" | Scenario: Publish / unpublish mount volume with multiple writers to multiple writer volume Given a VxFlexOS service And a valid volume When I call Probe + And I set protocol to And I call PublishVolume with "multiple-writer" And a valid PublishVolumeResponse is returned And the number of SDC mappings is 1 @@ -553,6 +662,9 @@ Feature: VxFlex OS CSI interface And I call UnpublishVolume And no error was received Then the number of SDC mappings is 0 + Examples: + | protocol | + | "NVMeTCP" | Scenario: Create NFS volume, enable quota with all key parameters Given a VxFlexOS service @@ -654,3 +766,15 @@ Feature: VxFlex OS CSI interface And I set quota with path "/fs" softLimit "200" graceperiod "86400" And I call CreateVolumeSize nfs "vol-inttest-nfs" "10" Then the error contains "requested softLimit: 200 perc is greater than volume size" + + Scenario: Try to create NFS volume and Replication on NFS & Replication not supported Array + Given a VxFlexOS service + And I set Platform Info + When I specify CreateVolumeMountRequest + And I call CreateVolume "volume1" with Error + Then I reset the Platform Info + + Examples: + | sourceVersion | sourceGenType | targetVersion | targetGenType | volumeType | errorMsg | + | "5.0" | "EC" | "5.0" | "EC" | "nfs" | "NFS is not supported on the System" | + | "4.0" | "" | "5.0" | "EC" | "ext4" | "Replication is not supported on this System" | \ No newline at end of file diff --git a/service/features/create_nvme_host.json b/service/features/create_nvme_host.json new file mode 100644 index 00000000..58ac059c --- /dev/null +++ b/service/features/create_nvme_host.json @@ -0,0 +1,3 @@ +{ + "id": "15d2cc7400010000" +} \ No newline at end of file diff --git a/service/features/csi_extension.feature b/service/features/csi_extension.feature index 726eadd1..af7e77c5 100644 --- a/service/features/csi_extension.feature +++ b/service/features/csi_extension.feature @@ -12,13 +12,34 @@ Feature: VxFlex OS CSI interface And I call ValidateConnectivity Then no error was received + @pmon + Scenario: Call ValidateConnectivity for NVMeTCP + Given a VxFlexOS service + When I call Probe + And I set protocol to "NVMeTCP" + And I call CreateVolume "podmon1" + Then a valid CreateVolumeResponse is returned + And I call ValidateConnectivity + Then no error was received + + @pmon + Scenario: Call ValidateConnectivity for GenType EC + Given a VxFlexOS service + When I call Probe + And I call CreateVolume "podmon1" + Then a valid CreateVolumeResponse is returned + And I set Platform Info "5.0" "EC" "5.0" "EC" + And I call ValidateConnectivity + Then no error was received + Then I reset the Platform Info + @pmon Scenario: Call ValidateConnectivity with node probe error Given a VxFlexOS service When I call Probe And I induce error "node-probe" And I call ValidateConnectivity - Then the error contains "NodeID is invalid" + Then the error contains "error getting host ID" @pmon Scenario: Call ValidateConnectivity with no Volume no Node @@ -42,7 +63,7 @@ Feature: VxFlex OS CSI interface And I call CreateVolume "podmon1" And I induce error "no-system" And I call ValidateConnectivity - Then the error contains "NodeID is invalid" + Then the error contains "error getting host ID" @pmon Scenario: Call ValidateConnectivity with contoller probe error @@ -50,7 +71,7 @@ Feature: VxFlex OS CSI interface When I call Probe And I induce error "controller-probe" And I call ValidateConnectivity - Then the error contains "NodeID is invalid" + Then the error contains "error getting host ID" @pmon Scenario: Call ValidateConnectivity with no System @@ -59,7 +80,7 @@ Feature: VxFlex OS CSI interface And I call CreateVolume "podmon1" And I induce error "no-sdc" And I call ValidateConnectivity - Then the error contains "NodeID is invalid" + Then the error contains "error getting host ID" @pmon Scenario: Call ValidateConnectivity with volume error diff --git a/service/features/ephemeral.feature b/service/features/ephemeral.feature index 3f4b071a..e79147a1 100644 --- a/service/features/ephemeral.feature +++ b/service/features/ephemeral.feature @@ -42,7 +42,7 @@ Scenario: Controller Unpublish Ephemeral Volume Fails And I call NodePublishVolume "SDC_GUID" And I induce error "BadVolIDError" And I call NodeUnpublishVolume "SDC_GUID" - Then the error contains "Inline ephemeral controller unpublish failed" + And no error was received Scenario Outline: Node publish and unpublish ephemeral volume Given a VxFlexOS service @@ -66,6 +66,35 @@ Examples: | "csi-d0f055a700000000" | "30Gi" | "viki_pool_HDD_20181031" | "does-not-exist" | "not recgonized" | | "csi-d0f055a700012345" | "30Gi" | "viki_pool_HDD_20181031" | "15dbbf5617523655" | "not published" | +Scenario Outline: Node publish and unpublish ephemeral volume for NVMeTCP Fails + Given a VxFlexOS service + And a controller published ephemeral volume + And a capability with voltype "mount" access "single-reader" fstype "none" + And I set protocol to "NVMeTCP" + And get Node Publish Ephemeral Volume Request with name size storagepool and systemName + And I call Probe + And I call NodePublishVolume NVME "mount" + And I call NodeUnpublishVolume "NVMeTCP" + Then the error contains +Examples: + | name | size | storagepool | systemName | errormsg | + | "csi-d0f055a700000000" | "30Gi" | "viki_pool_HDD_20181031" | "14dbbf5617523654" | "volumeID must be 16 characters" | + +Scenario Outline: Ephemeral Node Unpublish NVMeTCP with errors + Given a VxFlexOS service + And a controller published ephemeral volume + And a capability with voltype "mount" access "single-reader" fstype "none" + And I set protocol to "NVMeTCP" + And get Node Publish Ephemeral Volume Request with name size storagepool and systemName + And I call Probe + And I call NodePublishVolume NVME "mount" + And I call NodeUnpublishVolume "NVMeTCP" + And I call EphemeralNodeUnpublish + Then the error contains +Examples: + | name | size | storagepool | systemName | errormsg | + | "csi-d0f055a700000000" | "30Gi" | "viki_pool_HDD_20181031" | "14dbbf5617523654" | "inline NVMe ephemeral node stage volume failed" | + Scenario Outline: Ephemeral Node Unpublish with errors Given a VxFlexOS service And I induce error @@ -75,9 +104,19 @@ Scenario Outline: Ephemeral Node Unpublish with errors Examples: | error | errormsg | | "NoVolumeIDError" | "volume ID is required" | - | "none" | "Inline ephemeral. Was unable to read lockfile" | + | "none" | "Inline ephemeral controller unpublish failed" | Scenario Outline: Ephemeral Node Publish with errors Given a VxFlexOS service And I call EphemeralNodePublish Then the error contains "not recgonized" + +Scenario: Controller Unpublish Ephemeral NVME Volume Fails + Given a VxFlexOS service + And a controller published ephemeral volume + And a capability with voltype "mount" access "single-reader" fstype "none" + And get Node Publish Ephemeral Volume Request with name "csi-d0f055a700000000" size "30Gi" storagepool "viki_pool_HDD_20181031" and systemName "14dbbf5617523654" + And I call Probe + And I call NodePublishVolume NVME "mount" + And I call NodeUnpublishVolume "NVMeTCP" + Then the error contains "node ID is required" \ No newline at end of file diff --git a/service/features/get_all_sdt.json b/service/features/get_all_sdt.json new file mode 100644 index 00000000..d3a02788 --- /dev/null +++ b/service/features/get_all_sdt.json @@ -0,0 +1,159 @@ +[ + { + "ipList": [ + { + "role": "StorageAndHost", + "ip": "127.0.0.1:invalid" + } + ], + "sdtState": "Normal", + "name": "dummy-SDT1", + "systemId": "14dbbf5617523654", + "protectionDomainId": "df18067f00000000", + "storagePort": 12200, + "nvmePort": 4420, + "discoveryPort": 8009, + "faultSetId": null, + "softwareVersionInfo": "R4_5.5000.0", + "mdmConnectionState": "Connected", + "membershipState": "Joined", + "maintenanceState": "NoMaintenance", + "authenticationError": "None", + "persistentDiscoveryControllersNum": 0, + "id": "a6ef26bb00000002", + "links": [ + { + "rel": "self", + "href": "/api/instances/Sdt::a6ef26bb00000002" + }, + { + "rel": "/api/Sdt/relationship/Statistics", + "href": "/api/instances/Sdt::a6ef26bb00000002/relationships/Statistics" + }, + { + "rel": "/api/parent/relationship/protectionDomainId", + "href": "/api/instances/ProtectionDomain::df18067f00000000" + } + ] + }, + { + "ipList": [ + { + "role": "StorageAndHost", + "ip": "127.0.0.2:invalid" + } + ], + "sdtState": "Normal", + "name": "dummy-SDT2", + "systemId": "14dbbf5617523654", + "protectionDomainId": "df18067f00000000", + "storagePort": 12200, + "nvmePort": 4420, + "discoveryPort": 8009, + "faultSetId": null, + "softwareVersionInfo": "R4_5.5000.0", + "certificateInfo": { + "validFromAsn1Format": "251014070827Z", + "validToAsn1Format": "351013080827Z", + "subject": "/GN=sdt-comp-0/CN=lglou172/L=Hopkinton/ST=Massachusetts/C=US/O=EMC/OU=ASD", + "issuer": "/GN=MDM/CN=CA-14dbbf5617523654/L=Hopkinton/ST=Massachusetts/C=US/O=EMC/OU=ASD", + "thumbprint": "FE:E4:46:40:EB:0A:D3:70:63:71:A3:8A:EE:58:5C:7D:F9:F4:8D:F5", + "validTo": "Oct 13 08:08:27 2035 GMT", + "validFrom": "Oct 14 07:08:27 2025 GMT" + }, + "mdmConnectionState": "Connected", + "membershipState": "Joined", + "maintenanceState": "NoMaintenance", + "authenticationError": "None", + "persistentDiscoveryControllersNum": 0, + "id": "a6ef26b900000000", + "links": [ + { + "rel": "self", + "href": "/api/instances/Sdt::a6ef26b900000000" + }, + { + "rel": "/api/Sdt/relationship/Statistics", + "href": "/api/instances/Sdt::a6ef26b900000000/relationships/Statistics" + }, + { + "rel": "/api/parent/relationship/protectionDomainId", + "href": "/api/instances/ProtectionDomain::df18067f00000000" + } + ] + }, + { + "ipList": [ + { + "role": "StorageAndHost", + "ip": "127.0.0.3" + } + ], + "sdtState": "Normal", + "name": "dummy-SDT3", + "systemId": "14dbbf5617523654", + "protectionDomainId": "df18067f00000000", + "storagePort": 12200, + "nvmePort": 4420, + "discoveryPort": 8009, + "faultSetId": null, + "softwareVersionInfo": "R4_5.5000.0", + "mdmConnectionState": "Connected", + "membershipState": "Joined", + "maintenanceState": "NoMaintenance", + "authenticationError": "None", + "persistentDiscoveryControllersNum": 0, + "id": "a6ef26ba00000001", + "links": [ + { + "rel": "self", + "href": "/api/instances/Sdt::a6ef26ba00000001" + }, + { + "rel": "/api/Sdt/relationship/Statistics", + "href": "/api/instances/Sdt::a6ef26ba00000001/relationships/Statistics" + }, + { + "rel": "/api/parent/relationship/protectionDomainId", + "href": "/api/instances/ProtectionDomain::df18067f00000000" + } + ] + }, + { + "ipList": [ + { + "role": "StorageAndHost", + "ip": "127.0.0.4" + } + ], + "sdtState": "Normal", + "name": "dummy-SDT4", + "systemId": "14dbbf5617523654", + "protectionDomainId": "df18067f00000000", + "storagePort": 12200, + "nvmePort": 4420, + "discoveryPort": 8009, + "faultSetId": null, + "softwareVersionInfo": "R4_5.5000.0", + "mdmConnectionState": "Connected", + "membershipState": "Joined", + "maintenanceState": "NoMaintenance", + "authenticationError": "None", + "persistentDiscoveryControllersNum": 0, + "id": "a6ef26bc00000003", + "links": [ + { + "rel": "self", + "href": "/api/instances/Sdt::a6ef26bc00000003" + }, + { + "rel": "/api/Sdt/relationship/Statistics", + "href": "/api/instances/Sdt::a6ef26bc00000003/relationships/Statistics" + }, + { + "rel": "/api/parent/relationship/protectionDomainId", + "href": "/api/instances/ProtectionDomain::df18067f00000000" + } + ] + } +] \ No newline at end of file diff --git a/service/features/get_sdc_instances.json b/service/features/get_sdc_instances.json index 78ed8df4..02dc069f 100644 --- a/service/features/get_sdc_instances.json +++ b/service/features/get_sdc_instances.json @@ -331,5 +331,103 @@ "href": "/api/instances/System::14dbbf5617523654" } ] - } + }, + { + "hostOsFullType": "Generic", + "name": "192.168.0.1-513fda498aaf35e883d", + "systemId": "14dbbf5617523654", + "sdcIps": null, + "osType": null, + "sdtId": null, + "sdcApproved": null, + "sdcAgentActive": null, + "mdmIpAddressesCurrent": null, + "sdcIp": null, + "perfProfile": null, + "nqn": "nqn.2014-08.org.nvmexpress:uuid:a2f30442-bcf7-cf5b-d29c-ea4fae6a028b", + "peerMdmId": null, + "versionInfo": null, + "sdcType": null, + "maxNumPaths": 4, + "maxNumSysPorts": 10, + "softwareVersionInfo": null, + "mdmConnectionState": null, + "socketAllocationFailure": null, + "memoryAllocationFailure": null, + "hostType": "NVMeHost", + "sdcGuid": null, + "installedSoftwareVersionInfo": null, + "kernelVersion": null, + "kernelBuildNumber": null, + "sdcApprovedIps": null, + "sdrId": null, + "id": "15d2cc7400010000", + "links": [ + { + "rel": "self", + "href": "/api/instances/Host::15d2cc7400010000" + }, + { + "rel": "/api/Host/relationship/Volume", + "href": "/api/instances/Host::15d2cc7400010000/relationships/Volume" + }, + { + "rel": "/api/Host/relationship/NvmeController", + "href": "/api/instances/Host::15d2cc7400010000/relationships/NvmeController" + }, + { + "rel": "/api/parent/relationship/systemId", + "href": "/api/instances/System::14dbbf5617523654" + } + ] + }, + { + "hostOsFullType": "Generic", + "name": "192.168.0.2-25eb4476103ff9daccc", + "systemId": "14dbbf5617523654", + "sdcIps": null, + "osType": null, + "sdtId": null, + "sdcApproved": null, + "sdcAgentActive": null, + "mdmIpAddressesCurrent": null, + "sdcIp": null, + "perfProfile": null, + "nqn": "nqn.2014-08.org.nvmexpress:uuid:f4740442-g717-hbe4-i858-j6fd71c5c0f0", + "peerMdmId": null, + "versionInfo": null, + "sdcType": null, + "maxNumPaths": 4, + "maxNumSysPorts": 10, + "softwareVersionInfo": null, + "mdmConnectionState": null, + "socketAllocationFailure": null, + "memoryAllocationFailure": null, + "hostType": "NVMeHost", + "sdcGuid": null, + "installedSoftwareVersionInfo": null, + "kernelVersion": null, + "kernelBuildNumber": null, + "sdcApprovedIps": null, + "sdrId": null, + "id": "15d2cc7500010001", + "links": [ + { + "rel": "self", + "href": "/api/instances/Host::15d2cc7500010001" + }, + { + "rel": "/api/Host/relationship/Volume", + "href": "/api/instances/Host::15d2cc7500010001/relationships/Volume" + }, + { + "rel": "/api/Host/relationship/NvmeController", + "href": "/api/instances/Host::15d2cc7500010001/relationships/NvmeController" + }, + { + "rel": "/api/parent/relationship/systemId", + "href": "/api/instances/System::14dbbf5617523654" + } + ] + } ] diff --git a/service/features/get_system_statistics.json b/service/features/get_system_statistics.json index 874e4432..d24e5506 100644 --- a/service/features/get_system_statistics.json +++ b/service/features/get_system_statistics.json @@ -38,6 +38,7 @@ "rfcacheIosOutstanding": 0, "rmcacheBigBlockEvictionSizeCountInKb": 0, "capacityAvailableForVolumeAllocationInKb": 117440512, + "volumeAllocationLimitInKb": 117440512, "numOfMappedToAllVolumes": 0, "numOfScsiInitiators": 0, "rebuildPerReceiveJobNetThrottlingInKbps": 0, diff --git a/service/features/get_volume_metrics.json b/service/features/get_volume_metrics.json new file mode 100644 index 00000000..cda5499c --- /dev/null +++ b/service/features/get_volume_metrics.json @@ -0,0 +1,320 @@ +{ + "format": "ID_TIMESTAMP_METRIC", + "resource_type": "storage_pool", + "timestamps": [ + "2025-11-06T06:06:08Z" + ], + "resources": [ + { + "id": "e4aed8d700000000", + "metrics": [ + { + "name": "avg_host_read_latency", + "values": [ + 0 + ] + }, + { + "name": "physical_free", + "values": [ + 4340531632128 + ] + }, + { + "name": "raw_used", + "values": [ + 8793945538560 + ] + }, + { + "name": "logical_used", + "values": [ + 3607101440 + ] + }, + { + "name": "host_write_bandwidth", + "values": [ + 1 + ] + }, + { + "name": "host_write_iops", + "values": [ + 0 + ] + }, + { + "name": "storage_fe_write_bandwidth", + "values": [ + 0 + ] + }, + { + "name": "storage_fe_write_iops", + "values": [ + 0 + ] + }, + { + "name": "avg_fe_write_io_size", + "values": [ + 0 + ] + }, + { + "name": "storage_fe_read_bandwidth", + "values": [ + 0 + ] + }, + { + "name": "storage_fe_read_iops", + "values": [ + 0 + ] + }, + { + "name": "avg_fe_read_io_size", + "values": [ + 0 + ] + }, + { + "name": "utilization_ratio", + "values": [ + 0.014172318 + ] + }, + { + "name": "host_read_bandwidth", + "values": [ + 1 + ] + }, + { + "name": "host_read_iops", + "values": [ + 0 + ] + }, + { + "name": "compression_reducible_ratio", + "values": [ + 0.0 + ] + }, + { + "name": "data_reduction_ratio", + "values": [ + 0.41805917 + ] + }, + { + "name": "thin_provisioning_ratio", + "values": [ + 85.73023 + ] + }, + { + "name": "avg_wrc_read_latency", + "values": [ + 0 + ] + }, + { + "name": "storage_fe_read_latency", + "values": [ + 0 + ] + }, + { + "name": "unreducible_data", + "values": [ + 8302329856 + ] + }, + { + "name": "avg_wrc_write_latency", + "values": [ + 0 + ] + }, + { + "name": "over_provisioning_limit", + "values": [ + 4611686017353646080 + ] + }, + { + "name": "patterns_saving_ratio", + "values": [ + 1.0 + ] + }, + { + "name": "avg_host_write_latency", + "values": [ + 0 + ] + }, + { + "name": "storage_fe_write_latency", + "values": [ + 0 + ] + }, + { + "name": "logical_provisioned", + "values": [ + 309237645312 + ] + }, + { + "name": "efficiency_ratio", + "values": [ + 35.840313 + ] + }, + { + "name": "storage_fe_trim_latency", + "values": [ + 0 + ] + }, + { + "name": "physical_system", + "values": [ + 53687091200 + ] + }, + { + "name": "data_reduction_reducible_ratio", + "values": [ + 0.0 + ] + }, + { + "name": "compression_ratio", + "values": [ + 0.41805917 + ] + }, + { + "name": "reducible_ratio", + "values": [ + 1.0 + ] + }, + { + "name": "storage_fe_trim_bandwidth", + "values": [ + 0 + ] + }, + { + "name": "storage_fe_trim_iops", + "values": [ + 0 + ] + }, + { + "name": "avg_fe_trim_io_size", + "values": [ + 0 + ] + }, + { + "name": "physical_used", + "values": [ + 8628207616 + ] + }, + { + "name": "snapshot_saving_ratio", + "values": [ + 1.0 + ] + }, + { + "name": "physical_free", + "values": [ + 4334657470464 + ] + }, + { + "name": "host_trim_bandwidth", + "values": [ + 0 + ] + }, + { + "name": "host_trim_iops", + "values": [ + 0 + ] + }, + { + "name": "total_wrc_write_bandwidth", + "values": [ + 0 + ] + }, + { + "name": "total_wrc_write_iops", + "values": [ + 0 + ] + }, + { + "name": "avg_wrc_write_io_size", + "values": [ + 0 + ] + }, + { + "name": "total_wrc_read_bandwidth", + "values": [ + 0 + ] + }, + { + "name": "total_wrc_read_iops", + "values": [ + 0 + ] + }, + { + "name": "avg_wrc_read_io_size", + "values": [ + 0 + ] + }, + { + "name": "physical_total", + "values": [ + 4396972769280 + ] + }, + { + "name": "logical_owned", + "values": [ + 3607101440 + ] + }, + { + "name": "patterns_saving_reducible_ratio", + "values": [ + 0.0 + ] + }, + { + "name": "avg_host_trim_latency", + "values": [ + 0 + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/service/features/node_publish_unpublish.feature b/service/features/node_publish_unpublish.feature index 8be33da0..0882d991 100644 --- a/service/features/node_publish_unpublish.feature +++ b/service/features/node_publish_unpublish.feature @@ -599,3 +599,63 @@ Feature: VxFlex OS CSI interface Then the error contains "none" And I call NodeUnpublishVolume "SDC_GUID" Then the error contains "none" + + Scenario: a Basic NVME Node Publish Volume no error + Given a VxFlexOS service + And a controller published volume + And a capability with voltype access fstype + And get Node Publish Volume Request for NVME + When I call Probe + When I call NodePublishVolume NVME + Then the error contains "none" + + Examples: + | voltype | access | fstype | errormsg | + | "block" | "single-writer" | "none" | "none" | + | "mount" | "single-writer" | "xfs" | "none" | + + Scenario Outline: Node publish NVME volumes various induced error use cases from examples + Given a VxFlexOS service + And a controller published volume + And a capability with voltype access "single-writer" fstype "none" + And get Node Publish Volume Request for NVME + And I induce error + When I call Probe + When I call NodePublishVolume NVME + Then the error contains + + Examples: + | voltype | error | errormsg | + | "block" | "GOFSMockMountError" | "failed to publish NVMe block volume" | + | "mount" | "GOFSMockMountError" | "failed to publish NVMe filesystem volume" | + | "block" | "NodePublishNoTargetPath" | "target path required" | + | "block" | "NodePublishNoStagingTargetPath" | "staging target path required" | + | "block" | "NodePublishNoVolumeCapability" | "volume capability required" | + | "block" | "NodePublishNoAccessMode" | "Volume Access Mode is required" | + | "block" | "NodePublishNoAccessType" | "Volume Access Type is required" | + | "block" | "NodePublishBadTargetPath" | "cannot find the path specified@@no such file or directory" | + +Scenario: a Basic NVME Node Unpublish Volume no error + Given a VxFlexOS service + And a controller published volume + And a capability with voltype "mount" access "single-writer" fstype "xfs" + And get Node Publish Volume Request for NVME "mount" + When I call Probe + When I call NodeUnpublishVolume "NVMeTCP" + Then the error contains "none" + + Scenario Outline: Node unpublish NVME volumes various induced error use cases from examples + Given a VxFlexOS service + And a controller published volume + And a capability with voltype "mount" access "single-writer" fstype "none" + And get Node Publish Volume Request for NVME "mount" + And I induce error + When I call Probe + When I call NodeUnpublishVolume "NVMeTCP" + Then the error contains + + Examples: + | error | errormsg | + | "none" | "none" | + | "GOFSMockGetMounts_targetpath" | "none" | + | "GOFSMockGetMountsError" | "could not reliably determine existing mount" | \ No newline at end of file diff --git a/service/features/node_stage_unstage.feature b/service/features/node_stage_unstage.feature new file mode 100644 index 00000000..2710cb2a --- /dev/null +++ b/service/features/node_stage_unstage.feature @@ -0,0 +1,115 @@ +Feature: VxFlex OS CSI interface + As a consumer of the CSI interface + I want to test list service methods + So that they are known to work + + Scenario Outline: Node stage volume input validation + Given a VxFlexOS service + And I set protocol to + And a capability with voltype "block" access "single-writer" fstype "none" + And get Node Stage Volume Request + And I induce error + When I call Probe + When I call NodeStageVolume + Then the error contains + Examples: + | protocol | error | errormsg | + | "SDC" | "none" | "none" | + | "NVMeTCP" | "NodeStageNoVolumeID" | "volume ID is required" | + | "NVMeTCP" | "NodeStageInValidVolumeID" | "failed to build NGUID: volumeID must be 16 characters" | + | "NVMeTCP" | "NodeStageNoCapability" | "volume capability is required" | + | "NVMeTCP" | "NodeStageNoAccessMode" | "Volume Access Mode is required" | + | "NVMeTCP" | "NodeStageNoStagingPath" | "staging target path is required" | + + Scenario Outline: Node stage block volume various induced error use cases from examples + Given a VxFlexOS service + And I set protocol to "NVMeTCP" + And a capability with voltype "block" access "single-writer" fstype "none" + And I induce error + When I call Probe + When I call NodeStageVolume + Then the error contains + Examples: + | error | errormsg | + | "none" | "none" | + | "GobrickConnectError" | "unable to find device: induced ConnectVolumeError" | + | "GOFSMockGetDiskFormatError" | "failed to probe staging state: disk format probe: getDiskFormat induced error" | + | "GOFSMockGetDiskFormatType_mpath_member" | "none" | + | "GOFSMockGetMounts_deleted" | "none" | + | "GOFSMockGetMountsError" | "failed to probe staging state: get mounts: getMounts induced error" | + + Scenario Outline: Node stage mount volume various induced error use cases from examples + Given a VxFlexOS service + And I set protocol to "NVMeTCP" + And a capability with voltype "mount" access "single-writer" fstype "ext4" + And I induce error + When I call Probe + When I call NodeStageVolume + Then the error contains + Examples: + | error | errormsg | + | "none" | "none" | + | "GobrickConnectError" | "unable to find device: induced ConnectVolumeError" | + | "GOFSMockGetDiskFormatError" | "failed to probe staging state: disk format probe: getDiskFormat induced error" | + | "GOFSMockGetDiskFormatType_mpath_member" | "none" | + | "GOFSMockGetMounts_deleted" | "none" | + | "GOFSMockGetMountsError" | "failed to probe staging state: get mounts: getMounts induced error" | + + Scenario Outline: Node Unstage volume various induced error use cases from examples + Given a VxFlexOS service + And I set protocol to "NVMeTCP" + And a capability with voltype "mount" access "single-writer" fstype "ext4" + And I induce error + When I call Probe + And I call NodeUnstageVolume with "none" + Then the error contains + Examples: + | error | errormsg | + | "none" | "none" | + | "GobrickDisconnectError" | "Failed to disconnect NVME device" | + | "GOFSMockGetMountsError" | "none" | + | "GOFSMockGetMounts_deleted" | "none" | + | "GOFSMockUnmountError" | "none" | + | "GOFSMockGetMounts_unknowndevice" | "none" | + + Scenario Outline: Idempotent Node stage volume + Given a VxFlexOS service + And I set protocol to "NVMeTCP" + And a capability with voltype access "single-writer" fstype "none" + When I call Probe + When I call NodeStageVolume + Then the error contains "none" + When I call NodeStageVolume + Then the error contains "none" + Examples: + | volType | + | "block" | + | "mount" | + + Scenario Outline: Idempotent Node Unstage volume + Given a VxFlexOS service + And I set protocol to "NVMeTCP" + And a capability with voltype access "single-writer" fstype "none" + When I call Probe + When I call NodeUnstageVolume with "none" + Then the error contains "none" + When I call NodeUnstageVolume with "none" + Then the error contains "none" + Examples: + | volType | + | "block" | + | "mount" | + + Scenario Outline: Call NodeUnstageVolume to test podmon functionality + Given a VxFlexOS service + And I call Probe + When I call NodeUnstageVolume with + Then the error contains + Examples: + | error | errormsg | + | "none" | "none" | + | "NoRequestID" | "none" | + | "NoVolumeID" | "Volume ID is required" | + | "NoStagingTarget" | "StagingTargetPath is required" | + | "EphemeralVolume" | "none" | + | "UnmountError" | "Unable to remove staging target path" | \ No newline at end of file diff --git a/service/features/replication.feature b/service/features/replication.feature index 5a2e8401..5be6c0e6 100644 --- a/service/features/replication.feature +++ b/service/features/replication.feature @@ -262,3 +262,11 @@ Scenario Outline: Test ControllerExpandVolume on replication pair | "1srcVol" | 64 | "none" | "none" | | "1srcVol" | 64 | "GetReplicationPairError" | "GET ReplicationPair induced error" | | "1srcVol" | 64 | "GetRCGByIdError" | "could not GET RCG by ID" | + +Scenario: Try to create block volume with Replication on Replication not supported Array + Given a VxFlexOS service + And I use config "replication-config" + When I call CreateVolume "volume1" + And I set Platform Info "5.0" "EC" "5.0" "EC" + And I call CreateRemoteVolume with Error "Replication is not supported on this System" + Then I reset the Platform Info diff --git a/service/features/service.feature b/service/features/service.feature index a3da734b..e8ebd50c 100644 --- a/service/features/service.feature +++ b/service/features/service.feature @@ -23,7 +23,6 @@ Feature: VxFlex OS CSI interface | "14dbbf5617523654" | "none" | | "15dbbf5617523655" | "none" | - Scenario: Identity GetPluginInfo good call Given a VxFlexOS service When I call GetPluginInfo @@ -183,121 +182,167 @@ Feature: VxFlex OS CSI interface Scenario Outline: Create volume good scenario Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume Then a valid CreateVolumeResponse is returned - Examples: - | name | - | "volume1" | - | "thisnameiswaytoolongtopossiblybeunder31characters" | + | name | protocol | + | "volume1" | "SDC" | + | "thisnameiswaytoolongtopossiblybeunder31characters" | "SDC" | + | "volume1" | "NVMeTCP" | + | "thisnameiswaytoolongtopossiblybeunder31characters" | "NVMeTCP" | Scenario: Create volume with admin error Given a VxFlexOS service When I call Probe And I induce error "NoAdminError" + And I set protocol to And I call CreateVolume "volume1" Then a valid CreateVolumeResponse is returned + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Create Volume with invalid probe cache, no endpoint, and no admin Given a VxFlexOS service When I induce error "NoAdminError" And I induce error "NoEndpointError" And I invalidate the Probe cache + And I set protocol to And I call CreateVolume "volume1" Then the error contains "No system ID is found in parameters or as default" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Idempotent create volume with duplicate volume name Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "volume2" And I call CreateVolume "volume2" Then a valid CreateVolumeResponse is returned + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Idempotent create volume with different sizes Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolumeSize "volume3" "8" And I call CreateVolumeSize "volume3" "16" Then the error contains "different size than requested" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Idempotent create volume with different sizes and induced error in handleQueryVolumeIDByKey Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolumeSize "volume3" "8" And I induce error "FindVolumeIDError" And I call CreateVolumeSize "volume3" "16" Then the error contains "induced error" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Idempotent create volume with different sizes and induced error in handleInstances Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolumeSize "volume3" "8" And I induce error "GetVolByIDError" And I call CreateVolumeSize "volume3" "16" Then the error contains "induced error" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Idempotent create volume with different sizes and induced error in handleStoragePoolInstances Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolumeSize "volume3" "8" And I induce error "GetStoragePoolsError" And I call CreateVolumeSize "volume3" "16" Then the error contains "induced error" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Idempotent create volume with different storage pool Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "volume4" And I change the StoragePool "other_storage_pool" And I call CreateVolume "volume4" Then the error contains "different storage pool" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Idempotent create volume with bad storage pool Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "volume4" And I change the StoragePool "no_storage_pool" And I call CreateVolume "volume4" Then the error contains "Couldn't find storage pool" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario Outline: Create volume with Accessibility Requirements Given a VxFlexOS service When I call Probe And I specify AccessibilityRequirements with a SystemID of + And I set protocol to And I call CreateVolume "accessibility" Then the error contains - Examples: - | sysID | errormsg | - | "f.service.opt.SystemName" | "none" | - | "" | "is not accessible based on Preferred" | - | "Unknown" | "is not accessible based on Preferred" | - | "badSystem" | "is not accessible based on Preferred" | + | sysID | errormsg | protocol | + | "f.service.opt.SystemName" | "none" | "SDC" | + | "" | "is not accessible based on Preferred" | "SDC" | + | "Unknown" | "is not accessible based on Preferred" | "SDC" | + | "badSystem" | "is not accessible based on Preferred" | "SDC" | + | "f.service.opt.SystemName" | "none" | "NVMeTCP" | + | "" | "is not accessible based on Preferred" | "NVMeTCP" | + | "Unknown" | "is not accessible based on Preferred" | "NVMeTCP" | + | "badSystem" | "is not accessible based on Preferred" | "NVMeTCP" | Scenario Outline: Create volume with Accessibility Requirements Given a VxFlexOS service When I call Probe + And I set protocol to And I specify AccessibilityRequirements with a SystemID of And I call CreateVolume "accessibility" Then a valid CreateVolumeResponse with topology is returned Examples: - | sysID | - | "f.service.opt.SystemName" | - - + | sysID | protocol | + | "f.service.opt.SystemName" | "SDC" | + | "f.service.opt.SystemName" | "NVMeTCP" | Scenario Outline: Create volume with Accessiblity Requirements NFS volumes Invalid topology error Given a VxFlexOS service When I call Probe - And I specify bad NFS AccessibilityRequirements with a SystemID of + And I specify bad AccessibilityRequirements with a SystemID of "f.service.opt.SystemName" And I call CreateVolume "volume1" Then the error contains "Invalid topology requested for NFS Volume" - Examples: - | sysID | - | "f.service.opt.SystemName" | - - Scenario Outline: Create volume with Accessibility Requirements for NFS volumes with different examples Given a VxFlexOS service @@ -317,36 +362,60 @@ Feature: VxFlex OS CSI interface Given a VxFlexOS service When I call Probe And I specify MULTINODE_WRITER + And I set protocol to And I call CreateVolume "multi-writer" Then a valid CreateVolumeResponse is returned + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Attempt create volume with no name Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "" Then the error contains "Name cannot be empty" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Create volume with bad capacity Given a VxFlexOS service When I call Probe And I specify a BadCapacity + And I set protocol to And I call CreateVolume "bad capacity" Then the error contains "bad capacity" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Create volume with no storage pool Given a VxFlexOS service When I call Probe And I specify NoStoragePool + And I set protocol to And I call CreateVolume "no storage pool" Then the error contains "storagepool is a required parameter" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Create mount volume good scenario Given a VxFlexOS service When I call Probe When I specify CreateVolumeMountRequest "xfs" + And I set protocol to And I call CreateVolume "volume1" Then a valid CreateVolumeResponse is returned - + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Create mount volume NFS no error Given a VxFlexOS service @@ -400,10 +469,14 @@ Feature: VxFlex OS CSI interface Given a VxFlexOS service When I call Probe When I specify CreateVolumeMountRequest "xfs" + And I set protocol to And I call CreateVolume "volume2" And I call CreateVolume "volume2" Then a valid CreateVolumeResponse is returned - + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Create mount volume idempotent NFS no error Given a VxFlexOS service @@ -450,7 +523,7 @@ Feature: VxFlex OS CSI interface Scenario: Call GetNodeUID with invalid node Given a VxFlexOS service When I call GetNodeUID with invalid node - Then the error contains "Unable to fetch the node details" + Then the error contains "unable to fetch node" Scenario: Call NodeGetInfo with invalid volume limit node labels Given a VxFlexOS service @@ -467,6 +540,19 @@ Feature: VxFlex OS CSI interface When I call NodeGetInfo with a valid Node UID Then a valid NodeGetInfoResponse with node UID is returned + Scenario: Call NodeGetInfo with useNVME flag as true + Given a VxFlexOS service + And I induce error + When I call NodeGetInfo with useNVME flag as true + Then a valid NodeGetInfoResponse with in AccessibleTopology is returned + Examples: + | error | protocols | + | "none" | "nfs,nvmetcp" | + | "NoNfsServer" | "nvmetcp" | + | "SdtNotFoundError" | "nfs" | + | "EmptySdtError" | "nfs" | + | "NvmeDiscoveryError" | "nfs" | + Scenario: Call GetNodeUID Given a VxFlexOS service When I call GetNodeUID @@ -482,10 +568,10 @@ Feature: VxFlex OS CSI interface When I call GetNodeLabels with unset KubernetesClient Then the error contains "init client failed with error" - Scenario: Call GetNodeUID with invalid KubernetesClient + Scenario: Call GetNodeUID without KubeNodeName Given a VxFlexOS service - When I call GetNodeUID with unset KubernetesClient - Then the error contains "init client failed with error" + When I call GetNodeUID without KubeNodeName + Then the error contains "node name is empty" Scenario: Call GetCapacity without specifying Storage Pool Name (this returns overall capacity) Given a VxFlexOS service @@ -513,11 +599,32 @@ Feature: VxFlex OS CSI interface Scenario: Call GetMaximumVolumeSize with Systemid Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "volume123" Then a valid CreateVolumeResponse is returned + And I set Provision to And I call GetCapacity with storage pool "viki_pool_HDD_20181031" And I call get GetMaximumVolumeSize with systemid "14dbbf5617523654" Then a valid GetCapacityResponse1 is returned + Examples: + | protocol | provisionType | + | "SDC" | "Thick" | + | "NVMeTCP" | "Thin" | + + Scenario: Call GetMaximumVolumeSize with Systemid Without Storage Pool + Given a VxFlexOS service + When I call Probe + And I call CreateVolume "volume123" + Then a valid CreateVolumeResponse is returned + And I set Provision to + And I call GetCapacity with storage pool "" + And I call get GetMaximumVolumeSize with systemid "14dbbf5617523654" + Then a valid GetCapacityResponse1 is returned + + Examples: + | provisionType | + | "Thick" | + | "Thin" | Scenario: Call GetCapacity with invalid Storage Pool name Given a VxFlexOS service @@ -525,6 +632,28 @@ Feature: VxFlex OS CSI interface And I call GetCapacity with storage pool "xxx" Then the error contains "unable to look up storage pool" + Scenario: Call GetCapacity with invalid Storage Pool name For Gen 2 + Given a VxFlexOS service + When I call Probe + And I set Platform Info "5.0" "EC" "5.0" "EC" + And I call GetCapacity with storage pool "xxx" + Then the error contains "unable to look up storage pool" + Then I reset the Platform Info + + Scenario: Call GetCapacity with induced error retrieving statistics For Gen2 + Given a VxFlexOS service + When I call Probe + And I induce error "GetMetricsError" + And I set Platform Info "5.0" "EC" "5.0" "EC" + And I call GetCapacity with storage pool + Then the error contains "unable to get system stats" + Then I reset the Platform Info + + Examples: + | storagePool | + | "viki_pool_HDD_20181031" | + | "" | + Scenario: Call GetCapacity with induced error retrieving statistics Given a VxFlexOS service When I call Probe @@ -532,6 +661,28 @@ Feature: VxFlex OS CSI interface And I call GetCapacity with storage pool "viki_pool_HDD_20181031" Then the error contains "unable to get system stats" + Scenario: Call GetCapacity For Gen 2 With Storage Pool + Given a VxFlexOS service + When I call Probe + And I call CreateVolume "volume123" + Then a valid CreateVolumeResponse is returned + And I set Platform Info "5.0" "EC" "5.0" "EC" + And I call GetCapacity with storage pool "viki_pool_HDD_20181031" + And I call get GetMaximumVolumeSize with systemid "14dbbf5617523654" + Then a valid GetCapacityResponse1 is returned + Then I reset the Platform Info + + Scenario: Call GetCapacity For Gen 2 Without Storage Pool + Given a VxFlexOS service + When I call Probe + And I call CreateVolume "volume123" + Then a valid CreateVolumeResponse is returned + And I set Platform Info "5.0" "EC" "5.0" "EC" + And I call GetCapacity with storage pool "" + And I call get GetMaximumVolumeSize with systemid "14dbbf5617523654" + Then a valid GetCapacityResponse1 is returned + Then I reset the Platform Info + Scenario: Call ControllerGetCapabilities with health monitor enabled Given a VxFlexOS service When I call ControllerGetCapabilities "true" @@ -545,21 +696,30 @@ Feature: VxFlex OS CSI interface Scenario Outline: Calls to validate volume capabilities Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "volume1" And a valid CreateVolumeResponse is returned And I call ValidateVolumeCapabilities with voltype access fstype Then the error contains Examples: - | voltype | access | fstype | errormsg | - | "block" | "single-writer" | "none" | "none" | - | "block" | "multi-reader" | "none" | "none" | - | "mount" | "multi-writer" | "ext4" | "multi-node with writer(s) only supported for block access type" | - | "mount" | "multi-node-single-writer" | "ext4" | "multi-node with writer(s) only supported for block access type" | - | "mount" | "single-node-single-writer" | "ext4" | "none" | - | "mount" | "single-node-multi-writer" | "ext4" | "none" | - | "mount" | "unknown" | "ext4" | "access mode cannot be UNKNOWN" | - | "none " | "unknown" | "ext4" | "unknown access type is not Block or Mount" | + | voltype | access | fstype | errormsg | protocol | + | "block" | "single-writer" | "none" | "none" | "SDC" | + | "block" | "multi-reader" | "none" | "none" | "SDC" | + | "mount" | "multi-writer" | "ext4" | "multi-node with writer(s) only supported for block access type" | "SDC" | + | "mount" | "multi-node-single-writer" | "ext4" | "multi-node with writer(s) only supported for block access type" | "SDC" | + | "mount" | "single-node-single-writer" | "ext4" | "none" | "SDC" | + | "mount" | "single-node-multi-writer" | "ext4" | "none" | "SDC" | + | "mount" | "unknown" | "ext4" | "access mode cannot be UNKNOWN" | "SDC" | + | "none " | "unknown" | "ext4" | "unknown access type is not Block or Mount" | "SDC" | + | "block" | "single-writer" | "none" | "none" | "NVMeTCP" | + | "block" | "multi-reader" | "none" | "none" | "NVMeTCP" | + | "mount" | "multi-writer" | "ext4" | "multi-node with writer(s) only supported for block access type" | "NVMeTCP" | + | "mount" | "multi-node-single-writer" | "ext4" | "multi-node with writer(s) only supported for block access type" | "NVMeTCP" | + | "mount" | "single-node-single-writer" | "ext4" | "none" | "NVMeTCP" | + | "mount" | "single-node-multi-writer" | "ext4" | "none" | "NVMeTCP" | + | "mount" | "unknown" | "ext4" | "access mode cannot be UNKNOWN" | "NVMeTCP" | + | "none " | "unknown" | "ext4" | "unknown access type is not Block or Mount" | "NVMeTCP" | Scenario Outline: Call validate volume capabilities with non-existent volume Given a VxFlexOS service @@ -585,32 +745,16 @@ Feature: VxFlex OS CSI interface Scenario: Call with ValidateVolumeCapabilities with bad vol ID Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "volume1" And a valid CreateVolumeResponse is returned And I induce error "BadVolIDError" And I call ValidateVolumeCapabilities with voltype "block" access "single-writer" fstype "none" Then the error contains "volume not found" - - Scenario: Call NodeStageVolume, should get unimplemented - Given a VxFlexOS service - And I call Probe - When I call NodeStageVolume - Then the error contains "Unimplemented" - - Scenario Outline: Call NodeUnstageVolume to test podmon functionality - Given a VxFlexOS service - And I call Probe - When I call NodeUnstageVolume with - Then the error contains - Examples: - | error | errormsg | - | "none" | "none" | - | "NoRequestID" | "none" | - | "NoVolumeID" | "Volume ID is required" | - | "NoStagingTarget" | "StagingTargetPath is required" | - | "EphemeralVolume" | "none" | - | "UnmountError" | "Unable to remove staging target path" | + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Call NodeGetCapabilities with health monitor feature enabled Given a VxFlexOS service @@ -627,14 +771,20 @@ Feature: VxFlex OS CSI interface Scenario: Snapshot a single block volume Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "vol1" And a valid CreateVolumeResponse is returned And I call CreateSnapshot "snap1" Then a valid CreateSnapshotResponse is returned + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Idempotent test of snapshot a single block volume Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "vol1" And I induce error And a valid CreateVolumeResponse is returned @@ -644,13 +794,16 @@ Feature: VxFlex OS CSI interface Then the error contains Examples: - | error | errormsg | - | "none" | "none" | - | "BadVolIDJSON" | "Failed to create snapshot -- GetVolume returned unexpected error" | + | error | errormsg | protocol | + | "none" | "none" | "SDC" | + | "BadVolIDJSON" | "Failed to create snapshot -- GetVolume returned unexpected error" | "SDC" | + | "none" | "none" | "NVMeTCP" | + | "BadVolIDJSON" | "Failed to create snapshot -- GetVolume returned unexpected error" | "NVMeTCP" | Scenario: Request to create Snapshot with same name and different SourceVolumeID Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "vol1" And a valid CreateVolumeResponse is returned And I call CreateSnapshot "snap1" @@ -660,6 +813,10 @@ Feature: VxFlex OS CSI interface And I induce error "WrongVolIDError" And I call CreateSnapshot "snap1" Then the error contains "Failed to create snapshot" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Snapshot a single fileSystem Volume Given a VxFlexOS service @@ -708,10 +865,15 @@ Feature: VxFlex OS CSI interface Given a VxFlexOS service When I call Probe And I induce error "CreateSnapshotError" + And I set protocol to And I call CreateVolume "vol1" And a valid CreateVolumeResponse is returned And I call CreateSnapshot "" Then the error contains "snapshot name cannot be Nil" + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Call snapshot create with invalid volume Given a VxFlexOS service @@ -771,6 +933,7 @@ Feature: VxFlex OS CSI interface Scenario: Snapshot a block volume consistency group Given a VxFlexOS service When I call Probe + And I set protocol to And I call CreateVolume "vol1" And a valid CreateVolumeResponse is returned And I call CreateVolume "vol2" @@ -779,6 +942,10 @@ Feature: VxFlex OS CSI interface And a valid CreateVolumeResponse is returned And I call CreateSnapshot "snap1" Then a valid CreateSnapshotResponse is returned + Examples: + | protocol | + | "SDC" | + | "NVMeTCP" | Scenario: Delete a snapshot Given a VxFlexOS service @@ -1036,7 +1203,7 @@ Feature: VxFlex OS CSI interface And I induce error "CreateSnapshotError" When I call Probe And I call Create Volume from Snapshot - Then the error contains "Failed to create snapshot" + Then the error contains "Failed to call CreateSnapshotConsistencyGroup to create volume from snapshot" Scenario: Idempotent create a volume from a snapshot Given a VxFlexOS service @@ -1057,6 +1224,7 @@ Feature: VxFlex OS CSI interface Scenario Outline: Call ControllerExpandVolume Given a VxFlexOS service And I call Probe + And I set protocol to And I call CreateVolumeSize "volume10" "32" And a valid CreateVolumeResponse is returned And I induce error @@ -1066,17 +1234,24 @@ Feature: VxFlex OS CSI interface Then the error contains Examples: - | error | GB | errmsg | - | "none" | 32 | "none" | - | "SetVolumeSizeError" | 64 | "induced error" | - | "none" | 16 | "none" | - | "NoVolumeIDError" | 64 | "volume ID is required" | - | "none" | 64 | "none" | - | "GetVolByIDError" | 64 | "induced error" | - - Scenario Outline: Call NodeExpandVolume with non sysID and no defaultSysID + | error | GB | errmsg | protocol | + | "none" | 32 | "none" | "SDC" | + | "SetVolumeSizeError" | 64 | "induced error" | "SDC" | + | "none" | 16 | "none" | "SDC" | + | "NoVolumeIDError" | 64 | "volume ID is required" | "SDC" | + | "none" | 64 | "none" | "SDC" | + | "GetVolByIDError" | 64 | "induced error" | "SDC" | + | "none" | 32 | "none" | "NVMeTCP" | + | "SetVolumeSizeError" | 64 | "induced error" | "NVMeTCP" | + | "none" | 16 | "none" | "NVMeTCP" | + | "NoVolumeIDError" | 64 | "volume ID is required" | "NVMeTCP" | + | "none" | 64 | "none" | "NVMeTCP" | + | "GetVolByIDError" | 64 | "induced error" | "NVMeTCP" | + + Scenario Outline: Call NodeExpandVolume with non sysID and no defaultSysID for SDC Given setup Get SystemID to fail And a VxFlexOS service + And I set protocol to "SDC" And I call CreateVolumeSize "volume4" "32" And a controller published volume And a capability with voltype "mount" access "single-writer" fstype "xfs" @@ -1085,11 +1260,48 @@ Feature: VxFlex OS CSI interface And I induce error "EmptySysIDInNodeExpand" When I call NodeExpandVolume with volumePath as "test/00000000-1111-0000-0000-000000000000/datadir" Then the error contains "systemID is not found in the request and there is no default system" +| + Scenario Outline: Call NodeExpandVolume for NVMETCP + Given a VxFlexOS service + And I call Probe + And I set protocol to "NVMeTCP" + And I call CreateVolumeSize "volume4" "32" + And a controller published volume + And a capability with voltype "mount" access "single-writer" fstype "xfs" + And get Node Publish Volume Request + And get Node Publish Volume Request for NVME + And no error was received + And I induce error + When I call NodeExpandVolume with volumePath as + Then the error contains + + Examples: + | error | volPath | errormsg | voltype | + | "none" | "" | "Volume path required" | "block" | + | "none" | "test/00000000-1111-0000-0000-000000000000/datadir" | "none" | "block" | + | "GOFSInduceFSTypeError" | "test/00000000-1111-0000-0000-000000000000/datadir" | "Failed to fetch filesystem" | "mount" | + | "GOFSInduceResizeFSError" | "test/00000000-1111-0000-0000-000000000000/datadir" | "Failed to resize device" | "mount" | + Scenario Outline: Call NodeExpandVolume with non sysID and no defaultSysID for NVMETCP + Given setup Get SystemID to fail + And a VxFlexOS service + And I set protocol to + And I call CreateVolumeSize "volume4" "32" + And a controller published volume + And a capability with voltype "mount" access "single-writer" fstype "xfs" + And get Node Publish Volume Request for NVME + And I induce error "EmptySysIDInNodeExpand" + When I call NodeExpandVolume with volumePath as "test/00000000-1111-0000-0000-000000000000/datadir" + Then the error contains "systemID is not found in the request and there is no default system" + Examples: + | protocol | voltype | + | "NVMeTCP" | "block" | + Scenario Outline: Call NodeExpandVolume with invalid volID Given undo setup Get SystemID to fail And a VxFlexOS service And I call Probe + And I set protocol to "SDC" And I call CreateVolumeSize "volume4" "32" And a controller published volume And a capability with voltype "mount" access "single-writer" fstype "xfs" @@ -1103,6 +1315,7 @@ Feature: VxFlex OS CSI interface Scenario Outline: Call NodeExpandVolume Given a VxFlexOS service And I call Probe + And I set protocol to "SDC" And I call CreateVolumeSize "volume4" "32" And a controller published volume And a capability with voltype "mount" access "single-writer" fstype "xfs" @@ -1114,8 +1327,8 @@ Feature: VxFlex OS CSI interface Then the error contains Examples: - | error | volPath | errormsg | - | "none" | "" | "Volume path required" | + | error | volPath | errormsg | + | "none" | "" | "Volume path required" | | "none" | "test/00000000-1111-0000-0000-000000000000/datadir" | "none" | | "GOFSInduceFSTypeError" | "test/00000000-1111-0000-0000-000000000000/datadir" | "Failed to fetch filesystem" | | "GOFSInduceResizeFSError" | "test/00000000-1111-0000-0000-000000000000/datadir" | "Failed to resize device" | @@ -1125,7 +1338,7 @@ Feature: VxFlex OS CSI interface | "CorrectFormatBadCsiVolIDInNodeExpand" | "test/00000000-1111-0000-0000-000000000000/datadir" | "is not configured in the driver" | | "VolumeIDTooShortErrorInNodeExpand" | "test/00000000-1111-0000-0000-000000000000/datadir" | "is shorter than 3 chars, returning error" | | "TooManyDashesVolIDInNodeExpand" | "test/00000000-1111-0000-0000-000000000000/datadir" | "is not configured in the driver" | - + Scenario Outline: Call NodeGetVolumeStats with various errors Given a VxFlexOS service And a controller published volume @@ -1208,6 +1421,7 @@ Feature: VxFlex OS CSI interface Scenario: Idempotent clone of a volume Given a VxFlexOS service And I induce error + And I set protocol to And I call CreateVolume "vol1" And a valid CreateVolumeResponse is returned And I call Clone volume @@ -1216,9 +1430,11 @@ Feature: VxFlex OS CSI interface Then the error contains Examples: - | error | errormsg | - | "none" | "none" | - | "BadVolIDJSON" | "Failed to create clone -- GetVolume returned unexpected error" | + | error | errormsg | protocol | + | "none" | "none" | "SDC" | + | "BadVolIDJSON" | "Failed to create clone -- GetVolume returned unexpected error" | "SDC" | + | "none" | "none" | "NVMeTCP" | + | "BadVolIDJSON" | "Failed to create clone -- GetVolume returned unexpected error" | "NVMeTCP" | Scenario: Clone a volume Given a VxFlexOS service @@ -1411,6 +1627,31 @@ Feature: VxFlex OS CSI interface When I call Node Probe Then the error contains "The given GUID is invalid" + Scenario: Call Probe for NVMe host + Given a VxFlexOS service + And I call Probe + When I call Node Probe with NVMe flag enabled + Then the error contains "none" + + Scenario Outline: getArrayVersion good call + Given a VxFlexOS service + And I call getArrayVersion on + Then the error contains + Examples: + | systemID | errorMsg | + | "14dbbf5617523654" | "none" | + | "14dbbf5617523655" | "unable to get admin client of the array" | + + Scenario: Test setupNVMeHost with NVMe initiators + Given a VxFlexOS service + When I call Probe + And I call setupNVMeHost with NVMe intiators + Then the error contains + Examples: + | nvmeintiators | errorMsg | + | "nqn.123,nqn.124" | "none" | + | "" | "NVMe initiators not found on node" | + Scenario: Approve SDC using defaultSystemID fallback Given a VxFlexOS service And I set approveSDC with approveSDCEnabled "true" @@ -1593,22 +1834,25 @@ Feature: VxFlex OS CSI interface Given a VxFlexOS service And I call Probe And I induce error - When I call check NFS enabled + When I call check NFS enabled Then the error contains Examples: - | systemid | nasserver | error | errorMsg | - | "15dbbf5617523655" | "63ec8e0d-4551-29a7-e79c-b202f2b914f3" | "" | "none" | + | systemid | error | errorMsg | + | "15dbbf5617523655" | "" | "none" | Scenario: Create Volume for multi-available zone Given a VxFlexOS service And I use config When I call Probe + And I set protocol to And I call CreateVolume with zones Then the error contains Examples: - | name | config | errorMsg | - | "volume1" | "multi_az" | "none" | - | "volume1" | "invalid_multi_az" | "no zone topology found in accessibility requirements" | + | name | config | errorMsg | protocol | + | "volume1" | "multi_az" | "none" | "SDC" | + | "volume1" | "invalid_multi_az" | "no zone topology found in accessibility requirements" | "SDC" | + | "volume1" | "multi_az" | "none" | "NVMeTCP" | + | "volume1" | "invalid_multi_az" | "no zone topology found in accessibility requirements" | "NVMeTCP" | Scenario: Call NodeGetInfo without zone label Given a VxFlexOS service @@ -1630,6 +1874,7 @@ Feature: VxFlex OS CSI interface Given a VxFlexOS service And I use config When I call Probe + And I set protocol to And I call CreateVolume "volume1" with zones And a valid CreateVolumeResponse is returned And I call CreateSnapshot @@ -1637,20 +1882,23 @@ Feature: VxFlex OS CSI interface And I call Create Volume for zones from Snapshot Then a valid CreateVolumeResponse is returned Examples: - | name | config | errorMsg | - | "snap1" | "multi_az" | "none" | + | name | config | errorMsg | protocol | + | "snap1" | "multi_az" | "none" | "SDC" | + | "snap1" | "multi_az" | "none" | "NVMeTCP" | Scenario: Clone a single volume in zone Given a VxFlexOS service And I use config When I call Probe + And I set protocol to And I call CreateVolume with zones And a valid CreateVolumeResponse is returned And I call Clone volume for zones Then a valid CreateVolumeResponse is returned Examples: - | name | config | errorMsg | - | "volume1" | "multi_az" | "none" | + | name | config | errorMsg | protocol | + | "volume1" | "multi_az" | "none" | "SDC" | + | "volume1" | "multi_az" | "none" | "NVMeTCP" | Scenario: Probe all systems using availability zones Given a VxFlexOS service diff --git a/service/identity.go b/service/identity.go index 052a92d8..5c856633 100644 --- a/service/identity.go +++ b/service/identity.go @@ -1,4 +1,4 @@ -// Copyright © 2019-2024 Dell Inc. or its subsidiaries. All Rights Reserved. +// Copyright © 2019-2026 Dell Inc. or its subsidiaries. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,11 +19,9 @@ import ( "golang.org/x/net/context" - csi "github.com/container-storage-interface/spec/lib/go/csi" commonext "github.com/dell/dell-csi-extensions/common" + csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/protobuf/types/known/wrapperspb" - - "github.com/dell/csi-vxflexos/v2/core" ) func (s *service) GetPluginInfo( @@ -33,7 +31,7 @@ func (s *service) GetPluginInfo( ) { return &csi.GetPluginInfoResponse{ Name: Name, - VendorVersion: core.SemVer, + VendorVersion: ManifestSemver, Manifest: Manifest, }, nil } @@ -78,23 +76,23 @@ func (s *service) Probe( *csi.ProbeResponse, error, ) { if !strings.EqualFold(s.mode, "node") { - Log.Debug("systemProbe") + log.Debug("systemProbe") if err := s.systemProbeAll(ctx); err != nil { - Log.Printf("error in systemProbeAll: %s", err.Error()) + log.Infof("error in systemProbeAll: %s", err.Error()) return nil, err } } if !strings.EqualFold(s.mode, "controller") { - Log.Debug("nodeProbe") + log.Debug("nodeProbe") if err := s.nodeProbe(ctx); err != nil { - Log.Printf("error in nodeProbe: %s", err.Error()) + log.Infof("error in nodeProbe: %s", err.Error()) return nil, err } } rep := &csi.ProbeResponse{ Ready: wrapperspb.Bool(true), } - Log.Debug(fmt.Sprintf("Probe returning: %v", rep.Ready.GetValue())) + log.Debug(fmt.Sprintf("Probe returning: %v", rep.Ready.GetValue())) return rep, nil } @@ -104,16 +102,18 @@ func (s *service) ProbeController(ctx context.Context, *commonext.ProbeControllerResponse, error, ) { if !strings.EqualFold(s.mode, "node") { - Log.Debug("systemProbe") + log.Debug("systemProbe") if err := s.systemProbeAll(ctx); err != nil { - Log.Printf("error in systemProbeAll: %s", err.Error()) + log.Infof("error in systemProbeAll: %s", err.Error()) return nil, err } } rep := new(commonext.ProbeControllerResponse) rep.Name = Name - rep.VendorVersion = core.SemVer + rep.VendorVersion = ManifestSemver + Manifest["semver"] = ManifestSemver + rep.Manifest = Manifest return rep, nil diff --git a/service/mount.go b/service/mount.go index d7f2aa15..f4d608ab 100644 --- a/service/mount.go +++ b/service/mount.go @@ -21,16 +21,16 @@ import ( "strings" "time" - "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/dell/csmlog" "github.com/dell/gofsutil" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/google/uuid" - "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // Variables set only for unit testing. -var unitTestEmulateBlockDevice bool +var unitTestEmulateBlockDevice = false // Variables populdated from the environment var mountAllowRWOMultiPodAccess bool @@ -56,17 +56,16 @@ func GetDevice(path string) (*Device, error) { return nil, err } - // TODO does EvalSymlinks throw error if link is to non- - // existent file? assuming so by masking error below - ds, _ := os.Stat(d) - dm := ds.Mode() - if unitTestEmulateBlockDevice { - // For unit testing only, emulate a block device on windows - dm = dm | os.ModeDevice - } - if dm&os.ModeDevice == 0 { - return nil, fmt.Errorf( - "%s is not a block device", path) + if !unitTestEmulateBlockDevice { + // TODO does EvalSymlinks throw error if link is to non- + // existent file? assuming so by masking error below + ds, _ := os.Stat(d) + dm := ds.Mode() + + if dm&os.ModeDevice == 0 { + return nil, fmt.Errorf( + "%s is not a block device", path) + } } return &Device{ @@ -123,7 +122,7 @@ func publishVolume( // K8S probably removed part of the path. PrivtgtErr := cleanupPrivateTarget(sysDevice, reqID, privTgt) if PrivtgtErr != nil { - Log.Infof("Error removing private target or directory: %s", privTgt) + log.Errorf("Error removing private target or directory: %s", privTgt) } return status.Error(codes.FailedPrecondition, fmt.Sprintf("Could not create %s: %s", target, err.Error())) } @@ -144,7 +143,7 @@ func publishVolume( // check that target is right type for vol type // Path to mount device to - f := logrus.Fields{ + f := csmlog.Fields{ "id": id, "volumePath": sysDevice.FullPath, "device": sysDevice.RealDev, @@ -152,7 +151,7 @@ func publishVolume( "target": target, "privateMount": privTgt, } - Log.WithFields(f).Debugf("fields") + log.WithFields(f).Debugf("fields") ctx := context.WithValue(context.Background(), gofsutil.ContextKey("RequestID"), reqID) @@ -166,7 +165,7 @@ func publishVolume( if len(devMnts) == 0 { // Device isn't mounted anywhere, do the private mount - Log.WithFields(f).Printf("attempting mount to private area") + log.WithFields(f).Infof("attempting mount to private area") // Make sure private mount point exists created, err := mkdir(privTgt) @@ -177,7 +176,7 @@ func publishVolume( } alreadyMounted := false if !created { - Log.WithFields(f).Printf("directory for private mount target already exists") + log.WithFields(f).Infof("directory for private mount target already exists") // The place where our device is supposed to be mounted // already exists, but we also know that our device is not mounted anywhere @@ -196,11 +195,10 @@ func publishVolume( } for _, m := range mnts { if m.Path == privTgt { - Log.Debug(fmt.Sprintf("MOUNT: %#v", m)) + log.Debug(fmt.Sprintf("MOUNT: %#v", m)) resolvedMountDevice := evalSymlinks(m.Device) if resolvedMountDevice != sysDevice.RealDev { - Log.WithFields(f).WithField("mountedDevice", m.Device).Error( - "mount point already in use by device") + log.WithFields(f).WithFields(csmlog.Fields{"mountedDevice": m.Device}).Error("mount point already in use by device") return status.Error(codes.Internal, "Mount point already in use by device") } @@ -224,7 +222,7 @@ func publishVolume( // K8S may have removed the desired mount point. Clean up the private target. PrivtgtErr := cleanupPrivateTarget(sysDevice, reqID, privTgt) if PrivtgtErr != nil { - Log.Infof("Error removing private target or directory: %s", privTgt) + log.Infof("Error removing private target or directory: %s", privTgt) } return err } @@ -236,18 +234,18 @@ func publishVolume( mounted := false for _, m := range devMnts { if m.Path == target { - Log.Printf("mount %#v already mounted to requested target %s", m, target) + log.Infof("mount %#v already mounted to requested target %s", m, target) } else if m.Path == privTgt { - Log.WithFields(f).Printf("mount Path %s Source %s Device %s Opts %v", m.Path, m.Source, m.Device, m.Opts) + log.WithFields(f).Infof("mount Path %s Source %s Device %s Opts %v", m.Path, m.Source, m.Device, m.Opts) mounted = true rwo := multiAccessFlag if ro { rwo = "ro" } if rwo == "" || contains(m.Opts, rwo) { - Log.WithFields(f).Printf("private mount already in place") + log.WithFields(f).Infof("private mount already in place") } else { - Log.WithFields(f).Printf("mount %#v rwo %s", m, rwo) + log.WithFields(f).Infof("mount %#v rwo %s", m, rwo) return status.Error(codes.InvalidArgument, "Access mode conflicts with existing mounts") } @@ -263,7 +261,7 @@ func publishVolume( } // Private mount in place, now bind mount to target path - targetMnts, err := getPathMounts(target) + targetMnts, err := getPathMounts(ctx, target) if err != nil { return status.Errorf(codes.Internal, "could not reliably determine existing mount status: %s", @@ -282,12 +280,12 @@ func publishVolume( rwo = "ro" } if rwo != "" && !contains(m.Opts, rwo) { - Log.WithFields(f).Printf("mount %#v rwo %s\n", m, rwo) + log.WithFields(f).Infof("mount %#v rwo %s\n", m, rwo) return status.Error(codes.Internal, "volume previously published with different options") } // Existing mount satisfies request - Log.WithFields(f).Debug("volume already published to target") + log.WithFields(f).Debug("volume already published to target") return nil } } @@ -301,7 +299,7 @@ func publishVolume( // K8S probably removed part of the path. PrivtgtErr := cleanupPrivateTarget(sysDevice, reqID, privTgt) if PrivtgtErr != nil { - Log.Infof("Error removing private target or directory: %s", privTgt) + log.Infof("Error removing private target or directory: %s", privTgt) } return status.Error(codes.FailedPrecondition, fmt.Sprintf("Could not create %s: %s", target, err.Error())) } @@ -320,7 +318,7 @@ func publishVolume( // K8S probably removed part of the path. PrivtgtErr := cleanupPrivateTarget(sysDevice, reqID, privTgt) if PrivtgtErr != nil { - Log.Infof("Error removing private target or directory: %s", privTgt) + log.Infof("Error removing private target or directory: %s", privTgt) } return status.Errorf(codes.Internal, "error publish volume to target path: %s", @@ -330,6 +328,110 @@ func publishVolume( return nil } +// publishNVMEVolume bind mounts the nvme device to the target path +func publishNVMEVolume(req *csi.NodePublishVolumeRequest, reqID, devicePath string) error { + volID := req.GetVolumeId() + + targetPath := req.GetTargetPath() + if targetPath == "" { + return status.Error(codes.InvalidArgument, "target path required") + } + + f := csmlog.Fields{ + "CSIRequestID": reqID, + "VolumeID": volID, + "DevicePath": devicePath, + "TargetPath": targetPath, + } + + ctx := context.WithValue(context.Background(), gofsutil.ContextKey("RequestID"), reqID) + published, err := isAlreadyPublished(ctx, targetPath) + if err != nil { + return err + } + + if published { + log.WithFields(f).Infof("NVME volume: %s is already published", volID) + return nil + } + + stagingPath := req.GetStagingTargetPath() + if stagingPath == "" { + return status.Error(codes.InvalidArgument, "staging target path required") + } + f["StagingPath"] = stagingPath + + ro := req.GetReadonly() + + volCap := req.GetVolumeCapability() + if volCap == nil { + return status.Error(codes.InvalidArgument, + "volume capability required") + } + + isBlock, mntVol, accMode, _, err := validateVolumeCapability(volCap, ro) + if err != nil { + return err + } + + // Ensure target path exists + if err := createTarget(targetPath, isBlock); err != nil { + return status.Errorf(codes.FailedPrecondition, "Could not create target %s: %s", targetPath, err.Error()) + } + log.WithFields(f).Info("target path successfully created") + + if isBlock { + log.WithFields(f).Info("start publishing as block device") + + if ro { + return status.Error(codes.InvalidArgument, "read only not supported for Block Volume") + } + + sysDevice, err := GetDevice(devicePath) + if err != nil { + return status.Errorf(codes.Internal, "error getting block device for volume %s: %s", volID, err.Error()) + } + + mntFlags := mntVol.GetMountFlags() + err = mountBlock(sysDevice, targetPath, mntFlags, singleAccessMode(accMode)) + if err != nil { + return status.Errorf(codes.Internal, "failed to publish NVMe block volume: %s", err.Error()) + } + } else { + log.WithFields(f).Info("start publishing as filesystem volume") + + mntFlags := mntVol.GetMountFlags() + targetFS := mntVol.FsType + if targetFS == "xfs" { + mntFlags = append(mntFlags, "nouuid") + } + if ro { + mntFlags = append(mntFlags, "ro") + } + + err = gofsutil.BindMount(ctx, stagingPath, targetPath, mntFlags...) + if err != nil { + return status.Errorf(codes.Internal, "failed to publish NVMe filesystem volume: %s", err.Error()) + } + } + + log.WithFields(f).Info("Successfully published NVMe volume") + + return nil +} + +func isAlreadyPublished(ctx context.Context, targetPath string) (bool, error) { + mounts, err := getPathMounts(ctx, targetPath) + if err != nil { + return false, status.Errorf(codes.Internal, + "can't check mounts for path %s: %s", targetPath, err.Error()) + } + if len(mounts) == 0 { + return false, nil + } + return true, nil +} + // publishNFS mounts the NFS Volume to the targetpath func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, nfsExportURL string) error { volCap := req.GetVolumeCapability() @@ -354,7 +456,7 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, nfsExpor var mntOptions []string mntOptions = mountVol.GetMountFlags() - Log.Infof("The mountOptions received are: %s", mntOptions) + log.Infof("The mountOptions received are: %s", mntOptions) target := req.GetTargetPath() if target == "" { @@ -381,7 +483,7 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, nfsExpor "ExportPath": nfsExportURL, "AccessMode": am.GetMode(), } - Log.WithFields(fields).Info("Node publish volume params ") + log.WithFields(fields).Info("Node publish volume params ") mnts, err := gofsutil.GetMounts(ctx) if err != nil { @@ -398,33 +500,33 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, nfsExpor if m.Path == target { // as per specs, T1=T2, P1=P2 - return OK if contains(m.Opts, rwOption) { - Log.WithFields(fields).Debug( + log.WithFields(fields).Debug( "mount already in place with same options") return nil } // T1=T2, P1!=P2 - return AlreadyExists - Log.WithFields(fields).Error("Mount point already in use by device with different options") + log.WithFields(fields).Error("Mount point already in use by device with different options") return status.Error(codes.AlreadyExists, "Mount point already in use by device with different options") } // T1!=T2, P1==P2 || P1 != P2 - return FailedPrecondition for single node if am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER || am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY || am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER { - Log.WithFields(fields).Error("Mount point already in use for same device") + log.WithFields(fields).Error("Mount point already in use for same device") return status.Error(codes.FailedPrecondition, "Mount point already in use for same device") } } } } - Log.Infof("The mountOptions being used for mount are: %s", mntOptions) + log.Infof("The mountOptions being used for mount are: %s", mntOptions) if err := gofsutil.Mount(context.Background(), nfsExportURL, target, "nfs", mntOptions...); err != nil { count := 0 errmsg := err.Error() // Both substring validation is for NFSv3 and NFSv4 errors resp. for (strings.Contains(strings.ToLower(errmsg), "access denied by server while mounting") || (strings.Contains(strings.ToLower(errmsg), "no such file or directory"))) && count < 5 { time.Sleep(2 * time.Second) - Log.Infof("Mount re-trial attempt-%d", count) + log.Infof("Mount re-trial attempt-%d", count) err = gofsutil.Mount(context.Background(), nfsExportURL, target, "nfs", mntOptions...) if err != nil { errmsg = err.Error() @@ -434,7 +536,7 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, nfsExpor count++ } if err != nil { - Log.Errorf("%v", err) + log.Errorf("%v", err) return err } } @@ -444,7 +546,7 @@ func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, nfsExpor func unpublishNFS(ctx context.Context, req *csi.NodeUnpublishVolumeRequest, filterStr string) error { target := req.GetTargetPath() - Log.Debugf("attempting to unmount '%s'", target) + log.Debugf("attempting to unmount '%s'", target) isMounted, err := isVolumeMounted(ctx, filterStr, target) if err != nil { return err @@ -456,7 +558,7 @@ func unpublishNFS(ctx context.Context, req *csi.NodeUnpublishVolumeRequest, filt return status.Errorf(codes.Internal, "error unmounting target'%s': '%s'", target, err.Error()) } - Log.Debugf("unmounting '%s' succeeded", target) + log.Debugf("unmounting '%s' succeeded", target) return nil } @@ -478,12 +580,12 @@ func isVolumeMounted(ctx context.Context, filterStr string, target string) (bool } } } - Log.Debugf("target '%s' does not exist", target) + log.Debugf("target '%s' does not exist", target) return false, nil } // No mount exists also means not published - Log.Debugf("target '%s' does not exist", target) + log.Debugf("target '%s' does not exist", target) return false, nil } @@ -542,22 +644,20 @@ func contains(list []string, item string) bool { func mkfile(path string) (bool, error) { st, err := os.Stat(path) if err != nil { - Log.Warnf("Unable to check stat of file: %s with error: %v", path, err.Error()) + log.Warnf("Unable to check stat of file: %s with error: %v", path, err.Error()) if os.IsNotExist(err) { /* #nosec G302 G304 */ file, err := os.OpenFile(path, os.O_CREATE, 0o755) if err != nil { - Log.WithField("dir", path).WithError( - err).Error("Unable to create dir") + log.Errorf("Unable to create dir: %s error: %s", path, err.Error()) return false, err } err = file.Close() if err != nil { - // Log the error but keep going - Log.WithField("file", path).WithError( - err).Error("Unable to close file") + // csmlog the error but keep going + log.Errorf("Unable to close file: %s error: %s", path, err.Error()) } - Log.WithField("path", path).Debug("created file") + log.Debugf("created file: %s", path) return true, nil } return false, err @@ -573,15 +673,15 @@ func mkfile(path string) (bool, error) { func mkdir(path string) (bool, error) { st, err := os.Stat(path) if err != nil { - Log.Warnf("Unable to check stat of file: %s with error: %v", path, err.Error()) + log.Warnf("Unable to check stat of file: %s with error: %v", path, err.Error()) if os.IsNotExist(err) { err := os.Mkdir(path, 0o755) // #nosec G301 if err != nil { - Log.WithField("dir", path).WithError( - err).Error("Unable to create dir") + log.WithFields(csmlog.Fields{"dir": path}).Error("Unable to create dir" + err.Error()) return false, err } - Log.WithField("path", path).Debug("created directory") + log.WithFields(csmlog.Fields{"path": path}).Debug("created directory") + return true, nil } return false, err @@ -618,7 +718,7 @@ func unpublishVolume( // Path to mount device to privTgt := getPrivateMountPoint(privDir, volumeID) - f := logrus.Fields{ + f := csmlog.Fields{ "device": sysDevice.RealDev, "privTgt": privTgt, "CSIRequestID": reqID, @@ -640,28 +740,28 @@ func unpublishVolume( if m.Source == sysDevice.RealDev || m.Device == sysDevice.RealDev || m.Device == sysDevice.FullPath { if m.Path == privTgt { privMntExist = true - Log.Printf("Found private mount for device %#v, private mount path: %s .", sysDevice, privTgt) + log.Infof("Found private mount for device %#v, private mount path: %s .", sysDevice, privTgt) } else if m.Path == targetPath { tgtMntExist = true deviceMount = m - Log.Printf("Found target mount for device %#v, target mount path: %s .", sysDevice, targetPath) + log.Infof("Found target mount for device %#v, target mount path: %s .", sysDevice, targetPath) } else { // Check if this is a target mount for another pod in which case we should not unmount the private target thisPodID := getPodIDFromTargetPath(targetPath) thatPodID := getPodIDFromTargetPath(m.Path) if thisPodID != "" && thatPodID != "" && thisPodID != thatPodID { - Log.Infof("Will not unmount the private mount since another pod is using this volume: %s", m.Path) + log.Infof("Will not unmount the private mount since another pod is using this volume: %s", m.Path) keepPrivMnt = true } } } } if tgtMntExist && !privMntExist { - Log.Warnf("Device %#v has target mount without private mount. Target mount %#v", sysDevice, deviceMount) + log.Warnf("Device %#v has target mount without private mount. Target mount %#v", sysDevice, deviceMount) } if tgtMntExist { - Log.WithFields(f).Debug(fmt.Sprintf("Unmounting %s", targetPath)) + log.WithFields(f).Debug(fmt.Sprintf("Unmounting %s", targetPath)) if err := gofsutil.Unmount(ctx, targetPath); err != nil { return status.Errorf(codes.Internal, "Error unmounting target: %s", err.Error()) @@ -673,7 +773,7 @@ func unpublishVolume( } if privMntExist && !keepPrivMnt { - Log.WithFields(f).Debug(fmt.Sprintf("Unmounting %s", privTgt)) + log.WithFields(f).Debug(fmt.Sprintf("Unmounting %s", privTgt)) if err := unmountPrivMount(ctx, sysDevice, privTgt); err != nil { return status.Errorf(codes.Internal, "Error unmounting private mount: %s", err.Error()) @@ -683,6 +783,54 @@ func unpublishVolume( return nil } +func unpublishNVMEVolume( + volumeID, targetPath string, + reqID string, +) error { + ctx := context.Background() + + mounts, err := gofsutil.GetMounts(ctx) + if err != nil { + return status.Errorf(codes.Internal, + "could not reliably determine existing mount status: %s", + err.Error()) + } + + tgtMntExist := false + for _, mount := range mounts { + if mount.Path == targetPath { + log.Infof("found mount for target path: %s", targetPath) + tgtMntExist = true + break + } + } + + f := csmlog.Fields{ + "CSIRequestID": reqID, + "VolumeID": volumeID, + "TargetPath": targetPath, + } + + if !tgtMntExist { + log.WithFields(f).Info("no mounts found, unpublish complete") + return nil + } + + log.Infof("Unmounting %s", targetPath) + if err := gofsutil.Unmount(ctx, targetPath); err != nil { + return status.Errorf(codes.Internal, + "Error unmounting target: %s", err.Error()) + } + if err := removeWithRetry(targetPath); err != nil { + return status.Errorf(codes.Internal, + "Error remove target folder: %s", err.Error()) + } + + log.WithFields(f).Info("Successfully unpublished NVMe volume") + + return nil +} + var getTargetPathPrefix = func() string { return "/var/lib/kubelet/pods/" } @@ -720,10 +868,10 @@ func unmountPrivMount( if err := gofsutil.Unmount(ctx, target); err != nil { return err } - Log.WithField("directory", target).Debug( - "removing directory") + log.WithFields(csmlog.Fields{"directory": target}).Debug("created directory") + if err := os.Remove(target); err != nil { - Log.Errorf("Unable to remove directory: %v", err) + log.Errorf("Unable to remove directory: %v", err) } } return nil @@ -753,8 +901,7 @@ func replaceBackslashWithSlash(input string) string { } // getPathMounts finds all the mounts for a given path. -func getPathMounts(path string) ([]gofsutil.Info, error) { - ctx := context.Background() +func getPathMounts(ctx context.Context, path string) ([]gofsutil.Info, error) { devMnts := make([]gofsutil.Info, 0) mnts, err := gofsutil.GetMounts(ctx) @@ -774,10 +921,10 @@ func removeWithRetry(target string) error { for i := 0; i < 3; i++ { err = os.Remove(target) if err != nil && !os.IsNotExist(err) { - Log.Error("error removing private mount target: " + err.Error()) + log.Error("error removing private mount target: " + err.Error()) err = os.RemoveAll(target) if err != nil { - Log.Errorf("Error removing directory: %v", err.Error()) + log.Errorf("Error removing directory: %v", err.Error()) } time.Sleep(3 * time.Second) } else { @@ -797,7 +944,7 @@ func evalSymlinks(path string) string { // eval any symlinks and make sure it points to a device d, err := filepath.EvalSymlinks(path) if err != nil { - Log.Error("Could not evaluate symlinks for path: " + path) + log.Error("Could not evaluate symlinks for path: " + path) return path } return d @@ -832,7 +979,7 @@ func validateVolumeCapability(volCap *csi.VolumeCapability, readOnly bool) (bool multiAccessFlag = "rw" } if readOnly { - Log.Warnf("read only for Block Volume is not recommended") + log.Warnf("read only for Block Volume is not recommended") } } mntVol = volCap.GetMount() @@ -898,18 +1045,18 @@ func createTarget(target string, isBlock bool) error { // cleanupPrivateTarget unmounts and removes the private directory for the retry so clean start next time. func cleanupPrivateTarget(dev *Device, reqID, privTgt string) error { - Log.WithField("CSIRequestID", reqID).WithField("privTgt", privTgt).Info("Cleaning up private target") + log.WithFields(csmlog.Fields{"CSIRequestID": reqID}).WithFields(csmlog.Fields{"privTgt": privTgt}).Info("Cleaning up private target") mnts, err := getDevMounts(dev) if err != nil { return err } if len(mnts) == 1 && mnts[0].Path == privTgt { if privErr := gofsutil.Unmount(context.Background(), privTgt); privErr != nil { - Log.WithField("CSIRequestID", reqID).Printf("Error unmounting privTgt %s: %s", privTgt, privErr) + log.Errorf("Error unmounting privTgt %s: %s", privTgt, privErr) return privErr } if privErr := removeWithRetry(privTgt); privErr != nil { - Log.WithField("CSIRequestID", reqID).Printf("Error removing privTgt %s: %s", privTgt, privErr) + log.Errorf("Error removing privTgt %s: %s", privTgt, privErr) return privErr } } else if len(mnts) == 0 { @@ -919,12 +1066,12 @@ func cleanupPrivateTarget(dev *Device, reqID, privTgt string) error { } if st.IsDir() { if privErr := removeWithRetry(privTgt); privErr != nil { - Log.WithField("CSIRequestID", reqID).Printf("Error removing privTgt %s: %s", privTgt, privErr) + log.Errorf("Error removing privTgt %s: %s", privTgt, privErr) return privErr } } } else { - Log.WithField("CSIRequestID", reqID).Printf("Cannot delete private mount because there are target mounts : %s", privTgt) + log.Infof("Cannot delete private mount because there are target mounts : %s", privTgt) return status.Error(codes.Internal, "Cannot delete private mount as target mount exist") } return nil @@ -932,7 +1079,7 @@ func cleanupPrivateTarget(dev *Device, reqID, privTgt string) error { // mountBlock bind mounts the device to the required target func mountBlock(device *Device, target string, mntFlags []string, singleAccess bool) error { - Log.Printf("mountBlock called device %#v target %s mntFlags %#v", device, target, mntFlags) + log.Infof("mountBlock called device %#v target %s mntFlags %#v", device, target, mntFlags) // Check to see if already mounted mnts, err := getDevMounts(device) if err != nil { @@ -940,7 +1087,7 @@ func mountBlock(device *Device, target string, mntFlags []string, singleAccess b } for _, mnt := range mnts { if mnt.Path == target { - Log.Info("Block volume target is already mounted") + log.Info("Block volume target is already mounted") return nil } else if singleAccess { return status.Error(codes.InvalidArgument, "Access mode conflicts with existing mounts") diff --git a/service/node.go b/service/node.go index 55ca6529..0db29416 100644 --- a/service/node.go +++ b/service/node.go @@ -20,15 +20,16 @@ import ( "net" "os" "os/exec" + "path/filepath" "strconv" "strings" "time" - csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/dell/csmlog" "github.com/dell/gofsutil" "github.com/dell/goscaleio" siotypes "github.com/dell/goscaleio/types/v1" - "github.com/sirupsen/logrus" + csi "github.com/container-storage-interface/spec/lib/go/csi" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -51,12 +52,61 @@ const ( maxVxflexosVolumesPerNodeLabel = "max-vxflexos-volumes-per-node" ) -func (s *service) NodeStageVolume( - _ context.Context, - _ *csi.NodeStageVolumeRequest) ( - *csi.NodeStageVolumeResponse, error, -) { - return nil, status.Error(codes.Unimplemented, "") +func (s *service) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + if !s.useNVME { + // This stage path is a no-op for SDC nodes + // Return OK to preserve idempotency semantics if upper layers still call Stage. + return &csi.NodeStageVolumeResponse{}, nil + } + + logFields := csmlog.ExtractFieldsFromContext(ctx) + + if req.GetVolumeCapability() == nil { + return nil, status.Error(codes.InvalidArgument, "volume capability is required") + } + + csiVolID := req.GetVolumeId() + if csiVolID == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + + stagingPath := req.GetStagingTargetPath() + if stagingPath == "" { + return nil, status.Error(codes.InvalidArgument, "staging target path is required") + } + + volID := getVolumeIDFromCsiVolumeID(csiVolID) + log.Infof("[NodeStageVolume] volumeID: %s", volID) + + systemID := s.getSystemIDFromCsiVolumeID(csiVolID) + log.Infof("[NodeStageVolume] systemID: %s harvested from csiVolID: %s", systemID, csiVolID) + if systemID == "" { + systemID = s.opts.defaultSystemID + } + if systemID == "" { + return nil, status.Error(codes.InvalidArgument, "systemID is not found in the request and there is no default system") + } + + log.Infof("[NodeStageVolume] We are about to probe the system with systemID %s", systemID) + // Probe the system to make sure it is managed by driver + if err := s.requireProbe(ctx, systemID); err != nil { + return nil, err + } + + if err := s.discoverAndConnectNVMeTargets(s.systems[systemID]); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + var stager VolumeStager + stager = &NVMeStager{ + useNVME: s.useNVME, + systemID: systemID, + nvmeConnector: s.nvmeConnector, + targetNqn: s.nvmeTargetNqn, + adminClient: s.adminClients[systemID], + } + response, err := stager.Stage(ctx, req, stagingPath, logFields, volID) + return response, err } // NodeUnstageVolume will cleanup the staging path passed in the request. @@ -94,19 +144,30 @@ func (s *service) NodeUnstageVolume( // Skip ephemeral volumes. For ephemeral volumes, kubernetes gives us an internal ID, so we use the lockfile to find the Powerflex ID this is mapped to. lockFile := ephemeralStagingMountPath + csiVolID + "/id" if s.fileExist(lockFile) { - Log.WithFields(fields).Info("Skipping ephemeral volume") + log.WithFields(fields).Info("Skipping ephemeral volume") return &csi.NodeUnstageVolumeResponse{}, nil } + // Calling NVMeStager to unstage volume + if s.useNVME { + var stager VolumeStager + stager = &NVMeStager{ + useNVME: s.useNVME, + nvmeConnector: s.nvmeConnector, + } + response, err := stager.Unstage(ctx, stagingTargetPath, fields, csiVolID) + return response, err + } + // Unmount the staging target path. - Log.WithFields(fields).Info("unmounting directory") + log.WithFields(fields).Info("unmounting directory") if err := gofsutil.Unmount(ctx, stagingTargetPath); err != nil && !os.IsNotExist(err) { - Log.Errorf("Unable to Unmount staging target path: %s", err) + log.Errorf("Unable to Unmount staging target path: %s", err) } - Log.WithFields(fields).Info("removing directory") + log.WithFields(fields).Info("removing directory") if err := os.Remove(stagingTargetPath); err != nil && !os.IsNotExist(err) { - Log.Errorf("Unable to remove staging target path: %v", err) + log.Errorf("Unable to remove staging target path: %v", err) err := fmt.Errorf("Unable to remove staging target path: %s error: %v", stagingTargetPath, err) return &csi.NodeUnstageVolumeResponse{}, err } @@ -129,9 +190,9 @@ func (s *service) NodePublishVolume( s.logStatistics() volumeContext := req.GetVolumeContext() if volumeContext != nil { - Log.Info("VolumeContext:") + log.Info("VolumeContext:") for key, value := range volumeContext { - Log.WithFields(logrus.Fields{key: value}).Info("found in VolumeContext") + log.WithFields(csmlog.Fields{key: value}).Info("found in VolumeContext") } } @@ -139,7 +200,7 @@ func (s *service) NodePublishVolume( if ok && strings.ToLower(ephemeral) == "true" { resp, err := s.ephemeralNodePublish(ctx, req) if err != nil { - Log.Errorf("ephemeralNodePublish returned error: %v", err) + log.Errorf("ephemeralNodePublish returned error: %v", err) } return resp, err } @@ -149,7 +210,7 @@ func (s *service) NodePublishVolume( return nil, status.Error(codes.InvalidArgument, "volume ID is required") } - Log.Printf("[NodePublishVolume] csiVolID: %s", csiVolID) + log.Infof("[NodePublishVolume] csiVolID: %s", csiVolID) // Check for NFS protocol fsType := volumeContext[KeyFsType] @@ -159,10 +220,10 @@ func (s *service) NodePublishVolume( } volID := getVolumeIDFromCsiVolumeID(csiVolID) - Log.Printf("[NodePublishVolume] volumeID: %s", volID) + log.Infof("[NodePublishVolume] volumeID: %s", volID) systemID := s.getSystemIDFromCsiVolumeID(csiVolID) - Log.Printf("[NodePublishVolume] systemID: %s harvested from csiVolID: %s", systemID, csiVolID) + log.Infof("[NodePublishVolume] systemID: %s harvested from csiVolID: %s", systemID, csiVolID) if systemID == "" { // use default system systemID = s.opts.defaultSystemID @@ -172,7 +233,7 @@ func (s *service) NodePublishVolume( "systemID is not found in the request and there is no default system") } - Log.Printf("[NodePublishVolume] We are about to probe the system with systemID %s", systemID) + log.Infof("[NodePublishVolume] We are about to probe the system with systemID %s", systemID) // Probe the system to make sure it is managed by driver if err := s.requireProbe(ctx, systemID); err != nil { return nil, err @@ -219,13 +280,31 @@ func (s *service) NodePublishVolume( return &csi.NodePublishVolumeResponse{}, nil } - sdcMappedVol, err := s.getSDCMappedVol(volID, systemID, publishGetMappedVolMaxRetry) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } + if s.useNVME { + nguid, err := buildNGUID(volID, systemID) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to build NGUID: %s", err.Error()) + } - if err := publishVolume(req, s.privDir, sdcMappedVol.SdcDevice, reqID); err != nil { - return nil, err + symlinkPath, _, err := gofsutil.WWNToDevicePathX(context.Background(), nguid) + if err != nil || symlinkPath == "" { + errmsg := fmt.Sprintf("device path not found for nguid %s: %s", nguid, err) + log.Error(errmsg) + return nil, status.Error(codes.NotFound, errmsg) + } + + if err := publishNVMEVolume(req, reqID, symlinkPath); err != nil { + return nil, err + } + } else { + sdcMappedVol, err := s.getSDCMappedVol(volID, systemID, publishGetMappedVolMaxRetry) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + if err := publishVolume(req, s.privDir, sdcMappedVol.SdcDevice, reqID); err != nil { + return nil, err + } } return &csi.NodePublishVolumeResponse{}, nil @@ -260,32 +339,33 @@ func (s *service) NodeUnpublishVolume( isNFS := strings.Contains(csiVolID, "/") var ephemeralVolume bool // For ephemeral volumes, kubernetes gives us an internal ID, so we need to use the lockfile to find the Powerflex ID this is mapped to. - lockFile := ephemeralStagingMountPath + csiVolID + "/id" + + lockFile := filepath.Clean(filepath.Join(ephemeralStagingMountPath, csiVolID, "id")) + if s.fileExist(lockFile) { ephemeralVolume = true //while a file is being read from, it's a file determined by volID and is written by the driver /* #nosec G304 */ idFromFile, err := os.ReadFile(lockFile) if err != nil && os.IsNotExist(err) { - Log.Errorf("NodeUnpublish with ephemeral volume. Was unable to read lockfile: %v", err) + log.Errorf("NodeUnpublish with ephemeral volume. Was unable to read lockfile: %v", err) return nil, status.Error(codes.Internal, "NodeUnpublish with ephemeral volume. Was unable to read lockfile") } // Convert volume id from []byte to string format csiVolID = string(idFromFile) - Log.Infof("Read volume ID: %s from lockfile: %s ", csiVolID, lockFile) - + log.Infof("Read volume ID: %s from lockfile: %s ", csiVolID, lockFile) } if isNFS { fsID := getFilesystemIDFromCsiVolumeID(csiVolID) - Log.Printf("NodeUnpublishVolume fileSystemID: %s", fsID) + log.Infof("NodeUnpublishVolume fileSystemID: %s", fsID) systemID := s.getSystemIDFromCsiVolumeID(csiVolID) if systemID == "" { // use default system systemID = s.opts.defaultSystemID } - Log.Printf("NodeUnpublishVolume systemID: %s", systemID) + log.Infof("NodeUnpublishVolume systemID: %s", systemID) if systemID == "" { return nil, status.Error(codes.InvalidArgument, "systemID is not found in the request and there is no default system") @@ -320,14 +400,14 @@ func (s *service) NodeUnpublishVolume( } volID := getVolumeIDFromCsiVolumeID(csiVolID) - Log.Printf("NodeUnpublishVolume volumeID: %s", volID) + log.Infof("NodeUnpublishVolume volumeID: %s", volID) systemID := s.getSystemIDFromCsiVolumeID(csiVolID) if systemID == "" { // use default system systemID = s.opts.defaultSystemID } - Log.Printf("NodeUnpublishVolume systemID: %s", systemID) + log.Infof("NodeUnpublishVolume systemID: %s", systemID) if systemID == "" { return nil, status.Error(codes.InvalidArgument, "systemID is not found in the request and there is no default system") @@ -345,26 +425,46 @@ func (s *service) NodeUnpublishVolume( "checkVolumesMap for id: %s failed : %s", csiVolID, err.Error()) } + if s.useNVME { + log.Infof("NodeUnpublishVolume: NVME volume %s, doing mount cleanup", csiVolID) + + if ephemeralVolume { + log.Info("Detected ephemeral") + err := s.ephemeralNodeUnpublish(ctx, req) + if err != nil { + log.Errorf("ephemeralNodeUnpublish returned error: %v", err) + return nil, err + } + } + + if err := unpublishNVMEVolume(csiVolID, targetPath, reqID); err != nil { + return nil, err + } + + // Idempotent need to return ok if not published + return &csi.NodeUnpublishVolumeResponse{}, nil + } + sdcMappedVol, err := s.getSDCMappedVol(volID, systemID, unpublishGetMappedVolMaxRetry) if err != nil { - Log.Infof("Error from getSDCMappedVol is: %#v", err) - Log.Infof("Error message from getSDCMappedVol is: %s", err.Error()) + log.Infof("Error from getSDCMappedVol is: %#v", err) + log.Infof("Error message from getSDCMappedVol is: %s", err.Error()) // fix k8s 19 bug: ControllerUnpublishVolume is called before NodeUnpublishVolume // cleanup target from pod if err := gofsutil.Unmount(ctx, targetPath); err != nil { - Log.Errorf("cleanup target mount: %s", err.Error()) + log.Errorf("cleanup target mount: %s", err.Error()) } if err := removeWithRetry(targetPath); err != nil { - Log.Errorf("cleanup target path: %s", err.Error()) + log.Errorf("cleanup target path: %s", err.Error()) } // dont cleanup pvtMount in case it is in use elsewhere on the node if ephemeralVolume { - Log.Info("Detected ephemeral") + log.Info("Detected ephemeral") err := s.ephemeralNodeUnpublish(ctx, req) if err != nil { - Log.Errorf("ephemeralNodeUnpublish returned error: %s", err.Error()) + log.Errorf("ephemeralNodeUnpublish returned error: %s", err.Error()) return nil, err } } @@ -378,10 +478,10 @@ func (s *service) NodeUnpublishVolume( } if ephemeralVolume { - Log.Info("Detected ephemeral") + log.Info("Detected ephemeral") err := s.ephemeralNodeUnpublish(ctx, req) if err != nil { - Log.Errorf("ephemeralNodeUnpublish returned error: %v", err) + log.Errorf("ephemeralNodeUnpublish returned error: %v", err) return nil, err } @@ -398,18 +498,18 @@ func (s *service) getSDCMappedVol(volumeID string, systemID string, maxRetry int var err error for i := 0; i < maxRetry; i++ { if id, ok := s.connectedSystemNameToID[systemID]; ok { - Log.Printf("Node publish getMappedVol name: %s id: %s", systemID, id) + log.Infof("Node publish getMappedVol name: %s id: %s", systemID, id) systemID = id } sdcMappedVol, err = getMappedVol(volumeID, systemID) if sdcMappedVol != nil { break } - Log.Printf("Node publish getMappedVol retry: %d", i) + log.Infof("Node publish getMappedVol retry: %d", i) time.Sleep(getMappedVolDelay) } if err != nil { - Log.Printf("SDC returned volume %s on system %s not published to node", volumeID, systemID) + log.Infof("SDC returned volume %s on system %s not published to node", volumeID, systemID) return nil, err } return sdcMappedVol, err @@ -421,12 +521,12 @@ func getMappedVol(volID string, systemID string) (*goscaleio.SdcMappedVolume, er localVols, _ := goscaleio.GetLocalVolumeMap() var sdcMappedVol *goscaleio.SdcMappedVolume if len(localVols) == 0 { - Log.Printf("Length of localVols (goscaleio.GetLocalVolumeMap()) is 0 \n") + log.Infof("Length of localVols (goscaleio.GetLocalVolumeMap()) is 0 \n") } for _, v := range localVols { if v.VolumeID == volID && v.MdmID == systemID { sdcMappedVol = v - Log.Printf("Found matching SDC mapped volume %v", sdcMappedVol) + log.Infof("Found matching SDC mapped volume %v", sdcMappedVol) break } } @@ -443,7 +543,7 @@ func (s *service) getSystemName(_ context.Context, systems []string) bool { if id, ok := s.connectedSystemNameToID[systemID]; ok { for _, system := range systems { if id == system { - Log.Printf("nodeProbe found system Name: %s with id %s", systemID, id) + log.Infof("nodeProbe found system Name: %s with id %s", systemID, id) connectedSystemID = append(connectedSystemID, systemID) } } @@ -455,6 +555,12 @@ func (s *service) getSystemName(_ context.Context, systems []string) bool { // nodeProbe fetchs the SDC GUID by drv_cfg and the systemIDs/names by getSystemName method. // It also makes sure private directory(privDir) is created func (s *service) nodeProbe(ctx context.Context) error { + // skip SDC based probe if it is pure NVMe + if s.useNVME == true { + log.Info("skipping SDC based probe as the node is pure NVMe") + return nil + } + // make sure the kernel module is loaded if kmodLoaded(s.opts) { // fetch the SDC GUID @@ -467,12 +573,12 @@ func (s *service) nodeProbe(ctx context.Context) error { } s.opts.SdcGUID = guid - Log.WithField("guid", s.opts.SdcGUID).Info("set SDC GUID") + log.WithFields(csmlog.Fields{"guid": s.opts.SdcGUID}).Info("set SDC GUID") } // support for pre-approved guid if s.opts.IsApproveSDCEnabled { - Log.Infof("Approve SDC enabled") + log.Infof("Approve SDC enabled") if err := s.approveSDC(s.opts); err != nil { return err } @@ -507,7 +613,7 @@ func (s *service) nodeProbe(ctx context.Context) error { s.privDir, err.Error()) } } else { - Log.Infof("scini module not loaded, perhaps it was intentional") + log.Infof("scini module not loaded, perhaps it was intentional") } return nil @@ -538,7 +644,7 @@ func (s *service) approveSDC(opts Opts) error { // Check if SDC is already approved (only if SDC is found) if sdc != nil && sdc.Sdc.SdcApproved { - Log.Infof("SDC already approved, SDC GUID: %s", sdc.Sdc.SdcGUID) + log.Infof("SDC already approved, SDC GUID: %s", sdc.Sdc.SdcGUID) continue } @@ -546,7 +652,7 @@ func (s *service) approveSDC(opts Opts) error { switch mode { case "None": - Log.Infof("Approval not required, RestrictedSdcMode is: %s", mode) + log.Infof("Approval not required, RestrictedSdcMode is: %s", mode) case "Guid", "ApprovedIp": // Approve with SdcGUID (common for both modes) resp, err := system.ApproveSdc(&siotypes.ApproveSdcParam{ @@ -555,7 +661,7 @@ func (s *service) approveSDC(opts Opts) error { if err != nil { return status.Errorf(codes.FailedPrecondition, "%s", err) } - Log.Infof("SDC ID %s approved successfully using mode: %s", resp.SdcID, mode) + log.Infof("SDC ID %s approved successfully using mode: %s", resp.SdcID, mode) // Additional step for ApprovedIp mode if mode == "ApprovedIp" { @@ -568,7 +674,7 @@ func (s *service) approveSDC(opts Opts) error { if err != nil { return status.Errorf(codes.FailedPrecondition, "failed to set approved IPs: %s", err) } - Log.Infof("Approved IPs added successfully for SDC ID: %s", resp.SdcID) + log.Infof("Approved IPs added successfully for SDC ID: %s", resp.SdcID) } default: return status.Errorf(codes.InvalidArgument, "unsupported RestrictedSdcMode: %s", mode) @@ -641,9 +747,9 @@ func (s *service) renameSDC(opts Opts) error { newName = hostName } if sdc.Sdc.Name == newName { - Log.Infof("SDC is already named: %s.", newName) + log.Infof("SDC is already named: %s.", newName) } else { - Log.Infof("Assigning name: %s to SDC with GUID %s on system %s", newName, s.opts.SdcGUID, + log.Infof("Assigning name: %s to SDC with GUID %s on system %s", newName, s.opts.SdcGUID, systemID) err = s.adminClients[systemID].RenameSdc(sdcID, newName) if err != nil { @@ -663,7 +769,7 @@ func (s *service) getSDCName(sdcGUID string, systemID string) error { if err != nil { return status.Errorf(codes.FailedPrecondition, "%s", err) } - Log.Infof("SDC name set to: %s.", sdc.Sdc.Name) + log.Infof("SDC name set to: %s.", sdc.Sdc.Name) return nil } @@ -674,7 +780,7 @@ func kmodLoaded(opts Opts) bool { if opts.Lsmod == "" { out, err = exec.Command("lsmod").CombinedOutput() if err != nil { - Log.WithError(err).Error("error from lsmod") + log.Errorf("error from lsmod: %v", err) return false } } else { @@ -714,7 +820,8 @@ func getSystemsKnownToSDC() ([]string, error) { set[s.SystemID] = struct{}{} systems = append(systems, s.SystemID) - Log.WithField("ID", s.SystemID).Info("Found connected system") + log.WithFields(csmlog.Fields{"ID": s.SystemID}).Info("Found connected system") + } return systems, nil @@ -766,9 +873,23 @@ func (s *service) NodeGetCapabilities( }, } + stageCapabilities := []*csi.NodeServiceCapability{ + { + // Required for NodeStageVolume + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, + }, + }, + } + if s.opts.IsHealthMonitorEnabled { nodeCapabalities = append(nodeCapabalities, healthMonitorCapabalities...) } + + nodeCapabalities = append(nodeCapabalities, stageCapabilities...) + return &csi.NodeGetCapabilitiesResponse{ Capabilities: nodeCapabalities, }, nil @@ -786,14 +907,14 @@ func (s *service) NodeGetInfo( // Fetch SDC GUID if s.opts.SdcGUID == "" { if err := s.nodeProbe(ctx); err != nil { - Log.Infof("failed to probe node: %s", err) + log.Infof("failed to probe node: %s", err) } } // Fetch Node ID if len(connectedSystemID) == 0 { if err := s.nodeProbe(ctx); err != nil { - Log.Infof("failed to probe node: %s", err) + log.Infof("failed to probe node: %s", err) } } @@ -821,11 +942,11 @@ func (s *service) NodeGetInfo( } } - Log.Debugf("MaxVolumesPerNode: %v\n", maxVxflexosVolumesPerNode) + log.Debugf("MaxVolumesPerNode: %v\n", maxVxflexosVolumesPerNode) // Create the topology keys // csi-vxflexos.dellemc.com/: - Log.Infof("Arrays: %+v", s.opts.arrays) + log.Infof("Arrays: %+v", s.opts.arrays) topology := map[string]string{} if zone, ok := labels[s.opts.zoneLabelKey]; ok { @@ -833,7 +954,7 @@ func (s *service) NodeGetInfo( err = s.SetPodZoneLabel(ctx, topology) if err != nil { - Log.Warnf("Unable to set availability zone label '%s:%s' for this pod", topology[s.opts.zoneLabelKey], zone) + log.Warnf("Unable to set availability zone label '%s:%s' for this pod", topology[s.opts.zoneLabelKey], zone) } } @@ -846,6 +967,10 @@ func (s *service) NodeGetInfo( nodeID = s.opts.SdcGUID } + if s.useNVME { + nodeID = s.nodeID + } + for _, array := range s.opts.arrays { // Check if NFS protocol is enabled on the array isNFSEnabled, err := s.isNFSEnabled(ctx, array.SystemID) @@ -855,20 +980,19 @@ func (s *service) NodeGetInfo( if isNFSEnabled { topology[Name+"/"+array.SystemID+"-nfs"] = "true" } - if zone, ok := topology[s.opts.zoneLabelKey]; ok { if zone == string(array.AvailabilityZone.Name) { // Add only the secret values with the correct zone. - Log.Infof("Zone found for node ID: %s, adding system ID: %s to node topology", nodeID, array.SystemID) - topology[Name+"/"+array.SystemID] = SystemTopologySystemValue + log.Infof("Zone found for node ID: %s, adding system ID: %s to node topology", nodeID, array.SystemID) + s.populateNodeTopology(topology, array.SystemID) } } else { - Log.Infof("No zoning found for node ID: %s, adding system ID: %s", nodeID, array.SystemID) - topology[Name+"/"+array.SystemID] = SystemTopologySystemValue + log.Infof("No zoning found for node ID: %s, adding system ID: %s", nodeID, array.SystemID) + s.populateNodeTopology(topology, array.SystemID) } } - Log.Debugf("NodeId: %v\n", nodeID) + log.Debugf("NodeId: %v\n", nodeID) return &csi.NodeGetInfoResponse{ NodeId: nodeID, AccessibleTopology: &csi.Topology{ @@ -878,6 +1002,21 @@ func (s *service) NodeGetInfo( }, nil } +func (s *service) populateNodeTopology(topology map[string]string, systemID string) { + // Check if NVMe protocol is enabled on the array + if s.useNVME { + system := s.systems[systemID] + _, err := s.discoverNVMeTargets(system) + if err != nil { + log.Infof("Failed to connect to NVMe targets: %s", err) + } else { + topology[Name+"/"+systemID+"-nvmetcp"] = "true" + } + } else { + topology[Name+"/"+systemID] = SystemTopologySystemValue + } +} + // NodeGetVolumeStats will check the status of a volume given its ID and path // if volume is healthy, stats on volume usage will be returned // if volume is unhealthy, a message will be returned detailing the issue @@ -916,7 +1055,8 @@ func (s *service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume // make sure systemID we get is managed by the driver if err := s.requireProbe(ctx, systemID); err != nil { - Log.Infof("System: %s is not managed by driver; volume stats will not be collected", systemID) + log := log.WithContext(ctx) + log.Infof("System: %s is not managed by driver; volume stats will not be collected", systemID) return nil, err } @@ -950,11 +1090,11 @@ func (s *service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume if healthy { // check if path is mounted on node - mounts, err := getPathMounts(volPath) + mounts, err := getPathMounts(ctx, volPath) if len(mounts) > 0 { for _, m := range mounts { if m.Path == volPath { - Log.Infof("volPath: %s is mounted", volPath) + log.Infof("volPath: %s is mounted", volPath) mounted = true } } @@ -1027,6 +1167,7 @@ func (s *service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume func (s *service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { var reqID string var err error + var devName string headers, ok := metadata.FromIncomingContext(ctx) if ok { if req, ok := headers["csi.requestid"]; ok && len(req) > 0 { @@ -1036,12 +1177,12 @@ func (s *service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum err = s.nodeProbe(ctx) if err != nil { - Log.Error("nodeProbe failed with error :" + err.Error()) + log.Error("nodeProbe failed with error :" + err.Error()) } volumePath := req.GetVolumePath() if volumePath == "" { - Log.Error("Volume path required") + log.Error("Volume path required") return nil, status.Error(codes.InvalidArgument, "Volume path required") } @@ -1053,8 +1194,8 @@ func (s *service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum if err != nil { return nil, status.Error(codes.NotFound, "Could not stat volume path: "+volumePath) } - if !volumePathInfo.Mode().IsDir() { - Log.Infof("Volume path %s is not a directory- assuming a raw block device mount", volumePath) + if s.useSDC && !volumePathInfo.Mode().IsDir() { + log.Infof("Volume path %s is not a directory- assuming a raw block device mount", volumePath) return &csi.NodeExpandVolumeResponse{}, nil } @@ -1071,7 +1212,7 @@ func (s *service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum } volumeID := getVolumeIDFromCsiVolumeID(csiVolID) - Log.Printf("NodeExpandVolume volumeID: %s", volumeID) + log.Infof("NodeExpandVolume volumeID: %s", volumeID) if volumeID == "" { return nil, status.Error(codes.InvalidArgument, @@ -1083,7 +1224,7 @@ func (s *service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum // use default system systemID = s.opts.defaultSystemID } - Log.Printf("NodeExpandVolume systemID: %s", systemID) + log.Infof("NodeExpandVolume systemID: %s", systemID) if systemID == "" { return nil, status.Error(codes.InvalidArgument, "systemID is not found in the request and there is no default system") @@ -1094,43 +1235,151 @@ func (s *service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum return nil, err } + vol, err := s.getVolByID(volumeID, systemID) + if err != nil { + return nil, status.Errorf(codes.Unavailable, + "error retrieving volume details: %s", err.Error()) + } + volname := vol.Name + + if s.useNVME { + + nguid, err := buildNGUID(volumeID, systemID) + log.Infof("printing nguidddd %s", nguid) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to build NGUID: %s", err.Error()) + } + + devMnt, err := gofsutil.GetMountInfoFromDevice(ctx, volname) + if err != nil { + deviceNames, _ := gofsutil.GetSysBlockDevicesForVolumeWWN(context.Background(), nguid) + for _, deviceName := range deviceNames { + log.Infof("printing devicenames %s", deviceName) + } + + if len(deviceNames) > 0 { + for _, deviceName := range deviceNames { + if strings.HasPrefix(deviceName, "nvme") { + nvmeControllerDevice, err := gofsutil.GetNVMeController(deviceName) + if err != nil { + log.Errorf("Failed to rescan device (%s) with error (%s)", deviceName, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + if nvmeControllerDevice != "" { + devicePath := "/dev/" + nvmeControllerDevice + log.Infof("Rescanning unmounted (raw block) device %s to expand size", devicePath) + err = s.nvmeLib.DeviceRescan(devicePath) + if err != nil { + log.Errorf("Failed to rescan device (%s) with error (%s)", devicePath, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + } + } else { + devicePath := "/sys/block" + "/" + deviceName + log.Infof("Rescanning unmounted (raw block) device %s to expand size", deviceName) + err = gofsutil.DeviceRescan(context.Background(), devicePath) + if err != nil { + log.Errorf("Failed to rescan device (%s) with error (%s)", devicePath, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + } + devName = deviceName + } + + mpathDev, err := gofsutil.GetMpathNameFromDevice(ctx, devName) + if err != nil { + log.Errorf("Failed to fetch mpath name for device (%s) with error (%s)", devName, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + if mpathDev != "" { + err = gofsutil.ResizeMultipath(context.Background(), mpathDev) + if err != nil { + log.Errorf("Failed to resize filesystem: device (%s) with error (%s)", mpathDev, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + } + + return &csi.NodeExpandVolumeResponse{}, nil + } + log.Errorf("Failed to find mount info for (%s) with error (%s)", volname, err.Error()) + return nil, status.Error(codes.Internal, + fmt.Sprintf("Failed to find mount info for (%s) with error (%s)", volname, err.Error())) + } + log.Infof("Mount info for volume %s: %+v", volname, devMnt) + + // Expand the filesystem with the actual expanded volume size. + if devMnt.MPathName != "" { + err = gofsutil.ResizeMultipath(context.Background(), devMnt.MPathName) + if err != nil { + log.Errorf("Failed to resize filesystem: device (%s) with error (%s)", devMnt.MountPoint, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + } + // For a regular device, get the device path (devMnt.DeviceNames[1]) where the filesystem is mounted + // PublishVolume creates devMnt.DeviceNames[0] but is left unused for regular devices + var devicePath string + if len(devMnt.DeviceNames) > 1 { + devicePath = "/dev/" + devMnt.DeviceNames[1] + } else { + devicePath = "/dev/" + devMnt.DeviceNames[0] + } + + // Determine file system type + fsType, err := gofsutil.FindFSType(context.Background(), devMnt.MountPoint) + if err != nil { + log.Errorf("Failed to fetch filesystem for volume (%s) with error (%s)", devMnt.MountPoint, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + log.Infof("Found %s filesystem mounted on volume %s", fsType, devMnt.MountPoint) + + // Resize the filesystem + err = gofsutil.ResizeFS(context.Background(), devMnt.MountPoint, devicePath, devMnt.PPathName, devMnt.MPathName, fsType) + if err != nil { + log.Errorf("Failed to resize filesystem: mountpoint (%s) device (%s) with error (%s)", + devMnt.MountPoint, devicePath, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + + return &csi.NodeExpandVolumeResponse{}, nil + } + sdcMappedVolume, err := s.getSDCMappedVol(volumeID, systemID, publishGetMappedVolMaxRetry) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - Log.Infof("sdcMappedVolume %+v", sdcMappedVolume) + log.Infof("sdcMappedVolume %+v", sdcMappedVolume) sdcDevice := strings.Replace(sdcMappedVolume.SdcDevice, "/dev/", "", 1) - Log.Infof("sdcDevice %s", sdcDevice) + log.Infof("sdcDevice %s", sdcDevice) devicePath := sdcMappedVolume.SdcDevice - Log.Infof("devicePath %s", devicePath) + log.Infof("devicePath %s", devicePath) size := req.GetCapacityRange().GetRequiredBytes() - f := logrus.Fields{ + f := csmlog.Fields{ "CSIRequestID": reqID, "DevicePath": devicePath, "VolumeID": csiVolID, "VolumePath": volumePath, "Size": size, } - Log.WithFields(f).Info("resizing volume") + log.WithFields(f).Info("resizing volume") rc, err := goscaleio.DrvCfgQueryRescan() - Log.Infof("Rescan all SDC devices") + log.Infof("Rescan all SDC devices") if err != nil { - Log.Errorf("Rescan failed with ioctl error code %s with error %s, Run rescan manually on Powerflex host", rc, err.Error()) + log.Errorf("Rescan failed with ioctl error code %s with error %s, Run rescan manually on Powerflex host", rc, err.Error()) } fsType, err := gofsutil.FindFSType(context.Background(), volumePath) if err != nil { - Log.Errorf("Failed to fetch filesystem type for mount (%s) with error (%s)", volumePath, err.Error()) + log.Errorf("Failed to fetch filesystem type for mount (%s) with error (%s)", volumePath, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } - Log.Infof("Found %s filesystem mounted on volume %s", fsType, volumePath) + log.Infof("Found %s filesystem mounted on volume %s", fsType, volumePath) // Resize the filesystem err = gofsutil.ResizeFS(context.Background(), volumePath, devicePath, "", "", fsType) if err != nil { - Log.Errorf("Failed to resize filesystem: mountpoint (%s) device (%s) with error (%s)", + log.Errorf("Failed to resize filesystem: mountpoint (%s) device (%s) with error (%s)", volumePath, devicePath, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } diff --git a/service/node_connectivity_checker.go b/service/node_connectivity_checker.go new file mode 100644 index 00000000..70fa1a5e --- /dev/null +++ b/service/node_connectivity_checker.go @@ -0,0 +1,215 @@ +package service + +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "sync" + "sync/atomic" + "time" + + csictx "github.com/dell/gocsi/context" + "github.com/gorilla/mux" +) + +// pollingFrequency in seconds +var pollingFrequencyInSeconds int64 + +// startAPIService reads nodes to array status periodically +func (s *service) startAPIService(ctx context.Context) { + if !s.opts.IsPodmonEnabled { + log.Info("podmon is not enabled") + return + } + atomic.StoreInt64(&pollingFrequencyInSeconds, SetPollingFrequency(ctx)) + s.startNodeToArrayConnectivityCheck(ctx) + s.apiRouter(ctx) +} + +// apiRouter serves http requests +func (s *service) apiRouter(_ context.Context) { + log.Infof("starting http server on port %s", s.opts.PodmonPort) + // create a new mux router + router := mux.NewRouter() + // route to connectivity status + // connectivityStatus is the handlers + router.HandleFunc(ArrayStatus, s.connectivityStatus).Methods("GET") + router.HandleFunc(ArrayStatus+"/"+"{systemID}", s.getArrayConnectivityStatus).Methods("GET") + // start http server to serve requests + server := &http.Server{ + Addr: s.opts.PodmonPort, + Handler: router, + ReadTimeout: Timeout, + WriteTimeout: Timeout, + } + err := server.ListenAndServe() + if err != nil { + log.Errorf("unable to start http server to serve status requests due to %s", err) + } + log.Infof("started http server to serve status requests at %s", s.opts.PodmonPort) +} + +// connectivityStatus handler returns array connectivity status +func (s *service) connectivityStatus(w http.ResponseWriter, _ *http.Request) { + log.Infof("connectivityStatus called, status is %v \n", s.probeStatus) + // w.Header().Set("Content-Type", "application/json") + if s.probeStatus == nil { + log.Errorf("error probeStatus map in cache is empty") + w.WriteHeader(http.StatusInternalServerError) + w.Header().Set("Content-Type", "application/json") + return + } + + // convert struct to JSON + log.Debugf("ProbeStatus fetched from the cache has %+v", s.probeStatus) + + jsonResponse, err := MarshalSyncMapToJSON(s.probeStatus) + if err != nil { + log.Errorf("error %s during marshaling to json", err) + w.WriteHeader(http.StatusInternalServerError) + w.Header().Set("Content-Type", "application/json") + return + } + log.Info("sending connectivityStatus for all arrays ") + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(jsonResponse) + if err != nil { + log.Errorf("unable to write response %s", err) + } +} + +// MarshalSyncMapToJSON marshal the sync Map to Json +func MarshalSyncMapToJSON(m *sync.Map) ([]byte, error) { + tmpMap := make(map[string]ArrayConnectivityStatus) + m.Range(func(k, value interface{}) bool { + // this check is not necessary but just in case is someone in future play around this + switch value.(type) { + case ArrayConnectivityStatus: + tmpMap[k.(string)] = value.(ArrayConnectivityStatus) + return true + default: + log.Errorf("invalid data is stored in cache") + return false + } + }) + log.Debugf("map value is %+v", tmpMap) + if len(tmpMap) == 0 { + return nil, fmt.Errorf("invalid data is stored in cache") + } + return json.Marshal(tmpMap) +} + +// getArrayConnectivityStatus handler lists status of the requested array +func (s *service) getArrayConnectivityStatus(w http.ResponseWriter, r *http.Request) { + systemID := mux.Vars(r)["systemID"] + log.Infof("GetArrayConnectivityStatus called for array %s \n", systemID) + status, found := s.probeStatus.Load(systemID) + if !found { + // specify status code + w.WriteHeader(http.StatusNotFound) + w.Header().Set("Content-Type", "application/json") + // update response writer + fmt.Fprintf(w, "array %s not found \n", systemID) + return + } + // convert status struct to JSON + jsonResponse, err := json.Marshal(status) + if err != nil { + log.Errorf("error %s during marshaling to json", err) + w.WriteHeader(http.StatusInternalServerError) + w.Header().Set("Content-Type", "application/json") + return + } + log.Infof("sending response %+v for array %s \n", status, systemID) + // update response + _, err = w.Write(jsonResponse) + if err != nil { + log.Errorf("unable to write response %s", err) + } +} + +// startNodeToArrayConnectivityCheck starts connectivityTest as one goroutine for each array +func (s *service) startNodeToArrayConnectivityCheck(ctx context.Context) { + log.Debug("startNodeToArrayConnectivityCheck called") + s.probeStatus = new(sync.Map) + + for _, array := range s.opts.arrays { + go s.testConnectivityAndUpdateStatus(ctx, array.SystemID, Timeout) + } + + log.Infof("startNodeToArrayConnectivityCheck is running probes at pollingFrequency %d ", pollingFrequencyInSeconds/2) +} + +// testConnectivityAndUpdateStatus runs probe to test connectivity from node to array +// updates probeStatus map[array]ArrayConnectivityStatus +func (s *service) testConnectivityAndUpdateStatus(ctx context.Context, systemID string, timeout time.Duration) { + defer func() { + if err := recover(); err != nil { + log.Errorf("panic occurred in testConnectivityAndUpdateStatus: %s", err) + } + // if panic occurs restart new goroutine + go s.testConnectivityAndUpdateStatus(ctx, systemID, timeout) + }() + var status ArrayConnectivityStatus + for { + // add timeout to context + timeOutCtx, cancel := context.WithTimeout(ctx, timeout) + log.Debugf("Running probe for array %s at time %v \n", systemID, time.Now()) + if existingStatus, ok := s.probeStatus.Load(systemID); !ok { + log.Debugf("%s not in probeStatus ", systemID) + } else { + if status, ok = existingStatus.(ArrayConnectivityStatus); !ok { + log.Errorf("failed to extract ArrayConnectivityStatus for array '%s'", systemID) + } + } + // for the first time status will not be there. + log.Debugf("array %s , status is %+v", systemID, status) + // run nodeProbe to test connectivity + err := s.requireProbe(timeOutCtx, systemID) + if err == nil { + log.Debugf("Probe successful for %s", systemID) + status.LastSuccess = time.Now().Unix() + } else { + log.Warnf("Probe failed for array '%s' error:'%s'", systemID, err) + } + status.LastAttempt = time.Now().Unix() + log.Debugf("array %s , storing status %+v", systemID, status) + s.probeStatus.Store(systemID, status) + cancel() + // sleep for half the pollingFrequency and run check again + time.Sleep(time.Second * time.Duration(atomic.LoadInt64(&pollingFrequencyInSeconds)/2)) + } +} + +// SetPollingFrequency reads the pollingFrequency from Env, sets default vale if ENV not found +func SetPollingFrequency(ctx context.Context) int64 { + var pollingFrequency int64 + if pollRateEnv, ok := csictx.LookupEnv(ctx, EnvPodmonArrayConnectivityPollRate); ok { + if pollingFrequency, _ = strconv.ParseInt(pollRateEnv, 10, 32); pollingFrequency != 0 { + log.Debugf("use pollingFrequency as %d seconds", pollingFrequency) + return pollingFrequency + } + } + log.Debugf("use default pollingFrequency as %d seconds", DefaultPodmonPollRate) + return DefaultPodmonPollRate +} diff --git a/service/node_connectivity_checker_test.go b/service/node_connectivity_checker_test.go new file mode 100644 index 00000000..3074b8bb --- /dev/null +++ b/service/node_connectivity_checker_test.go @@ -0,0 +1,157 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package service + +import ( + "context" + "fmt" + "net/http" + "os" + "sync" + "testing" + "time" +) + +var s service + +func TestApiRouter1(t *testing.T) { + s.opts.PodmonPort = ":abc" + s.apiRouter(context.Background()) + + resp, err := http.Get("http://localhost:8083/node-status") + if err == nil || resp != nil { + t.Errorf("Error while probing node status") + } +} + +func TestApiRouter2(t *testing.T) { + s.opts.PodmonPort = ":8084" + go s.apiRouter(context.Background()) + time.Sleep(2 * time.Second) + + resp4, err := http.Get("http://localhost:8084/array-status") + if err != nil || resp4.StatusCode != 500 { + t.Errorf("Error while probing array status %v", err) + } + // fill some invalid dummy data in the cache and try to fetch + if s.probeStatus == nil { + s.probeStatus = new(sync.Map) + } else { + s.probeStatus.Clear() + } + s.probeStatus.Store("SystemID2", "status") + + resp5, err := http.Get("http://localhost:8084/array-status") + if err != nil || resp5.StatusCode != 500 { + t.Errorf("Error while probing array status %v, %d", err, resp5.StatusCode) + } + + // fill some dummy data in the cache and try to fetch + var status ArrayConnectivityStatus + status.LastSuccess = time.Now().Unix() + status.LastAttempt = time.Now().Unix() + s.probeStatus.Clear() + s.probeStatus.Store("SystemID", status) + + // array status + resp2, err := http.Get("http://localhost:8084/array-status") + if err != nil || resp2.StatusCode != 200 { + t.Errorf("Error while probing array status %v", err) + } + + resp3, err := http.Get("http://localhost:8084/array-status/SymIDNotPresent") + if err != nil || resp3.StatusCode != 404 { + t.Errorf("Error while probing array status %v", err) + } + value := make(chan int) + s.probeStatus.Store("SystemID3", value) + resp9, err := http.Get("http://localhost:8084/array-status/SystemID3") + if err != nil || resp9.StatusCode != 500 { + t.Errorf("Error while probing array status %v", err) + } +} + +func TestMarshalSyncMapToJSON(t *testing.T) { + type args struct { + m *sync.Map + } + sample := new(sync.Map) + sample2 := new(sync.Map) + var status ArrayConnectivityStatus + status.LastSuccess = time.Now().Unix() + status.LastAttempt = time.Now().Unix() + + sample.Store("SystemID", status) + sample2.Store("key", "2.adasd") + + tests := []struct { + name string + args args + }{ + {"storing valid value in map cache", args{m: sample}}, + {"storing valid value in map cache", args{m: sample2}}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, _ := MarshalSyncMapToJSON(tt.args.m) + if len(data) == 0 && i == 0 { + t.Errorf("MarshalSyncMapToJSON() expecting some data from cache in the response") + return + } + }) + } +} + +func TestStartAPIServiceNoPodmon(_ *testing.T) { + s.opts.IsPodmonEnabled = false + s.startAPIService(context.Background()) +} + +func TestStartAPIService(_ *testing.T) { + s.opts.IsPodmonEnabled = true + os.Setenv(EnvPodmonArrayConnectivityPollRate, "60") + defer os.Unsetenv(EnvPodmonArrayConnectivityPollRate) + s.opts.arrays = map[string]*ArrayConnectionData{ + "array1": { + SystemID: "array1", + }, + } + + // Create a valid ArrayConnectivityStatus instance + status := ArrayConnectivityStatus{ + LastSuccess: time.Now().Unix(), + LastAttempt: time.Now().Unix(), + } + + // Store valid data in probeStatus + if s.probeStatus == nil { + s.probeStatus = new(sync.Map) + } else { + s.probeStatus.Clear() + } + s.probeStatus.Store("SystemID", status) + defer s.probeStatus.Delete("SystemID") + + s.startAPIService(context.Background()) +} + +func TestSetPollingFrequencyDefaultPollRate(_ *testing.T) { + s.opts.PodmonPollingFreq = fmt.Sprintf("%d", DefaultPodmonPollRate) + SetPollingFrequency(context.Background()) +} diff --git a/service/nvme_utils.go b/service/nvme_utils.go new file mode 100644 index 00000000..2b2b4779 --- /dev/null +++ b/service/nvme_utils.go @@ -0,0 +1,157 @@ +// Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package service + +import ( + "context" + "fmt" + "net" + "sort" + "strings" + + "github.com/dell/gobrick" + "github.com/dell/gonvme" + "github.com/dell/goscaleio" +) + +const ( + oui = "64b94e" // Dell/PowerFlex OUI + volumeIDLen = 16 // 8 bytes + clusterLSBLen = 10 // 5 bytes + nguidLen = 32 // 16 bytes +) + +// NVMEConnector is wrapper of gobrick.NVMEConnector interface. +// It allows to connect NVMe volumes to the node. +type NVMEConnector interface { + ConnectVolume(ctx context.Context, info gobrick.NVMeVolumeInfo, useFC bool) (gobrick.Device, error) + DisconnectVolumeByDeviceName(ctx context.Context, name string) error + GetInitiatorName(ctx context.Context) ([]string, error) +} + +// GetNVMETCPTargetsInfoFromStorage returns list of gobrick compatible NVME TCP targets by querying PowerFlex array +func getNVMETCPTargetsInfoFromStorage(system *goscaleio.System) ([]string, error) { + allSdt, err := system.GetAllSdts() + if err != nil { + log.Infof("failed to get SDT from system %s : %v", system.System.ID, err) + return nil, err + } else if len(allSdt) == 0 { + log.Infof("system %s returned empty SDT: %v", system.System.ID, allSdt) + return nil, fmt.Errorf("system %s returned empty SDT", system.System.ID) + } + // sort data by id + sort.Slice(allSdt, func(i, j int) bool { + return allSdt[i].ID < allSdt[j].ID + }) + var portals []string + for _, t := range allSdt { + if len(t.IPList) == 0 || t.IPList[0].IP == "" { + log.Debugf("SDT %s has no IPs; skipping", t.ID) + continue + } + portals = append(portals, net.JoinHostPort(t.IPList[0].IP, "4420")) + } + return portals, nil +} + +func (s *service) discoverNVMeTargets(system *goscaleio.System) ([]gonvme.NVMeTarget, error) { + portals, err := getNVMETCPTargetsInfoFromStorage(system) + if err != nil { + return nil, fmt.Errorf("failed to get targets from array: %w", err) + } + + discoveredTargets := make(map[string]gonvme.NVMeTarget) + for _, portal := range portals { + var ip string + ip, _, err = net.SplitHostPort(portal) + if err != nil { + log.Infof("Invalid portal format, skipping: %s (%v)", portal, err) + continue + } + if _, exists := discoveredTargets[ip]; exists { + continue + } + log.Infof("Trying to discover NVMe target from portal %s", ip) + var targets []gonvme.NVMeTarget + targets, err = s.nvmeLib.DiscoverNVMeTCPTargets(ip, false) + if err != nil { + log.Infof("couldn't discover targets with portal %s: %s", ip, err) + continue + } + for _, target := range targets { + discoveredTargets[target.Portal] = target + s.nvmeTargetNqn[target.Portal] = target.TargetNqn + } + } + + if len(discoveredTargets) == 0 { + log.Warnf("failed to discover NVMe targets for array %s", system.System.ID) + } + + targets := make([]gonvme.NVMeTarget, 0, len(discoveredTargets)) + for _, target := range discoveredTargets { + targets = append(targets, target) + } + + return targets, err +} + +func (s *service) connectToNVMeTargets(system *goscaleio.System, targets []gonvme.NVMeTarget) error { + connected := false + for _, t := range targets { + log.Infof("Connecting to NVMe target %v", t) + if err := s.nvmeLib.NVMeTCPConnect(t, false); err != nil { + log.Errorf("couldn't connect to the nvme target %v: %s", t, err) + continue + } + connected = true + } + if connected { + return nil + } + return fmt.Errorf("failed to connect to any NVMe targets for array %s", system.System.ID) +} + +// discoverAndConnectNVMeTargets handles the discovery and connection to NVMe targets for a given array. +func (s *service) discoverAndConnectNVMeTargets(system *goscaleio.System) error { + targets, err := s.discoverNVMeTargets(system) + if err != nil { + return err + } + return s.connectToNVMeTargets(system, targets) +} + +// BuildNGUID creates NGUID as: +// [VolumeID (16 chars)] + [OUI (6 chars)] + [ClusterID last 10 chars] +func buildNGUID(volumeID, clusterID string) (string, error) { + vol := normalize(volumeID) + cluster := normalize(clusterID) + + if len(vol) != volumeIDLen { + return "", fmt.Errorf("volumeID must be %d characters, got %q", volumeIDLen, vol) + } + + if len(cluster) < clusterLSBLen { + return "", fmt.Errorf("clusterID must be at least %d characters, got %q", clusterLSBLen, cluster) + } + + clusterLSB := cluster[len(cluster)-clusterLSBLen:] + nguid := vol + oui + clusterLSB + + return nguid, nil +} + +func normalize(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} diff --git a/service/preinit.go b/service/preinit.go index 5066218e..ade761b5 100644 --- a/service/preinit.go +++ b/service/preinit.go @@ -71,7 +71,7 @@ var ( ) func (s *service) PreInit() error { - Log.Infof("PreInit running") + log.Infof("PreInit running") arrayConfig, err := arrayConfigurationProviderImpl.GetArrayConfiguration() if err != nil { @@ -86,7 +86,7 @@ func (s *service) PreInit() error { var mdmData string if labelKey == "" { - Log.Debug("No zone key found, will configure all MDMs") + log.Debug("No zone key found, will configure all MDMs") sb := strings.Builder{} for _, connectionData := range arrayConfig { if connectionData.Mdm != "" { @@ -98,7 +98,7 @@ func (s *service) PreInit() error { } mdmData = sb.String() } else { - Log.Infof("Zone key detected, will configure MDMs for this node, key: %s", labelKey) + log.Infof("Zone key detected, will configure MDMs for this node, key: %s", labelKey) nodeLabels, err := s.GetNodeLabels(context.Background()) if err != nil { return fmt.Errorf("unable to get node labels: %v", err) @@ -107,11 +107,11 @@ func (s *service) PreInit() error { zone, ok := nodeLabels[labelKey] if ok && zone == "" { - Log.Errorf("node key found but zone is missing, will not configure MDMs for this node, key: %s", labelKey) + log.Errorf("node key found but zone is missing, will not configure MDMs for this node, key: %s", labelKey) } if zone != "" { - Log.Infof("zone found, will configure MDMs for this node, zone: %s", zone) + log.Infof("zone found, will configure MDMs for this node, zone: %s", zone) mdmData, err = getMdmList(arrayConfig, labelKey, zone) if err != nil { return fmt.Errorf("unable to get MDM list: %v", err) @@ -119,7 +119,7 @@ func (s *service) PreInit() error { } } - Log.Infof("Saving MDM list to %s, MDM=%s", nodeMdmsFile, mdmData) + log.Infof("Saving MDM list to %s, MDM=%s", nodeMdmsFile, mdmData) err = fileWriterProviderImpl.WriteFile(nodeMdmsFile, []byte(fmt.Sprintf("MDM=%s\n", mdmData)), fs.FileMode(0o444)) return err } diff --git a/service/publisher.go b/service/publisher.go new file mode 100644 index 00000000..e5badfc9 --- /dev/null +++ b/service/publisher.go @@ -0,0 +1,223 @@ +// Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package service + +import ( + "context" + "strconv" + + "github.com/dell/goscaleio" + siotypes "github.com/dell/goscaleio/types/v1" + csi "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// VolumePublisher allows to publish a volume +type VolumePublisher interface { + // Publish does the steps necessary for volume to be available on the node + Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest, adminClient *goscaleio.Client, systemID, csiVolID string) (*csi.ControllerPublishVolumeResponse, error) +} + +// NVMePublisher implementation of VolumePublisher for NVMe volumes +type NVMePublisher struct { + svc *service + vol *siotypes.Volume +} + +func (p *NVMePublisher) Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest, adminClient *goscaleio.Client, systemID, csiVolID string) (*csi.ControllerPublishVolumeResponse, error) { + log.Debugf("ControllerPublish - in NVMePublisher") + volumeContext := req.GetVolumeContext() + nodeID := req.GetNodeId() + am := req.GetVolumeCapability().GetAccessMode() + vcs := []*csi.VolumeCapability{req.GetVolumeCapability()} + isBlock := accTypeIsBlock(vcs) + + nvmeHost, err := p.svc.systems[systemID].FindSdc("Name", nodeID) + if err != nil { + return nil, status.Errorf(codes.Internal, "error finding NVMe host %s. Error: %s", nodeID, err.Error()) + } + + if len(p.vol.MappedSdcInfo) > 0 { + for _, mappedSdcInfo := range p.vol.MappedSdcInfo { + if mappedSdcInfo.SdcName == nvmeHost.Sdc.Name { + log.Debug("volume already mapped") + if err := validateAndCompareQoS(volumeContext, p.vol.Name, mappedSdcInfo); err != nil { + return nil, err + } + return &csi.ControllerPublishVolumeResponse{}, nil + } + } + if isSingleNodeAccess(am) { + return nil, status.Errorf(codes.FailedPrecondition, "volume already published to NVMe host: %s", p.vol.MappedSdcInfo[0].SdcName) + } + } + + // All remaining cases are MULTI_NODE: + // This original code precludes block multi-writers, + // and is based on a faulty test that the Volume MappingToAllSdcsEnabled + // attribute must be set to allow multiple writers, which is not true. + // The proper way to control multiple mappings is with the allowMultipleMappings + // attribute passed in the MapVolumeSdcParameter. Unfortunately you cannot + // read this parameter back. + allowMultipleMappings, err := shouldAllowMultipleMappings(isBlock, am) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "%s", err.Error()) + } + + if err := validateAccessType(am, isBlock); err != nil { + return nil, err + } + + // Publish volume to NVMe host + targetVolume := goscaleio.NewVolume(adminClient) + targetVolume.Volume = &siotypes.Volume{ID: p.vol.ID} + + mapVolumeNVMeParam := &siotypes.MapVolumeNVMeParam{ + HostID: nvmeHost.Sdc.ID, + AllowMultipleMappings: allowMultipleMappings, + AllHosts: "", + } + + err = targetVolume.MapVolumeNVMe(mapVolumeNVMeParam) + if err != nil { + return nil, status.Errorf(codes.Internal, "error mapping volume to nvme host %s. Error: %s", req.NodeId, err.Error()) + } + + if err := setQoSIfNeeded(ctx, p.svc, systemID, nvmeHost.Sdc.ID, p.vol.Name, csiVolID, nodeID, volumeContext); err != nil { + return nil, err + } + + return &csi.ControllerPublishVolumeResponse{}, nil +} + +// SDCPublisher implementation of VolumePublisher for SDC volumes +type SDCPublisher struct { + svc *service + vol *siotypes.Volume +} + +func (p *SDCPublisher) Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest, adminClient *goscaleio.Client, systemID, csiVolID string) (*csi.ControllerPublishVolumeResponse, error) { + log.Debugf("ControllerPublish - in SDCPublisher") + volumeContext := req.GetVolumeContext() + nodeID := req.GetNodeId() + am := req.GetVolumeCapability().GetAccessMode() + vcs := []*csi.VolumeCapability{req.GetVolumeCapability()} + isBlock := accTypeIsBlock(vcs) + + sdcID, err := p.svc.getSDCID(nodeID, systemID) + if err != nil { + return nil, status.Errorf(codes.NotFound, "%s", err.Error()) + } + + if len(p.vol.MappedSdcInfo) > 0 { + for _, mappedSdcInfo := range p.vol.MappedSdcInfo { + if mappedSdcInfo.SdcID == sdcID { + log.Debug("volume already mapped") + if err := validateAndCompareQoS(volumeContext, p.vol.Name, mappedSdcInfo); err != nil { + return nil, err + } + return &csi.ControllerPublishVolumeResponse{}, nil + } + } + if isSingleNodeAccess(am) { + return nil, status.Errorf(codes.FailedPrecondition, "volume already published to SDC id: %s", p.vol.MappedSdcInfo[0].SdcID) + } + } + + // All remaining cases are MULTI_NODE: + // This original code precludes block multi-writers, + // and is based on a faulty test that the Volume MappingToAllSdcsEnabled + // attribute must be set to allow multiple writers, which is not true. + // The proper way to control multiple mappings is with the allowMultipleMappings + // attribute passed in the MapVolumeSdcParameter. Unfortunately you cannot + // read this parameter back. + allowMultipleMappings, err := shouldAllowMultipleMappings(isBlock, am) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "%s", err.Error()) + } + + if err := validateAccessType(am, isBlock); err != nil { + return nil, err + } + + // Publish volume to SDC + targetVolume := goscaleio.NewVolume(adminClient) + targetVolume.Volume = &siotypes.Volume{ID: p.vol.ID} + + mapVolumeSdcParam := &siotypes.MapVolumeSdcParam{ + SdcID: sdcID, + AllowMultipleMappings: allowMultipleMappings, + AllSdcs: "", + } + err = targetVolume.MapVolumeSdc(mapVolumeSdcParam) + if err != nil { + return nil, status.Errorf(codes.Internal, "error mapping volume to node: %s", err.Error()) + } + + if err := setQoSIfNeeded(ctx, p.svc, systemID, sdcID, p.vol.Name, csiVolID, nodeID, volumeContext); err != nil { + return nil, err + } + + return &csi.ControllerPublishVolumeResponse{}, nil +} + +func validateAndCompareQoS(volumeContext map[string]string, volName string, sdcInfo *siotypes.MappedSdcInfo) error { + bandwidthLimit := volumeContext[KeyBandwidthLimitInKbps] + iopsLimit := volumeContext[KeyIopsLimit] + + if err := validateQoSParameters(bandwidthLimit, iopsLimit, volName); err != nil { + return err + } + + if len(bandwidthLimit) > 0 && strconv.Itoa(sdcInfo.LimitBwInMbps*1024) != bandwidthLimit { + return status.Errorf(codes.InvalidArgument, + "volume %s already published with bandwidth limit: %d, but does not match the requested bandwidth limit: %s", + volName, sdcInfo.LimitBwInMbps*1024, bandwidthLimit) + } + if len(iopsLimit) > 0 && strconv.Itoa(sdcInfo.LimitIops) != iopsLimit { + return status.Errorf(codes.InvalidArgument, + "volume %s already published with IOPS limit: %d, but does not match the requested IOPS limits: %s", + volName, sdcInfo.LimitIops, iopsLimit) + } + return nil +} + +// isSingleNodeAccess checks if the given access mode is a single node access mode. +func isSingleNodeAccess(am *csi.VolumeCapability_AccessMode) bool { + switch am.Mode { + case csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER, + csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER, + csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY: + return true + } + return false +} + +func setQoSIfNeeded(ctx context.Context, svc *service, systemID, sdcID, volName, csiVolID, nodeID string, volumeContext map[string]string) error { + bandwidthLimit := volumeContext[KeyBandwidthLimitInKbps] + iopsLimit := volumeContext[KeyIopsLimit] + + // validate requested QoS parameters + if err := validateQoSParameters(bandwidthLimit, iopsLimit, volName); err != nil { + return err + } + + // check for atleast one of the QoS params should exist in storage class + if len(bandwidthLimit) > 0 || len(iopsLimit) > 0 { + return svc.setQoSParameters(ctx, systemID, sdcID, bandwidthLimit, iopsLimit, volName, csiVolID, nodeID) + } + return nil +} diff --git a/service/replication.go b/service/replication.go index 1ede99c4..68f4415e 100644 --- a/service/replication.go +++ b/service/replication.go @@ -20,14 +20,13 @@ package service import ( "fmt" - "log" "strconv" "strings" "time" - csi "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/dell-csi-extensions/replication" "github.com/dell/goscaleio" + csi "github.com/container-storage-interface/spec/lib/go/csi" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -145,7 +144,7 @@ func (s *service) GetReplicationCapabilities(_ context.Context, _ *replication.G } func (s *service) CreateRemoteVolume(ctx context.Context, req *replication.CreateRemoteVolumeRequest) (*replication.CreateRemoteVolumeResponse, error) { - Log.Printf("[CreateRemoteVolume] - req %+v", req) + log.Infof("[CreateRemoteVolume] - req %+v", req) volHandleCtx := req.GetVolumeHandle() parameters := req.GetParameters() @@ -157,7 +156,7 @@ func (s *service) CreateRemoteVolume(ctx context.Context, req *replication.Creat volumeID := getVolumeIDFromCsiVolumeID(volHandleCtx) systemID := s.getSystemIDFromCsiVolumeID(volHandleCtx) - Log.Printf("Volume ID: %s System ID: %s", volumeID, systemID) + log.Infof("Volume ID: %s System ID: %s", volumeID, systemID) if volumeID == "" || systemID == "" { return nil, status.Error(codes.InvalidArgument, "failed to provide system ID or volume ID") @@ -182,9 +181,17 @@ func (s *service) CreateRemoteVolume(ctx context.Context, req *replication.Creat return nil, status.Errorf(codes.InvalidArgument, "replication enabled but no remote system specified in storage class") } + log.Infof("Remote System ID: %s", remoteSystemID) + platformInfo, err := s.GetPlatformInfo(remoteSystemID) + if err != nil { + return nil, err + } else if s.isReplicationNotSupported(platformInfo.GenType) { + return nil, status.Errorf(codes.InvalidArgument, "Replication is not supported on this System %s with GenType: %s", remoteSystemID, platformInfo.GenType) + } + // Probe the remote system if err := s.requireProbe(ctx, remoteSystemID); err != nil { - Log.Infof("Remote probe failed: %s", err) + log.Errorf("Remote probe failed: %s", err) return nil, err } @@ -205,7 +212,7 @@ func (s *service) CreateRemoteVolume(ctx context.Context, req *replication.Creat protectionDomain, ok := parameters[s.WithRP(KeyReplicationProtectionDomain)] if !ok { - log.Printf("Remote protection domain not provided; there could be conflicts if two storage pools share a name") + log.Infof("Remote protection domain not provided; there could be conflicts if two storage pools share a name") } name := "replicated-" + vol.Name @@ -213,11 +220,11 @@ func (s *service) CreateRemoteVolume(ctx context.Context, req *replication.Creat createVolumeResponse, err := s.CreateVolume(ctx, volReq) if err != nil { - log.Printf("CreateVolume call failed: %s", err.Error()) + log.Errorf("CreateVolume call failed: %s", err) return nil, err } - log.Printf("Potentially created a remote volume: %+v", createVolumeResponse) + log.Infof("Potentially created a remote volume: %+v", createVolumeResponse) remoteParams := map[string]string{ "storagePool": remoteStoragePool, @@ -234,7 +241,8 @@ func (s *service) CreateRemoteVolume(ctx context.Context, req *replication.Creat // DeleteLocalVolume deletes the backend volume on the storage array. func (s *service) DeleteLocalVolume(ctx context.Context, req *replication.DeleteLocalVolumeRequest) (*replication.DeleteLocalVolumeResponse, error) { - Log.Printf("[DeleteLocalVolume] - req %+v", req) + log := log.WithContext(ctx) + log.Infof("[DeleteLocalVolume] - req %+v", req) volHandleCtx := req.GetVolumeHandle() @@ -245,7 +253,7 @@ func (s *service) DeleteLocalVolume(ctx context.Context, req *replication.Delete volumeID := getVolumeIDFromCsiVolumeID(volHandleCtx) systemID := s.getSystemIDFromCsiVolumeID(volHandleCtx) - Log.Printf("Volume ID: %s System ID: %s", volumeID, systemID) + log.Infof("Volume ID: %s System ID: %s", volumeID, systemID) if volumeID == "" || systemID == "" { return nil, status.Error(codes.InvalidArgument, "failed to provide system ID or volume ID") @@ -254,7 +262,7 @@ func (s *service) DeleteLocalVolume(ctx context.Context, req *replication.Delete vol, err := s.getVolByID(volumeID, systemID) if err != nil { if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) { - log.Printf("[DeleteLocalVolume] - volume already deleted.") + log.Infof("[DeleteLocalVolume] - volume already deleted.") return &replication.DeleteLocalVolumeResponse{}, nil } @@ -262,13 +270,13 @@ func (s *service) DeleteLocalVolume(ctx context.Context, req *replication.Delete } if vol.VolumeReplicationState != "UnmarkedForReplication" { - log.Printf("[DeleteLocalVolume] - target volume is marked for replication when deleting") + log.Infof("[DeleteLocalVolume] - target volume is marked for replication when deleting") return nil, status.Error(codes.InvalidArgument, "replication target volume marked for replication. Delete source volume.") } _, err = s.DeleteVolume(ctx, &csi.DeleteVolumeRequest{VolumeId: volHandleCtx}) if err != nil { - log.Printf("[DeleteLocalVolume] - call failed: %s", err.Error()) + log.Infof("[DeleteLocalVolume] - call failed: %s", err.Error()) return nil, err } @@ -276,7 +284,7 @@ func (s *service) DeleteLocalVolume(ctx context.Context, req *replication.Delete } func (s *service) CreateStorageProtectionGroup(ctx context.Context, req *replication.CreateStorageProtectionGroupRequest) (*replication.CreateStorageProtectionGroupResponse, error) { - Log.Printf("[CreateStorageProtectionGroup] - req %+v", req) + log.Infof("[CreateStorageProtectionGroup] - req %+v", req) volHandleCtx := req.GetVolumeHandle() if volHandleCtx == "" { @@ -309,7 +317,7 @@ func (s *service) CreateStorageProtectionGroup(ctx context.Context, req *replica return nil, status.Errorf(codes.Internal, "couldn't getProtectionDomain (local): %s", err.Error()) } - Log.Printf("[CreateStorageProtectionGroup] - Local Protection Domain: %+v", localProtectionDomain) + log.Infof("[CreateStorageProtectionGroup] - Local Protection Domain: %+v", localProtectionDomain) remoteSystemID, ok := parameters[s.WithRP(KeyReplicationRemoteSystem)] if !ok { @@ -349,13 +357,13 @@ func (s *service) CreateStorageProtectionGroup(ctx context.Context, req *replica clusterUID, ok := parameters["clusterUID"] if !ok { - Log.Warnf("[CreateStorageProtectionGroup] - source cluster UID not provided, using remote system ID in RCG name.") + log.Warnf("[CreateStorageProtectionGroup] - source cluster UID not provided, using remote system ID in RCG name.") clusterUID = remoteSystemID } rcgPrefix, ok := parameters[s.WithRP(KeyReplicationVGPrefix)] if !ok || rcgPrefix == "" { - Log.Warnf("[CreateStorageProtectionGroup] - RCG prefix not provided, using 'RCG' as prefix.") + log.Warnf("[CreateStorageProtectionGroup] - RCG prefix not provided, using 'RCG' as prefix.") rcgPrefix = "rcg" } @@ -364,7 +372,7 @@ func (s *service) CreateStorageProtectionGroup(ctx context.Context, req *replica if err != nil { return nil, err } - Log.Printf("[CreateStorageProtectionGroup] - consistencyGroupName: %+s", consistencyGroupName) + log.Infof("[CreateStorageProtectionGroup] - consistencyGroupName: %+s", consistencyGroupName) } localRcg, err := s.CreateReplicationConsistencyGroup(systemID, consistencyGroupName, @@ -381,7 +389,7 @@ func (s *service) CreateStorageProtectionGroup(ctx context.Context, req *replica remoteVolumeName := "replicated-" + vol.Name if len(remoteVolumeName) > 31 { - Log.Printf("remoteVolumeName: %s longer than 31 character max, will search for truncated name: %s", remoteVolumeName, remoteVolumeName[0:31]) + log.Infof("remoteVolumeName: %s longer than 31 character max, will search for truncated name: %s", remoteVolumeName, remoteVolumeName[0:31]) remoteVolumeName = remoteVolumeName[0:31] } @@ -420,7 +428,7 @@ func (s *service) CreateStorageProtectionGroup(ctx context.Context, req *replica s.opts.replicationContextPrefix + "remoteSystemID": localSystem.ID, } - Log.Printf("[CreateStorageProtectionGroup] - localRcg: %+s, group.ID: %s", localRcg.ID, group.ID) + log.Infof("[CreateStorageProtectionGroup] - localRcg: %+s, group.ID: %s", localRcg.ID, group.ID) return &replication.CreateStorageProtectionGroupResponse{ LocalProtectionGroupId: group.ID, @@ -432,7 +440,7 @@ func (s *service) CreateStorageProtectionGroup(ctx context.Context, req *replica } func (s *service) GetStorageProtectionGroupStatus(_ context.Context, req *replication.GetStorageProtectionGroupStatusRequest) (*replication.GetStorageProtectionGroupStatusResponse, error) { - Log.Printf("[GetStorageProtectionGroupStatus] - req %+v", req) + log.Infof("[GetStorageProtectionGroupStatus] - req %+v", req) localParams := req.GetProtectionGroupAttributes() @@ -455,7 +463,7 @@ func (s *service) GetStorageProtectionGroupStatus(_ context.Context, req *replic return nil, status.Errorf(codes.Internal, "no replication pairs exist") } - Log.Printf("[GetStorageProtectionGroupStatus] - group %+v", group) + log.Infof("[GetStorageProtectionGroupStatus] - group %+v", group) var state replication.StorageProtectionGroupStatus_State switch group.CurrConsistMode { @@ -464,7 +472,7 @@ func (s *service) GetStorageProtectionGroupStatus(_ context.Context, req *replic case goscaleio.Consistent: state = replication.StorageProtectionGroupStatus_SYNCHRONIZED default: - Log.Printf("The status (%s) does not match with known protection group states", group.CurrConsistMode) + log.Infof("The status (%s) does not match with known protection group states", group.CurrConsistMode) state = replication.StorageProtectionGroupStatus_UNKNOWN } @@ -485,7 +493,7 @@ func (s *service) GetStorageProtectionGroupStatus(_ context.Context, req *replic } func (s *service) DeleteStorageProtectionGroup(_ context.Context, req *replication.DeleteStorageProtectionGroupRequest) (*replication.DeleteStorageProtectionGroupResponse, error) { - Log.Printf("[DeleteStorageProtectionGroup] %+v", req) + log.Infof("[DeleteStorageProtectionGroup] %+v", req) localParams := req.GetProtectionGroupAttributes() protectionGroupSystem := localParams[s.opts.replicationContextPrefix+"systemName"] @@ -512,7 +520,7 @@ func (s *service) DeleteStorageProtectionGroup(_ context.Context, req *replicati } func (s *service) ExecuteAction(ctx context.Context, req *replication.ExecuteActionRequest) (*replication.ExecuteActionResponse, error) { - Log.Printf("[ExecuteAction] - req %+v", req) + log.Infof("[ExecuteAction] - req %+v", req) action := req.GetAction().GetActionTypes().String() protectionGroupID := req.GetProtectionGroupId() @@ -740,7 +748,7 @@ func (s *service) getReplicationPairs(systemID string, groupID string) ([]*sioty pairs, err := rcg.GetReplicationPairs() if err != nil { if !strings.EqualFold(err.Error(), sioReplicationPairsDoesNotExist) { - Log.Printf("Error getting replication pairs: %s", err.Error()) + log.Infof("Error getting replication pairs: %s", err.Error()) return nil, err } } diff --git a/service/service.go b/service/service.go index 0b4df4f2..bd657c73 100644 --- a/service/service.go +++ b/service/service.go @@ -1,4 +1,4 @@ -// Copyright © 2019-2025 Dell Inc. or its subsidiaries. All Rights Reserved. +// Copyright © 2019-2026 Dell Inc. or its subsidiaries. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,31 +16,40 @@ package service import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" "fmt" + "io" "net" + "net/http" "os" "path/filepath" "regexp" "runtime" + "slices" "sort" "strconv" "strings" "sync" "time" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-vxflexos/v2/core" "github.com/dell/csi-vxflexos/v2/k8sutils" + "github.com/dell/csmlog" "github.com/dell/dell-csi-extensions/podmon" "github.com/dell/dell-csi-extensions/replication" volumeGroupSnapshot "github.com/dell/dell-csi-extensions/volumeGroupSnapshot" + "github.com/dell/gobrick" "github.com/dell/gocsi" csictx "github.com/dell/gocsi/context" + "github.com/dell/gonvme" "github.com/dell/goscaleio" sio "github.com/dell/goscaleio" siotypes "github.com/dell/goscaleio/types/v1" + "github.com/apparentlymart/go-cidr/cidr" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/fsnotify/fsnotify" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -52,6 +61,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -71,14 +81,27 @@ const ( // SystemTopologySystemValue is the supported topology key SystemTopologySystemValue string = "csi-vxflexos.dellemc.com" - // DefaultLogLevel for csi logs - DefaultLogLevel = logrus.DebugLevel + DefaultLogLevel = csmlog.DebugLevel + + // ParamCSILogLevel csi driver csmlog level - // ParamCSILogLevel csi driver log level ParamCSILogLevel = "CSI_LOG_LEVEL" DriverNamespace = "vxflexos" DriverConfigMap = "vxflexos-config-params" ConfigMapFilePath = "/vxflexos-config-params/driver-config-params.yaml" + + // defaultNodeChrootPath for nvme commands + defaultNodeChrootPath = "/noderoot" + + // Supported protocols + NVMeTCP = "NVMeTCP" + SDC = "SDC" + + // Timeout for making http requests + Timeout = time.Second * 5 + + // DefaultPodmonPollRate for podmon + DefaultPodmonPollRate = 60 ) var ( @@ -86,6 +109,8 @@ var ( px = sync.Mutex{} ) +// var driverLog = csmlog.GetLogger() + // LookupEnv - Fetches the environment var value var LookupEnv = lookupEnv @@ -103,7 +128,6 @@ var K8sClientset kubernetes.Interface // Log controlls the logger // give default value, will be overwritten by configmap -var Log = logrus.New() // ArrayConnectionData contains data required to connect to array type ArrayConnectionData struct { @@ -118,6 +142,14 @@ type ArrayConnectionData struct { NasName string `json:"nasName"` Mdm string `json:"mdm,omitempty"` AvailabilityZone *AvailabilityZone `json:"zone,omitempty"` + BlockProtocol string `json:"blockProtocol,omitempty"` + AuthType string `json:"authType,omitempty"` + CiamClientID string `json:"ciamClientId,omitempty"` + CiamClientSecret string `json:"ciamClientSecret,omitempty"` + OidcClientID string `json:"oidcClientId,omitempty"` + OidcClientSecret string `json:"oidcClientSecret,omitempty"` + Issuer string `json:"issuer,omitempty"` + Scopes string `json:"scopes,omitempty"` } // Definitions to make AvailabilityZone decomposition easier to read. @@ -135,17 +167,24 @@ type AvailabilityZone struct { ProtectionDomains []ProtectionDomain `json:"protectionDomains"` } +// ArrayConnectivityStatus Status of the array probe +type ArrayConnectivityStatus struct { + LastSuccess int64 `json:"lastSuccess"` // connectivity status + LastAttempt int64 `json:"lastAttempt"` // last timestamp attempted to check connectivity +} + // ProtectionDomain provides protection domain information for a cluster's availability zone type ProtectionDomain struct { Name ProtectionDomainName `json:"name"` Pools []PoolName `json:"pools"` } +// Update when the manifest version changes. +var ManifestSemver string + // Manifest is the SP's manifest. var Manifest = map[string]string{ - "url": "http://github.com/dell/csi-vxflexos", - "semver": core.SemVer, - "commit": core.CommitSha32, + "semver": ManifestSemver, "formed": core.CommitTime.Format(time.RFC1123), } @@ -190,12 +229,24 @@ type Opts struct { KubeNodeName string zoneLabelKey string probeTimeout time.Duration + NodeChrootPath string + IsPodmonEnabled bool // used to indicate that podmon is enabled + PodmonPort string // to indicates the port to be used for exposing podmon API health + PodmonPollingFreq string // indicates the polling frequency to check array connectivity + AuthType string // indicate what auth type to use +} + +type PlatformInfo struct { + SystemID string + ArrayVersion float64 + GenType string } type service struct { opts Opts adminClients map[string]*sio.Client systems map[string]*sio.System + platformInfos map[string]*PlatformInfo mode string volCache []*siotypes.Volume volCacheRWL sync.RWMutex @@ -209,6 +260,14 @@ type service struct { // maps the first 24 bits of a volume ID to the volume's systemID volumePrefixToSystems map[string][]string connectedSystemNameToID map[string]string + nvmeTargetNqn map[string]string + nvmeConnector NVMEConnector + nvmeLib gonvme.NVMEinterface + useNVME bool + useSDC bool + nodeID string + probeStatus *sync.Map + probeLocks sync.Map // map[string]*sync.Mutex } type Config struct { @@ -230,12 +289,12 @@ func (s *service) ProcessMapSecretChange() error { // Update dynamic config params vc := viper.New() vc.AutomaticEnv() - Log.WithField("file", DriverConfigParamsFile).Info("driver configuration file ") + log.WithFields(csmlog.Fields{"file": DriverConfigParamsFile}).Info("driver configuration file") vc.SetConfigFile(DriverConfigParamsFile) if err := vc.ReadInConfig(); err != nil { - Log.WithError(err).Error("unable to read config file, using default values") + log.Errorf("unable to read config file, using default values: %v", err) } - if err := s.updateDriverConfigParams(Log, vc); err != nil { + if err := s.updateDriverConfigParams(vc); err != nil { return err } vc.WatchConfig() @@ -243,16 +302,17 @@ func (s *service) ProcessMapSecretChange() error { // Putting in mutex to allow tests to pass with race flag mx.Lock() defer mx.Unlock() - Log.WithField("file", DriverConfigParamsFile).Info("log configuration file changed") - if err := s.updateDriverConfigParams(Log, vc); err != nil { - Log.Warn(err) + log.WithFields(csmlog.Fields{"file": DriverConfigParamsFile}).Info("driver configuration file") + if err := s.updateDriverConfigParams(vc); err != nil { + log.Warn(err.Error()) } }) // dynamic array secret change va := viper.New() va.SetConfigFile(ArrayConfigFile) - Log.WithField("file", ArrayConfigFile).Info("array configuration file") + + log.WithFields(csmlog.Fields{"file": ArrayConfigFile}).Info("driver configuration file") va.WatchConfig() @@ -260,19 +320,19 @@ func (s *service) ProcessMapSecretChange() error { // Putting in mutex to allow tests to pass with race flag mx.Lock() defer mx.Unlock() - Log.WithField("file", ArrayConfigFile).Info("array configuration file changed") + log.WithFields(csmlog.Fields{"file": ArrayConfigFile}).Info("driver configuration file") var err error s.opts.arrays, err = getArrayConfig(context.Background()) if err != nil { - Log.WithError(err).Error("unable to reload multi array config file") + log.Errorf("unable to reload multi array config file: %v", err) } err = s.doProbe(context.Background()) if err != nil { - Log.WithError(err).Error("unable to probe array in multi array config") + log.Errorf("unable to probe array in multi array config: %v", err) } // log csiNode topology keys if err = s.logCsiNodeTopologyKeys(); err != nil { - Log.WithError(err).Error("unable to log csiNode topology keys") + log.Errorf("unable to log csiNode topology keys: %v", err) } }) return nil @@ -282,7 +342,7 @@ func (s *service) logCsiNodeTopologyKeys() error { if K8sClientset == nil { err := k8sutils.CreateKubeClientSet(KubeConfig) if err != nil { - Log.WithError(err).Error("unable to create k8s clientset for query") + log.Errorf("unable to create k8s clientset for query: %v", err) return err } K8sClientset = k8sutils.Clientset @@ -290,16 +350,16 @@ func (s *service) logCsiNodeTopologyKeys() error { csiNodes, err := K8sClientset.StorageV1().CSINodes().List(context.TODO(), metav1.ListOptions{}) if err != nil { - Log.WithError(err).Error("unable to get node list") + log.Errorf("unable to get node list: %v", err) return err } node, err := s.NodeGetInfo(context.Background(), nil) if node != nil { - Log.WithField("node info", node.NodeId).Info("NodeInfo ID") + log.WithFields(csmlog.Fields{"node info": node.NodeId}).Info("NodeInfo ID") segMap := node.AccessibleTopology.Segments for key := range segMap { - Log.WithField("node info key", key).Info("NodeInfo topologykeys") + log.WithFields(csmlog.Fields{"node info key": key}).Info("NodeInfo topologykeys") } if err == nil { @@ -309,30 +369,30 @@ func (s *service) logCsiNodeTopologyKeys() error { csiNodeName := csiNode.Spec.Drivers[i].Name if csinodeID == node.NodeId && csiNodeName == Name { csinodeID := csiNode.Spec.Drivers[i].NodeID - Log.WithField("csinode", csiNode.Name).Info("csiNode name") - Log.WithField("csinode ID", csinodeID).Info("csiNode id") + log.WithFields(csmlog.Fields{"csinode": csiNode.Name}).Info("csiNode name") + log.WithFields(csmlog.Fields{"csinode ID": csinodeID}).Info("csiNode id") tkeys := csiNode.Spec.Drivers[i].TopologyKeys if tkeys != nil { - Log.WithField("csinode topologykeys", len(tkeys)).Info("count") + log.WithFields(csmlog.Fields{"csinode topologykeys": len(tkeys)}).Info("count") needMap := make(map[string]string) for key := range segMap { for _, tkey := range tkeys { if tkey != key { needMap[key] = "missing" } else { - Log.WithField("csinode topologykeys", "ok").Info("found") + log.WithFields(csmlog.Fields{"csinode topologykeys": "ok"}).Info("found") } } } for akey := range needMap { - Log.WithField("csinode missing topology key", akey).Info("node key") + log.WithFields(csmlog.Fields{"csinode missing topology key": akey}).Info("node key") } } } } } } else { - Log.WithError(err).Error("unable to list csiNodes in cluster") + log.Errorf("unable to list csiNodes in cluster: %v", err) } } return nil @@ -347,18 +407,18 @@ func New() Service { } } -func (s *service) updateDriverConfigParams(logger *logrus.Logger, v *viper.Viper) error { +func (s *service) updateDriverConfigParams(v *viper.Viper) error { logFormat := v.GetString("CSI_LOG_FORMAT") logFormat = strings.ToLower(logFormat) - logger.WithField("format", logFormat).Info("Read CSI_LOG_FORMAT from log configuration file") + log.WithFields(csmlog.Fields{"format": logFormat}).Info("Read CSI_LOG_FORMAT from log configuration file") if strings.EqualFold(logFormat, "json") { - logger.SetFormatter(&logrus.JSONFormatter{}) + log.SetFormatter(&logrus.JSONFormatter{}) } else { // use text formatter by defualt if logFormat != "text" { - logger.WithField("format", logFormat).Info("CSI_LOG_FORMAT value not recognized, setting to text") + log.WithFields(csmlog.Fields{"format": logFormat}).Info("CSI_LOG_FORMAT value not recognized, setting to text") } - logger.SetFormatter(&logrus.TextFormatter{}) + log.SetFormatter(&logrus.TextFormatter{}) } level := DefaultLogLevel @@ -366,17 +426,17 @@ func (s *service) updateDriverConfigParams(logger *logrus.Logger, v *viper.Viper logLevel := v.GetString(ParamCSILogLevel) if logLevel != "" { logLevel = strings.ToLower(logLevel) - logger.WithField("level", logLevel).Info("Read CSI_LOG_LEVEL from log configuration file") + log.WithFields(csmlog.Fields{"level": logLevel}).Info("Read CSI_LOG_LEVEL from log configuration file") var err error - level, err = logrus.ParseLevel(logLevel) + level, err = csmlog.ParseLevel(logLevel) if err != nil { - Log.WithError(err).Errorf("CSI_LOG_LEVEL %s value not recognized, setting to debug error: %s ", logLevel, err.Error()) - logger.SetLevel(DefaultLogLevel) + log.Errorf("CSI_LOG_LEVEL %s value not recognized, setting to debug error: %s ", logLevel, err.Error()) + log.SetLevel(DefaultLogLevel) return fmt.Errorf("input log level %q is not valid", logLevel) } } } - logger.SetLevel(level) + csmlog.SetLevel(level) // set X_CSI_LOG_LEVEL so that gocsi doesn't overwrite the loglevel set by us _ = os.Setenv(gocsi.EnvVarLogLevel, level.String()) return nil @@ -402,9 +462,12 @@ func (s *service) BeforeServe( "IsQuotaEnabled": s.opts.IsQuotaEnabled, "ExternalAccess": s.opts.ExternalAccess, "KubeNodeName": s.opts.KubeNodeName, + "isPodmonEnabled": s.opts.IsPodmonEnabled, + "PodmonPort": s.opts.PodmonPort, + "PodmonFrequency": s.opts.PodmonPollingFreq, } - Log.WithFields(fields).Infof("configured %s", Name) + log.WithFields(fields).Infof("configured %s", Name) }() // Get the SP's operating mode. @@ -417,19 +480,19 @@ func (s *service) BeforeServe( // Process configuration file and initialize system clients opts.arrays, err = getArrayConfig(ctx) if err != nil { - Log.Warnf("unable to get arrays from config: %s", err.Error()) + log.Warnf("unable to get arrays from config: %s", err.Error()) return err } // if custom zoning is being used, find the common label from the array secret opts.zoneLabelKey, err = getZoneKeyLabelFromSecret(opts.arrays) if err != nil { - Log.Warnf("unable to get zone key from secret: %s", err.Error()) + log.Warnf("unable to get zone key from secret: %s", err.Error()) return err } if err = s.ProcessMapSecretChange(); err != nil { - Log.Warnf("unable to configure dynamic configMap secret change detection : %s", err.Error()) + log.Warnf("unable to configure dynamic configMap secret change detection : %s", err.Error()) return err } @@ -473,6 +536,15 @@ func (s *service) BeforeServe( opts.IsApproveSDCEnabled = true } } + + if nodeChrootPath, ok := csictx.LookupEnv(ctx, EnvNodeChrootPath); ok { + opts.NodeChrootPath = nodeChrootPath + } + + if opts.NodeChrootPath == "" { + opts.NodeChrootPath = defaultNodeChrootPath + } + if quotaEnabled, ok := csictx.LookupEnv(ctx, EnvQuotaEnabled); ok { if quotaEnabled == "true" { opts.IsQuotaEnabled = true @@ -491,7 +563,7 @@ func (s *service) BeforeServe( opts.replicationPrefix = replicationPrefix } if MaxVolumesPerNode, err := ParseInt64FromContext(ctx, EnvMaxVolumesPerNode); err != nil { - Log.Warnf("error while parsing env variable '%s', %s, defaulting to 0", EnvMaxVolumesPerNode, err) + log.Warnf("error while parsing env variable '%s', %s, defaulting to 0", EnvMaxVolumesPerNode, err) opts.MaxVolumesPerNode = 0 } else { opts.MaxVolumesPerNode = MaxVolumesPerNode @@ -500,12 +572,12 @@ func (s *service) BeforeServe( // Trimming spaces if any externalAccess = strings.TrimSpace(externalAccess) if externalAccess == "" { - Log.Infof("externalAccess is not provided") + log.Infof("externalAccess is not provided") opts.ExternalAccess = "" } else { opts.ExternalAccess, err = ParseCIDR(externalAccess) if err != nil { - Log.Warnf("error while parsing the externalAccess : %s, defaulting to empty", err) + log.Warnf("error while parsing the externalAccess : %s, defaulting to empty", err) opts.ExternalAccess = "" } } @@ -514,23 +586,39 @@ func (s *service) BeforeServe( opts.KubeNodeName = kubeNodeName } + if isPodmonEnabled, ok := csictx.LookupEnv(ctx, EnvPodmonEnabled); ok { + opts.IsPodmonEnabled = strings.EqualFold(isPodmonEnabled, "true") + } + + if podmonPort, ok := csictx.LookupEnv(ctx, EnvPodmonArrayConnectivityAPIPORT); ok { + opts.PodmonPort = fmt.Sprintf(":%s", podmonPort) + } + + if podmonPollRate, ok := csictx.LookupEnv(ctx, EnvPodmonArrayConnectivityPollRate); ok { + opts.PodmonPollingFreq = podmonPollRate + } + // log csiNode topology keys if err = s.logCsiNodeTopologyKeys(); err != nil { - Log.WithError(err).Error("unable to log csiNode topology keys") + log.Errorf("unable to log csiNode topology keys: %v", err) + } + + if EnvAuthType, ok := csictx.LookupEnv(ctx, EnvAuthType); ok { + opts.AuthType = EnvAuthType } opts.probeTimeout = DefaultAPITimeout if envProbeTimeout, ok := csictx.LookupEnv(ctx, EnvMaxProbeTimeout); ok { duration, err := time.ParseDuration(envProbeTimeout) if err != nil { - Log.Warnf("error while parsing env variable '%s', %s, defaulting to %s", EnvMaxProbeTimeout, err, DefaultAPITimeout) + log.Warnf("error while parsing env variable '%s', %s, defaulting to %s", EnvMaxProbeTimeout, err, DefaultAPITimeout) opts.probeTimeout = DefaultAPITimeout } else { - Log.Infof("env variable '%s' provided with value %s", EnvMaxProbeTimeout, envProbeTimeout) + log.Infof("env variable '%s' provided with value %s", EnvMaxProbeTimeout, envProbeTimeout) opts.probeTimeout = duration } } else { - Log.Infof("env variable '%s' not provided, defaulting to %s", EnvMaxProbeTimeout, DefaultAPITimeout) + log.Infof("env variable '%s' not provided, defaulting to %s", EnvMaxProbeTimeout, DefaultAPITimeout) } // pb parses an environment variable into a boolean value. If an error @@ -539,8 +627,7 @@ func (s *service) BeforeServe( if v, ok := csictx.LookupEnv(ctx, n); ok { b, err := strconv.ParseBool(v) if err != nil { - Log.WithField(n, v).Debug( - "invalid boolean value. defaulting to false") + log.WithFields(csmlog.Fields{n: v}).Debug("invalid boolean value. defaulting to false") return false } return b @@ -554,26 +641,172 @@ func (s *service) BeforeServe( s.opts = opts s.adminClients = make(map[string]*sio.Client) s.systems = make(map[string]*sio.System) + s.platformInfos = make(map[string]*PlatformInfo) + s.nvmeTargetNqn = make(map[string]string) + + // Initialize NVMe connectors + s.initConnectors() + + // Setup NVMe host for array version >= 4.0 + nvmeInitiators, err := s.getInitiators() + log.Infof("nvmeInitiators: %s", nvmeInitiators) + if err != nil { + log.Errorf("can not get initiators of the node: %s", err.Error()) + } + + for _, arr := range s.opts.arrays { + log.Infof("checking array version for array: %s", arr.SystemID) + version, err := s.getArrayVersion(ctx, arr.SystemID) + if err != nil { + log.Errorf("can not get version of the array: %s", err.Error()) + } else { + log.Infof("array version: %f", version) + } + + switch arr.BlockProtocol { + case NVMeTCP: + log.Infof("block protocol is set to NVMeTCP") + if version < 4.0 { + log.Warnf("NVMeTCP transport is not supported for array version %f", version) + } + if len(nvmeInitiators) == 0 { + log.Errorf("NVMeTCP transport was requested but NVMe initiator is not available") + } + s.useNVME = true + case SDC: + log.Infof("block protocol is set to SDC") + s.useSDC = true + case "auto": + log.Infof("block protocol is set to auto — node will be either SDC or NVMe, or SDC by default") + s.configureAutoBlockProtocol(ctx, version, len(nvmeInitiators)) + default: + log.Infof("block protocol is not set, defaulting to NFS") + } + + if s.useNVME { + if err := s.setupNVMeHost(nvmeInitiators, arr.SystemID); err != nil { + log.Errorf("can not setup NVMe host for array: %s", err.Error()) + } + } + } - // Update the ConfigMap with the Interface IPs if s.mode == "node" { + // Update the ConfigMap with the Interface IPs s.updateConfigMap(s.getIPAddressByInterface, ConfigMapFilePath) + + // Start the podmon API service + go s.startAPIService(ctx) } if _, ok := csictx.LookupEnv(ctx, "X_CSI_VXFLEXOS_NO_PROBE_ON_START"); !ok { - Log.Infof("BeforeServe probing starting %s", time.Now().Format("15:04:05.000000000")) + log.Infof("BeforeServe probing starting %s", time.Now().Format("15:04:05.000000000")) newContext, cancel := context.WithTimeout(ctx, s.opts.probeTimeout) defer cancel() err := s.doProbe(newContext) - Log.Infof("BeforeServe probing complete %s", time.Now().Format("15:04:05.000000000")) + log.Infof("BeforeServe probing complete %s", time.Now().Format("15:04:05.000000000")) return err } return nil } +func (s *service) configureAutoBlockProtocol(ctx context.Context, version float64, nvmeInitiators int) { + log := log.WithContext(ctx) + // do nodeProbe to detect SDC + if err := s.nodeProbe(ctx); err != nil { + log.Infof("nodeProbe failed: %s", err.Error()) + } + + isSDC := s.opts.SdcGUID != "" + isNVMe := nvmeInitiators != 0 && version >= 4.0 + + switch { + case isSDC: + log.Infof("SDC available, using SDC") + s.useSDC = true + case isNVMe: + log.Infof("NVMeTCP available, using NVMeTCP") + s.useNVME = true + default: + log.Info("neither SDC nor NVMeTCP detected, using NFS") + } +} + +func (s *service) setupNVMeHost(nvmeInitiators []string, systemID string) error { + log.Infof("setting up NVMe host for array %s", systemID) + defer log.Infof("finished setting up NVMe host for array %s", systemID) + + if len(nvmeInitiators) == 0 { + return fmt.Errorf("NVMe initiators not found on node") + } + log.Infof("NVMe initiators found on node: %s", nvmeInitiators) + + // Set up NVMe host + system, err := s.adminClients[systemID].FindSystem(systemID, "", "") + if err != nil { + log.Errorf("unable to find system: %s", err.Error()) + return err + } + + // Check if host with same name exists + hosts, err := system.GetAllNvmeHosts() + if err != nil { + log.Errorf("unable to get nvme hosts: %s", err.Error()) + return err + } + + // Get node ID + s.nodeID, err = s.generateNodeID() + if err != nil { + log.Errorf("failed to generate node ID: %s", err.Error()) + return err + } + + for _, host := range hosts { + if host.Name == s.nodeID { + log.Infof("host with same name already exists: %s", host.Name) + return nil + } + if host.Nqn != "" && slices.Contains(nvmeInitiators, host.Nqn) { + s.nodeID = host.Name + log.Infof("Found existing host with matching NQN: %s", host.Name) + return nil + } + } + + // If host not found → create new host + nvmeHostParams := siotypes.NvmeHostParam{ + Name: s.nodeID, + Nqn: nvmeInitiators[0], + } + + _, err = system.CreateNvmeHost(nvmeHostParams) + if err != nil { + log.Errorf("unable to create nvme host: %s", err.Error()) + return err + } + + return nil +} + +func (s *service) generateNodeID() (string, error) { + nodeUID, err := s.GetNodeUID(context.Background()) + if err != nil { + return "", err + } + + nodeIP, err := s.GetNodeIP(context.Background()) + if err != nil { + return "", err + } + + hashedNodeID := hashNodeID(nodeUID) + nodeID := fmt.Sprintf("%s-%s", nodeIP, hashedNodeID) + return nodeID[:31], nil +} + func (s *service) updateConfigMap(getIPAddressByInterfacefunc GetIPAddressByInterfacefunc, configFilePath string) { // Prepare driverConfigMap name using the release name releaseName := os.Getenv("RELEASE_NAME") @@ -581,14 +814,14 @@ func (s *service) updateConfigMap(getIPAddressByInterfacefunc GetIPAddressByInte configFileData, err := os.ReadFile(filepath.Clean(configFilePath)) if err != nil { - Log.Errorf("Failed to read ConfigMap file: %v", err) + log.Errorf("Failed to read ConfigMap file: %v", err) return } var config Config err = yaml.Unmarshal(configFileData, &config) if err != nil { - Log.Errorf("Failed to parse configMap data: %v", err) + log.Errorf("Failed to parse configMap data: %v", err) return } @@ -607,7 +840,7 @@ func (s *service) updateConfigMap(getIPAddressByInterfacefunc GetIPAddressByInte // Find the IP of the Interfaces ipAddress, err := getIPAddressByInterfacefunc(interfaceName, &service{}) if err != nil { - Log.Printf("Error while getting IP address for interface %s: %v\n", interfaceName, err) + log.Infof("Error while getting IP address for interface %s: %v\n", interfaceName, err) continue } ipAddresses = append(ipAddresses, ipAddress) @@ -621,7 +854,7 @@ func (s *service) updateConfigMap(getIPAddressByInterfacefunc GetIPAddressByInte if K8sClientset == nil { err = k8sutils.CreateKubeClientSet() if err != nil { - Log.Errorf("Failed to create Kubernetes ClientSet: %v", err) + log.Errorf("Failed to create Kubernetes ClientSet: %v", err) return } K8sClientset = k8sutils.Clientset @@ -630,7 +863,7 @@ func (s *service) updateConfigMap(getIPAddressByInterfacefunc GetIPAddressByInte // Get the vxflexos-config-params ConfigMap cm, err := K8sClientset.CoreV1().ConfigMaps(DriverNamespace).Get(context.TODO(), driverConfigMap, metav1.GetOptions{}) if err != nil { - Log.Errorf("Failed to get ConfigMap: %v", err) + log.Errorf("Failed to get ConfigMap: %v", err) return } @@ -639,7 +872,7 @@ func (s *service) updateConfigMap(getIPAddressByInterfacefunc GetIPAddressByInte err := yaml.Unmarshal([]byte(existingYaml), &configData) if err != nil { - Log.Errorf("Failed to parse ConfigMap data: %v", err) + log.Errorf("Failed to parse ConfigMap data: %v", err) return } @@ -649,13 +882,13 @@ func (s *service) updateConfigMap(getIPAddressByInterfacefunc GetIPAddressByInte interfaceNames[node] = ipAddressList } } else { - Log.Errorf("interfaceNames key missing or not in expected format") + log.Errorf("interfaceNames key missing or not in expected format") return } updatedYaml, err := yaml.Marshal(configData) if err != nil { - Log.Errorf("Failed to marshal updated data: %v", err) + log.Errorf("Failed to marshal updated data: %v", err) return } cm.Data["driver-config-params.yaml"] = string(updatedYaml) @@ -664,11 +897,11 @@ func (s *service) updateConfigMap(getIPAddressByInterfacefunc GetIPAddressByInte // Update the vxflexos-config-params ConfigMap _, err = K8sClientset.CoreV1().ConfigMaps("vxflexos").Update(context.TODO(), cm, metav1.UpdateOptions{}) if err != nil { - Log.Errorf("Failed to update ConfigMap: %v", err) + log.Errorf("Failed to update ConfigMap: %v", err) return } - Log.Infof("ConfigMap updated successfully") + log.Infof("ConfigMap updated successfully") } func (s *service) getIPAddressByInterface(interfaceName string, networkInterface NetworkInterface) (string, error) { @@ -695,26 +928,31 @@ func (s *service) getIPAddressByInterface(interfaceName string, networkInterface return "", fmt.Errorf("no IPv4 address found for interface %s", interfaceName) } +func (s *service) isGenTypeNotSupportsNfsAndReplication(genType string) bool { + return genType == "EC" +} + +func (s *service) isNfsNotSupported(version float64) bool { + // NFS is only supported in PowerFlex version 4.0+ and not in version <4.0 and in Nairobi + return version < 4.0 +} + +func (s *service) isReplicationNotSupported(genType string) bool { + // For Nairobi(5.0), replication is not supported. + return genType == "EC" +} + func (s *service) isNFSEnabled(ctx context.Context, systemID string) (bool, error) { if err := s.systemProbeAll(ctx); err != nil { return false, err } - c := s.adminClients[systemID] - if c == nil { - return false, nil - } - version, err := c.GetVersion() - if err != nil { - return false, err - } - ver, err := strconv.ParseFloat(version, 64) + version, err := s.GetPlatformVersion(systemID) if err != nil { return false, err } - // NFS is only supported in PowerFlex version 4.0+ - if ver < 4.0 { + if s.isNfsNotSupported(version) { return false, nil } @@ -730,7 +968,7 @@ func (s *service) isNFSEnabled(ctx context.Context, systemID string) (bool, erro // If no NAS name configured, NFS cannot be used if strings.TrimSpace(array.NasName) == "" { - Log.Warnf("nasName value not found in secret, it is mandatory parameter for NFS volume operations") + log.Warnf("nasName value not found in secret, it is mandatory parameter for NFS volume operations") return false, nil } @@ -749,7 +987,7 @@ func (s *service) doProbe(ctx context.Context) error { defer px.Unlock() if !s.isNodeMode() { - Log.Info("[doProbe] controllerProbe") + log.Info("[doProbe] controllerProbe") if err := s.systemProbeAll(ctx); err != nil { return err } @@ -758,13 +996,13 @@ func (s *service) doProbe(ctx context.Context) error { // Do a node probe if !s.isControllerMode() { // Probe all systems managed by driver - Log.Info("[doProbe] nodeProbe") + log.Info("[doProbe] nodeProbe") if err := s.systemProbeAll(ctx); err != nil { return err } if err := s.nodeProbe(ctx); err != nil { - Log.Infof("nodeProbe failed: %s", err.Error()) + log.Infof("nodeProbe failed: %s", err.Error()) } } return nil @@ -772,7 +1010,7 @@ func (s *service) doProbe(ctx context.Context) error { // RegisterAdditionalServers registers any additional grpc services that use the CSI socket. func (s *service) RegisterAdditionalServers(server *grpc.Server) { - Log.Info("Registering additional GRPC servers") + log.Info("Registering additional GRPC servers") podmon.RegisterPodmonServer(server, s) volumeGroupSnapshot.RegisterVolumeGroupSnapshotServer(server, s) replication.RegisterReplicationServer(server, s) @@ -790,7 +1028,7 @@ func (s *service) getVolProvisionType(params map[string]string) string { if tp, ok := params[KeyThickProvisioning]; ok { tpb, err := strconv.ParseBool(tp) if err != nil { - Log.Warnf("invalid boolean received %s=(%#v) in params", + log.Warnf("invalid boolean received %s=(%#v) in params", KeyThickProvisioning, tp) } else if tpb { volType = thickProvisioned @@ -886,7 +1124,7 @@ func (s *service) getCSIVolume(vol *siotypes.Volume, systemID string) *csi.Volum storagePoolName := s.getStoragePoolNameFromID(systemID, vol.StoragePoolID) installationID, err := s.getArrayInstallationID(systemID) if err != nil { - Log.Printf("getCSIVolume error system not found: %s with error: %v\n", systemID, err) + log.Infof("getCSIVolume error system not found: %s with error: %v\n", systemID, err) } // Make the additional volume attributes @@ -914,7 +1152,7 @@ func (s *service) getCSIVolumeFromFilesystem(fs *siotypes.FileSystem, systemID s storagePoolName := s.getStoragePoolNameFromID(systemID, fs.StoragePoolID) installationID, err := s.getArrayInstallationID(systemID) if err != nil { - Log.Printf("getCSIVolumeFromFilesystem error system not found: %s with error: %v\n", systemID, err) + log.Infof("getCSIVolumeFromFilesystem error system not found: %s with error: %v\n", systemID, err) } // Make the additional volume attributes @@ -993,7 +1231,7 @@ func (s *service) getStoragePoolNameFromID(systemID, id string) string { storagePoolName = pool.Name s.storagePoolIDToName[id] = pool.Name } else { - Log.Printf("Could not found StoragePool: %s on system %s", id, systemID) + log.Infof("Could not found StoragePool: %s on system %s", id, systemID) } } return storagePoolName @@ -1011,7 +1249,7 @@ func (s *service) logStatistics() { "HeapReleased": memstats.HeapReleased, "StackSys": memstats.StackSys, } - Log.WithFields(fields).Infof("resource statistics counter: %d", s.statisticsCounter) + log.WithFields(fields).Infof("resource statistics counter: %d", s.statisticsCounter) } } @@ -1020,7 +1258,7 @@ func getArrayConfig(_ context.Context) (map[string]*ArrayConnectionData, error) _, err := os.Stat(ArrayConfigFile) if err != nil { - Log.Errorf("Found error %v while checking stat of file %s ", err, ArrayConfigFile) + log.Errorf("Found error %v while checking stat of file %s ", err, ArrayConfigFile) if os.IsNotExist(err) { return nil, fmt.Errorf("file %s does not exist", ArrayConfigFile) } @@ -1062,10 +1300,14 @@ func getArrayConfig(_ context.Context) (map[string]*ArrayConnectionData, error) if c.Endpoint == "" { return nil, fmt.Errorf("invalid value for Endpoint at index %d", i) } + if strings.TrimSpace(c.BlockProtocol) == "" { + log.Infof("BlockProtocol is not set, defaulting to auto") + c.BlockProtocol = "auto" + } // ArrayConnectionData if c.AllSystemNames != "" { names := strings.Split(c.AllSystemNames, ",") - Log.Printf("Powerflex systemID %s AllSytemNames given %#v\n", systemID, names) + log.Infof("Powerflex systemID %s AllSytemNames given %#v\n", systemID, names) } // for PowerFlex v4.0 @@ -1084,9 +1326,10 @@ func getArrayConfig(_ context.Context) (map[string]*ArrayConnectionData, error) "systemID": c.SystemID, "allSystemNames": c.AllSystemNames, "nasName": c.NasName, + "blockProtocol": c.BlockProtocol, } - Log.WithFields(fields).Infof("configured %s", c.SystemID) + log.WithFields(fields).Infof("configured %s", c.SystemID) if c.IsDefault { noOfDefaultArray++ @@ -1123,7 +1366,7 @@ func getVolumeIDFromCsiVolumeID(csiVolID string) string { return tokens[index-1] } err := errors.New("csiVolID unexpected string") - Log.WithError(err).Errorf("%s format error", csiVolID) + log.Errorf("%s format error: %v", csiVolID, err) return "" } @@ -1145,7 +1388,7 @@ func getFilesystemIDFromCsiVolumeID(csiVolID string) string { } } err := errors.New("csiVolID unexpected string") - Log.WithError(err).Errorf("%s format error", csiVolID) + log.Errorf("%s format error: %v", csiVolID, err) return "" } @@ -1290,19 +1533,19 @@ func ParseCIDR(externalAccessCIDR string) (string, error) { if !strings.Contains(externalAccessCIDR, "/") { // if externalAccess is a plane ip we can add /32 from our end externalAccessCIDR += "/32" - Log.Debug("externalAccess after appending netMask bit:", externalAccessCIDR) + log.Debugf("externalAccess after appending netMask bit: %v", externalAccessCIDR) } ip, ipnet, err := net.ParseCIDR(externalAccessCIDR) if err != nil { return "", err } - Log.Debug("Parsed CIDR:", externalAccessCIDR, "-> ip:", ip, " net:", ipnet) + log.Debugf("Parsed CIDR: %s -> ip: %v net: %v", externalAccessCIDR, ip, ipnet) start, _ := cidr.AddressRange(ipnet) fromString, err := GetIPListWithMaskFromString(externalAccessCIDR) if err != nil { return "", err } - Log.Debug("IP with Mask:", fromString) + log.Debugf("IP with Mask: %v", fromString) part := strings.Split(fromString, "/") // ExernalAccess IP consists of Starting range IP of CIDR+Mask and hence concatenating the same to remove from the array @@ -1313,10 +1556,10 @@ func ParseCIDR(externalAccessCIDR string) (string, error) { // ExternalAccessAlreadyAdded return true if externalAccess is present on ARRAY in any access mode type func externalAccessAlreadyAdded(export *siotypes.NFSExport, externalAccess string) bool { if Contains(export.ReadWriteRootHosts, externalAccess) || Contains(export.ReadWriteHosts, externalAccess) || Contains(export.ReadOnlyRootHosts, externalAccess) || Contains(export.ReadOnlyHosts, externalAccess) { - Log.Debug("ExternalAccess is already added into Host Access list on array: ", externalAccess) + log.Debugf("ExternalAccess is already added into Host Access list on array: %v", externalAccess) return true } - Log.Debug("Going to add externalAccess into Host Access list on array: ", externalAccess) + log.Debugf("Going to add externalAccess into Host Access list on array: %v", externalAccess) return false } @@ -1338,7 +1581,7 @@ func (s *service) unexportFilesystem(_ context.Context, _ *csi.ControllerUnpubli } if !nfsExportExists { - Log.Infof("NFS Share: %s not found on array.", nfsExportName) + log.Infof("NFS Share: %s not found on array.", nfsExportName) return nil } @@ -1359,7 +1602,7 @@ func (s *service) unexportFilesystem(_ context.Context, _ *csi.ControllerUnpubli if len(nfsExportResp.ReadOnlyHosts) > 0 { if index >= 0 { modifyParam.RemoveReadOnlyHosts = append(modifyParam.RemoveReadOnlyHosts, nodeIP+"/255.255.255.255") // we can't remove without netmask - Log.Debug("Going to remove IP from ROHosts: ", nodeIP) + log.Debugf("Going to remove IP from ROHosts: %v", nodeIP) } } } @@ -1370,7 +1613,7 @@ func (s *service) unexportFilesystem(_ context.Context, _ *csi.ControllerUnpubli if len(nfsExportResp.ReadOnlyRootHosts) > 0 { if index >= 0 { modifyParam.RemoveReadOnlyRootHosts = append(modifyParam.RemoveReadOnlyRootHosts, nodeIP+"/255.255.255.255") // we can't remove without netmask - Log.Debug("Going to remove IP from RORootHosts: ", nodeIP) + log.Debugf("Going to remove IP from RORootHosts: %v", nodeIP) } } } @@ -1378,14 +1621,14 @@ func (s *service) unexportFilesystem(_ context.Context, _ *csi.ControllerUnpubli for _, nodeIP := range nodeIPs { if Contains(nfsExportResp.ReadWriteHosts, nodeIP+"/255.255.255.255") { modifyParam.RemoveReadWriteHosts = append(modifyParam.RemoveReadWriteHosts, nodeIP+"/255.255.255.255") // we can't remove without netmask - Log.Debug("Going to remove IP from RWHosts: ", nodeIP) + log.Debugf("Going to remove IP from RWHosts: %v", nodeIP) } } for _, nodeIP := range nodeIPs { if Contains(nfsExportResp.ReadWriteRootHosts, nodeIP+"/255.255.255.255") { modifyParam.RemoveReadWriteRootHosts = append(modifyParam.RemoveReadWriteRootHosts, nodeIP+"/255.255.255.255") // we can't remove without netmask - Log.Debug("Going to remove IP from RWRootHosts: ", nodeIP) + log.Debugf("Going to remove IP from RWRootHosts: %v", nodeIP) } } @@ -1393,8 +1636,8 @@ func (s *service) unexportFilesystem(_ context.Context, _ *csi.ControllerUnpubli if err != nil { return status.Errorf(codes.NotFound, "Allocating host %s access to NFS Export failed. Error: %v", nodeID, err) } - Log.Debugf("Host: %s access is removed from NFS Share: %s", nodeID, nfsExportID) - Log.Debugf("ControllerUnpublishVolume successful for volid: [%s]", volumeContextID) + log.Debugf("Host: %s access is removed from NFS Share: %s", nodeID, nfsExportID) + log.Debugf("ControllerUnpublishVolume successful for volid: [%s]", volumeContextID) return nil } @@ -1426,7 +1669,7 @@ func (s *service) exportFilesystem(_ context.Context, _ *csi.ControllerPublishVo // Create NFS export if it doesn't exist if !nfsExportExists { - Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export", fs.Name) + log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export", fs.Name) resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, FileSystemID: fs.ID, @@ -1509,13 +1752,13 @@ func (s *service) exportFilesystem(_ context.Context, _ *csi.ControllerPublishVo // Idempotent case if foundIdempotent { - Log.Info("Host has access to the given host and exists in the required state.") + log.Info("Host has access to the given host and exists in the required state.") return &csi.ControllerPublishVolumeResponse{PublishContext: pContext}, nil } // Check and remove the default host if given in external access if Contains(nodeIPs, externalAccess) { - Log.Debug("Setting externalAccess to empty as it contains the host ip") + log.Debug("Setting externalAccess to empty as it contains the host ip") externalAccess = "" } @@ -1540,8 +1783,8 @@ func (s *service) exportFilesystem(_ context.Context, _ *csi.ControllerPublishVo } } - Log.Debugf("NFS Export: %s is accessible to host: %s with access mode: %s", nfsExportID, nodeID, am.Mode) - Log.Debugf("ControllerPublishVolume successful for volid: [%s]", pContext["volumeContextId"]) + log.Debugf("NFS Export: %s is accessible to host: %s with access mode: %s", nfsExportID, nodeID, am.Mode) + log.Debugf("ControllerPublishVolume successful for volid: [%s]", pContext["volumeContextId"]) return &csi.ControllerPublishVolumeResponse{PublishContext: pContext}, nil } @@ -1554,20 +1797,20 @@ func (s *service) UpdateVolumePrefixToSystemsMap(systemID string) error { vols, _, err := s.listVolumes(systemID, 0, 1, true, false, "", "") if err != nil { - Log.WithError(err).Errorf("failed to list vols for array %s : %s ", systemID, err.Error()) + log.Errorf("failed to list vols for array %s : %s ", systemID, err.Error()) return fmt.Errorf("failed to list vols for array %s : %s ", systemID, err.Error()) } if len(vols) == 0 { // if system has no volumes, then there can't be a legacy vol on it - Log.Printf("systemID: %s has no volumes, not adding to volumePrefixToSystems map. \n", systemID) + log.Infof("systemID: %s has no volumes, not adding to volumePrefixToSystems map. \n", systemID) return nil } volID := vols[0].ID - Log.Printf("vol id in UpdateVolumePrefixToSystemsMap is: %s from systemID: %s \n", volID, systemID) + log.Infof("vol id in UpdateVolumePrefixToSystemsMap is: %s from systemID: %s \n", volID, systemID) // use first 24 bit from volume id as a key and system id as a value, and add this entry to the map @@ -1578,16 +1821,16 @@ func (s *service) UpdateVolumePrefixToSystemsMap(systemID string) error { // if key found: // make sure systemID isn't already added for the specific key if contains(s.volumePrefixToSystems[key], systemID) { - Log.Printf("volumePrefixToSystems: systemID: %s already added for key %s. Not adding for key again. \n", systemID, key) + log.Infof("volumePrefixToSystems: systemID: %s already added for key %s. Not adding for key again. \n", systemID, key) return nil } // systemID has not been added to key before, add it - Log.Printf("volumePrefixToSystems: Adding systemID %s to key %s \n", systemID, key) + log.Infof("volumePrefixToSystems: Adding systemID %s to key %s \n", systemID, key) s.volumePrefixToSystems[key] = append(s.volumePrefixToSystems[key], systemID) } else { // if key not found: - Log.Printf("volumePrefixToSystems: adding new key, value pair: key %s, systemID: %s \n", key, systemID) + log.Infof("volumePrefixToSystems: adding new key, value pair: key %s, systemID: %s \n", key, systemID) s.volumePrefixToSystems[key] = []string{systemID} } @@ -1600,12 +1843,12 @@ func (s *service) checkVolumesMap(volumeID string) error { // ID is legacy, so we ensure it's only found on default system if systemID == "" { - Log.Printf("volume id in checkVolumesMap is: %s \n", volumeID) - Log.Printf("volume %s ,assumed to be on default system. \n", volumeID) + log.Infof("volume id in checkVolumesMap is: %s \n", volumeID) + log.Infof("volume %s ,assumed to be on default system. \n", volumeID) if len(volumeID) < 3 { err := errors.New("vol ID too short") - Log.WithError(err).Errorf("volume id %s is shorter than 3 chars, returning error", volumeID) + log.Errorf("volume id %s is shorter than 3 chars, returning error: %v", volumeID, err) return fmt.Errorf("volume id %s is shorter than 3 chars, returning error", volumeID) } @@ -1617,13 +1860,13 @@ func (s *service) checkVolumesMap(volumeID string) error { for _, systemID := range s.volumePrefixToSystems[key] { vols, _, err := s.listVolumes(systemID, 0, 0, true, false, "", "") if err != nil { - Log.WithError(err).Errorf("failed to list vols for array %s : %s ", systemID, err.Error()) + log.Errorf("failed to list vols for array %s : %s ", systemID, err.Error()) return fmt.Errorf("failed to list vols for array %s : %s ", systemID, err.Error()) } for _, vol := range vols { if vol.ID == volumeID { // legacy volume found on non-default system, this is an error - Log.WithError(err).Errorf("found volume id %s on non-default system %s. expecting this volume id only on default system. aborting operation ", volumeID, systemID) + log.Errorf("found volume id %s on non-default system %s. expecting this volume id only on default system. aborting operation: %v", volumeID, systemID, err) return fmt.Errorf("found volume id %s on non-default system %s. expecting this volume id only on default system. aborting operation ", volumeID, systemID) } } @@ -1631,12 +1874,12 @@ func (s *service) checkVolumesMap(volumeID string) error { } // volume was not found on a non default system. - Log.Infof("checkVolumesMap returns OK") + log.Infof("checkVolumesMap returns OK") return nil } // volume was not legacy - Log.Printf("Volume ID: %s contains system ID: %s. checkVolumesMap passed", volumeID, systemID) + log.Infof("Volume ID: %s contains system ID: %s. checkVolumesMap passed", volumeID, systemID) return nil } @@ -1649,7 +1892,7 @@ func (s *service) calcKeyForMap(volumeID string) string { func (s *service) getProtectionDomainIDFromName(systemID, protectionDomainName string) (string, error) { if protectionDomainName == "" { - Log.Printf("Protection Domain not provided; there could be conflicts if two storage pools share a name") + log.Infof("Protection Domain not provided; there could be conflicts if two storage pools share a name") return "", nil } system, err := s.adminClients[systemID].FindSystem(systemID, "", "") @@ -1719,7 +1962,7 @@ func (s *service) getProtectionDomain(systemID string, pdName string) (string, e return "", errors.New("no protection domains found") } - Log.Printf("[getProtectionDomain] - PD not provived, using: %s, System: %s", pd[0].Name, systemID) + log.Infof("[getProtectionDomain] - PD not provived, using: %s, System: %s", pd[0].Name, systemID) pdID = pd[0].ID @@ -1770,22 +2013,22 @@ func (s *service) findReplicationPairByVolID(systemID, volumeID string) (*siotyp } func (s *service) expandReplicationPair(ctx context.Context, req *csi.ControllerExpandVolumeRequest, systemID, volumeID string) error { - Log.Printf("[expandReplicationPair] - Start: %s, %s", systemID, volumeID) + log.Infof("[expandReplicationPair] - Start: %s, %s", systemID, volumeID) pair, err := s.findReplicationPairByVolID(systemID, volumeID) if err != nil { return err } - Log.Printf("[expandReplicationPair] - Pair Found: %+v", pair) + log.Infof("[expandReplicationPair] - Pair Found: %+v", pair) group, err := s.getReplicationConsistencyGroupByID(systemID, pair.ReplicationConsistencyGroupID) if err != nil { return err } - Log.Printf("[expandReplicationPair] - Group Found: %+v", group) + log.Infof("[expandReplicationPair] - Group Found: %+v", group) // Avoid getting in a expand attempt cycle. if group.ReplicationDirection == "RemoteToLocal" { - Log.Printf("[expandReplicationPair] - Only want to expand from LocalToRemote, if first call, there might be an issue.") + log.Infof("[expandReplicationPair] - Only want to expand from LocalToRemote, if first call, there might be an issue.") return nil } @@ -1796,8 +2039,8 @@ func (s *service) expandReplicationPair(ctx context.Context, req *csi.Controller return err } - Log.Printf("[expandReplicationPair] - ControllerExpandVolume expanded the remote volume first: %+v", resp) - Log.Printf("[expandReplicationPair] - Ensuring remote has expanded...") + log.Infof("[expandReplicationPair] - ControllerExpandVolume expanded the remote volume first: %+v", resp) + log.Infof("[expandReplicationPair] - Ensuring remote has expanded...") requestedSize, err := validateVolSize(req.CapacityRange) if err != nil { @@ -1820,7 +2063,7 @@ func (s *service) expandReplicationPair(ctx context.Context, req *csi.Controller func (s *service) getNASServerIDFromName(systemID, nasName string) (string, error) { if nasName == "" { - Log.Printf("NAS server not provided.") + log.Infof("NAS server not provided.") return "", errors.New("NAS server not provided") } system, err := s.adminClients[systemID].FindSystem(systemID, "", "") @@ -1851,21 +2094,116 @@ func (s *service) GetNodeLabels(_ context.Context) (map[string]string, error) { nodeName := s.opts.KubeNodeName if nodeName == "" { - Log.Infof("Using env variable for node name") + log.Infof("Using env variable for node name") nodeName = os.Getenv("NODENAME") } - Log.Infof("Using: %s as nodeName", nodeName) + log.Infof("Using: %s as nodeName", nodeName) // access the API to fetch node object node, err := K8sClientset.CoreV1().Nodes().Get(context.TODO(), nodeName, v1.GetOptions{}) if err != nil { return nil, status.Error(codes.Internal, GetMessage("Unable to fetch the node labels. Error: %v", err)) } - Log.Debugf("Node labels: %v\n", node.Labels) + log.Debugf("Node labels: %v\n", node.Labels) return node.Labels, nil } +// GetNodeIPByCSINodeID returns cluster IP of the node corresponding to the given CSI nodeID +func (s *service) GetNodeIPByCSINodeID(nodeID string) string { + // 1. List CSINodes + csiNodes, err := K8sClientset.StorageV1().CSINodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + log.Errorf("Error listing CSINodes: %v", err) + return "" + } + + var kubeNodeName string + for _, csiNode := range csiNodes.Items { + for _, driver := range csiNode.Spec.Drivers { + if driver.Name == Name && driver.NodeID == nodeID { + kubeNodeName = csiNode.Name + break + } + } + if kubeNodeName != "" { + break + } + } + + if kubeNodeName == "" { + log.Warnf("No Kubernetes node found for CSI nodeID: %s", nodeID) + return "" + } + + // 2. Get Node object + node, err := s.getNode(context.TODO(), kubeNodeName) + if err != nil { + return "" + } + + // 3. Extract InternalIP + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + return addr.Address + } + } + return "" +} + +// QueryArrayStatus make API call to the specified url to retrieve connection status +func (s *service) QueryArrayStatus(ctx context.Context, url string) (bool, error) { + defer func() { + if err := recover(); err != nil { + log.Debugf("panic occurred in queryStatus: %v", err) + } + }() + client := http.Client{ + Timeout: Timeout, + } + resp, err := client.Get(url) + + log.Debugf("Received response %+v for url %s", resp, url) + if err != nil { + log.Errorf("failed to call API %s due to %s ", url, err.Error()) + return false, err + } + defer resp.Body.Close() // #nosec G307 + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + log.Errorf("failed to read API response due to %s ", err.Error()) + return false, err + } + if resp.StatusCode != 200 { + log.Errorf("Found unexpected response from the server while fetching array status %d ", resp.StatusCode) + return false, fmt.Errorf("unexpected response from the server") + } + var statusResponse ArrayConnectivityStatus + err = json.Unmarshal(bodyBytes, &statusResponse) + if err != nil { + log.Errorf("unable to unmarshal and determine connectivity due to %s ", err) + return false, err + } + log.Infof("API Response received is %+v\n", statusResponse) + // responseObject has last success and last attempt timestamp in Unix format + timeDiff := statusResponse.LastAttempt - statusResponse.LastSuccess + tolerance := SetPollingFrequency(ctx) + currTime := time.Now().Unix() + // checking if the status response is stale and connectivity test is still running + // since nodeProbe is run at frequency tolerance/2, ideally below check should never be true + if (currTime - statusResponse.LastAttempt) > tolerance*2 { + log.Errorf("seems like connectivity test is not being run, current time is %d and last run was at %d", currTime, statusResponse.LastAttempt) + // considering connectivity is broken + return false, nil + } + log.Debugf("last connectivity was %d sec back, tolerance is %d sec", timeDiff, tolerance) + // give 2s leeway for tolerance check + if timeDiff <= tolerance+2 { + return true, nil + } + return false, nil +} + func (s *service) SetPodZoneLabel(ctx context.Context, zoneLabel map[string]string) error { if K8sClientset == nil { err := k8sutils.CreateKubeClientSet() @@ -1897,7 +2235,7 @@ func (s *service) SetPodZoneLabel(ctx context.Context, zoneLabel map[string]stri } for key, value := range zoneLabel { - Log.Printf("Setting Label: Key: %s, Value: %s for pod: %s\n", key, value, podName) + log.Infof("Setting Label: Key: %s, Value: %s for pod: %s\n", key, value, podName) pod.Labels[key] = value } @@ -1909,23 +2247,49 @@ func (s *service) SetPodZoneLabel(ctx context.Context, zoneLabel map[string]stri return nil } -func (s *service) GetNodeUID(_ context.Context) (string, error) { - if K8sClientset == nil { - err := k8sutils.CreateKubeClientSet() - if err != nil { - return "", status.Error(codes.Internal, GetMessage("init client failed with error: %v", err)) +// getNode returns node corresponding to the s.opts.KubeNodeName +func (s *service) getNode(ctx context.Context, nodeName string) (*corev1.Node, error) { + if nodeName == "" { + return nil, status.Error(codes.InvalidArgument, "node name is empty") + } + node, err := K8sClientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return nil, status.Error(codes.Internal, GetMessage("unable to fetch node %q: %v", nodeName, err)) + } + return node, nil +} + +// GetNodeIP returns cluster IP of the node corresponding to the s.opts.KubeNodeName +func (s *service) GetNodeIP(ctx context.Context) (string, error) { + node, err := s.getNode(ctx, s.opts.KubeNodeName) + if err != nil { + return "", err + } + + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + return addr.Address, nil } - K8sClientset = k8sutils.Clientset } - // access the API to fetch node object - node, err := K8sClientset.CoreV1().Nodes().Get(context.TODO(), s.opts.KubeNodeName, v1.GetOptions{}) + return "", fmt.Errorf("no InternalIP found for node %q", s.opts.KubeNodeName) +} + +func (s *service) GetNodeUID(ctx context.Context) (string, error) { + node, err := s.getNode(ctx, s.opts.KubeNodeName) if err != nil { - return "", status.Error(codes.Internal, GetMessage("Unable to fetch the node details. Error: %v", err)) + return "", err } + return string(node.UID), nil } +func hashNodeID(input string) string { + hash := sha256.Sum256([]byte(input)) + encoded := hex.EncodeToString(hash[:]) + return encoded +} + // GetMessage - Get message func GetMessage(format string, args ...interface{}) string { str := fmt.Sprintf(format, args...) @@ -1957,7 +2321,7 @@ func getZoneKeyLabelFromSecret(arrays map[string]*ArrayConnectionData) (string, // Assumes that the key parameter is not empty zoneKeyLabel = array.AvailabilityZone.LabelKey } else if zoneKeyLabel != array.AvailabilityZone.LabelKey { - Log.Warnf("array %s zone key %s does not match %s", array.SystemID, array.AvailabilityZone.LabelKey, zoneKeyLabel) + log.Warnf("array %s zone key %s does not match %s", array.SystemID, array.AvailabilityZone.LabelKey, zoneKeyLabel) return "", fmt.Errorf("array %s zone key %s does not match %s", array.SystemID, array.AvailabilityZone.LabelKey, zoneKeyLabel) } } @@ -1980,3 +2344,157 @@ func (s *service) isControllerMode() bool { func (array *ArrayConnectionData) isInZone(zoneName string) bool { return array.AvailabilityZone != nil && array.AvailabilityZone.Name == ZoneName(zoneName) } + +func (s *service) initConnectors() { + if s.nvmeConnector == nil { + nvmeConnectorParams := gobrick.NVMeConnectorParams{ + Chroot: s.opts.NodeChrootPath, + } + s.nvmeConnector = gobrick.NewNVMeConnector(nvmeConnectorParams) + } + + if s.nvmeLib == nil { + nvmeOptions := map[string]string{ + "chrootDirectory": s.opts.NodeChrootPath, + } + s.nvmeLib = gonvme.NewNVMe(nvmeOptions) + } +} + +func (s *service) getInitiators() ([]string, error) { + ctx := context.Background() + + var nvmeAvailable bool + + nvmeInitiators, err := s.nvmeConnector.GetInitiatorName(ctx) + if err != nil { + log.Error("nodeStartup could not get Initiator NQNs") + } else if len(nvmeInitiators) == 0 { + log.Error("NVMe initiators not found on node") + } else { + log.Debug("NVMe initiators found on node") + nvmeAvailable = true + } + + if !nvmeAvailable { + // If we haven't found any initiators we still can use NFS + log.Info("NVMe initiators not found on node") + } + + return nvmeInitiators, nil +} + +func (s *service) GetPlatformInfo(systemID string) (*PlatformInfo, error) { + platformInfo, ok := s.platformInfos[systemID] + if !ok { + log.Infof("Start: Retrieving Platform Info from Array using SystemId: %s", systemID) + + platformInfo = &PlatformInfo{ + SystemID: systemID, + } + + version, err := s.GetPlatformVersion(systemID) + if err != nil { + return nil, err + } + + platformInfo.ArrayVersion = version + + genType, err := s.GetGenType(systemID) + if err != nil { + return nil, err + } + + platformInfo.GenType = genType + + s.platformInfos[systemID] = platformInfo + + log.Infof("End: Retrieved Platform Info from Array using SystemId: %s Version: %v GenType: %s", systemID, version, genType) + } + + return platformInfo, nil +} + +func (s *service) GetGenType(systemID string) (string, error) { + system := s.systems[systemID] + if system == nil { + return "", nil + } + + // Query all ProtectionDomains for this system and return genType of first one + pds, err := system.GetProtectionDomain("") + if err != nil { + return "", err + } + + if len(pds) > 0 { + return pds[0].GenType, nil + } + + return "", nil +} + +func (s *service) GetPlatformVersion(systemID string) (float64, error) { + c := s.adminClients[systemID] + if c == nil { + return 0, nil + } + + version, err := c.GetVersion() + if err != nil { + return 0, err + } + + ver, err := strconv.ParseFloat(version, 64) + if err != nil { + return 0, err + } + + return ver, nil +} + +func (s *service) getArrayVersion(ctx context.Context, systemID string) (float64, error) { + if err := s.systemProbeAll(ctx); err != nil { + return 0, err + } + + c := s.adminClients[systemID] + if c == nil { + return 0, fmt.Errorf("unable to get admin client of the array: %s", systemID) + } + + platformInfo, err := s.GetPlatformInfo(systemID) + if err != nil { + return 0, err + } + + return platformInfo.ArrayVersion, nil +} + +func (s *service) getHostIDAndType(systemID, nodeID string) (string, string, error) { + hostID := "" + hostType := "" + + sdcID, err := s.getSDCID(nodeID, systemID) + if err != nil { + log.Infof("No SDC host found for nodeID %s: %v", nodeID, err) + } + if sdcID != "" { + log.Infof("SDC Host with ID %s found for nodeID %s", sdcID, nodeID) + hostID = sdcID + hostType = SDC + } else { + nvmeHost, err := s.systems[systemID].FindSdc("Name", nodeID) + if err != nil { + log.Infof("No NVME host found for nodeID %s: %v", nodeID, err) + } + if nvmeHost != nil { + log.Infof("NVME Host with ID %s found for nodeID %s", nvmeHost.Sdc.ID, nodeID) + hostID = nvmeHost.Sdc.ID + hostType = NVMeTCP + } else { + return hostID, hostType, err + } + } + return hostID, hostType, nil +} diff --git a/service/service_test.go b/service/service_test.go index 54a3acce..cc8fa07c 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -18,28 +18,57 @@ import ( "fmt" "net/http" "os" + "path/filepath" "sync" "testing" "time" - "github.com/cucumber/godog" sio "github.com/dell/goscaleio" siotypes "github.com/dell/goscaleio/types/v1" + "github.com/cucumber/godog" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" ) +var ( + testStatus int + testStartTime time.Time +) + func TestMain(m *testing.M) { + testStatus = 0 + testStartTime = time.Now() + + if st := m.Run(); st > testStatus { + testStatus = st + } + + fmt.Printf("status %d\n", testStatus) + + os.Exit(testStatus) +} + +func TestFeatures(t *testing.T) { defaultGetTargetPathPrefix := getTargetPathPrefix + defaultEphemeralStagingMountPath := ephemeralStagingMountPath + unitTestEmulateBlockDevice = true defer func() { getTargetPathPrefix = defaultGetTargetPathPrefix + ephemeralStagingMountPath = defaultEphemeralStagingMountPath + unitTestEmulateBlockDevice = false }() + tempTargetPathPrefix := t.TempDir() getTargetPathPrefix = func() string { - return "test/" + return tempTargetPathPrefix } + nodePublishBlockDevicePath = filepath.Join(tempTargetPathPrefix, nodePublishBlockDevicePath) + nodePublishAltBlockDevPath = filepath.Join(tempTargetPathPrefix, nodePublishAltBlockDevPath) + nodePublishEphemDevPath = filepath.Join(tempTargetPathPrefix, nodePublishEphemDevPath) + ephemeralStagingMountPath = tempTargetPathPrefix + server := &http.Server{ Addr: "localhost:6060", ReadHeaderTimeout: 60 * time.Second, @@ -60,17 +89,9 @@ func TestMain(m *testing.M) { Options: &opts, }.Run() - fmt.Printf("godog finished\n") - - if st := m.Run(); st > status { - fmt.Printf("godog.TestSuite status %d\n", status) - fmt.Printf("m.Run status %d\n", st) - status = st + if status > 0 { + t.Error("godog tests failed") } - - fmt.Printf("status %d\n", status) - - os.Exit(status) } func Test_service_SetPodZoneLabel(t *testing.T) { diff --git a/service/service_unit_test.go b/service/service_unit_test.go index a5b21f9f..81ff3a01 100644 --- a/service/service_unit_test.go +++ b/service/service_unit_test.go @@ -15,22 +15,29 @@ package service import ( "context" + "encoding/json" "errors" "fmt" "net" + "net/http" + "net/http/httptest" + "os" "testing" + "time" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" - csi "github.com/container-storage-interface/spec/lib/go/csi" siotypes "github.com/dell/goscaleio/types/v1" + csi "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" ) -type mockService struct{} +type mockService struct { + service +} func (s *mockService) InterfaceByName(interfaceName string) (*net.Interface, error) { if interfaceName == "" { @@ -602,15 +609,19 @@ func TestGetZoneKeyLabelFromSecret(t *testing.T) { func TestFindNetworkInterfaceIPs(t *testing.T) { tests := []struct { - name string - expectedError string - client kubernetes.Interface - configMapData map[string]string - createConfigMap func(map[string]string, kubernetes.Interface) + name string + expectedError string + client kubernetes.Interface + createK8sClientSet func(kubeConfig ...string) error + configMapData map[string]string + createConfigMap func(map[string]string, kubernetes.Interface) }{ { name: "Error getting K8sClient", expectedError: "unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined", + createK8sClientSet: func(_ ...string) error { + return fmt.Errorf("unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined") + }, client: nil, configMapData: nil, createConfigMap: func(map[string]string, kubernetes.Interface) { @@ -643,7 +654,7 @@ func TestFindNetworkInterfaceIPs(t *testing.T) { // Create a ConfigMap using fake ClientSet _, err := clientSet.CoreV1().ConfigMaps(DriverNamespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) if err != nil { - Log.Fatalf("failed to create configMaps: %v", err) + log.Fatalf("failed to create configMaps: %v", err) } }, }, @@ -665,7 +676,7 @@ func TestFindNetworkInterfaceIPs(t *testing.T) { // Create a ConfigMap using fake ClientSet _, err := clientSet.CoreV1().ConfigMaps(DriverNamespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) if err != nil { - Log.Fatalf("failed to create configMaps: %v", err) + log.Fatalf("failed to create configMaps: %v", err) } }, }, @@ -687,7 +698,7 @@ func TestFindNetworkInterfaceIPs(t *testing.T) { // Create a ConfigMap using fake ClientSet _, err := clientSet.CoreV1().ConfigMaps(DriverNamespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) if err != nil { - Log.Fatalf("failed to create configMaps: %v", err) + log.Fatalf("failed to create configMaps: %v", err) } }, }, @@ -696,6 +707,16 @@ func TestFindNetworkInterfaceIPs(t *testing.T) { for _, tt := range tests { s := &service{} t.Run(tt.name, func(t *testing.T) { + defaultCreateKubeClientSet := CreateKubeClientSet + if tt.createK8sClientSet != nil { + CreateKubeClientSet = tt.createK8sClientSet + } + defer func() { + if tt.createK8sClientSet != nil { + CreateKubeClientSet = defaultCreateKubeClientSet + } + }() + K8sClientset = tt.client tt.createConfigMap(tt.configMapData, tt.client) _, err := s.findNetworkInterfaceIPs() @@ -707,3 +728,211 @@ func TestFindNetworkInterfaceIPs(t *testing.T) { }) } } + +func TestConfigureAutoBlockProtocol(t *testing.T) { + tests := []struct { + name string + version float64 + nvmeInitiators int + sdcGUID string + nodeProbeErr error + expectedUseSDC bool + expectedUseNVME bool + }{ + { + name: "Both SDC and NVMeTCP available", + version: 4.5, + nvmeInitiators: 1, + sdcGUID: "some-guid", + expectedUseSDC: true, + }, + { + name: "Only NVMeTCP available", + version: 4.5, + nvmeInitiators: 1, + sdcGUID: "", + expectedUseNVME: true, + }, + { + name: "Only SDC available", + version: 3.9, + nvmeInitiators: 0, + sdcGUID: "some-guid", + expectedUseSDC: true, + }, + { + name: "Neither SDC nor NVMeTCP available", + version: 3.9, + nvmeInitiators: 0, + sdcGUID: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc := &mockService{ + service: service{ + opts: Opts{ + SdcGUID: tt.sdcGUID, + }, + }, + } + + svc.configureAutoBlockProtocol(context.Background(), tt.version, tt.nvmeInitiators) + + if svc.useSDC != tt.expectedUseSDC { + t.Errorf("expected useSDC=%v, got %v", tt.expectedUseSDC, svc.useSDC) + } + if svc.useNVME != tt.expectedUseNVME { + t.Errorf("expected useNVME=%v, got %v", tt.expectedUseNVME, svc.useNVME) + } + }) + } +} + +// helper to build a server returning the given status and code +func newStatusServer(status ArrayConnectivityStatus, httpCode int) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(httpCode) + if httpCode == http.StatusOK { + _ = json.NewEncoder(w).Encode(status) + } else { + _, _ = w.Write([]byte(`{"error":"error message"}`)) + } + })) +} + +func TestQueryArrayStatus_AllScenarios(t *testing.T) { + ctx := context.Background() + os.Setenv(EnvPodmonArrayConnectivityPollRate, "60") + defer os.Unsetenv(EnvPodmonArrayConnectivityPollRate) + + tol := SetPollingFrequency(ctx) + type tc struct { + name string + makeURL func(t *testing.T) string + wantConn bool + wantErr bool + } + + now := time.Now().Unix() + + cases := []tc{ + { + name: "Connected_timeDiff<=tolerance+2", + makeURL: func(t *testing.T) string { + // timeDiff = LastAttempt - LastSuccess = 0 => connected + resp := ArrayConnectivityStatus{ + LastAttempt: now, + LastSuccess: now, + } + srv := newStatusServer(resp, http.StatusOK) + t.Cleanup(srv.Close) + return srv.URL + }, + wantConn: true, + wantErr: false, + }, + { + name: "NotConnected_timeDiff>tolerance+2", + makeURL: func(t *testing.T) string { + // Make timeDiff strictly greater than tolerance+2 + // timeDiff = (now-1) - ((now-1) - (tol+3)) = tol+3 + resp := ArrayConnectivityStatus{ + LastAttempt: now - 1, + LastSuccess: (now - 1) - (tol + 3), + } + srv := newStatusServer(resp, http.StatusOK) + t.Cleanup(srv.Close) + return srv.URL + }, + wantConn: false, + wantErr: false, + }, + { + name: "Stale_currTime-LastAttempt>tolerance*2", + makeURL: func(t *testing.T) string { + // Stale branch: (currTime - LastAttempt) > 2*tol + resp := ArrayConnectivityStatus{ + LastAttempt: now - (2*tol + 1), + LastSuccess: now - 100, // arbitrary older success + } + srv := newStatusServer(resp, http.StatusOK) + t.Cleanup(srv.Close) + return srv.URL + }, + wantConn: false, + wantErr: false, + }, + { + name: "HTTPNon200_returns_error", + makeURL: func(t *testing.T) string { + resp := ArrayConnectivityStatus{ + LastAttempt: now, + LastSuccess: now, + } + srv := newStatusServer(resp, http.StatusInternalServerError) + t.Cleanup(srv.Close) + return srv.URL + }, + wantConn: false, + wantErr: true, + }, + { + name: "BadJSON_returns_error", + makeURL: func(t *testing.T) string { + // 200 OK but invalid JSON to exercise unmarshal error path + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{invalid json`)) + })) + t.Cleanup(srv.Close) + return srv.URL + }, + wantConn: false, + wantErr: true, + }, + { + name: "HTTPClientError_connection_refused", + makeURL: func(_ *testing.T) string { + // Unreachable port typically triggers client.Get error + return "http://127.0.0.1:1" + }, + wantConn: false, + wantErr: true, + }, + { + name: "PanicRecovery_server_panics", + makeURL: func(t *testing.T) string { + // The server panics; your function has a defer recover() that logs. + // Behavior should result in an error (no crash). + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + panic("test panic") + })) + t.Cleanup(srv.Close) + return srv.URL + }, + wantConn: false, + wantErr: true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + s := &service{} // same package => can instantiate unexported type + + url := c.makeURL(t) + connected, err := s.QueryArrayStatus(ctx, url) + + if c.wantErr && err == nil { + t.Fatalf("expected error; got nil (connected=%v)", connected) + } + if !c.wantErr && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if connected != c.wantConn { + t.Fatalf("connected: got %v, want %v (err=%v)", connected, c.wantConn, err) + } + }) + } +} diff --git a/service/stager.go b/service/stager.go new file mode 100644 index 00000000..0a82ec61 --- /dev/null +++ b/service/stager.go @@ -0,0 +1,335 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package service + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "time" + + "github.com/dell/csmlog" + "github.com/dell/gobrick" + "github.com/dell/gofsutil" + "github.com/dell/goscaleio" + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + procMountsPath = "/proc/self/mountinfo" + procMountsRetries = 15 + defaultDirPerm = 0o700 +) + +// StageStatus represents the staging readiness of the volume at stagingPath. +type StageStatus int + +const ( + StageNotFound StageStatus = iota // no mount at stagingPath + StageReady // mounted and publish-ready + StageDeletedLink // mount source points to a deleted path + StageMpathMember // mounted device is a multipath member path + StageProbeError // failed to probe mounts or device format +) + +func (s StageStatus) String() string { + switch s { + case StageNotFound: + return "not_found" + case StageReady: + return "ready" + case StageDeletedLink: + return "deleted_link" + case StageMpathMember: + return "mpath_member" + case StageProbeError: + return "probe_error" + default: + return "unknown" + } +} + +type deviceInfo struct { + nguid string + nvmeTargets []gobrick.NVMeTargetInfo +} + +// VolumeStager allows node staging of a volume. +type VolumeStager interface { + Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPath string, logFields csmlog.Fields, volID string) (*csi.NodeStageVolumeResponse, error) + Unstage(ctx context.Context, stagingPath string, logFields csmlog.Fields, volID string) (*csi.NodeUnstageVolumeResponse, error) +} + +// NVMeStager implementation for staging volumes to NVMe hosts. +type NVMeStager struct { + useNVME bool + nvmeConnector NVMEConnector + systemID string + adminClient *goscaleio.Client + targetNqn map[string]string +} + +// Stage stages volume by connecting it through NVMe/TCP and creating bind mount to staging path. +func (n *NVMeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPath string, logFields csmlog.Fields, volID string) (*csi.NodeStageVolumeResponse, error) { + log := log.WithContext(ctx) + logFields["VolumeID"] = volID + logFields["StagingPath"] = stagingPath + + // Validate volume capability + isBlock, mount, accMode, _, err := validateVolumeCapability(req.GetVolumeCapability(), false) + if err != nil { + return nil, err + } + + // Validate volume existence + _, err = n.adminClient.GetVolume("", strings.TrimSpace(volID), "", "", false) + if err != nil { + return nil, status.Errorf(codes.NotFound, "volume %s not found: %s", volID, err.Error()) + } + + // Build NGUID + nguid, err := buildNGUID(volID, n.systemID) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to build NGUID: %s", err.Error()) + } + + // Get system and NVMe targets + system, err := n.adminClient.FindSystem(n.systemID, "", "") + if err != nil { + return nil, status.Errorf(codes.NotFound, "system %s not found: %s", n.systemID, err.Error()) + } + + targetPortals, err := getNVMETCPTargetsInfoFromStorage(system) + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to get NVMe/TCP targets: %s", err.Error()) + } + + nvmeTargets := buildNVMeTargetInfo(n.targetNqn, targetPortals) + + deviceInfo := deviceInfo{ + nguid: nguid, + nvmeTargets: nvmeTargets, + } + + logFields["Targets"] = nvmeTargets + logFields["WWN"] = nguid + ctx = csmlog.SetLogFields(ctx, logFields) + + // Ensure staging directory exists + if err := os.MkdirAll(stagingPath, defaultDirPerm); err != nil { + return nil, status.Errorf(codes.Internal, "failed to create staging path %s: %s", stagingPath, err.Error()) + } + log.WithFields(logFields).Info("staging path created") + + // Check staging status + stageStatus, err := isAlreadyStaged(ctx, stagingPath) + log.WithFields(logFields).Debugf("staging status detected: %s", stageStatus.String()) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to probe staging state: %v", err) + } + + switch stageStatus { + case StageReady: + log.WithFields(logFields).Info("device already staged") + return &csi.NodeStageVolumeResponse{}, nil + case StageDeletedLink, StageMpathMember: + log.WithFields(logFields).Warnf("unsafe staging state (%s); performing cleanup", stageStatus) + if _, err := n.Unstage(ctx, stagingPath, logFields, volID); err != nil { + return nil, status.Errorf(codes.Internal, "failed to cleanup staging path: %v", err) + } + case StageNotFound: + log.WithFields(logFields).Info("device not staged; proceeding with staging") + default: + return nil, status.Errorf(codes.Internal, "unknown stage status: %v", stageStatus) + } + + // Connect device + devicePath, err := n.connectDevice(ctx, deviceInfo) + if err != nil { + return nil, err + } + logFields["DevicePath"] = devicePath + + // Validate block device + sysDevice, err := GetDevice(devicePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "error getting block device for volume %s: %s", volID, err.Error()) + } + + // Mount if filesystem volume + if !isBlock { + fs := mount.GetFsType() + mntFlags := mount.GetMountFlags() + fsFormatOption := req.GetVolumeContext()[KeyMkfsFormatOption] + if fs == "xfs" { + mntFlags = append(mntFlags, "nouuid") + } + if err := handlePrivFSMount(ctx, accMode, sysDevice, mntFlags, fs, stagingPath, fsFormatOption); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount disk %s to staging path: %s", devicePath, err.Error()) + } + } + + log.WithFields(logFields).Info("stage complete") + return &csi.NodeStageVolumeResponse{}, nil +} + +func (n *NVMeStager) Unstage(ctx context.Context, stagingPath string, logFields csmlog.Fields, volID string) (*csi.NodeUnstageVolumeResponse, error) { + log := log.WithContext(ctx) + + mounts, err := getPathMounts(ctx, stagingPath) + if err != nil { + log.Errorf("NodeUnstageVolume: failed to get mounts for staging path: %v", err) + return &csi.NodeUnstageVolumeResponse{}, nil + } + + var devicePath string + for _, m := range mounts { + if m.Path == stagingPath { + devicePath = m.Device + break + } + } + + // Unmount the staging target path. + log.WithFields(logFields).Info("unmounting directory") + if err := gofsutil.Unmount(ctx, stagingPath); err != nil && !os.IsNotExist(err) { + log.Errorf("Unable to Unmount staging target path: %s", err) + } + + log.WithFields(logFields).Info("removing directory") + if err := os.Remove(stagingPath); err != nil && !os.IsNotExist(err) { + log.Errorf("Unable to remove staging target path: %v", err) + } + + // If we found a backing device and it looks like NVME, disconnect it. + if devicePath != "" { + log.Infof("NodeUnsatgeVolume: disconnecting NVME device %s for volumeID= %s", devicePath, volID) + if err := n.disconnectNVMEDevice(ctx, devicePath); err != nil { + log.Errorf("Node Unstage volume: failed to disconnect NVMEdevice %s: %v", devicePath, err) + return nil, status.Errorf(codes.Internal, "Failed to disconnect NVME device %s: %v", devicePath, err) + } + } else { + log.Infof("NodeUnsatgeVolume: no backing device found for stagingPath=%s; skipping NVME disconnect", stagingPath) + } + + log.WithFields(logFields).Info("unstage complete") + return &csi.NodeUnstageVolumeResponse{}, nil +} + +func (n *NVMeStager) connectDevice(ctx context.Context, data deviceInfo) (string, error) { + log := log.WithContext(ctx) + if !n.useNVME { + log.Warn("invalid operation: node is not NVMe-enabled") + return "", status.Errorf(codes.FailedPrecondition, "node does not support NVMe") + } + + device, err := n.connectNVMEDevice(ctx, data) + if err != nil { + log.Errorf("unable to find device after multiple discovery attempts: %s", err.Error()) + return "", status.Errorf(codes.Internal, "unable to find device: %s", err.Error()) + } + + return filepath.Join("/dev", device.Name), nil +} + +func (n *NVMeStager) connectNVMEDevice(ctx context.Context, data deviceInfo) (gobrick.Device, error) { + logFields := csmlog.ExtractFieldsFromContext(ctx) + var targets []gobrick.NVMeTargetInfo + for _, t := range data.nvmeTargets { + targets = append(targets, gobrick.NVMeTargetInfo{Target: t.Target, Portal: t.Portal}) + } + + // separate context to prevent 15 seconds cancel from kubernetes + connectorCtx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + connectorCtx = csmlog.SetLogFields(connectorCtx, logFields) + + return n.nvmeConnector.ConnectVolume(connectorCtx, gobrick.NVMeVolumeInfo{ + Targets: targets, + WWN: data.nguid, + }, false) +} + +func (n *NVMeStager) disconnectNVMEDevice(ctx context.Context, devicePath string) error { + devName := strings.TrimPrefix(devicePath, "/dev/") + if devName == "" { + return fmt.Errorf("empty device name for NVME disconnect") + } + + if !strings.HasPrefix(devName, "nvme") { + log.Infof("disconnectNVMEdevice: device %s does not look like nvme skipping", devName) + return nil + } + + log.Infof("disconnectNVMEDevices: disconnecting NVME device %s via connector", devName) + + if err := n.nvmeConnector.DisconnectVolumeByDeviceName(ctx, devName); err != nil { + return fmt.Errorf("disconnectNVMEDevice: connector failed for %s: %w", devName, err) + } + + log.Infof("disconnectNVMEDevice: successfully disconnect NVME device %s", devName) + return nil +} + +// isAlreadyStaged checks if stagingPath is mounted in a publish-ready way. +func isAlreadyStaged(ctx context.Context, stagingPath string) (StageStatus, error) { + mnts, err := getPathMounts(ctx, stagingPath) + if err != nil { + return StageProbeError, fmt.Errorf("get mounts: %w", err) + } + if len(mnts) == 0 { + log.Debug("isAlreadyStaged: no mounts found at stagingPath") + return StageNotFound, nil + } + + for _, m := range mnts { + if strings.HasSuffix(m.Source, "deleted") { + log.Warnf("isAlreadyStaged: mount source is deleted: src=%s", m.Source) + return StageDeletedLink, nil + } + } + + devFS, err := gofsutil.GetDiskFormat(ctx, stagingPath) + if err != nil { + return StageProbeError, fmt.Errorf("disk format probe: %w", err) + } + if devFS == "mpath_member" { + log.Warn("isAlreadyStaged: device is a multipath member (not the DM map)") + return StageMpathMember, nil + } + + return StageReady, nil +} + +func buildNVMeTargetInfo(targetNqn map[string]string, nvmetcpTargetPortals []string) []gobrick.NVMeTargetInfo { + nvmetcpTargets := []gobrick.NVMeTargetInfo{} + for _, portal := range nvmetcpTargetPortals { + ip, _, _ := net.SplitHostPort(portal) + nvmetcpTargets = append(nvmetcpTargets, gobrick.NVMeTargetInfo{ + Target: targetNqn[ip], + Portal: portal, + }) + } + return nvmetcpTargets +} diff --git a/service/step_defs_test.go b/service/step_defs_test.go index b94fbb0e..c435016d 100644 --- a/service/step_defs_test.go +++ b/service/step_defs_test.go @@ -17,7 +17,6 @@ import ( "context" "errors" "fmt" - "log" "net" "net/http/httptest" "os" @@ -26,6 +25,7 @@ import ( "runtime" "strconv" "strings" + "testing" "time" "github.com/container-storage-interface/spec/lib/go/csi" @@ -35,61 +35,84 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/wrapperspb" - "github.com/cucumber/godog" "github.com/dell/dell-csi-extensions/podmon" "github.com/dell/dell-csi-extensions/replication" volGroupSnap "github.com/dell/dell-csi-extensions/volumeGroupSnapshot" + "github.com/dell/gobrick" "github.com/dell/gocsi" "github.com/dell/gofsutil" + "github.com/dell/gonvme" "github.com/dell/goscaleio" types "github.com/dell/goscaleio/types/v1" + "github.com/cucumber/godog" "google.golang.org/grpc/metadata" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" ) const ( - testBaseDir = "test" - arrayID = "14dbbf5617523654" - arrayID2 = "15dbbf5617523655" - badVolumeID = "Totally Fake ID" - badCsiVolumeID = "ffff-f250" - goodVolumeID = "111" - badVolumeID2 = "9999" - badVolumeID3 = "99" - goodVolumeName = "vol1" - altVolumeID = "222" - goodNodeID = "9E56672F-2F4B-4A42-BFF4-88B6846FBFDA" - goodArrayConfig = "./features/array-config/config" - goodDriverConfig = "./features/driver-config/logConfig.yaml" - altNodeID = "7E012974-3651-4DCB-9954-25975A3C3CDF" - datafile = "test/00000000-1111-0000-0000-000000000000/datafile" - datadir = "test/00000000-1111-0000-0000-000000000000/datadir" - badtarget = "nonexistent/target/path" - altdatadir = "test/00000000-1111-0000-0000-000000000000/altdatadir" - altdatafile = "test/00000000-1111-0000-0000-000000000000/altdatafile" - sdcVolume1 = "d0f055a700000000" - sdcVolume2 = "c0f055aa00000000" - sdcVolume0 = "0000000000000000" - ephemVolumeSDC = "6373692d64306630353561373030303030303030" - mdmID = "14dbbf5617523654" - mdmIDEphem = "14dbbf5617523654" - mdmID1 = "24dbbf5617523654" - mdmID2 = "34dbbf5617523654" - badMdmID = "9999" - nodePublishBlockDevicePath = "test/dev/scinia" - nodePublishAltBlockDevPath = "test/dev/scinib" - nodePublishEphemDevPath = "test/dev/scinic" - nodePublishSymlinkDir = "test/dev/disk/by-id" - goodSnapID = "444" - altSnapID = "555" + testBaseDir = "test" + arrayID = "14dbbf5617523654" + arrayID2 = "15dbbf5617523655" + badVolumeID = "999999" + badCsiVolumeID = "ffff-f250" + goodVolumeID = "111" + badVolumeID2 = "9999" + badVolumeID3 = "99" + goodVolumeName = "vol1" + altVolumeID = "222" + goodNodeID = "9E56672F-2F4B-4A42-BFF4-88B6846FBFDA" + goodNodeIDNVMe = "192.168.0.1-513fda498aaf35e883d" + goodNodeIDNVMe1 = "192.168.0.2-25eb4476103ff9daccc" + goodSdcIDNVMe = "15d2cc7400010000" + goodArrayConfig = "./features/array-config/config" + goodDriverConfig = "./features/driver-config/logConfig.yaml" + altNodeID = "7E012974-3651-4DCB-9954-25975A3C3CDF" + altNodeIDNVMe = "192.168.0.2-25eb4476103ff9daccc" + altSdcIDNVMe = "15d2cc7500010001" + datafile = "test/00000000-1111-0000-0000-000000000000/datafile" + datadir = "test/00000000-1111-0000-0000-000000000000/datadir" + stageDir = "test/0efa4456de923f95192c0544979ba4c11d421e9368163084969db878af2394fc/globalmount" + nodePublishWWN = "77d3b2750000002064b94e6077518e0f" + badtarget = "nonexistent/target/path" + altdatadir = "test/00000000-1111-0000-0000-000000000000/altdatadir" + altdatafile = "test/00000000-1111-0000-0000-000000000000/altdatafile" + sdcVolume1 = "d0f055a700000000" + sdcVolume2 = "c0f055aa00000000" + sdcVolume0 = "0000000000000000" + ephemVolumeSDC = "6373692d64306630353561373030303030303030" + mdmID = "14dbbf5617523654" + mdmIDEphem = "14dbbf5617523654" + mdmID1 = "24dbbf5617523654" + mdmID2 = "34dbbf5617523654" + badMdmID = "9999" + nodePublishSymlinkDir = "test/dev/disk/by-id" + goodSnapID = "444" + altSnapID = "555" + nvmeNguid = "d0f055a70000000064b94e5617523654" + nodeInternalIP = "10.20.30.40" + imageVersion = "1.0.0" +) + +var ( + nodePublishBlockDevicePath = "scinia" + nodePublishAltBlockDevPath = "scinib" + nodePublishEphemDevPath = "scinic" + mockGobrickInducedErrors struct { + ConnectVolumeError bool + DisconnectVolumeError bool + } ) var setupGetSystemIDtoFail bool type feature struct { + t *testing.T + protocol string nGoRoutines int server *httptest.Server server2 *httptest.Server @@ -99,6 +122,8 @@ type feature struct { adminClient2 *goscaleio.Client countOfArrays int system2 *goscaleio.System + nvmeLibMock *gonvme.MockNVMe + nvmeConnectorMock *mockNVMeTCPConnector err error // return from the preceding call getPluginInfoResponse *csi.GetPluginInfoResponse getPluginCapabilitiesResponse *csi.GetPluginCapabilitiesResponse @@ -134,6 +159,7 @@ type feature struct { useAccessTypeMount bool capability *csi.VolumeCapability capabilities []*csi.VolumeCapability + nodeStageVolumeRequest *csi.NodeStageVolumeRequest nodePublishVolumeRequest *csi.NodePublishVolumeRequest createSnapshotRequest *csi.CreateSnapshotRequest volumeIDList []string @@ -154,12 +180,76 @@ type feature struct { nas types.NAS } +type mockNVMeTCPConnector struct{} + +func (m *mockNVMeTCPConnector) ConnectVolume(_ context.Context, _ gobrick.NVMeVolumeInfo, _ bool) (gobrick.Device, error) { + if mockGobrickInducedErrors.ConnectVolumeError { + return gobrick.Device{}, fmt.Errorf("induced ConnectVolumeError") + } + return gobrick.Device{}, nil +} + +func (m *mockNVMeTCPConnector) DisconnectVolumeByDeviceName(_ context.Context, _ string) error { + if mockGobrickInducedErrors.DisconnectVolumeError { + return fmt.Errorf("induced DisconnectVolumeError") + } + fmt.Printf("Removing WWN %s to path entry\n", nodePublishWWN) + delete(gofsutil.GOFSMockWWNToDevice, nodePublishWWN) + return nil +} + +func (m *mockNVMeTCPConnector) GetInitiatorName(_ context.Context) ([]string, error) { + result := make([]string, 0) + return result, nil +} + func (f *feature) checkGoRoutines(tag string) { goroutines := runtime.NumGoroutine() fmt.Printf("goroutines %s new %d old groutines %d\n", tag, goroutines, f.nGoRoutines) f.nGoRoutines = goroutines } +func (f *feature) setPlatformInfo(sourceVersion, sourceGenType, targetVersion, targetGenType string) error { + sourceVersionFloat, err := strconv.ParseFloat(sourceVersion, 64) + if err != nil { + return err + } + + f.service.platformInfos[arrayID] = &PlatformInfo{ + SystemID: arrayID, + ArrayVersion: sourceVersionFloat, + GenType: sourceGenType, + } + + targetVersionFloat, err := strconv.ParseFloat(targetVersion, 64) + if err != nil { + return err + } + + f.service.platformInfos[arrayID2] = &PlatformInfo{ + SystemID: arrayID2, + ArrayVersion: targetVersionFloat, + GenType: targetGenType, + } + + return nil +} + +func (f *feature) resetPlatformInfo() error { + // Remove platform info + delete(f.service.platformInfos, arrayID) + delete(f.service.platformInfos, arrayID2) + return nil +} + +func (f *feature) iSetProvisionTo(isThickProvision string) error { + f.service.opts.Thick = false + if isThickProvision == "Thick" { + f.service.opts.Thick = true + } + return nil +} + func (f *feature) aVxFlexOSService() error { return f.aVxFlexOSServiceWithTimeoutMilliseconds(150) } @@ -225,12 +315,19 @@ func (f *feature) aVxFlexOSServiceWithTimeoutMilliseconds(millis int) error { f.listedVolumeIDs = make(map[string]bool) f.capability = nil f.capabilities = make([]*csi.VolumeCapability, 0) + f.nodeStageVolumeRequest = nil f.nodePublishVolumeRequest = nil f.createSnapshotRequest = nil f.createSnapshotResponse = nil f.volumeIDList = f.volumeIDList[:0] f.snapshotIndex = 0 f.maxVolSize = 0 + f.nvmeLibMock = gonvme.NewMockNVMe(nil) + f.nvmeConnectorMock = new(mockNVMeTCPConnector) + f.protocol = SDC + gonvme.GONVMEMock.InduceDiscoveryError = false + mockGobrickInducedErrors.ConnectVolumeError = false + mockGobrickInducedErrors.DisconnectVolumeError = false // configure gofsutil; we use a mock interface gofsutil.UseMockFS() @@ -245,18 +342,22 @@ func (f *feature) aVxFlexOSServiceWithTimeoutMilliseconds(millis int) error { gofsutil.GOFSMock.InduceFSTypeError = false gofsutil.GOFSMock.InduceResizeFSError = false gofsutil.GOFSMockMounts = gofsutil.GOFSMockMounts[:0] + gofsutil.GOFSWWNPath = nodePublishSymlinkDir + "/nvme-eui." + clear(gofsutil.GOFSMockWWNToDevice) // configure variables in the driver publishGetMappedVolMaxRetry = 2 getMappedVolDelay = 10 * time.Millisecond // Get or reuse the cached service - f.getService() + f.getService(goodNodeID) goscaleio.SCINIMockMode = true // Get the httptest mock handler. Only set // a new server if there isn't one already. + // set with default api version. + apiVersion = "4.0" handler := getHandler() if handler != nil { if f.server == nil { @@ -289,16 +390,21 @@ func (f *feature) aVxFlexOSServiceWithTimeoutMilliseconds(millis int) error { systemArrays[addr].Link(systemArrays[addr2]) + // Configure ManifestSemver + ManifestSemver = imageVersion + f.checkGoRoutines("end aVxFlexOSService") return nil } -func (f *feature) getService() *service { +func (f *feature) getService(csiNodeID string) *service { testControllerHasNoConnection = false svc := new(service) svc.adminClients = make(map[string]*goscaleio.Client) svc.systems = make(map[string]*goscaleio.System) + svc.platformInfos = make(map[string]*PlatformInfo) + svc.nvmeTargetNqn = make(map[string]string) if f.adminClient != nil { svc.adminClients[arrayID] = f.adminClient @@ -309,11 +415,18 @@ func (f *feature) getService() *service { if f.system != nil { svc.systems[arrayID] = f.system + } else { + log.Infof("System is Empty: %s", arrayID) } + if f.system2 != nil { svc.systems[arrayID2] = f.system2 + } else { + log.Infof("System 2 is Empty: %s", arrayID) } + svc.nvmeLib = f.nvmeLibMock + svc.nvmeConnector = f.nvmeConnectorMock svc.storagePoolIDToName = map[string]string{} svc.volumePrefixToSystems = map[string][]string{} svc.connectedSystemNameToID = map[string]string{} @@ -333,7 +446,7 @@ func (f *feature) getService() *service { var err error opts.arrays, err = getArrayConfig(ctx) if err != nil { - log.Printf("Read arrays from config file failed: %s\n", err) + log.Infof("Read arrays from config file failed: %s\n", err) } opts.AutoProbe = true @@ -367,12 +480,17 @@ MDM-ID 14dbbf5617523654 SDC ID d0f33bd700000004 INSTALLATION ID 1c078b073d75512c if f.system != nil { svc.systems[arrayID] = f.system } + + if f.system2 != nil { + svc.systems[arrayID2] = f.system2 + } + f.service = svc f.countOfArrays = len(svc.systems) svc.statisticsCounter = 99 svc.logStatistics() if K8sClientset == nil { - f.CreateCSINode() + f.CreateCSINode(csiNodeID, "node1") svc.ProcessMapSecretChange() } server := grpc.NewServer() @@ -381,7 +499,7 @@ MDM-ID 14dbbf5617523654 SDC ID d0f33bd700000004 INSTALLATION ID 1c078b073d75512c } // CreateCSINode uses fakeclient to make csinode with topology key -func (f *feature) CreateCSINode() (*storage.CSINode, error) { +func (f *feature) CreateCSINode(csiNodeID string, nodeName string) (*storage.CSINode, error) { K8sClientset = fake.NewSimpleClientset() // csiKubeClient := nim.volumeHost.GetKubeClient() @@ -389,12 +507,12 @@ func (f *feature) CreateCSINode() (*storage.CSINode, error) { fakeCSINode := &storage.CSINode{ ObjectMeta: metav1.ObjectMeta{ - Name: "node1", + Name: nodeName, OwnerReferences: []metav1.OwnerReference{ { APIVersion: nodeKind.Version, Kind: nodeKind.Kind, - Name: "node1", + Name: nodeName, }, }, }, @@ -402,7 +520,7 @@ func (f *feature) CreateCSINode() (*storage.CSINode, error) { Drivers: []storage.CSINodeDriver{ { Name: "csi-vxflexos.dellemc.com", - NodeID: "9E56672F-2F4B-4A42-BFF4-88B6846FBFDA", + NodeID: csiNodeID, TopologyKeys: []string{"csi-vxflexos.dellemc.com/14dbbf5617523654"}, }, }, @@ -411,10 +529,32 @@ func (f *feature) CreateCSINode() (*storage.CSINode, error) { return K8sClientset.StorageV1().CSINodes().Create(context.TODO(), fakeCSINode, metav1.CreateOptions{}) } +// CreateKubernetesNode creates a Node object in the fake kubernetes client +func (f *feature) CreateKubernetesNode(k8s kubernetes.Interface, nodeName string) (*corev1.Node, error) { + internalIPs := []string{nodeInternalIP} + addrs := make([]corev1.NodeAddress, 0, len(internalIPs)) + for _, ip := range internalIPs { + addrs = append(addrs, corev1.NodeAddress{ + Type: corev1.NodeInternalIP, + Address: ip, + }) + } + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Status: corev1.NodeStatus{ + Addresses: addrs, + }, + } + return k8s.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}) +} + func (f *feature) aValidDynamicArrayChange() error { count := 0 for _, array := range f.service.opts.arrays { - log.Printf("array after config change array details %#v", array.SystemID) + log.Infof("array after config change array details %#v", array.SystemID) count++ } if count != 3 { @@ -430,36 +570,36 @@ func (f *feature) aValidDynamicArrayChange() error { func (f *feature) iCallDynamicArrayChange() error { f.countOfArrays = len(f.service.adminClients) - log.Printf("before config change array count=%d", f.countOfArrays) + log.Infof("before config change array count=%d", f.countOfArrays) for key := range f.service.adminClients { - log.Printf("before config change array ID %s", key) + log.Infof("before config change array ID %s", key) } backup := "features/config" os.Rename(ArrayConfigFile, backup) err := os.Rename("features/array-config/config.2", ArrayConfigFile) - log.Printf("wait for config change %s %#v", backup, err) + log.Infof("wait for config change %s %#v", backup, err) time.Sleep(10 * time.Second) return nil } func (f *feature) iCallDynamicLogChange(file string) error { - log.Printf("level before change: %s", Log.GetLevel()) + log.Infof("level before change: %s", log.GetLevel()) backup := "features/driver-config/logBackup.json" _ = os.Rename(DriverConfigParamsFile, backup) _ = os.Rename("features/driver-config/"+file, DriverConfigParamsFile) - log.Printf("wait for config change %s", backup) + log.Infof("wait for config change %s", backup) time.Sleep(10 * time.Second) return nil } func (f *feature) aValidDynamicLogChange(file, expectedLevel string) error { - log.Printf("level after change: %s", Log.GetLevel()) + log.Infof("level after change: %s", log.GetLevel()) backup := "features/driver-config/logBackup.json" - if Log.GetLevel().String() != expectedLevel { - err := fmt.Errorf("level was expected to be %s, but was %s instead", expectedLevel, Log.GetLevel().String()) + if log.GetLevel().String() != expectedLevel { + err := fmt.Errorf("level was expected to be %s, but was %s instead", expectedLevel, log.GetLevel().String()) return err } - log.Printf("Reverting log changes made") + log.Infof("Reverting log changes made") _ = os.Rename(DriverConfigParamsFile, "features/driver-config/"+file) _ = os.Rename(backup, DriverConfigParamsFile) return nil @@ -509,11 +649,11 @@ func (f *feature) theVolumeIsFromTheCorrectSystem(volID, sysID string) error { func (f *feature) aValidGetPlugInfoResponseIsReturned() error { rep := f.getPluginInfoResponse - url := rep.GetManifest()["url"] - if rep.GetName() == "" || rep.GetVendorVersion() == "" || url == "" { + if rep.GetName() == "" || rep.GetVendorVersion() == "" { return errors.New("Expected GetPluginInfo to return name and version") } - log.Printf("Name %s Version %s URL %s", rep.GetName(), rep.GetVendorVersion(), url) + + fmt.Printf("Name %s Version %s", rep.GetName(), rep.GetVendorVersion()) return nil } @@ -722,21 +862,58 @@ func (f *feature) iCallCreateVolume(name string) error { f.createVolumeResponse, f.err = f.service.CreateVolume(ctx, req) if f.err != nil { - log.Printf("CreateVolume called failed: %s\n", f.err.Error()) + log.Infof("CreateVolume called failed: %s\n", f.err.Error()) } if f.createVolumeResponse != nil { - log.Printf("vol id %s\n", f.createVolumeResponse.GetVolume().VolumeId) + log.Infof("vol id %s\n", f.createVolumeResponse.GetVolume().VolumeId) } return nil } -func (f *feature) iCallValidateVolumeHostConnectivity() error { +func (f *feature) iCallCreateVolumeWithError(name, errorMessage string) error { ctx := context.Background() + if f.createVolumeRequest == nil { + fmt.Println("createVolumeRequest is nil") + req := getTypicalCreateVolumeRequest() + f.createVolumeRequest = req + } else { + fmt.Println("createVolumeRequest is not nil") + } + + req := f.createVolumeRequest + req.Name = name + + req.Parameters = map[string]string{ + f.service.WithRP(KeyReplicationRemoteStoragePool): "viki_pool_HDD_20181031", + f.service.WithRP(KeyReplicationRemoteSystem): "15dbbf5617523655", + } - sdcID := f.service.opts.SdcGUID - sdcGUID := strings.ToUpper(sdcID) - csiNodeID := sdcGUID + fmt.Println("I am in iCallCreateVolumeWithError fn.....") + + f.createVolumeResponse, f.err = f.service.CreateVolume(ctx, req) + if f.err != nil { + if strings.Contains(f.err.Error(), errorMessage) { + log.Infof("Expected Error Message Found: %s\n", f.err.Error()) + } else { + return fmt.Errorf("expected error message not found: %s in the error: %s", errorMessage, f.err.Error()) + } + } + + return nil +} + +func (f *feature) iCallValidateVolumeHostConnectivity() error { + ctx := context.Background() + var csiNodeID string + if f.service.useNVME { + f.service.opts.SdcGUID = "" + csiNodeID = goodNodeIDNVMe1 + } else { + sdcID := f.service.opts.SdcGUID + sdcGUID := strings.ToUpper(sdcID) + csiNodeID = sdcGUID + } volIDs := make([]string, 0) @@ -764,6 +941,11 @@ func (f *feature) iCallValidateVolumeHostConnectivity() error { volIDs = append(volIDs, volid) } + clientSet := fake.NewSimpleClientset() + K8sClientset = clientSet + f.CreateCSINode(csiNodeID, "node2") + f.CreateKubernetesNode(K8sClientset, "node2") + req := &podmon.ValidateVolumeHostConnectivityRequest{ NodeId: csiNodeID, VolumeIds: volIDs, @@ -823,11 +1005,28 @@ func (f *feature) iSpecifyAccessibilityRequirementsWithASystemIDOf(requestedSyst capacityRange := new(csi.CapacityRange) capacityRange.RequiredBytes = 32 * 1024 * 1024 * 1024 req.CapacityRange = capacityRange - req.AccessibilityRequirements = new(csi.TopologyRequirement) + block := new(csi.VolumeCapability_BlockVolume) + capability := new(csi.VolumeCapability) + accessType := new(csi.VolumeCapability_Block) + accessType.Block = block + capability.AccessType = accessType + accessMode := new(csi.VolumeCapability_AccessMode) + accessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER + capability.AccessMode = accessMode + capabilities := make([]*csi.VolumeCapability, 0) + capabilities = append(capabilities, capability) + req.VolumeCapabilities = capabilities top := new(csi.Topology) - top.Segments = map[string]string{ - "csi-vxflexos.dellemc.com/" + requestedSystem: "powerflex.dellemc.com", + if f.protocol == NVMeTCP { + top.Segments = map[string]string{ + "csi-vxflexos.dellemc.com/" + requestedSystem + "-nvmetcp": "powerflex.dellemc.com", + } + } else { + top.Segments = map[string]string{ + "csi-vxflexos.dellemc.com/" + requestedSystem: "powerflex.dellemc.com", + } } + req.AccessibilityRequirements = new(csi.TopologyRequirement) req.AccessibilityRequirements.Preferred = append(req.AccessibilityRequirements.Preferred, top) f.createVolumeRequest = req return nil @@ -872,7 +1071,7 @@ func (f *feature) iSpecifyAccessibilityRequirementsNFSWithASystemIDOf(requestedS return nil } -func (f *feature) iSpecifyBadAccessibilityRequirementsNFSWithASystemIDOf(requestedSystem string) error { +func (f *feature) iSpecifyBadAccessibilityRequirementsWithASystemIDOf(requestedSystem string) error { if requestedSystem == "f.service.opt.SystemName" { requestedSystem = f.service.opts.defaultSystemID } @@ -893,9 +1092,7 @@ func (f *feature) iSpecifyBadAccessibilityRequirementsNFSWithASystemIDOf(request capability := new(csi.VolumeCapability) mountVolume := new(csi.VolumeCapability_MountVolume) mountVolume.FsType = "nfs" - if mountVolume.FsType == "nfs" { - req.Parameters["nasName"] = "dummy-nas-server" - } + req.Parameters["nasName"] = "dummy-nas-server" mountVolume.MountFlags = make([]string, 0) mount := new(csi.VolumeCapability_Mount) mount.Mount = mountVolume @@ -948,19 +1145,18 @@ func (f *feature) aValidCreateVolumeResponseWithTopologyIsReturned() error { constraint = tokens[1] } - log.Printf("Found topology constraint: VxFlex OS system: %s", constraint) - if isNFS { - nfsTokens := strings.Split(constraint, "-") - nfsLabel := "" - if len(nfsTokens) > 1 { - constraint = nfsTokens[0] - nfsLabel = nfsTokens[1] - if nfsLabel != "nfs" { - return status.Errorf(codes.InvalidArgument, - "Invalid topology requested for NFS Volume. Please validate your storage class has nfs topology.") - } + log.Infof("Found topology constraint: VxFlex OS system: %s", constraint) + constraints := strings.Split(constraint, "-") + if len(constraints) > 1 { + constraint = constraints[0] + if isNFS && constraints[1] != "nfs" { + return status.Errorf(codes.InvalidArgument, + "Invalid topology requested for NFS Volume. Please validate your storage class has nfs topology.") + } else if !isNFS && constraints[1] != "nvmetcp" { + return status.Errorf(codes.InvalidArgument, "Invalid topology \"%s\" requested. Please validate your storage class has correct topology.", constraints[1]) } } + if constraint != requestedSystem { fmt.Printf("Volume topology segement should have system %s. Found %s.", requestedSystem, constraint) return errors.New("wrong systemID in AccessibleTopology") @@ -1047,10 +1243,10 @@ func (f *feature) iCallCreateVolumeSize(name string, size int64) error { f.createVolumeResponse, f.err = f.service.CreateVolume(ctx, req) if f.err != nil { - log.Printf("CreateVolumeSize called failed: %s\n", f.err.Error()) + log.Infof("CreateVolumeSize called failed: %s\n", f.err.Error()) } if f.createVolumeResponse != nil { - log.Printf("vol id %s\n", f.createVolumeResponse.GetVolume().VolumeId) + log.Infof("vol id %s\n", f.createVolumeResponse.GetVolume().VolumeId) } return nil @@ -1073,10 +1269,10 @@ func (f *feature) iCallCreateVolumeSizeNFS(name string, size int64) error { f.createVolumeResponse, f.err = f.service.CreateVolume(ctx, req) if f.err != nil { - log.Printf("CreateVolumeSize called failed: %s\n", f.err.Error()) + log.Infof("CreateVolumeSize called failed: %s\n", f.err.Error()) } if f.createVolumeResponse != nil { - log.Printf("vol id %s\n", f.createVolumeResponse.GetVolume().VolumeId) + log.Infof("vol id %s\n", f.createVolumeResponse.GetVolume().VolumeId) } return nil @@ -1090,7 +1286,7 @@ func (f *feature) iChangeTheStoragePool(storagePoolName string) error { } func (f *feature) iInduceError(errtype string) error { - log.Printf("set induce error %s\n", errtype) + log.Infof("set induce error %s\n", errtype) inducedError = errors.New(errtype) switch errtype { case "WrongSysNameError": @@ -1140,7 +1336,7 @@ func (f *feature) iInduceError(errtype string) error { case "EmptySysIDInNodeExpand": stepHandlersErrors.EmptySysID = true case "WrongVolIDErrorInNodeExpand": - stepHandlersErrors.BadVolIDError = true + stepHandlersErrors.IncorrectVolID = true case "EmptyEphemeralID": f.nodePublishVolumeRequest.VolumeId = mdmID stepHandlersErrors.EmptyEphemeralID = true @@ -1157,8 +1353,12 @@ func (f *feature) iInduceError(errtype string) error { stepHandlersErrors.GetSdcInstancesError = true case "MapSdcError": stepHandlersErrors.MapSdcError = true + case "MapNVMeError": + stepHandlersErrors.MapNVMeError = true case "RemoveMappedSdcError": stepHandlersErrors.RemoveMappedSdcError = true + case "RemoveMappedHostError": + stepHandlersErrors.RemoveMappedHostError = true case "SDCLimitsError": stepHandlersErrors.SDCLimitsError = true case "require-probe": @@ -1230,7 +1430,6 @@ func (f *feature) iInduceError(errtype string) error { return err } case "NoBlockDevForNodePublish": - unitTestEmulateBlockDevice = false cmd := exec.Command("rm", nodePublishBlockDevicePath) _, err := cmd.CombinedOutput() if err != nil { @@ -1251,6 +1450,33 @@ func (f *feature) iInduceError(errtype string) error { f.service.privDir = filepath.Join(testBaseDir, "xxx/yyy") case "BlockMkfilePrivateDirectoryNodePublish": f.service.privDir = datafile + case "NodeStageNoVolumeID": + f.nodeStageVolumeRequest.VolumeId = "" + case "NodeStageInValidVolumeID": + f.nodeStageVolumeRequest.VolumeId = "invalid" + case "NodeStageNoCapability": + f.nodeStageVolumeRequest.VolumeCapability = nil + case "NodeStageNoAccessMode": + f.nodeStageVolumeRequest.VolumeCapability.AccessMode = nil + case "NodeStageNoStagingPath": + f.nodeStageVolumeRequest.StagingTargetPath = "" + case "GobrickConnectError": + mockGobrickInducedErrors.ConnectVolumeError = true + case "GobrickDisconnectError": + mockGobrickInducedErrors.DisconnectVolumeError = true + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Source: "/dev/nvme0n1", + Path: stageDir, + Device: "nvme0n1", + }, + } + case "GobrickConnectError2": + f.nodeStageVolumeRequest.StagingTargetPath = "" + case "GobrickConnectError3": + f.nodeStageVolumeRequest.StagingTargetPath = "" + case "GobrickConnectError4": + f.nodeStageVolumeRequest.StagingTargetPath = "" case "NodePublishNoVolumeCapability": f.nodePublishVolumeRequest.VolumeCapability = nil case "NodePublishNoAccessMode": @@ -1263,11 +1489,20 @@ func (f *feature) iInduceError(errtype string) error { fmt.Printf("Couldn't make: %s\n", datadir+"/"+sdcVolume1) } case "NodePublishPrivateTargetAlreadyMounted": - cmd := exec.Command("mknod", nodePublishAltBlockDevPath, "b", "0", "0") - _, err := cmd.CombinedOutput() + file, err := os.Create(nodePublishAltBlockDevPath) if err != nil { - fmt.Printf("Couldn't create block dev: %s\n", nodePublishAltBlockDevPath) + fmt.Printf("couldn't create file: %s: %v\n", nodePublishAltBlockDevPath, err) + } + + defer func() { + file.Close() + }() + + // Optionally set permissions to mimic a device node + if err := os.Chmod(nodePublishAltBlockDevPath, 0o750); err != nil { + fmt.Printf("Failed to set permissions on mock device file %s: %v\n", nodePublishAltBlockDevPath, err) } + err = os.MkdirAll("features/"+sdcVolume1, 0o777) if err != nil { fmt.Printf("Couldn't make: %s\n", datadir+"/"+sdcVolume1) @@ -1278,6 +1513,8 @@ func (f *feature) iInduceError(errtype string) error { } case "NodePublishNoTargetPath": f.nodePublishVolumeRequest.TargetPath = "" + case "NodePublishNoStagingTargetPath": + f.nodePublishVolumeRequest.StagingTargetPath = "" case "NodePublishBadTargetPath": f.nodePublishVolumeRequest.TargetPath = filepath.Join(testBaseDir, badtarget) case "NodePublishBlockTargetNotFile": @@ -1306,12 +1543,48 @@ func (f *feature) iInduceError(errtype string) error { gofsutil.GOFSMock.InduceMountError = true case "GOFSMockGetMountsError": gofsutil.GOFSMock.InduceGetMountsError = true + case "GOFSMockGetMounts_deleted": + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Source: "/dev/nvme0n1-deleted", + Path: stageDir, + Device: "nvme0n1-deleted", + }, + } + case "GOFSMockGetMounts_unknowndevice": + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Source: "/dev/unkowndevice", + Path: stageDir, + Device: "unkowndevice", + }, + } + case "GOFSMockGetMounts_targetpath": + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Path: datadir, + }, + } case "GOFSMockUnmountError": gofsutil.GOFSMock.InduceUnmountError = true case "GOFSMockGetDiskFormatError": gofsutil.GOFSMock.InduceGetDiskFormatError = true + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Source: "/dev/nvme0n1", + Path: stageDir, + }, + } case "GOFSMockGetDiskFormatType": gofsutil.GOFSMock.InduceGetDiskFormatType = "unknown-fs" + case "GOFSMockGetDiskFormatType_mpath_member": + gofsutil.GOFSMock.InduceGetDiskFormatType = "mpath_member" + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Source: "/dev/nvme0n1", + Path: stageDir, + }, + } case "GOFSMockFormatError": gofsutil.GOFSMock.InduceFormatError = true case "GOFSInduceFSTypeError": @@ -1360,6 +1633,16 @@ func (f *feature) iInduceError(errtype string) error { stepHandlersErrors.UpdateConfigFormatError = true case "ConfigMapNotFoundError": stepHandlersErrors.ConfigMapNotFoundError = true + case "NoNfsServer": + stepHandlersErrors.NoNfsServer = true + case "SdtNotFoundError": + stepHandlersErrors.SdtNotFoundError = true + case "EmptySdtError": + stepHandlersErrors.EmptySdtError = true + case "NvmeDiscoveryError": + stepHandlersErrors.NvmeDiscoveryError = true + case "GetMetricsError": + stepHandlersErrors.GetMetricsError = true default: fmt.Println("Ensure that the error is handled in the handlers section.") } @@ -1422,11 +1705,15 @@ func (f *feature) getControllerPublishVolumeRequest(accessType string) *csi.Cont if !f.noNodeID { req.NodeId = goodNodeID + if f.protocol == NVMeTCP { + req.NodeId = goodNodeIDNVMe + } } req.Readonly = false if !f.omitVolumeCapability { req.VolumeCapability = capability } + req.VolumeContext = make(map[string]string) return req } @@ -1625,10 +1912,10 @@ func (f *feature) iCallPublishVolumeWith(arg1 string) error { f.publishVolumeRequest = req } - log.Printf("Calling controllerPublishVolume") + log.Infof("Calling controllerPublishVolume") f.publishVolumeResponse, f.err = f.service.ControllerPublishVolume(ctx, req) if f.err != nil { - log.Printf("PublishVolume call failed: %s\n", f.err.Error()) + log.Infof("PublishVolume call failed: %s\n", f.err.Error()) } return nil } @@ -1690,10 +1977,10 @@ func (f *feature) iCallPublishVolumeWithNFS(arg1 string) error { log.Fatalf("failed to create configMap: %v", err) } - log.Printf("Calling controllerPublishVolume") + log.Infof("Calling controllerPublishVolume") f.publishVolumeResponse, f.err = f.service.ControllerPublishVolume(ctx, req) if f.err != nil { - log.Printf("PublishVolume call failed: %s\n", f.err.Error()) + log.Infof("PublishVolume call failed: %s\n", f.err.Error()) } return nil } @@ -1764,9 +2051,13 @@ func (f *feature) noAccessMode() error { } func (f *feature) thenIUseADifferentNodeID() error { - f.publishVolumeRequest.NodeId = altNodeID + nodeID := altNodeID + if f.protocol == NVMeTCP { + nodeID = altNodeIDNVMe + } + f.publishVolumeRequest.NodeId = nodeID if f.unpublishVolumeRequest != nil { - f.unpublishVolumeRequest.NodeId = altNodeID + f.unpublishVolumeRequest.NodeId = nodeID } return nil } @@ -1793,7 +2084,12 @@ func (f *feature) getControllerUnpublishVolumeRequest() *csi.ControllerUnpublish } } if !f.noNodeID { - req.NodeId = goodNodeID + if f.protocol == NVMeTCP { + req.NodeId = goodNodeIDNVMe + } + if f.protocol == SDC { + req.NodeId = goodNodeID + } } return req } @@ -1820,10 +2116,11 @@ func (f *feature) iCallUnpublishVolume() error { req = f.getControllerUnpublishVolumeRequest() f.unpublishVolumeRequest = req } - log.Printf("Calling controllerUnpublishVolume: %s", req.VolumeId) + log.Infof("Calling controllerUnpublishVolume: %s", req.NodeId) + log.Infof("Calling controllerUnpublishVolume: %s", req.VolumeId) f.unpublishVolumeResponse, f.err = f.service.ControllerUnpublishVolume(ctx, req) if f.err != nil { - log.Printf("UnpublishVolume call failed: %s\n", f.err.Error()) + log.Infof("UnpublishVolume call failed: %s\n", f.err.Error()) } return nil } @@ -1859,13 +2156,13 @@ func (f *feature) iCallUnpublishVolumeNFS() error { // Create a ConfigMap using fake ClientSet _, err := clientSet.CoreV1().ConfigMaps(DriverNamespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) if err != nil { - Log.Fatalf("failed to create configMaps: %v", err) + log.Fatalf("failed to create configMaps: %v", err) } - log.Printf("Calling controllerUnpublishVolume: %s", req.VolumeId) + log.Infof("Calling controllerUnpublishVolume: %s", req.VolumeId) f.unpublishVolumeResponse, f.err = f.service.ControllerUnpublishVolume(ctx, req) if f.err != nil { - log.Printf("UnpublishVolume call failed: %s\n", f.err.Error()) + log.Infof("UnpublishVolume call failed: %s\n", f.err.Error()) } return nil } @@ -1924,6 +2221,18 @@ func (f *feature) iCallNodeGetInfoWithValidNodeUID() error { return nil } +func (f *feature) iCallNodeGetInfoWithNVMeFlagTrue() error { + ctx := context.Background() + req := new(csi.NodeGetInfoRequest) + f.service.useNVME = true + gonvme.GONVMEMock.InduceDiscoveryError = stepHandlersErrors.NvmeDiscoveryError + f.nodeGetInfoResponse, f.err = f.service.NodeGetInfo(ctx, req) + f.nodeGetInfoResponse.NodeId = goodNodeIDNVMe + f.nodeGetInfoResponse.MaxVolumesPerNode = 0 + fmt.Printf("NodeGetInfoResponse: %v", f.nodeGetInfoResponse) + return nil +} + func (f *feature) iCallGetNodeUID() error { f.setFakeNode() ctx := context.Background() @@ -2019,6 +2328,15 @@ func (f *feature) iCallGetNodeLabelsWithInvalidNode() error { } func (f *feature) iCallGetNodeLabelsWithUnsetKubernetesClient() error { + defaultKubernetesHost := os.Getenv("KUBERNETES_SERVICE_HOST") + defaultKubernetesPort := os.Getenv("KUBERNETES_SERVICE_PORT") + defer func() { + os.Setenv("KUBERNETES_SERVICE_HOST", defaultKubernetesHost) + os.Setenv("KUBERNETES_SERVICE_PORT", defaultKubernetesPort) + }() + + os.Unsetenv("KUBERNETES_SERVICE_HOST") + os.Unsetenv("KUBERNETES_SERVICE_PORT") K8sClientset = nil ctx := context.Background() f.nodeLabels, f.err = f.service.GetNodeLabels(ctx) @@ -2031,10 +2349,9 @@ func (f *feature) iCallGetNodeUIDWithInvalidNode() error { return nil } -func (f *feature) iCallGetNodeUIDWithUnsetKubernetesClient() error { - K8sClientset = nil - ctx := context.Background() - f.nodeUID, f.err = f.service.GetNodeUID(ctx) +func (f *feature) iCallGetNodeUIDWithoutKubeNodeName() error { + f.service.opts.KubeNodeName = "" + f.nodeUID, f.err = f.service.GetNodeUID(context.Background()) return nil } @@ -2048,6 +2365,17 @@ func (f *feature) iCallNodeProbe() error { return nil } +func (f *feature) iCallNodeProbeNVMeFlagEnabled() error { + ctx := context.Background() + req := new(csi.ProbeRequest) + f.service.useNVME = true + f.checkGoRoutines("before probe") + f.service.mode = "node" + f.probeResponse, f.err = f.service.Probe(ctx, req) + f.checkGoRoutines("after probe") + return nil +} + func (f *feature) aValidNodeGetInfoResponseIsReturned() error { if f.err != nil { return f.err @@ -2074,6 +2402,56 @@ func (f *feature) aValidNodeGetInfoResponseWithNodeUIDIsReturned() error { return nil } +func (f *feature) aValidNodeGetInfoResponseWithTopologyIsReturned(protocols string) error { + if f.err != nil { + return f.err + } + + fmt.Printf("node: %s", f.nodeGetInfoResponse) + if f.nodeGetInfoResponse.NodeId == "" { + return errors.New("expected NodeGetInfoResponse to contain NodeID but it was null") + } + + if f.nodeGetInfoResponse.MaxVolumesPerNode != 0 { + return errors.New("expected NodeGetInfoResponse MaxVolumesPerNode to be 0") + } + + if f.nodeGetInfoResponse.AccessibleTopology == nil || len(f.nodeGetInfoResponse.AccessibleTopology.Segments) == 0 { + return errors.New("expected NodeGetInfoResponse AccessibleTopology to be not empty") + } + + expectedTopology := make(map[string]string) + for _, protocol := range strings.Split(protocols, ",") { + expectedTopology[Name+"/"+arrayID+"-"+protocol] = "true" + expectedTopology[Name+"/"+arrayID2+"-"+protocol] = "true" + } + + for key, expectedValue := range expectedTopology { + segment, ok := f.nodeGetInfoResponse.AccessibleTopology.Segments[key] + if !ok { + return fmt.Errorf("expected NodeGetInfoResponse AccessibleTopology to contain segment %s", key) + } + if segment != expectedValue { + return fmt.Errorf("expected NodeGetInfoResponse AccessibleTopology segment %s to be %s, but got %s", key, expectedValue, segment) + } + } + + // Check that no extra topologies are present + if len(f.nodeGetInfoResponse.AccessibleTopology.Segments) != len(expectedTopology) { + return fmt.Errorf("expected NodeGetInfoResponse AccessibleTopology to have %d segments, but got %d", + len(expectedTopology), len(f.nodeGetInfoResponse.AccessibleTopology.Segments)) + } + + for key := range f.nodeGetInfoResponse.AccessibleTopology.Segments { + if _, ok := expectedTopology[key]; !ok { + return fmt.Errorf("unexpected NodeGetInfoResponse AccessibleTopology segment %s", key) + } + } + + fmt.Printf("NodeID %s\n", f.nodeGetInfoResponse.NodeId) + return nil +} + func (f *feature) theVolumeLimitIsSet() error { if f.err != nil { return f.err @@ -2096,10 +2474,10 @@ func (f *feature) iCallDeleteVolumeWith(arg1 string) error { req = f.getControllerDeleteVolumeRequest(arg1) f.deleteVolumeRequest = req } - log.Printf("Calling DeleteVolume") + log.Infof("Calling DeleteVolume") f.deleteVolumeResponse, f.err = f.service.DeleteVolume(ctx, req) if f.err != nil { - log.Printf("DeleteVolume called failed: %s\n", f.err.Error()) + log.Infof("DeleteVolume called failed: %s\n", f.err.Error()) } return nil } @@ -2111,10 +2489,10 @@ func (f *feature) iCallDeleteVolumeWithBad(arg1 string) error { req = f.getControllerDeleteVolumeRequestBad(arg1) f.deleteVolumeRequest = req } - log.Printf("Calling DeleteVolume") + log.Infof("Calling DeleteVolume") f.deleteVolumeResponse, f.err = f.service.DeleteVolume(ctx, req) if f.err != nil { - log.Printf("DeleteVolume called failed: %s\n", f.err.Error()) + log.Infof("DeleteVolume called failed: %s\n", f.err.Error()) } return nil } @@ -2126,10 +2504,10 @@ func (f *feature) iCallDeleteVolumeNFSWith(arg1 string) error { req = f.getControllerDeleteVolumeRequestNFS(arg1) f.deleteVolumeRequest = req } - log.Printf("Calling DeleteVolume") + log.Infof("Calling DeleteVolume") f.deleteVolumeResponse, f.err = f.service.DeleteVolume(ctx, req) if f.err != nil { - log.Printf("DeleteVolume called failed: %s\n", f.err.Error()) + log.Infof("DeleteVolume called failed: %s\n", f.err.Error()) } return nil } @@ -2182,10 +2560,10 @@ func (f *feature) iCallGetCapacityWithStoragePool(arg1 string) error { parameters[KeyStoragePool] = arg1 req.Parameters = parameters } - log.Printf("Calling GetCapacity") + log.Infof("Calling GetCapacity") f.getCapacityResponse, f.err = f.service.GetCapacity(ctx, req) if f.err != nil { - log.Printf("GetCapacity call failed: %s\n", f.err.Error()) + log.Infof("GetCapacity call failed: %s\n", f.err.Error()) return nil } return nil @@ -2206,10 +2584,10 @@ func (f *feature) iCallGetCapacityWithAvailabilityZone(zoneLabelKey, zoneName st }, } - log.Printf("Calling GetCapacity") + log.Infof("Calling GetCapacity") f.getCapacityResponse, f.err = f.service.GetCapacity(ctx, req) if f.err != nil { - log.Printf("GetCapacity call failed: %s\n", f.err.Error()) + log.Infof("GetCapacity call failed: %s\n", f.err.Error()) return nil } return nil @@ -2219,7 +2597,7 @@ func (f *feature) iCallGetMaximumVolumeSize(arg1 string) { systemid := arg1 f.maxVolSize, f.err = f.service.getMaximumVolumeSize(systemid) if f.err != nil { - log.Printf("err while getting max vol size: %s\n", f.err.Error()) + log.Infof("err while getting max vol size: %s\n", f.err.Error()) } } @@ -2259,10 +2637,10 @@ func (f *feature) iCallControllerGetCapabilities(isHealthMonitorEnabled string) } ctx := context.Background() req := new(csi.ControllerGetCapabilitiesRequest) - log.Printf("Calling ControllerGetCapabilities") + log.Infof("Calling ControllerGetCapabilities") f.controllerGetCapabilitiesResponse, f.err = f.service.ControllerGetCapabilities(ctx, req) if f.err != nil { - log.Printf("ControllerGetCapabilities call failed: %s\n", f.err.Error()) + log.Infof("ControllerGetCapabilities call failed: %s\n", f.err.Error()) return f.err } return nil @@ -2340,10 +2718,10 @@ func (f *feature) iCallListVolumesWith(maxEntriesString, startingToken string) e f.listVolumesRequest = req } - log.Printf("Calling ListVolumes with req=%+v", f.listVolumesRequest) + log.Infof("Calling ListVolumes with req=%+v", f.listVolumesRequest) f.listVolumesResponse, f.err = f.service.ListVolumes(ctx, req) if f.err != nil { - log.Printf("ListVolume called failed: %s\n", f.err.Error()) + log.Infof("ListVolume called failed: %s\n", f.err.Error()) } else { f.listVolumesNextTokenCache = f.listVolumesResponse.NextToken } @@ -2471,7 +2849,7 @@ func (f *feature) iCallValidateVolumeCapabilitiesWithVoltypeAccessFstype(voltype capabilities := make([]*csi.VolumeCapability, 0) capabilities = append(capabilities, capability) req.VolumeCapabilities = capabilities - log.Printf("Calling ValidateVolumeCapabilities %#v", accessMode) + log.Infof("Calling ValidateVolumeCapabilities %#v", accessMode) f.validateVolumeCapabilitiesResponse, f.err = f.service.ValidateVolumeCapabilities(ctx, req) if f.err != nil { return nil @@ -2611,17 +2989,24 @@ func (f *feature) aControllerPublishedEphemeralVolume() error { _, err = os.Stat(nodePublishEphemDevPath) _, err2 := os.Stat(nodePublishSymlinkDir + "/emc-vol" + "-" + mdmIDEphem + "-" + ephemVolumeSDC) if err != nil || err2 != nil { - cmd := exec.Command("mknod", nodePublishEphemDevPath, "b", "0", "0") - output, err := cmd.CombinedOutput() + file, err := os.Create(nodePublishEphemDevPath) if err != nil { - fmt.Printf("scinic: %s\n", err.Error()) + fmt.Printf("couldn't create file: %s: %v\n", nodePublishEphemDevPath, err) + } + + defer func() { + file.Close() + }() + + // Optionally set permissions to mimic a device node + if err := os.Chmod(nodePublishEphemDevPath, 0o750); err != nil { + fmt.Printf("Failed to set permissions on mock device file %s: %v\n", nodePublishEphemDevPath, err) } - fmt.Printf("mknod output: %s\n", output) // Make the symlink - cmdstring := fmt.Sprintf("cd %s; ln -s ../../scinic emc-vol-%s-%s", nodePublishSymlinkDir, mdmIDEphem, ephemVolumeSDC) + cmdstring := fmt.Sprintf("cd %s; ln -s %s emc-vol-%s-%s", nodePublishSymlinkDir, nodePublishEphemDevPath, mdmIDEphem, ephemVolumeSDC) cmd = exec.Command("sh", "-c", cmdstring) - output, err = cmd.CombinedOutput() + output, err := cmd.CombinedOutput() fmt.Printf("symlink output: %s\n", output) if err != nil { fmt.Printf("link: %s\n", err.Error()) @@ -2653,8 +3038,6 @@ func (f *feature) aControllerPublishedEphemeralVolume() error { goscaleio.FSDevDirectoryPrefix = "test" // Empty WindowsMounts in gofsutil gofsutil.GOFSMockMounts = gofsutil.GOFSMockMounts[:0] - // Set variables in mount for unit testing - unitTestEmulateBlockDevice = true return nil } @@ -2689,18 +3072,49 @@ func (f *feature) controllerPublishVolume() { fmt.Printf("removed private staging directory\n") } + // remove the devices so that they can be re-created for each test + dir, err := os.ReadDir(getTargetPathPrefix()) + if err != nil { + fmt.Printf("couldn't read device directory: %s: %v\n", getTargetPathPrefix(), err) + } + for _, d := range dir { + os.RemoveAll(filepath.Join(getTargetPathPrefix(), d.Name())) + } + // Make the block device _, err = os.Stat(nodePublishBlockDevicePath) if err != nil { - cmd := exec.Command("mknod", nodePublishBlockDevicePath, "b", "0", "0") - output, err := cmd.CombinedOutput() + file, err := os.Create(nodePublishBlockDevicePath) if err != nil { - fmt.Printf("scinia: %s\n", err.Error()) + fmt.Printf("couldn't create file: %s: %v\n", nodePublishBlockDevicePath, err) + } + + defer func() { + file.Close() + }() + + // Optionally set permissions to mimic a device node + if err := os.Chmod(nodePublishBlockDevicePath, 0o750); err != nil { + fmt.Printf("Failed to set permissions on mock device file %s: %v\n", nodePublishBlockDevicePath, err) } - fmt.Printf("mknod output: %s\n", output) // Make the symlink - cmdstring := fmt.Sprintf("cd %s; ln -s ../../scinia emc-vol-%s-%s", nodePublishSymlinkDir, mdmID, sdcVolume1) + cmdstring := fmt.Sprintf("cd %s; ln -s %s emc-vol-%s-%s", nodePublishSymlinkDir, nodePublishBlockDevicePath, mdmID, sdcVolume1) + cmd = exec.Command("sh", "-c", cmdstring) + output, err := cmd.CombinedOutput() + fmt.Printf("symlink output: %s\n", output) + if err != nil { + fmt.Printf("link: %s\n", err.Error()) + err = nil + } + + linkTarget, err := os.Readlink(fmt.Sprintf("%s/emc-vol-%s-%s", nodePublishSymlinkDir, mdmID, sdcVolume1)) + if err != nil { + fmt.Printf("Symlink not created correctly: %v\n", err) + } + fmt.Printf("Symlink points to: %s\n", linkTarget) + + cmdstring = fmt.Sprintf("cd %s; ln -s %s nvme-eui.%s", nodePublishSymlinkDir, nodePublishBlockDevicePath, nvmeNguid) cmd = exec.Command("sh", "-c", cmdstring) output, err = cmd.CombinedOutput() fmt.Printf("symlink output: %s\n", output) @@ -2708,6 +3122,12 @@ func (f *feature) controllerPublishVolume() { fmt.Printf("link: %s\n", err.Error()) err = nil } + + linkTarget, err = os.Readlink(fmt.Sprintf("%s/nvme-eui.%s", nodePublishSymlinkDir, nvmeNguid)) + if err != nil { + fmt.Printf("Symlink not created correctly: %v\n", err) + } + fmt.Printf("Symlink points to: %s\n", linkTarget) } // Make the target directory if required @@ -2734,8 +3154,6 @@ func (f *feature) controllerPublishVolume() { goscaleio.FSDevDirectoryPrefix = "test" // Empty WindowsMounts in gofsutil gofsutil.GOFSMockMounts = gofsutil.GOFSMockMounts[:0] - // Set variables in mount for unit testing - unitTestEmulateBlockDevice = true } func (f *feature) twoIdenticalVolumesOnTwoDifferentSystems() error { @@ -2762,7 +3180,7 @@ func (f *feature) twoIdenticalVolumesOnTwoDifferentSystems() error { } func (f *feature) iCreateFalseEphemeralID() error { - fakeEphemeralIDFolder := ephemeralStagingMountPath + mdmIDEphem + fakeEphemeralIDFolder := filepath.Join(ephemeralStagingMountPath, mdmIDEphem) _, err := os.Stat(fakeEphemeralIDFolder) if err != nil { @@ -2805,12 +3223,24 @@ func (f *feature) iCreateFalseEphemeralID() error { return nil } +func (f *feature) getNodeStageVolumeRequest() error { + req := new(csi.NodeStageVolumeRequest) + req.VolumeId = sdcVolume1 + req.StagingTargetPath = stageDir + req.VolumeCapability = f.capability + f.nodeStageVolumeRequest = req + return nil +} + func (f *feature) getNodeEphemeralVolumePublishRequest(name, size, sp, systemName string) error { req := new(csi.NodePublishVolumeRequest) req.VolumeId = sdcVolume1 req.Readonly = false req.VolumeCapability = f.capability f.service.opts.defaultSystemID = "" + if f.service.useNVME { + f.service.nodeID = goodNodeIDNVMe + } req.VolumeContext = map[string]string{"csi.storage.k8s.io/ephemeral": "true", "volumeName": name, "size": size, "storagepool": sp, "systemID": systemName} // remove ephemeral mounting path before starting test @@ -2847,6 +3277,33 @@ func (f *feature) getNodePublishVolumeRequest() error { return nil } +func (f *feature) getNodePublishVolumeRequestNVME(voltype string) error { + req := new(csi.NodePublishVolumeRequest) + req.VolumeId = sdcVolume1 + + switch voltype { + case "block": + req.Readonly = false + case "mount": + req.Readonly = true + } + + gofsutil.GOFSMockWWNToDevice = map[string]string{nvmeNguid: nodePublishBlockDevicePath} + + req.VolumeCapability = f.capability + block := f.capability.GetBlock() + if block != nil { + req.TargetPath = datafile + } + mount := f.capability.GetMount() + if mount != nil { + req.TargetPath = datadir + } + req.StagingTargetPath = stageDir + f.nodePublishVolumeRequest = req + return nil +} + func (f *feature) getNodePublishVolumeRequestNFS() error { req := new(csi.NodePublishVolumeRequest) req.VolumeId = "14dbbf5617523654" + "/" + fileSystemNameToID["volume1"] @@ -2933,6 +3390,32 @@ func (f *feature) iCallNodePublishVolume(arg1 string) error { return nil } +//nolint:revive +func (f *feature) iCallNodePublishVolumeNVME(volType string) error { + header := metadata.New(map[string]string{"csi.requestid": "1"}) + ctx := metadata.NewIncomingContext(context.Background(), header) + req := f.nodePublishVolumeRequest + f.service.useNVME = true + if req == nil { + fmt.Printf("Request was Nil \n") + _ = f.getNodePublishVolumeRequestNVME(volType) + req = f.nodePublishVolumeRequest + } + + fmt.Printf("Calling NodePublishVolume\n") + fmt.Printf("nodePV req is: %v \n", req) + _, err := f.service.NodePublishVolume(ctx, req) + if err != nil { + fmt.Printf("NodePublishVolume failed: %s\n", err.Error()) + if f.err == nil { + f.err = err + } + } else { + fmt.Printf("NodePublishVolume completed successfully\n") + } + return nil +} + //nolint:revive func (f *feature) iCallNodePublishVolumeNFS(arg1 string) error { header := metadata.New(map[string]string{"csi.requestid": "1"}) @@ -3002,7 +3485,7 @@ func (f *feature) iCallUnmountPrivMount() error { func (f *feature) iCallGetPathMounts() error { gofsutil.GOFSMock.InduceGetMountsError = true - _, err := getPathMounts("/foo/bar") + _, err := getPathMounts(context.Background(), "/foo/bar") fmt.Printf(" getPathMounts error : %s\n", err) // getMounts induced err if err != nil { @@ -3118,7 +3601,7 @@ func (f *feature) iCallMountValidateVolCapabilities() error { func (f *feature) iCallCleanupPrivateTargetForErrors() error { gofsutil.GOFSMock.InduceDevMountsError = true - sysdevice, _ := GetDevice("test/dev/scinia") + sysdevice, _ := GetDevice(nodePublishBlockDevicePath) err := cleanupPrivateTarget(sysdevice, "1", "features/d0f055a700000000") if err != nil { fmt.Printf("Cleanupprivatetarget getDevice error : %s\n", err.Error()) @@ -3148,7 +3631,7 @@ func (f *feature) iCallCleanupPrivateTargetForErrors() error { } func (f *feature) iCallCleanupPrivateTarget() error { - sysdevice, terr := GetDevice("test/dev/scinia") + sysdevice, terr := GetDevice(nodePublishBlockDevicePath) if terr != nil { return terr } @@ -3176,8 +3659,6 @@ func (f *feature) iCallUnmountAndDeleteTarget() error { } func (f *feature) iCallEphemeralNodePublish() error { - save := ephemeralStagingMountPath - ephemeralStagingMountPath = "/tmp" header := metadata.New(map[string]string{"csi.requestid": "1"}) ctx := metadata.NewIncomingContext(context.Background(), header) req := new(csi.NodePublishVolumeRequest) @@ -3190,7 +3671,6 @@ func (f *feature) iCallEphemeralNodePublish() error { f.err = err } } - ephemeralStagingMountPath = save return nil } @@ -3203,9 +3683,13 @@ func (f *feature) iCallEphemeralNodeUnpublish() error { if stepHandlersErrors.NoVolumeIDError { req.VolumeId = "" } + err := os.MkdirAll(ephemeralStagingMountPath+"/"+goodVolumeID+"/id", 0o777) + if err != nil { + return err + } fmt.Printf("Calling ephemeralNodeUnpublishiVolume\n") - err := f.service.ephemeralNodeUnpublish(ctx, req) + err = f.service.ephemeralNodeUnpublish(ctx, req) if err != nil { fmt.Printf("ephemeralNodeUnpublishVolume failed: %s\n", err.Error()) if f.err == nil { @@ -3226,6 +3710,13 @@ func (f *feature) iCallNodeUnpublishVolume(arg1 string) error { req.VolumeId = f.nodePublishVolumeRequest.VolumeId req.TargetPath = f.nodePublishVolumeRequest.TargetPath fmt.Printf("Calling NodeUnpublishVolume\n") + if stepHandlersErrors.BadVolIDError { + req.VolumeId = badVolumeID + } + if arg1 == "NVMeTCP" { + f.service.useNVME = true + f.service.nodeID = goodNodeIDNVMe + } _, err := f.service.NodeUnpublishVolume(ctx, req) if err != nil { fmt.Printf("NodeUnpublishVolume failed: %s\n", err.Error()) @@ -3259,7 +3750,7 @@ func (f *feature) thereIsMount(path string) { split := strings.Split(path, ",") for _, p := range split { gofsutil.GOFSMockMounts = append(gofsutil.GOFSMockMounts, gofsutil.Info{ - Device: "test/dev/scinia", + Device: nodePublishBlockDevicePath, Path: p, }) } @@ -3296,17 +3787,17 @@ func (f *feature) theConfigMapIsUpdated() error { tmpFile, err := os.Create("driver-config-params.yaml") if err != nil { - Log.Errorf("Error creating temp file: %v", err) + log.Errorf("Error creating temp file: %v", err) } if _, err := tmpFile.Write([]byte(data)); err != nil { - Log.Errorf("Error writing to temp file: %v", err) + log.Errorf("Error writing to temp file: %v", err) } if !stepHandlersErrors.ConfigMapNotFoundError { // Create a ConfigMap using fake ClientSet _, err = clientSet.CoreV1().ConfigMaps(DriverNamespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) if err != nil { - Log.Errorf("failed to create configmap: %v", err) + log.Errorf("failed to create configmap: %v", err) } } @@ -3341,6 +3832,8 @@ func setUpBeforeServeContext() (interface{}, []string) { stringSlice = append(stringSlice, "X_CSI_REPLICATION_PREFIX=test") stringSlice = append(stringSlice, "X_CSI_POWERFLEX_EXTERNAL_ACCESS=test") stringSlice = append(stringSlice, "X_CSI_VXFLEXOS_THICKPROVISIONING=dummy") + stringSlice = append(stringSlice, "X_CSI_PODMON_ENABLED=true") + stringSlice = append(stringSlice, "X_CSI_PODMON_ARRAY_CONNECTIVITY_POLL_RATE=dummy") if os.Getenv("ALLOW_RWO_MULTI_POD") == "true" { fmt.Printf("debug set ALLOW_RWO_MULTI_POD\n") @@ -3390,9 +3883,24 @@ func (f *feature) executeBeforeServe(ctx context.Context) error { } func (f *feature) iCallNodeStageVolume() error { - ctx := context.Background() - req := new(csi.NodeStageVolumeRequest) - _, f.err = f.service.NodeStageVolume(ctx, req) + header := metadata.New(map[string]string{"csi.requestid": "1"}) + ctx := metadata.NewIncomingContext(context.Background(), header) + if f.nodeStageVolumeRequest == nil { + fmt.Printf("NodeStageVolumeRequest was nil \n") + _ = f.getNodeStageVolumeRequest() + } + + fmt.Println("Calling NodeStageVolume") + _, err := f.service.NodeStageVolume(ctx, f.nodeStageVolumeRequest) + + if err != nil { + fmt.Println("NodeStageVolume failed: ", err.Error()) + if f.err == nil { + f.err = err + } + } else { + fmt.Println("NodeStageVolume completed successfully") + } return nil } @@ -3433,7 +3941,7 @@ func (f *feature) iCallNodeExpandVolume(volPath string) error { req.VolumeId = badCsiVolumeID } else if stepHandlersErrors.EmptySysID { req.VolumeId = goodVolumeID - } else if stepHandlersErrors.BadVolIDError { + } else if stepHandlersErrors.IncorrectVolID { req.VolumeId = badVolumeID } _, f.err = f.service.NodeExpandVolume(ctx, req) @@ -3546,7 +4054,6 @@ func (f *feature) aCorrectNodeGetVolumeStatsResponse() error { func (f *feature) iCallNodeUnstageVolumeWith(errStr string) error { // Save the ephemeralStagingMountPath to restore below - ephemeralPath := ephemeralStagingMountPath header := metadata.New(map[string]string{"csi.requestid": "1"}) if errStr == "NoRequestID" { header = metadata.New(map[string]string{"csi.requestid": ""}) @@ -3557,7 +4064,7 @@ func (f *feature) iCallNodeUnstageVolumeWith(errStr string) error { if errStr == "NoVolumeID" { req.VolumeId = "" } - req.StagingTargetPath = datadir + req.StagingTargetPath = stageDir if errStr == "NoStagingTarget" { req.StagingTargetPath = "" } @@ -3567,14 +4074,13 @@ func (f *feature) iCallNodeUnstageVolumeWith(errStr string) error { } if errStr == "EphemeralVolume" { // Create an ephemeral volume id - ephemeralStagingMountPath = "test/" - err := os.MkdirAll("test"+"/"+goodVolumeID+"/id", 0o777) + // ephemeralStagingMountPath = "test/" + err := os.MkdirAll(ephemeralStagingMountPath+"/"+goodVolumeID+"/id", 0o777) if err != nil { return err } } _, f.err = f.service.NodeUnstageVolume(ctx, req) - ephemeralStagingMountPath = ephemeralPath os.Remove("test" + "/" + goodVolumeID + "/id") return nil } @@ -3610,16 +4116,18 @@ func (f *feature) aValidNodeGetCapabilitiesResponseIsReturned() error { count = count + 1 case csi.NodeServiceCapability_RPC_GET_VOLUME_STATS: count = count + 1 + case csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME: + count = count + 1 default: return fmt.Errorf("received unxexpcted capability: %v", typex) } } - if f.service.opts.IsHealthMonitorEnabled && count != 4 { + if f.service.opts.IsHealthMonitorEnabled && count != 5 { // Set default value f.service.opts.IsHealthMonitorEnabled = false return errors.New("Did not retrieve all the expected capabilities") - } else if !f.service.opts.IsHealthMonitorEnabled && count != 2 { + } else if !f.service.opts.IsHealthMonitorEnabled && count != 3 { return errors.New("Did not retrieve all the expected capabilities") } // Set default value @@ -3725,7 +4233,7 @@ func (f *feature) iCallControllerGetVolume() error { f.ControllerGetVolumeResponse, f.err = f.service.ControllerGetVolume(ctx, req) if f.err != nil { - log.Printf("Controller GetVolume call failed: %s\n", f.err.Error()) + log.Infof("Controller GetVolume call failed: %s\n", f.err.Error()) } fmt.Printf("Response from ControllerGetVolume is %v", f.ControllerGetVolumeResponse) return nil @@ -4029,10 +4537,10 @@ func (f *feature) iCallListSnapshotsWithMaxentriesAndStartingtoken(maxEntriesStr req := &csi.ListSnapshotsRequest{MaxEntries: int32(maxEntries), StartingToken: startingTokenString} f.listSnapshotsRequest = req - log.Printf("Calling ListSnapshots with req=%+v", f.listVolumesRequest) + log.Infof("Calling ListSnapshots with req=%+v", f.listVolumesRequest) f.listSnapshotsResponse, f.err = f.service.ListSnapshots(ctx, req) if f.err != nil { - log.Printf("ListSnapshots called failed: %s\n", f.err.Error()) + log.Infof("ListSnapshots called failed: %s\n", f.err.Error()) } return nil } @@ -4054,10 +4562,10 @@ func (f *feature) iCallListSnapshotsForVolume(arg1 string) error { } f.listSnapshotsRequest = req - log.Printf("Calling ListSnapshots with req=%+v", f.listSnapshotsRequest) + log.Infof("Calling ListSnapshots with req=%+v", f.listSnapshotsRequest) f.listSnapshotsResponse, f.err = f.service.ListSnapshots(ctx, req) if f.err != nil { - log.Printf("ListSnapshots called failed: %s\n", f.err.Error()) + log.Infof("ListSnapshots called failed: %s\n", f.err.Error()) } return nil } @@ -4066,10 +4574,10 @@ func (f *feature) iCallListSnapshotsForSnapshot(arg1 string) error { ctx := context.Background() req := &csi.ListSnapshotsRequest{SnapshotId: arg1} f.listSnapshotsRequest = req - log.Printf("Calling ListSnapshots with req=%+v", f.listVolumesRequest) + log.Infof("Calling ListSnapshots with req=%+v", f.listVolumesRequest) f.listSnapshotsResponse, f.err = f.service.ListSnapshots(ctx, req) if f.err != nil { - log.Printf("ListSnapshots called failed: %s\n", f.err.Error()) + log.Infof("ListSnapshots called failed: %s\n", f.err.Error()) } return nil } @@ -4463,7 +4971,7 @@ func (f *feature) iCallGetReplicationCapabilities() error { req := &replication.GetReplicationCapabilityRequest{} ctx := context.Background() f.replicationCapabilitiesResponse, f.err = f.service.GetReplicationCapabilities(ctx, req) - log.Printf("GetReplicationCapabilities returned %+v", f.replicationCapabilitiesResponse) + log.Infof("GetReplicationCapabilities returned %+v", f.replicationCapabilitiesResponse) return nil } @@ -4574,6 +5082,36 @@ func (f *feature) iCallCreateRemoteVolume() error { return nil } +func (f *feature) iCallCreateRemoteVolumeWithError(errorMessage string) error { + ctx := context.Background() + req := &replication.CreateRemoteVolumeRequest{} + if f.createVolumeResponse == nil { + return errors.New("iCallCreateRemoteVolume: f.createVolumeResponse is nil") + } + req.VolumeHandle = f.createVolumeResponse.Volume.VolumeId + if stepHandlersErrors.NoVolIDError { + req.VolumeHandle = "" + } + if stepHandlersErrors.BadVolIDError { + req.VolumeHandle = "/" + } + req.Parameters = map[string]string{ + f.service.WithRP(KeyReplicationRemoteStoragePool): "viki_pool_HDD_20181031", + f.service.WithRP(KeyReplicationRemoteSystem): "15dbbf5617523655", + } + _, f.err = f.service.CreateRemoteVolume(ctx, req) + + if f.err != nil { + if strings.Contains(f.err.Error(), errorMessage) { + log.Infof("Expected Error Message Found: %s\n", f.err.Error()) + } else { + return fmt.Errorf("expected error message not found: %s in the error: %s", errorMessage, f.err.Error()) + } + } + + return nil +} + func (f *feature) iCallDeleteLocalVolume(name string) error { ctx := context.Background() @@ -4587,7 +5125,7 @@ func (f *feature) iCallDeleteLocalVolume(name string) error { volumeHandle = "/" } - log.Printf("iCallDeleteLocalVolume name %s to ID %s", name, volumeHandle) + log.Infof("iCallDeleteLocalVolume name %s to ID %s", name, volumeHandle) req := &replication.DeleteLocalVolumeRequest{ VolumeHandle: volumeHandle, @@ -4712,7 +5250,7 @@ func (f *feature) iCallDeleteVolume(name string) error { ctx := context.Background() req := f.getControllerDeleteVolumeRequest("single-writer") id := arrayID + "-" + volumeNameToID[name] - log.Printf("iCallDeleteVolume name %s to ID %s", name, id) + log.Infof("iCallDeleteVolume name %s to ID %s", name, id) req.VolumeId = id f.deleteVolumeResponse, f.err = f.service.DeleteVolume(ctx, req) if f.err != nil { @@ -4883,6 +5421,71 @@ func (f *feature) iCallIsNFSEnabled(systemID string) error { return nil } +func (f *feature) iCallgetArrayVersion(systemID string) error { + ctx := context.Background() + f.checkGoRoutines("start aVxFlexOSService") + + // Save off the admin client and the system + if f.service != nil { + adminClient := f.service.adminClients[systemID] + if adminClient != nil { + f.adminClient = adminClient + f.adminClient.SetToken("xxxx") + } + + system := f.service.systems[systemID] + if system != nil { + f.system = system + } + } + + ver, err := f.service.getArrayVersion(ctx, systemID) + f.err = err + fmt.Printf("Array Version: %v\n", ver) + return nil +} + +func (f *feature) iCallSetupNVMeHostWithNVMeInitiators(initiatorsStr string) error { + var nvmeInitiators []string + if initiatorsStr != "" { + nvmeInitiators = strings.Split(initiatorsStr, ",") + for i := range nvmeInitiators { + nvmeInitiators[i] = strings.TrimSpace(nvmeInitiators[i]) + } + } + + f.checkGoRoutines("start aVxFlexOSService") + + // Save off the admin client and the system + if f.service != nil { + adminClient := f.service.adminClients[arrayID] + if adminClient != nil { + f.adminClient = adminClient + f.adminClient.SetToken("xxxx") + } + + system := f.service.systems[arrayID] + if system != nil { + f.system = system + } + } + + // Setup array config + f.service.opts.arrays = map[string]*ArrayConnectionData{ + arrayID: { + SystemID: arrayID, + BlockProtocol: NVMeTCP, + }, + } + + f.service.opts.KubeNodeName = "node2" + clientSet := fake.NewSimpleClientset() + K8sClientset = clientSet + f.CreateKubernetesNode(K8sClientset, "node2") + f.err = f.service.setupNVMeHost(nvmeInitiators, arrayID) + return nil +} + func getZoneEnabledRequest(zoneLabelName string) *csi.CreateVolumeRequest { req := new(csi.CreateVolumeRequest) params := make(map[string]string) @@ -4921,11 +5524,11 @@ func (f *feature) iCallCreateVolumeWithZones(name string) error { f.createVolumeResponse, f.err = f.service.CreateVolume(ctx, req) if f.err != nil { - log.Printf("CreateVolume with zones called failed: %s\n", f.err.Error()) + log.Infof("CreateVolume with zones called failed: %s\n", f.err.Error()) } if f.createVolumeResponse != nil { - log.Printf("vol id %s\n", f.createVolumeResponse.GetVolume().VolumeId) + log.Infof("vol id %s\n", f.createVolumeResponse.GetVolume().VolumeId) } return nil } @@ -4956,7 +5559,7 @@ func (f *feature) aValidNodeGetInfoIsReturnedWithNodeTopology() error { func (f *feature) aNodeGetInfoIsReturnedWithoutZoneTopology() error { accessibility := f.nodeGetInfoResponse.GetAccessibleTopology() - Log.Printf("Node Accessibility %+v", accessibility) + log.Infof("Node Accessibility %+v", accessibility) if _, ok := accessibility.Segments[f.service.opts.zoneLabelKey]; ok { return fmt.Errorf("zone found") } @@ -4965,7 +5568,7 @@ func (f *feature) aNodeGetInfoIsReturnedWithoutZoneTopology() error { func (f *feature) aNodeGetInfoIsReturnedWithoutZoneSystemTopology() error { accessibility := f.nodeGetInfoResponse.GetAccessibleTopology() - Log.Printf("Node Accessibility %+v", accessibility) + log.Infof("Node Accessibility %+v", accessibility) for _, array := range f.service.opts.arrays { if _, ok := accessibility.Segments[Name+"/"+array.SystemID]; ok { @@ -4975,10 +5578,29 @@ func (f *feature) aNodeGetInfoIsReturnedWithoutZoneSystemTopology() error { return nil } +func (f *feature) iSetProtocolTo(protocol string) error { + f.protocol = protocol + f.service.useNVME = false + f.service.useSDC = false + switch protocol { + case SDC: + f.service.useSDC = true + case NVMeTCP: + f.service.opts.SdcGUID = "" + f.service.useNVME = true + default: + fmt.Printf("unknown protocol %s", protocol) + } + return nil +} + func FeatureContext(s *godog.ScenarioContext) { f := &feature{} s.Step(`^a VxFlexOS service$`, f.aVxFlexOSService) + s.Step(`^I set Platform Info "([^"]*)" "([^"]*)" "([^"]*)" "([^"]*)"$`, f.setPlatformInfo) + s.Step(`^I reset the Platform Info$`, f.resetPlatformInfo) + s.Step(`^I set Provision to "([^"]*)"$`, f.iSetProvisionTo) s.Step(`^a VxFlexOS service with timeout (\d+) milliseconds$`, f.aVxFlexOSServiceWithTimeoutMilliseconds) s.Step(`^I call GetPluginInfo$`, f.iCallGetPluginInfo) s.Step(`^I call DynamicArrayChange$`, f.iCallDynamicArrayChange) @@ -5002,11 +5624,12 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^there is a Node Probe SdcGUID error$`, f.thereIsANodeProbeSdcGUIDError) s.Step(`^there is a Node Probe drvCfg error$`, f.thereIsANodeProbeDrvCfgError) s.Step(`^I call CreateVolume "([^"]*)"$`, f.iCallCreateVolume) + s.Step(`^I call CreateVolume "([^"]*)" with Error "([^"]*)"$`, f.iCallCreateVolumeWithError) s.Step(`^I call ValidateConnectivity$`, f.iCallValidateVolumeHostConnectivity) s.Step(`^a valid CreateVolumeResponse is returned$`, f.aValidCreateVolumeResponseIsReturned) s.Step(`^I specify AccessibilityRequirements with a SystemID of "([^"]*)"$`, f.iSpecifyAccessibilityRequirementsWithASystemIDOf) s.Step(`^I specify NFS AccessibilityRequirements with a SystemID of "([^"]*)"$`, f.iSpecifyAccessibilityRequirementsNFSWithASystemIDOf) - s.Step(`^I specify bad NFS AccessibilityRequirements with a SystemID of "([^"]*)"$`, f.iSpecifyBadAccessibilityRequirementsNFSWithASystemIDOf) + s.Step(`^I specify bad AccessibilityRequirements with a SystemID of "([^"]*)"$`, f.iSpecifyBadAccessibilityRequirementsWithASystemIDOf) s.Step(`^a valid CreateVolumeResponse with topology is returned$`, f.aValidCreateVolumeResponseWithTopologyIsReturned) s.Step(`^I specify MULTINODE_WRITER$`, f.iSpecifyMULTINODEWRITER) s.Step(`^I specify a BadCapacity$`, f.iSpecifyABadCapacity) @@ -5038,12 +5661,15 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^the number of SDC mappings is (\d+)$`, f.theNumberOfSDCMappingsIs) s.Step(`^I call NodeGetInfo$`, f.iCallNodeGetInfo) s.Step(`^I call Node Probe$`, f.iCallNodeProbe) + s.Step(`^I call Node Probe with NVMe flag enabled$`, f.iCallNodeProbeNVMeFlagEnabled) s.Step(`^a valid NodeGetInfoResponse is returned$`, f.aValidNodeGetInfoResponseIsReturned) s.Step(`^a valid NodeGetInfoResponse with node UID is returned$`, f.aValidNodeGetInfoResponseWithNodeUIDIsReturned) + s.Step(`^a valid NodeGetInfoResponse with "([^"]*)" in AccessibleTopology is returned$`, f.aValidNodeGetInfoResponseWithTopologyIsReturned) s.Step(`^the Volume limit is set$`, f.theVolumeLimitIsSet) s.Step(`^an invalid MaxVolumesPerNode$`, f.anInvalidMaxVolumesPerNode) s.Step(`^I call GetNodeLabels$`, f.iCallGetNodeLabels) s.Step(`^I call NodeGetInfo with a valid Node UID$`, f.iCallNodeGetInfoWithValidNodeUID) + s.Step(`^I call NodeGetInfo with useNVME flag as true$`, f.iCallNodeGetInfoWithNVMeFlagTrue) s.Step(`^a valid label is returned$`, f.aValidLabelIsReturned) s.Step(`^I set invalid EnvMaxVolumesPerNode$`, f.iSetInvalidEnvMaxVolumesPerNode) s.Step(`^I call GetNodeLabels with invalid node$`, f.iCallGetNodeLabelsWithInvalidNode) @@ -5076,6 +5702,7 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^a controller published volume$`, f.aControllerPublishedVolume) s.Step(`^a controller published volume with the private target equalling the mount path$`, f.aControllerPublishedVolumeWithPrivateTargetEqualMountPath) s.Step(`^I call NodePublishVolume "([^"]*)"$`, f.iCallNodePublishVolume) + s.Step(`^I call NodePublishVolume NVME "([^"]*)"$`, f.iCallNodePublishVolumeNVME) s.Step(`^I call NodePublishVolume NFS "([^"]*)"$`, f.iCallNodePublishVolumeNFS) s.Step(`^I call CleanupPrivateTarget$`, f.iCallCleanupPrivateTarget) s.Step(`^I call removeWithRetry$`, f.iCallRemoveWithRetry) @@ -5087,7 +5714,9 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^I call mountValidateBlockVolCapabilities$`, f.iCallMountValidateVolCapabilities) s.Step(`^I call blockValidateMountVolCapabilities$`, f.iCallBlockValidateVolCapabilities) s.Step(`^I call UnmountAndDeleteTarget$`, f.iCallUnmountAndDeleteTarget) + s.Step(`^get Node Stage Volume Request$`, f.getNodeStageVolumeRequest) s.Step(`^get Node Publish Volume Request$`, f.getNodePublishVolumeRequest) + s.Step(`^get Node Publish Volume Request for NVME "([^"]*)"$`, f.getNodePublishVolumeRequestNVME) s.Step(`^get Node Publish Volume Request NFS$`, f.getNodePublishVolumeRequestNFS) s.Step(`^get Node Publish Ephemeral Volume Request with name "([^"]*)" size "([^"]*)" storagepool "([^"]*)" and systemName "([^"]*)"$`, f.getNodeEphemeralVolumePublishRequest) s.Step(`^I mark request read only$`, f.iMarkRequestReadOnly) @@ -5181,6 +5810,7 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^I set approveSDC with approveSDCEnabled "([^"]*)"`, f.iSetApproveSdcEnabled) s.Step(`^I set RestrictedSDCMode with "([^"]*)"`, f.iSetRestrictedSDCMode) s.Step(`^I call CreateRemoteVolume$`, f.iCallCreateRemoteVolume) + s.Step(`^I call CreateRemoteVolume with Error "([^"]*)"$`, f.iCallCreateRemoteVolumeWithError) s.Step(`^I call DeleteLocalVolume "([^"]*)"$`, f.iCallDeleteLocalVolume) s.Step(`^I call CreateStorageProtectionGroup$`, f.iCallCreateStorageProtectionGroup) s.Step(`^I call CreateStorageProtectionGroup with "([^"]*)", "([^"]*)", "([^"]*)"$`, f.iCallCreateStorageProtectionGroupWith) @@ -5206,8 +5836,7 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^I call GetNodeUID$`, f.iCallGetNodeUID) s.Step(`^a valid node uid is returned$`, f.aValidNodeUIDIsReturned) s.Step(`^I call GetNodeUID with invalid node$`, f.iCallGetNodeUIDWithInvalidNode) - s.Step(`^I call GetNodeUID with unset KubernetesClient$`, f.iCallGetNodeUIDWithUnsetKubernetesClient) - + s.Step(`^I call GetNodeUID without KubeNodeName$`, f.iCallGetNodeUIDWithoutKubeNodeName) s.Step(`^I call CreateVolume "([^"]*)" with zones$`, f.iCallCreateVolumeWithZones) s.Step(`^I call NodeGetInfo with zone labels$`, f.iCallNodeGetInfoWithZoneLabels) s.Step(`^a valid NodeGetInfo is returned with node topology$`, f.aValidNodeGetInfoIsReturnedWithNodeTopology) @@ -5215,7 +5844,9 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^a NodeGetInfo is returned without zone system topology$`, f.aNodeGetInfoIsReturnedWithoutZoneSystemTopology) s.Step(`^I call systemProbeAll in mode "([^"]*)"`, f.iCallSystemProbeAll) s.Step(`^I call BeforeServe but change "([^"]*)" with "([^"]*)"$`, f.iCallBeforeServeButChangeWithValue) - + s.Step(`^I call getArrayVersion on "([^"]*)"`, f.iCallgetArrayVersion) + s.Step(`^I call setupNVMeHost with NVMe intiators "([^"]*)"$`, f.iCallSetupNVMeHostWithNVMeInitiators) + s.Step(`^I set protocol to "([^"]*)"$`, f.iSetProtocolTo) s.Before(func(ctx context.Context, _ *godog.Scenario) (context.Context, error) { // Cleanup test directory before each test if err := os.RemoveAll(testBaseDir); err != nil { diff --git a/service/step_handlers_test.go b/service/step_handlers_test.go index 83b4f0c4..c9179b6a 100644 --- a/service/step_handlers_test.go +++ b/service/step_handlers_test.go @@ -17,7 +17,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "log" "net/http" "os" "path/filepath" @@ -56,8 +55,10 @@ var ( GetSystemLimitError bool GetSdcInstancesError bool MapSdcError bool + MapNVMeError bool ApproveSdcError bool RemoveMappedSdcError bool + RemoveMappedHostError bool SDCLimitsError bool SIOGatewayVolumeNotFoundError bool GetStatisticsError bool @@ -71,6 +72,7 @@ var ( NasServerNotFoundError bool FileInterfaceNotFoundError bool BadVolIDError bool + IncorrectVolID bool NoCsiVolIDError bool WrongVolIDError bool WrongFileSystemIDError bool @@ -112,6 +114,11 @@ var ( UpdateConfigK8sClientError bool UpdateConfigFormatError bool ConfigMapNotFoundError bool + NoNfsServer bool + SdtNotFoundError bool + EmptySdtError bool + NvmeDiscoveryError bool + GetMetricsError bool } ) @@ -121,6 +128,7 @@ var ( scaleioRouter http.Handler testControllerHasNoConnection bool count int + apiVersion string ) var inducedError error @@ -136,12 +144,12 @@ const ( func getHandler() http.Handler { handler := http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - log.Printf("handler called: %s %s", r.Method, r.URL) + log.Infof("handler called: %s %s", r.Method, r.URL) if scaleioRouter == nil { getRouter().ServeHTTP(w, r) } }) - log.Printf("Clearing volume caches\n") + log.Infof("Clearing volume caches\n") volumeIDToName = make(map[string]string) fileSystemIDName = make(map[string]string) fileSystemIDToSizeTotal = make(map[string]string) @@ -181,7 +189,9 @@ func getHandler() http.Handler { stepHandlersErrors.PodmonVolumeError = false stepHandlersErrors.GetSdcInstancesError = false stepHandlersErrors.MapSdcError = false + stepHandlersErrors.MapNVMeError = false stepHandlersErrors.RemoveMappedSdcError = false + stepHandlersErrors.RemoveMappedHostError = false stepHandlersErrors.SDCLimitsError = false stepHandlersErrors.GetStatisticsError = false stepHandlersErrors.GetSystemSdcError = false @@ -195,6 +205,7 @@ func getHandler() http.Handler { stepHandlersErrors.NasServerNotFoundError = false stepHandlersErrors.BadCapacityError = false stepHandlersErrors.BadVolIDError = false + stepHandlersErrors.IncorrectVolID = false stepHandlersErrors.GetFileSystemsByIDError = false stepHandlersErrors.NoCsiVolIDError = false stepHandlersErrors.WrongVolIDError = false @@ -238,6 +249,11 @@ func getHandler() http.Handler { stepHandlersErrors.UpdateConfigK8sClientError = false stepHandlersErrors.UpdateConfigFormatError = false stepHandlersErrors.ConfigMapNotFoundError = false + stepHandlersErrors.NoNfsServer = false + stepHandlersErrors.SdtNotFoundError = false + stepHandlersErrors.EmptySdtError = false + stepHandlersErrors.NvmeDiscoveryError = false + stepHandlersErrors.GetMetricsError = false sdcMappings = sdcMappings[:0] sdcMappingsID = "" return handler @@ -252,6 +268,7 @@ func getRouter() http.Handler { scaleioRouter.HandleFunc("/api/login", handleLogin) scaleioRouter.HandleFunc("/api/version", handleVersion) scaleioRouter.HandleFunc("/api/types/System/instances", handleSystemInstances) + scaleioRouter.HandleFunc("/api/types/Host/instances", handleHostInstances) scaleioRouter.HandleFunc("/rest/v1/nas-servers", handleNasInstances) scaleioRouter.HandleFunc("/rest/v1/nas-servers/{id}", handleGetNasInstances) scaleioRouter.HandleFunc("/rest/v1/file-systems", handleFileSystems) @@ -273,6 +290,8 @@ func getRouter() http.Handler { scaleioRouter.HandleFunc("/rest/v1/file-tree-quotas/{id}", handleGetFileTreeQuotas) scaleioRouter.HandleFunc("/api/instances/System/action/querySystemLimits", handleGetSystemLimits) scaleioRouter.HandleFunc("/rest/v1/nfs-servers", handleIsNFSEnabled) + scaleioRouter.HandleFunc("/api/types/Sdt/instances", handleGetAllSdt) + scaleioRouter.HandleFunc("/dtapi/rest/v1/metrics/query", handleMetricsQuery) return scaleioRouter } @@ -323,7 +342,9 @@ func handleVersion(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusRequestTimeout) return } - w.Write([]byte("4.0")) + + log.Infof("Mock Api Version: %s", apiVersion) + w.Write([]byte(apiVersion)) } // handleSystemInstances implements GET /api/types/System/instances @@ -352,6 +373,11 @@ func handleSystemInstances(w http.ResponseWriter, _ *http.Request) { } } +// handleHostInstances implements POST /api/types/Host/instances +func handleHostInstances(w http.ResponseWriter, _ *http.Request) { + returnJSONFile("features", "create_nvme_host.json", w, nil) +} + func handleNFSSnapshots(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodPost: @@ -366,7 +392,7 @@ func handleNFSSnapshots(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } resp := types.CreateFileSystemSnapshotResponse{} resp.ID = hex.EncodeToString([]byte(req.Name)) @@ -385,15 +411,15 @@ func handleNFSSnapshots(w http.ResponseWriter, r *http.Request) { array.fileSystems[resp.ID]["size_total"] = sizeTotal } if debug { - log.Printf("request name: %s id: %s\n", req.Name, resp.ID) + log.Infof("request name: %s id: %s\n", req.Name, resp.ID) } encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } - log.Printf("end make fileSystemSnapshots") + log.Infof("end make fileSystemSnapshots") } } @@ -409,9 +435,23 @@ func handleNasInstances(w http.ResponseWriter, _ *http.Request) { // handleIsNFSEnabled implements GET rest/v1/nfs-servers?select=* func handleIsNFSEnabled(w http.ResponseWriter, _ *http.Request) { + if stepHandlersErrors.NoNfsServer { + return + } returnJSONFile("features", "get_nfs_server.json", w, nil) } +// handleGetAllSdt implements GET /api/types/Sdt/instances +func handleGetAllSdt(w http.ResponseWriter, _ *http.Request) { + if stepHandlersErrors.SdtNotFoundError { + writeError(w, "SDT not found", http.StatusNotFound, codes.NotFound) + return + } else if stepHandlersErrors.EmptySdtError { + return + } + returnJSONFile("features", "get_all_sdt.json", w, nil) +} + func handleGetNasInstances(w http.ResponseWriter, _ *http.Request) { if stepHandlersErrors.NasServerNotFoundError { writeError(w, "nas server not found", http.StatusNotFound, codes.NotFound) @@ -449,7 +489,7 @@ func handleNFSExports(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } // good response @@ -486,15 +526,15 @@ func handleNFSExports(w http.ResponseWriter, r *http.Request) { } if debug { - log.Printf("request name: %s id: %s\n", req.Name, resp.ID) + log.Infof("request name: %s id: %s\n", req.Name, resp.ID) } encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } - log.Printf("end make nfsExports") + log.Infof("end make nfsExports") // Read all the Volumes case http.MethodGet: if stepHandlersErrors.NFSExportInstancesError { @@ -522,7 +562,7 @@ func handleNFSExports(w http.ResponseWriter, r *http.Request) { nfsExp := new(types.NFSExport) err := json.Unmarshal(data, nfsExp) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, nfsExp) } @@ -558,7 +598,7 @@ func handleNFSExports(w http.ResponseWriter, r *http.Request) { nfsExp := new(types.NFSExport) err := json.Unmarshal(data, nfsExp) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, nfsExp) } @@ -566,7 +606,7 @@ func handleNFSExports(w http.ResponseWriter, r *http.Request) { encoder := json.NewEncoder(w) err := encoder.Encode(instances) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } } } @@ -588,7 +628,7 @@ func handleGetNFSExports(w http.ResponseWriter, r *http.Request) { // Insert to map if it doesn't exist. if nfsExportIDName[id] == "" { - log.Printf("Did not find id %s \n", id) + log.Infof("Did not find id %s \n", id) writeError(w, "could not find nfsExport ", http.StatusNotFound, codes.NotFound) return } @@ -599,7 +639,7 @@ func handleGetNFSExports(w http.ResponseWriter, r *http.Request) { nfsExp = array.nfsExports[id] } - log.Printf("Get id %s\n", id) + log.Infof("Get id %s\n", id) if nfsExp != nil { replacementMap["__ID__"] = nfsExp["id"] replacementMap["__NAME__"] = nfsExp["name"] @@ -642,13 +682,13 @@ func handleGetNFSExports(w http.ResponseWriter, r *http.Request) { nfsExp1 := new(types.NFSExport) err := json.Unmarshal(data, nfsExp1) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } encoder := json.NewEncoder(w) err = encoder.Encode(nfsExp1) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } case http.MethodDelete: vars := mux.Vars(r) @@ -656,7 +696,7 @@ func handleGetNFSExports(w http.ResponseWriter, r *http.Request) { // Insert to map if it doesn't exist. if nfsExportIDName[id] == "" { - log.Printf("Did not find id %s \n", id) + log.Infof("Did not find id %s \n", id) writeError(w, "could not find nfsExport ", http.StatusNotFound, codes.NotFound) return } @@ -678,7 +718,7 @@ func handleGetNFSExports(w http.ResponseWriter, r *http.Request) { fmt.Println("fsidname", nfsExportIDName[id]) if nfsExportIDName[id] == "" { - log.Printf("Did not find id %s \n", id) + log.Infof("Did not find id %s \n", id) writeError(w, "could not find nfsExport ", http.StatusNotFound, codes.NotFound) return } @@ -692,7 +732,7 @@ func handleGetNFSExports(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } fmt.Printf("patchReq:%#v\n", req) if len(req.AddReadOnlyRootHosts) != 0 { @@ -752,7 +792,7 @@ func handleFileSystems(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } // good response @@ -772,15 +812,15 @@ func handleFileSystems(w http.ResponseWriter, r *http.Request) { } if debug { - log.Printf("request name: %s id: %s\n", req.Name, resp.ID) + log.Infof("request name: %s id: %s\n", req.Name, resp.ID) } encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } - log.Printf("end make fileSystems") + log.Infof("end make fileSystems") // Read all the Volumes case http.MethodGet: instances := make([]*types.FileSystem, 0) @@ -800,7 +840,7 @@ func handleFileSystems(w http.ResponseWriter, r *http.Request) { fs := new(types.FileSystem) err := json.Unmarshal(data, fs) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, fs) } @@ -823,7 +863,7 @@ func handleFileSystems(w http.ResponseWriter, r *http.Request) { fs := new(types.FileSystem) err := json.Unmarshal(data, fs) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, fs) } @@ -831,7 +871,7 @@ func handleFileSystems(w http.ResponseWriter, r *http.Request) { encoder := json.NewEncoder(w) err := encoder.Encode(instances) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } } } @@ -849,22 +889,22 @@ func handleRestoreSnapshotNFS(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } // good response resp := new(types.RestoreFsSnapResponse) resp.ID = req.SnapshotID if debug { - log.Printf("response id: %s\n", resp.ID) + log.Infof("response id: %s\n", resp.ID) } encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } - log.Printf("end make restore fs from snaspshot") + log.Infof("end make restore fs from snaspshot") } } @@ -883,7 +923,7 @@ func handleGetFileSystems(w http.ResponseWriter, r *http.Request) { // Insert to map if it doesn't exist. if fileSystemIDName[id] == "" { - log.Printf("Did not find id %s \n", id) + log.Infof("Did not find id %s \n", id) writeError(w, "could not find filesystem ", http.StatusNotFound, codes.NotFound) return } @@ -894,7 +934,7 @@ func handleGetFileSystems(w http.ResponseWriter, r *http.Request) { fs = array.fileSystems[id] } - log.Printf("Get id %s\n", id) + log.Infof("Get id %s\n", id) if fs != nil { replacementMap["__ID__"] = fs["id"] replacementMap["__NAME__"] = fs["name"] @@ -925,13 +965,13 @@ func handleGetFileSystems(w http.ResponseWriter, r *http.Request) { fs1 := new(types.FileSystem) err := json.Unmarshal(data, fs1) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } encoder := json.NewEncoder(w) err = encoder.Encode(fs1) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } case http.MethodDelete: if inducedError.Error() == "DeleteSnapshotError" { @@ -943,7 +983,7 @@ func handleGetFileSystems(w http.ResponseWriter, r *http.Request) { // Insert to map if it doesn't exist. if fileSystemIDName[id] == "" { - log.Printf("Did not find id %s \n", id) + log.Infof("Did not find id %s \n", id) writeError(w, "could not find filesystem ", http.StatusNotFound, codes.NotFound) return } @@ -962,7 +1002,7 @@ func handleGetFileSystems(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } if inducedError.Error() == "ModifyFSError" { writeError(w, "Modify filesystem failed with error:", http.StatusRequestTimeout, codes.Internal) @@ -983,7 +1023,7 @@ func handleGetFileSystems(w http.ResponseWriter, r *http.Request) { array.fileSystems[id]["graceperiod"] = strconv.Itoa(req.GracePeriod) } w.WriteHeader(http.StatusNoContent) - log.Printf("end modify file systems") + log.Infof("end modify file systems") } // returnJSONFile("features", "get_file_system_response.json", w, nil) @@ -998,6 +1038,15 @@ func handleStoragePoolInstances(w http.ResponseWriter, _ *http.Request) { returnJSONFile("features", "get_storage_pool_instances.json", w, nil) } +// handleMetricsQuery implement POST /dtapi/rest/v1/metrics/query +func handleMetricsQuery(w http.ResponseWriter, _ *http.Request) { + if stepHandlersErrors.GetMetricsError { + writeError(w, "induced error", http.StatusRequestTimeout, codes.Internal) + return + } + returnJSONFile("features", "get_volume_metrics.json", w, nil) +} + func handlePeerMdmInstances(w http.ResponseWriter, _ *http.Request) { if inducedError.Error() == "PeerMdmError" { writeError(w, "PeerMdmError", http.StatusRequestTimeout, codes.Internal) @@ -1009,7 +1058,7 @@ func handlePeerMdmInstances(w http.ResponseWriter, _ *http.Request) { func returnJSONFile(directory, filename string, w http.ResponseWriter, replacements map[string]string) (jsonBytes []byte) { jsonBytes, err := os.ReadFile(filepath.Join(directory, filename)) if err != nil { - log.Printf("Couldn't read %s/%s\n", directory, filename) + log.Infof("Couldn't read %s/%s\n", directory, filename) if w != nil { w.WriteHeader(http.StatusNotFound) } @@ -1037,17 +1086,17 @@ func returnJSONFile(directory, filename string, w http.ResponseWriter, replaceme } if debug { - log.Printf("Edited payload:\n%s\n", jsonString) + log.Infof("Edited payload:\n%s\n", jsonString) } jsonBytes = []byte(jsonString) } if debug { - log.Printf("jsonBytes:\n%s\n", jsonBytes) + log.Infof("jsonBytes:\n%s\n", jsonBytes) } if w != nil { _, err = w.Write(jsonBytes) if err != nil { - log.Printf("Couldn't write to ResponseWriter") + log.Infof("Couldn't write to ResponseWriter") w.WriteHeader(http.StatusInternalServerError) return make([]byte, 0) } @@ -1187,12 +1236,12 @@ func handleVolumeInstances(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } if volumeNameToID[req.Name] != "" { w.WriteHeader(http.StatusInternalServerError) // duplicate volume name response - log.Printf("request for volume creation of duplicate name: %s\n", req.Name) + log.Infof("request for volume creation of duplicate name: %s\n", req.Name) resp := new(types.Error) resp.Message = sioGatewayVolumeNameInUse resp.HTTPStatusCode = http.StatusInternalServerError @@ -1200,7 +1249,7 @@ func handleVolumeInstances(w http.ResponseWriter, r *http.Request) { encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } return } @@ -1226,15 +1275,15 @@ func handleVolumeInstances(w http.ResponseWriter, r *http.Request) { } if debug { - log.Printf("request name: %s id: %s\n", req.Name, resp.ID) + log.Infof("request name: %s id: %s\n", req.Name, resp.ID) } encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } - log.Printf("end make volumes") + log.Infof("end make volumes") // Read all the Volumes case http.MethodGet: instances := make([]*types.Volume, 0) @@ -1256,7 +1305,7 @@ func handleVolumeInstances(w http.ResponseWriter, r *http.Request) { vol := new(types.Volume) err := json.Unmarshal(data, vol) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, vol) } @@ -1281,7 +1330,7 @@ func handleVolumeInstances(w http.ResponseWriter, r *http.Request) { vol := new(types.Volume) err := json.Unmarshal(data, vol) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, vol) } @@ -1289,7 +1338,7 @@ func handleVolumeInstances(w http.ResponseWriter, r *http.Request) { encoder := json.NewEncoder(w) err := encoder.Encode(instances) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } } } @@ -1299,7 +1348,7 @@ func handleAction(w http.ResponseWriter, r *http.Request) { from := vars["from"] id := vars["id"] action := vars["action"] - log.Printf("action from %s id %s action %s", from, id, action) + log.Infof("action from %s id %s action %s", from, id, action) switch action { case "setSdcName": if stepHandlersErrors.SetSdcNameError { @@ -1310,7 +1359,7 @@ func handleAction(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } fmt.Printf("SdcName: %s\n", req.SdcName) sdcIDToName = make(map[string]string, 0) @@ -1333,13 +1382,13 @@ func handleAction(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } resp := types.ApproveSdcResponse{SdcID: "d0f055a700000000"} encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } case "addMappedSdc": @@ -1351,15 +1400,32 @@ func handleAction(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } fmt.Printf("SdcID: %s\n", req.SdcID) if req.SdcID == "d0f055a700000000" { - sdcMappings = append(sdcMappings, types.MappedSdcInfo{SdcID: req.SdcID, SdcIP: "127.1.1.11"}) + sdcMappings = append(sdcMappings, types.MappedSdcInfo{SdcID: req.SdcID, SdcIP: "127.1.1.11", HostType: "SdcHost"}) } fmt.Printf("SdcID: %s\n", req.SdcID) if req.SdcID == "d0f055aa00000001" { - sdcMappings = append(sdcMappings, types.MappedSdcInfo{SdcID: req.SdcID, SdcIP: "127.1.1.10"}) + sdcMappings = append(sdcMappings, types.MappedSdcInfo{SdcID: req.SdcID, SdcIP: "127.1.1.10", HostType: "SdcHost"}) + } + case "addMappedHost": + if stepHandlersErrors.MapNVMeError { + writeError(w, "induced error", http.StatusRequestTimeout, codes.Internal) + return + } + req := types.MapVolumeNVMeParam{} + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(&req) + if err != nil { + log.Infof("error decoding json: %s\n", err.Error()) + } + fmt.Printf("HostID: %s\n", req.HostID) + if req.HostID == goodSdcIDNVMe { + sdcMappings = append(sdcMappings, types.MappedSdcInfo{SdcID: req.HostID, SdcName: goodNodeIDNVMe, HostType: "NVMeHost"}) + } else if req.HostID == altSdcIDNVMe { + sdcMappings = append(sdcMappings, types.MappedSdcInfo{SdcID: req.HostID, SdcName: altNodeIDNVMe, HostType: "NVMeHost"}) } case "removeMappedSdc": if stepHandlersErrors.RemoveMappedSdcError { @@ -1370,7 +1436,7 @@ func handleAction(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } for i, val := range sdcMappings { if val.SdcID == req.SdcID { @@ -1378,6 +1444,23 @@ func handleAction(w http.ResponseWriter, r *http.Request) { sdcMappings = sdcMappings[:len(sdcMappings)-1] } } + case "removeMappedHost": + if stepHandlersErrors.RemoveMappedHostError { + writeError(w, "induced error", http.StatusRequestTimeout, codes.Internal) + return + } + req := types.UnmapVolumeNVMeParam{} + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(&req) + if err != nil { + log.Infof("error decoding json: %s\n", err.Error()) + } + for i, val := range sdcMappings { + if val.SdcID == req.HostID { + copy(sdcMappings[i:], sdcMappings[i+1:]) + sdcMappings = sdcMappings[:len(sdcMappings)-1] + } + } case "setMappedSdcLimits": if stepHandlersErrors.SDCLimitsError { writeError(w, "induced error", http.StatusRequestTimeout, codes.Internal) @@ -1387,7 +1470,7 @@ func handleAction(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } fmt.Printf("SdcID: %s\n", req.SdcID) if req.SdcID == "d0f055a700000000" { @@ -1409,7 +1492,7 @@ func handleAction(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } for _, snapParam := range req.SnapshotDefs { // For now, only a single snapshot ID is supported @@ -1550,7 +1633,7 @@ func handleAction(w http.ResponseWriter, r *http.Request) { encoder := json.NewEncoder(w) err := encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } case "switchoverReplicationConsistencyGroup": fallthrough @@ -1582,7 +1665,7 @@ func getSdcMappings(volumeID string) string { bytes, err = json.Marshal(&emptyMappings) } if err != nil { - log.Printf("Json marshalling error: %s", err.Error()) + log.Infof("Json marshalling error: %s", err.Error()) return "" } if debug { @@ -1596,7 +1679,7 @@ func handleRelationships(w http.ResponseWriter, r *http.Request) { from := vars["from"] id := vars["id"] to := vars["to"] - log.Printf("relationship from %s id %s to %s", from, id, to) + log.Infof("relationship from %s id %s to %s", from, id, to) switch to { case "Sdc": if stepHandlersErrors.GetSdcInstancesError { @@ -1622,14 +1705,14 @@ func handleRelationships(w http.ResponseWriter, r *http.Request) { sdc := new(types.Sdc) err := json.Unmarshal(data, sdc) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, sdc) } encoder := json.NewEncoder(w) err := encoder.Encode(instances) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } setSdcNameSuccess = false return @@ -1682,16 +1765,16 @@ func handleRelationships(w http.ResponseWriter, r *http.Request) { pair := new(types.ReplicationPair) err := json.Unmarshal(data, pair) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } - log.Printf("pair +%v", pair) + log.Infof("pair +%v", pair) instances = append(instances, pair) } encoder := json.NewEncoder(w) err := encoder.Encode(instances) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } default: writeError(w, "Unsupported relationship to type", http.StatusRequestTimeout, codes.Internal) @@ -1724,7 +1807,7 @@ func handleInstances(w http.ResponseWriter, r *http.Request) { id := vars["id"] id = extractIDFromStruct(id) if true { - log.Printf("handle instances type %s id %s\n", objType, id) + log.Infof("handle instances type %s id %s\n", objType, id) } switch objType { case "Volume": @@ -1742,7 +1825,7 @@ func handleInstances(w http.ResponseWriter, r *http.Request) { vol = array.volumes[id] } - log.Printf("Get id %s for %s\n", id, objType) + log.Infof("Get id %s for %s\n", id, objType) if vol != nil { replacementMap["__ID__"] = vol["id"] replacementMap["__NAME__"] = vol["name"] @@ -1762,7 +1845,7 @@ func handleInstances(w http.ResponseWriter, r *http.Request) { } returnJSONFile("features", "volume.json.template", w, replacementMap) } else { - log.Printf("Did not find id %s for %s\n", id, objType) + log.Infof("Did not find id %s for %s\n", id, objType) writeError(w, "volume not found: "+id, http.StatusNotFound, codes.NotFound) } @@ -1826,12 +1909,12 @@ func handleQueryVolumeIDByKey(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } if volumeNameToID[req.Name] != "" { resp := new(types.VolumeResp) resp.ID = volumeNameToID[req.Name] - log.Printf("found volume %s id %s\n", req.Name, volumeNameToID[req.Name]) + log.Infof("found volume %s id %s\n", req.Name, volumeNameToID[req.Name]) encoder := json.NewEncoder(w) if stepHandlersErrors.BadVolIDJSON { err = encoder.Encode("thisWill://causeUnmarshalErr") @@ -1839,10 +1922,10 @@ func handleQueryVolumeIDByKey(w http.ResponseWriter, r *http.Request) { err = encoder.Encode(resp.ID) } if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } } else { - log.Printf("did not find volume %s\n", req.Name) + log.Infof("did not find volume %s\n", req.Name) volumeNameToID[req.Name] = "" writeError(w, fmt.Sprintf("Volume not found %s", req.Name), http.StatusNotFound, codes.NotFound) @@ -1861,14 +1944,14 @@ func handleReplicationConsistencyGroupInstances(w http.ResponseWriter, r *http.R decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } fmt.Printf("POST to ReplicationConsistencyGroup %s\n", req.Name) for _, ctx := range systemArrays[r.Host].replicationConsistencyGroups { if ctx["name"] == req.Name { w.WriteHeader(http.StatusInternalServerError) - log.Printf("request for rcg creation of duplicate name: %s\n", req.Name) + log.Infof("request for rcg creation of duplicate name: %s\n", req.Name) resp := types.Error{ Message: "The Replication Consistency Group already exists", HTTPStatusCode: http.StatusInternalServerError, ErrorCode: 6, @@ -1876,7 +1959,7 @@ func handleReplicationConsistencyGroupInstances(w http.ResponseWriter, r *http.R encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } return } @@ -1912,7 +1995,7 @@ func handleReplicationConsistencyGroupInstances(w http.ResponseWriter, r *http.R array.replicationConsistencyGroups[remoteRCGID]["replicationDirection"] = "RemoteToLocal" if debug { - log.Printf("request name: %s id: %s\n", req.Name, resp.ID) + log.Infof("request name: %s id: %s\n", req.Name, resp.ID) } if inducedError.Error() == "StorageGroupAlreadyExists" || inducedError.Error() == "StorageGroupAlreadyExistsUnretriavable" { @@ -1923,7 +2006,7 @@ func handleReplicationConsistencyGroupInstances(w http.ResponseWriter, r *http.R encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } case http.MethodGet: if inducedError.Error() == "GetReplicationConsistencyGroupsError" { @@ -1957,7 +2040,7 @@ func handleReplicationConsistencyGroupInstances(w http.ResponseWriter, r *http.R rcg := new(types.ReplicationConsistencyGroup) err := json.Unmarshal(data, rcg) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, rcg) @@ -1966,7 +2049,7 @@ func handleReplicationConsistencyGroupInstances(w http.ResponseWriter, r *http.R encoder := json.NewEncoder(w) err := encoder.Encode(instances) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } } @@ -1983,13 +2066,13 @@ func handleReplicationPairInstances(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } fmt.Printf("POST to ReplicationPair %s Request %+v\n", req.Name, req) for _, ctx := range systemArrays[r.Host].replicationPairs { if ctx["name"] == req.Name { w.WriteHeader(http.StatusInternalServerError) - log.Printf("request for replication pair creation of duplicate name: %s\n", req.Name) + log.Infof("request for replication pair creation of duplicate name: %s\n", req.Name) resp := new(types.Error) resp.Message = "Replication Pair name already in use" @@ -1998,7 +2081,7 @@ func handleReplicationPairInstances(w http.ResponseWriter, r *http.Request) { encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } return } @@ -2034,7 +2117,7 @@ func handleReplicationPairInstances(w http.ResponseWriter, r *http.Request) { volumeIDToReplicationState[req.DestinationVolumeID] = "Replicated" if debug { - log.Printf("request name: %s id: %s sourceVolume %s\n", req.Name, resp.ID, req.SourceVolumeID) + log.Infof("request name: %s id: %s sourceVolume %s\n", req.Name, resp.ID, req.SourceVolumeID) } if inducedError.Error() == "ReplicationPairAlreadyExists" || inducedError.Error() == "ReplicationPairAlreadyExistsUnretrievable" { @@ -2045,7 +2128,7 @@ func handleReplicationPairInstances(w http.ResponseWriter, r *http.Request) { encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } case http.MethodGet: if inducedError.Error() == "GetReplicationPairError" { @@ -2066,24 +2149,24 @@ func handleReplicationPairInstances(w http.ResponseWriter, r *http.Request) { replacementMap["__DESTINATION_VOLUME__"] = pair["remoteVolumeId"] replacementMap["__RP_GROUP__"] = pair["replicationConsistencyGroupId"] - log.Printf("replicatPair replacementMap %v\n", replacementMap) + log.Infof("replicatPair replacementMap %v\n", replacementMap) data := returnJSONFile("features", "replication_pair.template", nil, replacementMap) - log.Printf("replication-pair-data %s\n", string(data)) + log.Infof("replication-pair-data %s\n", string(data)) pair := new(types.ReplicationPair) err := json.Unmarshal(data, pair) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } - log.Printf("replication-pair +%v", pair) + log.Infof("replication-pair +%v", pair) instances = append(instances, pair) } encoder := json.NewEncoder(w) err := encoder.Encode(instances) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } } } @@ -2099,7 +2182,7 @@ func handleFileTreeQuotas(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } // good response @@ -2122,14 +2205,14 @@ func handleFileTreeQuotas(w http.ResponseWriter, r *http.Request) { array.treeQuotas[resp.ID]["hardlimit"] = strconv.Itoa(req.HardLimit) } if debug { - log.Printf("request \"dummy-name\" id: %s\n", resp.ID) + log.Infof("request \"dummy-name\" id: %s\n", resp.ID) } encoder := json.NewEncoder(w) err = encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } - log.Printf("end make tree quotas") + log.Infof("end make tree quotas") case http.MethodGet: if inducedError.Error() == "GetQuotaByFSIDError" { writeError(w, "Fetching tree quota for filesystem failed, error:", http.StatusRequestTimeout, codes.Internal) @@ -2152,7 +2235,7 @@ func handleFileTreeQuotas(w http.ResponseWriter, r *http.Request) { tq := new(types.TreeQuota) err := json.Unmarshal(data, tq) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, tq) } @@ -2174,7 +2257,7 @@ func handleFileTreeQuotas(w http.ResponseWriter, r *http.Request) { tq := new(types.TreeQuota) err := json.Unmarshal(data, tq) if err != nil { - log.Printf("error unmarshalling json: %s\n", string(data)) + log.Infof("error unmarshalling json: %s\n", string(data)) } instances = append(instances, tq) } @@ -2182,9 +2265,9 @@ func handleFileTreeQuotas(w http.ResponseWriter, r *http.Request) { encoder := json.NewEncoder(w) err := encoder.Encode(instances) if err != nil { - log.Printf("error encoding json: %s\n", err) + log.Infof("error encoding json: %s\n", err) } - log.Printf("end get tree quotas") + log.Infof("end get tree quotas") } } @@ -2203,7 +2286,7 @@ func handleGetFileTreeQuotas(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) err := decoder.Decode(&req) if err != nil { - log.Printf("error decoding json: %s\n", err.Error()) + log.Infof("error decoding json: %s\n", err.Error()) } fmt.Printf("patchReq:%#v\n", req) if array, ok := systemArrays[r.Host]; ok { @@ -2213,7 +2296,7 @@ func handleGetFileTreeQuotas(w http.ResponseWriter, r *http.Request) { array.treeQuotas[id]["hardlimit"] = strconv.Itoa(req.HardLimit) } w.WriteHeader(http.StatusNoContent) - log.Printf("end modify tree quotas") + log.Infof("end modify tree quotas") } } @@ -2227,6 +2310,6 @@ func writeError(w http.ResponseWriter, message string, httpStatus int, errorCode encoder := json.NewEncoder(w) err := encoder.Encode(resp) if err != nil { - log.Printf("error encoding json: %s\n", err.Error()) + log.Infof("error encoding json: %s\n", err.Error()) } } diff --git a/test/e2e-fsgroup/fs_scaleup_scaledown.go b/test/e2e-fsgroup/fs_scaleup_scaledown.go index 2797dfa2..5d2de64f 100644 --- a/test/e2e-fsgroup/fs_scaleup_scaledown.go +++ b/test/e2e-fsgroup/fs_scaleup_scaledown.go @@ -137,7 +137,7 @@ var _ = ginkgo.Describe("Nodes Scale-up and Scale-down", ginkgo.Label("csi-fsg") gomega.Expect(fss.CheckMount(ctx, client, statefulset, mountPath)).NotTo(gomega.HaveOccurred()) - ssPodsBeforeScaleDown := fss.GetPodList(ctx, client, statefulset) + ssPodsBeforeScaleDown, _ := fss.GetPodList(ctx, client, statefulset) gomega.Expect(ssPodsBeforeScaleDown.Items).NotTo(gomega.BeEmpty(), fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) gomega.Expect(len(ssPodsBeforeScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), @@ -180,7 +180,7 @@ var _ = ginkgo.Describe("Nodes Scale-up and Scale-down", ginkgo.Label("csi-fsg") _, scaledownErr := fss.Scale(ctx, client, statefulset, replicas-1) gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) fss.WaitForStatusReadyReplicas(ctx, client, statefulset, replicas-1) - ssPodsAfterScaleDown := fss.GetPodList(ctx, client, statefulset) + ssPodsAfterScaleDown, _ := fss.GetPodList(ctx, client, statefulset) gomega.Expect(ssPodsAfterScaleDown.Items).NotTo(gomega.BeEmpty(), fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas-1)).To(gomega.BeTrue(), @@ -228,7 +228,7 @@ var _ = ginkgo.Describe("Nodes Scale-up and Scale-down", ginkgo.Label("csi-fsg") fss.WaitForStatusReadyReplicas(ctx, client, statefulset, replicas) fss.WaitForStatusReadyReplicas(ctx, client, statefulset, replicas) - ssPodsAfterScaleUp := fss.GetPodList(ctx, client, statefulset) + ssPodsAfterScaleUp, _ := fss.GetPodList(ctx, client, statefulset) gomega.Expect(ssPodsAfterScaleUp.Items).NotTo(gomega.BeEmpty(), fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) gomega.Expect(len(ssPodsAfterScaleUp.Items) == int(replicas)).To(gomega.BeTrue(), @@ -256,7 +256,7 @@ var _ = ginkgo.Describe("Nodes Scale-up and Scale-down", ginkgo.Label("csi-fsg") _, scaledownErr = fss.Scale(ctx, client, statefulset, replicas) gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) fss.WaitForStatusReplicas(ctx, client, statefulset, replicas) - ssPodsAfterScaleDown = fss.GetPodList(ctx, client, statefulset) + ssPodsAfterScaleDown, _ = fss.GetPodList(ctx, client, statefulset) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") }) diff --git a/test/e2e-fsgroup/go.mod b/test/e2e-fsgroup/go.mod index 62caacb8..e59a22a9 100644 --- a/test/e2e-fsgroup/go.mod +++ b/test/e2e-fsgroup/go.mod @@ -1,169 +1,181 @@ module github.com/dell/csi-powerflex/tests/e2e-fsgroup -go 1.25 +go 1.25.0 require ( - github.com/onsi/ginkgo/v2 v2.21.0 - github.com/onsi/gomega v1.35.1 + github.com/onsi/ginkgo/v2 v2.27.3 + github.com/onsi/gomega v1.38.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.33.4 - k8s.io/apimachinery v0.33.4 - k8s.io/client-go v0.33.4 - k8s.io/kubectl v0.31.1 - k8s.io/kubernetes v1.32.8 - k8s.io/pod-security-admission v0.31.1 + k8s.io/api v0.34.3 + k8s.io/apimachinery v0.34.3 + k8s.io/client-go v0.34.3 + k8s.io/kubectl v0.34.3 + k8s.io/kubernetes v1.34.3 + k8s.io/pod-security-admission v0.34.3 ) require ( - cel.dev/expr v0.19.1 // indirect - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + cel.dev/expr v0.25.1 // indirect + cyphar.com/go-pathrs v0.2.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Microsoft/hnslib v0.1.1 // indirect + github.com/Microsoft/hnslib v0.1.2 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/chai2010/gettext-go v1.0.2 // indirect - github.com/container-storage-interface/spec v1.9.0 // indirect - github.com/containerd/containerd/api v1.9.0 // indirect - github.com/containerd/errdefs v0.1.0 // indirect + github.com/chai2010/gettext-go v1.0.3 // indirect + github.com/container-storage-interface/spec v1.12.0 // indirect + github.com/containerd/containerd/api v1.10.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/ttrpc v1.2.5 // indirect + github.com/containerd/ttrpc v1.2.7 // indirect + github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cyphar/filepath-securejoin v0.3.4 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-errors/errors v1.4.2 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-errors/errors v1.5.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/jsonpointer v0.22.4 // indirect + github.com/go-openapi/jsonreference v0.21.4 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/godbus/dbus/v5 v5.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/cadvisor v0.51.0 // indirect - github.com/google/cel-go v0.23.2 // indirect - github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/cadvisor v0.54.1 // indirect + github.com/google/cel-go v0.26.1 // indirect + github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/pprof v0.0.0-20251208000136-3d256cb9ff16 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/karrick/godirwalk v1.17.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/spdystream v0.5.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/userns v0.1.0 // indirect - github.com/moby/term v0.5.0 // indirect + github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/opencontainers/cgroups v0.0.6 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opencontainers/runc v1.2.1 // indirect - github.com/opencontainers/runtime-spec v1.2.0 // indirect - github.com/opencontainers/selinux v1.11.1 // indirect + github.com/opencontainers/runtime-spec v1.3.0 // indirect + github.com/opencontainers/selinux v1.13.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.4 // indirect + github.com/prometheus/procfs v0.19.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/stoewer/go-strcase v1.3.0 // indirect - github.com/stretchr/objx v0.5.2 // indirect - github.com/stretchr/testify v1.10.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stoewer/go-strcase v1.3.1 // indirect + github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.21 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.21 // indirect - go.etcd.io/etcd/client/v3 v3.5.21 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect - go.opentelemetry.io/otel v1.33.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect - go.opentelemetry.io/otel/metric v1.33.0 // indirect - go.opentelemetry.io/otel/sdk v1.33.0 // indirect - go.opentelemetry.io/otel/trace v1.33.0 // indirect - go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.etcd.io/etcd/api/v3 v3.6.6 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.6 // indirect + go.etcd.io/etcd/client/v3 v3.6.6 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.64.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.64.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/sdk v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect - golang.org/x/time v0.9.0 // indirect - golang.org/x/tools v0.26.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.36.5 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect + golang.org/x/mod v0.31.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/time v0.14.0 // indirect + golang.org/x/tools v0.40.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/grpc v1.77.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/apiextensions-apiserver v0.31.1 // indirect - k8s.io/apiserver v0.33.4 // indirect - k8s.io/cli-runtime v0.31.1 // indirect - k8s.io/cloud-provider v0.31.1 // indirect - k8s.io/component-base v0.33.4 // indirect - k8s.io/component-helpers v0.33.4 // indirect - k8s.io/controller-manager v0.31.1 // indirect - k8s.io/cri-api v0.33.4 // indirect - k8s.io/cri-client v0.31.1 // indirect - k8s.io/csi-translation-lib v0.31.1 // indirect - k8s.io/dynamic-resource-allocation v0.31.1 // indirect + k8s.io/apiextensions-apiserver v0.34.3 // indirect + k8s.io/apiserver v0.34.3 // indirect + k8s.io/cli-runtime v0.34.3 // indirect + k8s.io/cloud-provider v0.34.3 // indirect + k8s.io/component-base v0.34.3 // indirect + k8s.io/component-helpers v0.34.3 // indirect + k8s.io/controller-manager v0.34.3 // indirect + k8s.io/cri-api v0.34.3 // indirect + k8s.io/cri-client v0.34.3 // indirect + k8s.io/csi-translation-lib v0.34.3 // indirect + k8s.io/dynamic-resource-allocation v0.34.3 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kms v0.33.4 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - k8s.io/kube-scheduler v0.31.1 // indirect - k8s.io/kubelet v0.33.4 // indirect - k8s.io/mount-utils v0.31.1 // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/kustomize/api v0.18.0 // indirect - sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect + k8s.io/kms v0.34.3 // indirect + k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect + k8s.io/kube-scheduler v0.34.3 // indirect + k8s.io/kubelet v0.34.3 // indirect + k8s.io/mount-utils v0.34.3 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/kustomize/api v0.21.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.21.0 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/test/e2e-fsgroup/go.sum b/test/e2e-fsgroup/go.sum index a2cc242f..d0c8485c 100644 --- a/test/e2e-fsgroup/go.sum +++ b/test/e2e-fsgroup/go.sum @@ -1,19 +1,23 @@ -cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= -cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cyphar.com/go-pathrs v0.2.2 h1:y9w7hxbkr3zEL78Fjzeg4HEhs2xNy+fbwHiHGJJY2Xo= +cyphar.com/go-pathrs v0.2.2/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Microsoft/hnslib v0.1.1 h1:JsZy681SnvSOUAfCZVAxkX4LgQGp+CZZwPbLV0/pdF8= -github.com/Microsoft/hnslib v0.1.1/go.mod h1:DRQR4IjLae6WHYVhW7uqe44hmFUiNhmaWA+jwMbz5tM= +github.com/Microsoft/hnslib v0.1.2 h1:CshjwTQsNx1o7BIA1XO8HtgDsiCqn+b6kGjL/tIxXQQ= +github.com/Microsoft/hnslib v0.1.2/go.mod h1:5vTyBey4N/VI2ZTNh2gdWhkPMefSbCFYjpvVwye+qtI= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -22,126 +26,155 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= -github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= -github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= -github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= -github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= -github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= -github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= -github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= +github.com/chai2010/gettext-go v1.0.3 h1:9liNh8t+u26xl5ddmWLmsOsdNLwkdRTg5AG+JnTiM80= +github.com/chai2010/gettext-go v1.0.3/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/container-storage-interface/spec v1.12.0 h1:zrFOEqpR5AghNaaDG4qyedwPBqU2fU0dWjLQMP/azK0= +github.com/container-storage-interface/spec v1.12.0/go.mod h1:txsm+MA2B2WDa5kW69jNbqPnvTtfvZma7T/zsAZ9qX8= +github.com/containerd/containerd/api v1.10.0 h1:5n0oHYVBwN4VhoX9fFykCV9dF1/BvAXeg2F8W6UYq1o= +github.com/containerd/containerd/api v1.10.0/go.mod h1:NBm1OAk8ZL+LG8R0ceObGxT5hbUYj7CzTmR3xh0DlMM= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/ttrpc v1.2.5 h1:IFckT1EFQoFBMG4c3sMdT8EP3/aKfumK1msY+Ze4oLU= -github.com/containerd/ttrpc v1.2.5/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= -github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= -github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= +github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= +github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40= +github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= +github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8= -github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU= -github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= +github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= +github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= +github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= +github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= +github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= +github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= +github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= +github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/godbus/dbus/v5 v5.2.0 h1:3WexO+U+yg9T70v9FdHr9kCxYlazaAXUhx2VMkbfax8= +github.com/godbus/dbus/v5 v5.2.0/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= -github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cadvisor v0.51.0 h1:BspqSPdZoLKrnvuZNOvM/KiJ/A+RdixwagN20n+2H8k= -github.com/google/cadvisor v0.51.0/go.mod h1:czGE/c/P/i0QFpVNKTFrIEzord9Y10YfpwuaSWXELc0= -github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= -github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= -github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= -github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/cadvisor v0.54.1 h1:/qyUYrDJTPK1v6T6mB15WzlL0L7YcxZNHKQJTCmNqL8= +github.com/google/cadvisor v0.54.1/go.mod h1:0eXSPm4bB6uzDcoThFk95ADFZnIC+zezAu5pk1bxNJ8= +github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ= +github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= +github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/pprof v0.0.0-20251208000136-3d256cb9ff16 h1:ptucaU8cwiAc+/jqDblz0kb1ECLqPTeX/qQym8OBYzY= +github.com/google/pprof v0.0.0-20251208000136-3d256cb9ff16/go.mod h1:67FPmZWbr+KDT/VlpWtw6sO9XSjpJmLuHpoLmWiTGgY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= -github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= @@ -150,11 +183,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -163,8 +193,10 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= @@ -177,33 +209,34 @@ github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9Kou github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/ginkgo/v2 v2.27.3 h1:ICsZJ8JoYafeXFFlFAG75a7CxMsJHwgKwtO+82SE9L8= +github.com/onsi/ginkgo/v2 v2.27.3/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= +github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= +github.com/opencontainers/cgroups v0.0.6 h1:tfZFWTIIGaUUFImTyuTg+Mr5x8XRiSdZESgEBW7UxuI= +github.com/opencontainers/cgroups v0.0.6/go.mod h1:oWVzJsKK0gG9SCRBfTpnn16WcGEqDI8PAcpMGbqWxcs= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opencontainers/runc v1.2.1 h1:mQkmeFSUxqFaVmvIn1VQPeQIKpHFya5R07aJw0DKQa8= -github.com/opencontainers/runc v1.2.1/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc= -github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= -github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8= -github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= +github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= +github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -211,43 +244,52 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= +github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -258,183 +300,188 @@ github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= -go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.etcd.io/etcd/api/v3 v3.5.21 h1:A6O2/JDb3tvHhiIz3xf9nJ7REHvtEFJJ3veW3FbCnS8= -go.etcd.io/etcd/api/v3 v3.5.21/go.mod h1:c3aH5wcvXv/9dqIw2Y810LDXJfhSYdHQ0vxmP3CCHVY= -go.etcd.io/etcd/client/pkg/v3 v3.5.21 h1:lPBu71Y7osQmzlflM9OfeIV2JlmpBjqBNlLtcoBqUTc= -go.etcd.io/etcd/client/pkg/v3 v3.5.21/go.mod h1:BgqT/IXPjK9NkeSDjbzwsHySX3yIle2+ndz28nVsjUs= -go.etcd.io/etcd/client/v2 v2.305.21 h1:eLiFfexc2mE+pTLz9WwnoEsX5JTTpLCYVivKkmVXIRA= -go.etcd.io/etcd/client/v2 v2.305.21/go.mod h1:OKkn4hlYNf43hpjEM3Ke3aRdUkhSl8xjKjSf8eCq2J8= -go.etcd.io/etcd/client/v3 v3.5.21 h1:T6b1Ow6fNjOLOtM0xSoKNQt1ASPCLWrF9XMHcH9pEyY= -go.etcd.io/etcd/client/v3 v3.5.21/go.mod h1:mFYy67IOqmbRf/kRUvsHixzo3iG+1OF2W2+jVIQRAnU= -go.etcd.io/etcd/pkg/v3 v3.5.21 h1:jUItxeKyrDuVuWhdh0HtjUANwyuzcb7/FAeUfABmQsk= -go.etcd.io/etcd/pkg/v3 v3.5.21/go.mod h1:wpZx8Egv1g4y+N7JAsqi2zoUiBIUWznLjqJbylDjWgU= -go.etcd.io/etcd/raft/v3 v3.5.21 h1:dOmE0mT55dIUsX77TKBLq+RgyumsQuYeiRQnW/ylugk= -go.etcd.io/etcd/raft/v3 v3.5.21/go.mod h1:fmcuY5R2SNkklU4+fKVBQi2biVp5vafMrWUEj4TJ4Cs= -go.etcd.io/etcd/server/v3 v3.5.21 h1:9w0/k12majtgarGmlMVuhwXRI2ob3/d1Ik3X5TKo0yU= -go.etcd.io/etcd/server/v3 v3.5.21/go.mod h1:G1mOzdwuzKT1VRL7SqRchli/qcFrtLBTAQ4lV20sXXo= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 h1:KemlMZlVwBSEGaO91WKgp41BBFsnWqqj9sKRwmOqC40= -go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0/go.mod h1:uq8DrRaen3suIWTpdR/JNHCGpurSvMv9D5Nr5CU5TXc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 h1:PS8wXpbyaDJQ2VDHHncMe9Vct0Zn1fEjpsjrLxGJoSc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0/go.mod h1:HDBUsEjOuRC0EzKZ1bSaRGZWUBAzo+MhAcUUORSr4D0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= -go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= -go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ= -go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= -go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= -go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= -go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= -go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= -go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= -go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= -go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= -go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= -go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= +go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= +go.etcd.io/etcd/api/v3 v3.6.6 h1:mcaMp3+7JawWv69p6QShYWS8cIWUOl32bFLb6qf8pOQ= +go.etcd.io/etcd/api/v3 v3.6.6/go.mod h1:f/om26iXl2wSkcTA1zGQv8reJRSLVdoEBsi4JdfMrx4= +go.etcd.io/etcd/client/pkg/v3 v3.6.6 h1:uoqgzSOv2H9KlIF5O1Lsd8sW+eMLuV6wzE3q5GJGQNs= +go.etcd.io/etcd/client/pkg/v3 v3.6.6/go.mod h1:YngfUVmvsvOJ2rRgStIyHsKtOt9SZI2aBJrZiWJhCbI= +go.etcd.io/etcd/client/v3 v3.6.6 h1:G5z1wMf5B9SNexoxOHUGBaULurOZPIgGPsW6CN492ec= +go.etcd.io/etcd/client/v3 v3.6.6/go.mod h1:36Qv6baQ07znPR3+n7t+Rk5VHEzVYPvFfGmfF4wBHV8= +go.etcd.io/etcd/pkg/v3 v3.6.4 h1:fy8bmXIec1Q35/jRZ0KOes8vuFxbvdN0aAFqmEfJZWA= +go.etcd.io/etcd/pkg/v3 v3.6.4/go.mod h1:kKcYWP8gHuBRcteyv6MXWSN0+bVMnfgqiHueIZnKMtE= +go.etcd.io/etcd/server/v3 v3.6.4 h1:LsCA7CzjVt+8WGrdsnh6RhC0XqCsLkBly3ve5rTxMAU= +go.etcd.io/etcd/server/v3 v3.6.4/go.mod h1:aYCL/h43yiONOv0QIR82kH/2xZ7m+IWYjzRmyQfnCAg= +go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= +go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.64.0 h1:Okc6wG1R1x3z6sFrd40cJM3xvSWH7xoFztPWSgTDGxY= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.64.0/go.mod h1:IiRWCfCLUR/o1rFy+Jl4MUeTf4MZfu8uEIEIKNzK+vE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.64.0 h1:RN3ifU8y4prNWeEnQp2kRRHz8UwonAEYZl8tUzHEXAk= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.64.0/go.mod h1:habDz3tEWiFANTo6oUE99EmaFUrCNYAAg3wiVmusm70= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0/go.mod h1:GQ/474YrbE4Jx8gZ4q5I4hrhUzM6UPzyrqJYV2AqPoQ= +go.opentelemetry.io/contrib/propagators/b3 v1.39.0 h1:PI7pt9pkSnimWcp5sQhUA9OzLbc3Ba4sL+VEUTNsxrk= +go.opentelemetry.io/contrib/propagators/b3 v1.39.0/go.mod h1:5gV/EzPnfYIwjzj+6y8tbGW2PKWhcsz5e/7twptRVQY= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 h1:in9O8ESIOlwJAEGTkkf34DesGRAc/Pn8qJ7k3r/42LM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0/go.mod h1:Rp0EXBm5tfnv0WL+ARyO/PHBEaEAT8UUHQ6AGJcSq6c= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0 h1:8UPA4IbVZxpsD76ihGOQiFml99GPAEZLohDXvqHdi6U= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0/go.mod h1:MZ1T/+51uIVKlRzGw1Fo46KEWThjlCBZKl2LzY5nv4g= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 h1:MDfG8Cvcqlt9XXrmEiD4epKn7VJHZO84hejP9Jmp0MM= +golang.org/x/exp v0.0.0-20251209150349-8475f28825e9/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.33.4 h1:oTzrFVNPXBjMu0IlpA2eDDIU49jsuEorGHB4cvKupkk= -k8s.io/api v0.33.4/go.mod h1:VHQZ4cuxQ9sCUMESJV5+Fe8bGnqAARZ08tSTdHWfeAc= -k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= -k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= -k8s.io/apimachinery v0.33.4 h1:SOf/JW33TP0eppJMkIgQ+L6atlDiP/090oaX0y9pd9s= -k8s.io/apimachinery v0.33.4/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/apiserver v0.33.4 h1:6N0TEVA6kASUS3owYDIFJjUH6lgN8ogQmzZvaFFj1/Y= -k8s.io/apiserver v0.33.4/go.mod h1:8ODgXMnOoSPLMUg1aAzMFx+7wTJM+URil+INjbTZCok= -k8s.io/cli-runtime v0.31.1 h1:/ZmKhmZ6hNqDM+yf9s3Y4KEYakNXUn5sod2LWGGwCuk= -k8s.io/cli-runtime v0.31.1/go.mod h1:pKv1cDIaq7ehWGuXQ+A//1OIF+7DI+xudXtExMCbe9U= -k8s.io/client-go v0.33.4 h1:TNH+CSu8EmXfitntjUPwaKVPN0AYMbc9F1bBS8/ABpw= -k8s.io/client-go v0.33.4/go.mod h1:LsA0+hBG2DPwovjd931L/AoaezMPX9CmBgyVyBZmbCY= -k8s.io/cloud-provider v0.31.1 h1:40b6AgDizwm5eWratZbqubTHMob25VWr6NX2Ei5TwZA= -k8s.io/cloud-provider v0.31.1/go.mod h1:xAdkE7fdZdu9rKLuOZUMBfagu7bM+bas3iPux/2nLGg= -k8s.io/component-base v0.33.4 h1:Jvb/aw/tl3pfgnJ0E0qPuYLT0NwdYs1VXXYQmSuxJGY= -k8s.io/component-base v0.33.4/go.mod h1:567TeSdixWW2Xb1yYUQ7qk5Docp2kNznKL87eygY8Rc= -k8s.io/component-helpers v0.33.4 h1:DYHQPxWB3XIk7hwAQ4YczUelJ37PcUHfnLeee0qFqV8= -k8s.io/component-helpers v0.33.4/go.mod h1:kRgidIgCKFqOW/wy7D8IL3YOT3iaIRZu6FcTEyRr7WU= -k8s.io/controller-manager v0.31.1 h1:bwiy8y//EG5lJL2mdbOvZWrOgw2EXXIvwp95VYgoIis= -k8s.io/controller-manager v0.31.1/go.mod h1:O440MSE6EI1AEVhB2Fc8FYqv6r8BHrSXjm5aj3886No= -k8s.io/cri-api v0.33.4 h1:P49b1XSTqIKu79pTV6Ig+tMM20NupmZ8AVZ9rWSz1VQ= -k8s.io/cri-api v0.33.4/go.mod h1:OLQvT45OpIA+tv91ZrpuFIGY+Y2Ho23poS7n115Aocs= -k8s.io/cri-client v0.31.1 h1:w5D7BAhiaSVVDZqHs7YUZPpuUCybx8tCxfdBuDBw7zo= -k8s.io/cri-client v0.31.1/go.mod h1:voVfZexZQwvlf/JD8w30sGN0k22LRcHRfCj7+m4kAXE= -k8s.io/csi-translation-lib v0.31.1 h1:ps9kya8+ih0CVL59JO2B4AYH8U/e3WLQxl9sx19NjjM= -k8s.io/csi-translation-lib v0.31.1/go.mod h1:VeYSucPZJbAt6RT25AzfG7WjyxCcmqxtr4V/CaDdNZc= -k8s.io/dynamic-resource-allocation v0.31.1 h1:AiOVtBdeBmKMbwAVnHmL/v+m9gY2z734x0LKJb4WOMg= -k8s.io/dynamic-resource-allocation v0.31.1/go.mod h1:I1j9Vk9/rbzAckolbNZg8WasttD5yYnsZeDX2dpISKQ= +k8s.io/api v0.34.3 h1:D12sTP257/jSH2vHV2EDYrb16bS7ULlHpdNdNhEw2S4= +k8s.io/api v0.34.3/go.mod h1:PyVQBF886Q5RSQZOim7DybQjAbVs8g7gwJNhGtY5MBk= +k8s.io/apiextensions-apiserver v0.34.3 h1:p10fGlkDY09eWKOTeUSioxwLukJnm+KuDZdrW71y40g= +k8s.io/apiextensions-apiserver v0.34.3/go.mod h1:aujxvqGFRdb/cmXYfcRTeppN7S2XV/t7WMEc64zB5A0= +k8s.io/apimachinery v0.34.3 h1:/TB+SFEiQvN9HPldtlWOTp0hWbJ+fjU+wkxysf/aQnE= +k8s.io/apimachinery v0.34.3/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apiserver v0.34.3 h1:uGH1qpDvSiYG4HVFqc6A3L4CKiX+aBWDrrsxHYK0Bdo= +k8s.io/apiserver v0.34.3/go.mod h1:QPnnahMO5C2m3lm6fPW3+JmyQbvHZQ8uudAu/493P2w= +k8s.io/cli-runtime v0.34.3 h1:YRyMhiwX0dT9lmG0AtZDaeG33Nkxgt9OlCTZhRXj9SI= +k8s.io/cli-runtime v0.34.3/go.mod h1:GVwL1L5uaGEgM7eGeKjaTG2j3u134JgG4dAI6jQKhMc= +k8s.io/client-go v0.34.3 h1:wtYtpzy/OPNYf7WyNBTj3iUA0XaBHVqhv4Iv3tbrF5A= +k8s.io/client-go v0.34.3/go.mod h1:OxxeYagaP9Kdf78UrKLa3YZixMCfP6bgPwPwNBQBzpM= +k8s.io/cloud-provider v0.34.3 h1:+ZIj1mYPzrA0vWZMFFustsDCe1iP+xkhq0ZXZBhPW0o= +k8s.io/cloud-provider v0.34.3/go.mod h1:e0XM6MTHG4rPk1Fa7oWnQT9VqKca+jw7wcc+BJeUcn4= +k8s.io/component-base v0.34.3 h1:zsEgw6ELqK0XncCQomgO9DpUIzlrYuZYA0Cgo+JWpVk= +k8s.io/component-base v0.34.3/go.mod h1:5iIlD8wPfWE/xSHTRfbjuvUul2WZbI2nOUK65XL0E/c= +k8s.io/component-helpers v0.34.3 h1:Iws1GQfM89Lxo7IZITGmVdFOW0Bmyd7SVwwIu1/CCkE= +k8s.io/component-helpers v0.34.3/go.mod h1:S8HjjMTrUDVMVPo2EdNYRtQx9uIEIueQYdPMOe9UxJs= +k8s.io/controller-manager v0.34.3 h1:pEW6ExR3FteKkYkKRrLoi0Sy8dcbvUTAReP8OTxK5k0= +k8s.io/controller-manager v0.34.3/go.mod h1:YzXiwiubf6GdSC3ej2XFYhQQBwF5AvJq/3eymdsU9OU= +k8s.io/cri-api v0.34.3 h1:zFdQSHZuQlQXesw9ncjQRUyDpvLng/84Q4qLKd8x2zE= +k8s.io/cri-api v0.34.3/go.mod h1:4qVUjidMg7/Z9YGZpqIDygbkPWkg3mkS1PvOx/kpHTE= +k8s.io/cri-client v0.34.3 h1:P1LMB7Aa52OfIaJ383FfHYJTlokZsLYUBn6Y7IEc9Oc= +k8s.io/cri-client v0.34.3/go.mod h1:EEQXyjPYePB2RSBnrTs+4up14q/6lVKnHvPXUSMYR4A= +k8s.io/csi-translation-lib v0.34.3 h1:WGE/HPz5D3TIqffhYkk6s4KfW1mcSwSH30MzABK47Pg= +k8s.io/csi-translation-lib v0.34.3/go.mod h1:Lx11spUQnRzYFDrTok0/6cQMP3oXHi73+mXWvkRTxbE= +k8s.io/dynamic-resource-allocation v0.34.3 h1:8UGn1CTj1IljJa+r6HxnEDqLvcBZkv5c+Ooa6x1Oy+o= +k8s.io/dynamic-resource-allocation v0.34.3/go.mod h1:eYjQqNaHLfqXT94lbSXEy8ZLaUg1mGJ2JCEtNWM7e7M= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kms v0.33.4 h1:rvsVglcIFa9WeKk5vd3mBufSG4D5dqponz1Jz5d6FXU= -k8s.io/kms v0.33.4/go.mod h1:C1I8mjFFBNzfUZXYt9FZVJ8MJl7ynFbGgZFbBzkBJ3E= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= -k8s.io/kube-scheduler v0.31.1 h1:hbTiOUqEgPuXa85/J2ZYzIK7aYZruuOaQAirv5TQXjQ= -k8s.io/kube-scheduler v0.31.1/go.mod h1:pJKhtHJthZbxXpF+Mecb0wPXecYxsiMJbhuNi0xUsrE= -k8s.io/kubectl v0.31.1 h1:ih4JQJHxsEggFqDJEHSOdJ69ZxZftgeZvYo7M/cpp24= -k8s.io/kubectl v0.31.1/go.mod h1:aNuQoR43W6MLAtXQ/Bu4GDmoHlbhHKuyD49lmTC8eJM= -k8s.io/kubelet v0.33.4 h1:+sbpLmSq+Y8DF/OQeyw75OpuiF60tvlYcmc/yjN+nl4= -k8s.io/kubelet v0.33.4/go.mod h1:wboarviFRQld5rzZUjTliv7x00YVx+YhRd/p1OahX7Y= -k8s.io/kubernetes v1.32.8 h1:NePHsWPIT9NQZ9w5QT/chJMuwjFFGGZxalvD6FlOjlw= -k8s.io/kubernetes v1.32.8/go.mod h1:REY0Gok66BTTrbGyZaFMNKO9JhxvgBDW9B7aksWRFoY= -k8s.io/mount-utils v0.31.1 h1:f8UrH9kRynljmdNGM6BaCvFUON5ZPKDgE+ltmYqI4wA= -k8s.io/mount-utils v0.31.1/go.mod h1:HV/VYBUGqYUj4vt82YltzpWvgv8FPg0G9ItyInT3NPU= -k8s.io/pod-security-admission v0.31.1 h1:j++ISpfQU0mWpKhoS4tY06Wm5EKdn65teL4lPJhEMIM= -k8s.io/pod-security-admission v0.31.1/go.mod h1:0aE5T6MGm/50Nr/diBrC6+wwpxsT2E7NECe+TepUuEg= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo= -sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= -sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E= -sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo= -sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +k8s.io/kms v0.34.3 h1:QzBOD0sk1bGQVMcZQAHGjtbP1iKZJUyhC6D0I+BTxIE= +k8s.io/kms v0.34.3/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e h1:iW9ChlU0cU16w8MpVYjXk12dqQ4BPFBEgif+ap7/hqQ= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/kube-scheduler v0.34.3 h1:ki99I6opxUZNcIO/QIq1Jz5iTVRHPRdDRFxi5A/3Zjw= +k8s.io/kube-scheduler v0.34.3/go.mod h1:ECkAVWCHQjWJLasX6eznbZ/7J6YqDWiZchXpjG/w5ig= +k8s.io/kubectl v0.34.3 h1:vpM6//153gh5gvsYHXWHVJ4l4xmN5QFwTSmlfd8icm8= +k8s.io/kubectl v0.34.3/go.mod h1:zZQHtIZoUqTP1bAnPzq/3W1jfc0NeOeunFgcswrfg1c= +k8s.io/kubelet v0.34.3 h1:8QRev2FmasZ05yCC774qn6ULche72PYM7AQv0CVt9CM= +k8s.io/kubelet v0.34.3/go.mod h1:pMgblr+nVQ02UkyaTcgqzS3AIYVQkjlMFg1Pd5rGC1Q= +k8s.io/kubernetes v1.34.3 h1:0TfljWbhEF5DBks+WFMSrvKfxBLo4vnZuqORjLMiyT4= +k8s.io/kubernetes v1.34.3/go.mod h1:m6pZk6a179pRo2wsTiCPORJ86iOEQmfIzUvtyEF8BwA= +k8s.io/mount-utils v0.34.3 h1:+sk7PVMQhGoNkGnxmxhyjEXpFcTaD6s3a6NXZNhqERc= +k8s.io/mount-utils v0.34.3/go.mod h1:MIjjYlqJ0ziYQg0MO09kc9S96GIcMkhF/ay9MncF0GA= +k8s.io/pod-security-admission v0.34.3 h1:l9qgNG4X+Zl7H9d5KlYeUtRg9TfApTGNFHCy4F6aXJs= +k8s.io/pod-security-admission v0.34.3/go.mod h1:ureLqfSF+1BrkFIbIImze8LfRMVj48Kg8qYiPHLclmE= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kustomize/api v0.21.0 h1:I7nry5p8iDJbuRdYS7ez8MUvw7XVNPcIP5GkzzuXIIQ= +sigs.k8s.io/kustomize/api v0.21.0/go.mod h1:XGVQuR5n2pXKWbzXHweZU683pALGw/AMVO4zU4iS8SE= +sigs.k8s.io/kustomize/kyaml v0.21.0 h1:7mQAf3dUwf0wBerWJd8rXhVcnkk5Tvn/q91cGkaP6HQ= +sigs.k8s.io/kustomize/kyaml v0.21.0/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index d689a516..409fc60e 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -24,8 +24,8 @@ import ( "strings" "time" - "github.com/cucumber/godog" "github.com/dell/csi-vxflexos/v2/service" + "github.com/cucumber/godog" v1 "k8s.io/api/apps/v1" v1Core "k8s.io/api/core/v1" v1Storage "k8s.io/api/storage/v1" diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 30dc0d6c..00dd5819 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -12,6 +12,7 @@ // //go:build integration +// +build integration package integration_test @@ -24,11 +25,11 @@ import ( "testing" "time" - csi "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/cucumber/godog" "github.com/dell/csi-vxflexos/v2/provider" "github.com/dell/csi-vxflexos/v2/service" csiutils "github.com/dell/gocsi/utils/csi" + csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/cucumber/godog" "google.golang.org/grpc" ) diff --git a/test/integration/step_defs_test.go b/test/integration/step_defs_test.go index 336da71e..11ff561c 100644 --- a/test/integration/step_defs_test.go +++ b/test/integration/step_defs_test.go @@ -12,6 +12,7 @@ // //go:build integration +// +build integration package integration_test @@ -46,10 +47,10 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" - csi "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/cucumber/godog" csiext "github.com/dell/dell-csi-extensions/podmon" "github.com/dell/goscaleio" + csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/cucumber/godog" volGroupSnap "github.com/dell/dell-csi-extensions/volumeGroupSnapshot" )