From 41200323141b87c04b183c2855eb66fb9a02ae75 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 15 Jul 2026 21:56:37 +0200 Subject: [PATCH 1/2] HDDS-15876. Remove ozone-csi --- .github/workflows/repeat-acceptance.yml | 2 +- .run/CsiServer.run.xml | 33 - hadoop-hdds/common/src/main/conf/ozone-env.sh | 11 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 1 - hadoop-hdds/docs/content/interface/CSI.md | 93 -- hadoop-hdds/docs/content/interface/CSI.zh.md | 92 - .../csi/dev-support/findbugsExcludeFile.xml | 22 - hadoop-ozone/csi/pom.xml | 290 ---- .../hadoop/ozone/csi/ControllerService.java | 116 -- .../apache/hadoop/ozone/csi/CsiServer.java | 177 -- .../hadoop/ozone/csi/IdentityService.java | 72 - .../apache/hadoop/ozone/csi/NodeService.java | 148 -- .../apache/hadoop/ozone/csi/package-info.java | 21 - hadoop-ozone/csi/src/main/proto/csi.proto | 1323 --------------- .../csi/src/main/resources/proto.lock | 1479 ----------------- .../dev-support/intellij/ozone-site-ha.xml | 10 +- .../dev-support/intellij/ozone-site.xml | 8 - hadoop-ozone/dist/pom.xml | 5 - .../dist/src/main/compose/ozone-csi/.env | 20 - .../compose/ozone-csi/docker-compose.yaml | 64 - .../src/main/compose/ozone-csi/docker-config | 44 - .../dist/src/main/compose/ozone-csi/test.sh | 31 - .../definitions/ozone-csi/csi-controller.yaml | 53 - .../k8s/definitions/ozone-csi/csi-crd.yaml | 21 - .../k8s/definitions/ozone-csi/csi-node.yaml | 95 -- .../k8s/definitions/ozone-csi/csi-rbac.yaml | 66 - .../ozone-csi/csi-storageclass.yaml | 20 - .../ozone-csi/definitions/csi.yaml | 28 - .../test-webserver/flekszible.yaml | 16 - .../test-webserver/webserver-deployment.yaml | 50 - .../test-webserver/webserver-service.yaml | 29 - .../test-webserver/webserver-volume.yaml | 29 - .../main/k8s/examples/ozone-dev/Flekszible | 7 - .../examples/ozone-dev/config-configmap.yaml | 6 +- .../ozone-dev/csi/csi-node-daemonset.yaml | 97 -- .../ozone-dev/csi/csi-ozone-clusterrole.yaml | 98 -- .../csi/csi-ozone-clusterrolebinding.yaml | 28 - .../csi/csi-ozone-serviceaccount.yaml | 21 - .../csi/csi-provisioner-deployment.yaml | 54 - .../org.apache.hadoop.ozone-csidriver.yaml | 22 - .../ozone-dev/csi/ozone-storageclass.yaml | 21 - .../ozone-csi-test-webserver-deployment.yaml | 50 - ...-test-webserver-persistentvolumeclaim.yaml | 29 - .../ozone-csi-test-webserver-service.yaml | 29 - .../src/main/k8s/examples/ozone/Flekszible | 5 - .../k8s/examples/ozone/config-configmap.yaml | 6 +- .../ozone/csi/csi-node-daemonset.yaml | 97 -- .../ozone/csi/csi-ozone-clusterrole.yaml | 98 -- .../csi/csi-ozone-clusterrolebinding.yaml | 28 - .../ozone/csi/csi-ozone-serviceaccount.yaml | 21 - .../ozone/csi/csi-provisioner-deployment.yaml | 54 - .../org.apache.hadoop.ozone-csidriver.yaml | 22 - .../ozone/csi/ozone-storageclass.yaml | 21 - .../ozone-csi-test-webserver-deployment.yaml | 50 - ...-test-webserver-persistentvolumeclaim.yaml | 29 - .../ozone-csi-test-webserver-service.yaml | 29 - .../dist/src/main/license/jar-report.txt | 2 - .../dist/src/main/smoketest/csi.robot | 34 - hadoop-ozone/dist/src/shell/ozone/ozone | 7 - .../dist/src/shell/ozone/ozone-functions.sh | 2 +- hadoop-ozone/integration-test/pom.xml | 5 - hadoop-ozone/pom.xml | 1 - pom.xml | 5 - 63 files changed, 6 insertions(+), 5441 deletions(-) delete mode 100644 .run/CsiServer.run.xml delete mode 100644 hadoop-hdds/docs/content/interface/CSI.md delete mode 100644 hadoop-hdds/docs/content/interface/CSI.zh.md delete mode 100644 hadoop-ozone/csi/dev-support/findbugsExcludeFile.xml delete mode 100644 hadoop-ozone/csi/pom.xml delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/IdentityService.java delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/NodeService.java delete mode 100644 hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/package-info.java delete mode 100644 hadoop-ozone/csi/src/main/proto/csi.proto delete mode 100644 hadoop-ozone/csi/src/main/resources/proto.lock delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-csi/.env delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-csi/docker-compose.yaml delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config delete mode 100755 hadoop-ozone/dist/src/main/compose/ozone-csi/test.sh delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-controller.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-crd.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-node.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-rbac.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-storageclass.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/definitions/csi.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/flekszible.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-deployment.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-service.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-volume.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-node-daemonset.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrole.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrolebinding.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-serviceaccount.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-provisioner-deployment.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/org.apache.hadoop.ozone-csidriver.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/ozone-storageclass.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-deployment.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-service.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-node-daemonset.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrole.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrolebinding.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-serviceaccount.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-provisioner-deployment.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/org.apache.hadoop.ozone-csidriver.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/ozone-storageclass.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-deployment.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-service.yaml delete mode 100644 hadoop-ozone/dist/src/main/smoketest/csi.robot diff --git a/.github/workflows/repeat-acceptance.yml b/.github/workflows/repeat-acceptance.yml index 51ea4e497381..f8160bb3db73 100644 --- a/.github/workflows/repeat-acceptance.yml +++ b/.github/workflows/repeat-acceptance.yml @@ -18,7 +18,7 @@ # * "Test Suite", which should be ones of the existing suites from regular CI, # e.g. "cert-rotation", or # * "Test Filter", which is a regex pattern applied to filter test script's path -# (examples: "ozone-csi", "test-vault.sh", "ozone/test-ec.sh", "test-.*-rotation.sh") +# (examples: "ozone-ha", "test-vault.sh", "ozone/test-ec.sh", "test-.*-rotation.sh") name: repeat-acceptance-test on: diff --git a/.run/CsiServer.run.xml b/.run/CsiServer.run.xml deleted file mode 100644 index 3c153386e2a3..000000000000 --- a/.run/CsiServer.run.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - diff --git a/hadoop-hdds/common/src/main/conf/ozone-env.sh b/hadoop-hdds/common/src/main/conf/ozone-env.sh index a49b67b28157..0116c7786f9c 100644 --- a/hadoop-hdds/common/src/main/conf/ozone-env.sh +++ b/hadoop-hdds/common/src/main/conf/ozone-env.sh @@ -173,7 +173,7 @@ export OZONE_OS_TYPE=${OZONE_OS_TYPE:-$(uname -s)} # # Extra Java runtime options for all Ozone server daemons (OM, SCM, DataNode, -# S3 Gateway, Recon, HttpFS, CSI). These get appended to OZONE_OPTS for such +# S3 Gateway, Recon, HttpFS). These get appended to OZONE_OPTS for such # daemons and are a convenient way to apply common options to all of them. # export OZONE_SERVER_OPTS="" @@ -326,15 +326,6 @@ export OZONE_OS_TYPE=${OZONE_OS_TYPE:-$(uname -s)} # # export OZONE_HTTPFS_OPTS="" -### -# CSI server specific parameters -### -# Specify the JVM options to be used when starting the CSI server. -# These options will be appended to the options specified as OZONE_OPTS -# and therefore may override any similar flags set in OZONE_OPTS -# -# export OZONE_CSI_OPTS="" - ### # Client and tool command specific parameters ### diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 0fabba6df3bf..69c4c029ce10 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -228,7 +228,6 @@ public static List getConfigurationResourceFiles() { "hdds-server-framework", "hdds-server-scm", "ozone-common", - "ozone-csi", "ozone-manager", "ozone-recon", }; diff --git a/hadoop-hdds/docs/content/interface/CSI.md b/hadoop-hdds/docs/content/interface/CSI.md deleted file mode 100644 index 84bd89c049e2..000000000000 --- a/hadoop-hdds/docs/content/interface/CSI.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -title: CSI Protocol -weight: 6 -menu: - main: - parent: "Client Interfaces" -summary: Ozone supports Container Storage Interface(CSI) protocol. You can use Ozone by mounting an Ozone volume by Ozone CSI. ---- - - - - - -`Container Storage Interface` (CSI) will enable storage vendors (SP) to develop a plugin once and have it work across a number of container orchestration (CO) systems like Kubernetes or Yarn. - -To get more information about CSI at [SCI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md) - -CSI defined a simple GRPC interface with 3 interfaces (Identity, Controller, Node). It defined how the Container Orchestrator can request the creation of a new storage space or the mount of the newly created storage but doesn't define how the storage can be mounted. - -![CSI](CSI.png) - -By default Ozone CSI service uses a S3 fuse driver ([goofys](https://github.com/kahing/goofys)) to mount the created Ozone bucket. Implementation of other mounting options such as a dedicated NFS server or native Fuse driver is work in progress. - - - -Ozone CSI is an implementation of CSI, it can make possible of using Ozone as a storage volume for a container. - -## Getting started - -First of all, we need an Ozone cluster with s3gateway, and its OM rpc port and s3gateway port must be visible to CSI pod, -because CSIServer will access OM to create or delete a bucket, also, CSIServer will publish volume by creating a mount point to s3g -through goofys. - -If you don't have an Ozone cluster on kubernetes, you can reference [Kubernetes]({{< ref "start/Kubernetes.md" >}}) to create one. Use the resources from `kubernetes/examples/ozone` where you can find all the required Kubernetes resources to run cluster together with the dedicated Ozone CSI daemon (check `kubernetes/examples/ozone/csi`) - -Now, create the CSI related resources by execute the follow command. - -```bash -kubectl create -f /ozone/kubernetes/examples/ozone/csi -``` - -## Create pv-test and visit the result. - -Create pv-test related resources by execute the follow command. - -```bash -kubectl create -f /ozone/kubernetes/examples/ozone/pv-test -``` - -Attach the pod scm-0 and put a key into the /s3v/pvc* bucket. - -```bash -kubectl exec -it scm-0 bash -[hadoop@scm-0 ~]$ ozone sh bucket list s3v -[ { - "metadata" : { }, - "volumeName" : "s3v", - "name" : "pvc-861e2d8b-2232-4cd1-b43c-c0c26697ab6b", - "storageType" : "DISK", - "versioning" : false, - "creationTime" : "2020-06-11T08:19:47.469Z", - "encryptionKeyName" : null -} ] -[hadoop@scm-0 ~]$ ozone sh key put /s3v/pvc-861e2d8b-2232-4cd1-b43c-c0c26697ab6b/A LICENSE.txt -``` - -Now, let's forward port of the `ozone-csi-test-webserver-7cbdc5d65c-h5mnn` to see the UI through the web browser. - -```bash -kubectl port-forward ozone-csi-test-webserver-7cbdc5d65c-h5mnn 8000:8000 -``` - -Eventually, we can see the result from `http://localhost:8000/` - -![pvtest-webui](pvtest-webui.png) diff --git a/hadoop-hdds/docs/content/interface/CSI.zh.md b/hadoop-hdds/docs/content/interface/CSI.zh.md deleted file mode 100644 index b243d1e5c8e4..000000000000 --- a/hadoop-hdds/docs/content/interface/CSI.zh.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -title: CSI 协议 -weight: 3 -menu: - main: - parent: "编程接口" -summary: Ozone 支持 容器存储接口 (CSI) 协议。你可以通过 Ozone CSI 挂载 Ozone 桶的方式使用 Ozone。 ---- - - - -容器存储接口 `Container Storage Interface` (CSI) 使存储供应商（SP）能够一次性开发一个插件，并让它跨多个容器编排工作, -就像 Kubernetes 或者 YARN。 - -获取更多 CSI 的信息，可以参考[SCI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md) - -CSI 定义了一个简单的，包含3个接口（Identity， Controller， Node）的 GRPC 接口，它定义了容器编排器如何请求创建新的存储空间或挂载新创建的存储， -但没有定义如何挂载存储。 - -![CSI](CSI.png) - -默认情况下，Ozone CSI 服务使用 S3 FUSE 驱动程序（[goofys](https://github.com/kahing/goofys)）挂载 Ozone 桶。 -其他挂载方式（如专用 NFS 服务或本机FUSE驱动程序）的实现正在进行中。 - - - -Ozone CSI 是 CSI 的一种实现，它可以将 Ozone 用作容器的存储卷。 - -## 入门 - -首先，我们需要一个带有 s3gateway 的 Ozone 集群，并且它的 OM 和 s3gateway 的端口都可以对 CSI pod 可见， -因为 CSIServer 将会访问 OM 来创建或者删除桶，同时 CSIServer 通过 goofys 创建一个可以访问 s3g 的挂载点来发布卷。 - -如果你没有一个运行在 Kubernetes 上的 Ozone 集群，你可以参考[Kubernetes]({{< ref "start/Kubernetes.zh.md" >}}) 来创建一个。 -使用来自 `kubernetes/examples/ozone`的资源，你可以找到所有需要的 Kubernetes 资源来和指定的 CSI 运行在一起 -(参考 `kubernetes/examples/ozone/csi`) - -现在，使用如下命令，创建 CSI 相关的资源。 - -```bash -kubectl create -f /ozone/kubernetes/examples/ozone/csi -``` - -## 创建 pv-test 并查看结果 - -通过执行以下命令，创建 pv-test 相关的资源。 - -```bash -kubectl create -f /ozone/kubernetes/examples/ozone/pv-test -``` - -连接 pod scm-0 并在 /s3v/pvc* 桶中创建一个键值。 - -```bash -kubectl exec -it scm-0 bash -[hadoop@scm-0 ~]$ ozone sh bucket list s3v -[ { - "metadata" : { }, - "volumeName" : "s3v", - "name" : "pvc-861e2d8b-2232-4cd1-b43c-c0c26697ab6b", - "storageType" : "DISK", - "versioning" : false, - "creationTime" : "2020-06-11T08:19:47.469Z", - "encryptionKeyName" : null -} ] -[hadoop@scm-0 ~]$ ozone sh key put /s3v/pvc-861e2d8b-2232-4cd1-b43c-c0c26697ab6b/A LICENSE.txt -``` - -现在，通过映射 `ozone-csi-test-webserver-7cbdc5d65c-h5mnn` 端口，我们可以使用浏览器展示其 UI 页面。 - -```bash -kubectl port-forward ozone-csi-test-webserver-7cbdc5d65c-h5mnn 8000:8000 -``` - -最终，我们可以通过 `http://localhost:8000/` 看到结果 - -![pvtest-webui](pvtest-webui.png) diff --git a/hadoop-ozone/csi/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/csi/dev-support/findbugsExcludeFile.xml deleted file mode 100644 index 62d72d26a830..000000000000 --- a/hadoop-ozone/csi/dev-support/findbugsExcludeFile.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - diff --git a/hadoop-ozone/csi/pom.xml b/hadoop-ozone/csi/pom.xml deleted file mode 100644 index 6355d3cda673..000000000000 --- a/hadoop-ozone/csi/pom.xml +++ /dev/null @@ -1,290 +0,0 @@ - - - - 4.0.0 - - org.apache.ozone - ozone - 2.3.0-SNAPSHOT - - ozone-csi - 2.3.0-SNAPSHOT - jar - Apache Ozone CSI service - Apache Ozone CSI service - - - false - true - - true - - - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - commons-io - commons-io - - - info.picocli - picocli - - - io.grpc - grpc-api - - - com.google.code.findbugs - jsr305 - - - - - io.grpc - grpc-netty - - - io.grpc - grpc-protobuf - - - com.google.code.findbugs - jsr305 - - - com.google.protobuf - protobuf-java - - - - - io.grpc - grpc-stub - - - io.netty - netty-transport - - - io.netty - netty-transport-classes-epoll - - - io.netty - netty-transport-native-unix-common - - - org.apache.ozone - hdds-cli-common - - - org.apache.ozone - hdds-common - - - org.apache.ozone - hdds-config - - - org.apache.hadoop - hadoop-common - - - org.apache.hadoop - hadoop-hdfs - - - - - org.apache.ozone - hdds-server-framework - - - org.apache.ozone - ozone-client - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - - - io.netty - netty - - - io.netty - netty-all - - - - - org.apache.ozone - ozone-common - - - org.slf4j - slf4j-api - - - com.google.code.findbugs - jsr305 - 3.0.2 - provided - - - com.google.protobuf - protobuf-java-util - ${protobuf.version} - provided - - - com.google.code.findbugs - jsr305 - - - com.google.j2objc - j2objc-annotations - - - com.google.protobuf - protobuf-java - - - - - - javax.annotation - javax.annotation-api - provided - - - io.netty - netty-codec-http2 - runtime - - - io.netty - netty-handler-proxy - runtime - - - io.netty - netty-transport-native-epoll - linux-x86_64 - runtime - - - org.slf4j - slf4j-reload4j - runtime - - - - - - - com.salesforce.servicelibs - proto-backwards-compatibility - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.apache.ozone - hdds-config - ${hdds.version} - - - - org.apache.hadoop.hdds.conf.ConfigFileGenerator - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - compile-proto-${protobuf.version} - - compile - test-compile - compile-custom - test-compile-custom - - - grpc-java - io.grpc:protoc-gen-grpc-java:${io.grpc.version}:exe:${os.detected.classifier} - com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} - - csi.proto - - false - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - - - ban-annotations - - - - - Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - - org.apache.hadoop.hdds.scm.metadata.Replicate - org.kohsuke.MetaInfServices - - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/dev-support/findbugsExcludeFile.xml - - - - - - kr.motd.maven - os-maven-plugin - - - - diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java deleted file mode 100644 index f0fa375c3fc6..000000000000 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.csi; - -import csi.v1.ControllerGrpc.ControllerImplBase; -import csi.v1.Csi.CapacityRange; -import csi.v1.Csi.ControllerGetCapabilitiesRequest; -import csi.v1.Csi.ControllerGetCapabilitiesResponse; -import csi.v1.Csi.ControllerServiceCapability; -import csi.v1.Csi.ControllerServiceCapability.RPC; -import csi.v1.Csi.ControllerServiceCapability.RPC.Type; -import csi.v1.Csi.CreateVolumeRequest; -import csi.v1.Csi.CreateVolumeResponse; -import csi.v1.Csi.DeleteVolumeRequest; -import csi.v1.Csi.DeleteVolumeResponse; -import csi.v1.Csi.Volume; -import io.grpc.stub.StreamObserver; -import java.io.IOException; -import org.apache.hadoop.ozone.client.OzoneClient; - -/** - * CSI controller service. - *

- * This service usually runs only once and responsible for the creation of - * the volume. - */ -public class ControllerService extends ControllerImplBase { - - private long defaultVolumeSize; - - private OzoneClient ozoneClient; - - public ControllerService(OzoneClient ozoneClient, long volumeSize) { - this.defaultVolumeSize = volumeSize; - this.ozoneClient = ozoneClient; - } - - @Override - public void createVolume(CreateVolumeRequest request, - StreamObserver responseObserver) { - try { - ozoneClient.getObjectStore().createS3Bucket(request.getName()); - - long size = findSize(request.getCapacityRange()); - - CreateVolumeResponse response = CreateVolumeResponse.newBuilder() - .setVolume(Volume.newBuilder() - .setVolumeId(request.getName()) - .setCapacityBytes(size)) - .build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IOException e) { - responseObserver.onError(e); - } - } - - private long findSize(CapacityRange capacityRange) { - if (capacityRange.getRequiredBytes() != 0) { - return capacityRange.getRequiredBytes(); - } else { - if (capacityRange.getLimitBytes() != 0) { - return Math.min(defaultVolumeSize, capacityRange.getLimitBytes()); - } else { - //~1 gig - return defaultVolumeSize; - } - } - } - - @Override - public void deleteVolume(DeleteVolumeRequest request, - StreamObserver responseObserver) { - try { - ozoneClient.getObjectStore().deleteS3Bucket(request.getVolumeId()); - - DeleteVolumeResponse response = DeleteVolumeResponse.newBuilder() - .build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IOException e) { - responseObserver.onError(e); - } - } - - @Override - public void controllerGetCapabilities( - ControllerGetCapabilitiesRequest request, - StreamObserver responseObserver) { - ControllerGetCapabilitiesResponse response = - ControllerGetCapabilitiesResponse.newBuilder() - .addCapabilities( - ControllerServiceCapability.newBuilder().setRpc( - RPC.newBuilder().setType(Type.CREATE_DELETE_VOLUME))) - .build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } -} diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java deleted file mode 100644 index cf1133a53401..000000000000 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.csi; - -import io.grpc.Server; -import io.grpc.netty.NettyServerBuilder; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.epoll.EpollServerDomainSocketChannel; -import io.netty.channel.unix.DomainSocketAddress; -import java.util.concurrent.Callable; -import org.apache.hadoop.hdds.cli.GenericCli; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.conf.Config; -import org.apache.hadoop.hdds.conf.ConfigGroup; -import org.apache.hadoop.hdds.conf.ConfigTag; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.HddsServerUtil; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; -import org.apache.hadoop.ozone.util.OzoneVersionInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine.Command; - -/** - * CLI entrypoint of the CSI service daemon. - */ -@Command(name = "ozone csi", - hidden = true, description = "CSI service daemon.", - versionProvider = HddsVersionProvider.class, - mixinStandardHelpOptions = true) -public class CsiServer extends GenericCli implements Callable { - - private static final Logger LOG = LoggerFactory.getLogger(CsiServer.class); - - @Override - public Void call() throws Exception { - String[] originalArgs = getCmd().getParseResult().originalArgs() - .toArray(new String[0]); - OzoneConfiguration ozoneConfiguration = getOzoneConf(); - HddsServerUtil.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, - CsiServer.class, originalArgs, LOG, ozoneConfiguration); - CsiConfig csiConfig = ozoneConfiguration.getObject(CsiConfig.class); - - OzoneClient rpcClient = OzoneClientFactory.getRpcClient(ozoneConfiguration); - - EpollEventLoopGroup group = new EpollEventLoopGroup(); - - if (csiConfig.getVolumeOwner().isEmpty()) { - throw new IllegalArgumentException( - "ozone.csi.owner is not set. You should set this configuration " - + "variable to define which user should own all the created " - + "buckets."); - } - - Server server = - NettyServerBuilder - .forAddress(new DomainSocketAddress(csiConfig.getSocketPath())) - .channelType(EpollServerDomainSocketChannel.class) - .workerEventLoopGroup(group) - .bossEventLoopGroup(group) - .addService(new IdentityService()) - .addService(new ControllerService(rpcClient, - csiConfig.getDefaultVolumeSize())) - .addService(new NodeService(csiConfig)) - .build(); - - server.start(); - server.awaitTermination(); - rpcClient.close(); - return null; - } - - public static void main(String[] args) { - new CsiServer().run(args); - } - - /** - * Configuration settings specific to the CSI server. - */ - @ConfigGroup(prefix = "ozone.csi") - public static class CsiConfig { - - @Config(key = "ozone.csi.socket", - defaultValue = "/var/lib/csi.sock", - description = - "The socket where all the CSI services will listen (file name).", - tags = ConfigTag.STORAGE) - private String socketPath; - - @Config(key = "ozone.csi.default-volume-size", - defaultValue = "1000000000", - description = - "The default size of the create volumes (if not specified).", - tags = ConfigTag.STORAGE) - private long defaultVolumeSize; - - @Config(key = "ozone.csi.s3g.address", - defaultValue = "http://localhost:9878", - description = - "The address of S3 Gateway endpoint.", - tags = ConfigTag.STORAGE) - private String s3gAddress; - - @Config(key = "ozone.csi.owner", - defaultValue = "", - description = - "This is the username which is used to create the requested " - + "storage. Used as a hadoop username and the generated ozone" - + " volume used to store all the buckets. WARNING: It can " - + "be a security hole to use CSI in a secure environments as " - + "ALL the users can request the mount of a specific bucket " - + "via the CSI interface.", - tags = ConfigTag.STORAGE) - private String volumeOwner; - - @Config(key = "ozone.csi.mount.command", - defaultValue = "goofys --endpoint %s %s %s", - description = - "This is the mount command which is used to publish volume." - + " these %s will be replicated by s3gAddress, volumeId " - + " and target path.", - tags = ConfigTag.STORAGE) - private String mountCommand; - - public String getSocketPath() { - return socketPath; - } - - public String getVolumeOwner() { - return volumeOwner; - } - - public void setVolumeOwner(String volumeOwner) { - this.volumeOwner = volumeOwner; - } - - public void setSocketPath(String socketPath) { - this.socketPath = socketPath; - } - - public long getDefaultVolumeSize() { - return defaultVolumeSize; - } - - public void setDefaultVolumeSize(long defaultVolumeSize) { - this.defaultVolumeSize = defaultVolumeSize; - } - - public String getS3gAddress() { - return s3gAddress; - } - - public void setS3gAddress(String s3gAddress) { - this.s3gAddress = s3gAddress; - } - - public String getMountCommand() { - return mountCommand; - } - } -} diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/IdentityService.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/IdentityService.java deleted file mode 100644 index b65de8cb5f43..000000000000 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/IdentityService.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.csi; - -import static csi.v1.Csi.PluginCapability.Service.Type.CONTROLLER_SERVICE; - -import com.google.protobuf.BoolValue; -import csi.v1.Csi.GetPluginCapabilitiesResponse; -import csi.v1.Csi.GetPluginInfoResponse; -import csi.v1.Csi.PluginCapability; -import csi.v1.Csi.PluginCapability.Service; -import csi.v1.Csi.ProbeResponse; -import csi.v1.IdentityGrpc.IdentityImplBase; -import io.grpc.stub.StreamObserver; -import org.apache.hadoop.ozone.util.OzoneVersionInfo; - -/** - * Implementation of the CSI identity service. - */ -public class IdentityService extends IdentityImplBase { - - @Override - public void getPluginInfo(csi.v1.Csi.GetPluginInfoRequest request, - StreamObserver responseObserver) { - GetPluginInfoResponse response = GetPluginInfoResponse.newBuilder() - .setName("org.apache.hadoop.ozone") - .setVendorVersion(OzoneVersionInfo.OZONE_VERSION_INFO.getVersion()) - .build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } - - @Override - public void getPluginCapabilities( - csi.v1.Csi.GetPluginCapabilitiesRequest request, - StreamObserver responseObserver) { - GetPluginCapabilitiesResponse response = - GetPluginCapabilitiesResponse.newBuilder() - .addCapabilities(PluginCapability.newBuilder().setService( - Service.newBuilder().setType(CONTROLLER_SERVICE))) - .build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } - - @Override - public void probe(csi.v1.Csi.ProbeRequest request, - StreamObserver responseObserver) { - ProbeResponse response = ProbeResponse.newBuilder() - .setReady(BoolValue.of(true)) - .build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } -} diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/NodeService.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/NodeService.java deleted file mode 100644 index 7220c31137b3..000000000000 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/NodeService.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.csi; - -import csi.v1.Csi.NodeGetCapabilitiesRequest; -import csi.v1.Csi.NodeGetCapabilitiesResponse; -import csi.v1.Csi.NodeGetInfoRequest; -import csi.v1.Csi.NodeGetInfoResponse; -import csi.v1.Csi.NodePublishVolumeRequest; -import csi.v1.Csi.NodePublishVolumeResponse; -import csi.v1.Csi.NodeUnpublishVolumeRequest; -import csi.v1.Csi.NodeUnpublishVolumeResponse; -import csi.v1.NodeGrpc.NodeImplBase; -import io.grpc.stub.StreamObserver; -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.ozone.csi.CsiServer.CsiConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implementation of the CSI node service. - */ -public class NodeService extends NodeImplBase { - - private static final Logger LOG = LoggerFactory.getLogger(NodeService.class); - - private final String mountCommand; - private String s3Endpoint; - - public NodeService(CsiConfig configuration) { - this.s3Endpoint = configuration.getS3gAddress(); - this.mountCommand = configuration.getMountCommand(); - } - - @Override - public void nodePublishVolume(NodePublishVolumeRequest request, - StreamObserver responseObserver) { - - try { - Files.createDirectories(Paths.get(request.getTargetPath())); - String command = - String.format(mountCommand, - s3Endpoint, - request.getVolumeId(), - request.getTargetPath()); - LOG.info("Executing {}", command); - - executeCommand(command); - - responseObserver.onNext(NodePublishVolumeResponse.newBuilder() - .build()); - responseObserver.onCompleted(); - - } catch (IOException e) { - responseObserver.onError(e); - } catch (InterruptedException e) { - responseObserver.onError(e); - Thread.currentThread().interrupt(); - } - - } - - private void executeCommand(String command) - throws IOException, InterruptedException { - Process exec = Runtime.getRuntime().exec(command); - exec.waitFor(10, TimeUnit.SECONDS); - - LOG.info("Command is executed with stdout: {}, stderr: {}", - IOUtils.toString(exec.getInputStream(), StandardCharsets.UTF_8), - IOUtils.toString(exec.getErrorStream(), StandardCharsets.UTF_8)); - if (exec.exitValue() != 0) { - throw new RuntimeException(String - .format("Return code of the command %s was %d", command, - exec.exitValue())); - } - } - - @Override - public void nodeUnpublishVolume(NodeUnpublishVolumeRequest request, - StreamObserver responseObserver) { - String umountCommand = - String.format("fusermount -u %s", request.getTargetPath()); - LOG.info("Executing {}", umountCommand); - - try { - executeCommand(umountCommand); - - responseObserver.onNext(NodeUnpublishVolumeResponse.newBuilder() - .build()); - responseObserver.onCompleted(); - - } catch (IOException e) { - responseObserver.onError(e); - } catch (InterruptedException e) { - responseObserver.onError(e); - Thread.currentThread().interrupt(); - } - - } - - @Override - public void nodeGetCapabilities(NodeGetCapabilitiesRequest request, - StreamObserver responseObserver) { - NodeGetCapabilitiesResponse response = - NodeGetCapabilitiesResponse.newBuilder() - .build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } - - @Override - public void nodeGetInfo(NodeGetInfoRequest request, - StreamObserver responseObserver) { - NodeGetInfoResponse response = null; - try { - response = NodeGetInfoResponse.newBuilder() - .setNodeId(InetAddress.getLocalHost().getHostName()) - .build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (UnknownHostException e) { - responseObserver.onError(e); - } - - } -} diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/package-info.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/package-info.java deleted file mode 100644 index 5083ae080a8e..000000000000 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Container Storage Interface server implementation for Ozone. - */ -package org.apache.hadoop.ozone.csi; diff --git a/hadoop-ozone/csi/src/main/proto/csi.proto b/hadoop-ozone/csi/src/main/proto/csi.proto deleted file mode 100644 index 3bd53a0758b4..000000000000 --- a/hadoop-ozone/csi/src/main/proto/csi.proto +++ /dev/null @@ -1,1323 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by make; DO NOT EDIT. -syntax = "proto3"; -package csi.v1; - -import "google/protobuf/descriptor.proto"; -import "google/protobuf/timestamp.proto"; -import "google/protobuf/wrappers.proto"; - -option go_package = "csi"; - -extend google.protobuf.FieldOptions { - // Indicates that a field MAY contain information that is sensitive - // and MUST be treated as such (e.g. not logged). - bool csi_secret = 1059; -} -service Identity { - rpc GetPluginInfo(GetPluginInfoRequest) - returns (GetPluginInfoResponse) {} - - rpc GetPluginCapabilities(GetPluginCapabilitiesRequest) - returns (GetPluginCapabilitiesResponse) {} - - rpc Probe (ProbeRequest) - returns (ProbeResponse) {} -} - -service Controller { - rpc CreateVolume (CreateVolumeRequest) - returns (CreateVolumeResponse) {} - - rpc DeleteVolume (DeleteVolumeRequest) - returns (DeleteVolumeResponse) {} - - rpc ControllerPublishVolume (ControllerPublishVolumeRequest) - returns (ControllerPublishVolumeResponse) {} - - rpc ControllerUnpublishVolume (ControllerUnpublishVolumeRequest) - returns (ControllerUnpublishVolumeResponse) {} - - rpc ValidateVolumeCapabilities (ValidateVolumeCapabilitiesRequest) - returns (ValidateVolumeCapabilitiesResponse) {} - - rpc ListVolumes (ListVolumesRequest) - returns (ListVolumesResponse) {} - - rpc GetCapacity (GetCapacityRequest) - returns (GetCapacityResponse) {} - - rpc ControllerGetCapabilities (ControllerGetCapabilitiesRequest) - returns (ControllerGetCapabilitiesResponse) {} - - rpc CreateSnapshot (CreateSnapshotRequest) - returns (CreateSnapshotResponse) {} - - rpc DeleteSnapshot (DeleteSnapshotRequest) - returns (DeleteSnapshotResponse) {} - - rpc ListSnapshots (ListSnapshotsRequest) - returns (ListSnapshotsResponse) {} - - rpc ControllerExpandVolume (ControllerExpandVolumeRequest) - returns (ControllerExpandVolumeResponse) {} -} - -service Node { - rpc NodeStageVolume (NodeStageVolumeRequest) - returns (NodeStageVolumeResponse) {} - - rpc NodeUnstageVolume (NodeUnstageVolumeRequest) - returns (NodeUnstageVolumeResponse) {} - - rpc NodePublishVolume (NodePublishVolumeRequest) - returns (NodePublishVolumeResponse) {} - - rpc NodeUnpublishVolume (NodeUnpublishVolumeRequest) - returns (NodeUnpublishVolumeResponse) {} - - rpc NodeGetVolumeStats (NodeGetVolumeStatsRequest) - returns (NodeGetVolumeStatsResponse) {} - - - rpc NodeExpandVolume(NodeExpandVolumeRequest) - returns (NodeExpandVolumeResponse) {} - - - rpc NodeGetCapabilities (NodeGetCapabilitiesRequest) - returns (NodeGetCapabilitiesResponse) {} - - rpc NodeGetInfo (NodeGetInfoRequest) - returns (NodeGetInfoResponse) {} -} -message GetPluginInfoRequest { - // Intentionally empty. -} - -message GetPluginInfoResponse { - // The name MUST follow domain name notation format - // (https://tools.ietf.org/html/rfc1035#section-2.3.1). It SHOULD - // include the plugin's host company name and the plugin name, - // to minimize the possibility of collisions. It MUST be 63 - // characters or less, beginning and ending with an alphanumeric - // character ([a-z0-9A-Z]) with dashes (-), dots (.), and - // alphanumerics between. This field is REQUIRED. - string name = 1; - - // This field is REQUIRED. Value of this field is opaque to the CO. - string vendor_version = 2; - - // This field is OPTIONAL. Values are opaque to the CO. - map manifest = 3; -} -message GetPluginCapabilitiesRequest { - // Intentionally empty. -} - -message GetPluginCapabilitiesResponse { - // All the capabilities that the controller service supports. This - // field is OPTIONAL. - repeated PluginCapability capabilities = 1; -} - -// Specifies a capability of the plugin. -message PluginCapability { - message Service { - enum Type { - UNKNOWN = 0; - // CONTROLLER_SERVICE indicates that the Plugin provides RPCs for - // the ControllerService. Plugins SHOULD provide this capability. - // In rare cases certain plugins MAY wish to omit the - // ControllerService entirely from their implementation, but such - // SHOULD NOT be the common case. - // The presence of this capability determines whether the CO will - // attempt to invoke the REQUIRED ControllerService RPCs, as well - // as specific RPCs as indicated by ControllerGetCapabilities. - CONTROLLER_SERVICE = 1; - - // VOLUME_ACCESSIBILITY_CONSTRAINTS indicates that the volumes for - // this plugin MAY NOT be equally accessible by all nodes in the - // cluster. The CO MUST use the topology information returned by - // CreateVolumeRequest along with the topology information - // returned by NodeGetInfo to ensure that a given volume is - // accessible from a given node when scheduling workloads. - VOLUME_ACCESSIBILITY_CONSTRAINTS = 2; - } - Type type = 1; - } - - message VolumeExpansion { - enum Type { - UNKNOWN = 0; - - // ONLINE indicates that volumes may be expanded when published to - // a node. When a Plugin implements this capability it MUST - // implement either the EXPAND_VOLUME controller capability or the - // EXPAND_VOLUME node capability or both. When a plugin supports - // ONLINE volume expansion and also has the EXPAND_VOLUME - // controller capability then the plugin MUST support expansion of - // volumes currently published and available on a node. When a - // plugin supports ONLINE volume expansion and also has the - // EXPAND_VOLUME node capability then the plugin MAY support - // expansion of node-published volume via NodeExpandVolume. - // - // Example 1: Given a shared filesystem volume (e.g. GlusterFs), - // the Plugin may set the ONLINE volume expansion capability and - // implement ControllerExpandVolume but not NodeExpandVolume. - // - // Example 2: Given a block storage volume type (e.g. EBS), the - // Plugin may set the ONLINE volume expansion capability and - // implement both ControllerExpandVolume and NodeExpandVolume. - // - // Example 3: Given a Plugin that supports volume expansion only - // upon a node, the Plugin may set the ONLINE volume - // expansion capability and implement NodeExpandVolume but not - // ControllerExpandVolume. - ONLINE = 1; - - // OFFLINE indicates that volumes currently published and - // available on a node SHALL NOT be expanded via - // ControllerExpandVolume. When a plugin supports OFFLINE volume - // expansion it MUST implement either the EXPAND_VOLUME controller - // capability or both the EXPAND_VOLUME controller capability and - // the EXPAND_VOLUME node capability. - // - // Example 1: Given a block storage volume type (e.g. Azure Disk) - // that does not support expansion of "node-attached" (i.e. - // controller-published) volumes, the Plugin may indicate - // OFFLINE volume expansion support and implement both - // ControllerExpandVolume and NodeExpandVolume. - OFFLINE = 2; - } - } - - oneof type { - // Service that the plugin supports. - Service service = 1; - VolumeExpansion volume_expansion = 2; - } -} -message ProbeRequest { - // Intentionally empty. -} - -message ProbeResponse { - // Readiness allows a plugin to report its initialization status back - // to the CO. Initialization for some plugins MAY be time consuming - // and it is important for a CO to distinguish between the following - // cases: - // - // 1) The plugin is in an unhealthy state and MAY need restarting. In - // this case a gRPC error code SHALL be returned. - // 2) The plugin is still initializing, but is otherwise perfectly - // healthy. In this case a successful response SHALL be returned - // with a readiness value of `false`. Calls to the plugin's - // Controller and/or Node services MAY fail due to an incomplete - // initialization state. - // 3) The plugin has finished initializing and is ready to service - // calls to its Controller and/or Node services. A successful - // response is returned with a readiness value of `true`. - // - // This field is OPTIONAL. If not present, the caller SHALL assume - // that the plugin is in a ready state and is accepting calls to its - // Controller and/or Node services (according to the plugin's reported - // capabilities). - .google.protobuf.BoolValue ready = 1; -} -message CreateVolumeRequest { - // The suggested name for the storage space. This field is REQUIRED. - // It serves two purposes: - // 1) Idempotency - This name is generated by the CO to achieve - // idempotency. The Plugin SHOULD ensure that multiple - // `CreateVolume` calls for the same name do not result in more - // than one piece of storage provisioned corresponding to that - // name. If a Plugin is unable to enforce idempotency, the CO's - // error recovery logic could result in multiple (unused) volumes - // being provisioned. - // In the case of error, the CO MUST handle the gRPC error codes - // per the recovery behavior defined in the "CreateVolume Errors" - // section below. - // The CO is responsible for cleaning up volumes it provisioned - // that it no longer needs. If the CO is uncertain whether a volume - // was provisioned or not when a `CreateVolume` call fails, the CO - // MAY call `CreateVolume` again, with the same name, to ensure the - // volume exists and to retrieve the volume's `volume_id` (unless - // otherwise prohibited by "CreateVolume Errors"). - // 2) Suggested name - Some storage systems allow callers to specify - // an identifier by which to refer to the newly provisioned - // storage. If a storage system supports this, it can optionally - // use this name as the identifier for the new volume. - // Any Unicode string that conforms to the length limit is allowed - // except those containing the following banned characters: - // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. - // (These are control characters other than commonly used whitespace.) - string name = 1; - - // This field is OPTIONAL. This allows the CO to specify the capacity - // requirement of the volume to be provisioned. If not specified, the - // Plugin MAY choose an implementation-defined capacity range. If - // specified it MUST always be honored, even when creating volumes - // from a source; which MAY force some backends to internally extend - // the volume after creating it. - CapacityRange capacity_range = 2; - - // The capabilities that the provisioned volume MUST have. SP MUST - // provision a volume that will satisfy ALL of the capabilities - // specified in this list. Otherwise SP MUST return the appropriate - // gRPC error code. - // The Plugin MUST assume that the CO MAY use the provisioned volume - // with ANY of the capabilities specified in this list. - // For example, a CO MAY specify two volume capabilities: one with - // access mode SINGLE_NODE_WRITER and another with access mode - // MULTI_NODE_READER_ONLY. In this case, the SP MUST verify that the - // provisioned volume can be used in either mode. - // This also enables the CO to do early validation: If ANY of the - // specified volume capabilities are not supported by the SP, the call - // MUST return the appropriate gRPC error code. - // This field is REQUIRED. - repeated VolumeCapability volume_capabilities = 3; - - // Plugin specific parameters passed in as opaque key-value pairs. - // This field is OPTIONAL. The Plugin is responsible for parsing and - // validating these parameters. COs will treat these as opaque. - map parameters = 4; - - // Secrets required by plugin to complete volume creation request. - // This field is OPTIONAL. Refer to the `Secrets Requirements` - // section on how to use this field. - map secrets = 5 [(csi_secret) = true]; - - // If specified, the new volume will be pre-populated with data from - // this source. This field is OPTIONAL. - VolumeContentSource volume_content_source = 6; - - // Specifies where (regions, zones, racks, etc.) the provisioned - // volume MUST be accessible from. - // An SP SHALL advertise the requirements for topological - // accessibility information in documentation. COs SHALL only specify - // topological accessibility information supported by the SP. - // This field is OPTIONAL. - // This field SHALL NOT be specified unless the SP has the - // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. - // If this field is not specified and the SP has the - // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability, the SP MAY - // choose where the provisioned volume is accessible from. - TopologyRequirement accessibility_requirements = 7; -} - -// Specifies what source the volume will be created from. One of the -// type fields MUST be specified. -message VolumeContentSource { - message SnapshotSource { - // Contains identity information for the existing source snapshot. - // This field is REQUIRED. Plugin is REQUIRED to support creating - // volume from snapshot if it supports the capability - // CREATE_DELETE_SNAPSHOT. - string snapshot_id = 1; - } - - message VolumeSource { - // Contains identity information for the existing source volume. - // This field is REQUIRED. Plugins reporting CLONE_VOLUME - // capability MUST support creating a volume from another volume. - string volume_id = 1; - } - - oneof type { - SnapshotSource snapshot = 1; - VolumeSource volume = 2; - } -} - -message CreateVolumeResponse { - // Contains all attributes of the newly created volume that are - // relevant to the CO along with information required by the Plugin - // to uniquely identify the volume. This field is REQUIRED. - Volume volume = 1; -} - -// Specify a capability of a volume. -message VolumeCapability { - // Indicate that the volume will be accessed via the block device API. - message BlockVolume { - // Intentionally empty, for now. - } - - // Indicate that the volume will be accessed via the filesystem API. - message MountVolume { - // The filesystem type. This field is OPTIONAL. - // An empty string is equal to an unspecified field value. - string fs_type = 1; - - // The mount options that can be used for the volume. This field is - // OPTIONAL. `mount_flags` MAY contain sensitive information. - // Therefore, the CO and the Plugin MUST NOT leak this information - // to untrusted entities. The total size of this repeated field - // SHALL NOT exceed 4 KiB. - repeated string mount_flags = 2; - } - - // Specify how a volume can be accessed. - message AccessMode { - enum Mode { - UNKNOWN = 0; - - // Can only be published once as read/write on a single node, at - // any given time. - SINGLE_NODE_WRITER = 1; - - // Can only be published once as readonly on a single node, at - // any given time. - SINGLE_NODE_READER_ONLY = 2; - - // Can be published as readonly at multiple nodes simultaneously. - MULTI_NODE_READER_ONLY = 3; - - // Can be published at multiple nodes simultaneously. Only one of - // the node can be used as read/write. The rest will be readonly. - MULTI_NODE_SINGLE_WRITER = 4; - - // Can be published as read/write at multiple nodes - // simultaneously. - MULTI_NODE_MULTI_WRITER = 5; - } - - // This field is REQUIRED. - Mode mode = 1; - } - - // Specifies what API the volume will be accessed using. One of the - // following fields MUST be specified. - oneof access_type { - BlockVolume block = 1; - MountVolume mount = 2; - } - - // This is a REQUIRED field. - AccessMode access_mode = 3; -} - -// The capacity of the storage space in bytes. To specify an exact size, -// `required_bytes` and `limit_bytes` SHALL be set to the same value. At -// least one of the these fields MUST be specified. -message CapacityRange { - // Volume MUST be at least this big. This field is OPTIONAL. - // A value of 0 is equal to an unspecified field value. - // The value of this field MUST NOT be negative. - int64 required_bytes = 1; - - // Volume MUST not be bigger than this. This field is OPTIONAL. - // A value of 0 is equal to an unspecified field value. - // The value of this field MUST NOT be negative. - int64 limit_bytes = 2; -} - -// Information about a specific volume. -message Volume { - // The capacity of the volume in bytes. This field is OPTIONAL. If not - // set (value of 0), it indicates that the capacity of the volume is - // unknown (e.g., NFS share). - // The value of this field MUST NOT be negative. - int64 capacity_bytes = 1; - - // The identifier for this volume, generated by the plugin. - // This field is REQUIRED. - // This field MUST contain enough information to uniquely identify - // this specific volume vs all other volumes supported by this plugin. - // This field SHALL be used by the CO in subsequent calls to refer to - // this volume. - // The SP is NOT responsible for global uniqueness of volume_id across - // multiple SPs. - string volume_id = 2; - - // Opaque static properties of the volume. SP MAY use this field to - // ensure subsequent volume validation and publishing calls have - // contextual information. - // The contents of this field SHALL be opaque to a CO. - // The contents of this field SHALL NOT be mutable. - // The contents of this field SHALL be safe for the CO to cache. - // The contents of this field SHOULD NOT contain sensitive - // information. - // The contents of this field SHOULD NOT be used for uniquely - // identifying a volume. The `volume_id` alone SHOULD be sufficient to - // identify the volume. - // A volume uniquely identified by `volume_id` SHALL always report the - // same volume_context. - // This field is OPTIONAL and when present MUST be passed to volume - // validation and publishing calls. - map volume_context = 3; - - // If specified, indicates that the volume is not empty and is - // pre-populated with data from the specified source. - // This field is OPTIONAL. - VolumeContentSource content_source = 4; - - // Specifies where (regions, zones, racks, etc.) the provisioned - // volume is accessible from. - // A plugin that returns this field MUST also set the - // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. - // An SP MAY specify multiple topologies to indicate the volume is - // accessible from multiple locations. - // COs MAY use this information along with the topology information - // returned by NodeGetInfo to ensure that a given volume is accessible - // from a given node when scheduling workloads. - // This field is OPTIONAL. If it is not specified, the CO MAY assume - // the volume is equally accessible from all nodes in the cluster and - // MAY schedule workloads referencing the volume on any available - // node. - // - // Example 1: - // accessible_topology = {"region": "R1", "zone": "Z2"} - // Indicates a volume accessible only from the "region" "R1" and the - // "zone" "Z2". - // - // Example 2: - // accessible_topology = - // {"region": "R1", "zone": "Z2"}, - // {"region": "R1", "zone": "Z3"} - // Indicates a volume accessible from both "zone" "Z2" and "zone" "Z3" - // in the "region" "R1". - repeated Topology accessible_topology = 5; -} - -message TopologyRequirement { - // Specifies the list of topologies the provisioned volume MUST be - // accessible from. - // This field is OPTIONAL. If TopologyRequirement is specified either - // requisite or preferred or both MUST be specified. - // - // If requisite is specified, the provisioned volume MUST be - // accessible from at least one of the requisite topologies. - // - // Given - // x = number of topologies provisioned volume is accessible from - // n = number of requisite topologies - // The CO MUST ensure n >= 1. The SP MUST ensure x >= 1 - // If x==n, then the SP MUST make the provisioned volume available to - // all topologies from the list of requisite topologies. If it is - // unable to do so, the SP MUST fail the CreateVolume call. - // For example, if a volume should be accessible from a single zone, - // and requisite = - // {"region": "R1", "zone": "Z2"} - // then the provisioned volume MUST be accessible from the "region" - // "R1" and the "zone" "Z2". - // Similarly, if a volume should be accessible from two zones, and - // requisite = - // {"region": "R1", "zone": "Z2"}, - // {"region": "R1", "zone": "Z3"} - // then the provisioned volume MUST be accessible from the "region" - // "R1" and both "zone" "Z2" and "zone" "Z3". - // - // If xn, then the SP MUST make the provisioned volume available from - // all topologies from the list of requisite topologies and MAY choose - // the remaining x-n unique topologies from the list of all possible - // topologies. If it is unable to do so, the SP MUST fail the - // CreateVolume call. - // For example, if a volume should be accessible from two zones, and - // requisite = - // {"region": "R1", "zone": "Z2"} - // then the provisioned volume MUST be accessible from the "region" - // "R1" and the "zone" "Z2" and the SP may select the second zone - // independently, e.g. "R1/Z4". - repeated Topology requisite = 1; - - // Specifies the list of topologies the CO would prefer the volume to - // be provisioned in. - // - // This field is OPTIONAL. If TopologyRequirement is specified either - // requisite or preferred or both MUST be specified. - // - // An SP MUST attempt to make the provisioned volume available using - // the preferred topologies in order from first to last. - // - // If requisite is specified, all topologies in preferred list MUST - // also be present in the list of requisite topologies. - // - // If the SP is unable to to make the provisioned volume available - // from any of the preferred topologies, the SP MAY choose a topology - // from the list of requisite topologies. - // If the list of requisite topologies is not specified, then the SP - // MAY choose from the list of all possible topologies. - // If the list of requisite topologies is specified and the SP is - // unable to to make the provisioned volume available from any of the - // requisite topologies it MUST fail the CreateVolume call. - // - // Example 1: - // Given a volume should be accessible from a single zone, and - // requisite = - // {"region": "R1", "zone": "Z2"}, - // {"region": "R1", "zone": "Z3"} - // preferred = - // {"region": "R1", "zone": "Z3"} - // then the the SP SHOULD first attempt to make the provisioned volume - // available from "zone" "Z3" in the "region" "R1" and fall back to - // "zone" "Z2" in the "region" "R1" if that is not possible. - // - // Example 2: - // Given a volume should be accessible from a single zone, and - // requisite = - // {"region": "R1", "zone": "Z2"}, - // {"region": "R1", "zone": "Z3"}, - // {"region": "R1", "zone": "Z4"}, - // {"region": "R1", "zone": "Z5"} - // preferred = - // {"region": "R1", "zone": "Z4"}, - // {"region": "R1", "zone": "Z2"} - // then the the SP SHOULD first attempt to make the provisioned volume - // accessible from "zone" "Z4" in the "region" "R1" and fall back to - // "zone" "Z2" in the "region" "R1" if that is not possible. If that - // is not possible, the SP may choose between either the "zone" - // "Z3" or "Z5" in the "region" "R1". - // - // Example 3: - // Given a volume should be accessible from TWO zones (because an - // opaque parameter in CreateVolumeRequest, for example, specifies - // the volume is accessible from two zones, aka synchronously - // replicated), and - // requisite = - // {"region": "R1", "zone": "Z2"}, - // {"region": "R1", "zone": "Z3"}, - // {"region": "R1", "zone": "Z4"}, - // {"region": "R1", "zone": "Z5"} - // preferred = - // {"region": "R1", "zone": "Z5"}, - // {"region": "R1", "zone": "Z3"} - // then the the SP SHOULD first attempt to make the provisioned volume - // accessible from the combination of the two "zones" "Z5" and "Z3" in - // the "region" "R1". If that's not possible, it should fall back to - // a combination of "Z5" and other possibilities from the list of - // requisite. If that's not possible, it should fall back to a - // combination of "Z3" and other possibilities from the list of - // requisite. If that's not possible, it should fall back to a - // combination of other possibilities from the list of requisite. - repeated Topology preferred = 2; -} - -// Topology is a map of topological domains to topological segments. -// A topological domain is a sub-division of a cluster, like "region", -// "zone", "rack", etc. -// A topological segment is a specific instance of a topological domain, -// like "zone3", "rack3", etc. -// For example {"com.company/zone": "Z1", "com.company/rack": "R3"} -// Valid keys have two segments: an OPTIONAL prefix and name, separated -// by a slash (/), for example: "com.company.example/zone". -// The key name segment is REQUIRED. The prefix is OPTIONAL. -// The key name MUST be 63 characters or less, begin and end with an -// alphanumeric character ([a-z0-9A-Z]), and contain only dashes (-), -// underscores (_), dots (.), or alphanumerics in between, for example -// "zone". -// The key prefix MUST be 63 characters or less, begin and end with a -// lower-case alphanumeric character ([a-z0-9]), contain only -// dashes (-), dots (.), or lower-case alphanumerics in between, and -// follow domain name notation format -// (https://tools.ietf.org/html/rfc1035#section-2.3.1). -// The key prefix SHOULD include the plugin's host company name and/or -// the plugin name, to minimize the possibility of collisions with keys -// from other plugins. -// If a key prefix is specified, it MUST be identical across all -// topology keys returned by the SP (across all RPCs). -// Keys MUST be case-insensitive. Meaning the keys "Zone" and "zone" -// MUST not both exist. -// Each value (topological segment) MUST contain 1 or more strings. -// Each string MUST be 63 characters or less and begin and end with an -// alphanumeric character with '-', '_', '.', or alphanumerics in -// between. -message Topology { - map segments = 1; -} -message DeleteVolumeRequest { - // The ID of the volume to be deprovisioned. - // This field is REQUIRED. - string volume_id = 1; - - // Secrets required by plugin to complete volume deletion request. - // This field is OPTIONAL. Refer to the `Secrets Requirements` - // section on how to use this field. - map secrets = 2 [(csi_secret) = true]; -} - -message DeleteVolumeResponse { - // Intentionally empty. -} -message ControllerPublishVolumeRequest { - // The ID of the volume to be used on a node. - // This field is REQUIRED. - string volume_id = 1; - - // The ID of the node. This field is REQUIRED. The CO SHALL set this - // field to match the node ID returned by `NodeGetInfo`. - string node_id = 2; - - // Volume capability describing how the CO intends to use this volume. - // SP MUST ensure the CO can use the published volume as described. - // Otherwise SP MUST return the appropriate gRPC error code. - // This is a REQUIRED field. - VolumeCapability volume_capability = 3; - - // Indicates SP MUST publish the volume in readonly mode. - // CO MUST set this field to false if SP does not have the - // PUBLISH_READONLY controller capability. - // This is a REQUIRED field. - bool readonly = 4; - - // Secrets required by plugin to complete controller publish volume - // request. This field is OPTIONAL. Refer to the - // `Secrets Requirements` section on how to use this field. - map secrets = 5 [(csi_secret) = true]; - - // Volume context as returned by CO in CreateVolumeRequest. This field - // is OPTIONAL and MUST match the volume_context of the volume - // identified by `volume_id`. - map volume_context = 6; -} - -message ControllerPublishVolumeResponse { - // Opaque static publish properties of the volume. SP MAY use this - // field to ensure subsequent `NodeStageVolume` or `NodePublishVolume` - // calls calls have contextual information. - // The contents of this field SHALL be opaque to a CO. - // The contents of this field SHALL NOT be mutable. - // The contents of this field SHALL be safe for the CO to cache. - // The contents of this field SHOULD NOT contain sensitive - // information. - // The contents of this field SHOULD NOT be used for uniquely - // identifying a volume. The `volume_id` alone SHOULD be sufficient to - // identify the volume. - // This field is OPTIONAL and when present MUST be passed to - // subsequent `NodeStageVolume` or `NodePublishVolume` calls - map publish_context = 1; -} -message ControllerUnpublishVolumeRequest { - // The ID of the volume. This field is REQUIRED. - string volume_id = 1; - - // The ID of the node. This field is OPTIONAL. The CO SHOULD set this - // field to match the node ID returned by `NodeGetInfo` or leave it - // unset. If the value is set, the SP MUST unpublish the volume from - // the specified node. If the value is unset, the SP MUST unpublish - // the volume from all nodes it is published to. - string node_id = 2; - - // Secrets required by plugin to complete controller unpublish volume - // request. This SHOULD be the same secrets passed to the - // ControllerPublishVolume call for the specified volume. - // This field is OPTIONAL. Refer to the `Secrets Requirements` - // section on how to use this field. - map secrets = 3 [(csi_secret) = true]; -} - -message ControllerUnpublishVolumeResponse { - // Intentionally empty. -} -message ValidateVolumeCapabilitiesRequest { - // The ID of the volume to check. This field is REQUIRED. - string volume_id = 1; - - // Volume context as returned by CO in CreateVolumeRequest. This field - // is OPTIONAL and MUST match the volume_context of the volume - // identified by `volume_id`. - map volume_context = 2; - - // The capabilities that the CO wants to check for the volume. This - // call SHALL return "confirmed" only if all the volume capabilities - // specified below are supported. This field is REQUIRED. - repeated VolumeCapability volume_capabilities = 3; - - // See CreateVolumeRequest.parameters. - // This field is OPTIONAL. - map parameters = 4; - - // Secrets required by plugin to complete volume validation request. - // This field is OPTIONAL. Refer to the `Secrets Requirements` - // section on how to use this field. - map secrets = 5 [(csi_secret) = true]; -} - -message ValidateVolumeCapabilitiesResponse { - message Confirmed { - // Volume context validated by the plugin. - // This field is OPTIONAL. - map volume_context = 1; - - // Volume capabilities supported by the plugin. - // This field is REQUIRED. - repeated VolumeCapability volume_capabilities = 2; - - // The volume creation parameters validated by the plugin. - // This field is OPTIONAL. - map parameters = 3; - } - - // Confirmed indicates to the CO the set of capabilities that the - // plugin has validated. This field SHALL only be set to a non-empty - // value for successful validation responses. - // For successful validation responses, the CO SHALL compare the - // fields of this message to the originally requested capabilities in - // order to guard against an older plugin reporting "valid" for newer - // capability fields that it does not yet understand. - // This field is OPTIONAL. - Confirmed confirmed = 1; - - // Message to the CO if `confirmed` above is empty. This field is - // OPTIONAL. - // An empty string is equal to an unspecified field value. - string message = 2; -} -message ListVolumesRequest { - // If specified (non-zero value), the Plugin MUST NOT return more - // entries than this number in the response. If the actual number of - // entries is more than this number, the Plugin MUST set `next_token` - // in the response which can be used to get the next page of entries - // in the subsequent `ListVolumes` call. This field is OPTIONAL. If - // not specified (zero value), it means there is no restriction on the - // number of entries that can be returned. - // The value of this field MUST NOT be negative. - int32 max_entries = 1; - - // A token to specify where to start paginating. Set this field to - // `next_token` returned by a previous `ListVolumes` call to get the - // next page of entries. This field is OPTIONAL. - // An empty string is equal to an unspecified field value. - string starting_token = 2; -} - -message ListVolumesResponse { - message Entry { - Volume volume = 1; - } - - repeated Entry entries = 1; - - // This token allows you to get the next page of entries for - // `ListVolumes` request. If the number of entries is larger than - // `max_entries`, use the `next_token` as a value for the - // `starting_token` field in the next `ListVolumes` request. This - // field is OPTIONAL. - // An empty string is equal to an unspecified field value. - string next_token = 2; -} -message GetCapacityRequest { - // If specified, the Plugin SHALL report the capacity of the storage - // that can be used to provision volumes that satisfy ALL of the - // specified `volume_capabilities`. These are the same - // `volume_capabilities` the CO will use in `CreateVolumeRequest`. - // This field is OPTIONAL. - repeated VolumeCapability volume_capabilities = 1; - - // If specified, the Plugin SHALL report the capacity of the storage - // that can be used to provision volumes with the given Plugin - // specific `parameters`. These are the same `parameters` the CO will - // use in `CreateVolumeRequest`. This field is OPTIONAL. - map parameters = 2; - - // If specified, the Plugin SHALL report the capacity of the storage - // that can be used to provision volumes that in the specified - // `accessible_topology`. This is the same as the - // `accessible_topology` the CO returns in a `CreateVolumeResponse`. - // This field is OPTIONAL. This field SHALL NOT be set unless the - // plugin advertises the VOLUME_ACCESSIBILITY_CONSTRAINTS capability. - Topology accessible_topology = 3; -} - -message GetCapacityResponse { - // The available capacity, in bytes, of the storage that can be used - // to provision volumes. If `volume_capabilities` or `parameters` is - // specified in the request, the Plugin SHALL take those into - // consideration when calculating the available capacity of the - // storage. This field is REQUIRED. - // The value of this field MUST NOT be negative. - int64 available_capacity = 1; -} -message ControllerGetCapabilitiesRequest { - // Intentionally empty. -} - -message ControllerGetCapabilitiesResponse { - // All the capabilities that the controller service supports. This - // field is OPTIONAL. - repeated ControllerServiceCapability capabilities = 1; -} - -// Specifies a capability of the controller service. -message ControllerServiceCapability { - message RPC { - enum Type { - UNKNOWN = 0; - CREATE_DELETE_VOLUME = 1; - PUBLISH_UNPUBLISH_VOLUME = 2; - LIST_VOLUMES = 3; - GET_CAPACITY = 4; - // Currently the only way to consume a snapshot is to create - // a volume from it. Therefore plugins supporting - // CREATE_DELETE_SNAPSHOT MUST support creating volume from - // snapshot. - CREATE_DELETE_SNAPSHOT = 5; - LIST_SNAPSHOTS = 6; - - // Plugins supporting volume cloning at the storage level MAY - // report this capability. The source volume MUST be managed by - // the same plugin. Not all volume sources and parameters - // combinations MAY work. - CLONE_VOLUME = 7; - - // Indicates the SP supports ControllerPublishVolume.readonly - // field. - PUBLISH_READONLY = 8; - - // See VolumeExpansion for details. - EXPAND_VOLUME = 9; - } - - Type type = 1; - } - - oneof type { - // RPC that the controller supports. - RPC rpc = 1; - } -} -message CreateSnapshotRequest { - // The ID of the source volume to be snapshotted. - // This field is REQUIRED. - string source_volume_id = 1; - - // The suggested name for the snapshot. This field is REQUIRED for - // idempotency. - // Any Unicode string that conforms to the length limit is allowed - // except those containing the following banned characters: - // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. - // (These are control characters other than commonly used whitespace.) - string name = 2; - - // Secrets required by plugin to complete snapshot creation request. - // This field is OPTIONAL. Refer to the `Secrets Requirements` - // section on how to use this field. - map secrets = 3 [(csi_secret) = true]; - - // Plugin specific parameters passed in as opaque key-value pairs. - // This field is OPTIONAL. The Plugin is responsible for parsing and - // validating these parameters. COs will treat these as opaque. - // Use cases for opaque parameters: - // - Specify a policy to automatically clean up the snapshot. - // - Specify an expiration date for the snapshot. - // - Specify whether the snapshot is readonly or read/write. - // - Specify if the snapshot should be replicated to some place. - // - Specify primary or secondary for replication systems that - // support snapshotting only on primary. - map parameters = 4; -} - -message CreateSnapshotResponse { - // Contains all attributes of the newly created snapshot that are - // relevant to the CO along with information required by the Plugin - // to uniquely identify the snapshot. This field is REQUIRED. - Snapshot snapshot = 1; -} - -// Information about a specific snapshot. -message Snapshot { - // This is the complete size of the snapshot in bytes. The purpose of - // this field is to give CO guidance on how much space is needed to - // create a volume from this snapshot. The size of the volume MUST NOT - // be less than the size of the source snapshot. This field is - // OPTIONAL. If this field is not set, it indicates that this size is - // unknown. The value of this field MUST NOT be negative and a size of - // zero means it is unspecified. - int64 size_bytes = 1; - - // The identifier for this snapshot, generated by the plugin. - // This field is REQUIRED. - // This field MUST contain enough information to uniquely identify - // this specific snapshot vs all other snapshots supported by this - // plugin. - // This field SHALL be used by the CO in subsequent calls to refer to - // this snapshot. - // The SP is NOT responsible for global uniqueness of snapshot_id - // across multiple SPs. - string snapshot_id = 2; - - // Identity information for the source volume. Note that creating a - // snapshot from a snapshot is not supported here so the source has to - // be a volume. This field is REQUIRED. - string source_volume_id = 3; - - // Timestamp when the point-in-time snapshot is taken on the storage - // system. This field is REQUIRED. - .google.protobuf.Timestamp creation_time = 4; - - // Indicates if a snapshot is ready to use as a - // `volume_content_source` in a `CreateVolumeRequest`. The default - // value is false. This field is REQUIRED. - bool ready_to_use = 5; -} -message DeleteSnapshotRequest { - // The ID of the snapshot to be deleted. - // This field is REQUIRED. - string snapshot_id = 1; - - // Secrets required by plugin to complete snapshot deletion request. - // This field is OPTIONAL. Refer to the `Secrets Requirements` - // section on how to use this field. - map secrets = 2 [(csi_secret) = true]; -} - -message DeleteSnapshotResponse {} -// List all snapshots on the storage system regardless of how they were -// created. -message ListSnapshotsRequest { - // If specified (non-zero value), the Plugin MUST NOT return more - // entries than this number in the response. If the actual number of - // entries is more than this number, the Plugin MUST set `next_token` - // in the response which can be used to get the next page of entries - // in the subsequent `ListSnapshots` call. This field is OPTIONAL. If - // not specified (zero value), it means there is no restriction on the - // number of entries that can be returned. - // The value of this field MUST NOT be negative. - int32 max_entries = 1; - - // A token to specify where to start paginating. Set this field to - // `next_token` returned by a previous `ListSnapshots` call to get the - // next page of entries. This field is OPTIONAL. - // An empty string is equal to an unspecified field value. - string starting_token = 2; - - // Identity information for the source volume. This field is OPTIONAL. - // It can be used to list snapshots by volume. - string source_volume_id = 3; - - // Identity information for a specific snapshot. This field is - // OPTIONAL. It can be used to list only a specific snapshot. - // ListSnapshots will return with current snapshot information - // and will not block if the snapshot is being processed after - // it is cut. - string snapshot_id = 4; -} - -message ListSnapshotsResponse { - message Entry { - Snapshot snapshot = 1; - } - - repeated Entry entries = 1; - - // This token allows you to get the next page of entries for - // `ListSnapshots` request. If the number of entries is larger than - // `max_entries`, use the `next_token` as a value for the - // `starting_token` field in the next `ListSnapshots` request. This - // field is OPTIONAL. - // An empty string is equal to an unspecified field value. - string next_token = 2; -} -message ControllerExpandVolumeRequest { - // The ID of the volume to expand. This field is REQUIRED. - string volume_id = 1; - - // This allows CO to specify the capacity requirements of the volume - // after expansion. This field is REQUIRED. - CapacityRange capacity_range = 2; - - // Secrets required by the plugin for expanding the volume. - // This field is OPTIONAL. - map secrets = 3 [(csi_secret) = true]; -} - -message ControllerExpandVolumeResponse { - // Capacity of volume after expansion. This field is REQUIRED. - int64 capacity_bytes = 1; - - // Whether node expansion is required for the volume. When true - // the CO MUST make NodeExpandVolume RPC call on the node. This field - // is REQUIRED. - bool node_expansion_required = 2; -} -message NodeStageVolumeRequest { - // The ID of the volume to publish. This field is REQUIRED. - string volume_id = 1; - - // The CO SHALL set this field to the value returned by - // `ControllerPublishVolume` if the corresponding Controller Plugin - // has `PUBLISH_UNPUBLISH_VOLUME` controller capability, and SHALL be - // left unset if the corresponding Controller Plugin does not have - // this capability. This is an OPTIONAL field. - map publish_context = 2; - - // The path to which the volume MAY be staged. It MUST be an - // absolute path in the root filesystem of the process serving this - // request, and MUST be a directory. The CO SHALL ensure that there - // is only one `staging_target_path` per volume. The CO SHALL ensure - // that the path is directory and that the process serving the - // request has `read` and `write` permission to that directory. The - // CO SHALL be responsible for creating the directory if it does not - // exist. - // This is a REQUIRED field. - string staging_target_path = 3; - - // Volume capability describing how the CO intends to use this volume. - // SP MUST ensure the CO can use the staged volume as described. - // Otherwise SP MUST return the appropriate gRPC error code. - // This is a REQUIRED field. - VolumeCapability volume_capability = 4; - - // Secrets required by plugin to complete node stage volume request. - // This field is OPTIONAL. Refer to the `Secrets Requirements` - // section on how to use this field. - map secrets = 5 [(csi_secret) = true]; - - // Volume context as returned by CO in CreateVolumeRequest. This field - // is OPTIONAL and MUST match the volume_context of the volume - // identified by `volume_id`. - map volume_context = 6; -} - -message NodeStageVolumeResponse { - // Intentionally empty. -} -message NodeUnstageVolumeRequest { - // The ID of the volume. This field is REQUIRED. - string volume_id = 1; - - // The path at which the volume was staged. It MUST be an absolute - // path in the root filesystem of the process serving this request. - // This is a REQUIRED field. - string staging_target_path = 2; -} - -message NodeUnstageVolumeResponse { - // Intentionally empty. -} -message NodePublishVolumeRequest { - // The ID of the volume to publish. This field is REQUIRED. - string volume_id = 1; - - // The CO SHALL set this field to the value returned by - // `ControllerPublishVolume` if the corresponding Controller Plugin - // has `PUBLISH_UNPUBLISH_VOLUME` controller capability, and SHALL be - // left unset if the corresponding Controller Plugin does not have - // this capability. This is an OPTIONAL field. - map publish_context = 2; - - // The path to which the volume was staged by `NodeStageVolume`. - // It MUST be an absolute path in the root filesystem of the process - // serving this request. - // It MUST be set if the Node Plugin implements the - // `STAGE_UNSTAGE_VOLUME` node capability. - // This is an OPTIONAL field. - string staging_target_path = 3; - - // The path to which the volume will be published. It MUST be an - // absolute path in the root filesystem of the process serving this - // request. The CO SHALL ensure uniqueness of target_path per volume. - // The CO SHALL ensure that the parent directory of this path exists - // and that the process serving the request has `read` and `write` - // permissions to that parent directory. - // For volumes with an access type of block, the SP SHALL place the - // block device at target_path. - // For volumes with an access type of mount, the SP SHALL place the - // mounted directory at target_path. - // Creation of target_path is the responsibility of the SP. - // This is a REQUIRED field. - string target_path = 4; - - // Volume capability describing how the CO intends to use this volume. - // SP MUST ensure the CO can use the published volume as described. - // Otherwise SP MUST return the appropriate gRPC error code. - // This is a REQUIRED field. - VolumeCapability volume_capability = 5; - - // Indicates SP MUST publish the volume in readonly mode. - // This field is REQUIRED. - bool readonly = 6; - - // Secrets required by plugin to complete node publish volume request. - // This field is OPTIONAL. Refer to the `Secrets Requirements` - // section on how to use this field. - map secrets = 7 [(csi_secret) = true]; - - // Volume context as returned by CO in CreateVolumeRequest. This field - // is OPTIONAL and MUST match the volume_context of the volume - // identified by `volume_id`. - map volume_context = 8; -} - -message NodePublishVolumeResponse { - // Intentionally empty. -} -message NodeUnpublishVolumeRequest { - // The ID of the volume. This field is REQUIRED. - string volume_id = 1; - - // The path at which the volume was published. It MUST be an absolute - // path in the root filesystem of the process serving this request. - // The SP MUST delete the file or directory it created at this path. - // This is a REQUIRED field. - string target_path = 2; -} - -message NodeUnpublishVolumeResponse { - // Intentionally empty. -} -message NodeGetVolumeStatsRequest { - // The ID of the volume. This field is REQUIRED. - string volume_id = 1; - - // It can be any valid path where volume was previously - // staged or published. - // It MUST be an absolute path in the root filesystem of - // the process serving this request. - // This is a REQUIRED field. - string volume_path = 2; -} - -message NodeGetVolumeStatsResponse { - // This field is OPTIONAL. - repeated VolumeUsage usage = 1; -} - -message VolumeUsage { - enum Unit { - UNKNOWN = 0; - BYTES = 1; - INODES = 2; - } - // The available capacity in specified Unit. This field is OPTIONAL. - // The value of this field MUST NOT be negative. - int64 available = 1; - - // The total capacity in specified Unit. This field is REQUIRED. - // The value of this field MUST NOT be negative. - int64 total = 2; - - // The used capacity in specified Unit. This field is OPTIONAL. - // The value of this field MUST NOT be negative. - int64 used = 3; - - // Units by which values are measured. This field is REQUIRED. - Unit unit = 4; -} -message NodeGetCapabilitiesRequest { - // Intentionally empty. -} - -message NodeGetCapabilitiesResponse { - // All the capabilities that the node service supports. This field - // is OPTIONAL. - repeated NodeServiceCapability capabilities = 1; -} - -// Specifies a capability of the node service. -message NodeServiceCapability { - message RPC { - enum Type { - UNKNOWN = 0; - STAGE_UNSTAGE_VOLUME = 1; - // If Plugin implements GET_VOLUME_STATS capability - // then it MUST implement NodeGetVolumeStats RPC - // call for fetching volume statistics. - GET_VOLUME_STATS = 2; - // See VolumeExpansion for details. - EXPAND_VOLUME = 3; - } - - Type type = 1; - } - - oneof type { - // RPC that the controller supports. - RPC rpc = 1; - } -} -message NodeGetInfoRequest { -} - -message NodeGetInfoResponse { - // The identifier of the node as understood by the SP. - // This field is REQUIRED. - // This field MUST contain enough information to uniquely identify - // this specific node vs all other nodes supported by this plugin. - // This field SHALL be used by the CO in subsequent calls, including - // `ControllerPublishVolume`, to refer to this node. - // The SP is NOT responsible for global uniqueness of node_id across - // multiple SPs. - string node_id = 1; - - // Maximum number of volumes that controller can publish to the node. - // If value is not set or zero CO SHALL decide how many volumes of - // this type can be published by the controller to the node. The - // plugin MUST NOT set negative values here. - // This field is OPTIONAL. - int64 max_volumes_per_node = 2; - - // Specifies where (regions, zones, racks, etc.) the node is - // accessible from. - // A plugin that returns this field MUST also set the - // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. - // COs MAY use this information along with the topology information - // returned in CreateVolumeResponse to ensure that a given volume is - // accessible from a given node when scheduling workloads. - // This field is OPTIONAL. If it is not specified, the CO MAY assume - // the node is not subject to any topological constraint, and MAY - // schedule workloads that reference any volume V, such that there are - // no topological constraints declared for V. - // - // Example 1: - // accessible_topology = - // {"region": "R1", "zone": "R2"} - // Indicates the node exists within the "region" "R1" and the "zone" - // "Z2". - Topology accessible_topology = 3; -} -message NodeExpandVolumeRequest { - // The ID of the volume. This field is REQUIRED. - string volume_id = 1; - - // The path on which volume is available. This field is REQUIRED. - string volume_path = 2; - - // This allows CO to specify the capacity requirements of the volume - // after expansion. If capacity_range is omitted then a plugin MAY - // inspect the file system of the volume to determine the maximum - // capacity to which the volume can be expanded. In such cases a - // plugin MAY expand the volume to its maximum capacity. - // This field is OPTIONAL. - CapacityRange capacity_range = 3; -} - -message NodeExpandVolumeResponse { - // The capacity of the volume in bytes. This field is OPTIONAL. - int64 capacity_bytes = 1; -} diff --git a/hadoop-ozone/csi/src/main/resources/proto.lock b/hadoop-ozone/csi/src/main/resources/proto.lock deleted file mode 100644 index 410598cbb668..000000000000 --- a/hadoop-ozone/csi/src/main/resources/proto.lock +++ /dev/null @@ -1,1479 +0,0 @@ -{ - "definitions": [ - { - "protopath": "csi.proto", - "def": { - "enums": [ - { - "name": "Service.Type", - "enum_fields": [ - { - "name": "UNKNOWN" - }, - { - "name": "CONTROLLER_SERVICE", - "integer": 1 - }, - { - "name": "VOLUME_ACCESSIBILITY_CONSTRAINTS", - "integer": 2 - } - ] - }, - { - "name": "VolumeExpansion.Type", - "enum_fields": [ - { - "name": "UNKNOWN" - }, - { - "name": "ONLINE", - "integer": 1 - }, - { - "name": "OFFLINE", - "integer": 2 - } - ] - }, - { - "name": "AccessMode.Mode", - "enum_fields": [ - { - "name": "UNKNOWN" - }, - { - "name": "SINGLE_NODE_WRITER", - "integer": 1 - }, - { - "name": "SINGLE_NODE_READER_ONLY", - "integer": 2 - }, - { - "name": "MULTI_NODE_READER_ONLY", - "integer": 3 - }, - { - "name": "MULTI_NODE_SINGLE_WRITER", - "integer": 4 - }, - { - "name": "MULTI_NODE_MULTI_WRITER", - "integer": 5 - } - ] - }, - { - "name": "RPC.Type", - "enum_fields": [ - { - "name": "UNKNOWN" - }, - { - "name": "CREATE_DELETE_VOLUME", - "integer": 1 - }, - { - "name": "PUBLISH_UNPUBLISH_VOLUME", - "integer": 2 - }, - { - "name": "LIST_VOLUMES", - "integer": 3 - }, - { - "name": "GET_CAPACITY", - "integer": 4 - }, - { - "name": "CREATE_DELETE_SNAPSHOT", - "integer": 5 - }, - { - "name": "LIST_SNAPSHOTS", - "integer": 6 - }, - { - "name": "CLONE_VOLUME", - "integer": 7 - }, - { - "name": "PUBLISH_READONLY", - "integer": 8 - }, - { - "name": "EXPAND_VOLUME", - "integer": 9 - } - ] - }, - { - "name": "VolumeUsage.Unit", - "enum_fields": [ - { - "name": "UNKNOWN" - }, - { - "name": "BYTES", - "integer": 1 - }, - { - "name": "INODES", - "integer": 2 - } - ] - }, - { - "name": "RPC.Type", - "enum_fields": [ - { - "name": "UNKNOWN" - }, - { - "name": "STAGE_UNSTAGE_VOLUME", - "integer": 1 - }, - { - "name": "GET_VOLUME_STATS", - "integer": 2 - }, - { - "name": "EXPAND_VOLUME", - "integer": 3 - } - ] - } - ], - "messages": [ - { - "name": "google.protobuf.FieldOptions", - "fields": [ - { - "id": 1059, - "name": "csi_secret", - "type": "bool" - } - ] - }, - { - "name": "GetPluginInfoRequest" - }, - { - "name": "GetPluginInfoResponse", - "fields": [ - { - "id": 1, - "name": "name", - "type": "string" - }, - { - "id": 2, - "name": "vendor_version", - "type": "string" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 3, - "name": "manifest", - "type": "string" - } - } - ] - }, - { - "name": "GetPluginCapabilitiesRequest" - }, - { - "name": "GetPluginCapabilitiesResponse", - "fields": [ - { - "id": 1, - "name": "capabilities", - "type": "PluginCapability", - "is_repeated": true - } - ] - }, - { - "name": "PluginCapability", - "fields": [ - { - "id": 1, - "name": "service", - "type": "Service", - "oneof_parent": "type" - }, - { - "id": 2, - "name": "volume_expansion", - "type": "VolumeExpansion", - "oneof_parent": "type" - } - ], - "messages": [ - { - "name": "Service", - "fields": [ - { - "id": 1, - "name": "type", - "type": "Type" - } - ] - }, - { - "name": "VolumeExpansion" - } - ] - }, - { - "name": "ProbeRequest" - }, - { - "name": "ProbeResponse", - "fields": [ - { - "id": 1, - "name": "ready", - "type": ".google.protobuf.BoolValue" - } - ] - }, - { - "name": "CreateVolumeRequest", - "fields": [ - { - "id": 1, - "name": "name", - "type": "string" - }, - { - "id": 2, - "name": "capacity_range", - "type": "CapacityRange" - }, - { - "id": 3, - "name": "volume_capabilities", - "type": "VolumeCapability", - "is_repeated": true - }, - { - "id": 6, - "name": "volume_content_source", - "type": "VolumeContentSource" - }, - { - "id": 7, - "name": "accessibility_requirements", - "type": "TopologyRequirement" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 4, - "name": "parameters", - "type": "string" - } - }, - { - "key_type": "string", - "field": { - "id": 5, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - } - ] - }, - { - "name": "VolumeContentSource", - "fields": [ - { - "id": 1, - "name": "snapshot", - "type": "SnapshotSource", - "oneof_parent": "type" - }, - { - "id": 2, - "name": "volume", - "type": "VolumeSource", - "oneof_parent": "type" - } - ], - "messages": [ - { - "name": "SnapshotSource", - "fields": [ - { - "id": 1, - "name": "snapshot_id", - "type": "string" - } - ] - }, - { - "name": "VolumeSource", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - } - ] - } - ] - }, - { - "name": "CreateVolumeResponse", - "fields": [ - { - "id": 1, - "name": "volume", - "type": "Volume" - } - ] - }, - { - "name": "VolumeCapability", - "fields": [ - { - "id": 1, - "name": "block", - "type": "BlockVolume", - "oneof_parent": "access_type" - }, - { - "id": 2, - "name": "mount", - "type": "MountVolume", - "oneof_parent": "access_type" - }, - { - "id": 3, - "name": "access_mode", - "type": "AccessMode" - } - ], - "messages": [ - { - "name": "BlockVolume" - }, - { - "name": "MountVolume", - "fields": [ - { - "id": 1, - "name": "fs_type", - "type": "string" - }, - { - "id": 2, - "name": "mount_flags", - "type": "string", - "is_repeated": true - } - ] - }, - { - "name": "AccessMode", - "fields": [ - { - "id": 1, - "name": "mode", - "type": "Mode" - } - ] - } - ] - }, - { - "name": "CapacityRange", - "fields": [ - { - "id": 1, - "name": "required_bytes", - "type": "int64" - }, - { - "id": 2, - "name": "limit_bytes", - "type": "int64" - } - ] - }, - { - "name": "Volume", - "fields": [ - { - "id": 1, - "name": "capacity_bytes", - "type": "int64" - }, - { - "id": 2, - "name": "volume_id", - "type": "string" - }, - { - "id": 4, - "name": "content_source", - "type": "VolumeContentSource" - }, - { - "id": 5, - "name": "accessible_topology", - "type": "Topology", - "is_repeated": true - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 3, - "name": "volume_context", - "type": "string" - } - } - ] - }, - { - "name": "TopologyRequirement", - "fields": [ - { - "id": 1, - "name": "requisite", - "type": "Topology", - "is_repeated": true - }, - { - "id": 2, - "name": "preferred", - "type": "Topology", - "is_repeated": true - } - ] - }, - { - "name": "Topology", - "maps": [ - { - "key_type": "string", - "field": { - "id": 1, - "name": "segments", - "type": "string" - } - } - ] - }, - { - "name": "DeleteVolumeRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 2, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - } - ] - }, - { - "name": "DeleteVolumeResponse" - }, - { - "name": "ControllerPublishVolumeRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 2, - "name": "node_id", - "type": "string" - }, - { - "id": 3, - "name": "volume_capability", - "type": "VolumeCapability" - }, - { - "id": 4, - "name": "readonly", - "type": "bool" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 5, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - }, - { - "key_type": "string", - "field": { - "id": 6, - "name": "volume_context", - "type": "string" - } - } - ] - }, - { - "name": "ControllerPublishVolumeResponse", - "maps": [ - { - "key_type": "string", - "field": { - "id": 1, - "name": "publish_context", - "type": "string" - } - } - ] - }, - { - "name": "ControllerUnpublishVolumeRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 2, - "name": "node_id", - "type": "string" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 3, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - } - ] - }, - { - "name": "ControllerUnpublishVolumeResponse" - }, - { - "name": "ValidateVolumeCapabilitiesRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 3, - "name": "volume_capabilities", - "type": "VolumeCapability", - "is_repeated": true - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 2, - "name": "volume_context", - "type": "string" - } - }, - { - "key_type": "string", - "field": { - "id": 4, - "name": "parameters", - "type": "string" - } - }, - { - "key_type": "string", - "field": { - "id": 5, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - } - ] - }, - { - "name": "ValidateVolumeCapabilitiesResponse", - "fields": [ - { - "id": 1, - "name": "confirmed", - "type": "Confirmed" - }, - { - "id": 2, - "name": "message", - "type": "string" - } - ], - "messages": [ - { - "name": "Confirmed", - "fields": [ - { - "id": 2, - "name": "volume_capabilities", - "type": "VolumeCapability", - "is_repeated": true - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 1, - "name": "volume_context", - "type": "string" - } - }, - { - "key_type": "string", - "field": { - "id": 3, - "name": "parameters", - "type": "string" - } - } - ] - } - ] - }, - { - "name": "ListVolumesRequest", - "fields": [ - { - "id": 1, - "name": "max_entries", - "type": "int32" - }, - { - "id": 2, - "name": "starting_token", - "type": "string" - } - ] - }, - { - "name": "ListVolumesResponse", - "fields": [ - { - "id": 1, - "name": "entries", - "type": "Entry", - "is_repeated": true - }, - { - "id": 2, - "name": "next_token", - "type": "string" - } - ], - "messages": [ - { - "name": "Entry", - "fields": [ - { - "id": 1, - "name": "volume", - "type": "Volume" - } - ] - } - ] - }, - { - "name": "GetCapacityRequest", - "fields": [ - { - "id": 1, - "name": "volume_capabilities", - "type": "VolumeCapability", - "is_repeated": true - }, - { - "id": 3, - "name": "accessible_topology", - "type": "Topology" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 2, - "name": "parameters", - "type": "string" - } - } - ] - }, - { - "name": "GetCapacityResponse", - "fields": [ - { - "id": 1, - "name": "available_capacity", - "type": "int64" - } - ] - }, - { - "name": "ControllerGetCapabilitiesRequest" - }, - { - "name": "ControllerGetCapabilitiesResponse", - "fields": [ - { - "id": 1, - "name": "capabilities", - "type": "ControllerServiceCapability", - "is_repeated": true - } - ] - }, - { - "name": "ControllerServiceCapability", - "fields": [ - { - "id": 1, - "name": "rpc", - "type": "RPC", - "oneof_parent": "type" - } - ], - "messages": [ - { - "name": "RPC", - "fields": [ - { - "id": 1, - "name": "type", - "type": "Type" - } - ] - } - ] - }, - { - "name": "CreateSnapshotRequest", - "fields": [ - { - "id": 1, - "name": "source_volume_id", - "type": "string" - }, - { - "id": 2, - "name": "name", - "type": "string" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 3, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - }, - { - "key_type": "string", - "field": { - "id": 4, - "name": "parameters", - "type": "string" - } - } - ] - }, - { - "name": "CreateSnapshotResponse", - "fields": [ - { - "id": 1, - "name": "snapshot", - "type": "Snapshot" - } - ] - }, - { - "name": "Snapshot", - "fields": [ - { - "id": 1, - "name": "size_bytes", - "type": "int64" - }, - { - "id": 2, - "name": "snapshot_id", - "type": "string" - }, - { - "id": 3, - "name": "source_volume_id", - "type": "string" - }, - { - "id": 4, - "name": "creation_time", - "type": ".google.protobuf.Timestamp" - }, - { - "id": 5, - "name": "ready_to_use", - "type": "bool" - } - ] - }, - { - "name": "DeleteSnapshotRequest", - "fields": [ - { - "id": 1, - "name": "snapshot_id", - "type": "string" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 2, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - } - ] - }, - { - "name": "DeleteSnapshotResponse" - }, - { - "name": "ListSnapshotsRequest", - "fields": [ - { - "id": 1, - "name": "max_entries", - "type": "int32" - }, - { - "id": 2, - "name": "starting_token", - "type": "string" - }, - { - "id": 3, - "name": "source_volume_id", - "type": "string" - }, - { - "id": 4, - "name": "snapshot_id", - "type": "string" - } - ] - }, - { - "name": "ListSnapshotsResponse", - "fields": [ - { - "id": 1, - "name": "entries", - "type": "Entry", - "is_repeated": true - }, - { - "id": 2, - "name": "next_token", - "type": "string" - } - ], - "messages": [ - { - "name": "Entry", - "fields": [ - { - "id": 1, - "name": "snapshot", - "type": "Snapshot" - } - ] - } - ] - }, - { - "name": "ControllerExpandVolumeRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 2, - "name": "capacity_range", - "type": "CapacityRange" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 3, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - } - ] - }, - { - "name": "ControllerExpandVolumeResponse", - "fields": [ - { - "id": 1, - "name": "capacity_bytes", - "type": "int64" - }, - { - "id": 2, - "name": "node_expansion_required", - "type": "bool" - } - ] - }, - { - "name": "NodeStageVolumeRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 3, - "name": "staging_target_path", - "type": "string" - }, - { - "id": 4, - "name": "volume_capability", - "type": "VolumeCapability" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 2, - "name": "publish_context", - "type": "string" - } - }, - { - "key_type": "string", - "field": { - "id": 5, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - }, - { - "key_type": "string", - "field": { - "id": 6, - "name": "volume_context", - "type": "string" - } - } - ] - }, - { - "name": "NodeStageVolumeResponse" - }, - { - "name": "NodeUnstageVolumeRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 2, - "name": "staging_target_path", - "type": "string" - } - ] - }, - { - "name": "NodeUnstageVolumeResponse" - }, - { - "name": "NodePublishVolumeRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 3, - "name": "staging_target_path", - "type": "string" - }, - { - "id": 4, - "name": "target_path", - "type": "string" - }, - { - "id": 5, - "name": "volume_capability", - "type": "VolumeCapability" - }, - { - "id": 6, - "name": "readonly", - "type": "bool" - } - ], - "maps": [ - { - "key_type": "string", - "field": { - "id": 2, - "name": "publish_context", - "type": "string" - } - }, - { - "key_type": "string", - "field": { - "id": 7, - "name": "secrets", - "type": "string", - "options": [ - { - "name": "(csi_secret)", - "value": "true" - } - ] - } - }, - { - "key_type": "string", - "field": { - "id": 8, - "name": "volume_context", - "type": "string" - } - } - ] - }, - { - "name": "NodePublishVolumeResponse" - }, - { - "name": "NodeUnpublishVolumeRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 2, - "name": "target_path", - "type": "string" - } - ] - }, - { - "name": "NodeUnpublishVolumeResponse" - }, - { - "name": "NodeGetVolumeStatsRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 2, - "name": "volume_path", - "type": "string" - } - ] - }, - { - "name": "NodeGetVolumeStatsResponse", - "fields": [ - { - "id": 1, - "name": "usage", - "type": "VolumeUsage", - "is_repeated": true - } - ] - }, - { - "name": "VolumeUsage", - "fields": [ - { - "id": 1, - "name": "available", - "type": "int64" - }, - { - "id": 2, - "name": "total", - "type": "int64" - }, - { - "id": 3, - "name": "used", - "type": "int64" - }, - { - "id": 4, - "name": "unit", - "type": "Unit" - } - ] - }, - { - "name": "NodeGetCapabilitiesRequest" - }, - { - "name": "NodeGetCapabilitiesResponse", - "fields": [ - { - "id": 1, - "name": "capabilities", - "type": "NodeServiceCapability", - "is_repeated": true - } - ] - }, - { - "name": "NodeServiceCapability", - "fields": [ - { - "id": 1, - "name": "rpc", - "type": "RPC", - "oneof_parent": "type" - } - ], - "messages": [ - { - "name": "RPC", - "fields": [ - { - "id": 1, - "name": "type", - "type": "Type" - } - ] - } - ] - }, - { - "name": "NodeGetInfoRequest" - }, - { - "name": "NodeGetInfoResponse", - "fields": [ - { - "id": 1, - "name": "node_id", - "type": "string" - }, - { - "id": 2, - "name": "max_volumes_per_node", - "type": "int64" - }, - { - "id": 3, - "name": "accessible_topology", - "type": "Topology" - } - ] - }, - { - "name": "NodeExpandVolumeRequest", - "fields": [ - { - "id": 1, - "name": "volume_id", - "type": "string" - }, - { - "id": 2, - "name": "volume_path", - "type": "string" - }, - { - "id": 3, - "name": "capacity_range", - "type": "CapacityRange" - } - ] - }, - { - "name": "NodeExpandVolumeResponse", - "fields": [ - { - "id": 1, - "name": "capacity_bytes", - "type": "int64" - } - ] - } - ], - "services": [ - { - "name": "Identity", - "rpcs": [ - { - "name": "GetPluginInfo", - "in_type": "GetPluginInfoRequest", - "out_type": "GetPluginInfoResponse" - }, - { - "name": "GetPluginCapabilities", - "in_type": "GetPluginCapabilitiesRequest", - "out_type": "GetPluginCapabilitiesResponse" - }, - { - "name": "Probe", - "in_type": "ProbeRequest", - "out_type": "ProbeResponse" - } - ] - }, - { - "name": "Controller", - "rpcs": [ - { - "name": "CreateVolume", - "in_type": "CreateVolumeRequest", - "out_type": "CreateVolumeResponse" - }, - { - "name": "DeleteVolume", - "in_type": "DeleteVolumeRequest", - "out_type": "DeleteVolumeResponse" - }, - { - "name": "ControllerPublishVolume", - "in_type": "ControllerPublishVolumeRequest", - "out_type": "ControllerPublishVolumeResponse" - }, - { - "name": "ControllerUnpublishVolume", - "in_type": "ControllerUnpublishVolumeRequest", - "out_type": "ControllerUnpublishVolumeResponse" - }, - { - "name": "ValidateVolumeCapabilities", - "in_type": "ValidateVolumeCapabilitiesRequest", - "out_type": "ValidateVolumeCapabilitiesResponse" - }, - { - "name": "ListVolumes", - "in_type": "ListVolumesRequest", - "out_type": "ListVolumesResponse" - }, - { - "name": "GetCapacity", - "in_type": "GetCapacityRequest", - "out_type": "GetCapacityResponse" - }, - { - "name": "ControllerGetCapabilities", - "in_type": "ControllerGetCapabilitiesRequest", - "out_type": "ControllerGetCapabilitiesResponse" - }, - { - "name": "CreateSnapshot", - "in_type": "CreateSnapshotRequest", - "out_type": "CreateSnapshotResponse" - }, - { - "name": "DeleteSnapshot", - "in_type": "DeleteSnapshotRequest", - "out_type": "DeleteSnapshotResponse" - }, - { - "name": "ListSnapshots", - "in_type": "ListSnapshotsRequest", - "out_type": "ListSnapshotsResponse" - }, - { - "name": "ControllerExpandVolume", - "in_type": "ControllerExpandVolumeRequest", - "out_type": "ControllerExpandVolumeResponse" - } - ] - }, - { - "name": "Node", - "rpcs": [ - { - "name": "NodeStageVolume", - "in_type": "NodeStageVolumeRequest", - "out_type": "NodeStageVolumeResponse" - }, - { - "name": "NodeUnstageVolume", - "in_type": "NodeUnstageVolumeRequest", - "out_type": "NodeUnstageVolumeResponse" - }, - { - "name": "NodePublishVolume", - "in_type": "NodePublishVolumeRequest", - "out_type": "NodePublishVolumeResponse" - }, - { - "name": "NodeUnpublishVolume", - "in_type": "NodeUnpublishVolumeRequest", - "out_type": "NodeUnpublishVolumeResponse" - }, - { - "name": "NodeGetVolumeStats", - "in_type": "NodeGetVolumeStatsRequest", - "out_type": "NodeGetVolumeStatsResponse" - }, - { - "name": "NodeExpandVolume", - "in_type": "NodeExpandVolumeRequest", - "out_type": "NodeExpandVolumeResponse" - }, - { - "name": "NodeGetCapabilities", - "in_type": "NodeGetCapabilitiesRequest", - "out_type": "NodeGetCapabilitiesResponse" - }, - { - "name": "NodeGetInfo", - "in_type": "NodeGetInfoRequest", - "out_type": "NodeGetInfoResponse" - } - ] - } - ], - "imports": [ - { - "path": "google/protobuf/descriptor.proto" - }, - { - "path": "google/protobuf/timestamp.proto" - }, - { - "path": "google/protobuf/wrappers.proto" - } - ], - "package": { - "name": "csi.v1" - }, - "options": [ - { - "name": "go_package", - "value": "csi" - } - ] - } - } - ] -} \ No newline at end of file diff --git a/hadoop-ozone/dev-support/intellij/ozone-site-ha.xml b/hadoop-ozone/dev-support/intellij/ozone-site-ha.xml index 1e4f14b257b2..8b4c457d9a5d 100644 --- a/hadoop-ozone/dev-support/intellij/ozone-site-ha.xml +++ b/hadoop-ozone/dev-support/intellij/ozone-site-ha.xml @@ -23,14 +23,6 @@ ozone.scm.block.client.address localhost - - ozone.csi.owner - hadoop - - - ozone.csi.socket - /tmp/csi.sock - ozone.scm.client.address localhost @@ -167,4 +159,4 @@ ozone.security.enabled false - \ No newline at end of file + diff --git a/hadoop-ozone/dev-support/intellij/ozone-site.xml b/hadoop-ozone/dev-support/intellij/ozone-site.xml index c06449cee709..9f9c7fe7791e 100644 --- a/hadoop-ozone/dev-support/intellij/ozone-site.xml +++ b/hadoop-ozone/dev-support/intellij/ozone-site.xml @@ -27,14 +27,6 @@ ozone.scm.block.client.address localhost - - ozone.csi.owner - hadoop - - - ozone.csi.socket - /tmp/csi.sock - ozone.scm.client.address localhost diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index 4a281d0d1153..d399b1f9cbcd 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -102,11 +102,6 @@ ozone-common runtime - - org.apache.ozone - ozone-csi - runtime - org.apache.ozone ozone-datanode diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/.env b/hadoop-ozone/dist/src/main/compose/ozone-csi/.env deleted file mode 100644 index 2de359fc5dbf..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-csi/.env +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} -OZONE_RUNNER_VERSION=${docker.ozone-runner.version} -OZONE_RUNNER_IMAGE=apache/ozone-runner -OZONE_OPTS= diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-compose.yaml deleted file mode 100644 index 99044feb6b37..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-compose.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -services: - datanode: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - env_file: - - docker-config - environment: - OZONE_OPTS: - ports: - - 19864 - - 9882 - command: ["ozone","datanode"] - om: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - env_file: - - docker-config - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - OZONE_OPTS: - ports: - - 9874:9874 - - 9862:9862 - command: ["ozone","om"] - scm: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - env_file: - - docker-config - ports: - - 9876:9876 - - 9860:9860 - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - OZONE_OPTS: - command: ["ozone","scm"] - csi: - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - env_file: - - docker-config - environment: - OZONE_OPTS: - command: ["ozone","csi"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config deleted file mode 100644 index 88d069ebfbbd..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -CORE-SITE.XML_fs.defaultFS=ofs://om - -OZONE-SITE.XML_ozone.csi.owner=hadoop -OZONE-SITE.XML_ozone.csi.socket=/tmp/csi.sock - -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.om.http-address=om:9874 -OZONE-SITE.XML_ozone.scm.http-address=scm:9876 -OZONE-SITE.XML_ozone.scm.container.size=1GB -OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB -OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s -OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB -OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 -OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s -OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http - -OZONE_CONF_DIR=/etc/hadoop -OZONE_LOG_DIR=/var/log/hadoop - -no_proxy=om,scm,csi,s3g,recon,kdc,localhost,127.0.0.1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-csi/test.sh deleted file mode 100755 index 1fe220a720bf..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-csi/test.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -u -o pipefail - -COMPOSE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -export COMPOSE_DIR - -export SECURITY_ENABLED=false -export OZONE_REPLICATION_FACTOR=3 - -# shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" - -start_docker_env - -execute_robot_test csi csi.robot diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-controller.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-controller.yaml deleted file mode 100644 index 511f48ff54d8..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-controller.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -kind: Deployment -apiVersion: apps/v1 -metadata: - name: csi-provisioner -spec: - replicas: 1 - selector: - matchLabels: - app: csi-provisioner - template: - metadata: - labels: - app: csi-provisioner - spec: - serviceAccount: csi-ozone - containers: - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.0.1 - args: - - "--csi-address=/var/lib/csi/csi.sock" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/ - - name: ozone-csi - image: "@docker.image@" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/ - imagePullPolicy: IfNotPresent - envFrom: - - configMapRef: - name: config - args: - - ozone - - csi - volumes: - - name: socket-dir - emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-crd.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-crd.yaml deleted file mode 100644 index fa5e296b1784..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-crd.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: org.apache.hadoop.ozone -spec: - attachRequired: false diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-node.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-node.yaml deleted file mode 100644 index dceeb19512ad..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-node.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-node -spec: - selector: - matchLabels: - app: csi-node - template: - metadata: - labels: - app: csi-node - spec: - serviceAccount: csi-ozone - containers: - - name: driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 - args: - - "--v=4" - - "--csi-address=/var/lib/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/org.apache.hadoop.ozone/csi.sock" - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/csi - - name: registration-dir - mountPath: /registration/ - - name: csi-node - image: "@docker.image@" - securityContext: - runAsUser: 0 - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - args: - - ozone - - csi - envFrom: - - configMapRef: - name: config - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/csi - - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods - mountPropagation: "Bidirectional" - - name: fuse-device - mountPath: /dev/fuse - - name: dbus - mountPath: /var/run/dbus - - name: systemd - mountPath: /run/systemd - volumes: - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/org.apache.hadoop.ozone - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - - name: pods-mount-dir - hostPath: - path: /var/lib/kubelet/pods - type: Directory - - name: fuse-device - hostPath: - path: /dev/fuse - - name: dbus - hostPath: - path: /var/run/dbus - - name: systemd - hostPath: - path: /run/systemd diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-rbac.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-rbac.yaml deleted file mode 100644 index d83ffb3e1f1e..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-rbac.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: default - name: csi-ozone ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-ozone -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update","watch"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "create"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-ozone -subjects: - - kind: ServiceAccount - name: csi-ozone - namespace: default -roleRef: - kind: ClusterRole - name: csi-ozone - apiGroup: rbac.authorization.k8s.io diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-storageclass.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-storageclass.yaml deleted file mode 100644 index 978016055096..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/csi-storageclass.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ozone -provisioner: org.apache.hadoop.ozone diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/definitions/csi.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/definitions/csi.yaml deleted file mode 100644 index 14c2ea30affa..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone-csi/definitions/csi.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -name: ozone/csi -description: Configuration for CSI interface ---- -- type: Add - trigger: - metadata: - name: config - path: - - data - value: - OZONE-SITE.XML_ozone.csi.s3g.address: http://s3g-0.s3g:9878 - OZONE-SITE.XML_ozone.csi.socket: /var/lib/csi/csi.sock - OZONE-SITE.XML_ozone.csi.owner: hadoop diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/flekszible.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/flekszible.yaml deleted file mode 100644 index 54203bdb664f..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/flekszible.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -description: Simple python based webserver with persistent volume claim. diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-deployment.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-deployment.yaml deleted file mode 100644 index d8e75782371d..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-deployment.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ozone-csi-test-webserver - labels: - app: ozone-csi-test-webserver - annotations: {} -spec: - replicas: 1 - selector: - matchLabels: - app: ozone-csi-test-webserver - template: - metadata: - labels: - app: ozone-csi-test-webserver - spec: - containers: - - name: web - image: python:3.7.3-alpine3.8 - args: - - python - - -m - - http.server - - --directory - - /www - volumeMounts: - - mountPath: /www - name: webroot - volumes: - - name: webroot - persistentVolumeClaim: - claimName: ozone-csi-test-webserver - readOnly: false diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-service.yaml deleted file mode 100644 index 6a53a4397f02..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: ozone-csi-test-webserver - labels: {} - annotations: {} -spec: - type: NodePort - ports: - - port: 8000 - name: web - selector: - app: ozone-csi-test-webserver diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-volume.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-volume.yaml deleted file mode 100644 index 4b1e44b206a8..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/definitions/test-webserver/webserver-volume.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ozone-csi-test-webserver - labels: {} - annotations: {} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: ozone diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/Flekszible b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/Flekszible index aad4836c61c2..7d11451c7539 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/Flekszible +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/Flekszible @@ -25,7 +25,6 @@ import: - type: ozone/tracing - type: ozone/profiler - type: ozone/emptydir - - type: ozone/csi - type: ozone/onenode - path: prometheus - path: jaeger @@ -37,12 +36,6 @@ import: - type: Image image: "@docker.image@" - type: ozone/tracing - - path: test-webserver - destination: pv-test - - path: ozone-csi - destination: csi - - path: test-webserver - destination: pv-test transformations: - type: Namespace - type: kustomize diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml index 55c865fe224b..87f542bed060 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -41,11 +41,7 @@ data: LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout - LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: '%d{yyyy-MM-dd - HH:mm:ss} %-5p %c{1}:%L - %m%n' + LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: '%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n' OZONE-SITE.XML_hdds.prometheus.endpoint.enabled: "true" OZONE-SITE.XML_hdds.profiler.endpoint.enabled: "true" ASYNC_PROFILER_HOME: /opt/profiler - OZONE-SITE.XML_ozone.csi.s3g.address: http://s3g-0.s3g:9878 - OZONE-SITE.XML_ozone.csi.socket: /var/lib/csi/csi.sock - OZONE-SITE.XML_ozone.csi.owner: hadoop diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-node-daemonset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-node-daemonset.yaml deleted file mode 100644 index 1272053720f3..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-node-daemonset.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-node -spec: - selector: - matchLabels: - app: csi-node - template: - metadata: - labels: - app: csi-node - spec: - serviceAccount: csi-ozone - containers: - - name: driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 - args: - - --v=4 - - --csi-address=/var/lib/csi/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/org.apache.hadoop.ozone/csi.sock - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/csi - - name: registration-dir - mountPath: /registration/ - - name: csi-node - image: '@docker.image@' - securityContext: - runAsUser: 0 - privileged: true - capabilities: - add: - - SYS_ADMIN - allowPrivilegeEscalation: true - args: - - ozone - - csi - envFrom: - - configMapRef: - name: config - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/csi - - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - - name: fuse-device - mountPath: /dev/fuse - - name: dbus - mountPath: /var/run/dbus - - name: systemd - mountPath: /run/systemd - volumes: - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/org.apache.hadoop.ozone - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - - name: pods-mount-dir - hostPath: - path: /var/lib/kubelet/pods - type: Directory - - name: fuse-device - hostPath: - path: /dev/fuse - - name: dbus - hostPath: - path: /var/run/dbus - - name: systemd - hostPath: - path: /run/systemd diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrole.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrole.yaml deleted file mode 100644 index 927ba6ff7b7f..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrole.yaml +++ /dev/null @@ -1,98 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-ozone-default -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - get - - list - - watch - - update -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - create -- apiGroups: - - storage.k8s.io - resources: - - volumeattachments - verbs: - - get - - list - - watch - - update -- apiGroups: - - storage.k8s.io - resources: - - csinodes - verbs: - - get - - list - - watch diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrolebinding.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrolebinding.yaml deleted file mode 100644 index 948e759fbe35..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-clusterrolebinding.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-ozone-default -subjects: -- kind: ServiceAccount - name: csi-ozone - namespace: default -roleRef: - kind: ClusterRole - name: csi-ozone-default - apiGroup: rbac.authorization.k8s.io diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-serviceaccount.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-serviceaccount.yaml deleted file mode 100644 index 628d2a1c5957..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-ozone-serviceaccount.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: default - name: csi-ozone diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-provisioner-deployment.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-provisioner-deployment.yaml deleted file mode 100644 index 81837122d16c..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/csi-provisioner-deployment.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: Deployment -apiVersion: apps/v1 -metadata: - name: csi-provisioner -spec: - replicas: 1 - selector: - matchLabels: - app: csi-provisioner - template: - metadata: - labels: - app: csi-provisioner - spec: - serviceAccount: csi-ozone - containers: - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.0.1 - args: - - --csi-address=/var/lib/csi/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/ - - name: ozone-csi - image: '@docker.image@' - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/ - imagePullPolicy: IfNotPresent - envFrom: - - configMapRef: - name: config - args: - - ozone - - csi - volumes: - - name: socket-dir - emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/org.apache.hadoop.ozone-csidriver.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/org.apache.hadoop.ozone-csidriver.yaml deleted file mode 100644 index aa578e95f84e..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/org.apache.hadoop.ozone-csidriver.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: org.apache.hadoop.ozone -spec: - attachRequired: false diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/ozone-storageclass.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/ozone-storageclass.yaml deleted file mode 100644 index c6c1c6c9d1e1..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/csi/ozone-storageclass.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ozone -provisioner: org.apache.hadoop.ozone diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-deployment.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-deployment.yaml deleted file mode 100644 index 04edcec9814d..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-deployment.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ozone-csi-test-webserver - labels: - app: ozone-csi-test-webserver - annotations: {} -spec: - replicas: 1 - selector: - matchLabels: - app: ozone-csi-test-webserver - template: - metadata: - labels: - app: ozone-csi-test-webserver - spec: - containers: - - name: web - image: python:3.7.3-alpine3.8 - args: - - python - - -m - - http.server - - --directory - - /www - volumeMounts: - - mountPath: /www - name: webroot - volumes: - - name: webroot - persistentVolumeClaim: - claimName: ozone-csi-test-webserver - readOnly: false diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml deleted file mode 100644 index 4b1e44b206a8..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ozone-csi-test-webserver - labels: {} - annotations: {} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: ozone diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-service.yaml deleted file mode 100644 index 6a53a4397f02..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/pv-test/ozone-csi-test-webserver-service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: ozone-csi-test-webserver - labels: {} - annotations: {} -spec: - type: NodePort - ports: - - port: 8000 - name: web - selector: - app: ozone-csi-test-webserver diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/Flekszible b/hadoop-ozone/dist/src/main/k8s/examples/ozone/Flekszible index 5562aac43c6a..57b167754bcb 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/Flekszible +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/Flekszible @@ -21,16 +21,11 @@ import: - type: Image image: "@docker.image@" - type: ozone/persistence - - type: ozone/csi - path: ozone/freon destination: freon transformations: - type: Image image: "@docker.image@" - - path: ozone-csi - destination: csi - - path: test-webserver - destination: pv-test transformations: - type: Namespace - type: kustomize diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml index 68a5697992c0..7395db8d2ba6 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml @@ -41,8 +41,4 @@ data: LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout - LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: '%d{yyyy-MM-dd - HH:mm:ss} %-5p %c{1}:%L - %m%n' - OZONE-SITE.XML_ozone.csi.s3g.address: http://s3g-0.s3g:9878 - OZONE-SITE.XML_ozone.csi.socket: /var/lib/csi/csi.sock - OZONE-SITE.XML_ozone.csi.owner: hadoop + LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: '%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n' diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-node-daemonset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-node-daemonset.yaml deleted file mode 100644 index 1272053720f3..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-node-daemonset.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-node -spec: - selector: - matchLabels: - app: csi-node - template: - metadata: - labels: - app: csi-node - spec: - serviceAccount: csi-ozone - containers: - - name: driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 - args: - - --v=4 - - --csi-address=/var/lib/csi/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/org.apache.hadoop.ozone/csi.sock - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/csi - - name: registration-dir - mountPath: /registration/ - - name: csi-node - image: '@docker.image@' - securityContext: - runAsUser: 0 - privileged: true - capabilities: - add: - - SYS_ADMIN - allowPrivilegeEscalation: true - args: - - ozone - - csi - envFrom: - - configMapRef: - name: config - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/csi - - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - - name: fuse-device - mountPath: /dev/fuse - - name: dbus - mountPath: /var/run/dbus - - name: systemd - mountPath: /run/systemd - volumes: - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/org.apache.hadoop.ozone - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - - name: pods-mount-dir - hostPath: - path: /var/lib/kubelet/pods - type: Directory - - name: fuse-device - hostPath: - path: /dev/fuse - - name: dbus - hostPath: - path: /var/run/dbus - - name: systemd - hostPath: - path: /run/systemd diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrole.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrole.yaml deleted file mode 100644 index 927ba6ff7b7f..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrole.yaml +++ /dev/null @@ -1,98 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-ozone-default -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - get - - list - - watch - - update -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - create -- apiGroups: - - storage.k8s.io - resources: - - volumeattachments - verbs: - - get - - list - - watch - - update -- apiGroups: - - storage.k8s.io - resources: - - csinodes - verbs: - - get - - list - - watch diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrolebinding.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrolebinding.yaml deleted file mode 100644 index 948e759fbe35..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-clusterrolebinding.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-ozone-default -subjects: -- kind: ServiceAccount - name: csi-ozone - namespace: default -roleRef: - kind: ClusterRole - name: csi-ozone-default - apiGroup: rbac.authorization.k8s.io diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-serviceaccount.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-serviceaccount.yaml deleted file mode 100644 index 628d2a1c5957..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-ozone-serviceaccount.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: default - name: csi-ozone diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-provisioner-deployment.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-provisioner-deployment.yaml deleted file mode 100644 index 81837122d16c..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/csi-provisioner-deployment.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: Deployment -apiVersion: apps/v1 -metadata: - name: csi-provisioner -spec: - replicas: 1 - selector: - matchLabels: - app: csi-provisioner - template: - metadata: - labels: - app: csi-provisioner - spec: - serviceAccount: csi-ozone - containers: - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.0.1 - args: - - --csi-address=/var/lib/csi/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/ - - name: ozone-csi - image: '@docker.image@' - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/ - imagePullPolicy: IfNotPresent - envFrom: - - configMapRef: - name: config - args: - - ozone - - csi - volumes: - - name: socket-dir - emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/org.apache.hadoop.ozone-csidriver.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/org.apache.hadoop.ozone-csidriver.yaml deleted file mode 100644 index aa578e95f84e..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/org.apache.hadoop.ozone-csidriver.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: org.apache.hadoop.ozone -spec: - attachRequired: false diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/ozone-storageclass.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/ozone-storageclass.yaml deleted file mode 100644 index c6c1c6c9d1e1..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/csi/ozone-storageclass.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ozone -provisioner: org.apache.hadoop.ozone diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-deployment.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-deployment.yaml deleted file mode 100644 index 04edcec9814d..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-deployment.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ozone-csi-test-webserver - labels: - app: ozone-csi-test-webserver - annotations: {} -spec: - replicas: 1 - selector: - matchLabels: - app: ozone-csi-test-webserver - template: - metadata: - labels: - app: ozone-csi-test-webserver - spec: - containers: - - name: web - image: python:3.7.3-alpine3.8 - args: - - python - - -m - - http.server - - --directory - - /www - volumeMounts: - - mountPath: /www - name: webroot - volumes: - - name: webroot - persistentVolumeClaim: - claimName: ozone-csi-test-webserver - readOnly: false diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml deleted file mode 100644 index 4b1e44b206a8..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-persistentvolumeclaim.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ozone-csi-test-webserver - labels: {} - annotations: {} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: ozone diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-service.yaml deleted file mode 100644 index 6a53a4397f02..000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/pv-test/ozone-csi-test-webserver-service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: ozone-csi-test-webserver - labels: {} - annotations: {} -spec: - type: NodePort - ports: - - port: 8000 - name: web - selector: - app: ozone-csi-test-webserver diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index 1ab2063cf5c4..6d24551b5d03 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -205,7 +205,6 @@ share/ozone/lib/netty-tcnative-boringssl-static.Final-windows-x86_64.jar share/ozone/lib/netty-tcnative-boringssl-static.Final.jar share/ozone/lib/netty-tcnative-classes.Final.jar share/ozone/lib/netty-transport-classes-epoll.Final.jar -share/ozone/lib/netty-transport-native-epoll.Final-linux-x86_64.jar share/ozone/lib/netty-transport-native-epoll.Final.jar share/ozone/lib/netty-transport-native-unix-common.Final.jar share/ozone/lib/netty-transport.Final.jar @@ -238,7 +237,6 @@ share/ozone/lib/ozone-cli-repair.jar share/ozone/lib/ozone-cli-shell.jar share/ozone/lib/ozone-client.jar share/ozone/lib/ozone-common.jar -share/ozone/lib/ozone-csi.jar share/ozone/lib/ozone-datanode.jar share/ozone/lib/ozone-filesystem-common.jar share/ozone/lib/ozone-filesystem-hadoop2.jar diff --git a/hadoop-ozone/dist/src/main/smoketest/csi.robot b/hadoop-ozone/dist/src/main/smoketest/csi.robot deleted file mode 100644 index cd64b67ef5ac..000000000000 --- a/hadoop-ozone/dist/src/main/smoketest/csi.robot +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Smoketest Ozone CSI service -Library OperatingSystem -Library BuiltIn -Library String -Resource commonlib.robot -Test Timeout 1 minutes - -*** Keywords *** -CSI Socket check - Execute [ -S /tmp/csi.sock ] - -*** Test Cases *** -Check if CSI server is started - Wait Until Keyword Succeeds 3min 1sec CSI Socket check - -Test CSI identity service - ${result} = Execute csc -e unix:///tmp/csi.sock identity plugin-info - Should Contain ${result} org.apache.hadoop.ozone diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index 72712191068a..79dfa1a1166e 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -61,7 +61,6 @@ function ozone_usage ozone_add_subcommand "scm" daemon "run the Storage Container Manager service" ozone_add_subcommand "s3g" daemon "run the S3 compatible REST gateway" ozone_add_subcommand "httpfs" daemon "run the HTTPFS compatible REST gateway" - ozone_add_subcommand "csi" daemon "run the standalone CSI daemon" ozone_add_subcommand "recon" daemon "run the Recon service" ozone_add_subcommand "sh" client "command line interface for object store operations (alias: shell)" ozone_add_subcommand "s3" client "command line interface for s3 related operations" @@ -191,12 +190,6 @@ function ozonecmd_case OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.tenant.TenantShell OZONE_RUN_ARTIFACT_NAME="ozone-cli-shell" ;; - csi) - OZONE_SUBCMD_SUPPORTDAEMONIZATION="true" - OZONE_CLASSNAME='org.apache.hadoop.ozone.csi.CsiServer' - OZONE_CSI_OPTS="${OZONE_CSI_OPTS} -Dlog4j.configuration=file:${OZONE_CONF_DIR}/log4j.properties" - OZONE_RUN_ARTIFACT_NAME="ozone-csi" - ;; recon) OZONE_SUBCMD_SUPPORTDAEMONIZATION="true" OZONE_CLASSNAME='org.apache.hadoop.ozone.recon.ReconServer' diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh index ce364d1cdec0..e7ed930a9530 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh +++ b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh @@ -49,7 +49,7 @@ function ozone_debug function ozone_validate_classpath_usage { description=$'The --validate flag checks that all jars required for the command are present on the classpath\n\n' - usage_text=$'Usage I: ozone --validate classpath \nUsage II: ozone --validate [OPTIONS] --daemon start|status|stop csi|datanode|om|recon|s3g|scm\n\n' + usage_text=$'Usage I: ozone --validate classpath \nUsage II: ozone --validate [OPTIONS] --daemon start|status|stop datanode|om|recon|s3g|scm\n\n' options=$' OPTIONS is none or any of:\n\ncontinue\tcommand execution shall continue even if validation fails' ozone_error "${description}${usage_text}${options}" exit 1 diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index f07bb9140957..74fe6709e3a3 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -401,11 +401,6 @@ ozone-common test - - org.apache.ozone - ozone-csi - test - org.apache.ozone ozone-filesystem diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index a1141d41e5bc..9d0b7ec71100 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -32,7 +32,6 @@ cli-shell client common - csi datanode dist fault-injection-test diff --git a/pom.xml b/pom.xml index b63164afa94a..7de27a294cd4 100644 --- a/pom.xml +++ b/pom.xml @@ -1288,11 +1288,6 @@ ${ozone.version} test-jar - - org.apache.ozone - ozone-csi - ${ozone.version} - org.apache.ozone ozone-datanode From d4f68ce4e22147b877a30d55902329b3d9cc1b38 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 16 Jul 2026 14:22:47 +0200 Subject: [PATCH 2/2] remove leftover CSI.png --- hadoop-hdds/docs/content/interface/CSI.png | Bin 27210 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 hadoop-hdds/docs/content/interface/CSI.png diff --git a/hadoop-hdds/docs/content/interface/CSI.png b/hadoop-hdds/docs/content/interface/CSI.png deleted file mode 100644 index 38720c3019cfdb186d792bf65aed4190a3988a81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27210 zcmbrlbyStnyEVE2>6DW05RmRhq$LC-rMp|Y5$TfdE=fVUyGx{`Tco?;uKhdbjB&>O z#<+iccQ{md_ulVb>se3EIp44kiqhyPL?{pl1YK4}LInbW)dW93AR&N%%Uu4TLm*g@ zo*y+`R1DoH9GvXUENx6FTs$01DNNlh%^(o>KNV?~PE0N6!OzZkTCmcwIv9x6jDmw_ z=K&Q=_kVm8R}5^y%P%PtUD@wLKv@=H_HG_(OY! z((&!)ZrJo<-pxI{e$VxGtMA0ZA&HMVZ_S#rhw1Ont~cChyfxJsN5Jd&V%mksnny-X_=#-^;Qt?I+Xl+~{S>&{S0tY<5bo@x5*@YukO*@nvZ2DFMXs9( zouts@e|1*8BZU$5O^&5drnIo>*GMdlR(R+~_E=)>8J+iu+SX-l(Mj5Ng|cN8bt_Ua z!T|w=@2lD_R|Z1u*rp81yB>o$Imfxq->8~p+FtSF@710$zE{_^z2tUXvV_C=yUAYAU)998o*@G1*oA z?7(Zd(TH=#Z+@2EBFVicYaq|7kJ_D$npX|pQOzz9IU@YQ@7uO}xcI|Tm|+jr&2zM! zey?Rae(q)e!O{<-1quz!vJuzDW2AMIT9;8WyjJbs(_{vF{k9AW3(f};4p(-j73)o$ z6}8-d>*-Y)Kh)4ZcCMAchRthHYB zXZ%X~eQp-d+kK{O_Ex$bj+9;O={U<4wQ(2k0*_HakuR4ij!_0#)?cp)j8bDw4^^yF zDo9tTrZmpxlb7Ibw_yE8(inuaV6m56h<8b&Si4@6`6cTP{$`i5{3mHHX-M&NXWVua zIU&2W7|qLMQ{4v7Bm*fJv7lV4+WDfQppfbH6-S1V)&ZxQo7DcE%{gcE-{P;|mvi1E z$41DuhziN6sW>uyNUwR%xSc@C*=<;t+|rcCY2%-!fqfUh1=0Qb)Rrxeh9XKX??H>@08cHFVSSNKEW<4qRr|yC+#V zG1j8Ly^g@ffpN3?>&Rj-F{?tbBs0zH9L+qA7%LWrB4W{e!S@@DK(dBPJISVJqnU|7 zI|la8K#ZJ}GHNgm5-B9SeBSM~%5?l<<8z6%smdNc!!N>^fq>#Q<8TT{I9UlT9^R<1 zbxpT&6RDUjEE&Xete@4K|2xa);cvIH=KLQ^CGFmm)^vAO`O@_yk$8q_!T8{6RZe5l zu4{!*N|FDi?)HBUptunmFQ8_>m$7?SoURt-@JdHBhe9UsWk9zW$@>>{a)>f9ID7%` znR*kEO+Trd)LV>e;>8)MnHsUNQ$HwClB)!Dt|J?~xPNp&6(90NxUAKX?R5U6>T;}B z%!t-_HQ2(MDGv`ThxF24S648iM#7!W#ZTCnz`P);^nt_=1_ni8jWj{>KB*I;Tw%}HZhhm`bfNp;NX2r}|ALVH9m7!$h7qeNSGi;$O>DTQ zl6W;o%}?pv&^j0Gp9FRVv9!M*M(LA2Ya7+0r2liJPUZ?=WXQea9{OmNcU)S6Rg~tH zl-Vy{SWcJ2R2O5D3&T*k7FYnFG@C&ra@FjxR;gS-2t$XjjCn{RK zE-5Gvd?StgA%L3oExbv{?|}UPv_@H6lsAL1IaYN|BRZCxv$QZu@pZ|wny)QFaV2@) z(2FNoDvx+jUl7I09>gh9kdeWC`1q~(aN&K4bd0kMR!|t@PJbMOeCUERaJZlUhcU)g&s5OT7g9|-)_4;CWZ3p8TQO^yt|;UasoV5W zIObn$i7$^NLu3Auoun0n5aw{onHYp17QWTR`6dTrqXVy4iI;gOV?pg@Nj_076wvza z9hLW2*r#e$#Km9 z33Q)aj$)j}pW>*(IMOk?5cInP+g_O99KILEG7M(L#9q!%H_j|*N7>FCk~@V{r~P5p z(U@6fTJps(Xl2YoVsHO4^r{lM@ULKdQf1(OpTUup z2wil*P@$LfWb{@^W=+~zY;OJVn@a|M7XOI2r07@U3%OS>G+`8Ct&v)|W;{w?{|O{m z3AjQdl_{RH`1@BFPR82Hp|mW`LHgaN_l+;g4-ycx8oW#gC-c&Ot{v*xoqN&WswJZn zt=jL(N0ybbtJJgMd`ub3 z?ORQzMfWy;aBy&Hjw+6OpW=&&CJA5uGR|-|%9AaUj&s9laq8`#kg|)(RW|0}_sMX6 zMsJ_FdSY;NqV_+rO*t+`nyBTF)jKPZD;&fl`C>1=X0ScX(%_6!FsGk&|K-wsSQ04} zu{ddx(Z8db_=Y-rDaiyS&B7E;Nyza~Z1mb15#6$8ocYL|z4cc^rlglgRhq+@(cLRG zRLedx3u&`_;dzs+!`k==i(_obzfIseHtD7f08?#G)DSD&s%MkT9^@+)<6B-E{i_q)v)KE@d$Rx zf*~sNI@Tcv=bvw+*LF<*;oI*bwl!|y-Z9q;CfIgF`)gK3bzK#Hp&!pXQCf4{QhjM* zOn)yQ60RL{fh8sO@<@6%mS7oUI<)P=+^2|UJLyY>gGUW^`x~q2Rfm$rJybg!XL@Y1DIXj|*^biUYebrJDt~V}`b|-|dB;Jd&9B{aPbh&=Xh3+oTw*R|+KW!$tIyuEarsl2CidmC%@J55ZkPOqmvwq(q*04F-` zdRLTfhT>yRK7B6H8q?Onz^d4Zb;{H25Yk2hZGe=p|L2wV>mSQ$_9*u;F*LAc{W_@n zt?A2>gQG?*)g^zzsf9?yQ8Tln{yM|3$H+ucr=JcPZ@<$a%FYz~xrC;n1yeVXjA*D8 zSz$^(%>~KFcX5%K;zM3&KOo+tOS4DHEXLGt2_ z;;2ZPqL2?1qYsZTBbD8*&7a*H_#^)ELc7x=C4kO3e@s|S=k9S8{&j+tB!>Zfn!~Yg zhth1sPN+Q7%NtU^*E`c@xw$-agRD3<#Fr-jRy}2J;Qh<~5+9h-(1cO$)awf5xJuom zSJHex43@2V#tQw*s2!{MyU`;(Yb?HAo_;;O1i6VZEe74rmIg&15x8G)&s=R$wkE_4*NM7 zSG(8UVJXJ#n#}DOlB;Z(Hp7i!(QI23kIbI_3GZDM5@k3W0#-t*>8PvDDfjm%U#%*Qi)D+9pz_ShF{f?m1Q*WStk-%FUNPVWz1O2buFY&$o&9W znfROT)Vany*|Kvv`)6PeE;hGPg}1Ydl1;nd;B&UywaeAjajLqXC}xM)y z!`}IHLy|?WRYpbS8}8TtTGRND;hMLT}oS#!JP-i@;6pE z(#L4%=>El&sg+4>4<{m{qQb3B=Dzz~s)|^MYr9zQvlcUF^keP#cw(kZgFSuBvSMUH1%vJe#&l}Rr2ke;^_S;D0sOR>AUx`fV0IXz>4aVhv6 zeX?)sOXIedQBtC3VTq=a3}4&df8XYL$>+Qi+4>R~UtiIxx>wkgBbCqV>P-|8cb@0v zKRyx+0k z|N9rNGHgkVPR#SW7#;zk#%hMq^ZIBE?5fHAlqyE(3`w;}-tg~a?(gqvsPRJ*byhQc z9%nSb(9?6Z?x#k;UNTKi+bP`E_|CiIreHHSx3`X4gD)Y_+jIw`S}r$pkEQd@Ww&_W zI+=}UgiTreE|!##VDr4Nc!M7S-3-(*uA_^K=|eC`H~voLkEXEql+G4w*IC^iawujB zNXC!}LuB5+&o>$Rruhb6&-+wTN2g{bNh+UxP9c&RAC+1AXO3(<-N!QZDBz-O=Hnzj zPt-9&uwpVYNZ{ahll6UJa+I<~65eS2veZpvR5O|`Qt0jPuK^plI$C}&CIc$B1+Oxi{h>`muPZ$gRMC{rUdt@ZoHP^^NvV zB>yj8UWDT_e@840Mty}%MixC~u?4(UOkLV1q$hXvj+&0{Ys0E0-qTvDJcoi5ag0s#_lfn z7drezfm7v~jwIcAQc+Wv>b2eJCNB&pGHLxV3U2iPzD}7}N-Wrjboz=^$UXK~;rq?q zUBu4qkz`ppxsB4|ELaM5_BddD7J&G{e3o^0GBVD&MDztN z*z{X)oq;8s|9a25tn9I9!$YT-F<5Qbvo%JKvXUHG^b!wGax9Je^z5v;cpn25)tLxE zDl||c4XIB!nKY>ek5w;Yqc37V#>dC!^zw4d*@K*%{QJjJr|HXG5>X~|)y*jSOSweG z?UrcZP#k_w-g$X>8Qpkdw`beVejjvnN_3iPo*F;oir=k*ka&2%4N`?>t2?`Hi|b64 zt4^(1)XMI@VS~fExSr=8ZEIWG@q&OY5Apf#MDgeSSzH#K#?Is}U_DS3#?GxYKp~I? zYwL(sK~b2#Sqs9nn9O+_FkcKJi!gnxNYrlTJ6>AR50Ftp^2+U9O-<)2bwM;ek1RI3 z9ufh+m?l5|Q}_P)9$4=HxJ+l#G|0wbQBjGQ#aP(YU)&um)A~W>+2=Cp5gi*Fi|UJb zwXgBP%z7#h9^BV*x(M-_#<)?mIJGAPv)1R{m7=WQakgB`^Kix=29D4B#vyaB zKe7E_h$<4Yrg*4V1=xvVohCoG<*yeC)AP-r_u>39KoS+7=6ZA=HLDe2rKAiX6ZZUj zj)+MnudPi45q-SCNh{iWwodB@(TUy(0>E-I2j*P}#!Sf2Bv%xt`S^zlZEoMYJ$lcp zgHloZv#lY;dt6*xiNim2&WG0C-a_Dq>FhaBDmFGZFK=tj#(wR3xVt;tU06a?R8@gX z;?OJRSQ0I_xJgfQ6|*mMaB_l+Od6FrmT4j-%$>Ze-uYG6i5hy2p8Ao5oKo2u9lk!r zk1xe>Nq!-|e_#eTS!i6|7YRxBR2T+Mcy!ld7Q|}(Y8z<{+KegW<2|^Gs$woot8l=zm&O`hG z?3YNUbK^`yFu47#FdlrU(99#@ihs{!B95~nWA^Pmtw6yS$5Jb_36F3m~ zk_?4&5}>Mn18-9PX`u|vJ^=Rqcdm*YVm^^o2x>DflX_&4LOM190g}n3tk?M%FQ{l; z)6?;wKJ39AAs`~wdR^PKJ+wdFS)<@Fe$gqTk$@@qazm`s?yUuLc5MIDU_HlDn4n@j z5X~HW3h9r}UP zI0Nig1?s)in6sOkhS$}>P@U-hY{hg#2dK6_dL||X|0Z*TrfYx&!DrKWY`^RIT&jWZ z0mjl;Uthm-3JVLf0ynzj!NI|Cwmsh(iif|_fs~Y#ye^)*)FTw}Zp^Te44|Ds6&LFK zH(T*OUn+_$;kXP=8g;yhOd7Mt zG*}=#_lDt|lSI0LhL47ZW^H>r81ynyQbu*!0p+Jlfq{V+K4W%ikr5Ho+1a((`k4Zs zHS28%A%uj4kVs-)mN}jZ^)glG!k8FLZmZvu4SW0h>DK8B`#8XJp%E^2d9<7kjb4xQ z9YnM2C@E?N1_qDu5i;r_v)bGQr@Lh5lo(>GwPPz$^$P96RT=Ny@yr*9h`UW04Gp}Y zxuE6bj0X|TZaw?v?qV;0VztB1uhPfoux_q0+rXJC&O8Q{7n~1BU1A`}ozP!?)0IL- zM&?B?<+bD7#i9}q4SaiN5Y`c#o=&_!SCuTH>Bo^Of3ZIoqa^yswJM^?!onh-C6o#h z(=NYgOT5eRN*f`Yfj^iVeMvV_^ZflGz%ix4z5UC*RLn*&kas7t=d3G1pf`D5exIH< z=Y8yRPd{!LAoD_tAL^~QU6hfP9lG|Ajv*@qL7E~@r(JI&`@^WOM6XQ%+8TgpIaqGd zsL~zA&;eO`)XnEpgWYzzeY;`z7vzhhWx)wka7f)hEvGLy{t!zB>bvSfElU*HkHnCoa>8jY$cjHjUSw1>W&p>QXsBE7;P6Ln3K8=nnfp_xya; zS@c}+eBbfB*|eLrSvRLg4H_A+D;zAWoQda+2v&E{)k5n!TdMrWa?Q92k^6&sSa2S7 zDS~b+pjB84qt6=T7Nw#J1hsp=>!;iCiqQ3v^TSCG=Iu((JxJlkAOn_b)gsg`jK)&F z`@7m98awk-v)$Xn+uOTDw}l7f&@GT|CXZjlCWB(Vcl~#?YSw16KMI#wJ3&L=C(&tp z_$4lGFX$oy$`#KQ(zp!IcgBS7{^qxyj|&n+NzGR3GLf@U(ik99p4Aw1!J;9+VTANB zXttg96MLU@A$ml!JyWD`n1<`+pC8mtoNdO4I<0lWL`qR7v*>Ps6;7uQuXye=gGRV~ zRpm7!qg;m*2QULN4jo^GS8wP`Pg@P(da^Py1|ZO({sA>uPfrh=e?kIo`{UIoaI!^+ z_fD`}c1tOsL|_pT4s`_~?#-0F$`o`R! zk6nuR5PDD3;q_8nT~+hy(5)s&4dh=!0T+w*2Do9mYpPa3YUyYb&{jab3T5;7;ai;7 z%Wyx2(k|>gh-RY$rBwdQmSx2TyJb)kOtQvwHmX`YFBw7GZFKQ>>JBg9zq;EHc~Hk5 z@2_`(e;9*okieiSg|cn`(`j4I_xaHc1Q-QrPw;=rwx%uU3PIDLT4N&15P46r%ZB>j z_xR{qYzd}6OiaQ8ng2bj{`d3;3iSW~tD*O!7F=?aNR$#V8&r}My+I1oT+n#|boYg9 z5&?M$d5)X?D*u@;HoAy(ffjB$Mu9WKpRX|u z0XucVNGIQBEu|W}C`Xvb;;*5>j|AvBxRE+Z^2upt zp7mFS9}TI~=vX7M+`fYZr-B5getTq14EP^I810`j2fKw+4KxV2*-Wp8W;#(d>qO&& z@AE7_El6bZv(7-uN2_y(S3!i3|9gO znl*T*4&m(?6WJ;G|APTM-}!&u$cqY&lO3H;l$GUtv9zu(mtweSkB?6txgAWeaAC2n zuzY6bQ3oBxPmfVGrF|RNyc9S_FtlE8p%gwD(J`9*n)ex(+xp3u{3O-KCI}I{g*6uj>eY-}p7w6PRtp?2@}flab@IKZOz!H>@?@DYb+)?S zLH8ge4>MvFawyGlV`YotHLi8q6k)LaLCdGgu0ecw8vZG?meWCzDyC9#y-i(@t71DK zy{t7B>J9Gxa7kkrpD+r~6--d>{Tep*zOOsKf{|__O!>7C@oXn+e66)eDm*s^Z&bBF zi0zABwLOGpxh*?#m(|!}#nBrC#Wlz>{7rxKD0!)R_&K8VqX{0d5M@=mC7T^!Xr9!Q zCzA219x}Ln$7q(DYC*Cz%m2#V(^XP9bfJ?We`CGERk0ryBCGNrtOdoPQ?B84J&DZb zvgDAv7M>nL=XMo50es^FKSolE>}J)9M3;p>JsR6t$55*fYZ!c$`&3Q)t6&oscc>K} z;La2%FyPA?idDF77~gfrpVXO0wfeFWpgU0ZV{{HhsJ9-^bXk0u)-^tRR$;*L4z~A| z={|~mr3H-(cnSjsc&7RyZ2UM0{+>2e^#!gqaUJze0k@&>kkb|N`FJ~`S}Ik-6G0$GD|#pj%uLjn3yTDA1c}dlMUrH7zR~;#jCtsR**`EPP8531 znvb^$dMpk9sM&ka&0twyvJ}_-}&fK7z|A5xVp74^tfh;y5sfRrWm((0hf8C zS*7hK)5Ppqjtjen_fn0(@T)8KSfM(C> zREweK(rXL(RgxKPQ zPqjxA4|%xc*6`=NBjx3ZW*z=BCCT2rt1oQIV;1}$qlMm6YM{?Hy;mPzHJMbZRqnai z<iWmhv(B(_K8QY9m@L~3EUQt|3O_$S9b%%=qO^=Kv5*P4 zwq$~tIX*R&A(#>cw-8soEV1x?vYa89Z<37;8wtaGp9=*4$>t8TpF0UKhqrWBOXJ#L8%E9|l5GESS^r{ONKY)52;ZRSU!LYWa$~Mo9`*=U z_*Bm)=(WHb)G^@O$r<+kGJzcv5BVTBb13tDH z5yEfn6YPyv{!|j$?ptVZe2PY-7*^bEdr=|aFfLvs$v+c|d|F+-V55SR*ZBmgUOco& z``SgJK@{ePS?uOjgxAhz(^Nr!*J6d16))5BW9sf_#yTUz%-bMCjnsC5V6k)b-2MXo z8Jb&9f*Q}h=b?#$9}b@c3Pl^my`$4gGxnc5p7k~}Q;{3+Dj{eXGK%rOpnN%sW(s+H zJ6vptPfDT-rhICHVd9yRni6!cLC}e=J6|Mkdbe4fHE$xH z$eWLqa_(ZD22tN$QF^(ny_sOmJMv)|3NH2fU})0&HJpUrj;mv>nXZ*eJ70&BugqlI zBfMCPlaAL3&KqZrH>;(bn+dICA<7rHKhf&wZpzZ1ALz^vKJw0~+!p+`jrX5hJ)&!R zDSbL_#JTBoT_JcPDUk~xOW;Dc60tMuY0Ue`yhlqBI* zhyp`KQkeT{wm0J7@IX<1s?)hSG@~tb_!r{7nNN3=R$|#Q&5^9we@C9tW>fh5>QvKq zEl4plq9U9Hl0d?anGskO788m*6G0mbNmjulyO^b zi|X~x$^&2fYtpRG#fIog#UAM2n@&&5M9K+X$~Yjd>fotvQe)q^d}hp+EaICnQaD}@ zY7UClltMSc+3uf!X39n-nEh@wrtPDg=R{bkhXH-R3_Ims8nYP;T6#hE_53mu%?#^^ z)+IEwNTOrBbg-S2rY`5 zTWbGhcx26D7EL>W&c9JhI-QH>o+{e4^ypsEe+s_PAS!&7#>sYaZrVASsZ(>&IoA@M|t>`i)1sc+aKAHk6Z-4iKtVq#fsNN4mZCpKNE6Us4loj-)ANq zb>77`U<3FMPhlDWI5417d^gQSMMZTq)LbXnT5SMG35zr-clXBW3xE#hSrRdSsxNpO zgaEMZEMYH5wRQz~*}86u zb$NY8`@SYnFGu`eSQr^0t(I$*2q@5&0GznyEPQ^Dn5I!>o}K#R1Xc# z%?PvgQQ>t1F`=Lyo89;^} zkC$^Wp`oEr4A|x0Z{&z=&+|Pn%v$~`KnDHklr(aceuoGIz@M;d@k%1O0HwP4H^Vd$ z*L!`qC=HrLD5>yTxA~{YV@1PC_v$J+faOh0XaFWmDI$_JI4C1*01pq(ZaR!xq?j45 z67F+uR#q!#R!1vU|NJ1cuM= zNeIBI)NkHI0_@hPVR3J|7&D5PcL$(^9b^~NzbpWR#@c!U3#Q`Mvtg#Y);T?|C_05S zHsgLYpFaTmxcQqOv-KbQ?yK_;JXfpSVT2@}oi(lq{rwPYNIA#sTA6z)hmK?a^Ta9b z>6G^eO`%yI73azDZ}%(3Mv>k<=NVywKPz1cgCBpXkfbp#m6fGmCC-Z+`%6RT05)y`u znf2K4Y&!61HrSB@3<`|sFG)$G%F3AJ7>Iam1_3_KpixQkd^_O>MY`b-&MeK9_j^zPf7r`T2EscSCNDT8=Gd%TvjmMDG4lXZbyO z=;&nJ|82Y3h!{=hjR7OMdX?VT(Q?Zkz?{Lc^sPHoJ8nurQ6NCPgmf8n8n8J4OvlK` zh(Od=2qOeeVLFA+kqUqpdB6=0mYU+hAa?$Af4sT32i=KT?fTA+6hJSPWl?i+~s>Yj})pg-JyMuuO|M#ik*zE}I{|*OmVj4FN z4QZ-ADS0&WE1@xN3wiNbk00BJ9NjB8!@uItUzuGr-(ee6DrDN`Z%f z@fo1_uS6a%RDE$Sa$X}JE{TwN%Vy5EZ~3y5eI-AHequ>J z=$^RnF4b`CCE|;W&f`U7t7O!^F5-5WXs93R8SBJ}A-E~>qcqdRbqsSYF2jmH+IR|l zk7;^RN-EESxe&4U?HZ=AxOnsEXdJAlRHYCFf)NrV6-5L9neg(OS*`??g5qM;d@1yA z-@Z9GI~$*@b$L2(fR(25+LI|OD+9bg*!S^nuTJdta!de(GZ>NG(vkoI_fb`^taQS7&G;_mKzAnNIWUBHw-S!=EYskdJ} z`k4RI^U?}@q7xiws*ndq>`dhCw2~-0;Ieddb=ircUCT#qW`^naexOFByKTmB1@LG}!3^c7WEL2252l1U9Y&_@DPb4Zg07DYA3y zWQr25tda;CAZK6yOuUYgBmeT1Z~DiC!gr{r13!&~G`6qOpdGX{tS;M@Owi{nj`I-o zgXr?dmQ1D>5shM2cRh2MOaL$eI4fUpJSY)s>v5ObfP|E*@&H(NzI2QcPdZ>)(MSan zzznUJ!S~j{fMR83rLnmg`{m2GBT3AcmrX4#s^32{1O^2;e*WBN!!uo`!3wbLbBqlJ z1}>ZV$x>Gvfb!WaH>+U=S5{U6FfP;e@Q-2bg8RjkbmlCSBnC+5K9}u4G$~;05Lj7R zS3Gt}0jMKyWyJ_C_DoDD=^y=@DT5rWw55X-v3Ynn_7$D@BFE!kp8dNh6_ z^z%Clw_2$MX$5yLKHgVZ%_gQs60pQq#V@lrIz1y$zSEOAZg0^`YI42y-}8A$nOqLB zQc7W4DUcht`5pS+lTX@3)cb}FP+IAFWj|dHwS^xJRy(r8@t9!Oc6R!~{}TKe0#HCPvVE>npO#`{v5G&yX=UzmE93*V5^PHu zlah>#44}YR`1nPl9g-3F<$7&tNz9+V#l>}j@^#^nR%FPNetWx{ojI%D;%eLMc3jEI z36NP>SlHL9MQFeWX#@n)Qc?(j8v-18G+Q*=zWw$qK*IzoiUBJR@TMRrbq*2(L|j}P znti|=AvMhfu<344q>(!T{&#z|Xn%R=d)oI3^07iY5qxZU#(D=Z`G6eI{9y#IBy{>h zx7lSK;NecL*y5pxDfNy@--+1hm5;f6^Ubpy)~B!G?6o+zKZMgNmu3i2utKK`ZgI2bus57r~RU!XNOh9os;cMOQL`C$r)9dz2W~X+o z^wIJl^5Ej1*XHoioMKz zsBNkT+~k+wVC8o2+w{sgvs!k?4GG|fS20No074uEl^qN$DPv}T8tjz-mHu=(_X|{^ zfB%N7I-cbE!tvM%(4fal#BC+SM{lJ-4Knc@W-6lTYJOf`iB@gVbWEKWC1@+yjr#|Y z-{X!FmB+t&bNiJEGM2Nq&Pe<2bo;x6r$Xe;4EvP4miOVElf~$A!f=gGD#=Zcl-1X6 z-4P;^-#KrWyQOq=?3VrKf7BDAvk`WVAgPya=wxJmmM)iHj{O?4c=rxQL&rXfac%X)hJU=)Co@M1XC=lX| zA4|lpkCyoX^B=&>1b|3TW<7RN%^e&ab3qrS)94TYIHm9-lX$PFZ@( zS2~sg(C4B)Ta|e}l}UZ)F;-%?XyxN;XP@dxy0>H(x=F)0ygHHF4xNas?ndM6PWL$; zVHxcS+yr}j1cICm`h`dNED0B!)q4e(MRj(Fo20M%-nr4{Hu8etN^Ln({oK&(vM1i* z`*<<1u!xC?X};3x0nJmGA~(=P@~WgJ4YV4d*`cPTl{|h}Em5WhT$?;7J?_`bZpdh8 zFAM-0snhDNY)|C-ue2Bv8W{`Y5Fz0}mJl#pR z=kR@ShZ1+7=Y$dhfI+7c7S05M1A{v4)2pkUDBhLEibqk0H3;CY-Z4mml5DZk${#!9 zcU)H0o(Eui2?>eGich?cd^K7X>Y%%R=jkZ`=mJpWv!KlL`COI$#1G>Ez%{O`9RDC~g12XzVburW_dn*%ZD9nX(3Adf(4v=#4jg*I5Gv>2hYOR6q6ggXxmuX{|pXS zxU}I-eHzJGkHjBQ=<1($mR1?3a+JT9oX@DmzoM_%Oh~Nq3)|h=%5#5cIx>1EG!V#jfSpGmiEg#l0Mry18w`QQMRB?|Xtvek97=u&dtOlR0obSHS*b?z4Uj}d zLd$~w)17&}iEQ!YOujVc&UoesFqw>kLZSEV8ML_soU;>1aY=8qx`9zs`JL&Ygeb_! zbYQRmVgv&=f)b4?GN6wbSa)Ev`u!TDKXK4Rv8T%S_Vp<@Iv9eJ>EWCZ>IsBLJvlj{ zU}iony6>=A;D8E>a^Ho3Mr9N5sV0Dk23#2&o70vIoqS3s=m8@sW7y5*>2LtEvI*$q z^_$a8p|e41AkInyuV%!EKBWMl8Z_{-Iyxk3-#=1vaC`&05rZF^DWLxV^=~_ZwLJ$k z$-gUg7a0tIydkonfCieY`5bb5KOcbm5%Sm&NJo=^_n^CbdSc_?yj$-M_SlT#4Gjx3 z{PWW?gU>M#NUcuxt2$IlRcLRqP#m)@=~=bZnhd9YU$nRUT;DA{-Mlfn+q{sM+^jCs zbdKy5X>8-oB-XX}TJ?Dobd*iIW;ojX&;A23YctpR?C!B(*4sHHoRbIV-vPDw^ViCT zWK6EY1((l4N!d!8$9J9Pt%noSc*|rruTDU;oS|Ooe+OGZBjJnvu2#ljx76rYfT<4X z4G<5_rndT5kP4o3g=hAYz2a;RgHA|-%4i0EyxXczGH6`ni^_>?6{h)I{$c_91r-5BaY}sYdbKjv71`vJl%m4hYlV7By_3cXt`+X1M zV{$Q8vWpL*mC)YT#cbYhE7_HyBx}dGl^B^rH2+dASws&e^O}k4vYp~J^o1s&IRYF; z(Q5bKY=@Sy5+J&We=<#nsx=^rb@B^cYp%2UptF5E2JFpgFOCx5m>=l*pt2~1YbILS zU_gk~fO3M*qB98M3$pv;PfgHhl(&zM2jGHV>$eMmbZNZW?gOMpNnk?6#>Vz9FKRvb z{QPh}VYAYz<>w0;xJzyJpe`{G=Hi zfk`78P^PEz#xc+_9l&?I*1r1H1~9}tHWWZS_@~Yq+Gqm@x-cHrvj-gpXd37MgFKt< zwOR69re5*um|6h%{-Yu+FGBZ$m}~w)(Qd|~_tWjJ{p}wl$|o|Gg*t(tmK&$Pm`uBQ z$E}Pb>;BQ%y+15nDEc02|6HQn6KFj5NZ5N+(+?w_I%( zhUR9#c)x)FIcLkEvHQV~feWH`sJmHA<>`!=tkyF!Gb>;=06eO5WCT@TU%%M&KU@t= zV>+7mN&{95TAQ3ggCHhF7J|E-3EI9ms_Io-d29hpd)8dXjd{8JmO**u2Hn`6D=!8- z43WIvx8C3FS~I6BWC*QQ&!IHTsxQwZNp|y(k49*gi!=pG*nIUcHLy-EF17$)AMbVs z*Eul}Q=IK9uzFisTH1i7A?SM0xF_(0JTx@q?&Wn3IsIMLJtj@v{f%YEsTa>GyB2N^9bOicwIMA=PF)-86Y!&TSDCZ*p)uuU^ z6CgJ8)d8R|Y!L@K|F;N1r^zWePcp)&cA?Q}8-*@j*1nR2+X|Q5_l6Rp?|ll-q+ULm zPI>;T(a{(HW^yhrE?Kz~0|OyocCB{VQ-K~YP@O>C5Kzy*s(jHfF^hrn0qAdmpi&Xc z;{-r<;SC9HoXGekbqb;GKw}tH8(CIcxVM`0^nSC5vFo#+>9%-NV=6&`(|pVJGm4)4 z=atFdku2}uL+RT=DD3qb5D(j1@1IfX`Ns3ke}1t9)*7 zi?+qJ5;RdR`?Kf@(-%Zl8I&bJY9pU1kO(pkmqI>3wVKlIoLI6SucJu?OH_-aMRv3_ zH3@(O!HFp5jVL&i9UubD2V=~gFat1}B5p0Acab5aX$tn4x7m9?8q?8TN{whG>pb{-!rV8Db zq}^;kJrqem5%>GUuNlrFVX{bEu212%^b~dD3V!)~D#<|bphi;6vIrv>GOi`tz!0s~ zS?Vcoiavv7Fx-4>2sydiMYbXPlz=cJ{92VJSk&m~&0mJKjvyelpw*ZQ3$z8O_`|W^ zbIUfKJs=v$NO%><1AL{Z5b>zSoy?gY|B?DFjUm`z^kuD9>0cJZkg*VP3v*dD@80a@ zDi0R$n^u<1kvuptT32^~xnH(D$V{rxp!2=n$2Sazfq84zi}e*>m;lqu3;)J{*i>Hr z=wHwy59{jrInS9*U%8@cON_Zu$H>bK;i&5K53@6&h2hP=isKL)ILUHhv#RWN>gZM! z^QBM$z$T8H;txN0q@|=+;_DRzbLU}2fKG&x(oBP#+!5<&W;@Y&XD-I{&)G}CCkBDw zrn;pg@;3{VpYa>Dt;;3kBAA+q3EItTN^?K2Z(gnXL%<=J6@iQ~-n($i^jU_kEN@17 zO6b!{;&uy%9pS8wF4qnwa_<2Zw+57dx!GNE#dN^rjq(VBZE}qW_?;LS)nRhBc&z?p zHL5J;otLQwK9>4|v~#o_=BDv|u`6`NLKHBlIn^-%QPimMhOC;1>mRTlYIwOvhGh<+ zH+AoX#TVPlWAR?PS{Bomi7S4{4>S+_CCtPBb5e&h@%bn~<D-?8ny+dX3{wwF+($AsZ=cP&9Y;OP=1{3mp{CL>x}z9Weo-e=zfu&jBs+HuY4I) z5MW0>#UcRWO`xAk1`lZ{f(E7#>pu@@z}--~I=*F){P-~eDr-4#ZNUwMK8xYb26QW2 z3M@}p@aJs||Mk{V(5XWy(r+iMQ1uj<3__4jEg)>*cQW2cQb51+|Mg(W|M95Cf7bJV zarOUG?z3UnDDx7W5Kv;Tcf&IvDN`&Qx(+8WfahcWv&>f3YQR~Y0x7kuVJK|h%uFK4 z(kB2Nm4y=F0QOOez-LB6MveoP83zh7z(PMI7GMGuM`*_wbZl*Oqz3g-h2X!QKyj=B zEe4lDd>~Z8^uIUs-;4Y|u2Lxh-k@y8b7Ib`YOXY)+pKokDcytl+>J#4Eh8~FE9~lC zbOD#YodAc*ETW~S&$lE3D{3xYBog*?skHREJ~9P^5#)_l&5MXWz}yKg$AIwy)R+*J za4;{f#QQw2hmg;=`*^jVLn$dRvMTf8-?yE>JAi^oFo{)v+&tT7hl^P-0s7#};P@Ib<&L##2bAa4madF66-wCiA*gz$`18_G`-}@|YAD+hd37k9ux^FRG zZPccL;%2*fWpY==xE)9M};#O_kc$C&4lUs;zW9zx*Ws*Q`&XM zQ`!Ih%W5G`C8bbQB!!I3kfWipSCY}NO3En8ib5nRqq0d>Mn=dk-K2y#$q3!DjwH$G zSkL?FdH#5w|9-F6^T*w*m(Dq^b6ubBXS~Pf244gM?nuta9}ios^n3YN$$r?qg|v_+Ma}L8yh)&$>9`D$r;HFvv56(z*zyxh_Hep5=`TABBn8* zCfukSVBu2?WF&}@Ye`BTS68E)9z{1xHa@NU!2{mLSCeuXJO&@mdjM_UWqdG8moG9# zNj&(0h|X#J24K1%vpAubZE!mVC9Ak;o-fAo=A2VX4bc*$V<-dTIUw${xID7cCL#Es zpcU}hyvOX}`|M=??A#nFNGE0VWaC+le*5;FJFI>1f+Rz{>*IbF`(xI-ouU- z;FUuW$W?p?8{An^vq*k4`LH8I)!|vO{SbS(a|rzxc@uzvlGPnCAq>m_zuS9mY#$)D zxF{cBMkK(C)+rStP<*XTz8I1|W*CT6GU&UYi>z8&R3a7^7eRN1ki!(dt^xGxFgqZC zPS>tLclOve-2d4Qa}yI2f|-Q{=*+Jg3evpN_KOZc4jd4-4!{=ZkaKIDGdh=!`<96t zG#u9Y@neL(R%&W$#`n`~hFV&|`dS$XY9n;iP%gvILjOI|%8<*C_L-uY(9qX7kFg|v zY;83HyW9z0H0KW_&;|&AD`v(!QxPmi77`Qiu4Ktw_}o|Mm>L0JSA#>5SZV~lE#m!oVLzSf&=1(2Him2JlU+f73g0VPOq?Ft7s} z;O;g&(W!o2tqf2-S$OouP<-g$x_NWBHOnv~8E{V)+O1>|=!S^u;JbH?jk1i5pbBo9 zsmMp8>sAnE2-bNqY9^iLU|)7H6u&WMmAzK%6pW-Y2U2w_)O+ zI)|H=NWKL>_-&7VJZTjtC#@+C7^Hzl0EeCz<97?mPzB@1Mu=zwt(|V@8F|*MNhZk? zF91}7>CMf$3v(h;QhIF5mn+5TnB zsus%uKa+3eS-UnRLZ4oLPuKw#5$!p73~%JCr@wcQKgW#~lb8Q&ufoo|$1o`=NlaPU zd}pfRwQDa|tz4;xYix-=QFbMpygqAdYepf=kmS34!oI^k!pW~_Sg|nO#n=m&_%`_( z955pY_&07Gng6m!a4F_wq20aCQkS&OJZt~XlFKP@FvOUZTuM^(oXvYIbtd4Y=b!vc zSYs+ZVOlh(ZkUEAFCE2YC;R}!LMRzDh*gvS(cWXk((cj$U)nB>!{YX>{+{{VEIlhX zlMDw82I)uv^j07#&R$<_&z=>9_!{p-ESA1}Nw3*HmxN7V7g5X4i0WG`lkbyymWedZmy~~P-YsE4h?k4@8*XK)(=&E zDxsqeGo(^)wk`tl6jfV*aGUY`8A?j;%n_u)Hk?`~hDC#f)c{ZCcB_>uO}VaCGa#dO zjvIgDpNr9%ObcuBE*=7z)Yeg8O9M-hNy7XK>|W?f9FXDYfO<`uRAyo;=Rt@XK}K)n z?3{0LCi2v-eB9u3#a*anAJb;OcNk!xSL)6C3G`U!RzLuKvY>T7R75g0LeYD^1Rs^U ze*sC~0Zgp+yBwVT-KKeJTqhydFI>zLl9X(=kLxHpfu`F4I)8^B-KE-Ft;w5>fv5rO zpQ=Z|ty>uw-|6q*$sDcvdth_t0K{G^5e1@GS(h83k8fiD?l@PDIGC&fGe{@-5EqI9T2i!uh&f6uZamgn3qgoBl@%KF%AJ4 z^+mQ#0yHFbqZb!WklBsYUl@T&cLPM3HUKYAiplvQbEhjCaNi!0>B54)+`e@OEm@PD zM?fHlL<33NON2g|4vH>x=6R>5k9q>z2WNgFvI5{uF3g?wjd8TYizkT~=zHl2=HL}sh#jH0JPyF|T=ti5?neDbE4Lwb%96E(TBHQDM-W1#R6Yn zZe4o`CqPR}OCM-+>XRqwxDGFseFC<7M`1mYoCj#_)_kWzIX55 z-+wxm@?(s~Wvs|Y#C5{g;d5RR`pj_FXG${Ytw2sMJ^;EI%LqrX&9FTK@wBBWYQ@B* zKi?kcM1;6bRw)PyCY)S!l-vOK&xa{_1m z&JLaU_sa!6t5FZ^fpg}x_oMHG(O~dy`*nb_<_XXJK!Dsrf^E1xW%&$sb)QOZP@o2G zBS-6&iXUP>1cN)Y|2Ns&a)|2I@S>c??ZAgKu3S#fQYE5Xs-uw z@vWkwK`6!wO?A(YJr0Ay*;-Hgm zt@81~G;sOq)iktAq~Zp+95XjB?{ZQYa|DJq_lbYiKKWJwANz(*dzQhV4?RRGeeN-7 zbMVdvb!X?j$ed4}Jn0)7JD#xL>*wU;1Fklf@31k7h>IWd&Rb;P>yPs@>&FUSCwyH% z$@v-jLIL<@;~euN$j(zr%azl~dFo7<%0ZjKrQ)mOdPb3dHdQBh$vdyrt>$K(TrcB( zjx&C#gWEUdUD#`2bFd((NXvxpUpA>r_HD-;{CQtpPZkvuqW7&a*<0eP&#_WZ zoho@zEINFYsj^Dd)OJak$=-p3K6xUf=q?xT8EIWkQNXK2 z$%`QaDW^X~FIn$|7lu@qYA~9%U=0D``~107a!Ly9$86X+AsXz5$!PPZPoEI?yu3JZ zNtKlXPoF+5f%zX5YE`yz*C1DRadC0y=-}{hP=5YiI~hJ!Hny*b7?V1>si|on{#J*| zC-A3z?nrx&S^n?Y6II?S5|lM64Tuq1-_LZEj2mzT$~pC1$U9XfFc7zI_b}PQi0<0u zkK9o+-I8*M#qLz)3Sx00%x8nS4U5&K+j>nj8V`Jr?wCK@j(RZ$9*gbT1w+2mPh|*6 zp4PnTL9o%Qrc?}p#$yMIyL)=gPi=@N9q%3&T<_Fd84AqQUlFz~TDqq z40W4Ty*C>f849D?@kMTRtHI;Z(V=Fmk5cE!jKFJ+rP_hTBh zP*6}P`jgks1+JKC2qp*ezdRd(;&}#o% zRT8IOvL;Ri5bW&i+ji{W^ISKLT8IMt^aD(9RG_89n5FWoPmIB9U*D|aU1+>yeLH) zR#u6O&w!7u&{KjNRl_@byyqOzBc^@HI2i9WeFClSG75$Axxh9|G4epJMa56dyrhW@ zn-6N>rJ(Tf1)ErSu*u2E@e71xXYXcAkA)1yUtbLogH!Q)fJ&i)KJyfu3gs)~J)D|) zKReIos;9J&%0`DK6Z}H*E)@4%%lh&zwN3VTyz)lOmH(X2tq3SAlySRmVIeRsH2(+# zsK10HS7vfDGg?#rA5IPq2Mi6@ffhcScZ$l*!!xDZ`4j444a&k{Z6U^Sk89Wd1#rXv zqc}hR075Wk4E>eE!{OoALddXCO?jXFpsB6B8fT#E*zwB)4LsfD9&5>wZn=AOWaL+* zLT<|E&d$?le;qvHv;#3i2naEYh={bcx2NDtqorI%$n4wKkKWkM-u?#C0MccPb<8Su zc0GN4X37oYtT!0iIH!G<7G|>h#bo$!_YLFTiDC@`Zq_Ayb5ToueZTp6Pk$wqO?-Ue zj~?Cb#T-h10dKB)T5C~PC5UVpS=pNU`W5gYdTx{0y))SF+qbhEg8RFz{EkyR`ubO{ zUJd>DPp41Q%N_e+#SDSQG|Q3|=3?wD;Bayp-=oqojI{&YYt}fo6H@TO2JVeNrnfF= z1r}&(RRe! zK7)jkHi<{FsDbjXOLZQ;XSt&eF}w&hvX+}$QO$kbt5Hl_$Z;|#C?YEA#ZKuf-%b4; zckkX!A7`Y#1SHa>{Gp{qZ}aBO+f-F8sZ<~WB9MYeijrGhE?>G7K0E79p}cKqxQ{V% zW5hLYifFc{aEwv3F7+=nrW_=aTTuz?mMY3l8V~y!~@(@jzKn{o4m3=W6`e?#IOJ0_PmfYvHa=rLGp-FOG8%i3&;L zE71NJkmCp@YJEW^fCs=pjs58heafcPr87V+-T)E95`+df`tbALl6CF#^S|6pKxK=! z(P&>GHh=~vza}*Wa6orEHRY=I{n7DpE+r+U`KeqV&4UM*frh+;dzm>JC7|HU3bgqZ zXhucPN?uD(3gkE}&5Cv1NZ!_G&afZ}yIJh302-R%KRGqosGz9$s=j{s#)npfiq(Y+ z&pt@E7du&Ei-lENy56$ZJh=H+6Owk#Tiky2VAM;&lOqTxCnr*7i)f~2E=TP%MUh9e zCbih%Ypf#jii-nNQ@20M&ZZzx@4iv=ViYPKG|*dDTUXaRlYbBeFxD<&*avOlMXrIz zsuVYVEPHr^)6{jr$FzU=_dexek-Y5mbjgS5?JX@6IalML4{x36&bfbojiHed63oV9 z!*huj;4|d3zj`v)W|4&MS&N9#9 zfkyAQ(Bf^(3tDV1nP1b=Zwq2!?4o7<*|!osH?FBtS<#?NJyay|-Jerajs`EDJ$v7G zdCY$NuC0zvlU!FP`#~CwmRDHF+N@}ufXNDd@mN5!r9)TT^muO=#)FMS z9#%{|J5{zGD+2VdA6uB8508lnL8Gls4G#}boOb`RF`bW(&+7DPKQyR;(~r?hX}79T zsck2aUIfx@zYT3YoYtC@le5~`*cfE*nsu5ta&sjb-o2ZcsaoR4ipeS9 z`0?Y*tVhwwb5oFDH-jhkO-SgK_$0V)-Axoa;a$7lS(QabMs^yUwYR_B-EAIpxa_>$ z;+T_<0n2?0Y?IKlv<$=6m@A_+9TS$qX)8=4>w=R>hm4-Y&`1;ZwVZzrnN9iBr<2Wg zGW6{OgM&7PCMFswEq&-FC1quA;o8XDbPc(6MlH7q5;ghbEPA%3T{4_Aqk6GVZ^UoL z2coXQZvIUp;jz`o$5h81XUFIOO@e?&e`$}k;X30GlA~Mq+O7#2DJxss<(N_T%$XG| z4+#oNe?YDho0^)M;Wt(i=Du3w+anOCJrnaqc|sEHjWfr33PsQMva|EGKBe&$?m*hh zT<0*`Z^(liNbwznd+uxE2|VDh+Ybl{C4bO2F{!gZ5l@hts;Yt7(!~Y6e&18n+o4j) zw`mh%1n&yT$QThWwcEF#hy(LKufnbdd+Dz`RJ_H-#pUbcaA5_Ivj5tq%(Q>Dk7e2l z4TsxPr~zF^6k>-LM=aZ6mfHK@KgC9;nf{Ady9X@=yxw+XFN`d$l zbp1lJ-QC^Y?n2%+(2>=@duQO&`@(TkenG)k>^TB1kAv9gUdFN3+gB%p&SSU%fhjDeKjjFZ=-e zO^Y0OTow6Wy?)I^V6LyPulI$~XMTQO7r}(7(5}eL6P-5Ui|Oqv7Fpa#7B$6w7~fB9 zc`=V4=be7D>fz(ZLc4c!qR2rgtp=ABu>~CtRua5IXCif}PYvV~8)ioG62axM5hUP> zbHt7WL?$M{OQiIh%8KI8M%32U_JQ~Duc%NV`?9A`caUqcP)BAlUoa6U7Ssa=nCWyn z7(_)v?b0Pl{GEHlh9Ep4ILBC&uKu8UcqpLXOs>pKuX9Dx&KJ_@%o}bK$b9Yb{_O&IS@?#`GmCnkKr6G;#Sm2Pa!i(P^g!XOYko zTSv%kaFw5_9HvxiRRx#oJU6K&!UJmg{CNesS@b-fjFB|bR7B#lQTz~|!XDP#-F+u2 zYCU?+udwvMtaa${;lt7UwX==cHT7ow){+;rFijf4YHO)?1XTsTyma$7&eBA*k*+$B zhgMkkLHGs-uOg~(#8!kJWYN^KI(wECqZJCVHQ28ibAWWKzJLGDajgd3AZ-B+3ET=n zFLf{!Ko&z^#8@XRCbkOHd?2E|z=m_3rn;kJwz>+#3+Z=R^a@Q@EDMrK_aUM2eg_8! zm^5tie6Uc7zKG5ZZCbO{5;WhlHbSS7m6$8%e;p-!0|O70%?-VO#DXS~@Ws8mbLWmE z*vxN(gM1OvNWREZDzuKawwsuqg|}^EMd)!;P${i(U%Bz63JRq0&C`BVUG86oooy;; zf3l`_o0o8d3p;>Zf(NslfGpq7Ovl=KD^cCrx)vYd(8ufxe^PR0<|-5zObY6HdK559 zm#4p7bwhcAVQ}N?GzJx1surQ#%39Tl`;w#eN%okH8e02vRiuIVtPjdd-L8s3hE{@ z0){0i_n%fWa&oU~Ygr)*HRy7uEqiytF*{ zfY%aj@hN334n?2KJ)}{wbuBFJf+YMGzj_!8SuhxSH8Ev9dBWON?1X3jw^y6ZCOJ9B zWi7jc{m>)-ofk{NXLj{@RFs#u3G5~wN?J1mdg*GjG%*2QQ&3<~jQI$}WA*#@93Tx3 zX=rG~Fu<#F3kq@(L7}0VSFXt6n?y(R!d27!`E&igbrcqqytd04Gkkf!?}Ub0J34YA zK7hHcMDltA?q>YY1Q^bMpgufaNFvmAb;CP1oyE7Ns2dopr9NqFigPS!Jt|SbN|27! zU%!441Z;oKWlUUztKz9L$I+gOvd)0J^nKFO{y~Z5<(uKpqe8fL?AWnOlY_@AGyXo; zBSTmZMP7PWSJ{^Z)Keq>CQ#3Ky4@E`)8enuO=vL>9|-aezIAB0NI zqLQkrAU4D0y?7B262eYp#iT*S%+PE2H)~X*84iz#xCv%ttD+(wh68BOYXB%?A6nAt zZULn$;^Z}aP#W)ph5^7Xxj^tac>DRUTt3whGyce^?CdmyZmtiMAocihP9i;;cR9xf z5=tSevB{`SeE_}dt-!##Q`dtm+`qD6L!tmUO4B?m^}qoz+a$CA#1LUNr!niFnV+4YZ<%9aSw{K@_^)5VkO#-%8Ng4T8WcBcvv_87Bu&bb9N-Zs zSngOgKOjM$#!#>YJBm1!VJIeR75WzP`1lVU*4Dl#ecKkDb=lOTn3#UFf^Y)X0##4E z*!1q5Kh6P>n3TkX-EveQp#(8KJ&pE41E2^&VEd`~G&bG}4do>Ia3V<`3vHUHMha{J za$A`c^iopOd(3x*&`SwO0OZ~1>vULKG)OM|WwJkLn%dNy16F?j E1r1&