From ef524f64d2d449a7ebb70c3dc24fa5a3fd2cb733 Mon Sep 17 00:00:00 2001 From: analect Date: Tue, 3 Oct 2017 08:49:51 +0100 Subject: [PATCH] prelim set-up of resources to test kafka-connect source/sink demo --- addon-cp/rest.yml | 2 ++ connect-test-debezium/Dockerfile | 9 +++++ connect-test-debezium/README.md | 5 +++ connect-test-debezium/connect-deployment.yaml | 34 +++++++++++++++++++ connect-test-debezium/connect-service.yaml | 17 ++++++++++ connect-test-debezium/mysql-deployment.yaml | 29 ++++++++++++++++ connect-test-debezium/mysql-service.yaml | 14 ++++++++ .../postgres-deployment.yaml | 29 ++++++++++++++++ connect-test-debezium/postgres-service.yaml | 14 ++++++++ 9 files changed, 153 insertions(+) create mode 100644 connect-test-debezium/Dockerfile create mode 100644 connect-test-debezium/README.md create mode 100644 connect-test-debezium/connect-deployment.yaml create mode 100644 connect-test-debezium/connect-service.yaml create mode 100644 connect-test-debezium/mysql-deployment.yaml create mode 100644 connect-test-debezium/mysql-service.yaml create mode 100644 connect-test-debezium/postgres-deployment.yaml create mode 100644 connect-test-debezium/postgres-service.yaml diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml index 60fcc31b..97b81349 100644 --- a/addon-cp/rest.yml +++ b/addon-cp/rest.yml @@ -21,6 +21,8 @@ spec: env: - name: KAFKAREST_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties + - name: REST_PORT + value: "80" command: - kafka-rest-start - /etc/kafka-rest/kafka-rest.properties diff --git a/connect-test-debezium/Dockerfile b/connect-test-debezium/Dockerfile new file mode 100644 index 00000000..c220f62b --- /dev/null +++ b/connect-test-debezium/Dockerfile @@ -0,0 +1,9 @@ +FROM debezium/connect:0.6 + +# Deploy PostgreSQL JDBC Driver +RUN cd /kafka/libs && curl -sO https://jdbc.postgresql.org/download/postgresql-42.1.4.jar + +# Deploy Kafka Connect JDBC +ENV KAFKA_CONNECT_JDBC_DIR=$KAFKA_CONNECT_PLUGINS_DIR/kafka-connect-jdbc +RUN mkdir $KAFKA_CONNECT_JDBC_DIR && cd $KAFKA_CONNECT_JDBC_DIR &&\ + curl -sO http://packages.confluent.io/maven/io/confluent/kafka-connect-jdbc/3.3.0/kafka-connect-jdbc-3.3.0.jar diff --git a/connect-test-debezium/README.md b/connect-test-debezium/README.md new file mode 100644 index 00000000..c3cb9659 --- /dev/null +++ b/connect-test-debezium/README.md @@ -0,0 +1,5 @@ +### Testing kafka-connect in a kubernetes context + +This `connect-debezium-test` feature branch is an attempt to get this demonstration [blog post](http://debezium.io/blog/2017/09/25/streaming-to-another-database/) from debezium running. Debezium are behind a number of source connectors that facilitate change-data-capture from mySQL and postgres. The also have a source connector for mongodb. + +This particular demo uses the mySQL source connector and combines with the Confluent [kafka-connect-jdbc](https://github.com/confluentinc/kafka-connect-jdbc) sink connector in order to replicate changes from the mySQL database to the postgres database via a kafka message bus. \ No newline at end of file diff --git a/connect-test-debezium/connect-deployment.yaml b/connect-test-debezium/connect-deployment.yaml new file mode 100644 index 00000000..9d456d44 --- /dev/null +++ b/connect-test-debezium/connect-deployment.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: connect + namespace: kafka +spec: + replicas: 1 + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + io.kompose.service: connect + spec: + containers: + - env: + - name: BOOTSTRAP_SERVERS + value: broker.kafka:9092 + - name: CONFIG_STORAGE_TOPIC + value: my_connect_configs + - name: GROUP_ID + value: "1" + - name: OFFSET_STORAGE_TOPIC + value: my_connect_offsets + - name: REST_PORT + value: "80" + image: analect/connect-deb-jdbc:0.6 + name: connect + ports: + - containerPort: 8083 + - containerPort: 5005 + resources: {} + restartPolicy: Always +status: {} diff --git a/connect-test-debezium/connect-service.yaml b/connect-test-debezium/connect-service.yaml new file mode 100644 index 00000000..282c4e99 --- /dev/null +++ b/connect-test-debezium/connect-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: connect + namespace: kafka +spec: + ports: + - name: "8083" + port: 8083 + targetPort: 8083 + - name: "5005" + port: 5005 + targetPort: 5005 + selector: + io.kompose.service: connect +status: + loadBalancer: {} diff --git a/connect-test-debezium/mysql-deployment.yaml b/connect-test-debezium/mysql-deployment.yaml new file mode 100644 index 00000000..5ef6fdac --- /dev/null +++ b/connect-test-debezium/mysql-deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: mysql + namespace: kafka +spec: + replicas: 1 + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + io.kompose.service: mysql + spec: + containers: + - env: + - name: MYSQL_PASSWORD + value: mysqlpw + - name: MYSQL_ROOT_PASSWORD + value: debezium + - name: MYSQL_USER + value: mysqluser + image: debezium/example-mysql:0.6 + name: mysql + ports: + - containerPort: 3306 + resources: {} + restartPolicy: Always +status: {} diff --git a/connect-test-debezium/mysql-service.yaml b/connect-test-debezium/mysql-service.yaml new file mode 100644 index 00000000..5e35db38 --- /dev/null +++ b/connect-test-debezium/mysql-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: kafka +spec: + ports: + - name: "3306" + port: 3306 + targetPort: 3306 + selector: + io.kompose.service: mysql +status: + loadBalancer: {} diff --git a/connect-test-debezium/postgres-deployment.yaml b/connect-test-debezium/postgres-deployment.yaml new file mode 100644 index 00000000..803add97 --- /dev/null +++ b/connect-test-debezium/postgres-deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: postgres + namespace: kafka +spec: + replicas: 1 + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + io.kompose.service: postgres + spec: + containers: + - env: + - name: POSTGRES_DB + value: inventory + - name: POSTGRES_PASSWORD + value: postgrespw + - name: POSTGRES_USER + value: postgresuser + image: debezium/postgres:9.6 + name: postgres + ports: + - containerPort: 5432 + resources: {} + restartPolicy: Always +status: {} diff --git a/connect-test-debezium/postgres-service.yaml b/connect-test-debezium/postgres-service.yaml new file mode 100644 index 00000000..e6284c83 --- /dev/null +++ b/connect-test-debezium/postgres-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: kafka +spec: + ports: + - name: "5432" + port: 5432 + targetPort: 5432 + selector: + io.kompose.service: postgres +status: + loadBalancer: {}