spark-connect-cpp/docker-compose.yaml at main · devsimiyu/spark-connect-cpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
services:
  spark:
    image: "apache/spark:3.5.3-scala2.12-java17-python3-ubuntu"
    command: >
      /opt/spark/sbin/start-connect-server.sh
      --packages "org.apache.spark:spark-connect_2.12:3.5.3,io.delta:delta-spark_2.12:3.2.0"
      --conf "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp"
      --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension"
      --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog"
    environment:
      - SPARK_NO_DAEMONIZE=true
    ports:
      - "4040:4040"
      - "15002:15002"
    volumes:
      - ./datasets:/opt/spark/work-dir/datasets
    networks:
      - default

  spark_client:
    build:
      context: .
    depends_on:
      - spark

    command:
      - /bin/sh
      - -c
      - |
        echo "Waiting for Spark Connect Server to start..."
        until nc -z spark 15002; do
          echo "Spark Connect Server is not ready yet. Retrying in 1 second..."
          sleep 1
        done
        echo "Spark Connect Server is ready."
        cd build
        cmake ..
        cmake --build .
        ctest --output-on-failure
    networks:
      - default

networks:
  default:
    driver: bridge
    name: spark_connect_cpp