ScalarLM/docker-compose.yaml at main · tensorwavecloud/ScalarLM · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
services:

    cray: &cray
        command: /app/cray/scripts/start_one_server.sh
        build:
            context: .
            dockerfile: Dockerfile
            platforms:
                - ${DOCKER_PLATFORM:-linux/arm64}
            args:
                - TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST}
                - BASE_NAME=${BASE_NAME}
                - VLLM_TARGET_DEVICE=${VLLM_TARGET_DEVICE}
                - VLLM_SOURCE=${VLLM_SOURCE:-remote}
                - VLLM_BRANCH=${VLLM_BRANCH:-main}
                - VLLM_REPO=${VLLM_REPO:-https://github.com/supermassive-intelligence/vllm-fork.git}
        ports:
            - "3000:3000"
            - "8000:8000"
            - "8001:8001"
        volumes:
            - type: bind
              source: ./models
              target: /root/.cache/huggingface
            - type: bind
              source: ./infra/cray_infra
              target: /app/cray/infra/cray_infra
            - type: bind
              source: ./scripts
              target: /app/cray/scripts
            - type: bind
              source: ./ml
              target: /app/cray/ml
            - type: bind
              source: ./test
              target: /app/cray/test
        networks:
            - cray-network


    cray-nvidia:
        <<: *cray
        runtime: nvidia
        restart: unless-stopped
        cap_add:
          - SYS_PTRACE
        deploy:
            resources:
                reservations:
                    devices:
                        - driver: nvidia
                          capabilities: [gpu]

    cray-amd:
        <<: *cray
        devices:
          - /dev/kfd
          - /dev/dri
        security_opt:
          - seccomp:unconfined
        cap_add:
          - SYS_PTRACE

networks:
  cray-network:
    name: cray_network