Skip to content
This repository was archived by the owner on Jan 5, 2023. It is now read-only.

Cluster config: Update instance type to smaller node, preallocate Plasma memory, revert to single-node until preallocation is validated. #4

Open
clarkzinzow wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Cluster config: Update instance type to smaller node, preallocate Plasma memory, revert to single-node until preallocation is validated. #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 11 additions & 12 deletions benchmarks/cluster.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ cluster_name: shuffle-data-loader-benchmarking

# The minimum number of workers nodes to launch in addition to the head
# node. This number should be >= 0.
min_workers: 3
min_workers: 0

# The maximum number of workers nodes to launch in addition to the head
# node. This takes precedence over min_workers.
max_workers: 3
max_workers: 0

# The autoscaler will scale up the cluster faster with higher upscaling speed.
# E.g., if the task requires adding more nodes then autoscaler will gradually
Expand Down Expand Up @@ -62,7 +62,7 @@ auth:
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
head_node:
InstanceType: i3.8xlarge
InstanceType: i3.4xlarge
ImageId: ami-0edc3c56e8af8d35a # Deep Learning AMI (Ubuntu) Version 30

# You can provision additional disk space with a conf as follows
Expand All @@ -85,7 +85,7 @@ head_node:
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
worker_nodes:
InstanceType: i3.8xlarge
InstanceType: i3.4xlarge
ImageId: ami-0edc3c56e8af8d35a # Deep Learning AMI (Ubuntu) Version 30
IamInstanceProfile:
Arn: arn:aws:iam::959243851260:instance-profile/ray-autoscaler-v1
Expand Down Expand Up @@ -148,12 +148,12 @@ setup_commands:
- pip install -U --user numpy pandas pyarrow fastparquet smart-open[s3]
- if ! sudo mountpoint -q /mnt/disk0; then sudo mkfs -t ext4 /dev/nvme0n1 && sudo mkdir -p /mnt/disk0 && sudo mount /dev/nvme0n1 /mnt/disk0 && sudo chown -R ubuntu:ubuntu /mnt/disk0 && sudo chmod 777 /mnt/disk0; fi
- if ! sudo mountpoint -q /mnt/disk1; then sudo mkfs -t ext4 /dev/nvme1n1 && sudo mkdir -p /mnt/disk1 && sudo mount /dev/nvme1n1 /mnt/disk1 && sudo chown -R ubuntu:ubuntu /mnt/disk1 && sudo chmod 777 /mnt/disk1; fi
- if ! sudo mountpoint -q /mnt/disk2; then sudo mkfs -t ext4 /dev/nvme2n1 && sudo mkdir -p /mnt/disk2 && sudo mount /dev/nvme2n1 /mnt/disk2 && sudo chown -R ubuntu:ubuntu /mnt/disk2 && sudo chmod 777 /mnt/disk2; fi
- if ! sudo mountpoint -q /mnt/disk3; then sudo mkfs -t ext4 /dev/nvme3n1 && sudo mkdir -p /mnt/disk3 && sudo mount /dev/nvme3n1 /mnt/disk3 && sudo chown -R ubuntu:ubuntu /mnt/disk3 && sudo chmod 777 /mnt/disk3; fi
# - if ! sudo mountpoint -q /mnt/disk2; then sudo mkfs -t ext4 /dev/nvme2n1 && sudo mkdir -p /mnt/disk2 && sudo mount /dev/nvme2n1 /mnt/disk2 && sudo chown -R ubuntu:ubuntu /mnt/disk2 && sudo chmod 777 /mnt/disk2; fi
# - if ! sudo mountpoint -q /mnt/disk3; then sudo mkfs -t ext4 /dev/nvme3n1 && sudo mkdir -p /mnt/disk3 && sudo mount /dev/nvme3n1 /mnt/disk3 && sudo chown -R ubuntu:ubuntu /mnt/disk3 && sudo chmod 777 /mnt/disk3; fi
- mkdir -p /mnt/disk0/benchmark_scratch && rm -f /mnt/disk0/benchmark_scratch/*.parquet.*
- mkdir -p /mnt/disk1/benchmark_scratch && rm -f /mnt/disk1/benchmark_scratch/*.parquet.*
- mkdir -p /mnt/disk2/benchmark_scratch && rm -f /mnt/disk2/benchmark_scratch/*.parquet.*
- mkdir -p /mnt/disk3/benchmark_scratch && rm -f /mnt/disk3/benchmark_scratch/*.parquet.*
# - mkdir -p /mnt/disk2/benchmark_scratch && rm -f /mnt/disk2/benchmark_scratch/*.parquet.*
# - mkdir -p /mnt/disk3/benchmark_scratch && rm -f /mnt/disk3/benchmark_scratch/*.parquet.*
- mkdir -p /mnt/disk0/benchmark_scratch/results
# Note: if you're developing Ray, you probably want to create a Docker image that
# has your Ray repo pre-cloned. Then, you can replace the pip installs
Expand All @@ -170,12 +170,11 @@ worker_setup_commands: []
# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
- ray stop
# - ulimit -n 65536; RAY_BACKEND_LOG_LEVEL=debug ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"max_io_workers":4,"object_spilling_config":"{\"type\":\"filesystem\",\"params\":{\"directory_path\":[\"/mnt/disk0/benchmark_scratch\",\"/mnt/disk1/benchmark_scratch\",\"/mnt/disk2/benchmark_scratch\",\"/mnt/disk3/benchmark_scratch\"]}}"}'
# - ulimit -n 65536; RAY_BACKEND_LOG_LEVEL=debug RAY_RAYLET_PERFTOOLS_PROFILER=1 PERFTOOLS_PATH=/usr/lib/x86_64-linux-gnu/libprofiler.so PERFTOOLS_LOGFILE=/tmp/ray/session_latest/pprof.out ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"max_io_workers":1,"object_spilling_config":"{\"type\":\"filesystem\",\"params\":{\"directory_path\":[\"/mnt/disk0/benchmark_scratch\"]}}"}'
- ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --object-store-memory=$(( 110 * 1024 * 1024 * 1024 )) --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"automatic_object_spilling_enabled":false,"idle_worker_killing_time_threshold_ms":1000000}'

# - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --object-store-memory=$(( 110 * 1024 * 1024 * 1024 )) --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"automatic_object_spilling_enabled":false,"idle_worker_killing_time_threshold_ms":1000000}'
- ulimit -n 65536; RAY_PREALLOCATE_PLASMA_MEMORY=1 ray start --head --port=6379 --object-manager-port=8076 --object-store-memory=$(( 55 * (10 ** 9) )) --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"max_io_workers":4,"object_spilling_config":"{\"type\":\"filesystem\",\"params\":{\"directory_path\":[\"/mnt/disk0/benchmark_scratch\"]}}","idle_worker_killing_time_threshold_ms":1000000}'
# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
- ray stop
# - ulimit -n 65536; RAY_BACKEND_LOG_LEVEL=debug RAY_RAYLET_PERFTOOLS_PROFILER=1 PERFTOOLS_PATH=/usr/lib/x86_64-linux-gnu/libprofiler.so PERFTOOLS_LOGFILE=/tmp/ray/session_latest/pprof.out ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076
- ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 --object-store-memory=$(( 110 * 1024 * 1024 * 1024 ))
- ulimit -n 65536; RAY_PREALLOCATE_PLASMA_MEMORY=1 ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 --object-store-memory=$(( 55 * (10 ** 9) ))
3 changes: 1 addition & 2 deletions examples/horovod/cluster.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,7 @@ file_mounts: {

setup_commands:
- pip install -q boto3 tqdm torch torchvision tensorboard
# - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
- pip install -U ray
- pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
- pip install git+https://github.com/ray-project/ray_shuffling_data_loader.git@main#egg=ray_shuffling_data_loader
# - docker pull richardliaw/horovod
# - pip install ray[tune]
Expand Down