From 7ca8e522232baeec68d2b69f5a09ec4924c3f838 Mon Sep 17 00:00:00 2001 From: Clark Zinzow Date: Tue, 18 May 2021 16:01:43 +0000 Subject: [PATCH] Update instance type to smaller node, preallocate Plasma memory, revert to single-node until preallocation is validated. --- benchmarks/cluster.yaml | 23 +++++++++++------------ examples/horovod/cluster.yaml | 3 +-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/benchmarks/cluster.yaml b/benchmarks/cluster.yaml index d7caa80..5943f1c 100644 --- a/benchmarks/cluster.yaml +++ b/benchmarks/cluster.yaml @@ -3,11 +3,11 @@ cluster_name: shuffle-data-loader-benchmarking # The minimum number of workers nodes to launch in addition to the head # node. This number should be >= 0. -min_workers: 3 +min_workers: 0 # The maximum number of workers nodes to launch in addition to the head # node. This takes precedence over min_workers. -max_workers: 3 +max_workers: 0 # The autoscaler will scale up the cluster faster with higher upscaling speed. # E.g., if the task requires adding more nodes then autoscaler will gradually @@ -62,7 +62,7 @@ auth: # For more documentation on available fields, see: # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances head_node: - InstanceType: i3.8xlarge + InstanceType: i3.4xlarge ImageId: ami-0edc3c56e8af8d35a # Deep Learning AMI (Ubuntu) Version 30 # You can provision additional disk space with a conf as follows @@ -85,7 +85,7 @@ head_node: # For more documentation on available fields, see: # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances worker_nodes: - InstanceType: i3.8xlarge + InstanceType: i3.4xlarge ImageId: ami-0edc3c56e8af8d35a # Deep Learning AMI (Ubuntu) Version 30 IamInstanceProfile: Arn: arn:aws:iam::959243851260:instance-profile/ray-autoscaler-v1 @@ -148,12 +148,12 @@ setup_commands: - pip install -U --user numpy pandas pyarrow fastparquet smart-open[s3] - if ! sudo mountpoint -q /mnt/disk0; then sudo mkfs -t ext4 /dev/nvme0n1 && sudo mkdir -p /mnt/disk0 && sudo mount /dev/nvme0n1 /mnt/disk0 && sudo chown -R ubuntu:ubuntu /mnt/disk0 && sudo chmod 777 /mnt/disk0; fi - if ! sudo mountpoint -q /mnt/disk1; then sudo mkfs -t ext4 /dev/nvme1n1 && sudo mkdir -p /mnt/disk1 && sudo mount /dev/nvme1n1 /mnt/disk1 && sudo chown -R ubuntu:ubuntu /mnt/disk1 && sudo chmod 777 /mnt/disk1; fi - - if ! sudo mountpoint -q /mnt/disk2; then sudo mkfs -t ext4 /dev/nvme2n1 && sudo mkdir -p /mnt/disk2 && sudo mount /dev/nvme2n1 /mnt/disk2 && sudo chown -R ubuntu:ubuntu /mnt/disk2 && sudo chmod 777 /mnt/disk2; fi - - if ! sudo mountpoint -q /mnt/disk3; then sudo mkfs -t ext4 /dev/nvme3n1 && sudo mkdir -p /mnt/disk3 && sudo mount /dev/nvme3n1 /mnt/disk3 && sudo chown -R ubuntu:ubuntu /mnt/disk3 && sudo chmod 777 /mnt/disk3; fi + # - if ! sudo mountpoint -q /mnt/disk2; then sudo mkfs -t ext4 /dev/nvme2n1 && sudo mkdir -p /mnt/disk2 && sudo mount /dev/nvme2n1 /mnt/disk2 && sudo chown -R ubuntu:ubuntu /mnt/disk2 && sudo chmod 777 /mnt/disk2; fi + # - if ! sudo mountpoint -q /mnt/disk3; then sudo mkfs -t ext4 /dev/nvme3n1 && sudo mkdir -p /mnt/disk3 && sudo mount /dev/nvme3n1 /mnt/disk3 && sudo chown -R ubuntu:ubuntu /mnt/disk3 && sudo chmod 777 /mnt/disk3; fi - mkdir -p /mnt/disk0/benchmark_scratch && rm -f /mnt/disk0/benchmark_scratch/*.parquet.* - mkdir -p /mnt/disk1/benchmark_scratch && rm -f /mnt/disk1/benchmark_scratch/*.parquet.* - - mkdir -p /mnt/disk2/benchmark_scratch && rm -f /mnt/disk2/benchmark_scratch/*.parquet.* - - mkdir -p /mnt/disk3/benchmark_scratch && rm -f /mnt/disk3/benchmark_scratch/*.parquet.* + # - mkdir -p /mnt/disk2/benchmark_scratch && rm -f /mnt/disk2/benchmark_scratch/*.parquet.* + # - mkdir -p /mnt/disk3/benchmark_scratch && rm -f /mnt/disk3/benchmark_scratch/*.parquet.* - mkdir -p /mnt/disk0/benchmark_scratch/results # Note: if you're developing Ray, you probably want to create a Docker image that # has your Ray repo pre-cloned. Then, you can replace the pip installs @@ -170,12 +170,11 @@ worker_setup_commands: [] # Command to start ray on the head node. You don't need to change this. head_start_ray_commands: - ray stop - # - ulimit -n 65536; RAY_BACKEND_LOG_LEVEL=debug ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"max_io_workers":4,"object_spilling_config":"{\"type\":\"filesystem\",\"params\":{\"directory_path\":[\"/mnt/disk0/benchmark_scratch\",\"/mnt/disk1/benchmark_scratch\",\"/mnt/disk2/benchmark_scratch\",\"/mnt/disk3/benchmark_scratch\"]}}"}' # - ulimit -n 65536; RAY_BACKEND_LOG_LEVEL=debug RAY_RAYLET_PERFTOOLS_PROFILER=1 PERFTOOLS_PATH=/usr/lib/x86_64-linux-gnu/libprofiler.so PERFTOOLS_LOGFILE=/tmp/ray/session_latest/pprof.out ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"max_io_workers":1,"object_spilling_config":"{\"type\":\"filesystem\",\"params\":{\"directory_path\":[\"/mnt/disk0/benchmark_scratch\"]}}"}' - - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --object-store-memory=$(( 110 * 1024 * 1024 * 1024 )) --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"automatic_object_spilling_enabled":false,"idle_worker_killing_time_threshold_ms":1000000}' - + # - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --object-store-memory=$(( 110 * 1024 * 1024 * 1024 )) --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"automatic_object_spilling_enabled":false,"idle_worker_killing_time_threshold_ms":1000000}' + - ulimit -n 65536; RAY_PREALLOCATE_PLASMA_MEMORY=1 ray start --head --port=6379 --object-manager-port=8076 --object-store-memory=$(( 55 * (10 ** 9) )) --autoscaling-config=~/ray_bootstrap_config.yaml --system-config='{"max_io_workers":4,"object_spilling_config":"{\"type\":\"filesystem\",\"params\":{\"directory_path\":[\"/mnt/disk0/benchmark_scratch\"]}}","idle_worker_killing_time_threshold_ms":1000000}' # Command to start ray on worker nodes. You don't need to change this. worker_start_ray_commands: - ray stop # - ulimit -n 65536; RAY_BACKEND_LOG_LEVEL=debug RAY_RAYLET_PERFTOOLS_PROFILER=1 PERFTOOLS_PATH=/usr/lib/x86_64-linux-gnu/libprofiler.so PERFTOOLS_LOGFILE=/tmp/ray/session_latest/pprof.out ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 - - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 --object-store-memory=$(( 110 * 1024 * 1024 * 1024 )) + - ulimit -n 65536; RAY_PREALLOCATE_PLASMA_MEMORY=1 ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 --object-store-memory=$(( 55 * (10 ** 9) )) diff --git a/examples/horovod/cluster.yaml b/examples/horovod/cluster.yaml index 4a832c1..2ed22de 100644 --- a/examples/horovod/cluster.yaml +++ b/examples/horovod/cluster.yaml @@ -78,8 +78,7 @@ file_mounts: { setup_commands: - pip install -q boto3 tqdm torch torchvision tensorboard - # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl - - pip install -U ray + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl - pip install git+https://github.com/ray-project/ray_shuffling_data_loader.git@main#egg=ray_shuffling_data_loader # - docker pull richardliaw/horovod # - pip install ray[tune]