Merge pull request #458 from ClickHouse/faster-init

alexey-milovidov · web-flow · commit 88478c82012f · 2025-07-04T15:16:08.000+03:00
Faster initialization
diff --git a/cedardb/benchmark.sh b/cedardb/benchmark.sh
@@ -6,9 +6,10 @@ sudo apt-get install -y  docker.io postgresql-client gzip
 
 # download dataset
 echo "Downloading dataset..."
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
 echo "Unpacking dataset..."
-gzip -d hits.tsv.gz
+pigz -d -f hits.tsv.gz
 mkdir data
 mv hits.tsv data
 chmod -R 777 data
diff --git a/chdb/benchmark.sh b/chdb/benchmark.sh
@@ -7,8 +7,9 @@ pip install psutil
 pip install chdb
 
 # Load the data
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d -f hits.csv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
+pigz -d -f hits.csv.gz
 ./load.py
 
 # Run the queries
diff --git a/citus/benchmark.sh b/citus/benchmark.sh
@@ -6,8 +6,9 @@ sudo apt-get install -y postgresql-client
 
 sudo docker run -d --name citus -p 5432:5432 -e POSTGRES_PASSWORD=mypass citusdata/citus:11.0
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 echo "*:*:*:*:mypass" > .pgpass
 chmod 400 .pgpass
diff --git a/clickhouse-tencent/benchmark.sh b/clickhouse-tencent/benchmark.sh
@@ -19,8 +19,9 @@ clickhouse-client < create.sql
 
 if [ ! -f hits.tsv ]
 then
-    wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-    gzip -d -f hits.tsv.gz
+    sudo apt-get install -y axel pigz
+    axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+    pigz -d -f hits.tsv.gz
 fi
 
 clickhouse-client --time --query "INSERT INTO hits FORMAT TSV" < hits.tsv
diff --git a/clickhouse/benchmark.sh b/clickhouse/benchmark.sh
@@ -39,8 +39,9 @@ clickhouse-client < create"$SUFFIX".sql
 
 if [ ! -f hits.tsv ]
 then
-    wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-    gzip -d -f hits.tsv.gz
+    sudo apt-get install -y axel pigz
+    axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+    pigz -d -f hits.tsv.gz
 fi
 
 echo -n "Load time: "
diff --git a/databend/benchmark.sh b/databend/benchmark.sh
@@ -26,8 +26,9 @@ CONF
 # Docs: https://databend.rs/doc/use-cases/analyze-hits-dataset-with-databend
 curl 'http://default@localhost:8124/' --data-binary @create.sql
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 ## Aws gp2 write performance is not stable, we must load the data when disk's write around ~500MB/s (Don't know much about the rules of gp2)
 time curl -XPUT 'http://root:@127.0.0.1:8000/v1/streaming_load' -H 'insert_sql: insert into hits FILE_FORMAT = (type = TSV)' -F 'upload=@"./hits.tsv"'
diff --git a/doris/benchmark.sh b/doris/benchmark.sh
@@ -88,8 +88,9 @@ mysql -h 127.0.0.1 -P9030 -uroot hits <"$ROOT"/create.sql
 
 # Download data
 if [[ ! -f hits.tsv.gz ]] && [[ ! -f hits.tsv ]]; then
-    wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-    gzip -d -f hits.tsv.gz
+    sudo apt-get install -y axel pigz
+    axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+    pigz -d -f hits.tsv.gz
 fi
 
 # Load data
diff --git a/druid/benchmark.sh b/druid/benchmark.sh
@@ -26,8 +26,9 @@ echo "druid.query.groupBy.maxMergingDictionarySize=5000000000" >> apache-druid-$
 
 # Load the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 ./apache-druid-${VERSION}/bin/post-index-task --file ingest.json --url http://localhost:8081
 
diff --git a/duckdb-memory/benchmark.sh b/duckdb-memory/benchmark.sh
@@ -8,8 +8,9 @@ pip install duckdb==1.1.3 psutil
 
 # Load the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d -f hits.csv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
+pigz -d -f hits.csv.gz
 
 # Run the queries
 
diff --git a/duckdb/benchmark.sh b/duckdb/benchmark.sh
@@ -14,8 +14,9 @@ export PATH="$PATH:`pwd`/build/release/"
 cd ..
 
 # Load the data
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 time duckdb hits.db -f create.sql -c "COPY hits FROM 'hits.tsv' (QUOTE '')"
 
diff --git a/elasticsearch/benchmark.sh b/elasticsearch/benchmark.sh
@@ -37,8 +37,9 @@ curl -k -X PUT "https://localhost:9200/hits?pretty" -u "elastic:${PASSWORD}"  -H
 ###### Data loading (JSON dump via ES Bulk API insert)
 
 # Download and unzip dataset
-wget --continue --progress=dot:giga https://datasets.clickhouse.com/hits_compatible/hits.json.gz
-gzip -d -f hits.json.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.json.gz'
+pigz -d -f hits.json.gz
 
 # Prepare Elasticsearch for large bulk insert. To do the large upload, you have to break up JSON file into smaller files to prevent 'curl' from OOM while doing it, and adjust ELasticsearch HTTP upload size minimum. This creates roughly 250M files (note it takes a while)
 split -l 10000000 hits.json hits_
diff --git a/greenplum/benchmark.sh b/greenplum/benchmark.sh
@@ -63,8 +63,9 @@ sudo mkdir /gpmaster /gpdata1 /gpdata2 /gpdata3 /gpdata4 /gpdata5 /gpdata6 /gpda
 sudo chmod 777 /gpmaster /gpdata1 /gpdata2 /gpdata3 /gpdata4 /gpdata5 /gpdata6 /gpdata7 /gpdata8 /gpdata9 /gpdata10 /gpdata11 /gpdata12 /gpdata13 /gpdata14
 gpinitsystem -ac gpinitsystem_singlenode
 export MASTER_DATA_DIRECTORY=/gpmaster/gpsne-1/
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 chmod 777 ~ hits.tsv
 psql -d postgres -f create.sql
 nohup gpfdist &
diff --git a/heavyai/benchmark.sh b/heavyai/benchmark.sh
@@ -27,8 +27,9 @@ sudo systemctl enable heavydb
 
 # Load the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d -f hits.csv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
+pigz -d -f hits.csv.gz
 chmod 777 ~ hits.csv
 
 sudo bash -c "echo 'allowed-import-paths = [\"/home/ubuntu/\"]' > /var/lib/heavyai/heavy.conf_"
diff --git a/hologres/benchmark.sh b/hologres/benchmark.sh
@@ -19,8 +19,9 @@ FILENAME="hits.tsv"
 # Check if the file exists
 if [ ! -f "$FILENAME" ]; then
     echo "The file $FILENAME does not exist. Starting to download..."
-    wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-    gzip -d hits.tsv.gz
+    sudo apt-get install -y axel pigz
+    axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+    pigz -d -f hits.tsv.gz
     chmod 777 ~ hits.tsv
     if [ $? -eq 0 ]; then
         echo "File download completed!"
diff --git a/hyper/benchmark.sh b/hyper/benchmark.sh
@@ -4,8 +4,9 @@ sudo apt-get update
 sudo apt-get install -y python3-pip
 pip install tableauhyperapi
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d -f hits.csv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
+pigz -d -f hits.csv.gz
 
 ./load.py
 
diff --git a/infobright/benchmark.sh b/infobright/benchmark.sh
@@ -13,8 +13,9 @@ sudo docker run -it --rm --network host mysql:5 mysql --host 127.0.0.1 --port 50
 
 # Load the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 # ERROR 2 (HY000) at line 1: Wrong data or column definition. Row: 93557187, field: 100.
 head -n 90000000 hits.tsv > hits90m.tsv
diff --git a/locustdb/benchmark.sh b/locustdb/benchmark.sh
@@ -14,8 +14,9 @@ sudo apt-get install -y g++ capnproto libclang-14-dev
 
 cargo build --features "enable_rocksdb" --features "enable_lz4" --release
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d -f hits.csv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
+pigz -d -f hits.csv.gz
 
 target/release/repl --load hits.csv --db-path db
 
diff --git a/mariadb-columnstore/benchmark.sh b/mariadb-columnstore/benchmark.sh
@@ -16,8 +16,9 @@ mysql --password="${PASSWORD}" --host 127.0.0.1 test < create.sql
 
 # Load the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 time mysql --password="${PASSWORD}" --host 127.0.0.1 test -e "
     LOAD DATA LOCAL INFILE 'hits.tsv' INTO TABLE hits
diff --git a/mariadb/benchmark.sh b/mariadb/benchmark.sh
@@ -14,8 +14,9 @@ sudo service mariadb restart
 
 # Load the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 sudo mariadb -e "CREATE DATABASE test"
 sudo mariadb test < create.sql
diff --git a/monetdb/benchmark.sh b/monetdb/benchmark.sh
@@ -22,8 +22,9 @@ sudo apt-get install -y expect
 
 ./query.expect "$(cat create.sql)"
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 chmod 777 ~ hits.tsv
 
 ./query.expect "COPY INTO hits FROM '$(pwd)/hits.tsv' USING DELIMITERS '\t'"
diff --git a/mongodb/benchmark.sh b/mongodb/benchmark.sh
@@ -55,8 +55,9 @@ time mongosh --quiet --eval 'db.hits.createIndex({"ClientIP": 1, "WatchID": 1, "
 
 #################################
 # Load data and import
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 # Use mongo import to load data into mongo. By default numInsertionWorkers is 1 so change to half of VM where it would be run
 #time mongoimport --collection hits --type tsv hits.tsv --fieldFile=create.txt --columnsHaveTypes --numInsertionWorkers=8
diff --git a/mysql-myisam/benchmark.sh b/mysql-myisam/benchmark.sh
@@ -9,8 +9,9 @@ sudo service mysql restart
 
 # Load the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 sudo mysql -e "CREATE DATABASE test"
 sudo mysql test < create.sql
diff --git a/mysql/benchmark.sh b/mysql/benchmark.sh
@@ -9,8 +9,9 @@ sudo service mysql restart
 
 # Load the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 sudo mysql -e "CREATE DATABASE test"
 sudo mysql test < create.sql
diff --git a/oxla/benchmark.sh b/oxla/benchmark.sh
@@ -9,9 +9,10 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential
 
 # download dataset
 echo "Download dataset."
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
 echo "Unpack dataset."
-gzip -d -f hits.csv.gz
+pigz -d -f hits.csv.gz
 mkdir data
 mv hits.csv data
 
diff --git a/pg_duckdb-indexed/benchmark.sh b/pg_duckdb-indexed/benchmark.sh
@@ -6,8 +6,9 @@ set -eux
 #sudo apt-get install -y docker.io
 #sudo apt-get install -y postgresql-client
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 memory=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
 threads=$(nproc)
diff --git a/pg_duckdb/benchmark.sh b/pg_duckdb/benchmark.sh
@@ -6,8 +6,9 @@ set -eux
 #sudo apt-get install -y docker.io
 #sudo apt-get install -y postgresql-client
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 memory=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
 threads=$(nproc)
diff --git a/postgresql-indexed/benchmark.sh b/postgresql-indexed/benchmark.sh
@@ -51,8 +51,9 @@ EOF
 
 sudo systemctl restart postgresql@$PGVERSION-main
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 sudo -u postgres psql -t -c 'CREATE DATABASE test'
 sudo -u postgres psql test -t <create.sql
diff --git a/postgresql/benchmark.sh b/postgresql/benchmark.sh
@@ -51,8 +51,9 @@ EOF
 
 sudo systemctl restart postgresql@$PGVERSION-main
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 
 sudo -u postgres psql -t -c 'CREATE DATABASE test'
 sudo -u postgres psql test -t <create.sql
diff --git a/questdb/benchmark.sh b/questdb/benchmark.sh
@@ -17,8 +17,9 @@ questdb/bin/questdb.sh start
 
 # Import the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d -f hits.csv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
+pigz -d -f hits.csv.gz
 
 curl -G --data-urlencode "query=$(cat create.sql)" 'http://localhost:9000/exec'
 
diff --git a/selectdb/benchmark.sh b/selectdb/benchmark.sh
@@ -89,8 +89,9 @@ mysql -h 127.0.0.1 -P9030 -uroot hits <"$ROOT"/create.sql
 
 # Download data
 if [[ ! -f hits.tsv.gz ]] && [[ ! -f hits.tsv ]]; then
-    wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-    gzip -d -f hits.tsv.gz
+    sudo apt-get install -y axel pigz
+    axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+    pigz -d -f hits.tsv.gz
 fi
 
 # Load data
diff --git a/siglens/benchmark.sh b/siglens/benchmark.sh
@@ -14,8 +14,9 @@ go build -o siglens cmd/siglens/main.go
 cd ..
 
 echo "Download and unzip dataset"
-wget --continue --progress=dot:giga https://datasets.clickhouse.com/hits_compatible/hits.json.gz
-gzip -d hits.json.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.json.gz'
+pigz -d -f hits.json.gz
 
 echo "Load data into SigLens, this can take a few hours"
 time python3 send_datawithactionline.py
diff --git a/singlestore/benchmark.sh b/singlestore/benchmark.sh
@@ -21,8 +21,9 @@ sudo docker exec -it memsql-ciab memsql -p"${ROOT_PASSWORD}"
 
 # Load the data
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d -f hits.tsv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+pigz -d -f hits.tsv.gz
 sudo docker cp hits.tsv memsql-ciab:/
 
 sudo docker exec -it memsql-ciab memsql -p"${ROOT_PASSWORD}" -e "CREATE DATABASE test"
diff --git a/sqlite/benchmark.sh b/sqlite/benchmark.sh
@@ -5,8 +5,9 @@ sudo apt-get install -y sqlite3
 
 sqlite3 mydb < create.sql
 
-wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d -f hits.csv.gz
+sudo apt-get install -y axel pigz
+axel --num-connections=32 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
+pigz -d -f hits.csv.gz
 
 time sqlite3 mydb '.import --csv hits.csv hits'
 wc -c mydb
diff --git a/starrocks/benchmark.sh b/starrocks/benchmark.sh
diff --git a/tablespace/benchmark.sh b/tablespace/benchmark.sh
diff --git a/tembo-olap/benchmark.sh b/tembo-olap/benchmark.sh
diff --git a/timescaledb-no-columnstore/benchmark.sh b/timescaledb-no-columnstore/benchmark.sh
diff --git a/timescaledb/benchmark.sh b/timescaledb/benchmark.sh
diff --git a/umbra/benchmark.sh b/umbra/benchmark.sh
diff --git a/ursa/benchmark.sh b/ursa/benchmark.sh
diff --git a/vertica/benchmark.sh b/vertica/benchmark.sh
diff --git a/yugabytedb/benchmark.sh b/yugabytedb/benchmark.sh
