From cb911be434a10330ad9c92ede6bb8245fd00efae Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Mon, 8 Feb 2016 15:56:14 +0100 Subject: [PATCH 1/9] Cluster 3.0 works --- Docker/ArangoDBClusterWithDocker.sh | 41 +++++-------------- .../GoogleComputeEngine_ArangoDB_Cluster.sh | 2 +- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/Docker/ArangoDBClusterWithDocker.sh b/Docker/ArangoDBClusterWithDocker.sh index 18a0e92..7ffc139 100755 --- a/Docker/ArangoDBClusterWithDocker.sh +++ b/Docker/ArangoDBClusterWithDocker.sh @@ -48,7 +48,7 @@ startArangoDBClusterWithDocker() { - DOCKER_IMAGE_NAME=neunhoef/arangodb_cluster:2.6.dev-2.5 + DOCKER_IMAGE_NAME=m0ppers/arangodb:3.0 # Two docker images are needed: # microbox/etcd for the agency and @@ -195,41 +195,28 @@ startArangoDBClusterWithDocker() { wait echo Starting agency... - until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -p 4001:4001 -p 7001:7001 --name=agency -e "ETCD_NONO_WAL_SYNC=1" -v $AGENCY_DIR:/data ${DOCKER_IMAGE_NAME} /usr/lib/arangodb/etcd-arango --data-dir /data --listen-client-urls "http://0.0.0.0:4001" --listen-peer-urls "http://0.0.0.0:7001" >/home/$SSH_USER/agency.log" + until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -p 4001:4001 -p 7001:7001 --name=agency -e "ETCD_NONO_WAL_SYNC=1" -v $AGENCY_DIR:/data ${DOCKER_IMAGE_NAME} /usr/lib/arangodb/etcd-arango --data-dir /data --listen-client-urls "http://0.0.0.0:4001" --listen-peer-urls "http://0.0.0.0:7001" --advertise-client-urls "http://${SERVERS_INTERNAL_ARR[0]}:4001" >/home/$SSH_USER/agency.log" do echo "Error in remote docker run, retrying..." done sleep 1 - echo Initializing agency... - until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --link=agency:agency --rm ${DOCKER_IMAGE_NAME} arangosh --javascript.execute /scripts/init_agency.js > /home/$SSH_USER/init_agency.log" - do - echo "Error in remote docker run, retrying..." - done - echo Starting discovery... - until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true --link=agency:agency --name discovery ${DOCKER_IMAGE_NAME} arangosh --javascript.execute scripts/discover.js > /home/$SSH_USER/discovery.log" - do - echo "Error in remote docker run, retrying..." - done start_dbserver () { i=$1 echo Starting DBserver on ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$i]} $SSH_SUFFIX \ - docker run --detach=true -v $DBSERVER_DATA:/data \ + docker run --detach=true -e ARANGO_NO_AUTH=1 -v $DBSERVER_DATA:/var/lib/arangodb \ -v $DBSERVER_LOGS:/logs --net=host \ --name=dbserver$PORT_DBSERVER ${DOCKER_IMAGE_NAME} \ - arangod --database.directory /data \ - --frontend-version-check false \ + arangod --frontend-version-check false \ --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ --cluster.my-address tcp://${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER \ --server.endpoint tcp://0.0.0.0:$PORT_DBSERVER \ --cluster.my-local-info dbserver:${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER \ - --log.file /logs/$PORT_DBSERVER.log \ + --cluster.my-role PRIMARY \ --dispatcher.report-interval 15 \ - --server.foxx-queues false \ - --server.disable-statistics true \ --scheduler.threads 3 \ --server.threads 5 \ $DBSERVER_ARGS \ @@ -244,20 +231,17 @@ startArangoDBClusterWithDocker() { echo Starting Coordinator on ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$i]} $SSH_SUFFIX \ - docker run --detach=true -v $COORDINATOR_DATA:/data \ + docker run --detach=true -e ARANGO_NO_AUTH=1 -v $COORDINATOR_DATA:/var/lib/arangodb \ -v $COORDINATOR_LOGS:/logs --net=host \ --name=coordinator$PORT_COORDINATOR \ ${DOCKER_IMAGE_NAME} \ - arangod --database.directory /data \ - --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ + arangod --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ --cluster.my-address tcp://${SERVERS_INTERNAL_ARR[$i]}:$PORT_COORDINATOR \ --server.endpoint tcp://0.0.0.0:$PORT_COORDINATOR \ --cluster.my-local-info \ coordinator:${SERVERS_INTERNAL_ARR[$i]}:$PORT_COORDINATOR \ - --log.file /logs/$PORT_COORDINATOR.log \ + --cluster.my-role COORDINATOR \ --dispatcher.report-interval 15 \ - --server.foxx-queues false \ - --server.disable-statistics true \ --scheduler.threads 4 \ --server.threads 40 \ $COORDINATOR_ARGS \ @@ -331,16 +315,13 @@ startArangoDBClusterWithDocker() { echo " ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER on ${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA" until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$j]} $SSH_SUFFIX \ - docker run --detach=true -v $REPLICA_DATA:/data \ + docker run --detach=true -e ARANGO_NO_AUTH=1 -v $REPLICA_DATA:/var/lib/arangodb \ -v $REPLICA_LOGS:/logs --net=host \ --name=replica$PORT_REPLICA ${DOCKER_IMAGE_NAME} \ - arangod --database.directory /data \ - --frontend-version-check false \ + arangod --frontend-version-check false \ --server.endpoint tcp://0.0.0.0:$PORT_REPLICA \ - --log.file /logs/$PORT_REPLICA.log \ + --cluster.my-role SECONDARY \ --dispatcher.report-interval 15 \ - --server.foxx-queues false \ - --server.disable-statistics true \ --scheduler.threads 1 \ --server.threads 2 \ $REPLICA_ARGS \ diff --git a/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh b/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh index b5a2d46..bfb8376 100755 --- a/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh +++ b/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh @@ -375,7 +375,7 @@ export SERVERS_EXTERNAL export SERVERS_IDS export SSH_USER="core" export SSH_CMD="ssh" -export SSH_SUFFIX="-i $DEFAULT_KEY_PATH -l $SSH_USER" +export SSH_SUFFIX="" export ZONE export PROJECT export DBSERVER_DATA=/data/dbserver From 49c15a62d12d67063a4c7402b9b41f0251e10364 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Mon, 8 Feb 2016 17:26:13 +0100 Subject: [PATCH 2/9] Update AMIs, fix security group stuff --- .../AmazonWebServices_ArangoDB_Cluster.sh | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh b/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh index 944f1a4..01268f3 100755 --- a/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh +++ b/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh @@ -143,42 +143,43 @@ fi #We have to keep all ami's up to date # Current Version: CoreOS 607.0.0 -# URL: https://coreos.com/docs/running-coreos/cloud-providers/ec2/ +# mop: updated 2016-02-08 CoreOS CoreOS 835.12.0 +# URL: https://coreos.com/os/docs/latest/booting-on-ec2.html if [ "$zone" == "eu-central-1" ]; then - IMAGE="ami-0e300d13" + IMAGE="ami-e8ebf384" fi if [ "$zone" == "ap-northeast-1" ]; then - IMAGE="ami-af28dcaf" + IMAGE="ami-673b0109" fi if [ "$zone" == "sa-east-1" ]; then - IMAGE="ami-2354ec3e" + IMAGE="ami-6c1a9a00" fi if [ "$zone" == "ap-southeast-2" ]; then - IMAGE="ami-b9b5c583" + IMAGE="ami-d0a783b3" fi if [ "$zone" == "ap-southeast-1" ]; then - IMAGE="mi-f80b3aaa" + IMAGE="ami-4a65aa29" fi if [ "$zone" == "us-east-1" ]; then - IMAGE="ami-323b195a" + IMAGE="ami-dfb699b5" fi if [ "$zone" == "us-west-2" ]; then - IMAGE="ami-0789a437" + IMAGE="ami-abc82ecb" fi if [ "$zone" == "us-west-1" ]; then - IMAGE="ami-8dd533c9" + IMAGE="ami-4d2d5b2d" fi if [ "$zone" == "eu-west-1" ]; then - IMAGE="ami-55950a22" + IMAGE="ami-1461d767" fi @@ -337,7 +338,7 @@ function setMachineName () { function setMachineSecurity () { echo "Adding security groups." id=`cat "$OUTPUT/temp/IDS$i"` - secureid=`aws ec2 describe-security-groups --output json --group-names ${PREFIX}security |python -mjson.tool|grep GroupId| awk {'print $2'}| cut -c 2- | rev | cut -c 3- | rev` + secureid=$(aws ec2 describe-security-groups --output json --group-names ${PREFIX}security | grep GroupId | sed -e "s/.*: \"\([^\"]\+\).*/\1/g") aws ec2 modify-instance-attribute --instance-id "$id" --groups "$secureid" } From 0be59409a5b03de50d1b341108aa2cf7feece883 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 9 Feb 2016 13:23:07 +0100 Subject: [PATCH 3/9] Adjust for current azure --- platformAZURE/Azure_ArangoDB_Cluster.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformAZURE/Azure_ArangoDB_Cluster.sh b/platformAZURE/Azure_ArangoDB_Cluster.sh index 2fb21f0..0131889 100755 --- a/platformAZURE/Azure_ArangoDB_Cluster.sh +++ b/platformAZURE/Azure_ArangoDB_Cluster.sh @@ -12,11 +12,11 @@ trap "kill 0" SIGINT -ZONE="West US" +ZONE="West Europe" MACHINE_TYPE="Medium" NUMBER="3" OUTPUT="azure" -IMAGE="2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-607.0.0" +IMAGE="2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-835.12.0" SSH_KEY_PATH="" DEFAULT_KEY_PATH="$OUTPUT/arangodb_azure_key" @@ -165,7 +165,7 @@ then DEFAULT_KEY_PATH="$HOME/.ssh/arangodb_azure_key" else echo "No SSH-Key-Path given. Creating a new SSH-Key." - ssh-keygen -t dsa -f "$DEFAULT_KEY_PATH" -C "arangodb@arangodb.com" + ssh-keygen -t rsa -f "$DEFAULT_KEY_PATH" -C "arangodb@arangodb.com" # openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout "$DEFAULT_KEY_PATH" -out "$DEFAULT_KEY_PATH.pem" openssl req -x509 -key "$DEFAULT_KEY_PATH" -nodes -days 365 -newkey rsa:2048 -out "${DEFAULT_KEY_PATH}.pem" From ef3e93a35db1e162a57583b790f64dfa24de5251 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 9 Feb 2016 14:13:05 +0100 Subject: [PATCH 4/9] Use rsa instead of dsa --- platformDO/DigitalOcean_ArangoDB_Cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformDO/DigitalOcean_ArangoDB_Cluster.sh b/platformDO/DigitalOcean_ArangoDB_Cluster.sh index 261f742..1536ecc 100755 --- a/platformDO/DigitalOcean_ArangoDB_Cluster.sh +++ b/platformDO/DigitalOcean_ArangoDB_Cluster.sh @@ -192,7 +192,7 @@ if test -z "$SSHID"; then then echo "No ArangoDB SSH-Key found. Generating a new one.!" - ssh-keygen -t dsa -f $OUTPUT/$SSH_KEY -C "arangodb@arangodb.com" + ssh-keygen -t rsa -f $OUTPUT/$SSH_KEY -C "arangodb@arangodb.com" if [ $? -eq 0 ]; then echo OK From 64ae376da321c5df1ef9ae7372ec8a5abda7dc5e Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Fri, 17 Jun 2016 14:55:10 +0200 Subject: [PATCH 5/9] fix aws deployment --- Docker/ArangoDBClusterWithDocker.sh | 98 ++++++++++------------------- 1 file changed, 34 insertions(+), 64 deletions(-) diff --git a/Docker/ArangoDBClusterWithDocker.sh b/Docker/ArangoDBClusterWithDocker.sh index 7ffc139..4d6cc07 100755 --- a/Docker/ArangoDBClusterWithDocker.sh +++ b/Docker/ArangoDBClusterWithDocker.sh @@ -195,7 +195,7 @@ startArangoDBClusterWithDocker() { wait echo Starting agency... - until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -p 4001:4001 -p 7001:7001 --name=agency -e "ETCD_NONO_WAL_SYNC=1" -v $AGENCY_DIR:/data ${DOCKER_IMAGE_NAME} /usr/lib/arangodb/etcd-arango --data-dir /data --listen-client-urls "http://0.0.0.0:4001" --listen-peer-urls "http://0.0.0.0:7001" --advertise-client-urls "http://${SERVERS_INTERNAL_ARR[0]}:4001" >/home/$SSH_USER/agency.log" + until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -e ARANGO_NO_AUTH=1 -p 4001:8529 --name=agency -v $AGENCY_DIR:/var/lib/arangodb3 ${DOCKER_IMAGE_NAME} --server.authentication false --agency.id 0 --agency.size 1 --javascript.v8-contexts 2" do echo "Error in remote docker run, retrying..." done @@ -207,20 +207,16 @@ startArangoDBClusterWithDocker() { echo Starting DBserver on ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$i]} $SSH_SUFFIX \ - docker run --detach=true -e ARANGO_NO_AUTH=1 -v $DBSERVER_DATA:/var/lib/arangodb \ - -v $DBSERVER_LOGS:/logs --net=host \ + docker run -p $PORT_DBSERVER:8529 --detach=true -e ARANGO_NO_AUTH=1 -v $DBSERVER_DATA:/var/lib/arangodb3 \ --name=dbserver$PORT_DBSERVER ${DOCKER_IMAGE_NAME} \ - arangod --frontend-version-check false \ - --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ + arangod --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ --cluster.my-address tcp://${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER \ - --server.endpoint tcp://0.0.0.0:$PORT_DBSERVER \ --cluster.my-local-info dbserver:${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER \ --cluster.my-role PRIMARY \ - --dispatcher.report-interval 15 \ --scheduler.threads 3 \ --server.threads 5 \ - $DBSERVER_ARGS \ - >/dev/null + --javascript.v8-contexts 6 \ + $DBSERVER_ARGS do echo "Error in remote docker run, retrying..." done @@ -231,21 +227,18 @@ startArangoDBClusterWithDocker() { echo Starting Coordinator on ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$i]} $SSH_SUFFIX \ - docker run --detach=true -e ARANGO_NO_AUTH=1 -v $COORDINATOR_DATA:/var/lib/arangodb \ - -v $COORDINATOR_LOGS:/logs --net=host \ + docker run -p $PORT_COORDINATOR:8529 --detach=true -e ARANGO_NO_AUTH=1 -v $COORDINATOR_DATA:/var/lib/arangodb3 \ --name=coordinator$PORT_COORDINATOR \ ${DOCKER_IMAGE_NAME} \ arangod --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ --cluster.my-address tcp://${SERVERS_INTERNAL_ARR[$i]}:$PORT_COORDINATOR \ - --server.endpoint tcp://0.0.0.0:$PORT_COORDINATOR \ --cluster.my-local-info \ coordinator:${SERVERS_INTERNAL_ARR[$i]}:$PORT_COORDINATOR \ --cluster.my-role COORDINATOR \ - --dispatcher.report-interval 15 \ --scheduler.threads 4 \ - --server.threads 40 \ - $COORDINATOR_ARGS \ - >/dev/null + --javascript.v8-contexts 11 \ + --server.threads 10 \ + $COORDINATOR_ARGS do echo "Error in remote docker run, retrying..." done @@ -278,81 +271,58 @@ startArangoDBClusterWithDocker() { fi done } - - #for i in `seq 0 $LASTDBSERVER` ; do - # testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER - #done - - for i in $COORDINATOR_MACHINES ; do - testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR + + DBSERVER_IDS=() + for i in `seq 0 $LASTDBSERVER` ; do + testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER + DBSERVER_IDS[$i]=$(curl http://"${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER"/_admin/server/id) done - echo Bootstrapping DBServers... - curl -s -X POST "http://${SERVERS_EXTERNAL_ARR[$FIRST_COORDINATOR]}:$PORT_COORDINATOR/_admin/cluster/bootstrapDbServers" \ - -d '{"isRelaunch":false}' >/dev/null 2>&1 - - echo Running DB upgrade on cluster... - curl -s -X POST "http://${SERVERS_EXTERNAL_ARR[$FIRST_COORDINATOR]}:$PORT_COORDINATOR/_admin/cluster/upgradeClusterDatabase" \ - -d '{"isRelaunch":false}' >/dev/null 2>&1 - - echo Bootstrapping Coordinators... for i in $COORDINATOR_MACHINES ; do - echo Doing ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR - curl -s -X POST "http://${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR/_admin/cluster/bootstrapCoordinator" \ - -d '{"isRelaunch":false}' >/dev/null 2>&1 & + testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR done - - wait - + if [ ! -z "$REPLICAS" ] ; then start_replica () { i=$1 j=`expr $i + 1` + ID="Secondary$j" if [ $j -gt $LASTDBSERVER ] ; then j=0 fi echo Starting asynchronous replica for echo " ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER on ${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA" + + while true; do + curl -f -X PUT --data "{\"primary\": \"${DBSERVER_IDS[$i]}\", \"oldSecondary\": \"none\", \"newSecondary\": \"${ID}\"}" -H "Content-Type: application/json" "${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR"/_admin/cluster/replaceSecondary + if [ "$?" == "0" ]; then + break + fi + echo "Failed registering secondary...Retrying..." + sleep 1 + done until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$j]} $SSH_SUFFIX \ - docker run --detach=true -e ARANGO_NO_AUTH=1 -v $REPLICA_DATA:/var/lib/arangodb \ - -v $REPLICA_LOGS:/logs --net=host \ + docker run -p 8529:$PORT_REPLICA --detach=true -e ARANGO_NO_AUTH=1 -v $REPLICA_DATA:/var/lib/arangodb3 \ --name=replica$PORT_REPLICA ${DOCKER_IMAGE_NAME} \ - arangod --frontend-version-check false \ - --server.endpoint tcp://0.0.0.0:$PORT_REPLICA \ + arangod \ + --cluster.my-id "$ID" \ --cluster.my-role SECONDARY \ - --dispatcher.report-interval 15 \ - --scheduler.threads 1 \ - --server.threads 2 \ - $REPLICA_ARGS \ - >/dev/null + --scheduler.threads 3 \ + --server.threads 5 \ + --javascript.v8-contexts 6 \ + $REPLICA_ARGS do echo "Error in remote docker run, retrying..." done } for i in `seq 0 $LASTDBSERVER` ; do - start_replica $i & + start_replica $i done echo Waiting 10 seconds till replicas are up and running... sleep 10 - - for i in `seq 0 $LASTDBSERVER` ; do - j=`expr $i + 1` - if [ $j -gt $LASTDBSERVER ] ; then - j=0 - fi - echo Attaching replica on $j for $i ... - curl -s -X PUT "http://${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA/_api/replication/applier-config" -d '{"endpoint":"tcp://'${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER'","database":"_system","includeSystem":false}' --dump - - # >/dev/null 2>&1 - TICK=`curl -X PUT "http://${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA/_api/replication/sync" -d '{"endpoint":"tcp://'${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER'"}' | sed -e 's/^.*lastLogTick":"\([0-9]*\)"}.*$/\1/'` - # >/dev/null 2>&1 - curl -X PUT "http://${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA/_api/replication/applier-start?from=$TICK" --dump - && echo - # >/dev/null 2>&1 - done - wait - fi echo "" From bc4e6670adf3cf95fd4a12e7c90b83b927b00148 Mon Sep 17 00:00:00 2001 From: hkernbach Date: Fri, 17 Jun 2016 15:56:55 +0200 Subject: [PATCH 6/9] supervision param --- Docker/ArangoDBClusterWithDocker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/ArangoDBClusterWithDocker.sh b/Docker/ArangoDBClusterWithDocker.sh index 4d6cc07..6747624 100755 --- a/Docker/ArangoDBClusterWithDocker.sh +++ b/Docker/ArangoDBClusterWithDocker.sh @@ -195,7 +195,7 @@ startArangoDBClusterWithDocker() { wait echo Starting agency... - until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -e ARANGO_NO_AUTH=1 -p 4001:8529 --name=agency -v $AGENCY_DIR:/var/lib/arangodb3 ${DOCKER_IMAGE_NAME} --server.authentication false --agency.id 0 --agency.size 1 --javascript.v8-contexts 2" + until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -e ARANGO_NO_AUTH=1 -p 4001:8529 --name=agency -v $AGENCY_DIR:/var/lib/arangodb3 ${DOCKER_IMAGE_NAME} --agency.id 0 --agency.size 1 --javascript.v8-contexts 2 --agency.supervision true" do echo "Error in remote docker run, retrying..." done From 5b138a469070efd991090d3207a26da9e69c1435 Mon Sep 17 00:00:00 2001 From: hkernbach Date: Fri, 17 Jun 2016 15:57:27 +0200 Subject: [PATCH 7/9] fixed sed, was not working on a mac --- platformAWS/AmazonWebServices_ArangoDB_Cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh b/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh index 01268f3..50f1a7a 100755 --- a/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh +++ b/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh @@ -338,7 +338,7 @@ function setMachineName () { function setMachineSecurity () { echo "Adding security groups." id=`cat "$OUTPUT/temp/IDS$i"` - secureid=$(aws ec2 describe-security-groups --output json --group-names ${PREFIX}security | grep GroupId | sed -e "s/.*: \"\([^\"]\+\).*/\1/g") + secureid=$(aws ec2 describe-security-groups --output json --group-names ${PREFIX}security | grep GroupId | sed -e 's/.*: "\([^"]*\)"/\1/g') aws ec2 modify-instance-attribute --instance-id "$id" --groups "$secureid" } From 77b3337ce0ebc96edac7ff228809035edb26f2fe Mon Sep 17 00:00:00 2001 From: hkernbach Date: Fri, 17 Jun 2016 16:25:04 +0200 Subject: [PATCH 8/9] added function to fetch latest core os ami id automatically --- .../AmazonWebServices_ArangoDB_Cluster.sh | 55 +++++-------------- 1 file changed, 13 insertions(+), 42 deletions(-) diff --git a/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh b/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh index 50f1a7a..dac5971 100755 --- a/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh +++ b/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh @@ -12,9 +12,6 @@ trap "kill 0" SIGINT -#CoreOS AWS Image List -#https://coreos.com/docs/running-coreos/cloud-providers/ec2/ - MACHINE_TYPE="t1.medium" NUMBER="3" OUTPUT="aws" @@ -126,7 +123,7 @@ done PREFIX="arangodb-test-$$-" echo "OUTPUT DIRECTORY: $OUTPUT" -echo "ZONE: $ZONE" +echo "ZONE: $zone" echo "PROJECT: $PROJECT" DEFAULT_KEY_PATH="$HOME/.ssh/${PREFIX}aws-ssh-key" @@ -141,47 +138,21 @@ else ZONE=$zone fi -#We have to keep all ami's up to date -# Current Version: CoreOS 607.0.0 -# mop: updated 2016-02-08 CoreOS CoreOS 835.12.0 -# URL: https://coreos.com/os/docs/latest/booting-on-ec2.html - -if [ "$zone" == "eu-central-1" ]; then - IMAGE="ami-e8ebf384" -fi - -if [ "$zone" == "ap-northeast-1" ]; then - IMAGE="ami-673b0109" -fi - -if [ "$zone" == "sa-east-1" ]; then - IMAGE="ami-6c1a9a00" -fi - -if [ "$zone" == "ap-southeast-2" ]; then - IMAGE="ami-d0a783b3" -fi - -if [ "$zone" == "ap-southeast-1" ]; then - IMAGE="ami-4a65aa29" -fi +# Function to get latest core os ami ids (stable channel) +# URL: https://coreos.com/dist/aws/aws-stable.json -if [ "$zone" == "us-east-1" ]; then - IMAGE="ami-dfb699b5" -fi - -if [ "$zone" == "us-west-2" ]; then - IMAGE="ami-abc82ecb" -fi +echo "Your aws zone is: $ZONE" +echo "Searching ami id of latest core os version..." -if [ "$zone" == "us-west-1" ]; then - IMAGE="ami-4d2d5b2d" -fi - -if [ "$zone" == "eu-west-1" ]; then - IMAGE="ami-1461d767" -fi +function jsonval { + temp=`echo $json | sed 's/\\\\\//\//g' | sed 's/[{}]//g' | awk -v k="text" '{n=split($0,a,","); for (i=1; i<=n; i++) print a[i]}' | sed 's/\"\:\"/\|/g' | sed 's/[\,]/ /g' | sed 's/\"//g' | grep -w $ZONE` + echo ${temp##*|} +} +json=`curl -s -X GET https://coreos.com/dist/aws/aws-stable.json` +amiid=`jsonval` +IMAGE=`echo $amiid | awk {'print $3'}` +echo "Found image. ID is: $IMAGE" if test -e "$OUTPUT"; then if [ "$REMOVE" == "1" ] ; then From af8c6fb6ceef2ac25ef879988bad8e94ee9e2c7a Mon Sep 17 00:00:00 2001 From: hkernbach Date: Wed, 12 Oct 2016 10:48:27 +0200 Subject: [PATCH 9/9] gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 3035329..8014db8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,9 @@ Azure_ArangoDB_Cluster.sh DigitalOcean_ArangoDB_Cluster.sh GoogleComputeEngine_ArangoDB_Cluster.sh GoogleComputeEngine_Mesos_Cluster.sh +vCloud_ArangoDB_Cluster.sh digital_ocean/ azure/ gce/ aws/ +*.swp