diff --git a/.gitignore b/.gitignore index 3035329..8014db8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,9 @@ Azure_ArangoDB_Cluster.sh DigitalOcean_ArangoDB_Cluster.sh GoogleComputeEngine_ArangoDB_Cluster.sh GoogleComputeEngine_Mesos_Cluster.sh +vCloud_ArangoDB_Cluster.sh digital_ocean/ azure/ gce/ aws/ +*.swp diff --git a/Docker/ArangoDBClusterWithDocker.sh b/Docker/ArangoDBClusterWithDocker.sh index 18a0e92..6747624 100755 --- a/Docker/ArangoDBClusterWithDocker.sh +++ b/Docker/ArangoDBClusterWithDocker.sh @@ -48,7 +48,7 @@ startArangoDBClusterWithDocker() { - DOCKER_IMAGE_NAME=neunhoef/arangodb_cluster:2.6.dev-2.5 + DOCKER_IMAGE_NAME=m0ppers/arangodb:3.0 # Two docker images are needed: # microbox/etcd for the agency and @@ -195,45 +195,28 @@ startArangoDBClusterWithDocker() { wait echo Starting agency... - until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -p 4001:4001 -p 7001:7001 --name=agency -e "ETCD_NONO_WAL_SYNC=1" -v $AGENCY_DIR:/data ${DOCKER_IMAGE_NAME} /usr/lib/arangodb/etcd-arango --data-dir /data --listen-client-urls "http://0.0.0.0:4001" --listen-peer-urls "http://0.0.0.0:7001" >/home/$SSH_USER/agency.log" + until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true -e ARANGO_NO_AUTH=1 -p 4001:8529 --name=agency -v $AGENCY_DIR:/var/lib/arangodb3 ${DOCKER_IMAGE_NAME} --agency.id 0 --agency.size 1 --javascript.v8-contexts 2 --agency.supervision true" do echo "Error in remote docker run, retrying..." done sleep 1 - echo Initializing agency... - until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --link=agency:agency --rm ${DOCKER_IMAGE_NAME} arangosh --javascript.execute /scripts/init_agency.js > /home/$SSH_USER/init_agency.log" - do - echo "Error in remote docker run, retrying..." - done - echo Starting discovery... - until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[0]} $SSH_SUFFIX "docker run --detach=true --link=agency:agency --name discovery ${DOCKER_IMAGE_NAME} arangosh --javascript.execute scripts/discover.js > /home/$SSH_USER/discovery.log" - do - echo "Error in remote docker run, retrying..." - done start_dbserver () { i=$1 echo Starting DBserver on ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$i]} $SSH_SUFFIX \ - docker run --detach=true -v $DBSERVER_DATA:/data \ - -v $DBSERVER_LOGS:/logs --net=host \ + docker run -p $PORT_DBSERVER:8529 --detach=true -e ARANGO_NO_AUTH=1 -v $DBSERVER_DATA:/var/lib/arangodb3 \ --name=dbserver$PORT_DBSERVER ${DOCKER_IMAGE_NAME} \ - arangod --database.directory /data \ - --frontend-version-check false \ - --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ + arangod --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ --cluster.my-address tcp://${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER \ - --server.endpoint tcp://0.0.0.0:$PORT_DBSERVER \ --cluster.my-local-info dbserver:${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER \ - --log.file /logs/$PORT_DBSERVER.log \ - --dispatcher.report-interval 15 \ - --server.foxx-queues false \ - --server.disable-statistics true \ + --cluster.my-role PRIMARY \ --scheduler.threads 3 \ --server.threads 5 \ - $DBSERVER_ARGS \ - >/dev/null + --javascript.v8-contexts 6 \ + $DBSERVER_ARGS do echo "Error in remote docker run, retrying..." done @@ -244,24 +227,18 @@ startArangoDBClusterWithDocker() { echo Starting Coordinator on ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$i]} $SSH_SUFFIX \ - docker run --detach=true -v $COORDINATOR_DATA:/data \ - -v $COORDINATOR_LOGS:/logs --net=host \ + docker run -p $PORT_COORDINATOR:8529 --detach=true -e ARANGO_NO_AUTH=1 -v $COORDINATOR_DATA:/var/lib/arangodb3 \ --name=coordinator$PORT_COORDINATOR \ ${DOCKER_IMAGE_NAME} \ - arangod --database.directory /data \ - --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ + arangod --cluster.agency-endpoint tcp://${SERVERS_INTERNAL_ARR[0]}:4001 \ --cluster.my-address tcp://${SERVERS_INTERNAL_ARR[$i]}:$PORT_COORDINATOR \ - --server.endpoint tcp://0.0.0.0:$PORT_COORDINATOR \ --cluster.my-local-info \ coordinator:${SERVERS_INTERNAL_ARR[$i]}:$PORT_COORDINATOR \ - --log.file /logs/$PORT_COORDINATOR.log \ - --dispatcher.report-interval 15 \ - --server.foxx-queues false \ - --server.disable-statistics true \ + --cluster.my-role COORDINATOR \ --scheduler.threads 4 \ - --server.threads 40 \ - $COORDINATOR_ARGS \ - >/dev/null + --javascript.v8-contexts 11 \ + --server.threads 10 \ + $COORDINATOR_ARGS do echo "Error in remote docker run, retrying..." done @@ -294,84 +271,58 @@ startArangoDBClusterWithDocker() { fi done } - - #for i in `seq 0 $LASTDBSERVER` ; do - # testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER - #done - - for i in $COORDINATOR_MACHINES ; do - testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR + + DBSERVER_IDS=() + for i in `seq 0 $LASTDBSERVER` ; do + testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER + DBSERVER_IDS[$i]=$(curl http://"${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER"/_admin/server/id) done - echo Bootstrapping DBServers... - curl -s -X POST "http://${SERVERS_EXTERNAL_ARR[$FIRST_COORDINATOR]}:$PORT_COORDINATOR/_admin/cluster/bootstrapDbServers" \ - -d '{"isRelaunch":false}' >/dev/null 2>&1 - - echo Running DB upgrade on cluster... - curl -s -X POST "http://${SERVERS_EXTERNAL_ARR[$FIRST_COORDINATOR]}:$PORT_COORDINATOR/_admin/cluster/upgradeClusterDatabase" \ - -d '{"isRelaunch":false}' >/dev/null 2>&1 - - echo Bootstrapping Coordinators... for i in $COORDINATOR_MACHINES ; do - echo Doing ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR - curl -s -X POST "http://${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR/_admin/cluster/bootstrapCoordinator" \ - -d '{"isRelaunch":false}' >/dev/null 2>&1 & + testServer ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR done - - wait - + if [ ! -z "$REPLICAS" ] ; then start_replica () { i=$1 j=`expr $i + 1` + ID="Secondary$j" if [ $j -gt $LASTDBSERVER ] ; then j=0 fi echo Starting asynchronous replica for echo " ${SERVERS_EXTERNAL_ARR[$i]}:$PORT_DBSERVER on ${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA" + + while true; do + curl -f -X PUT --data "{\"primary\": \"${DBSERVER_IDS[$i]}\", \"oldSecondary\": \"none\", \"newSecondary\": \"${ID}\"}" -H "Content-Type: application/json" "${SERVERS_EXTERNAL_ARR[$i]}:$PORT_COORDINATOR"/_admin/cluster/replaceSecondary + if [ "$?" == "0" ]; then + break + fi + echo "Failed registering secondary...Retrying..." + sleep 1 + done until $SSH_CMD "${SSH_ARGS}" ${SSH_USER}@${SERVERS_EXTERNAL_ARR[$j]} $SSH_SUFFIX \ - docker run --detach=true -v $REPLICA_DATA:/data \ - -v $REPLICA_LOGS:/logs --net=host \ + docker run -p 8529:$PORT_REPLICA --detach=true -e ARANGO_NO_AUTH=1 -v $REPLICA_DATA:/var/lib/arangodb3 \ --name=replica$PORT_REPLICA ${DOCKER_IMAGE_NAME} \ - arangod --database.directory /data \ - --frontend-version-check false \ - --server.endpoint tcp://0.0.0.0:$PORT_REPLICA \ - --log.file /logs/$PORT_REPLICA.log \ - --dispatcher.report-interval 15 \ - --server.foxx-queues false \ - --server.disable-statistics true \ - --scheduler.threads 1 \ - --server.threads 2 \ - $REPLICA_ARGS \ - >/dev/null + arangod \ + --cluster.my-id "$ID" \ + --cluster.my-role SECONDARY \ + --scheduler.threads 3 \ + --server.threads 5 \ + --javascript.v8-contexts 6 \ + $REPLICA_ARGS do echo "Error in remote docker run, retrying..." done } for i in `seq 0 $LASTDBSERVER` ; do - start_replica $i & + start_replica $i done echo Waiting 10 seconds till replicas are up and running... sleep 10 - - for i in `seq 0 $LASTDBSERVER` ; do - j=`expr $i + 1` - if [ $j -gt $LASTDBSERVER ] ; then - j=0 - fi - echo Attaching replica on $j for $i ... - curl -s -X PUT "http://${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA/_api/replication/applier-config" -d '{"endpoint":"tcp://'${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER'","database":"_system","includeSystem":false}' --dump - - # >/dev/null 2>&1 - TICK=`curl -X PUT "http://${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA/_api/replication/sync" -d '{"endpoint":"tcp://'${SERVERS_INTERNAL_ARR[$i]}:$PORT_DBSERVER'"}' | sed -e 's/^.*lastLogTick":"\([0-9]*\)"}.*$/\1/'` - # >/dev/null 2>&1 - curl -X PUT "http://${SERVERS_EXTERNAL_ARR[$j]}:$PORT_REPLICA/_api/replication/applier-start?from=$TICK" --dump - && echo - # >/dev/null 2>&1 - done - wait - fi echo "" diff --git a/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh b/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh index 944f1a4..dac5971 100755 --- a/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh +++ b/platformAWS/AmazonWebServices_ArangoDB_Cluster.sh @@ -12,9 +12,6 @@ trap "kill 0" SIGINT -#CoreOS AWS Image List -#https://coreos.com/docs/running-coreos/cloud-providers/ec2/ - MACHINE_TYPE="t1.medium" NUMBER="3" OUTPUT="aws" @@ -126,7 +123,7 @@ done PREFIX="arangodb-test-$$-" echo "OUTPUT DIRECTORY: $OUTPUT" -echo "ZONE: $ZONE" +echo "ZONE: $zone" echo "PROJECT: $PROJECT" DEFAULT_KEY_PATH="$HOME/.ssh/${PREFIX}aws-ssh-key" @@ -141,46 +138,21 @@ else ZONE=$zone fi -#We have to keep all ami's up to date -# Current Version: CoreOS 607.0.0 -# URL: https://coreos.com/docs/running-coreos/cloud-providers/ec2/ - -if [ "$zone" == "eu-central-1" ]; then - IMAGE="ami-0e300d13" -fi - -if [ "$zone" == "ap-northeast-1" ]; then - IMAGE="ami-af28dcaf" -fi - -if [ "$zone" == "sa-east-1" ]; then - IMAGE="ami-2354ec3e" -fi - -if [ "$zone" == "ap-southeast-2" ]; then - IMAGE="ami-b9b5c583" -fi - -if [ "$zone" == "ap-southeast-1" ]; then - IMAGE="mi-f80b3aaa" -fi +# Function to get latest core os ami ids (stable channel) +# URL: https://coreos.com/dist/aws/aws-stable.json -if [ "$zone" == "us-east-1" ]; then - IMAGE="ami-323b195a" -fi - -if [ "$zone" == "us-west-2" ]; then - IMAGE="ami-0789a437" -fi +echo "Your aws zone is: $ZONE" +echo "Searching ami id of latest core os version..." -if [ "$zone" == "us-west-1" ]; then - IMAGE="ami-8dd533c9" -fi - -if [ "$zone" == "eu-west-1" ]; then - IMAGE="ami-55950a22" -fi +function jsonval { + temp=`echo $json | sed 's/\\\\\//\//g' | sed 's/[{}]//g' | awk -v k="text" '{n=split($0,a,","); for (i=1; i<=n; i++) print a[i]}' | sed 's/\"\:\"/\|/g' | sed 's/[\,]/ /g' | sed 's/\"//g' | grep -w $ZONE` + echo ${temp##*|} +} +json=`curl -s -X GET https://coreos.com/dist/aws/aws-stable.json` +amiid=`jsonval` +IMAGE=`echo $amiid | awk {'print $3'}` +echo "Found image. ID is: $IMAGE" if test -e "$OUTPUT"; then if [ "$REMOVE" == "1" ] ; then @@ -337,7 +309,7 @@ function setMachineName () { function setMachineSecurity () { echo "Adding security groups." id=`cat "$OUTPUT/temp/IDS$i"` - secureid=`aws ec2 describe-security-groups --output json --group-names ${PREFIX}security |python -mjson.tool|grep GroupId| awk {'print $2'}| cut -c 2- | rev | cut -c 3- | rev` + secureid=$(aws ec2 describe-security-groups --output json --group-names ${PREFIX}security | grep GroupId | sed -e 's/.*: "\([^"]*\)"/\1/g') aws ec2 modify-instance-attribute --instance-id "$id" --groups "$secureid" } diff --git a/platformAZURE/Azure_ArangoDB_Cluster.sh b/platformAZURE/Azure_ArangoDB_Cluster.sh index 2fb21f0..0131889 100755 --- a/platformAZURE/Azure_ArangoDB_Cluster.sh +++ b/platformAZURE/Azure_ArangoDB_Cluster.sh @@ -12,11 +12,11 @@ trap "kill 0" SIGINT -ZONE="West US" +ZONE="West Europe" MACHINE_TYPE="Medium" NUMBER="3" OUTPUT="azure" -IMAGE="2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-607.0.0" +IMAGE="2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-835.12.0" SSH_KEY_PATH="" DEFAULT_KEY_PATH="$OUTPUT/arangodb_azure_key" @@ -165,7 +165,7 @@ then DEFAULT_KEY_PATH="$HOME/.ssh/arangodb_azure_key" else echo "No SSH-Key-Path given. Creating a new SSH-Key." - ssh-keygen -t dsa -f "$DEFAULT_KEY_PATH" -C "arangodb@arangodb.com" + ssh-keygen -t rsa -f "$DEFAULT_KEY_PATH" -C "arangodb@arangodb.com" # openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout "$DEFAULT_KEY_PATH" -out "$DEFAULT_KEY_PATH.pem" openssl req -x509 -key "$DEFAULT_KEY_PATH" -nodes -days 365 -newkey rsa:2048 -out "${DEFAULT_KEY_PATH}.pem" diff --git a/platformDO/DigitalOcean_ArangoDB_Cluster.sh b/platformDO/DigitalOcean_ArangoDB_Cluster.sh index 261f742..1536ecc 100755 --- a/platformDO/DigitalOcean_ArangoDB_Cluster.sh +++ b/platformDO/DigitalOcean_ArangoDB_Cluster.sh @@ -192,7 +192,7 @@ if test -z "$SSHID"; then then echo "No ArangoDB SSH-Key found. Generating a new one.!" - ssh-keygen -t dsa -f $OUTPUT/$SSH_KEY -C "arangodb@arangodb.com" + ssh-keygen -t rsa -f $OUTPUT/$SSH_KEY -C "arangodb@arangodb.com" if [ $? -eq 0 ]; then echo OK diff --git a/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh b/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh index b5a2d46..bfb8376 100755 --- a/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh +++ b/platformGCE/GoogleComputeEngine_ArangoDB_Cluster.sh @@ -375,7 +375,7 @@ export SERVERS_EXTERNAL export SERVERS_IDS export SSH_USER="core" export SSH_CMD="ssh" -export SSH_SUFFIX="-i $DEFAULT_KEY_PATH -l $SSH_USER" +export SSH_SUFFIX="" export ZONE export PROJECT export DBSERVER_DATA=/data/dbserver