diff --git a/README.md b/README.md index 8caea37..dcaef40 100644 --- a/README.md +++ b/README.md @@ -49,17 +49,18 @@ DisTRaC creates a ceph instance that runs using RAM. As the system is volatile, DisTRaC takes several parameters to run: ``` ./distrac.sh - -i= | --interface= Network interface to use, i.e. -i=ib0 or --interface=ib0 (Required) - -s= | --size= Size of RAM to use, i.e. -s=50GB or --size=100GB (Required) - -n= | --number= Number of RAM OSDs on each host, if -s=50GB and -n=5 that will create 5 OSDs using 250GB of RAM (Required) - -t= | --type= The type of RAM Block gram, ram (brd), zram i.e. -t=gram or -t=ram or --type=zram (Required) - -f= | --folder= Folder to locate Ceph keys this allows for multiple deployments when different folders speficed. - -hf= | --hostfile= When not using UGE with a parallel environment, provide a file with a list of comma separated hosts - -pn= | --poolname= Define the name of a pool if using RADOS, i.e. -pn=example or --poolname=example - -rgw | --rgw To use a rados gateway set the flag set on i.e. -rgw or --rgw - -uid=| --uid= To create an s3 user for the rados gateway i.e. -uid=test or --uid=test - -sk= | --secretkey= This will create an acess and secret key for the user define in -uid, i.e. -sk=test or --secretkey=test - -h | --help Display help message + -i= | --interface= Network interface to use, i.e. -i=ib0 or --interface=ib0 (Required) + -s= | --size= Size of RAM to use, i.e. -s=50GB or --size=100GB (Required) + -n= | --number= Number of RAM OSDs on each host, if -s=50GB and -n=5 that will create 5 OSDs using 250GB of RAM (Required) + -t= | --type= The type of RAM Block gram, ram (brd), zram i.e. -t=gram or -t=ram or --type=zram (Required) + -f= | --folder= Folder to locate Ceph keys this allows for multiple deployments when different folders speficed. + -hf= | --hostfile= When not using UGE with a parallel environment, provide a file with a list of comma separated hosts + -pn= | --poolname= Define the name of a pool if using RADOS, i.e. -pn=example or --poolname=example + -fs= | --filesystem= To use a cephfs specifiy a location that you want the system to mount i.e. -fs=/mnt/mycephfs + -rgw | --rgw To use a rados gateway set the flag set on i.e. -rgw or --rgw + -uid=| --uid= To create an s3 user for the rados gateway i.e. -uid=test or --uid=test + -sk= | --secretkey= This will create an acess and secret key for the user define in -uid, i.e. -sk=test or --secretkey=test + -h | --help Display help message ``` Removal only requires a folder and hostfile is they where specified when running DisTRaC otherwise `./remove-distrac.sh` will suffice. @@ -67,10 +68,11 @@ Removal only requires a folder and hostfile is they where specified when running ``` ./remove-distrac.sh -h - -f= | --folder= Folder with Ceph keys to remove - -t= | --type= The type of RAM Block gram, ram (brd), zram i.e. -t=gram or -t=ram or --type=zram (Required) - -hf= | --hostfile= When not using UGE with a parallel environment, provide a file with a list of comma separated hosts for Ceph removal. - -h | --help Display help message + -f= | --folder= Folder with Ceph keys to remove + -t= | --type= The type of RAM Block gram, ram (brd), zram i.e. -t=gram or -t=ram or --type=zram (Required) + -hf= | --hostfile= When not using UGE with a parallel environment, provide a file with a list of comma separated hosts for Ceph removal. + -fs= | --filesystem= To remove the filesystem specifiy the mount point used i.e. -fs=/mnt/mycephfs + -h | --help Display help message ``` @@ -121,6 +123,14 @@ To create a ceph instance with a rados gateway, three additional parameters need This creates a ceph instance that has rados gateway and an s3 user with the id of example and a secret and access key of example. +### Ceph instance with a mounted CephFS filesystem + +To create a ceph instance with a mounted filesystem, an additional parameter needs to be passed. `-fs=` creates a filesystem, with the MDS residing on the primary node, mounting it at the location specificied by the parameter across all nodes. + +``` +./distrac.sh -i=ib0 -n=1 -s=80G -t=gram -fs=/mnt/mycephfs +``` +This creates a cephfs instance mounted at /mnt/mycephfs for all compute nodes. ### Multiple instances of ceph at once diff --git a/distrac/create-brd.sh b/distrac/create-brd.sh index f6853ff..1a40a1f 100755 --- a/distrac/create-brd.sh +++ b/distrac/create-brd.sh @@ -15,7 +15,7 @@ case $i in ;; -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null shift # past argument=value ;; *) @@ -25,5 +25,4 @@ esac done # Load brd ram block module sudo modprobe brd rd_size=`echo $(( $(numfmt $size --from iec) / 1024 )) ` max_part=1 rd_nr=$amount -sudo pvcreate /dev/ram[0-$((amount-1))] & -wait \ No newline at end of file + diff --git a/distrac/create-fs.sh b/distrac/create-fs.sh new file mode 100755 index 0000000..7a9ae45 --- /dev/null +++ b/distrac/create-fs.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +filesystem="" +for i in "$@" +do +case $i in + -fs=*|--filesystem=*) + filesystem="${i#*=}" + mkdir $filesystem 2> /dev/null + shift # past argument=value + ;; + *) + # unknown option + ;; +esac +done + +sudo mount -t ceph :/ $filesystem -o name=admin,secret=`ceph auth print-key client.admin` +sudo chown $USER:$USER $filesystem +sudo chmod 777 $filesystem diff --git a/distrac/create-gram.sh b/distrac/create-gram.sh index 020ceda..e64aac9 100755 --- a/distrac/create-gram.sh +++ b/distrac/create-gram.sh @@ -15,7 +15,7 @@ case $i in ;; -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null shift # past argument=value ;; *) @@ -26,7 +26,7 @@ done # This ignores root squash cp gram.ko /tmp/ # Changes LVM so pvcreate can be used -./create-gram-lvm.sh +create-gram-lvm.sh sudo insmod /tmp/gram.ko num_devices=$amount & wait for num in $(seq 0 $[amount-1]) @@ -34,5 +34,4 @@ do echo $size | sudo tee /sys/block/gram$num/disksize & wait done -sudo pvcreate /dev/gram[0-$((amount-1))] & -wait + diff --git a/distrac/create-mds.sh b/distrac/create-mds.sh new file mode 100755 index 0000000..fe0671c --- /dev/null +++ b/distrac/create-mds.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +folder="." +for i in "$@" +do +case $i in + -f=*|--folder=*) + folder="${i#*=}" + mkdir $folder 2> /dev/null + shift # past argument=value + ;; + *) + # unknown option + ;; +esac +done +sudo mkdir /var/lib/ceph/mds/ceph-$HOSTNAME +ceph auth get-or-create mds.$HOSTNAME osd "allow rwx" mds "allow" mon "allow profile mds" > $folder/ceph.mds.keyring +chmod 644 $folder/ceph.mds.keyring +sudo cp $folder/ceph.mds.keyring /var/lib/ceph/mds/ceph-$HOSTNAME/keyring +sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-$HOSTNAME/ +sudo systemctl start ceph-mds@$HOSTNAME +create-pool.sh -pn=cephfs_data -per=0.90 -f=$folder & +wait +create-pool.sh -pn=cephfs_metadata -per=0.10 -f=$folder & +wait +ceph fs new cephfs cephfs_metadata cephfs_data & +wait +state=0 +while [ $state -le 0 ] +do + state=$(ceph fs status | grep -c "active") +done diff --git a/distrac/create-mgr.sh b/distrac/create-mgr.sh index 8dbe882..903b67c 100755 --- a/distrac/create-mgr.sh +++ b/distrac/create-mgr.sh @@ -5,7 +5,7 @@ do case $i in -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null shift # past argument=value ;; *) @@ -27,4 +27,4 @@ sudo chown -R ceph:ceph /var/lib/ceph/mgr/ceph-$HOSTNAME/ # Stating MGR daemon sudo systemctl start ceph-mgr@$HOSTNAME # Starting Dashboard -ceph mgr module enable dashboard +#ceph mgr module enable dashboard diff --git a/distrac/create-mon.sh b/distrac/create-mon.sh index 4cbddd3..0bd0c94 100755 --- a/distrac/create-mon.sh +++ b/distrac/create-mon.sh @@ -40,8 +40,9 @@ osd pool default size = 1 mon pg warn min per osd = 30 mon pg warn max per osd = 166496 mon max pg per osd = 166496 +osd pool default pg autoscale mode = off " > $folder/ceph.conf -cat log.conf >> ceph.conf +distrac-config-log.sh # Copy ceph.conf to system sudo cp $folder/ceph.conf /etc/ceph/ # Create Keyrings diff --git a/distrac/create-osd.sh b/distrac/create-osd.sh index 6d58aff..04e59b7 100755 --- a/distrac/create-osd.sh +++ b/distrac/create-osd.sh @@ -31,22 +31,20 @@ done sudo cp $folder/ceph.client.admin.keyring /etc/ceph/ sudo cp $folder/ceph.conf /etc/ceph/ sudo cp $folder/ceph.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring - +sudo cp $folder/ceph.bootstrap-osd.keyring /etc/ceph/ceph.keyring if [ $type == gram ] then - ./create-gram.sh -s=$size -n=$amount -f=$folder + create-gram.sh -s=$size -n=$amount -f=$folder elif [ $type == ram ] then - ./create-brd.sh -s=$size -n=$amount -f=$folder + create-brd.sh -s=$size -n=$amount -f=$folder elif [ $type == zram ] then - ./create-zram.sh -s=$size -n=$amount -f=$folder + create-zram.sh -s=$size -n=$amount -f=$folder fi - -# Creating OSDs using ceph-volume for num in $(seq 0 $[amount-1]) do - sudo ceph-volume --log-path /dev/null lvm create --data /dev/$type$num & + sudo ceph-volume --log-path /dev/null lvm prepare --data /dev/$type$num done -wait +sudo ceph-volume --log-path /dev/null lvm activate --all diff --git a/distrac/create-pool.sh b/distrac/create-pool.sh index 5d92641..a77416f 100755 --- a/distrac/create-pool.sh +++ b/distrac/create-pool.sh @@ -13,7 +13,7 @@ case $i in ;; -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null shift # past argument=value ;; *) @@ -28,18 +28,23 @@ amountOfOSDs=`cat $folder/amountOfOSDs.num` # Gets current PGS in ceph -currentPGs=$(ceph pg stat | awk '{print $1}') -source ./calculate-pool-pg.sh +currentPGs=$(ceph pg stat 2> /dev/null | awk '{print $1}') +if [ "$currentPGs" -eq "0" ]; +then + currentPGs=1 +fi + +source calculate-pool-pg.sh # Works out the PGs need for pool CalculatePoolPG $percentage $amountOfHosts $amountOfOSDs # Creates a pool with the name passed and amout of PGs -ceph osd pool create $poolname $result & -wait +ceph osd pool create $poolname $result +echo "Creating PG's" # Update the expected PGs active and clean to current plus new result=$(expr $result + $currentPGs) # Check if all pgs are active and clean -pgstat=$(ceph pg stat | grep -c "$result active+clean") +pgstat=$(ceph -s | grep -c "$result active+clean") while [ $pgstat -le 0 ] do - pgstat=$(ceph pg stat | grep -c "$result active+clean") + pgstat=$(ceph -s | grep -c "$result active+clean") done diff --git a/distrac/create-rgw.sh b/distrac/create-rgw.sh index 4598661..6ba5101 100755 --- a/distrac/create-rgw.sh +++ b/distrac/create-rgw.sh @@ -5,7 +5,7 @@ do case $i in -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null shift # past argument=value ;; *) @@ -24,17 +24,17 @@ sudo cp $folder/ceph.client.radosgw.keyring /var/lib/ceph/radosgw/ceph-radosgw.$ # Setting folder permission to ceph user sudo chown -R ceph:ceph /var/lib/ceph/radosgw/ceph-radosgw.$HOSTNAME/keyring # Creating RGW Pools -./create-pool.sh -pn=.rgw.root -per=0.05 -f=$folder & +create-pool.sh -pn=.rgw.root -per=0.05 -f=$folder & wait -./create-pool.sh -pn=default.rgw.control -per=0.02 -f=$folder & +create-pool.sh -pn=default.rgw.control -per=0.02 -f=$folder & wait -./create-pool.sh -pn=default.rgw.meta -per=0.02 -f=$folder & +create-pool.sh -pn=default.rgw.meta -per=0.02 -f=$folder & wait -./create-pool.sh -pn=default.rgw.log -per=0.02 -f=$folder & +create-pool.sh -pn=default.rgw.log -per=0.02 -f=$folder & wait -./create-pool.sh -pn=default.rgw.buckets.index -per=0.05 -f=$folder & +create-pool.sh -pn=default.rgw.buckets.index -per=0.05 -f=$folder & wait -./create-pool.sh -pn=default.rgw.buckets.data -per=0.84 -f=$folder & +create-pool.sh -pn=default.rgw.buckets.data -per=0.84 -f=$folder & wait # Start rados gateway sudo systemctl start ceph-radosgw@radosgw.$HOSTNAME diff --git a/distrac/create-zram.sh b/distrac/create-zram.sh index db388e7..d222067 100755 --- a/distrac/create-zram.sh +++ b/distrac/create-zram.sh @@ -15,7 +15,7 @@ case $i in ;; -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null shift # past argument=value ;; *) @@ -32,7 +32,6 @@ do wait done # Changes LVM so pvcreate can be used -./create-zram-lvm.sh -sudo pvcreate /dev/zram[0-$((amount-1))] & -wait +create-zram-lvm.sh + diff --git a/distrac/distrac-config-log.sh b/distrac/distrac-config-log.sh new file mode 100755 index 0000000..3965df8 --- /dev/null +++ b/distrac/distrac-config-log.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +folder="." +for i in "$@" +do +case $i in + -f=*|--folder=*) + folder="${i#*=}" + mkdir $folder + shift # past argument=value + ;; + *) + # unknown option + ;; +esac +done + +echo " +#Log to /dev/null +log flush on exit = false +log file = /dev/null +mon cluster log file = /dev/null +#Log level 0 and memory 0 +debug default = 0/0 +debug lockdep = 0/0 +debug context = 0/0 +debug crush = 0/0 +debug mds = 0/0 +debug mds balancer = 0/0 +debug mds locker = 0/0 +debug mds log = 0/0 +debug mds log expire = 0/0 +debug mds migrator = 0/0 +debug buffer = 0/0 +debug timer = 0/0 +debug filer = 0/0 +debug striper = 0/0 +debug objecter = 0/0 +debug rados = 0/0 +debug rbd = 0/0 +debug rbd mirror = 0/0 +debug rbd replay = 0/0 +debug journaler = 0/0 +debug objectcacher = 0/0 +debug client = 0/0 +debug osd = 0/0 +debug optracker = 0/0 +debug objclass = 0/0 +debug filestore = 0/0 +debug journal = 0/0 +debug ms = 0/0 +debug mon = 0/0 +debug monc = 0/0 +debug paxos = 0/0 +debug tp = 0/0 +debug auth = 0/0 +debug crypto = 0/0 +debug finisher = 0/0 +debug reserver = 0/0 +debug heartbeatmap = 0/0 +debug perfcounter = 0/0 +debug rgw = 0/0 +debug rgw sync = 0/0 +debug civetweb = 0/0 +debug javaclient = 0/0 +debug asok = 0/0 +debug throttle = 0/0 +debug refs = 0/0 +debug compressor = 0/0 +debug bluestore = 0/0 +debug bluefs = 0/0 +debug bdev = 0/0 +debug kstore = 0/0 +debug rocksdb = 0/0 +debug leveldb = 0/0 +debug memdb = 0/0 +debug fuse = 0/0 +debug mgr = 0/0 +debug mgrc = 0/0 +debug dpdk = 0/0 +debug eventtrace = 0/0 +" >> $folder/ceph.conf diff --git a/distrac/distrac.sh b/distrac/distrac.sh index a1e89d7..2ef41af 100755 --- a/distrac/distrac.sh +++ b/distrac/distrac.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash rgw=false folder="." -source ./helpmsg.sh +source helpmsg.sh if [ -z "$@" ] then @@ -14,7 +14,11 @@ do case $i in -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null + shift # past argument=value + ;; + -fs=*|--filesystem=*) + filesystem="${i#*=}" shift # past argument=value ;; -i=*|--interface=*) @@ -67,32 +71,44 @@ done echo $amount> $folder/amountOfOSDs.num if [ ! -z "$hostfile" ] then - source ./process-hostfile.sh -f=$folder -hf=$hostfile + source process-hostfile.sh -f=$folder -hf=$hostfile else - source ./uge-hostfile.sh -f=$folder + source uge-hostfile.sh -f=$folder fi - -./create-mon.sh -i=$interface -f=$folder -./create-mgr.sh -f=$folder +echo "Creating MON" +create-mon.sh -i=$interface -f=$folder +echo "Creating MGR" +create-mgr.sh -f=$folder echo HOSTS: cat $folder/hostfile module load openmpi amountOfHosts=`cat $folder/amountOfHosts.num` -mpirun -np $amountOfHosts --map-by ppr:1:node --hostfile $folder/hostfile ./create-osd.sh -s=$size -n=$amount -f=$folder -t=$type +echo "Creating OSDs" +mpirun -np $amountOfHosts --map-by ppr:1:node --hostfile $folder/hostfile create-osd.sh -s=$size -n=$amount -f=$folder -t=$type +if ([ ! -z ${filesystem+x} ]) +then + echo "Creating MDS" + create-mds.sh -f=$folder + echo "Mounting Filesystem" + mpirun -np $amountOfHosts --map-by ppr:1:node --hostfile $folder/hostfile create-fs.sh -fs=$filesystem +fi + if ([ ! -z "$poolname" ] && [ "$rgw" = "false" ]) then - ./create-pool.sh -pn=$poolname -per=1 -f=$folder + echo "Creating POOL" + create-pool.sh -pn=$poolname -per=1 -f=$folder fi module unload openmpi if ([ "$rgw" = "true" ] && [ -z "$poolname" ]) - then - ./create-rgw.sh -f=$folder + then + echo "Creating RGW" + create-rgw.sh -f=$folder if ([ -z "$id" ] && [ -z "$secret" ]) then - echo "No user will be create" + echo "No user will be created" else - ./create-s3-user.sh --uid=$id -sk=$secret + create-s3-user.sh --uid=$id -sk=$secret fi fi diff --git a/distrac/helpmsg.sh b/distrac/helpmsg.sh index ed190b2..fdfd66a 100755 --- a/distrac/helpmsg.sh +++ b/distrac/helpmsg.sh @@ -3,24 +3,26 @@ #Prints out a help message for users function helpmsg(){ echo " - -i= | --interface= Network interface to use, i.e. -i=ib0 or --interface=ib0 (Required) - -s= | --size= Size of RAM to use, i.e. -s=50GB or --size=100GB (Required) - -n= | --number= Number of RAM OSDs on each host, if -s=50GB and -n=5 that will create 5 OSDs using 250GB of RAM (Required) - -t= | --type= The type of RAM Block gram, ram (brd), zram i.e. -t=gram or -t=ram or --type=zram (Required) - -f= | --folder= Folder to locate Ceph keys this allows for multiple deployments when different folders speficed. - -hf= | --hostfile= When not using UGE with a parallel environment, provide a file with a list of comma separated hosts - -pn= | --poolname= Define the name of a pool if using RADOS, i.e. -pn=example or --poolname=example - -rgw | --rgw To use a rados gateway set the flag set on i.e. -rgw or --rgw - -uid=| --uid= To create an s3 user for the rados gateway i.e. -uid=test or --uid=test - -sk= | --secretkey= This will create an acess and secret key for the user define in -uid, i.e. -sk=test or --secretkey=test - -h | --help Display help message + -i= | --interface= Network interface to use, i.e. -i=ib0 or --interface=ib0 (Required) + -s= | --size= Size of RAM to use, i.e. -s=50GB or --size=100GB (Required) + -n= | --number= Number of RAM OSDs on each host, if -s=50GB and -n=5 that will create 5 OSDs using 250GB of RAM (Required) + -t= | --type= The type of RAM Block gram, ram (brd), zram i.e. -t=gram or -t=ram or --type=zram (Required) + -f= | --folder= Folder to locate Ceph keys this allows for multiple deployments when different folders speficed. + -hf= | --hostfile= When not using UGE with a parallel environment, provide a file with a list of comma separated hosts + -pn= | --poolname= Define the name of a pool if using RADOS, i.e. -pn=example or --poolname=example + -fs= | --filesystem= To use a cephfs specifiy a location that you want the system to mount i.e. -fs=/mnt/mycephfs + -rgw | --rgw To use a rados gateway set the flag set on i.e. -rgw or --rgw + -uid=| --uid= To create an s3 user for the rados gateway i.e. -uid=test or --uid=test + -sk= | --secretkey= This will create an acess and secret key for the user define in -uid, i.e. -sk=test or --secretkey=test + -h | --help Display help message " } function helpmsgremove(){ echo " - -f= | --folder= Folder with Ceph keys to remove - -t= | --type= The type of RAM Block gram, ram (brd), zram i.e. -t=gram or -t=ram or --type=zram (Required) - -hf= | --hostfile= When not using UGE with a parallel environment, provide a file with a list of comma separated hosts for Ceph removal. - -h | --help Display help message + -f= | --folder= Folder with Ceph keys to remove + -t= | --type= The type of RAM Block gram, ram (brd), zram i.e. -t=gram or -t=ram or --type=zram (Required) + -hf= | --hostfile= When not using UGE with a parallel environment, provide a file with a list of comma separated hosts for Ceph removal. + -fs= | --filesystem= To remove the filesystem specifiy the mount point used i.e. -fs=/mnt/mycephfs + -h | --help Display help message " } diff --git a/distrac/process-hostfile.sh b/distrac/process-hostfile.sh index a4682ee..2e01ca6 100755 --- a/distrac/process-hostfile.sh +++ b/distrac/process-hostfile.sh @@ -6,7 +6,7 @@ do case $i in -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null shift # past argument=value ;; -hf=*|--hostfile=*) @@ -21,7 +21,7 @@ done # Convert the hostfile to a column of hosts and store in $folder/hostlist tr , '\n' < $hostfile > $folder/hostfile # Make sure the first host in hostfile is the headnode -sed -i "/\b\("$HOSTNAME"\)\b/d" $folder/hostfile -echo "$(printf "$HOSTNAME \n"; cat $folder/hostfile)" > $folder/hostfile +sed -i "/\b\("`hostname`"\)\b/d" $folder/hostfile +echo "$(printf "`hostname` \n"; cat $folder/hostfile)" > $folder/hostfile # Read hostfile and put the amount of hosts in amountOfHosts.num cat $folder/hostfile | wc -l > $folder/amountOfHosts.num diff --git a/distrac/remove-brd.sh b/distrac/remove-brd.sh index 266b8df..a978f3b 100755 --- a/distrac/remove-brd.sh +++ b/distrac/remove-brd.sh @@ -1,10 +1,5 @@ #!/usr/bin/env bash # Remove BRD block device -# Get amount of ram's -amount=` find /dev/ -name "ram*" | wc -l` -# Remove PV's -sudo pvremove /dev/ram[0-$((amount-1))] & -wait # Unload brd sudo rmmod brd diff --git a/distrac/remove-ceph.sh b/distrac/remove-ceph.sh index e145fde..ad0f0dc 100755 --- a/distrac/remove-ceph.sh +++ b/distrac/remove-ceph.sh @@ -4,7 +4,9 @@ sudo systemctl stop ceph.target sudo systemctl stop ceph-mon.target sudo systemctl stop ceph-mgr.target +sudo systemctl stop ceph-mds.target sudo systemctl stop ceph-radosgw.target +sudo systemctl stop ceph-crash.service sudo systemctl stop ceph-osd.target & osd=$! wait $osd @@ -14,14 +16,20 @@ sudo find "/var/lib/ceph/bootstrap-mgr/" -mindepth 1 -delete sudo find "/var/lib/ceph/bootstrap-osd/" -mindepth 1 -delete sudo find "/var/lib/ceph/bootstrap-rbd/" -mindepth 1 -delete sudo find "/var/lib/ceph/bootstrap-rgw/" -mindepth 1 -delete +sudo find "/var/lib/ceph/bootstrap-mds/" -mindepth 1 -delete sudo find "/var/lib/ceph/mgr/" -mindepth 1 -delete sudo find "/var/lib/ceph/mon/" -mindepth 1 -delete sudo find "/var/lib/ceph/radosgw/" -mindepth 1 -delete +sudo find "/var/lib/ceph/mds/" -mindepth 1 -delete sudo find "/var/lib/ceph/tmp/" -mindepth 1 -delete sudo find "/var/log/ceph/" -mindepth 1 -delete +sudo find "/var/lib/ceph/crash/" -mindepth 1 -delete +sudo find "/etc/ceph/" -mindepth 1 -delete + # Stop deamons sudo systemctl stop system-ceph\\x2dmgr.slice sudo systemctl stop system-ceph\\x2dmon.slice +sudo systemctl stop system-ceph\\x2dmds.slice sudo systemctl stop system-ceph\\x2dosd.slice sudo systemctl stop system-ceph\\x2dradosgw.slice sudo systemctl stop system-ceph\\x2dvolume.slice diff --git a/distrac/remove-distrac.sh b/distrac/remove-distrac.sh index 7e2004e..83f145d 100755 --- a/distrac/remove-distrac.sh +++ b/distrac/remove-distrac.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -source ./helpmsg.sh +source helpmsg.sh folder="." for i in "$@" do @@ -17,6 +17,10 @@ case $i in type="${i#*=}" shift ;; + -fs=*|--filesystem=*) + filesystem="${i#*=}" + shift # past argument=value + ;; -h| --help) helpmsgremove exit 0 @@ -31,17 +35,23 @@ done if [ ! -z "$hostfile" ] then - source ./process-hostfile.sh -f=$folder -hf=$hostfile + source process-hostfile.sh -f=$folder -hf=$hostfile else - source ./uge-hostfile.sh -f=$folder + source uge-hostfile.sh -f=$folder fi -module load openmpi amountOfHosts=`cat $folder/amountOfHosts.num` + +if ([ ! -z ${filesystem+x} ]) + then + echo "Removing FS" + mpirun -np $amountOfHosts --map-by ppr:1:node --hostfile $folder/hostfile remove-fs.sh -fs=$filesystem +fi + echo "Removing Ceph" -mpirun -np $amountOfHosts --map-by ppr:1:node --hostfile $folder/hostfile ./remove-ceph.sh +mpirun -np $amountOfHosts --map-by ppr:1:node --hostfile $folder/hostfile remove-ceph.sh echo "Removing OSDs" -mpirun -np $amountOfHosts --map-by ppr:1:node --hostfile $folder/hostfile ./remove-osd.sh -t=$type +mpirun -np $amountOfHosts --map-by ppr:1:node --hostfile $folder/hostfile remove-osd.sh -t=$type echo "Removing Temp Files" -./remove-temp-files.sh -f=$folder +remove-temp-files.sh -f=$folder echo "Done" diff --git a/distrac/remove-fs.sh b/distrac/remove-fs.sh new file mode 100755 index 0000000..7b1b788 --- /dev/null +++ b/distrac/remove-fs.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +filesystem="" +for i in "$@" +do +case $i in + -fs=*|--filesystem=*) + filesystem="${i#*=}" + mkdir $filesystem 2> /dev/null + shift # past argument=value + ;; + *) + # unknown option + ;; +esac +done + +sudo umount $filesystem \ No newline at end of file diff --git a/distrac/remove-gram.sh b/distrac/remove-gram.sh index a72d139..65c18f5 100755 --- a/distrac/remove-gram.sh +++ b/distrac/remove-gram.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -# Get amount of Grams -amount=` find /dev/ -name "gram*" | wc -l` -sudo pvremove /dev/gram[0-$((amount-1))] & -wait +# Remove GRAM sudo rmmod gram rm /tmp/gram.ko -./remove-gram-lvm.sh +remove-gram-lvm.sh diff --git a/distrac/remove-osd.sh b/distrac/remove-osd.sh index cc0e4a0..462595a 100755 --- a/distrac/remove-osd.sh +++ b/distrac/remove-osd.sh @@ -30,11 +30,11 @@ sudo find "/var/lib/ceph/osd/" -mindepth 1 -delete if [ $type == gram ] then - ./remove-gram.sh + remove-gram.sh elif [ $type == ram ] then - ./remove-brd.sh + remove-brd.sh elif [ $type == zram ] then - ./remove-zram.sh + remove-zram.sh fi \ No newline at end of file diff --git a/distrac/remove-temp-files.sh b/distrac/remove-temp-files.sh index 5721dbe..ce5e1d6 100755 --- a/distrac/remove-temp-files.sh +++ b/distrac/remove-temp-files.sh @@ -5,7 +5,7 @@ do case $i in -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null shift # past argument=value ;; *) diff --git a/distrac/remove-zram.sh b/distrac/remove-zram.sh index c00233c..9bc6719 100755 --- a/distrac/remove-zram.sh +++ b/distrac/remove-zram.sh @@ -1,11 +1,6 @@ #!/usr/bin/env bash # Remove ZRAM block device -# Get amount of zram's -amount=` find /dev/ -name "zram*" | wc -l` -# Remove PV's -sudo pvremove /dev/zram[0-$((amount-1))] & -wait # Unload zram sudo rmmod zram -./remove-zram-lvm.sh \ No newline at end of file +remove-zram-lvm.sh \ No newline at end of file diff --git a/distrac/uge-hostfile.sh b/distrac/uge-hostfile.sh index 003d9b7..2214bd0 100755 --- a/distrac/uge-hostfile.sh +++ b/distrac/uge-hostfile.sh @@ -5,7 +5,7 @@ do case $i in -f=*|--folder=*) folder="${i#*=}" - mkdir $folder + mkdir $folder 2> /dev/null shift # past argument=value ;; *) diff --git a/sudoers_file/distrac b/sudoers_file/distrac old mode 100755 new mode 100644 index 6fd14c3..7f5e187 --- a/sudoers_file/distrac +++ b/sudoers_file/distrac @@ -1,94 +1,116 @@ -# allow DisTRaC user to create a ceph cluster +# allow distrac group to create a ceph cluster # Create the Monitor node -distrac ALL=NOPASSWD: /usr/bin/cp */ceph.conf /etc/ceph/ -distrac ALL=NOPASSWD: /usr/bin/mkdir /var/lib/ceph/mon/ceph-* -distrac ALL=NOPASSWD: /usr/bin/ceph-mon --cluster ceph --mkfs -i * --monmap */ceph.monmap --keyring */ceph.mon.keyring -distrac ALL=NOPASSWD: /usr/bin/chown -R ceph\:ceph /var/lib/ceph/mon/ceph-* -distrac ALL=NOPASSWD: /usr/bin/cp */ceph.client.admin.keyring /etc/ceph/ -distrac ALL=NOPASSWD: /usr/bin/systemctl start ceph-mon@* +%distrac ALL=NOPASSWD: /usr/bin/cp */ceph.conf /etc/ceph/ +%distrac ALL=NOPASSWD: /usr/bin/mkdir /var/lib/ceph/mon/ceph-* +%distrac ALL=NOPASSWD: /usr/bin/ceph-mon --cluster ceph --mkfs -i * --monmap */ceph.monmap --keyring */ceph.mon.keyring +%distrac ALL=NOPASSWD: /usr/bin/chown -R ceph\:ceph /var/lib/ceph/mon/ceph-* +%distrac ALL=NOPASSWD: /usr/bin/cp */ceph.client.admin.keyring /etc/ceph/ +%distrac ALL=NOPASSWD: /usr/bin/systemctl start ceph-mon@* # Create the MGR node -distrac ALL=NOPASSWD: /usr/bin/mkdir /var/lib/ceph/mgr/ceph-* -distrac ALL=NOPASSWD: /usr/bin/cp */ceph.mgr.keyring /var/lib/ceph/mgr/ceph-*/keyring -distrac ALL=NOPASSWD: /usr/bin/chown -R ceph\:ceph /var/lib/ceph/mgr/ceph-* -distrac ALL=NOPASSWD: /usr/bin/systemctl start ceph-mgr@* +%distrac ALL=NOPASSWD: /usr/bin/mkdir /var/lib/ceph/mgr/ceph-* +%distrac ALL=NOPASSWD: /usr/bin/cp */ceph.mgr.keyring /var/lib/ceph/mgr/ceph-*/keyring +%distrac ALL=NOPASSWD: /usr/bin/chown -R ceph\:ceph /var/lib/ceph/mgr/ceph-* +%distrac ALL=NOPASSWD: /usr/bin/systemctl start ceph-mgr@* + +# Create the MDS node +%distrac ALL=NOPASSWD: /usr/bin/mkdir /var/lib/ceph/mds/ceph-* +%distrac ALL=NOPASSWD: /usr/bin/cp */ceph.mds.keyring /var/lib/ceph/mds/ceph-*/keyring +%distrac ALL=NOPASSWD: /usr/bin/chown -R ceph\:ceph /var/lib/ceph/mds/ceph-* +%distrac ALL=NOPASSWD: /usr/bin/systemctl start ceph-mds@* # Create BRD Block Device -distrac ALL=NOPASSWD: /usr/sbin/modprobe brd rd_size=* max_part=1 rd_nr=* -distrac ALL=NOPASSWD: /usr/sbin/pvcreate /dev/ram* +%distrac ALL=NOPASSWD: /usr/sbin/modprobe brd rd_size=* max_part=1 rd_nr=* + # Create ZRAM Block Device -distrac ALL=NOPASSWD: /usr/bin/sed -i /\#\\\\s*types\\\\s*\=\\\\s*\\\\\[\\\\s*\\"fd\\"\,\\\\s*16\\\\s*\]/atypes=\\\\\[\\"zram\\"\,1000\] /etc/lvm/lvm.conf -distrac ALL=NOPASSWD: /usr/sbin/modprobe zram num_devices=* -distrac ALL=NOPASSWD: /usr/bin/echo * -distrac ALL=NOPASSWD: /usr/bin/tee /sys/block/zram*/disksize -distrac ALL=NOPASSWD: /usr/sbin/pvcreate /dev/zram* +%distrac ALL=NOPASSWD: /usr/bin/sed -i /\#\\\\s*types\\\\s*\=\\\\s*\\\\\[\\\\s*\\"fd\\"\,\\\\s*16\\\\s*\]/atypes=\\\\\[\\"zram\\"\,1000\] /etc/lvm/lvm.conf +%distrac ALL=NOPASSWD: /usr/sbin/modprobe zram num_devices=* +%distrac ALL=NOPASSWD: /usr/bin/echo * +%distrac ALL=NOPASSWD: /usr/bin/tee /sys/block/zram*/disksize + # Create GRAM Block Device -distrac ALL=NOPASSWD: /usr/bin/sed -i /\#\\\\s*types\\\\s*\=\\\\s*\\\\\[\\\\s*\\"fd\\"\,\\\\s*16\\\\s*\]/atypes=\\\\\[\\"gram\\"\,1000\] /etc/lvm/lvm.conf -distrac ALL=NOPASSWD: /usr/sbin/insmod /tmp/gram.ko num_devices=* -distrac ALL=NOPASSWD: /usr/bin/echo * -distrac ALL=NOPASSWD: /usr/bin/tee /sys/block/gram*/disksize -distrac ALL=NOPASSWD: /usr/sbin/pvcreate /dev/gram* +%distrac ALL=NOPASSWD: /usr/bin/sed -i /\#\\\\s*types\\\\s*\=\\\\s*\\\\\[\\\\s*\\"fd\\"\,\\\\s*16\\\\s*\]/atypes=\\\\\[\\"gram\\"\,1000\] /etc/lvm/lvm.conf +%distrac ALL=NOPASSWD: /usr/sbin/insmod /tmp/gram.ko num_devices=* +%distrac ALL=NOPASSWD: /usr/bin/echo * +%distrac ALL=NOPASSWD: /usr/bin/tee /sys/block/gram*/disksize -# Create OSD -distrac ALL=NOPASSWD: /usr/bin/cp */ceph.client.admin.keyring /etc/ceph/ -distrac ALL=NOPASSWD: /usr/bin/cp */ceph.conf /etc/ceph/ -distrac ALL=NOPASSWD: /usr/bin/cp */ceph.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -distrac ALL=NOPASSWD: /usr/sbin/ceph-volume --log-path /dev/null lvm create --data /dev/ram* -distrac ALL=NOPASSWD: /usr/sbin/ceph-volume --log-path /dev/null lvm create --data /dev/zram* -distrac ALL=NOPASSWD: /usr/sbin/ceph-volume --log-path /dev/null lvm create --data /dev/gram* +# Create OSD +%distrac ALL=NOPASSWD: /usr/bin/cp */ceph.client.admin.keyring /etc/ceph/ +%distrac ALL=NOPASSWD: /usr/bin/cp */ceph.conf /etc/ceph/ +%distrac ALL=NOPASSWD: /usr/bin/cp */ceph.bootstrap-osd.keyring /etc/ceph/ceph.keyring +%distrac ALL=NOPASSWD: /usr/bin/cp */ceph.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring +%distrac ALL=NOPASSWD: /usr/sbin/ceph-volume --log-path /dev/null lvm prepare --data /dev/ram* +%distrac ALL=NOPASSWD: /usr/sbin/ceph-volume --log-path /dev/null lvm prepare --data /dev/zram* +%distrac ALL=NOPASSWD: /usr/sbin/ceph-volume --log-path /dev/null lvm prepare --data /dev/gram* +%distrac ALL=NOPASSWD: /usr/sbin/ceph-volume --log-path /dev/null lvm activate --all # Create RGW -distrac ALL=NOPASSWD: /usr/bin/mkdir /var/lib/ceph/radosgw/ceph-radosgw.* -distrac ALL=NOPASSWD: /usr/bin/cp */ceph.client.radosgw.keyring /var/lib/ceph/radosgw/ceph-radosgw.*/keyring -distrac ALL=NOPASSWD: /usr/bin/chown -R ceph\:ceph /var/lib/ceph/radosgw/ceph-radosgw.*/keyring -distrac ALL=NOPASSWD: /usr/bin/systemctl start ceph-radosgw@radosgw.* +%distrac ALL=NOPASSWD: /usr/bin/mkdir /var/lib/ceph/radosgw/ceph-radosgw.* +%distrac ALL=NOPASSWD: /usr/bin/cp */ceph.client.radosgw.keyring /var/lib/ceph/radosgw/ceph-radosgw.*/keyring +%distrac ALL=NOPASSWD: /usr/bin/chown -R ceph\:ceph /var/lib/ceph/radosgw/ceph-radosgw.*/keyring +%distrac ALL=NOPASSWD: /usr/bin/systemctl start ceph-radosgw@radosgw.* + + +# Create FS +%distrac ALL=NOPASSWD: /usr/bin/mkdir * +%distrac ALL=NOPASSWD: /usr/bin/mount -t ceph \:/ * -o name=admin\,secret=* +%distrac ALL=NOPASSWD: /usr/bin/chown -R *\:* * +%distrac ALL=NOPASSWD: /usr/bin/chmod 770 * # Remove Ceph +## Remove FS mount + +%distrac ALL=NOPASSWD: /usr/bin/umount * + ## Stopping services -distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph.target -distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-mon.target -distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-mgr.target -distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-radosgw.target -distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-osd.target +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph.target +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-mon.target +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-mgr.target +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-mds.target +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-radosgw.target +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-osd.target +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop ceph-crash.service ## Cleaning folders -distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/bootstrap-mgr/ -mindepth 1 -delete -distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/bootstrap-osd/ -mindepth 1 -delete -distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/bootstrap-rbd/ -mindepth 1 -delete -distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/bootstrap-rgw/ -mindepth 1 -delete -distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/mgr/ -mindepth 1 -delete -distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/mon/ -mindepth 1 -delete -distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/radosgw/ -mindepth 1 -delete -distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/osd/ -mindepth 1 -delete -distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/tmp/ -mindepth 1 -delete -distrac ALL=NOPASSWD: /usr/bin/find /var/log/ceph/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/bootstrap-mgr/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/bootstrap-osd/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/bootstrap-rbd/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/bootstrap-rgw/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/bootstrap-mds/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/mgr/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/mon/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/mds/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/radosgw/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/osd/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/tmp/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/lib/ceph/crash/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /var/log/ceph/ -mindepth 1 -delete +%distrac ALL=NOPASSWD: /usr/bin/find /etc/ceph/ -mindepth 1 -delete ## Stopping extra deamons -distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dmgr.slice -distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dmon.slice -distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dosd.slice -distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dradosgw.slice -distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dvolume.slice +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dmgr.slice +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dmon.slice +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dosd.slice +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dmds.slice +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dradosgw.slice +%distrac ALL=NOPASSWD: /usr/bin/systemctl stop system-ceph\\\\x2dvolume.slice # Removing OSDs -distrac ALL=NOPASSWD: /usr/sbin/vgdisplay -distrac ALL=NOPASSWD: /usr/sbin/vgremove ceph* -y -distrac ALL=NOPASSWD: /usr/bin/umount /var/lib/ceph/osd/* -distrac ALL=NOPASSWD: /usr/bin/systemctl reset-failed +%distrac ALL=NOPASSWD: /usr/sbin/vgdisplay +%distrac ALL=NOPASSWD: /usr/sbin/vgremove ceph* -y +%distrac ALL=NOPASSWD: /usr/bin/umount /var/lib/ceph/osd/* +%distrac ALL=NOPASSWD: /usr/bin/systemctl reset-failed ## Removing BRD block device -distrac ALL=NOPASSWD: /usr/sbin/pvremove /dev/ram* -distrac ALL=NOPASSWD: /usr/sbin/rmmod brd +%distrac ALL=NOPASSWD: /usr/sbin/rmmod brd ## Removing ZRAM block devices -distrac ALL=NOPASSWD: /usr/sbin/pvremove /dev/zram* -distrac ALL=NOPASSWD: /usr/sbin/rmmod zram -distrac ALL=NOPASSWD: /usr/bin/sed -i /types=\\\\\[\\"zram\\"\,1000]/d /etc/lvm/lvm.conf +%distrac ALL=NOPASSWD: /usr/sbin/rmmod zram +%distrac ALL=NOPASSWD: /usr/bin/sed -i /types=\\\\\[\\"zram\\"\,1000]/d /etc/lvm/lvm.conf ## Removing GRAM block devices -distrac ALL=NOPASSWD: /usr/sbin/pvremove /dev/gram* -distrac ALL=NOPASSWD: /usr/sbin/rmmod gram -distrac ALL=NOPASSWD: /usr/bin/sed -i /types=\\\\\[\\"gram\\"\,1000]/d /etc/lvm/lvm.conf +%distrac ALL=NOPASSWD: /usr/sbin/rmmod gram +%distrac ALL=NOPASSWD: /usr/bin/sed -i /types=\\\\\[\\"gram\\"\,1000]/d /etc/lvm/lvm.conf