From 680e35511f3a96fd934a00da8d826dbd0685e171 Mon Sep 17 00:00:00 2001 From: Slach Date: Sat, 17 Mar 2018 15:08:25 +0500 Subject: [PATCH 01/11] actualize Vagrant environment and fix problems on orchestrator-agent api with sqlite backend Signed-off-by: Slach --- .gitignore | 5 + Vagrantfile | 4 +- build.sh | 16 +-- conf/orchestrator-sample-sqlite.conf.json | 120 ++++++++++++++++++++++ go/agent/agent_dao.go | 2 +- go/db/generate_base.go | 2 +- vagrant/admin-build.sh | 21 +++- vagrant/base-build.sh | 39 ++++--- 8 files changed, 178 insertions(+), 31 deletions(-) create mode 100644 conf/orchestrator-sample-sqlite.conf.json diff --git a/.gitignore b/.gitignore index 0e4cea53a..c4f3b1a36 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ conf/orchestrator.conf.json .DS_Store .vagrant/ +vagrant/.sqlite vagrant/admin-post-install.sh vagrant/db-post-install.sh vagrant/db1-post-install.sh @@ -13,3 +14,7 @@ vagrant/vagrant-ssh-key.pub Godeps/_workspace .gopath/ main +.idea/ +*.deb +*.pcap +*.log diff --git a/Vagrantfile b/Vagrantfile index bd981b4c0..106fee857 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -16,8 +16,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = BOX config.vm.box_download_insecure = true config.vm.box_check_update = false - config.vm.synced_folder '.', '/orchestrator', type: 'rsync', - rsync__auto: true + config.vm.synced_folder '.', '/orchestrator' + #, type: 'rsync', rsync__auto: true (0..4).each do |n| name = (n > 0 ? ("db" + n.to_s) : "admin") diff --git a/build.sh b/build.sh index a560f3a59..2de0c4190 100755 --- a/build.sh +++ b/build.sh @@ -87,7 +87,7 @@ function oinstall() { cd $mydir gofmt -s -w go/ rsync -qa ./resources $builddir/orchestrator${prefix}/orchestrator/ - rsync -qa ./conf/orchestrator-sample.* $builddir/orchestrator${prefix}/orchestrator/ + rsync -qa ./conf/orchestrator-sample*.conf.json $builddir/orchestrator${prefix}/orchestrator/ cp etc/init.d/orchestrator.bash $builddir/orchestrator/etc/init.d/orchestrator chmod +x $builddir/orchestrator/etc/init.d/orchestrator } @@ -134,12 +134,14 @@ function package() { esac echo "---" - if cat /etc/centos-release | grep 'CentOS release 6' ; then - rm ${TOPDIR:-?}/orchestrator*.deb - rm ${TOPDIR:-?}/orchestrator*.tar.gz - # n CentOD 6 box: we only want the rpms for CentOS6 - # Add "-centos6" to the file name. - ls ${TOPDIR:-?}/*.rpm | while read f; do centos_file=$(echo $f | sed -r -e "s/^(.*)-${RELEASE_VERSION}(.*)/\1-centos6-${RELEASE_VERSION}\2/g") ; mv $f $centos_file ; done + if [[ -e /etc/centos-release ]]; then + if cat /etc/centos-release | grep 'CentOS release 6' ; then + rm ${TOPDIR:-?}/orchestrator*.deb + rm ${TOPDIR:-?}/orchestrator*.tar.gz + # n CentOD 6 box: we only want the rpms for CentOS6 + # Add "-centos6" to the file name. + ls ${TOPDIR:-?}/*.rpm | while read f; do centos_file=$(echo $f | sed -r -e "s/^(.*)-${RELEASE_VERSION}(.*)/\1-centos6-${RELEASE_VERSION}\2/g") ; mv $f $centos_file ; done + fi fi echo "Done. Find releases in $TOPDIR" } diff --git a/conf/orchestrator-sample-sqlite.conf.json b/conf/orchestrator-sample-sqlite.conf.json new file mode 100644 index 000000000..cd5690e4e --- /dev/null +++ b/conf/orchestrator-sample-sqlite.conf.json @@ -0,0 +1,120 @@ +{ + "Debug": true, + "EnableSyslog": false, + "ListenAddress": ":3000", + "MySQLTopologyUser": "orc_client_user", + "MySQLTopologyPassword": "orc_client_password", + "MySQLTopologyCredentialsConfigFile": "", + "MySQLTopologySSLPrivateKeyFile": "", + "MySQLTopologySSLCertFile": "", + "MySQLTopologySSLCAFile": "", + "MySQLTopologySSLSkipVerify": true, + "MySQLTopologyUseMutualTLS": false, + "BackendDB": "sqlite", + "SQLite3DataFile": "/usr/local/orchestrator/orchestrator.sqlite3", + "DefaultInstancePort": 3306, + "DiscoverByShowSlaveHosts": true, + "InstancePollSeconds": 5, + "UnseenInstanceForgetHours": 240, + "SnapshotTopologiesIntervalHours": 0, + "InstanceBulkOperationsWaitTimeoutSeconds": 10, + "HostnameResolveMethod": "default", + "MySQLHostnameResolveMethod": "@@hostname", + "SkipBinlogServerUnresolveCheck": true, + "ExpiryHostnameResolvesMinutes": 60, + "RejectHostnameResolvePattern": "", + "ReasonableReplicationLagSeconds": 10, + "ProblemIgnoreHostnameFilters": [], + "VerifyReplicationFilters": false, + "ReasonableMaintenanceReplicationLagSeconds": 20, + "CandidateInstanceExpireMinutes": 60, + "AuditLogFile": "", + "AuditToSyslog": false, + "RemoveTextFromHostnameDisplay": ".mydomain.com:3306", + "ReadOnly": false, + "AuthenticationMethod": "", + "HTTPAuthUser": "", + "HTTPAuthPassword": "", + "AuthUserHeader": "", + "PowerAuthUsers": [ + "*" + ], + "ClusterNameToAlias": { + "127.0.0.1": "test suite" + }, + "SlaveLagQuery": "", + "DetectClusterAliasQuery": "SELECT SUBSTRING_INDEX(@@hostname, '.', 1)", + "DetectClusterDomainQuery": "", + "DetectInstanceAliasQuery": "", + "DetectPromotionRuleQuery": "", + "DataCenterPattern": "[.]([^.]+)[.][^.]+[.]mydomain[.]com", + "PhysicalEnvironmentPattern": "[.]([^.]+[.][^.]+)[.]mydomain[.]com", + "PromotionIgnoreHostnameFilters": [], + "DetectSemiSyncEnforcedQuery": "", + "ServeAgentsHttp": false, + "AgentsServerPort": ":3001", + "AgentsUseSSL": false, + "AgentsUseMutualTLS": false, + "AgentSSLSkipVerify": false, + "AgentSSLPrivateKeyFile": "", + "AgentSSLCertFile": "", + "AgentSSLCAFile": "", + "AgentSSLValidOUs": [], + "UseSSL": false, + "UseMutualTLS": false, + "SSLSkipVerify": false, + "SSLPrivateKeyFile": "", + "SSLCertFile": "", + "SSLCAFile": "", + "SSLValidOUs": [], + "URLPrefix": "", + "StatusEndpoint": "/api/status", + "StatusSimpleHealth": true, + "StatusOUVerify": false, + "AgentPollMinutes": 60, + "UnseenAgentForgetHours": 6, + "StaleSeedFailMinutes": 60, + "SeedAcceptableBytesDiff": 8192, + "PseudoGTIDPattern": "", + "PseudoGTIDPatternIsFixedSubstring": false, + "PseudoGTIDMonotonicHint": "asc:", + "DetectPseudoGTIDQuery": "", + "BinlogEventsChunkSize": 10000, + "SkipBinlogEventsContaining": [], + "ReduceReplicationAnalysisCount": true, + "FailureDetectionPeriodBlockMinutes": 60, + "RecoveryPeriodBlockSeconds": 3600, + "RecoveryIgnoreHostnameFilters": [], + "RecoverMasterClusterFilters": [ + "_master_pattern_" + ], + "RecoverIntermediateMasterClusterFilters": [ + "_intermediate_master_pattern_" + ], + "OnFailureDetectionProcesses": [ + "echo 'Detected {failureType} on {failureCluster}. Affected replicas: {countSlaves}' >> /tmp/recovery.log" + ], + "PreFailoverProcesses": [ + "echo 'Will recover from {failureType} on {failureCluster}' >> /tmp/recovery.log" + ], + "PostFailoverProcesses": [ + "echo '(for all types) Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log" + ], + "PostUnsuccessfulFailoverProcesses": [], + "PostMasterFailoverProcesses": [ + "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Promoted: {successorHost}:{successorPort}' >> /tmp/recovery.log" + ], + "PostIntermediateMasterFailoverProcesses": [ + "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log" + ], + "CoMasterRecoveryMustPromoteOtherCoMaster": true, + "DetachLostSlavesAfterMasterFailover": true, + "ApplyMySQLPromotionAfterMasterFailover": false, + "MasterFailoverDetachSlaveMasterHost": false, + "MasterFailoverLostInstancesDowntimeMinutes": 0, + "PostponeSlaveRecoveryOnLagMinutes": 0, + "OSCIgnoreHostnameFilters": [], + "GraphiteAddr": "", + "GraphitePath": "", + "GraphiteConvertHostnameDotsToUnderscores": true +} diff --git a/go/agent/agent_dao.go b/go/agent/agent_dao.go index c5b4d423d..0eac89b35 100644 --- a/go/agent/agent_dao.go +++ b/go/agent/agent_dao.go @@ -164,7 +164,7 @@ func ReadOutdatedAgentsHosts() ([]string, error) { from host_agent where - IFNULL(last_checked < now() - interval ? minute, true) + IFNULL(last_checked < now() - interval ? minute, 1) ` err := db.QueryOrchestrator(query, sqlutils.Args(config.Config.AgentPollMinutes), func(m sqlutils.RowMap) error { hostname := m.GetString("hostname") diff --git a/go/db/generate_base.go b/go/db/generate_base.go index 5e0a22c90..892d49763 100644 --- a/go/db/generate_base.go +++ b/go/db/generate_base.go @@ -138,7 +138,7 @@ var generateSQLBase = []string{ last_checked timestamp NULL DEFAULT NULL, last_seen timestamp NULL DEFAULT NULL, mysql_port smallint(5) unsigned DEFAULT NULL, - count_mysql_snapshots smallint(5) unsigned NOT NULL, + count_mysql_snapshots smallint(5) unsigned NOT NULL DEFAULT 0, PRIMARY KEY (hostname) ) ENGINE=InnoDB DEFAULT CHARSET=ascii `, diff --git a/vagrant/admin-build.sh b/vagrant/admin-build.sh index ed70bae37..18d4bbb08 100755 --- a/vagrant/admin-build.sh +++ b/vagrant/admin-build.sh @@ -1,18 +1,28 @@ #!/bin/bash - +set -xeuo pipefail # Install orchestrator -rpm -i /tmp/orchestrator-release/orchestrator*.rpm +if [[ -e /etc/redhat-release ]]; then + rpm -i /tmp/orchestrator-release/orchestrator*.rpm +fi + +if [[ -e /etc/debian_version ]]; then + dpkg -i /tmp/orchestrator-release/orchestrator*.deb +fi + +if [[ -e /orchestrator/vagrant/.sqlite ]]; then + cp -fv /usr/local/orchestrator/orchestrator-sample-sqlite.conf.json /etc/orchestrator.conf.json +else + cp -fv /usr/local/orchestrator/orchestrator-sample.conf.json /etc/orchestrator.conf.json +fi if [[ -e /etc/redhat-release ]]; then /sbin/chkconfig orchestrator on - cp /usr/local/orchestrator/orchestrator-sample.conf.json /etc/orchestrator.conf.json /sbin/service orchestrator start elif [[ -e /etc/debian_version ]]; then update-rc.d orchestrator defaults - cp /usr/local/orchestrator/orchestrator-sample.conf.json /etc/orchestrator.conf.json /usr/sbin/service orchestrator start fi @@ -20,4 +30,5 @@ fi echo '* * * * * root /usr/bin/orchestrator -c discover -i db1' > /etc/cron.d/orchestrator-discovery # Discover instances -/usr/bin/orchestrator -c discover -i localhost +/usr/bin/orchestrator --verbose --debug --stack -c redeploy-internal-db +/usr/bin/orchestrator --verbose --debug --stack -c discover -i localhost diff --git a/vagrant/base-build.sh b/vagrant/base-build.sh index 134535707..8e2b77c7a 100755 --- a/vagrant/base-build.sh +++ b/vagrant/base-build.sh @@ -1,8 +1,8 @@ #!/bin/bash - +set -xeuo pipefail if [[ -e /etc/redhat-release ]]; then # Percona's Yum Repository - yum -d 0 -y install http://www.percona.com/downloads/percona-release/redhat/0.1-3/percona-release-0.1-3.noarch.rpm epel-release + yum -d 0 -y install http://www.percona.com/downloads/percona-release/redhat/0.1-4/percona-release-0.1-4.noarch.rpm epel-release # All the project dependencies to build plus some utilities # No reason not to install this stuff in all the places :) @@ -12,6 +12,10 @@ if [[ -e /etc/redhat-release ]]; then # Pin to 1.4 due to 1.5 no longer working on EL6 gem install fpm --version 1.4 + echo "PATH=$PATH:/usr/local/bin" | sudo tee -a /etc/environment + export PATH="PATH=$PATH:/usr/local/bin" + + # Build orchestrator and orchestrator agent mkdir -p /home/vagrant/go/{bin,pkg,src} /tmp/orchestrator-release mkdir -p /home/vagrant/go/src/github.com/github/orchestrator @@ -26,7 +30,10 @@ if [[ -e /etc/redhat-release ]]; then chown -R vagrant.vagrant /home/vagrant /tmp/orchestrator-release # Setup mysql + set +e /sbin/chkconfig mysql on + /sbin/chkconfig mysqld on + set -e if [[ -e "/orchestrator/vagrant/${HOSTNAME}-my.cnf" ]]; then rm -f /etc/my.cnf @@ -41,13 +48,13 @@ elif [[ -e /etc/debian_version ]]; then # Percona's Apt Repository - sudo apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A 9334A25F8507EFA5 echo "deb http://repo.percona.com/apt "$(lsb_release -sc)" main" | sudo tee /etc/apt/sources.list.d/percona.list sudo apt-get -y update sudo apt-get -y install debconf-utils echo "golang-go golang-go/dashboard boolean true" | sudo debconf-set-selections - echo "percona-server-server-5.6 percona-server-server/root_password password vagrant" | sudo debconf-set-selections - echo "percona-server-server-5.6 percona-server-server/root_password_again password vagrant" | sudo debconf-set-selections + echo percona-server-server-5.6 percona-server-server/root_password password "" | sudo debconf-set-selections + echo percona-server-server-5.6 percona-server-server/root_password_again password "" | sudo debconf-set-selections export DEBIAN_FRONTEND=noninteractive # No reason not to install this stuff in all the places :) @@ -56,24 +63,24 @@ elif [[ -e /etc/debian_version ]]; then # add the mysql community packages # from https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/ - sudo apt-key adv --keyserver pgp.mit.edu --recv-keys 5072E1F5 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 5072E1F5 8C718D3B5072E1F5 export CODENAME=$(/usr/bin/lsb_release -c | cut -f2) echo "deb http://repo.mysql.com/apt/ubuntu/ ${CODENAME} mysql-5.7" | sudo tee /etc/apt/sources.list.d/mysql.list apt-get -y update - echo "mysql-community-server mysql-community-server/root-pass password vagrant" | sudo debconf-set-selections - echo "mysql-community-server mysql-community-server/re-root-pass password vagrant" | sudo debconf-set-selections + echo mysql-community-server mysql-community-server/root-pass password "" | sudo debconf-set-selections + echo mysql-community-server mysql-community-server/re-root-pass password "" | sudo debconf-set-selections apt-get -y --force-yes install mysql-server chmod a+w /var/log # All the project dependencies to build - sudo apt-get -y install ruby-dev gcc git rubygems rpm jq + sudo apt-get -y install ruby-dev gcc git rubygems rpm jq make # Jump though some hoops to get a non-decrepit version of golang - sudo apt-get remove golang-go + sudo apt-get remove -y golang-go cd /tmp - wget --quiet "https://redirector.gvt1.com/edgedl/go/go1.9.2.linux-amd64.tar.gz" - sudo tar -C /usr/local -xzf go1.9.2.linux-amd64.tar.gz - echo "PATH=$PATH:/usr/local/go/bin" | sudo tee -a /etc/environment - export PATH="PATH=$PATH:/usr/local/go/bin" + wget -c --quiet "https://redirector.gvt1.com/edgedl/go/go1.9.4.linux-amd64.tar.gz" + sudo tar -C /usr/local -xzf go1.9.4.linux-amd64.tar.gz + echo "PATH=$PATH:/usr/local/go/bin:/usr/local/bin" | sudo tee -a /etc/environment + export PATH="PATH=$PATH:/usr/local/go/bin:/usr/local/bin" # newest versions of java aren't compatable with the installed version of ruby (1.8.7) gem install json --version 1.8.6 @@ -100,7 +107,7 @@ elif [[ -e /etc/debian_version ]]; then /usr/sbin/service mysql start fi -mysql -uroot -pvagrant -e "grant all on *.* to 'root'@'localhost' identified by ''" +sudo mysql -e "grant all on *.* to 'root'@'localhost' identified by ''" cat <<-EOF | mysql -u root CREATE DATABASE IF NOT EXISTS orchestrator; GRANT ALL PRIVILEGES ON orchestrator.* TO 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_password'; @@ -119,7 +126,9 @@ cat <<-EOF >> /etc/hosts EOF if [[ -e /etc/redhat-release ]]; then + set +e sudo service iptables stop + set -e fi if [[ $HOSTNAME == 'admin' ]]; then From 8bb825bf0740dd5bd56d3d07b43ef33b31b5b797 Mon Sep 17 00:00:00 2001 From: Slach Date: Sat, 17 Mar 2018 20:51:09 +0500 Subject: [PATCH 02/11] add sourceHost parameter into agent /api/post-copy/ --- go/agent/agent_dao.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/agent/agent_dao.go b/go/agent/agent_dao.go index 0eac89b35..e72355871 100644 --- a/go/agent/agent_dao.go +++ b/go/agent/agent_dao.go @@ -461,7 +461,7 @@ func MountLV(hostname string, lv string) (Agent, error) { return executeAgentCommand(hostname, fmt.Sprintf("mountlv?lv=%s", lv), nil) } -// RemoveLV requests an agent to remvoe a snapshot +// RemoveLV requests an agent to remove a snapshot func RemoveLV(hostname string, lv string) (Agent, error) { return executeAgentCommand(hostname, fmt.Sprintf("removelv?lv=%s", lv), nil) } @@ -547,8 +547,8 @@ func AbortSeed(seedId int64) error { } // PostCopy will request an agent to invoke post-copy commands -func PostCopy(hostname string) (Agent, error) { - return executeAgentCommand(hostname, "post-copy", nil) +func PostCopy(hostname, sourceHostname string) (Agent, error) { + return executeAgentCommand(hostname, fmt.Sprintf("post-copy/%s", sourceHostname), nil) } // SubmitSeedEntry submits a new seed operation entry, returning its unique ID @@ -774,7 +774,7 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err // Cleanup: seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Executing post-copy command on %s", targetHostname), "") - _, err = PostCopy(targetHostname) + _, err = PostCopy(targetHostname, sourceHostname) if err != nil { return updateSeedStateEntry(seedStateId, err) } From 17f5a32dc032f5a2e2c4624da5677f630e291afe Mon Sep 17 00:00:00 2001 From: Slach Date: Sun, 18 Mar 2018 18:22:48 +0500 Subject: [PATCH 03/11] sourceHostname to logging --- go/agent/agent_dao.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/agent/agent_dao.go b/go/agent/agent_dao.go index e72355871..313db0d7b 100644 --- a/go/agent/agent_dao.go +++ b/go/agent/agent_dao.go @@ -693,7 +693,7 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err } seedFromLogicalVolume := sourceAgent.LogicalVolumes[0] - seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Mounting logical volume: %s", seedFromLogicalVolume.Path), "") + seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s Mounting logical volume: %s", sourceHostname, seedFromLogicalVolume.Path), "") _, err = MountLV(sourceHostname, seedFromLogicalVolume.Path) if err != nil { return updateSeedStateEntry(seedStateId, err) From 9d9dde5cc6bd876d92c545759b8bc283d9b34919 Mon Sep 17 00:00:00 2001 From: Slach Date: Mon, 19 Mar 2018 08:35:14 +0500 Subject: [PATCH 04/11] add include /etc/ files for iject DAEMONOPTS or other settings before daemon running --- etc/init.d/orchestrator.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/init.d/orchestrator.bash b/etc/init.d/orchestrator.bash index ae0961235..ecbde46f8 100644 --- a/etc/init.d/orchestrator.bash +++ b/etc/init.d/orchestrator.bash @@ -54,8 +54,9 @@ start_daemon () { post_start_daemon_hook 1>&2 } -# The file /etc/orchestrator_profile can be used to inject pre-service execution +# This files can be used to inject pre-service execution # scripts, such as exporting variables or whatever. It's yours! +[ -f /etc/default/orchestrator ] && . /etc/default/orchestrator [ -f /etc/orchestrator_profile ] && . /etc/orchestrator_profile case "$1" in From 2d1b5d9276516aa05eb99de35e78331bbb643edc Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 10:04:50 +0500 Subject: [PATCH 05/11] more clarified log message --- go/agent/agent_dao.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/agent/agent_dao.go b/go/agent/agent_dao.go index 313db0d7b..8ac66d9da 100644 --- a/go/agent/agent_dao.go +++ b/go/agent/agent_dao.go @@ -779,7 +779,7 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err return updateSeedStateEntry(seedStateId, err) } - seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Unmounting logical volume: %s", seedFromLogicalVolume.Path), "") + seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s Unmounting logical volume: %s", sourceHostname, seedFromLogicalVolume.Path), "") _, err = Unmount(sourceHostname) if err != nil { return updateSeedStateEntry(seedStateId, err) From c96b2fb413d5247920a20d54c493421133618e0b Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 15:09:58 +0500 Subject: [PATCH 06/11] add SeedWaitSecondsBeforeSend config parameter --- docs/configuration-sample.md | 1 + go/agent/agent_dao.go | 4 ++-- go/config/config.go | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/configuration-sample.md b/docs/configuration-sample.md index 5d2e73f79..af8fe9e25 100644 --- a/docs/configuration-sample.md +++ b/docs/configuration-sample.md @@ -93,6 +93,7 @@ The following is a production configuration file, with some details redacted. "HttpTimeoutSeconds": 60, "StaleSeedFailMinutes": 60, "SeedAcceptableBytesDiff": 8192, + "SeedWaitSecondsBeforeSend": 2, "PseudoGTIDPattern": "drop view if exists `meta`.`_pseudo_gtid_hint__asc:", "PseudoGTIDPatternIsFixedSubstring": true, "PseudoGTIDMonotonicHint": "asc:", diff --git a/go/agent/agent_dao.go b/go/agent/agent_dao.go index 8ac66d9da..09cfd9495 100644 --- a/go/agent/agent_dao.go +++ b/go/agent/agent_dao.go @@ -722,8 +722,8 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s will now receive data in background", targetHostname), "") ReceiveMySQLSeedData(targetHostname, seedId) - seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Waiting some time for %s to start listening for incoming data", targetHostname), "") - time.Sleep(2 * time.Second) + seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Waiting %d seconds for %s to start listening for incoming data", config.Config.SeedWaitSecondsBeforeSend, targetHostname), "") + time.Sleep(time.Duration(config.Config.SeedWaitSecondsBeforeSend) * time.Second) seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s will now send data to %s in background", sourceHostname, targetHostname), "") SendMySQLSeedData(sourceHostname, targetHostname, seedId) diff --git a/go/config/config.go b/go/config/config.go index 46e174f00..d3003bb4c 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -204,6 +204,7 @@ type Configuration struct { UnseenAgentForgetHours uint // Number of hours after which an unseen agent is forgotten StaleSeedFailMinutes uint // Number of minutes after which a stale (no progress) seed is considered failed. SeedAcceptableBytesDiff int64 // Difference in bytes between seed source & target data size that is still considered as successful copy + SeedWaitSecondsBeforeSend int64 // Number of seconds for waiting before start send data command on agent AutoPseudoGTID bool // Should orchestrator automatically inject Pseudo-GTID entries to the masters PseudoGTIDPattern string // Pattern to look for in binary logs that makes for a unique entry (pseudo GTID). When empty, Pseudo-GTID based refactoring is disabled. PseudoGTIDPatternIsFixedSubstring bool // If true, then PseudoGTIDPattern is not treated as regular expression but as fixed substring, and can boost search time @@ -363,6 +364,7 @@ func newConfiguration() *Configuration { UnseenAgentForgetHours: 6, StaleSeedFailMinutes: 60, SeedAcceptableBytesDiff: 8192, + SeedWaitSecondsBeforeSend: 2, AutoPseudoGTID: false, PseudoGTIDPattern: "", PseudoGTIDPatternIsFixedSubstring: false, From 3c4a64fd4f1c1580add68d76442d56c33c7a80b3 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 19:44:52 +0500 Subject: [PATCH 07/11] backwards compatibility for api/post-copy agent command --- go/agent/agent_dao.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/agent/agent_dao.go b/go/agent/agent_dao.go index 09cfd9495..2da0bef76 100644 --- a/go/agent/agent_dao.go +++ b/go/agent/agent_dao.go @@ -548,7 +548,7 @@ func AbortSeed(seedId int64) error { // PostCopy will request an agent to invoke post-copy commands func PostCopy(hostname, sourceHostname string) (Agent, error) { - return executeAgentCommand(hostname, fmt.Sprintf("post-copy/%s", sourceHostname), nil) + return executeAgentCommand(hostname, fmt.Sprintf("post-copy/?sourceHost=%s", sourceHostname), nil) } // SubmitSeedEntry submits a new seed operation entry, returning its unique ID From fe728b37cc3d67a2db3c818cb23728cd3973dad7 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 21 Mar 2018 19:54:22 +0500 Subject: [PATCH 08/11] add /etc/profile.d/orchestrator usage --- etc/init.d/orchestrator.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/init.d/orchestrator.bash b/etc/init.d/orchestrator.bash index ecbde46f8..d7c86d512 100644 --- a/etc/init.d/orchestrator.bash +++ b/etc/init.d/orchestrator.bash @@ -58,6 +58,7 @@ start_daemon () { # scripts, such as exporting variables or whatever. It's yours! [ -f /etc/default/orchestrator ] && . /etc/default/orchestrator [ -f /etc/orchestrator_profile ] && . /etc/orchestrator_profile +[ -f /etc/profile.d/orchestrator ] && . /etc/profile.d/orchestrator case "$1" in start) From 989017bc22e52ad17105def5073e3a2d3ef8a5fc Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 29 Mar 2018 15:17:16 +0500 Subject: [PATCH 09/11] return rsync shared folder back rollback generate_base.go after https://github.com/github/orchestrator/pull/445#discussion_r177999552 --- Vagrantfile | 3 +-- build.sh | 2 +- go/db/generate_base.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 106fee857..ba34be324 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -16,8 +16,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = BOX config.vm.box_download_insecure = true config.vm.box_check_update = false - config.vm.synced_folder '.', '/orchestrator' - #, type: 'rsync', rsync__auto: true + config.vm.synced_folder '.', '/orchestrator', type: 'rsync', rsync__auto: true (0..4).each do |n| name = (n > 0 ? ("db" + n.to_s) : "admin") diff --git a/build.sh b/build.sh index 2de0c4190..9e7cfb8f9 100755 --- a/build.sh +++ b/build.sh @@ -134,7 +134,7 @@ function package() { esac echo "---" - if [[ -e /etc/centos-release ]]; then + if [[ -f /etc/centos-release ]]; then if cat /etc/centos-release | grep 'CentOS release 6' ; then rm ${TOPDIR:-?}/orchestrator*.deb rm ${TOPDIR:-?}/orchestrator*.tar.gz diff --git a/go/db/generate_base.go b/go/db/generate_base.go index 892d49763..5e0a22c90 100644 --- a/go/db/generate_base.go +++ b/go/db/generate_base.go @@ -138,7 +138,7 @@ var generateSQLBase = []string{ last_checked timestamp NULL DEFAULT NULL, last_seen timestamp NULL DEFAULT NULL, mysql_port smallint(5) unsigned DEFAULT NULL, - count_mysql_snapshots smallint(5) unsigned NOT NULL DEFAULT 0, + count_mysql_snapshots smallint(5) unsigned NOT NULL, PRIMARY KEY (hostname) ) ENGINE=InnoDB DEFAULT CHARSET=ascii `, From 9fbf9f780bbe2842dbde3316ade3af95ce98156b Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 29 Mar 2018 15:45:50 +0500 Subject: [PATCH 10/11] fix agent --- go/agent/agent_dao.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/agent/agent_dao.go b/go/agent/agent_dao.go index 2da0bef76..f1e2a87d8 100644 --- a/go/agent/agent_dao.go +++ b/go/agent/agent_dao.go @@ -106,9 +106,9 @@ func SubmitAgent(hostname string, port int, token string) (string, error) { _, err := db.ExecOrchestrator(` replace into host_agent ( - hostname, port, token, last_submitted + hostname, port, token, last_submitted, count_mysql_snapshots ) VALUES ( - ?, ?, ?, NOW() + ?, ?, ?, NOW(), 0 ) `, hostname, From 0c9a7d0b69a008480994fcf9eefbd9008cfab1d6 Mon Sep 17 00:00:00 2001 From: Slach Date: Sun, 8 Apr 2018 16:30:38 +0500 Subject: [PATCH 11/11] remove redeploy-internal-db --- vagrant/admin-build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/vagrant/admin-build.sh b/vagrant/admin-build.sh index 18d4bbb08..44b360976 100755 --- a/vagrant/admin-build.sh +++ b/vagrant/admin-build.sh @@ -30,5 +30,4 @@ fi echo '* * * * * root /usr/bin/orchestrator -c discover -i db1' > /etc/cron.d/orchestrator-discovery # Discover instances -/usr/bin/orchestrator --verbose --debug --stack -c redeploy-internal-db /usr/bin/orchestrator --verbose --debug --stack -c discover -i localhost