diff --git a/.gitignore b/.gitignore index 0e4cea53a..c4f3b1a36 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ conf/orchestrator.conf.json .DS_Store .vagrant/ +vagrant/.sqlite vagrant/admin-post-install.sh vagrant/db-post-install.sh vagrant/db1-post-install.sh @@ -13,3 +14,7 @@ vagrant/vagrant-ssh-key.pub Godeps/_workspace .gopath/ main +.idea/ +*.deb +*.pcap +*.log diff --git a/Vagrantfile b/Vagrantfile index bd981b4c0..ba34be324 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -16,8 +16,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = BOX config.vm.box_download_insecure = true config.vm.box_check_update = false - config.vm.synced_folder '.', '/orchestrator', type: 'rsync', - rsync__auto: true + config.vm.synced_folder '.', '/orchestrator', type: 'rsync', rsync__auto: true (0..4).each do |n| name = (n > 0 ? ("db" + n.to_s) : "admin") diff --git a/build.sh b/build.sh index a560f3a59..9e7cfb8f9 100755 --- a/build.sh +++ b/build.sh @@ -87,7 +87,7 @@ function oinstall() { cd $mydir gofmt -s -w go/ rsync -qa ./resources $builddir/orchestrator${prefix}/orchestrator/ - rsync -qa ./conf/orchestrator-sample.* $builddir/orchestrator${prefix}/orchestrator/ + rsync -qa ./conf/orchestrator-sample*.conf.json $builddir/orchestrator${prefix}/orchestrator/ cp etc/init.d/orchestrator.bash $builddir/orchestrator/etc/init.d/orchestrator chmod +x $builddir/orchestrator/etc/init.d/orchestrator } @@ -134,12 +134,14 @@ function package() { esac echo "---" - if cat /etc/centos-release | grep 'CentOS release 6' ; then - rm ${TOPDIR:-?}/orchestrator*.deb - rm ${TOPDIR:-?}/orchestrator*.tar.gz - # n CentOD 6 box: we only want the rpms for CentOS6 - # Add "-centos6" to the file name. - ls ${TOPDIR:-?}/*.rpm | while read f; do centos_file=$(echo $f | sed -r -e "s/^(.*)-${RELEASE_VERSION}(.*)/\1-centos6-${RELEASE_VERSION}\2/g") ; mv $f $centos_file ; done + if [[ -f /etc/centos-release ]]; then + if cat /etc/centos-release | grep 'CentOS release 6' ; then + rm ${TOPDIR:-?}/orchestrator*.deb + rm ${TOPDIR:-?}/orchestrator*.tar.gz + # n CentOD 6 box: we only want the rpms for CentOS6 + # Add "-centos6" to the file name. + ls ${TOPDIR:-?}/*.rpm | while read f; do centos_file=$(echo $f | sed -r -e "s/^(.*)-${RELEASE_VERSION}(.*)/\1-centos6-${RELEASE_VERSION}\2/g") ; mv $f $centos_file ; done + fi fi echo "Done. Find releases in $TOPDIR" } diff --git a/conf/orchestrator-sample-sqlite.conf.json b/conf/orchestrator-sample-sqlite.conf.json new file mode 100644 index 000000000..cd5690e4e --- /dev/null +++ b/conf/orchestrator-sample-sqlite.conf.json @@ -0,0 +1,120 @@ +{ + "Debug": true, + "EnableSyslog": false, + "ListenAddress": ":3000", + "MySQLTopologyUser": "orc_client_user", + "MySQLTopologyPassword": "orc_client_password", + "MySQLTopologyCredentialsConfigFile": "", + "MySQLTopologySSLPrivateKeyFile": "", + "MySQLTopologySSLCertFile": "", + "MySQLTopologySSLCAFile": "", + "MySQLTopologySSLSkipVerify": true, + "MySQLTopologyUseMutualTLS": false, + "BackendDB": "sqlite", + "SQLite3DataFile": "/usr/local/orchestrator/orchestrator.sqlite3", + "DefaultInstancePort": 3306, + "DiscoverByShowSlaveHosts": true, + "InstancePollSeconds": 5, + "UnseenInstanceForgetHours": 240, + "SnapshotTopologiesIntervalHours": 0, + "InstanceBulkOperationsWaitTimeoutSeconds": 10, + "HostnameResolveMethod": "default", + "MySQLHostnameResolveMethod": "@@hostname", + "SkipBinlogServerUnresolveCheck": true, + "ExpiryHostnameResolvesMinutes": 60, + "RejectHostnameResolvePattern": "", + "ReasonableReplicationLagSeconds": 10, + "ProblemIgnoreHostnameFilters": [], + "VerifyReplicationFilters": false, + "ReasonableMaintenanceReplicationLagSeconds": 20, + "CandidateInstanceExpireMinutes": 60, + "AuditLogFile": "", + "AuditToSyslog": false, + "RemoveTextFromHostnameDisplay": ".mydomain.com:3306", + "ReadOnly": false, + "AuthenticationMethod": "", + "HTTPAuthUser": "", + "HTTPAuthPassword": "", + "AuthUserHeader": "", + "PowerAuthUsers": [ + "*" + ], + "ClusterNameToAlias": { + "127.0.0.1": "test suite" + }, + "SlaveLagQuery": "", + "DetectClusterAliasQuery": "SELECT SUBSTRING_INDEX(@@hostname, '.', 1)", + "DetectClusterDomainQuery": "", + "DetectInstanceAliasQuery": "", + "DetectPromotionRuleQuery": "", + "DataCenterPattern": "[.]([^.]+)[.][^.]+[.]mydomain[.]com", + "PhysicalEnvironmentPattern": "[.]([^.]+[.][^.]+)[.]mydomain[.]com", + "PromotionIgnoreHostnameFilters": [], + "DetectSemiSyncEnforcedQuery": "", + "ServeAgentsHttp": false, + "AgentsServerPort": ":3001", + "AgentsUseSSL": false, + "AgentsUseMutualTLS": false, + "AgentSSLSkipVerify": false, + "AgentSSLPrivateKeyFile": "", + "AgentSSLCertFile": "", + "AgentSSLCAFile": "", + "AgentSSLValidOUs": [], + "UseSSL": false, + "UseMutualTLS": false, + "SSLSkipVerify": false, + "SSLPrivateKeyFile": "", + "SSLCertFile": "", + "SSLCAFile": "", + "SSLValidOUs": [], + "URLPrefix": "", + "StatusEndpoint": "/api/status", + "StatusSimpleHealth": true, + "StatusOUVerify": false, + "AgentPollMinutes": 60, + "UnseenAgentForgetHours": 6, + "StaleSeedFailMinutes": 60, + "SeedAcceptableBytesDiff": 8192, + "PseudoGTIDPattern": "", + "PseudoGTIDPatternIsFixedSubstring": false, + "PseudoGTIDMonotonicHint": "asc:", + "DetectPseudoGTIDQuery": "", + "BinlogEventsChunkSize": 10000, + "SkipBinlogEventsContaining": [], + "ReduceReplicationAnalysisCount": true, + "FailureDetectionPeriodBlockMinutes": 60, + "RecoveryPeriodBlockSeconds": 3600, + "RecoveryIgnoreHostnameFilters": [], + "RecoverMasterClusterFilters": [ + "_master_pattern_" + ], + "RecoverIntermediateMasterClusterFilters": [ + "_intermediate_master_pattern_" + ], + "OnFailureDetectionProcesses": [ + "echo 'Detected {failureType} on {failureCluster}. Affected replicas: {countSlaves}' >> /tmp/recovery.log" + ], + "PreFailoverProcesses": [ + "echo 'Will recover from {failureType} on {failureCluster}' >> /tmp/recovery.log" + ], + "PostFailoverProcesses": [ + "echo '(for all types) Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log" + ], + "PostUnsuccessfulFailoverProcesses": [], + "PostMasterFailoverProcesses": [ + "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Promoted: {successorHost}:{successorPort}' >> /tmp/recovery.log" + ], + "PostIntermediateMasterFailoverProcesses": [ + "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log" + ], + "CoMasterRecoveryMustPromoteOtherCoMaster": true, + "DetachLostSlavesAfterMasterFailover": true, + "ApplyMySQLPromotionAfterMasterFailover": false, + "MasterFailoverDetachSlaveMasterHost": false, + "MasterFailoverLostInstancesDowntimeMinutes": 0, + "PostponeSlaveRecoveryOnLagMinutes": 0, + "OSCIgnoreHostnameFilters": [], + "GraphiteAddr": "", + "GraphitePath": "", + "GraphiteConvertHostnameDotsToUnderscores": true +} diff --git a/docs/configuration-sample.md b/docs/configuration-sample.md index c29186d32..c8e3feebc 100644 --- a/docs/configuration-sample.md +++ b/docs/configuration-sample.md @@ -93,6 +93,7 @@ The following is a production configuration file, with some details redacted. "HttpTimeoutSeconds": 60, "StaleSeedFailMinutes": 60, "SeedAcceptableBytesDiff": 8192, + "SeedWaitSecondsBeforeSend": 2, "PseudoGTIDPattern": "drop view if exists `meta`.`_pseudo_gtid_hint__asc:", "PseudoGTIDPatternIsFixedSubstring": true, "PseudoGTIDMonotonicHint": "asc:", diff --git a/etc/init.d/orchestrator.bash b/etc/init.d/orchestrator.bash index ae0961235..d7c86d512 100644 --- a/etc/init.d/orchestrator.bash +++ b/etc/init.d/orchestrator.bash @@ -54,9 +54,11 @@ start_daemon () { post_start_daemon_hook 1>&2 } -# The file /etc/orchestrator_profile can be used to inject pre-service execution +# This files can be used to inject pre-service execution # scripts, such as exporting variables or whatever. It's yours! +[ -f /etc/default/orchestrator ] && . /etc/default/orchestrator [ -f /etc/orchestrator_profile ] && . /etc/orchestrator_profile +[ -f /etc/profile.d/orchestrator ] && . /etc/profile.d/orchestrator case "$1" in start) diff --git a/go/agent/agent_dao.go b/go/agent/agent_dao.go index c5b4d423d..f1e2a87d8 100644 --- a/go/agent/agent_dao.go +++ b/go/agent/agent_dao.go @@ -106,9 +106,9 @@ func SubmitAgent(hostname string, port int, token string) (string, error) { _, err := db.ExecOrchestrator(` replace into host_agent ( - hostname, port, token, last_submitted + hostname, port, token, last_submitted, count_mysql_snapshots ) VALUES ( - ?, ?, ?, NOW() + ?, ?, ?, NOW(), 0 ) `, hostname, @@ -164,7 +164,7 @@ func ReadOutdatedAgentsHosts() ([]string, error) { from host_agent where - IFNULL(last_checked < now() - interval ? minute, true) + IFNULL(last_checked < now() - interval ? minute, 1) ` err := db.QueryOrchestrator(query, sqlutils.Args(config.Config.AgentPollMinutes), func(m sqlutils.RowMap) error { hostname := m.GetString("hostname") @@ -461,7 +461,7 @@ func MountLV(hostname string, lv string) (Agent, error) { return executeAgentCommand(hostname, fmt.Sprintf("mountlv?lv=%s", lv), nil) } -// RemoveLV requests an agent to remvoe a snapshot +// RemoveLV requests an agent to remove a snapshot func RemoveLV(hostname string, lv string) (Agent, error) { return executeAgentCommand(hostname, fmt.Sprintf("removelv?lv=%s", lv), nil) } @@ -547,8 +547,8 @@ func AbortSeed(seedId int64) error { } // PostCopy will request an agent to invoke post-copy commands -func PostCopy(hostname string) (Agent, error) { - return executeAgentCommand(hostname, "post-copy", nil) +func PostCopy(hostname, sourceHostname string) (Agent, error) { + return executeAgentCommand(hostname, fmt.Sprintf("post-copy/?sourceHost=%s", sourceHostname), nil) } // SubmitSeedEntry submits a new seed operation entry, returning its unique ID @@ -693,7 +693,7 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err } seedFromLogicalVolume := sourceAgent.LogicalVolumes[0] - seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Mounting logical volume: %s", seedFromLogicalVolume.Path), "") + seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s Mounting logical volume: %s", sourceHostname, seedFromLogicalVolume.Path), "") _, err = MountLV(sourceHostname, seedFromLogicalVolume.Path) if err != nil { return updateSeedStateEntry(seedStateId, err) @@ -722,8 +722,8 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s will now receive data in background", targetHostname), "") ReceiveMySQLSeedData(targetHostname, seedId) - seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Waiting some time for %s to start listening for incoming data", targetHostname), "") - time.Sleep(2 * time.Second) + seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Waiting %d seconds for %s to start listening for incoming data", config.Config.SeedWaitSecondsBeforeSend, targetHostname), "") + time.Sleep(time.Duration(config.Config.SeedWaitSecondsBeforeSend) * time.Second) seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s will now send data to %s in background", sourceHostname, targetHostname), "") SendMySQLSeedData(sourceHostname, targetHostname, seedId) @@ -774,12 +774,12 @@ func executeSeed(seedId int64, targetHostname string, sourceHostname string) err // Cleanup: seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Executing post-copy command on %s", targetHostname), "") - _, err = PostCopy(targetHostname) + _, err = PostCopy(targetHostname, sourceHostname) if err != nil { return updateSeedStateEntry(seedStateId, err) } - seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("Unmounting logical volume: %s", seedFromLogicalVolume.Path), "") + seedStateId, _ = submitSeedStateEntry(seedId, fmt.Sprintf("%s Unmounting logical volume: %s", sourceHostname, seedFromLogicalVolume.Path), "") _, err = Unmount(sourceHostname) if err != nil { return updateSeedStateEntry(seedStateId, err) diff --git a/go/config/config.go b/go/config/config.go index bd2fdc419..224574bc0 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -204,6 +204,7 @@ type Configuration struct { UnseenAgentForgetHours uint // Number of hours after which an unseen agent is forgotten StaleSeedFailMinutes uint // Number of minutes after which a stale (no progress) seed is considered failed. SeedAcceptableBytesDiff int64 // Difference in bytes between seed source & target data size that is still considered as successful copy + SeedWaitSecondsBeforeSend int64 // Number of seconds for waiting before start send data command on agent AutoPseudoGTID bool // Should orchestrator automatically inject Pseudo-GTID entries to the masters PseudoGTIDPattern string // Pattern to look for in binary logs that makes for a unique entry (pseudo GTID). When empty, Pseudo-GTID based refactoring is disabled. PseudoGTIDPatternIsFixedSubstring bool // If true, then PseudoGTIDPattern is not treated as regular expression but as fixed substring, and can boost search time @@ -365,6 +366,7 @@ func newConfiguration() *Configuration { UnseenAgentForgetHours: 6, StaleSeedFailMinutes: 60, SeedAcceptableBytesDiff: 8192, + SeedWaitSecondsBeforeSend: 2, AutoPseudoGTID: false, PseudoGTIDPattern: "", PseudoGTIDPatternIsFixedSubstring: false, diff --git a/vagrant/admin-build.sh b/vagrant/admin-build.sh index ed70bae37..44b360976 100755 --- a/vagrant/admin-build.sh +++ b/vagrant/admin-build.sh @@ -1,18 +1,28 @@ #!/bin/bash - +set -xeuo pipefail # Install orchestrator -rpm -i /tmp/orchestrator-release/orchestrator*.rpm +if [[ -e /etc/redhat-release ]]; then + rpm -i /tmp/orchestrator-release/orchestrator*.rpm +fi + +if [[ -e /etc/debian_version ]]; then + dpkg -i /tmp/orchestrator-release/orchestrator*.deb +fi + +if [[ -e /orchestrator/vagrant/.sqlite ]]; then + cp -fv /usr/local/orchestrator/orchestrator-sample-sqlite.conf.json /etc/orchestrator.conf.json +else + cp -fv /usr/local/orchestrator/orchestrator-sample.conf.json /etc/orchestrator.conf.json +fi if [[ -e /etc/redhat-release ]]; then /sbin/chkconfig orchestrator on - cp /usr/local/orchestrator/orchestrator-sample.conf.json /etc/orchestrator.conf.json /sbin/service orchestrator start elif [[ -e /etc/debian_version ]]; then update-rc.d orchestrator defaults - cp /usr/local/orchestrator/orchestrator-sample.conf.json /etc/orchestrator.conf.json /usr/sbin/service orchestrator start fi @@ -20,4 +30,4 @@ fi echo '* * * * * root /usr/bin/orchestrator -c discover -i db1' > /etc/cron.d/orchestrator-discovery # Discover instances -/usr/bin/orchestrator -c discover -i localhost +/usr/bin/orchestrator --verbose --debug --stack -c discover -i localhost diff --git a/vagrant/base-build.sh b/vagrant/base-build.sh index 134535707..8e2b77c7a 100755 --- a/vagrant/base-build.sh +++ b/vagrant/base-build.sh @@ -1,8 +1,8 @@ #!/bin/bash - +set -xeuo pipefail if [[ -e /etc/redhat-release ]]; then # Percona's Yum Repository - yum -d 0 -y install http://www.percona.com/downloads/percona-release/redhat/0.1-3/percona-release-0.1-3.noarch.rpm epel-release + yum -d 0 -y install http://www.percona.com/downloads/percona-release/redhat/0.1-4/percona-release-0.1-4.noarch.rpm epel-release # All the project dependencies to build plus some utilities # No reason not to install this stuff in all the places :) @@ -12,6 +12,10 @@ if [[ -e /etc/redhat-release ]]; then # Pin to 1.4 due to 1.5 no longer working on EL6 gem install fpm --version 1.4 + echo "PATH=$PATH:/usr/local/bin" | sudo tee -a /etc/environment + export PATH="PATH=$PATH:/usr/local/bin" + + # Build orchestrator and orchestrator agent mkdir -p /home/vagrant/go/{bin,pkg,src} /tmp/orchestrator-release mkdir -p /home/vagrant/go/src/github.com/github/orchestrator @@ -26,7 +30,10 @@ if [[ -e /etc/redhat-release ]]; then chown -R vagrant.vagrant /home/vagrant /tmp/orchestrator-release # Setup mysql + set +e /sbin/chkconfig mysql on + /sbin/chkconfig mysqld on + set -e if [[ -e "/orchestrator/vagrant/${HOSTNAME}-my.cnf" ]]; then rm -f /etc/my.cnf @@ -41,13 +48,13 @@ elif [[ -e /etc/debian_version ]]; then # Percona's Apt Repository - sudo apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A 9334A25F8507EFA5 echo "deb http://repo.percona.com/apt "$(lsb_release -sc)" main" | sudo tee /etc/apt/sources.list.d/percona.list sudo apt-get -y update sudo apt-get -y install debconf-utils echo "golang-go golang-go/dashboard boolean true" | sudo debconf-set-selections - echo "percona-server-server-5.6 percona-server-server/root_password password vagrant" | sudo debconf-set-selections - echo "percona-server-server-5.6 percona-server-server/root_password_again password vagrant" | sudo debconf-set-selections + echo percona-server-server-5.6 percona-server-server/root_password password "" | sudo debconf-set-selections + echo percona-server-server-5.6 percona-server-server/root_password_again password "" | sudo debconf-set-selections export DEBIAN_FRONTEND=noninteractive # No reason not to install this stuff in all the places :) @@ -56,24 +63,24 @@ elif [[ -e /etc/debian_version ]]; then # add the mysql community packages # from https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/ - sudo apt-key adv --keyserver pgp.mit.edu --recv-keys 5072E1F5 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 5072E1F5 8C718D3B5072E1F5 export CODENAME=$(/usr/bin/lsb_release -c | cut -f2) echo "deb http://repo.mysql.com/apt/ubuntu/ ${CODENAME} mysql-5.7" | sudo tee /etc/apt/sources.list.d/mysql.list apt-get -y update - echo "mysql-community-server mysql-community-server/root-pass password vagrant" | sudo debconf-set-selections - echo "mysql-community-server mysql-community-server/re-root-pass password vagrant" | sudo debconf-set-selections + echo mysql-community-server mysql-community-server/root-pass password "" | sudo debconf-set-selections + echo mysql-community-server mysql-community-server/re-root-pass password "" | sudo debconf-set-selections apt-get -y --force-yes install mysql-server chmod a+w /var/log # All the project dependencies to build - sudo apt-get -y install ruby-dev gcc git rubygems rpm jq + sudo apt-get -y install ruby-dev gcc git rubygems rpm jq make # Jump though some hoops to get a non-decrepit version of golang - sudo apt-get remove golang-go + sudo apt-get remove -y golang-go cd /tmp - wget --quiet "https://redirector.gvt1.com/edgedl/go/go1.9.2.linux-amd64.tar.gz" - sudo tar -C /usr/local -xzf go1.9.2.linux-amd64.tar.gz - echo "PATH=$PATH:/usr/local/go/bin" | sudo tee -a /etc/environment - export PATH="PATH=$PATH:/usr/local/go/bin" + wget -c --quiet "https://redirector.gvt1.com/edgedl/go/go1.9.4.linux-amd64.tar.gz" + sudo tar -C /usr/local -xzf go1.9.4.linux-amd64.tar.gz + echo "PATH=$PATH:/usr/local/go/bin:/usr/local/bin" | sudo tee -a /etc/environment + export PATH="PATH=$PATH:/usr/local/go/bin:/usr/local/bin" # newest versions of java aren't compatable with the installed version of ruby (1.8.7) gem install json --version 1.8.6 @@ -100,7 +107,7 @@ elif [[ -e /etc/debian_version ]]; then /usr/sbin/service mysql start fi -mysql -uroot -pvagrant -e "grant all on *.* to 'root'@'localhost' identified by ''" +sudo mysql -e "grant all on *.* to 'root'@'localhost' identified by ''" cat <<-EOF | mysql -u root CREATE DATABASE IF NOT EXISTS orchestrator; GRANT ALL PRIVILEGES ON orchestrator.* TO 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_password'; @@ -119,7 +126,9 @@ cat <<-EOF >> /etc/hosts EOF if [[ -e /etc/redhat-release ]]; then + set +e sudo service iptables stop + set -e fi if [[ $HOSTNAME == 'admin' ]]; then