diff --git a/autocomplete b/autocomplete index 88d66c5..3a5fc77 100644 --- a/autocomplete +++ b/autocomplete @@ -88,6 +88,12 @@ function _cumulus { COMPREPLY=( $(compgen -W "$configs" -- $cur) ) fi ;; + rds) + if [ "$prev" == "diff" ] || [ "$prev" == "sync" ]; then + configs=$(${cumulus_command} rds list) + COMPREPLY=( $(compgen -W "$configs" -- $cur) ) + fi + ;; ec2) COMPREPLY=( $(compgen -W "diff list migrate sync" -- $cur) ) ;; @@ -126,6 +132,9 @@ function _cumulus { sqs) COMPREPLY=( $(compgen -W "diff list migrate sync urls" -- $cur) ) ;; + rds) + COMPREPLY=( $(compgen -W "diff list migrate sync" -- $cur) ) + ;; ec2) COMPREPLY=( $(compgen -W "ebs instances" -- $cur) ) ;; diff --git a/bin/cumulus b/bin/cumulus index 9f8a069..1b7d0ce 100755 --- a/bin/cumulus +++ b/bin/cumulus @@ -19,6 +19,7 @@ def help_message "\telb\t\t- Manages configuration for elastic load balancers", "\tiam\t\t- Compiles IAM roles and policies that are defined with configuration files and syncs the resulting IAM roles and policies with AWS", "\tkinesis\t\t- Manages configuration for Kinesis streams", + "\trds\t\t- Manages configuration for the Relational Database Service", "\troute53\t\t- Manages configuration for Route53", "\ts3\t\t- Manages configuration of S3 buckets", "\tsecurity-groups\t- Manages configuration for EC2 Security Groups", @@ -64,10 +65,7 @@ OptionParser.new do |opts| end end.parse! -if ARGV.size == 0 or (ARGV[0] != "iam" and ARGV[0] != "help" and ARGV[0] != "--help" and ARGV[0] != "autoscaling" and - ARGV[0] != "route53" and ARGV[0] != "s3" and ARGV[0] != "security-groups" and - ARGV[0] != "cloudfront" and ARGV[0] != "elb" and ARGV[0] != "vpc" and ARGV[0] != "kinesis" and - ARGV[0] != "sqs" and ARGV[0] != "ec2") +unless ARGV.size > 0 and ["iam", "help", "--help", "autoscaling", "route53", "s3", "security-groups", "cloudfront", "elb", "vpc", "kinesis", "sqs", "ec3", "rds"].include?(ARGV[0]) puts usage_message exit @@ -116,6 +114,9 @@ elsif ARGV[0] == "elb" elsif ARGV[0] == "kinesis" require "kinesis/Commands" Cumulus::Kinesis::Commands.parse(ARGV[1..-1]) +elsif ARGV[0] == "rds" + require "rds/Commands" + Cumulus::RDS::Commands.parse(ARGV[1..-1]) elsif ARGV[0] == "route53" require "route53/Commands" Cumulus::Route53::Commands.parse(ARGV[1..-1]) diff --git a/lib/conf/Configuration.rb b/lib/conf/Configuration.rb index fd14b39..fadbba4 100644 --- a/lib/conf/Configuration.rb +++ b/lib/conf/Configuration.rb @@ -85,7 +85,7 @@ class Configuration include Config attr_reader :colors_enabled - attr_reader :iam, :autoscaling, :route53, :s3, :security, :cloudfront, :elb, :vpc, :kinesis, :sqs, :ec2 + attr_reader :iam, :autoscaling, :route53, :s3, :security, :cloudfront, :elb, :vpc, :kinesis, :sqs, :ec2, :rds attr_reader :client # Internal: Constructor. Sets up the `instance` variable, which is the access @@ -113,6 +113,7 @@ def initialize(conf_dir, profile, assume_role, autoscaling_force_size, json = ni @kinesis = KinesisConfig.new @sqs = SQSConfig.new @ec2 = EC2Config.new + @rds = RDSConfig.new region = conf "region" credentials = if assume_role @@ -378,5 +379,16 @@ def initialize end end + # Public: Inner class that contains EC2 configuration options + class RDSConfig + include Config + + attr_reader :instances_directory + + def initialize + @instances_directory = absolute_path "rds/instances" + end + end + end end diff --git a/lib/rds/Commands.rb b/lib/rds/Commands.rb new file mode 100644 index 0000000..0e3eb00 --- /dev/null +++ b/lib/rds/Commands.rb @@ -0,0 +1,20 @@ +module Cumulus + module RDS + require "common/Commands" + class Commands < Cumulus::Common::Commands + + def self.banner_message + format_message [ + "rds: Manage Relational Database Service.", + "\tCompiles rds resources that are defined in configuration files and syncs the resulting RDS assets with AWS.", + ] + end + + def self.manager + require "rds/manager/Manager" + Cumulus::RDS::Manager.new + end + + end + end +end diff --git a/lib/rds/RDS.rb b/lib/rds/RDS.rb new file mode 100644 index 0000000..db6d646 --- /dev/null +++ b/lib/rds/RDS.rb @@ -0,0 +1,29 @@ +require "conf/Configuration" +require "aws-sdk" + +module Cumulus + module RDS + class << self + @@client = Aws::RDS::Client.new(Configuration.instance.client) + + def client + @@client + end + + def instances + @instances ||= init_instances + end + + def named_instances + Hash[instances.map { |instance| [instance[:db_instance_identifier], instance] }] + end + + private + + def init_instances + @@client.describe_db_instances.db_instances + end + + end + end +end diff --git a/lib/rds/loader/Loader.rb b/lib/rds/loader/Loader.rb new file mode 100644 index 0000000..093d25f --- /dev/null +++ b/lib/rds/loader/Loader.rb @@ -0,0 +1,20 @@ +require "rds/models/InstanceConfig" +require "common/BaseLoader" +require "conf/Configuration" + +module Cumulus + module RDS + # public load RDS assets + module Loader + include Common::BaseLoader + + @@instance_dir = Configuration.instance.rds.instances_directory + @@instance_loader = Proc.new { |name, json| InstanceConfig.new(name, json) } + + def self.instances + Common::BaseLoader.resources(@@instance_dir, &@@instance_loader) + end + + end + end +end diff --git a/lib/rds/manager/Manager.rb b/lib/rds/manager/Manager.rb new file mode 100644 index 0000000..b42f3a2 --- /dev/null +++ b/lib/rds/manager/Manager.rb @@ -0,0 +1,192 @@ +require "rds/RDS" +require "rds/loader/Loader" +require "common/manager/Manager" +require "rds/models/InstanceDiff" +require "conf/Configuration" +require "io/console" + +module Cumulus + module RDS + class Manager < Common::Manager + def resource_name + "RDS Database Instance" + end + + def local_resources + @local_resources ||= Hash[Loader.instances.map { |local| [local.name, local] }] + end + + def aws_resources + @aws_resources ||= RDS::named_instances + end + + def unmanaged_diff(aws) + InstanceDiff.unmanaged(aws) + end + + def added_diff(local) + InstanceDiff.added(local) + end + + def diff_resource(local, aws) + puts Colors.blue("Processing #{local.name}...") + cumulus_version = InstanceConfig.new(local.name).populate!(aws) + local.diff(cumulus_version) + end + + def migrate + puts Colors.blue("Will migrate #{RDS.instances.length} instances") + + # Create the directories + rds_dir = "#{@migration_root}/rds" + instances_dir = "#{rds_dir}/instances" + + if !Dir.exists?(@migration_root) + Dir.mkdir(@migration_root) + end + if !Dir.exists?(rds_dir) + Dir.mkdir(rds_dir) + end + if !Dir.exists?(instances_dir) + Dir.mkdir(instances_dir) + end + + RDS.named_instances.each do |name, instance| + puts "Migrating #{name}..." + + cumulus_instance = InstanceConfig.new(name).populate!(instance) + + json = JSON.pretty_generate(cumulus_instance.to_hash) + File.open("#{instances_dir}/#{name}.json", "w") { |f| f.write(json) } + end + end + + def update(local, diffs) + aws_instance = RDS::named_instances[local.name] + + all_changes = diffs.map do |diff| + case diff.type + when InstanceChange::ENGINE, + InstanceChange::USERNAME, + InstanceChange::SUBNET, + InstanceChange::DATABASE, + InstanceChange::STORAGE_TYPE + puts Colors.red("Cannot change #{diff.asset_type}") + # no change for this diff, ignore it + nil + when InstanceChange::PORT + puts "Updating the database port number..." + {db_port_number: local.port} + when InstanceChange::TYPE + puts "Updating the database instance type..." + {db_instance_class: "db." + local.type} + when InstanceChange::ENGINE_VERSION + puts "Updating the engine version..." + {engine_version: local.engine_version} + when InstanceChange::STORAGE_SIZE + puts "Updating the allocated storage..." + {allocated_storage: local.storage_size} + when InstanceChange::SECURITY_GROUPS + puts "Updating the security groups..." + security_group_ids = local.security_groups.map do |sg| + # TODO: once the db subnet config is created, find the security group id based on the vpc in that subnet. + # right now, if security groups from different vpc have the same name, this might not return the right id. + sg_id = SecurityGroups.vpc_security_group_id_names.values.inject(:merge).key(sg) + if sg_id.nil? + raise Exception.new("security group #{sg} does not exist") + end + sg_id + end + {vpc_security_group_ids: security_group_ids} + when InstanceChange::PUBLIC + if local.public_facing + puts "making the database public..." + else + puts "blocking the database from the public..." + end + {publicly_accessible: local.public_facing} + when InstanceChange::BACKUP + puts "Updating the backup preferences..." + {preferred_backup_window: local.backup_window, backup_retention_period: local.backup_period} + when InstanceChange::UPGRADE + puts "Updating the upgrade preferences..." + {preferred_maintenance_window: local.upgrade_window, auto_minor_version_upgrade: local.auto_upgrade} + end + end.reject {|v| v.nil?}.reduce(&:merge) + + # make all the updates in the same call + RDS::client.modify_db_instance({db_instance_identifier: local.name, apply_immediately: true}.merge(all_changes)) unless all_changes.nil? + end + + def create(local) + errors = Array.new + + if local.name.nil? + errors << "instance name is required" + end + + if local.type.nil? + errors << "instance type is required" + end + + if local.engine.nil? + errors << "database engine is required" + end + + unless errors.empty? + puts Colors.red("Could not create #{local.name}:") + errors.each { |e| puts Colors.red("\t#{e}")} + exit StatusCodes::EXCEPTION + end + + master_password = unless local.master_username.nil? + password_prompt(local.master_username) + else + nil + end + + security_group_ids = local.security_groups.map do |sg| + # TODO: once the db subnet config is created, find the security group id based on the vpc in that subnet. + # right now, if security groups from different vpc have the same name, this might not return the right id. + sg_id = SecurityGroups.vpc_security_group_id_names.values.inject(:merge).key(sg) + if sg_id.nil? + raise Exception.new("security group #{sg} does not exist") + end + sg_id + end + + RDS::client.create_db_instance({ + db_name: local.database, + db_instance_identifier: local.name, # required + allocated_storage: local.storage_size, + db_instance_class: "db." + local.type, # required + engine: local.engine, # required + master_username: local.master_username, + master_user_password: master_password, + vpc_security_group_ids: security_group_ids, + db_subnet_group_name: local.subnet, + preferred_maintenance_window: local.upgrade_window, + backup_retention_period: local.backup_period, + preferred_backup_window: local.backup_window, + port: local.port, + engine_version: local.engine_version, + auto_minor_version_upgrade: local.auto_upgrade, + publicly_accessible: local.public_facing, + storage_type: local.storage_type, + }.reject { |k, v| v.nil? }) + + end + + private + + def password_prompt(username) + # prompt for the user's password (discreetly) + print "enter a password for #{username}: " + password = STDIN.noecho(&:gets).chomp + puts "\n" + password + end + + end + end +end diff --git a/lib/rds/models/InstanceConfig.rb b/lib/rds/models/InstanceConfig.rb new file mode 100644 index 0000000..fd783b8 --- /dev/null +++ b/lib/rds/models/InstanceConfig.rb @@ -0,0 +1,140 @@ +require "common/models/ListChange" +require "security/SecurityGroups" +require "rds/models/InstanceDiff" + +module Cumulus + module RDS + class InstanceConfig + attr_reader :name, :port, :type, :engine, :engine_version, :storage_type, :storage_size, :master_username, :security_groups, :subnet, :database, :public_facing, :backup_period, :backup_window, :auto_upgrade, :upgrade_window + + def initialize(name, json = nil) + @name = name + unless json.nil? + @port = json["port"] + @type = json["type"] + @engine = json["engine"] + @engine_version = json["engine_version"] + @storage_type = json["storage_type"] + @storage_size = json["storage_size"] + @master_username = json["master_username"] + @security_groups = json["security-groups"] + @subnet = json["subnet"] + @database = json["database"] + @public_facing = json["public"] + @backup_period = json["backup_period"] + @backup_window = json["backup_window"] + @auto_upgrade = json["auto_upgrade"] + @upgrade_window = json["upgrade_window"].downcase + end + end + + def to_hash + { + "port" => @port, + "type" => @type, + "engine" => @engine, + "engine_version" => @engine_version, + "storage_type" => @storage_type, + "storage_size" => @storage_size, + "master_username" => @master_username, + "security-groups" => @security_groups, + "subnet" => @subnet, + "database" => @database, + "public" => @public_facing, + "backup_period" => @backup_period, + "backup_window" => @backup_window, + "auto_upgrade" => @auto_upgrade, + "upgrade_window" => @upgrade_window, + } + end + + def populate!(aws_instance) + raise Exception.new("the rds instance (#{@name}) is still booting up.") if aws_instance[:db_instance_status] == "creating" + @port = aws_instance[:endpoint][:port] + @type = aws_instance[:db_instance_class].reverse.chomp("db.".reverse).reverse # remove the 'db.' that prefixes the string + @engine = aws_instance[:engine] + @engine_version = aws_instance[:engine_version] + @storage_type = aws_instance[:storage_type] + @storage_size = aws_instance[:allocated_storage] + @master_username = aws_instance[:master_username] + @security_groups = aws_instance[:vpc_security_groups].map(&:vpc_security_group_id).map { |id| SecurityGroups::id_security_groups[id].group_name }.sort + @subnet = aws_instance[:db_subnet_group][:db_subnet_group_name] + @database = aws_instance[:db_name] + @public_facing = aws_instance[:publicly_accessible] + @backup_period = aws_instance[:backup_retention_period] + @backup_window = aws_instance[:preferred_backup_window] + @auto_upgrade = aws_instance[:auto_minor_version_upgrade] + @upgrade_window = aws_instance[:preferred_maintenance_window] + self # return the instanceconfig + end + + def diff(aws) + diffs = Array.new + + if aws.port != @port + diffs << InstanceDiff.new(InstanceChange::PORT, aws.port, @port) + end + + if aws.type != @type + diffs << InstanceDiff.new(InstanceChange::TYPE, aws.type, @type) + end + + if aws.engine != @engine + diffs << InstanceDiff.new(InstanceChange::ENGINE, aws.engine, @engine) + end + + if aws.engine_version != @engine_version + diffs << InstanceDiff.new(InstanceChange::ENGINE_VERSION, aws.engine_version, @engine_version) + end + + if aws.storage_type != @storage_type + diffs << InstanceDiff.new(InstanceChange::STORAGE_TYPE, aws.storage_type, @storage_type) + end + + if aws.storage_size != @storage_size + diffs << InstanceDiff.new(InstanceChange::STORAGE_SIZE, aws.storage_size, @storage_size) + end + + if aws.master_username != @master_username + diffs << InstanceDiff.new(InstanceChange::USERNAME, aws.master_username, @master_username) + end + + if aws.security_groups != @security_groups + changes = Common::ListChange::simple_list_diff(aws.security_groups, @security_groups) + diffs << InstanceDiff.new(InstanceChange::SECURITY_GROUPS, aws.security_groups, @security_groups, changes) + end + + if aws.subnet != @subnet + diffs << InstanceDiff.new(InstanceChange::SUBNET, aws.subnet, @subnet) + end + + if aws.database != @database + diffs << InstanceDiff.new(InstanceChange::DATABASE, aws.database, @database) + end + + if aws.public_facing != @public_facing + diffs << InstanceDiff.new(InstanceChange::PUBLIC, aws.public_facing, @public_facing) + end + + if aws.backup_period != @backup_period + diffs << InstanceDiff.new(InstanceChange::BACKUP, aws.backup_period, @backup_period) + end + + if aws.backup_window != @backup_window + diffs << InstanceDiff.new(InstanceChange::BACKUP, aws.backup_window, @backup_window) + end + + if aws.auto_upgrade != @auto_upgrade + diffs << InstanceDiff.new(InstanceChange::UPGRADE, aws.auto_upgrade, @auto_upgrade) + end + + if aws.upgrade_window != @upgrade_window + diffs << InstanceDiff.new(InstanceChange::UPGRADE, aws.upgrade_window, @upgrade_window) + end + + diffs + end + + end + end +end diff --git a/lib/rds/models/InstanceDiff.rb b/lib/rds/models/InstanceDiff.rb new file mode 100644 index 0000000..87918b8 --- /dev/null +++ b/lib/rds/models/InstanceDiff.rb @@ -0,0 +1,69 @@ +require "common/models/Diff" +require "common/models/TagsDiff" + +module Cumulus + module RDS + module InstanceChange + include Common::DiffChange + + PORT = Common::DiffChange.next_change_id + TYPE = Common::DiffChange.next_change_id + ENGINE = Common::DiffChange.next_change_id + ENGINE_VERSION = Common::DiffChange.next_change_id + STORAGE_SIZE = Common::DiffChange.next_change_id + STORAGE_TYPE = Common::DiffChange.next_change_id + USERNAME = Common::DiffChange.next_change_id + SECURITY_GROUPS = Common::DiffChange.next_change_id + SUBNET = Common::DiffChange.next_change_id + DATABASE = Common::DiffChange.next_change_id + PUBLIC = Common::DiffChange.next_change_id + BACKUP = Common::DiffChange.next_change_id + UPGRADE = Common::DiffChange.next_change_id + end + + # Public: Represents a single difference between local configuration and AWS configuration + class InstanceDiff < Common::Diff + include InstanceChange + + def aws_name + @aws[:db_instance_identifier] + end + + def asset_type + case @type + when PORT then "Port" + when TYPE then "Type" + when ENGINE then "Engine" + when ENGINE_VERSION then "Engine Version" + when STORAGE_SIZE then "Storage Size" + when STORAGE_TYPE then "Storage Type" + when USERNAME then "Username" + when SECURITY_GROUPS then "Security Groups" + when SUBNET then "Subnet" + when DATABASE then "Database Name" + when PUBLIC then "Public Facing" + when BACKUP then "Backup" + when UPGRADE then "Upgrade" + else + "RDS Instance" + end + end + + def diff_string + unless @type == SECURITY_GROUPS + [ + "#{asset_type}:", + Colors.aws_changes("\tAWS - #{aws}"), + Colors.local_changes("\tLocal - #{local}"), + ].join("\n") + else + [ + "#{asset_type}:", + @changes.removed.map { |sg| Colors.unmanaged("\t#{sg}") }, + @changes.added.map { |sg| Colors.added("\t#{sg}") } + ].flatten.join("\n") + end + end + end + end +end diff --git a/rakefile.rb b/rakefile.rb index 04126db..9060cc7 100644 --- a/rakefile.rb +++ b/rakefile.rb @@ -5,9 +5,15 @@ task :spec do Rake::Task['spec_sqs'].execute + Rake::Task['spec_rds'].execute end RSpec::Core::RakeTask.new(:spec_sqs) do |task| task.rspec_opts = ['-r ./spec/rspec_config.rb'] task.pattern = 'spec/sqs/*_spec.rb' end + +RSpec::Core::RakeTask.new(:spec_rds) do |task| + task.rspec_opts = ['-r ./spec/rspec_config.rb'] + task.pattern = 'spec/rds/*_spec.rb' +end diff --git a/spec/rds/RDSUtil.rb b/spec/rds/RDSUtil.rb new file mode 100644 index 0000000..6b4af08 --- /dev/null +++ b/spec/rds/RDSUtil.rb @@ -0,0 +1,210 @@ +require "conf/Configuration" +require "mocks/MockedConfiguration" +Cumulus::Configuration.send :include, Cumulus::Test::MockedConfiguration + +require "common/BaseLoader" +require "mocks/MockedLoader" +Cumulus::Common::BaseLoader.send :include, Cumulus::Test::MockedLoader + +require "common/manager/Manager" +require "util/ManagerUtil" +Cumulus::Common::Manager.send :include, Cumulus::Test::ManagerUtil + +require "util/StatusCodes" +require "mocks/MockedStatusCodes" +Cumulus::StatusCodes.send :include, Cumulus::Test::MockedStatusCodes + +require "aws-sdk" +require "json" +require "rds/manager/Manager" +require "rds/RDS" +require "util/DeepMerge" +require "mocks/ClientSpy" + +module Cumulus + module Test + # Monkey patch Cumulus::RDS such that the cached values from the AWS client + # can be reset between tests. + module ResetRDS + def self.included(base) + base.instance_eval do + def reset_instances + @instances = nil + end + + def client=(client) + @@client = client + end + end + end + end + + # Monkey patch Cumulus::RDS::Manager so that a user doesn't have to enter a password for new users + DEFAULT_PASSWORD = "passwordforrdstest" + Cumulus::RDS::Manager.send(:define_method, :password_prompt) do |*args| + DEFAULT_PASSWORD + end + + module RDS + @instances_directory = "/mocked/rds/instances" + @default_instance_name = "test-instance" + + DEFAULT_PORT = 3306 + DEFAULT_TYPE = "t2.micro" + SECONDARY_TYPE = "m4.large" + DEFAULT_ENGINE = "mysql" + SECONDARY_ENGINE = "aurora" + DEFAULT_ENGINE_VERSION = "5.6.27" + SECONDARY_ENGINE_VERSION = "6.0.0" + DEFAULT_STORAGE_TYPE = "gp2" + SECONDARY_STORAGE_TYPE = "standard" + DEFAULT_STORAGE_SIZE = 5 + DEFAULT_USERNAME = "testing" + SECONDARY_USERNAME = "diffme" + DEFAULT_SUBNET_GROUP = "default" + SECONDARY_SUBNET_GROUP = "secondary" + DEFAULT_DATABASE_NAME = "testdb" + SECONDARY_DATABASE_NAME = "seconddb" + DEFAULT_PUBLIC_FACING = false + DEFAULT_BACKUP_PERIOD = 7 + DEFAULT_BACKUP_WINDOW = "02:30-03:00" + SECONDARY_BACKUP_WINDOW = "01:00-02:30" + DEFAULT_AUTO_UPGRADE = true + DEFAULT_UPGRADE_WINDOW = "mon:03:27-mon:03:57" + SECONDARY_UPGRADE_WINDOW = "tue:03:30-tue:04:00" + + + @default_instance_attributes = { + "port" => DEFAULT_PORT, + "type" => DEFAULT_TYPE, + "engine" => DEFAULT_ENGINE, + "engine_version" => DEFAULT_ENGINE_VERSION, + "storage_type" => DEFAULT_STORAGE_TYPE, + "storage_size" => DEFAULT_STORAGE_SIZE, + "master_username" => DEFAULT_USERNAME, + "security-groups" => [ + + ], + "subnet" => DEFAULT_SUBNET_GROUP, + "database" => DEFAULT_DATABASE_NAME, + "public" => DEFAULT_PUBLIC_FACING, + "backup_period" => DEFAULT_BACKUP_PERIOD, + "backup_window" => DEFAULT_BACKUP_WINDOW, + "auto_upgrade" => DEFAULT_AUTO_UPGRADE, + "upgrade_window" => DEFAULT_UPGRADE_WINDOW, + } + + @default_aws_instance_attributes = { + endpoint: { + port: DEFAULT_PORT, + }, + db_instance_class: "db." + DEFAULT_TYPE, + engine: DEFAULT_ENGINE, + engine_version: DEFAULT_ENGINE_VERSION, + storage_type: DEFAULT_STORAGE_TYPE, + allocated_storage: DEFAULT_STORAGE_SIZE, + master_username: DEFAULT_USERNAME, + vpc_security_groups: [ + + ], + db_subnet_group: { + db_subnet_group_name: DEFAULT_SUBNET_GROUP + }, + db_name: DEFAULT_DATABASE_NAME, + publicly_accessible: DEFAULT_PUBLIC_FACING, + backup_retention_period: DEFAULT_BACKUP_PERIOD, + preferred_backup_window: DEFAULT_BACKUP_WINDOW, + auto_minor_version_upgrade: DEFAULT_AUTO_UPGRADE, + preferred_maintenance_window: DEFAULT_UPGRADE_WINDOW, + } + + # Public: Returns a Hash containing default instance attributes for a local + # instance definition with values overridden by the Hash passed in. + # + # overrides - optionally provide a Hash that will override default + # attributes + def self.default_instance_attributes(overrides = nil) + Util::DeepMerge.deep_merge(@default_instance_attributes, overrides) + end + + def self.do_diff(config, &test) + self.prepare_test(config) + + # get the diffs and call the tester to determine the result of the test + manager = Cumulus::RDS::Manager.new + diffs = manager.diff_strings + test.call(diffs) + end + + # Public: Sync stubbed local configuration and stubbed AWS configuration. + # + # config - a Hash that contains two values, :local and :aws, which contain + # the values to stub out. + # :local contains :queues which is an Array of queues to stub the + # directory with, and :policies, which is an Array of policy files + # to stub out. + # :aws is a hash of method names from the AWS Client to stub mapped + # to the value the AWS Client should return + # test - a block that tests the AWS Client after the syncing has been done + def self.do_sync(config, &test) + self.prepare_test(config) + + # get the diffs and call the tester to determine the result of the test + manager = Cumulus::RDS::Manager.new + manager.sync + test.call(Cumulus::RDS::client) + end + + # Public: Returns a mocked rds instance object similar to what the aws client would return + def self.aws_instance(name, overrides = nil) + overrides = if overrides.nil? + {db_instance_identifier: name} + else + {db_instance_identifier: name}.merge(overrides) + end + Util::DeepMerge.deep_merge(@default_aws_instance_attributes, overrides) + end + + private + + def self.prepare_test(config) + self.reset + + # stub out local queues + if config[:local][:instances] + Cumulus::Common::BaseLoader.stub_directory( + @instances_directory, config[:local][:instances] + ) + end + + # stub out aws responses + config[:aws].map do |call, value| + if value + Cumulus::RDS::client.stub_responses(call, value) + else + Cumulus::RDS::client.stub_responses(call) + end + end + end + + # Public: Reset the SQS module in between tests + def self.reset + Cumulus::Configuration.stub + + # reset Cumulus::RDS to forget cached aws resources + if !Cumulus::RDS.respond_to? :reset_instances + Cumulus::RDS.send :include, Cumulus::Test::ResetRDS + end + + Cumulus::RDS::reset_instances + end + + def self.client_spy + unless Cumulus::RDS::client.respond_to? :have_received + Cumulus::RDS::client = ClientSpy.new(Cumulus::RDS::client) + end + Cumulus::RDS::client.clear_spy + end + end + end +end diff --git a/spec/rds/SingleChangeTest.rb b/spec/rds/SingleChangeTest.rb new file mode 100644 index 0000000..6effdbc --- /dev/null +++ b/spec/rds/SingleChangeTest.rb @@ -0,0 +1,47 @@ +module Cumulus + module Test + module RDS + class SingleChangeTest + + DEFAULT_AWS_INSTANCE_NAME = "cumulus-test-instance" + attr_reader :diffs, :client + + def initialize(args) + @local_overrides = args[:local] + @aws_overrides = args[:aws] + end + + def self.execute_diff(args = Hash.new) + test = SingleChangeTest.new(args) + test.execute_diff + yield test.diffs + end + + def execute_diff + RDS::do_diff({ + local: {instances: [{name: DEFAULT_AWS_INSTANCE_NAME, value: RDS::default_instance_attributes(@local_overrides)}]}, + aws: {describe_db_instances: {db_instances: [RDS::aws_instance(DEFAULT_AWS_INSTANCE_NAME, @aws_overrides)]}}, + }) do |diffs| + @diffs = diffs + end + end + + def self.execute_sync(args = Hash.new) + test = SingleChangeTest.new(args) + test.execute_sync + yield test.client + end + + def execute_sync + RDS::client_spy + RDS::do_sync({ + local: {instances: [{name: DEFAULT_AWS_INSTANCE_NAME, value: RDS::default_instance_attributes(@local_overrides)}]}, + aws: {describe_db_instances: {db_instances: [RDS::aws_instance(DEFAULT_AWS_INSTANCE_NAME, @aws_overrides)]}}, + }) do |client| + @client = client + end + end + end + end + end +end diff --git a/spec/rds/diff_spec.rb b/spec/rds/diff_spec.rb new file mode 100644 index 0000000..2306639 --- /dev/null +++ b/spec/rds/diff_spec.rb @@ -0,0 +1,217 @@ +require "rds/RDSUtil" +require "rds/SingleChangeTest" + +module Cumulus + module Test + module RDS + describe Cumulus::RDS::Manager do + context "The RDS module's diffing functionality" do + it "should detect new instances defined locally" do + instance_name = "not-in-aws" + RDS::do_diff({ + local: {instances: [{name: instance_name, value: RDS::default_instance_attributes}]}, + aws: {describe_db_instances: {db_instances: []}}, + }) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq "RDS Instance #{instance_name} will be created." + end + end + + it "should detect new instances added in AWS" do + instance_name = "not-local" + RDS::do_diff({ + local: {instances: []}, + aws: {describe_db_instances: {db_instances: [RDS::aws_instance(instance_name)]}}, + }) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq "RDS Instance #{instance_name} is not managed by Cumulus." + end + end + + it "should detect changes made to the port" do + SingleChangeTest.execute_diff( + local: {"port" => DEFAULT_PORT - 1}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Port:", + "\tAWS - #{DEFAULT_PORT}", + "\tLocal - #{DEFAULT_PORT - 1}", + ].join("\n") + end + end + + it "should detect changes made to the instance type" do + SingleChangeTest.execute_diff( + local: {"type" => SECONDARY_TYPE}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Type:", + "\tAWS - #{DEFAULT_TYPE}", + "\tLocal - #{SECONDARY_TYPE}", + ].join("\n") + end + end + + it "should detect changes made to the engine" do + SingleChangeTest.execute_diff( + local: {"engine" => SECONDARY_ENGINE}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Engine:", + "\tAWS - #{DEFAULT_ENGINE}", + "\tLocal - #{SECONDARY_ENGINE}", + ].join("\n") + end + end + + it "should detect changes made to the engine version" do + SingleChangeTest.execute_diff( + local: {"engine_version" => SECONDARY_ENGINE_VERSION}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Engine Version:", + "\tAWS - #{DEFAULT_ENGINE_VERSION}", + "\tLocal - #{SECONDARY_ENGINE_VERSION}", + ].join("\n") + end + end + + it "should detect changes made to the storage type" do + SingleChangeTest.execute_diff( + local: {"storage_type" => SECONDARY_STORAGE_TYPE}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Storage Type:", + "\tAWS - #{DEFAULT_STORAGE_TYPE}", + "\tLocal - #{SECONDARY_STORAGE_TYPE}", + ].join("\n") + end + end + + it "should detect changes made to the storage size" do + SingleChangeTest.execute_diff( + local: {"storage_size" => DEFAULT_STORAGE_SIZE - 1}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Storage Size:", + "\tAWS - #{DEFAULT_STORAGE_SIZE}", + "\tLocal - #{DEFAULT_STORAGE_SIZE - 1}", + ].join("\n") + end + end + + it "should detect changes made to the username" do + SingleChangeTest.execute_diff( + local: {"master_username" => SECONDARY_USERNAME}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Username:", + "\tAWS - #{DEFAULT_USERNAME}", + "\tLocal - #{SECONDARY_USERNAME}", + ].join("\n") + end + end + + it "should detect changes made to the db subnet group" do + SingleChangeTest.execute_diff( + local: {"subnet" => SECONDARY_SUBNET_GROUP}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Subnet:", + "\tAWS - #{DEFAULT_SUBNET_GROUP}", + "\tLocal - #{SECONDARY_SUBNET_GROUP}", + ].join("\n") + end + end + + it "should detect changes made to the database name" do + SingleChangeTest.execute_diff( + local: {"database" => SECONDARY_DATABASE_NAME}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Database Name:", + "\tAWS - #{DEFAULT_DATABASE_NAME}", + "\tLocal - #{SECONDARY_DATABASE_NAME}", + ].join("\n") + end + end + + it "should detect changes made to public access" do + SingleChangeTest.execute_diff( + local: {"public" => !DEFAULT_PUBLIC_FACING}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Public Facing:", + "\tAWS - #{DEFAULT_PUBLIC_FACING}", + "\tLocal - #{!DEFAULT_PUBLIC_FACING}", + ].join("\n") + end + end + + it "should detect changes made to the backup period" do + SingleChangeTest.execute_diff( + local: {"backup_period" => DEFAULT_BACKUP_PERIOD - 1}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Backup:", + "\tAWS - #{DEFAULT_BACKUP_PERIOD}", + "\tLocal - #{DEFAULT_BACKUP_PERIOD - 1}", + ].join("\n") + end + end + + it "should detect changes made to the backup window" do + SingleChangeTest.execute_diff( + local: {"backup_window" => SECONDARY_BACKUP_WINDOW}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Backup:", + "\tAWS - #{DEFAULT_BACKUP_WINDOW}", + "\tLocal - #{SECONDARY_BACKUP_WINDOW}", + ].join("\n") + end + end + + it "should detect changes made to automatic upgrades" do + SingleChangeTest.execute_diff( + local: {"auto_upgrade" => !DEFAULT_AUTO_UPGRADE}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Upgrade:", + "\tAWS - #{DEFAULT_AUTO_UPGRADE}", + "\tLocal - #{!DEFAULT_AUTO_UPGRADE}", + ].join("\n") + end + end + + it "should detect changes made to the upgrade window" do + SingleChangeTest.execute_diff( + local: {"upgrade_window" => SECONDARY_UPGRADE_WINDOW}, + ) do |diffs| + expect(diffs.size).to eq 1 + expect(diffs.first.to_s).to eq [ + "Upgrade:", + "\tAWS - #{DEFAULT_UPGRADE_WINDOW}", + "\tLocal - #{SECONDARY_UPGRADE_WINDOW}", + ].join("\n") + end + end + + end + end + end + end +end diff --git a/spec/rds/sync_spec.rb b/spec/rds/sync_spec.rb new file mode 100644 index 0000000..7267e81 --- /dev/null +++ b/spec/rds/sync_spec.rb @@ -0,0 +1,274 @@ +require "rds/RDSUtil" +require "rds/SingleChangeTest" + +module Cumulus + module Test + module RDS + describe Cumulus::RDS::Manager do + context "The SQS module's syncing functionality" do + it "should correctly create a new instance that's defined locally" do + instance_name = "not-in-aws" + RDS::client_spy + RDS::do_sync({ + local: {instances: [{name: instance_name, value: RDS::default_instance_attributes}]}, + aws: {describe_db_instances: {db_instances: []}}, + }) do |client| + create = client.spied_method(:create_db_instance) + expect(create.num_calls).to eq 1 + expect(create.arguments.first).to eq ({ + db_name: DEFAULT_DATABASE_NAME, + db_instance_identifier: instance_name, + allocated_storage: DEFAULT_STORAGE_SIZE, + db_instance_class: "db." + DEFAULT_TYPE, + engine: DEFAULT_ENGINE, + master_username: DEFAULT_USERNAME, + master_user_password: DEFAULT_PASSWORD, + vpc_security_group_ids: Array.new, + db_subnet_group_name: DEFAULT_SUBNET_GROUP, + preferred_maintenance_window: DEFAULT_UPGRADE_WINDOW, + backup_retention_period: DEFAULT_BACKUP_PERIOD, + preferred_backup_window: DEFAULT_BACKUP_WINDOW, + port: DEFAULT_PORT, + engine_version: DEFAULT_ENGINE_VERSION, + auto_minor_version_upgrade: DEFAULT_AUTO_UPGRADE, + publicly_accessible: DEFAULT_PUBLIC_FACING, + storage_type: DEFAULT_STORAGE_TYPE, + }) + end + end + + it "should not delete instances added in AWS" do + instance_name = "not-local" + RDS::client_spy + RDS::do_sync({ + local: {instances: []}, + aws: {describe_db_instances: {db_instances: [RDS::aws_instance(instance_name)]}}, + }) do |client| + # no calls were made to change anything + expect(client.method_calls.size).to eq 2 # one call to :stub responses, and one call to :describe_db_instances (for diff) + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + end + end + + it "should update the port" do + SingleChangeTest.execute_sync( + local: {"port" => DEFAULT_PORT - 1}, + ) do |client| + expect(client.method_calls.size).to eq 3 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + update = client.spied_method(:modify_db_instance) + expect(update.nil?).to eq false + expect(update.num_calls).to eq 1 + expect(update.arguments.first).to eq ({ + db_instance_identifier: "cumulus-test-instance", + apply_immediately: true, + db_port_number: DEFAULT_PORT - 1 + }) + end + end + + it "should update the instance type" do + SingleChangeTest.execute_sync( + local: {"type" => SECONDARY_TYPE}, + ) do |client| + expect(client.method_calls.size).to eq 3 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + update = client.spied_method(:modify_db_instance) + expect(update.nil?).to eq false + expect(update.num_calls).to eq 1 + expect(update.arguments.first).to eq ({ + db_instance_identifier: "cumulus-test-instance", + apply_immediately: true, + db_instance_class: "db." + SECONDARY_TYPE + }) + end + end + + it "should update the engine" do + SingleChangeTest.execute_sync( + local: {"engine" => SECONDARY_ENGINE}, + ) do |client| + expect(client.method_calls.size).to eq 2 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + end + end + + it "should update the engine version" do + SingleChangeTest.execute_sync( + local: {"engine_version" => SECONDARY_ENGINE_VERSION}, + ) do |client| + expect(client.method_calls.size).to eq 3 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + update = client.spied_method(:modify_db_instance) + expect(update.nil?).to eq false + expect(update.num_calls).to eq 1 + expect(update.arguments.first).to eq ({ + db_instance_identifier: "cumulus-test-instance", + apply_immediately: true, + engine_version: SECONDARY_ENGINE_VERSION + }) + end + end + + it "should update the storage type" do + SingleChangeTest.execute_sync( + local: {"storage_type" => SECONDARY_STORAGE_TYPE}, + ) do |client| + expect(client.method_calls.size).to eq 2 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + end + end + + it "should update the storage size" do + SingleChangeTest.execute_sync( + local: {"storage_size" => DEFAULT_STORAGE_SIZE - 1}, + ) do |client| + expect(client.method_calls.size).to eq 3 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + update = client.spied_method(:modify_db_instance) + expect(update.nil?).to eq false + expect(update.num_calls).to eq 1 + expect(update.arguments.first).to eq ({ + db_instance_identifier: "cumulus-test-instance", + apply_immediately: true, + allocated_storage: DEFAULT_STORAGE_SIZE - 1 + }) + end + end + + it "should update the username" do + SingleChangeTest.execute_sync( + local: {"master_username" => SECONDARY_USERNAME}, + ) do |client| + expect(client.method_calls.size).to eq 2 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + end + end + + it "should update the db subnet group" do + SingleChangeTest.execute_sync( + local: {"subnet" => SECONDARY_SUBNET_GROUP}, + ) do |client| + expect(client.method_calls.size).to eq 2 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + end + end + + it "should update the database name" do + SingleChangeTest.execute_sync( + local: {"database" => SECONDARY_DATABASE_NAME}, + ) do |client| + expect(client.method_calls.size).to eq 2 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + end + end + + it "should update public access" do + SingleChangeTest.execute_sync( + local: {"public" => !DEFAULT_PUBLIC_FACING}, + ) do |client| + expect(client.method_calls.size).to eq 3 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + update = client.spied_method(:modify_db_instance) + expect(update.nil?).to eq false + expect(update.num_calls).to eq 1 + expect(update.arguments.first).to eq ({ + db_instance_identifier: "cumulus-test-instance", + apply_immediately: true, + publicly_accessible: !DEFAULT_PUBLIC_FACING + }) + end + end + + it "should update the backup period" do + SingleChangeTest.execute_sync( + local: {"backup_period" => DEFAULT_BACKUP_PERIOD - 1}, + ) do |client| + expect(client.method_calls.size).to eq 3 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + update = client.spied_method(:modify_db_instance) + expect(update.nil?).to eq false + expect(update.num_calls).to eq 1 + expect(update.arguments.first).to eq ({ + db_instance_identifier: "cumulus-test-instance", + apply_immediately: true, + preferred_backup_window: DEFAULT_BACKUP_WINDOW, + backup_retention_period: DEFAULT_BACKUP_PERIOD - 1 + }) + end + end + + it "should update the backup window" do + SingleChangeTest.execute_sync( + local: {"backup_window" => SECONDARY_BACKUP_WINDOW}, + ) do |client| + expect(client.method_calls.size).to eq 3 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + update = client.spied_method(:modify_db_instance) + expect(update.nil?).to eq false + expect(update.num_calls).to eq 1 + expect(update.arguments.first).to eq ({ + db_instance_identifier: "cumulus-test-instance", + apply_immediately: true, + preferred_backup_window: SECONDARY_BACKUP_WINDOW, + backup_retention_period: DEFAULT_BACKUP_PERIOD + }) + end + end + + it "should update automatic upgrades" do + SingleChangeTest.execute_sync( + local: {"auto_upgrade" => !DEFAULT_AUTO_UPGRADE}, + ) do |client| + expect(client.method_calls.size).to eq 3 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + update = client.spied_method(:modify_db_instance) + expect(update.nil?).to eq false + expect(update.num_calls).to eq 1 + expect(update.arguments.first).to eq ({ + db_instance_identifier: "cumulus-test-instance", + apply_immediately: true, + preferred_maintenance_window: DEFAULT_UPGRADE_WINDOW, + auto_minor_version_upgrade: !DEFAULT_AUTO_UPGRADE + }) + end + end + + it "should update the upgrade window" do + SingleChangeTest.execute_sync( + local: {"upgrade_window" => SECONDARY_UPGRADE_WINDOW}, + ) do |client| + expect(client.method_calls.size).to eq 3 + expect(client.spied_method(:stub_responses).nil?).to eq false + expect(client.spied_method(:describe_db_instances).nil?).to eq false + update = client.spied_method(:modify_db_instance) + expect(update.nil?).to eq false + expect(update.num_calls).to eq 1 + expect(update.arguments.first).to eq ({ + db_instance_identifier: "cumulus-test-instance", + apply_immediately: true, + preferred_maintenance_window: SECONDARY_UPGRADE_WINDOW, + auto_minor_version_upgrade: DEFAULT_AUTO_UPGRADE + }) + end + end + + end + end + end + end +end diff --git a/spec/util/ManagerUtil.rb b/spec/util/ManagerUtil.rb index 7c50979..56e1fc8 100644 --- a/spec/util/ManagerUtil.rb +++ b/spec/util/ManagerUtil.rb @@ -7,6 +7,10 @@ def diff_strings each_difference(local_resources, true) { |key, diffs| @diff_strings.concat diffs } @diff_strings end + + # Public - override 'puts' method to prevent annoying output during tests. + def puts(msg) + end end end end