RDS module, fixes #150

autocomplete

@@ -88,6 +88,12 @@ function _cumulus {
             COMPREPLY=( $(compgen -W "$configs" -- $cur) )
           fi
         ;;
+        rds)
+          if [ "$prev" == "diff" ] || [ "$prev" == "sync" ]; then
+            configs=$(${cumulus_command} rds list)
+            COMPREPLY=( $(compgen -W "$configs" -- $cur) )
+          fi
+        ;;
         ec2)
           COMPREPLY=( $(compgen -W "diff list migrate sync" -- $cur) )
         ;;
@@ -126,6 +132,9 @@ function _cumulus {
             sqs)
               COMPREPLY=( $(compgen -W "diff list migrate sync urls" -- $cur) )
             ;;
+            rds)
+              COMPREPLY=( $(compgen -W "diff list migrate sync" -- $cur) )
+            ;;
             ec2)
               COMPREPLY=( $(compgen -W "ebs instances" -- $cur) )
             ;;

bin/cumulus

@@ -19,6 +19,7 @@ def help_message
     "\telb\t\t- Manages configuration for elastic load balancers",
     "\tiam\t\t- Compiles IAM roles and policies that are defined with configuration files and syncs the resulting IAM roles and policies with AWS",
     "\tkinesis\t\t- Manages configuration for Kinesis streams",
+    "\trds\t\t- Manages configuration for the Relational Database Service",
     "\troute53\t\t- Manages configuration for Route53",
     "\ts3\t\t- Manages configuration of S3 buckets",
     "\tsecurity-groups\t- Manages configuration for EC2 Security Groups",
@@ -64,10 +65,7 @@ OptionParser.new do |opts|
   end
 end.parse!
 
-if ARGV.size == 0 or (ARGV[0] != "iam" and ARGV[0] != "help" and ARGV[0] != "--help" and ARGV[0] != "autoscaling" and
-  ARGV[0] != "route53" and ARGV[0] != "s3" and ARGV[0] != "security-groups" and
-  ARGV[0] != "cloudfront" and ARGV[0] != "elb" and ARGV[0] != "vpc" and ARGV[0] != "kinesis" and
-  ARGV[0] != "sqs" and ARGV[0] != "ec2")
+unless ARGV.size > 0 and ["iam", "help", "--help", "autoscaling", "route53", "s3", "security-groups", "cloudfront", "elb", "vpc", "kinesis", "sqs", "ec3", "rds"].include?(ARGV[0])
 
   puts usage_message
   exit
@@ -116,6 +114,9 @@ elsif ARGV[0] == "elb"
 elsif ARGV[0] == "kinesis"
   require "kinesis/Commands"
   Cumulus::Kinesis::Commands.parse(ARGV[1..-1])
+elsif ARGV[0] == "rds"
+  require "rds/Commands"
+  Cumulus::RDS::Commands.parse(ARGV[1..-1])
 elsif ARGV[0] == "route53"
   require "route53/Commands"
   Cumulus::Route53::Commands.parse(ARGV[1..-1])

lib/conf/Configuration.rb

@@ -85,7 +85,7 @@ class Configuration
     include Config
 
     attr_reader :colors_enabled
-    attr_reader :iam, :autoscaling, :route53, :s3, :security, :cloudfront, :elb, :vpc, :kinesis, :sqs, :ec2
+    attr_reader :iam, :autoscaling, :route53, :s3, :security, :cloudfront, :elb, :vpc, :kinesis, :sqs, :ec2, :rds
     attr_reader :client
 
     # Internal: Constructor. Sets up the `instance` variable, which is the access
@@ -113,6 +113,7 @@ def initialize(conf_dir, profile, assume_role, autoscaling_force_size, json = ni
       @kinesis = KinesisConfig.new
       @sqs = SQSConfig.new
       @ec2 = EC2Config.new
+      @rds = RDSConfig.new
 
       region = conf "region"
       credentials = if assume_role
@@ -378,5 +379,16 @@ def initialize
       end
     end
 
+    # Public: Inner class that contains EC2 configuration options
+    class RDSConfig
+      include Config
+
+      attr_reader :instances_directory
+
+      def initialize
+        @instances_directory = absolute_path "rds/instances"
+      end
+    end
+
   end
 end

lib/rds/Commands.rb

@@ -0,0 +1,20 @@
+module Cumulus
+  module RDS
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.banner_message
+        format_message [
+          "rds: Manage Relational Database Service.",
+          "\tCompiles rds resources that are defined in configuration files and syncs the resulting RDS assets with AWS.",
+        ]
+      end
+
+      def self.manager
+        require "rds/manager/Manager"
+        Cumulus::RDS::Manager.new
+      end
+
+    end
+  end
+end

lib/rds/RDS.rb

@@ -0,0 +1,29 @@
+require "conf/Configuration"
+require "aws-sdk"
+
+module Cumulus
+  module RDS
+    class << self
+      @@client = Aws::RDS::Client.new(Configuration.instance.client)
+
+      def client
+        @@client
+      end
+
+      def instances
+        @instances ||= init_instances
+      end
+
+      def named_instances
+        Hash[instances.map { |instance| [instance[:db_instance_identifier], instance] }]
+      end
+
+      private
+
+      def init_instances
+        @@client.describe_db_instances.db_instances
+      end
+
+    end
+  end
+end

lib/rds/loader/Loader.rb

@@ -0,0 +1,20 @@
+require "rds/models/InstanceConfig"
+require "common/BaseLoader"
+require "conf/Configuration"
+
+module Cumulus
+  module RDS
+    # public load RDS assets
+    module Loader
+      include Common::BaseLoader
+
+      @@instance_dir = Configuration.instance.rds.instances_directory
+      @@instance_loader = Proc.new { |name, json| InstanceConfig.new(name, json) }
+
+      def self.instances
+        Common::BaseLoader.resources(@@instance_dir, &@@instance_loader)
+      end
+
+    end
+  end
+end

lib/rds/manager/Manager.rb

@@ -0,0 +1,192 @@
+require "rds/RDS"
+require "rds/loader/Loader"
+require "common/manager/Manager"
+require "rds/models/InstanceDiff"
+require "conf/Configuration"
+require "io/console"
+
+module Cumulus
+  module RDS
+    class Manager < Common::Manager
+      def resource_name
+        "RDS Database Instance"
+      end
+
+      def local_resources
+        @local_resources ||= Hash[Loader.instances.map { |local| [local.name, local] }]
+      end
+
+      def aws_resources
+        @aws_resources ||= RDS::named_instances
+      end
+
+      def unmanaged_diff(aws)
+        InstanceDiff.unmanaged(aws)
+      end
+
+      def added_diff(local)
+        InstanceDiff.added(local)
+      end
+
+      def diff_resource(local, aws)
+        puts Colors.blue("Processing #{local.name}...")
+        cumulus_version = InstanceConfig.new(local.name).populate!(aws)
+        local.diff(cumulus_version)
+      end
+
+      def migrate
+        puts Colors.blue("Will migrate #{RDS.instances.length} instances")
+
+        # Create the directories
+        rds_dir = "#{@migration_root}/rds"
+        instances_dir = "#{rds_dir}/instances"
+
+        if !Dir.exists?(@migration_root)
+          Dir.mkdir(@migration_root)
+        end
+        if !Dir.exists?(rds_dir)
+          Dir.mkdir(rds_dir)
+        end
+        if !Dir.exists?(instances_dir)
+          Dir.mkdir(instances_dir)
+        end
+
+        RDS.named_instances.each do |name, instance|
+          puts "Migrating #{name}..."
+
+          cumulus_instance = InstanceConfig.new(name).populate!(instance)
+
+          json = JSON.pretty_generate(cumulus_instance.to_hash)
+          File.open("#{instances_dir}/#{name}.json", "w") { |f| f.write(json) }
+        end
+      end
+
+      def update(local, diffs)
+        aws_instance = RDS::named_instances[local.name]
+
+        all_changes = diffs.map do |diff|
+          case diff.type
+          when InstanceChange::ENGINE,
+               InstanceChange::USERNAME,
+               InstanceChange::SUBNET,
+               InstanceChange::DATABASE,
+               InstanceChange::STORAGE_TYPE
+            puts Colors.red("Cannot change #{diff.asset_type}")
+            # no change for this diff, ignore it
+            nil
+          when InstanceChange::PORT
+            puts "Updating the database port number..."
+            {db_port_number: local.port}
+          when InstanceChange::TYPE
+            puts "Updating the database instance type..."
+            {db_instance_class: "db." + local.type}
+          when InstanceChange::ENGINE_VERSION
+            puts "Updating the engine version..."
+            {engine_version: local.engine_version}
+          when InstanceChange::STORAGE_SIZE
+            puts "Updating the allocated storage..."
+            {allocated_storage: local.storage_size}
+          when InstanceChange::SECURITY_GROUPS
+            puts "Updating the security groups..."
+            security_group_ids = local.security_groups.map do |sg|
+              # TODO: once the db subnet config is created, find the security group id based on the vpc in that subnet.
+              # right now, if security groups from different vpc have the same name, this might not return the right id.
+              sg_id = SecurityGroups.vpc_security_group_id_names.values.inject(:merge).key(sg)
+              if sg_id.nil?
+                raise Exception.new("security group #{sg} does not exist")
+              end
+              sg_id
+            end
+            {vpc_security_group_ids: security_group_ids}
+          when InstanceChange::PUBLIC
+            if local.public_facing
+              puts "making the database public..."
+            else
+              puts "blocking the database from the public..."
+            end
+            {publicly_accessible: local.public_facing}
+          when InstanceChange::BACKUP
+            puts "Updating the backup preferences..."
+            {preferred_backup_window: local.backup_window, backup_retention_period: local.backup_period}
+          when InstanceChange::UPGRADE
+            puts "Updating the upgrade preferences..."
+            {preferred_maintenance_window: local.upgrade_window, auto_minor_version_upgrade: local.auto_upgrade}
+          end
+        end.reject {|v| v.nil?}.reduce(&:merge)
+
+        # make all the updates in the same call
+        RDS::client.modify_db_instance({db_instance_identifier: local.name, apply_immediately: true}.merge(all_changes)) unless all_changes.nil?
+      end
+
+      def create(local)
+        errors = Array.new
+
+        if local.name.nil?
+          errors << "instance name is required"
+        end
+
+        if local.type.nil?
+          errors << "instance type is required"
+        end
+
+        if local.engine.nil?
+          errors << "database engine is required"
+        end
+
+        unless errors.empty?
+          puts Colors.red("Could not create #{local.name}:")
+          errors.each { |e| puts Colors.red("\t#{e}")}
+          exit StatusCodes::EXCEPTION
+        end
+
+        master_password = unless local.master_username.nil?
+          password_prompt(local.master_username)
+        else
+          nil
+        end
+
+        security_group_ids = local.security_groups.map do |sg|
+          # TODO: once the db subnet config is created, find the security group id based on the vpc in that subnet.
+          # right now, if security groups from different vpc have the same name, this might not return the right id.
+          sg_id = SecurityGroups.vpc_security_group_id_names.values.inject(:merge).key(sg)
+          if sg_id.nil?
+            raise Exception.new("security group #{sg} does not exist")
+          end
+          sg_id
+        end
+
+        RDS::client.create_db_instance({
+          db_name: local.database,
+          db_instance_identifier: local.name, # required
+          allocated_storage: local.storage_size,
+          db_instance_class: "db." + local.type, # required
+          engine: local.engine, # required
+          master_username: local.master_username,
+          master_user_password: master_password,
+          vpc_security_group_ids: security_group_ids,
+          db_subnet_group_name: local.subnet,
+          preferred_maintenance_window: local.upgrade_window,
+          backup_retention_period: local.backup_period,
+          preferred_backup_window: local.backup_window,
+          port: local.port,
+          engine_version: local.engine_version,
+          auto_minor_version_upgrade: local.auto_upgrade,
+          publicly_accessible: local.public_facing,
+          storage_type: local.storage_type,
+        }.reject { |k, v| v.nil? })
+
+      end
+
+      private
+
+      def password_prompt(username)
+        # prompt for the user's password (discreetly)
+        print "enter a password for #{username}: "
+        password = STDIN.noecho(&:gets).chomp
+        puts "\n"
+        password
+      end
+
+    end
+  end
+end