diff --git a/glacier/glacier.py b/glacier/glacier.py index d55c97a..ab12425 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -6,6 +6,8 @@ :synopsis: Command line interface for amazon glacier """ +# import modules + import sys import os import ConfigParser @@ -13,173 +15,17 @@ import re import locale import glob -import csv -import json - -import glacierexception -import constants - -from prettytable import PrettyTable -from GlacierWrapper import GlacierWrapper -from functools import wraps - -def output_headers(headers, output): - """ - Prints a list of headers - single item output. - - :param headers: the output to be printed as {'header1':'data1',...} - :type headers: dict - """ - rows = [(k, headers[k]) for k in headers.keys()] - if output == 'print': - table = PrettyTable(["Header", "Value"]) - for row in rows: - if len(str(row[1])) <= 138: - table.add_row(row) - - print table - - if output not in constants.HEADERS_OUTPUT_FORMAT: - raise ValueError("Output format must be {}, got" - ":{}".format(constants.HEADERS_OUTPUT_FORMAT, - output)) - - if output == 'csv': - csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - for row in rows: - csvwriter.writerow(row) - - if output == 'json': - print json.dumps(headers) - -def output_table(results, output, keys=None, sort_key=None): - """ - Prettyprints results. Expects a list of identical dicts. - Use the dict keys as headers unless keys is given; - one line for each item. - - Expected format of data is a list of dicts: - [{'key1':'data1.1', 'key2':'data1.2', ... }, - {'key1':'data1.2', 'key2':'data2.2', ... }, - ...] - keys: dict of headers to be printed for each key: - {'key1':'header1', 'key2':'header2',...} - - sort_key: the key to use for sorting the table. - """ - - if output not in constants.TABLE_OUTPUT_FORMAT: - raise ValueError("Output format must be{}, " - "got {}".format(constants.TABLE_OUTPUT_FORMAT, - output)) - if output == 'print': - if len(results) == 0: - print 'No output!' - return - - headers = [keys[k] for k in keys.keys()] if keys else results[0].keys() - table = PrettyTable(headers) - for line in results: - table.add_row([line[k] if k in line else '' for k in (keys.keys() if keys else headers)]) - - if sort_key: - table.sortby = keys[sort_key] if keys else sort_key - - print table - - if output == 'csv': - csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - keys = results[0].keys() - csvwriter.writerow(keys) - for row in results: - csvwriter.writerow([row[k] for k in keys]) - - if output == 'json': - print json.dumps(results) - -def output_msg(msg, output, success=True): - """ - In case of a single message output, e.g. nothing found. - - :param msg: a single message to output. - :type msg: str - :param success: whether the operation was a success or not. - :type success: boolean - """ - - if output not in constants.TABLE_OUTPUT_FORMAT: - raise ValueError("Output format must be{}, " - "got {}".format(constants.TABLE_OUTPUT_FORMAT, - output)) - - if msg is not None: - if output == 'print': - print msg - - if output == 'csv': - csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - csvwriter.writerow(msg) - - if output == 'json': - print json.dumps(msg) - - if not success: - sys.exit(125) - -def size_fmt(num, decimals = 1): - """ - Formats file sizes in human readable format. Anything bigger than - TB is returned is TB. Number of decimals is optional, - defaults to 1. - """ - fmt = "%%3.%sf %%s"% decimals - for x in ['bytes','KB','MB','GB']: - if num < 1024.0: - return fmt % (num, x) - - num /= 1024.0 - - return fmt % (num, 'TB') - -def default_glacier_wrapper(args, **kwargs): - """ - Convenience function to call an instance of GlacierWrapper - with all required arguments. - """ - return GlacierWrapper(args.aws_access_key, - args.aws_secret_key, - args.region, - bookkeeping=args.bookkeeping, - no_bookkeeping=args.no_bookkeeping, - bookkeeping_domain_name=args.bookkeeping_domain_name, - sdb_access_key=args.sdb_access_key, - sdb_secret_key=args.sdb_secret_key, - sdb_region=args.sdb_region, - # sns_enable=args.sns_enable, - # sns_topic=args.sns_topic, - # sns_monitored_vaults=args.sns_monitored_vaults, - # sns_options=args.sns_options, - # config_object=args.config_object, - logfile=args.logfile, - loglevel=args.loglevel, - logtostdout=args.logtostdout) -def handle_errors(fn): - """ - Decorator for exception handling. - """ - @wraps(fn) - def wrapper(*args, **kwargs): - try: - return fn(*args, **kwargs) - except glacierexception.GlacierException as e: +# import our modules +from modules.glacier_utils import (default_glacier_wrapper, size_fmt, + output_msg, output_headers, output_table) +from modules.sns_utils import (snssync, snssubscribe, + snslistsubscriptions, snslisttopics, snsunsubscribe) +from modules.decorators import handle_errors +from modules import glacierexception, constants +from modules.GlacierWrapper import GlacierWrapper - # We are only interested in the error message in case - # it is a self-caused exception. - e.write(indentation='|| ', stack=False, message=True) - sys.exit(e.exitcode) - - return wrapper +# function definition @handle_errors def lsvault(args): @@ -208,6 +54,7 @@ def rmvault(args): """ Remove a vault. """ + glacier = default_glacier_wrapper(args) response = glacier.rmvault(args.vault) output_headers(response, args.output) @@ -217,6 +64,7 @@ def describevault(args): """ Give the description of a vault. """ + glacier = default_glacier_wrapper(args) response = glacier.describevault(args.vault) headers = {'LastInventoryDate': "LastInventory", @@ -231,6 +79,7 @@ def listmultiparts(args): """ Give an overview of all multipart uploads that are not finished. """ + glacier = default_glacier_wrapper(args) response = glacier.listmultiparts(args.vault) if not response: @@ -244,6 +93,7 @@ def abortmultipart(args): """ Abort a multipart upload which is in progress. """ + glacier = default_glacier_wrapper(args) response = glacier.abortmultipart(args.vault, args.uploadId) output_headers(response, args.output) @@ -253,6 +103,7 @@ def listjobs(args): """ List all the active jobs for a vault. """ + glacier = default_glacier_wrapper(args) job_list = glacier.list_jobs(args.vault) if job_list == []: @@ -281,6 +132,7 @@ def download(args): """ Download an archive. """ + glacier = default_glacier_wrapper(args) response = glacier.download(args.vault, args.archive, args.partsize, out_file_name=args.outfile, @@ -368,6 +220,7 @@ def getarchive(args): """ Initiate an archive retrieval job. """ + glacier = default_glacier_wrapper(args) status, job, jobid = glacier.getarchive(args.vault, args.archive) output_headers(job, args.output) @@ -377,6 +230,7 @@ def rmarchive(args): """ Remove an archive from a vault. """ + glacier = default_glacier_wrapper(args) glacier.rmarchive(args.vault, args.archive) output_msg("Archive removed.", args.output, success=True) @@ -386,6 +240,7 @@ def search(args): """ Search the database for file name or description. """ + glacier = default_glacier_wrapper(args) response = glacier.search(vault=args.vault, region=args.region, @@ -398,6 +253,7 @@ def inventory(args): """ Fetch latest inventory (or start a retrieval job if not ready). """ + glacier = default_glacier_wrapper(args) output = args.output if sys.stdout.isatty() and output == 'print': @@ -439,6 +295,7 @@ def treehash(args): """ Calculates the tree hash of the given file(s). """ + glacier = default_glacier_wrapper(args) hash_results = [] for f in args.filename: @@ -461,66 +318,7 @@ def treehash(args): output_table(hash_results, args.output) -def snssync(args): - """ - If monitored_vaults is specified in configuration file, subscribe - vaults specificed in it to notifications, otherwiser - subscribe all vault. - """ - glacier = default_glacier_wrapper(args) - response = glacier.sns_sync(sns_options=args.sns_options, - output=args.output) - output_table(response, args.output) -def snssubscribe(args): - """ - Subscribe individual vaults to notifications by method - specified by user. - """ - protocol = args.protocol - endpoint = args.endpoint - vault_names = args.vault - topic = args.topic - - glacier = default_glacier_wrapper(args) - response = glacier.sns_subscribe(protocol, endpoint, topic, - vault_names=vault_names, - sns_options=args.sns_options) - output_table(response, args.output) - -def snslistsubscriptions(args): - """ - List subscriptions. - """ - protocol = args.protocol - endpoint = args.endpoint - topic = args.topic - - glacier = default_glacier_wrapper(args) - response = glacier.sns_list_subscriptions(protocol, endpoint, - topic, - sns_options=args.sns_options) - output_table(response, args.output) - -def snslisttopics(args): - glacier = default_glacier_wrapper(args) - response = glacier.sns_list_topics(sns_options=args.sns_options) - output_table(response, args.output) - -def snsunsubscribe(args): - """ - Unsubscribe individual vaults from notifications for - specified protocol, endpoint and vault. - """ - protocol = args.protocol - endpoint = args.endpoint - topic = args.topic - - glacier = default_glacier_wrapper(args) - response = glacier.sns_unsubscribe(protocol, endpoint, - topic, - sns_options=args.sns_options) - output_table(response, args.output) def main(): program_description = u""" diff --git a/glacier/GlacierWrapper.py b/glacier/modules/GlacierWrapper.py similarity index 80% rename from glacier/GlacierWrapper.py rename to glacier/modules/GlacierWrapper.py index 7053533..2968732 100755 --- a/glacier/GlacierWrapper.py +++ b/glacier/modules/GlacierWrapper.py @@ -2,10 +2,11 @@ """ .. module:: GlacierWrapper :platform: Unix, Windows - :synopsis: Wrapper for accessing Amazon Glacier, with Amazon SimpleDB + :synopsis: Wrapper for accessing Amazon Glacier, with Amazon SimpleDB support and other features. """ +# import modules import math import json import pytz @@ -16,7 +17,7 @@ import time import sys import traceback -import glaciercorecalls + import select import hashlib import fcntl @@ -25,6 +26,7 @@ import boto import boto.sdb + from boto import sns from functools import wraps @@ -32,9 +34,11 @@ from datetime import datetime from pprint import pformat -from glaciercorecalls import GlacierConnection, GlacierWriter +# import our modules -from glacierexception import * +from modules import constants +from modules import glaciercorecalls +from modules import glacierexception class log_class_call(object): """ @@ -109,27 +113,6 @@ class GlacierWrapper(object): and other features. """ - VAULT_NAME_ALLOWED_CHARACTERS = "[a-zA-Z\.\-\_0-9]+" - ID_ALLOWED_CHARACTERS = "[a-zA-Z\-\_0-9]+" - MAX_VAULT_NAME_LENGTH = 255 - MAX_VAULT_DESCRIPTION_LENGTH = 1024 - MAX_PARTS = 10000 - AVAILABLE_REGIONS = ('us-east-1', 'us-west-2', 'us-west-1', - 'eu-west-1', 'eu-central-1', 'sa-east-1', - 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2') - AVAILABLE_REGIONS_MESSAGE = """\ -Invalid region. Available regions for Amazon Glacier are: -us-east-1 (US - Virginia) -us-west-1 (US - N. California) -us-west-2 (US - Oregon) -eu-west-1 (EU - Ireland) -eu-central-1 (EU - Frankfurt) -sa-east-1 (South America - Sao Paulo) -ap-northeast-1 (Asia-Pacific - Tokyo) -ap-southeast-1 (Asia Pacific (Singapore) -ap-southeast-2 (Asia-Pacific - Sydney)\ -""" - def setuplogging(self, logfile, loglevel, logtostdout): """ Set up the logging facility. @@ -225,11 +208,10 @@ def glacier_connect_wrap(*args, **kwargs): self.aws_access_key, self.aws_secret_key, self.region) - self.glacierconn = GlacierConnection(self.aws_access_key, - self.aws_secret_key, - region_name=self.region) + self.glacierconn = glaciercorecalls.GlacierConnection(self.aws_access_key, + self.aws_secret_key, region_name=self.region) except boto.exception.AWSConnectionError as e: - raise ConnectionException( + raise glacierexception.ConnectionException( "Cannot connect to Amazon Glacier.", cause=e.cause, code="GlacierConnectionError") @@ -261,7 +243,7 @@ def sdb_connect_wrap(*args, **kwargs): # we need to glaciercorecalls? if not self.bookkeeping_domain_name: - raise InputException( + raise glacierexception.InputException( '''\ Bookkeeping enabled but no Amazon SimpleDB domain given. Provide a domain in either the config file or via the @@ -284,8 +266,9 @@ def sdb_connect_wrap(*args, **kwargs): aws_secret_access_key=self.sdb_secret_key) domain_name = self.bookkeeping_domain_name self.sdb_domain = self.sdb_conn.create_domain(domain_name) - except (boto.exception.AWSConnectionError, boto.exception.SDBResponseError) as e: - raise ConnectionException( + except (boto.exception.AWSConnectionError, + boto.exception.SDBResponseError) as e: + raise glacierexception.ConnectionException( "Cannot connect to Amazon SimpleDB.", cause=e, code="SdbConnectionError") @@ -316,7 +299,7 @@ def sns_connect_wrap(*args, **kwargs): aws_secret_access_key=self.aws_secret_key, region_name=self.region) except boto.exception.AWSConnectionError as e: - raise ConnectionException( + raise glacierexception.ConnectionException( "Cannot connect to Amazon SNS.", cause=e.cause, code="SNSConnectionError") @@ -337,14 +320,16 @@ def _check_vault_name(self, name): :raises: :py:exc:`glacier.glacierexception.InputException` """ - if len(name) > self.MAX_VAULT_NAME_LENGTH: - raise InputException( - u"Vault name can be at most %s characters long." % self.MAX_VAULT_NAME_LENGTH, - cause="Vault name more than %s characters long." % self.MAX_VAULT_NAME_LENGTH, + if len(name) > constants.MAX_VAULT_NAME_LENGTH: + raise glacierexception.InputException( + "Vault name can be at most {} characters "\ + "long.".format(onstants.MAX_VAULT_NAME_LENGTH), + cause="Vault name more than {} characters "\ + "long.".format(constants.MAX_VAULT_NAME_LENGTH), code="VaultNameError") if len(name) == 0: - raise InputException( + raise glacierexception.InputException( u"Vault name has to be at least 1 character long.", cause='Vault name has to be at least 1 character long.', code="VaultNameError") @@ -352,10 +337,11 @@ def _check_vault_name(self, name): # If the name starts with an illegal character, then result # m is None. In that case the expression becomes '0 != len(name)' # which of course is always True. - m = re.match(self.VAULT_NAME_ALLOWED_CHARACTERS, name) + m = re.match(constants.VAULT_NAME_ALLOWED_CHARACTERS, name) if (m.end() if m else 0) != len(name): - raise InputException( - u"""Allowed characters are a-z, A-Z, 0-9, '_' (underscore), '-' (hyphen), and '.' (period)""", + raise glacierexception.InputException( + u"allowed characters are a-z, A-Z, 0-9, '_' "\ + "(underscore), '-' (hyphen), and '.' (period)", cause='Illegal characters in the vault name.', code="VaultNameError") @@ -365,8 +351,8 @@ def _check_vault_name(self, name): 'Vault description is valid.') def _check_vault_description(self, description): """ - Checks whether a vault description is valid (at least one character, - not too long, no illegal characters). + Checks whether a vault description is valid (at least one + character, not too long, no illegal characters). :param description: Vault description :type description: str @@ -376,19 +362,21 @@ def _check_vault_description(self, description): :raises: :py:exc:`glacier.glacierexception.InputException` """ - if len(description) > self.MAX_VAULT_DESCRIPTION_LENGTH: - raise InputException( - u"Description must be no more than %s characters."% self.MAX_VAULT_DESCRIPTION_LENGTH, - cause='Vault description contains more than %s characters.'% self.MAX_VAULT_DESCRIPTION_LENGTH, + if len(description) > constants.MAX_VAULT_DESCRIPTION_LENGTH: + raise glacierexception.InputException( + "Description must be no more than {} characters"\ + ".".format(constants.MAX_VAULT_DESCRIPTION_LENGTH), + cause='Vault description contains more than {} "\ + "characters.'.format(constants.MAX_VAULT_DESCRIPTION_LENGTH), code="VaultDescriptionError") for char in description: n = ord(char) if n < 32 or n > 126: - raise InputException( - u"""The allowed characters are 7-bit ASCII without \ -control codes, specifically ASCII values 32-126 decimal \ -or 0x20-0x7E hexadecimal.""", + raise glacierexception.InputException( + u"The allowed characters are 7-bit ASCII without "\ + "control codes, specifically ASCII values 32-126 "\ + "decimal or 0x20-0x7E hexadecimal.", cause="Invalid characters in the vault name.", code="VaultDescriptionError") @@ -405,7 +393,8 @@ def _check_id(self, amazon_id, id_type): :param amazon_id: id to be validated :type amazon_id: str - :param id_type: the case-sensity type of id (JobId, UploadId, ArchiveId). + :param id_type: the case-sensity type of id + (JobId, UploadId, ArchiveId). :type id_type: str :returns: True if valid, raises exception otherwise. @@ -416,20 +405,23 @@ def _check_id(self, amazon_id, id_type): length = {'JobId': 92, 'UploadId': 92, 'ArchiveId': 138} - self.logger.debug('Checking a %s.' % id_type) + self.logger.debug('Checking a {}.'.format(id_type)) if len(amazon_id) != length[id_type]: - raise InputException( - 'A %s must be %s characters long. This ID is %s characters.'% (id_type, length[id_type], len(amazon_id)), - cause='Incorrect length of the %s string.'% id_type, + raise glacierexception.InputException( + 'A {} must be {} characters long. This ID is {} "\ + "characters.'.format(id_type, + length[id_type], + len(amazon_id)), + cause='Incorrect length of the {} string.'.format(id_type), code="IdError") - m = re.match(self.ID_ALLOWED_CHARACTERS, amazon_id) + m = re.match(constants.ID_ALLOWED_CHARACTERS, amazon_id) if (m.end() if m else 0) != len(amazon_id): - raise InputException(u"""\ -This %s contains invalid characters. \ + raise glacierexception.InputException(u"""\ +This {} contains invalid characters. \ Allowed characters are a-z, A-Z, 0-9, '_' (underscore) and '-' (hyphen)\ -""" % id_type, - cause='Illegal characters in the %s string.' % id_type, +""".format(id_type), + cause='Illegal characters in the {} string.'.format(id_type), code="IdError") return True @@ -448,10 +440,10 @@ def _check_region(self, region): :raises: GlacierWrapper.InputException """ - if not region in self.AVAILABLE_REGIONS: - raise InputException( - self.AVAILABLE_REGIONS_MESSAGE, - cause='Invalid region code: %s.' % region, + if not region in constants.AVAILABLE_REGIONS: + raise glacierexception.InputException( + constants.AVAILABLE_REGIONS_MESSAGE, + cause='Invalid region code: {}.'.format(region), code='RegionError') return True @@ -471,29 +463,29 @@ def _check_part_size(self, part_size, total_size): def _part_size_for_total_size(total_size): return self._next_power_of_2( int(math.ceil( - float(total_size) / (1024 * 1024 * self.MAX_PARTS) + float(total_size) / (1024 * 1024 * constants.MAX_PARTS) ))) if part_size < 0: if total_size > 0: part_size = _part_size_for_total_size(total_size) else: - part_size = GlacierWriter.DEFAULT_PART_SIZE + part_size = glaciercorecalls.GlacierWriter.DEFAULT_PART_SIZE else: ps = self._next_power_of_2(part_size) if not ps == part_size: - self.logger.warning("""\ -Part size in MB must be a power of 2, \ -e.g. 1, 2, 4, 8 MB; automatically increased part size from %s to %s.\ -""" % (part_size, ps)) - + self.logger.warning("Part size in MB must be a "\ + "power of 2, "\ + "e.g. 1, 2, 4, 8 MB; "\ + "automatically increased part "\ + "size from {} to {}.".format(part_size, ps)) part_size = ps # Check if user specified value is big enough, and adjust if needed. - if total_size > part_size * 1024 * 1024 * self.MAX_PARTS: + if total_size > part_size * 1024 * 1024 * constants.MAX_PARTS: part_size = _part_size_for_total_size(total_size) self.logger.warning("Part size given is too small; \ -using %s MB parts to upload." % part_size) +using {} MB parts to upload.".format(part_size)) return part_size @@ -539,7 +531,9 @@ def _progress(self, msg): if sys.stdout.isatty(): # Get the current screen width. - cols = struct.unpack('hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, '1234'))[1] + cols = struct.unpack('hh', fcntl.ioctl(sys.stdout, + termios.TIOCGWINSZ, + '1234'))[1] # Make sure the message fits on a single line, strip if not, # and add spaces to fill the line if it's shorter (to erase @@ -615,7 +609,7 @@ def lsvault(self, limit=None): try: response = self.glacierconn.list_vaults(marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( 'Failed to recieve vault list.', cause=self._decode_error_message(e.body), code=e.code) @@ -650,8 +644,8 @@ def mkvault(self, vault_name): try: response = self.glacierconn.create_vault(vault_name) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to create vault with name %s.' % vault_name, + raise glacierexception.ResponseException( + 'Failed to create vault with name {}.'.format(vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -683,23 +677,27 @@ def rmvault(self, vault_name): try: response = self.glacierconn.delete_vault(vault_name) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to remove vault with name %s.' % vault_name, + raise glacierexception.ResponseException( + 'Failed to remove vault with name {}.'.format(vault_name), cause=self._decode_error_message(e.body), code=e.code) # Check for orphaned entries in the bookkeeping database, and # remove them. if self.bookkeeping: - query = "select * from `%s` where vault='%s'" % (self.bookkeeping_domain_name, vault_name) + query = "select * from `{}` where "\ + "vault='{}'".format(self.bookkeeping_domain_name, + vault_name) result = self.sdb_domain.select(query) try: for item in result: self.sdb_domain.delete_item(item) - self.logger.debug('Deleted orphaned archive from the database: %s.' % item.name) + self.logger.debug("Deleted orphaned archive from "\ + "the database: {}.".format(item.name)) except boto.exception.SDBResponseError as e: - raise ResponseException( - 'SimpleDB did not respond correctly to our orphaned listings check.', + raise glacierexception.ResponseException( + "SimpleDB did not respond correctly to "\ + "our orphaned listings check.", cause=self._decode_error_message(e.body), code=e.code) @@ -734,8 +732,8 @@ def describevault(self, vault_name): try: response = self.glacierconn.describe_vault(vault_name) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to get description of vault with name %s.' % vault_name, + raise glacierexception.ResponseException( + 'Failed to get description of vault with name {}.'.format(vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -783,13 +781,15 @@ def list_jobs(self, vault_name, completed=None, job_list = [] while True: try: - response = self.glacierconn.list_jobs(vault_name, - completed=completed, - status_code=status_code, - marker=marker) + response = self.glacierconn.list_jobs( + vault_name, + completed=completed, + status_code=status_code, + marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to recieve the jobs list for vault %s.' % vault_name, + raise glacierexception.ResponseException( + "Failed to recieve the jobs list for vault "\ + "{}.".format(vault_name), cause=self._decode_error_message(e.body), code=e.code) job_list += response.copy()['JobList'] @@ -843,8 +843,8 @@ def describejob(self, vault_name, job_id): try: response = self.glacierconn.describe_job(vault_name, job_id) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to get description of job with job id %s.' % job_id, + raise glacierexception.ResponseException( + 'Failed to get description of job with job id {}.'.format(job_id), cause=self._decode_error_message(e.body), code=e.code) @@ -855,12 +855,13 @@ def describejob(self, vault_name, job_id): "Multipart upload successfully aborted.") def abortmultipart(self, vault_name, upload_id): """ - Aborts an incomplete multipart upload, causing any uploaded data to be - removed from Amazon Glacier. + Aborts an incomplete multipart upload, causing any + uploaded data to be removed from Amazon Glacier. :param vault_name: Name of the vault. :type vault_name: str - :param upload_id: the UploadId of the multipart upload to be aborted. + :param upload_id: the UploadId of the multipart upload + to be aborted. :type upload_id: str :returns: server response. @@ -877,10 +878,12 @@ def abortmultipart(self, vault_name, upload_id): self._check_vault_name(vault_name) self._check_id(upload_id, "UploadId") try: - response = self.glacierconn.abort_multipart_upload(vault_name, upload_id) + response = self.glacierconn.abort_multipart_upload( + vault_name, + upload_id) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to abort multipart upload with id %s.' % upload_id, + raise glacierexception.ResponseException( + 'Failed to abort multipart upload with id {}.'.format(upload_id), cause=self._decode_error_message(e.body), code=e.code) @@ -919,8 +922,8 @@ def listmultiparts(self, vault_name, limit=None): response = self.glacierconn.list_multipart_uploads(vault_name, marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to get a list of multipart uploads for vault %s.' % vault_name, + raise glacierexception.ResponseException( + 'Failed to get a list of multipart uploads for vault {}.'.format(vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -981,8 +984,12 @@ def upload(self, vault_name, file_name, description, region, self._check_id(uploadid, 'UploadId') if resume and stdin: - raise InputException( - 'You must provide the UploadId to resume upload of streams from stdin.\nUse glacier-cmd listmultiparts to find the UploadId.', + raise glacierexception.InputException( + "You must provide the UploadId to "\ + "resume upload of streams from "\ + "stdin.\nUse glacier-cmd "\ + "listmultiparts "\ + "to find the UploadId.", code='CommandError') # If file_name is given, try to use this file(s). @@ -998,7 +1005,7 @@ def upload(self, vault_name, file_name, description, region, if not stdin and not is_pipe: if not file_name: - raise InputException( + raise glacierexception.InputException( "No file name given for upload.", code='CommandError') @@ -1007,8 +1014,8 @@ def upload(self, vault_name, file_name, description, region, mmapped_file = mmap(f) total_size = os.path.getsize(file_name) except IOError as e: - raise InputException( - "Could not access file: %s."% file_name, + raise glacierexception.InputException( + "Could not access file: {}.".format(file_name), cause=e, code='FileError') @@ -1017,62 +1024,77 @@ def upload(self, vault_name, file_name, description, region, reader = open(file_name, 'rb') total_size = 0 except IOError: - raise InputException( - "Could not access pipe: %s."% file_name, + raise glacierexception.InputException( + "Could not access pipe: {}.".format(file_name), cause = e, code = 'FileError') elif select.select([sys.stdin,],[],[],0.0)[0]: reader = sys.stdin total_size = 0 else: - raise InputException( + raise glacierexception.InputException( "There is nothing to upload.", code='CommandError') # Log the kind of upload we're going to do. if uploadid: - self.logger.info('Attempting resumption of upload of %s to %s.'% (file_name if file_name else 'data from stdin', vault_name)) + self.logger.info('Attempting resumption of upload of %s to {}.'.format(file_name if file_name else 'data from stdin', vault_name)) elif resume: - self.logger.info('Attempting resumption of upload of %s to %s.'% (file_name, vault_name)) + self.logger.info('Attempting resumption of upload of {} to {}.'.format(file_name, vault_name)) else: - self.logger.info('Starting upload of %s to %s.\nDescription: %s'% (file_name if file_name else 'data from stdin', vault_name, description)) + self.logger.info('Starting upload of {} to {}.\nDescription: {}'.format(file_name if file_name else 'data from stdin', vault_name, description)) - # If user did not specify part_size, compute the optimal (i.e. lowest - # value to stay within the self.MAX_PARTS (10,000) block limit). + # If user did not specify part_size, compute the + # optimal (i.e. lowest value to stay within the + # constants.MAX_PARTS (10,000) block limit). part_size = self._check_part_size(part_size, total_size) part_size_in_bytes = part_size * 1024 * 1024 - # If we have an UploadId, check whether it is linked to a current - # job. If so, check whether uploaded data matches the input data and - # try to resume uploading. + # If we have an UploadId, check whether it is linked + # to a current job. If so, check whether uploaded data + # matches the input data and try to resume uploading. upload = None if uploadid: uploads = self.listmultiparts(vault_name) for upload in uploads: if uploadid == upload['MultipartUploadId']: - self.logger.debug('Found a matching upload id. Continuing upload resumption attempt.') + self.logger.debug("Found a matching upload id. "\ + "Continuing upload resumption "\ + "attempt.") self.logger.debug(upload) part_size_in_bytes = upload['PartSizeInBytes'] break else: - raise InputException( - 'Can not resume upload of this data as no existing job with this uploadid could be found.', - code='IdError') + raise glacierexception.InputException( + "Can not resume upload of "\ + "this data as no existing "\ + "job with this uploadid "\ + "could be found.", + code='IdError') # Initialise the writer task. - writer = GlacierWriter(self.glacierconn, vault_name, description=description, - part_size_in_bytes=part_size_in_bytes, uploadid=uploadid, logger=self.logger) + writer = glaciercorecalls.GlacierWriter(self.glacierconn, vault_name, + description=description, + part_size_in_bytes=part_size_in_bytes, + uploadid=uploadid, logger=self.logger) if upload: marker = None while True: - # Fetch a list of already uploaded parts and their SHA hashes. + # Fetch a list of already uploaded parts and their + # SHA hashes. try: - response = self.glacierconn.list_parts(vault_name, uploadid, marker=marker) + response = self.glacierconn.list_parts( + vault_name, + uploadid, + marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to get a list already uploaded parts for interrupted upload %s.'% uploadid, + raise glacierexception.ResponseException( + "Failed to get a list "\ + "already uploaded parts "\ + "for interrupted "\ + "upload ".format(uploadid), cause=self._decode_error_message(e.body), code=e.code) @@ -1080,29 +1102,34 @@ def upload(self, vault_name, file_name, description, region, current_position = 0 stop = 0 # Process the parts list. - # For each part of data, take the matching data range from + # For each part of data, take the matching + # data range from # the local file, and compare hashes. - # If recieving data over stdin, the parts must be sequential - # and the first must start at 0. For file, we can use the seek() + # If recieving data over stdin, the parts + # must be sequential + # and the first must start at 0. For file, + # we can use the seek() # function to handle non-sequential parts. for part in list_parts_response['Parts']: start, stop = (int(p) for p in part['RangeInBytes'].split('-')) stop += 1 if not start == current_position: if stdin or is_pipe: - raise InputException( - 'Cannot verify non-sequential upload data from stdin or pipe.', + raise glacierexception.InputException( + "Cannot verify non-sequential upload "\ + "data from stdin or pipe.", code='ResumeError') if reader: reader.seek(start) if mmapped_file and stop > mmapped_file.size: - raise InputException( - 'File does not match uploaded data; please check your uploadid and try again.', - cause='File is smaller than uploaded data.', + raise glacierexception.InputException( + constants.UPLOAD_DATA_ERROR_MSG, + cause="File is smaller than uploaded data.", code='ResumeError') - # Try to read the chunk of data, and take the hash if we + # Try to read the chunk of data, and take + # the hash if we # have received anything. # If no data or hash mismatch, stop checking raise an # exception. @@ -1111,17 +1138,17 @@ def upload(self, vault_name, file_name, description, region, if data: data_hash = glaciercorecalls.tree_hash(glaciercorecalls.chunk_hashes(data)) if glaciercorecalls.bytes_to_hex(data_hash) == part['SHA256TreeHash']: - self.logger.debug('Part %s hash matches.'% part['RangeInBytes']) + self.logger.debug('Part {} hash matches.'.format(part['RangeInBytes'])) writer.tree_hashes.append(data_hash) else: - raise InputException( - 'Received data does not match uploaded data; please check your uploadid and try again.', - cause='SHA256 hash mismatch.', - code='ResumeError') + raise glacierexception.InputException( + constants.UPLOAD_DATA_ERROR_MSG, + cause="SHA256 hash mismatch.", + code="ResumeError") else: - raise InputException( - 'Received data does not match uploaded data; please check your uploadid and try again.', + raise glacierexception.InputException( + constants.UPLOAD_DATA_ERROR_MSG, cause='No or not enough data to match.', code='ResumeError') @@ -1140,22 +1167,22 @@ def upload(self, vault_name, file_name, description, region, self._size_fmt(total_size), self._bold(str(int(100 * writer.uploaded_size/total_size)))) else: - msg = 'Checked %s.' \ - % (self._size_fmt(writer.uploaded_size)) + msg = "Checked "\ + "{}.".format(self._size_fmt(writer.uploaded_size)) self._progress(msg) - # Finished checking; log this and print the final status update - # before resuming the upload. - self.logger.info('Already uploaded: %s. Continuing from there.'% self._size_fmt(stop)) + # Finished checking; log this and print the final + # status update before resuming the upload. + self.logger.info("Already uploaded: {}"\ + ". Continuing from there.".format(self._size_fmt(stop))) if total_size > 0: msg = 'Checked %s of %s (%s%%). Check done; resuming upload.' \ % (self._size_fmt(writer.uploaded_size), self._size_fmt(total_size), self._bold(str(int(100 * writer.uploaded_size/total_size)))) else: - msg = 'Checked %s. Check done; resuming upload.' \ - % (self._size_fmt(writer.uploaded_size)) + msg = "Checked {}. Check done; resuming upload.".format(self._size_fmt(writer.uploaded_size)) self._progress(msg) @@ -1231,12 +1258,15 @@ def upload(self, vault_name, file_name, description, region, location = writer.get_location() if self.bookkeeping: - self.logger.info('Writing upload information into the bookkeeping database.') + self.logger.info("Writing upload information into "\ + "the bookkeeping database.") - # Use the alternative name as given by --name if we have it. + # Use the alternative name as given by + # --name if we have it. file_name = alternative_name if alternative_name else file_name - # If still no name this is an stdin job, so set name accordingly. + # If still no name this is an stdin job, + # so set name accordingly. file_name = file_name if file_name else 'Data from stdin.' file_attrs = { 'region': region, @@ -1250,11 +1280,6 @@ def upload(self, vault_name, file_name, description, region, 'size': writer.uploaded_size } -## if file_name: -## file_attrs['filename'] = file_name -## elif stdin: -## file_attrs['filename'] = 'data from stdin' - self.sdb_domain.put_attributes(file_attrs['filename'], file_attrs) return (archive_id, sha256hash) @@ -1280,7 +1305,8 @@ def getarchive(self, vault_name, archive_id): - return tuple ("ready", job, jobId). - :param vault: Vault name from where we want to retrieve the archive. + :param vault: Vault name from where we want to + retrieve the archive. :type vault: str :param archive: ArchiveID of archive to be retrieved. :type archive: str @@ -1312,8 +1338,9 @@ def getarchive(self, vault_name, archive_id): try: response = self.glacierconn.initiate_job(vault_name, job_data) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to initiate an archive retrieval job for archive %s in vault %s.'% (archive_id, vault_name), + raise glacierexception.ResponseException( + "Failed to initiate an archive retrieval "\ + "job for archive {} in vault {}.".format(archive_id, vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -1342,30 +1369,36 @@ def download(self, vault_name, archive_id, part_size, if job['ArchiveId'] == archive_id: download_job = job if not job['Completed']: - raise CommunicationException( - "Archive retrieval request not completed yet. Please try again later.", + raise glacierexception.CommunicationException( + "Archive retrieval request not completed "\ + "yet. Please try again later.", code='NotReady') - self.logger.debug('Archive retrieval completed; archive is available for download now.') + self.logger.debug("Archive retrieval completed; "\ + "archive is available for "\ + "download now.") break else: - raise InputException( - "Requested archive not available. Please make sure \ -your archive ID is correct, and start a retrieval job using \ -'getarchive' if necessary.", + raise glacierexception.InputException( + "Requested archive not available. Please make sure "\ + "your archive ID is correct, and start a retrieval "\ + "job using getarchive' if necessary.", code='IdError') - # Check whether we can access the file the archive has to be written to. + # Check whether we can access the file the archive + # has to be written to. out_file = None if out_file_name: if os.path.isfile(out_file_name) and not overwrite: - raise InputException( - "File exists already, aborting. Use the overwrite flag to overwrite existing file.", + raise glacierexception.InputException( + "File exists already, aborting. "\ + "Use the overwrite flag to overwrite "\ + "existing file.", code="FileError") try: out_file = open(out_file_name, 'w') except IOError as e: - raise InputException( + raise glacierexception.InputException( "Cannot access the ouput file.", cause=e, code='FileError') @@ -1379,24 +1412,28 @@ def download(self, vault_name, archive_id, part_size, # Log our pending action. if out_file: - self.logger.debug('Starting download of archive to file %s.'% out_file_name) + self.logger.debug("Starting download of archive "\ + "to file {}.".format(out_file_name)) else: - self.logger.debug('Starting download of archive to stdout.') + self.logger.debug("Starting download of archive to stdout.") # Download the data, one part at a time. while downloaded_size < total_size: # Read a part of data. from_bytes = downloaded_size - to_bytes = min(downloaded_size + part_size_in_bytes, total_size) + to_bytes = min(downloaded_size + part_size_in_bytes, + total_size) try: - response = self.glacierconn.get_job_output(vault_name, - download_job['JobId'], - byte_range=(from_bytes, to_bytes-1)) + response = self.glacierconn.get_job_output( + vault_name, + download_job['JobId'], + byte_range=(from_bytes, to_bytes-1)) + data = response.read() except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to download archive %s.'% archive_id, + raise glacierexception.ResponseException( + 'Failed to download archive {}.'.format(archive_id), cause=self._decode_error_message(e.body), code=e.code) @@ -1406,7 +1443,7 @@ def download(self, vault_name, archive_id, part_size, try: out_file.write(response.read()) except IOError as e: - raise InputException( + raise glacierexception.InputException( "Cannot write data to the specified file.", cause=e, code='FileError') @@ -1443,7 +1480,7 @@ def download(self, vault_name, archive_id, part_size, if out_file: out_file.close() if glaciercorecalls.bytes_to_hex(glaciercorecalls.tree_hash(hash_list)) != download_job['SHA256TreeHash']: - raise CommunicationException( + raise glacierexception.CommunicationException( "Downloaded data hash mismatch", code="DownloadError", cause=None) @@ -1460,7 +1497,8 @@ def download(self, vault_name, archive_id, part_size, @sdb_connect @log_class_call("Searching for archive.", "Search done.") - def search(self, vault=None, region=None, file_name=None, search_term=None): + def search(self, vault=None, region=None, + file_name=None, search_term=None): """ Searches for archives using SimpleDB @@ -1484,7 +1522,7 @@ def search(self, vault=None, region=None, file_name=None, search_term=None): # Sanity checking. if not self.bookkeeping: - raise InputException( + raise glacierexception.InputException( "You must enable bookkeeping to be able to do searches.", cause='Bookkeeping not enabled.', code='BookkeepingError') @@ -1495,17 +1533,6 @@ def search(self, vault=None, region=None, file_name=None, search_term=None): if region: self._check_region(region) -## if file_name and ('"' in file_name or "'" in file_name): -## raise InputException( -## 'Quotes like \' and \" are not allowed in search terms.', -## cause='Invalid search term %s: contains quotes.'% file_name) -## -## -## if search_term and ('"' in search_term or "'" in search_term): -## raise InputException( -## 'Quotes like \' and \" are not allowed in search terms.', -## cause='Invalid search term %s: contains quotes.'% search_term) - self.logger.debug('Search terms: vault %s, region %s, file name %s, search term %s'% (vault, region, file_name, search_term)) search_params = [] @@ -1539,7 +1566,7 @@ def search(self, vault=None, region=None, file_name=None, search_term=None): if item.has_key('archive_id'): items.append(item) except boto.exception.SDBResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( 'SimpleDB did not like your query with parameters %s.'% search_params, cause=self._decode_error_message(e.body), code=e.code) @@ -1567,7 +1594,7 @@ def rmarchive(self, vault_name, archive_id): try: self.glacierconn.delete_archive(vault_name, archive_id) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( 'Failed to remove archive %s from vault %s.'% (archive_id, vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -1579,7 +1606,7 @@ def rmarchive(self, vault_name, archive_id): if item: self.sdb_domain.delete_item(item) except boto.exception.SDBResponseError as e: - raise CommunicationException( + raise glacierexception.CommunicationException( "Cannot delete item from Amazon SimpleDB.", code="SdbWriteError", cause=e) @@ -1654,20 +1681,24 @@ def inventory(self, vault_name, refresh): # If inventory retrieval is complete, process it. if inventory_done: - self.logger.debug('Fetching results of finished inventory retrieval.') + self.logger.debug("Fetching results of finished "\ + "inventory retrieval.") response = self.glacierconn.get_job_output(vault_name, inventory_job['JobId']) inventory = response.copy() archives = [] # If bookkeeping is enabled, update cache. - # Add all inventory information to the database, then check - # for any archives listed in the database for that vault and + # Add all inventory information to the database, + # then check for any archives listed + # in the database for that vault and # remove those. if self.bookkeeping and len(inventory['ArchiveList']) > 0: - self.logger.debug('Updating the bookkeeping with the latest inventory.') + self.logger.debug("Updating the bookkeeping "\ + "with the latest inventory.") items = {} - # Add items to the inventory, 25 at a time (maximum batch). + # Add items to the inventory, 25 at a time + # (maximum batch). for item in inventory['ArchiveList']: items[item['ArchiveId']] = { 'vault': vault_name, @@ -1680,12 +1711,15 @@ def inventory(self, vault_name, refresh): } archives.append(item['ArchiveId']) if len(items) == 25: - self.logger.debug('Writing batch of 25 inventory items to the bookkeeping db.') + self.logger.debug( + "Writing batch of 25 inventory "\ + "items to the bookkeeping db.") try: self.sdb_domain.batch_put_attributes(items) except boto.exception.SDBResponseError as e: - raise CommunicationException( - "Cannot update inventory cache, Amazon SimpleDB is not happy.", + raise glacierexception.CommunicationException( + "Cannot update inventory cache, "\ + "Amazon SimpleDB is not happy.", cause=e, code="SdbWriteError") items = {} @@ -1693,46 +1727,57 @@ def inventory(self, vault_name, refresh): # Add the remaining batch of items, if any, to the # database. if items: - self.logger.debug('Writing final batch of %s inventory items to the bookkeeping db.'% len(items)) + self.logger.debug("Writing final batch of %s "\ + "inventory items to the"\ + " bookkeeping db."% len(items)) try: self.sdb_domain.batch_put_attributes(items) except boto.exception.SDBResponseError as e: - raise CommunicationException( - "Cannot update inventory cache, Amazon SimpleDB is not happy.", + raise glacierexception.CommunicationException( + "Cannot update inventory cache, "\ + "Amazon SimpleDB is not happy.", cause=e, code="SdbWriteError") - # Get the inventory from the database for this vault, - # and delete any orphaned items. + # Get the inventory from the database for this + # vault, and delete any orphaned items. query = "select * from `%s` where vault='%s'" % (self.bookkeeping_domain_name, vault_name) result = self.sdb_domain.select(query) try: for item in result: if not item.name in archives: self.sdb_domain.delete_item(item) - self.logger.debug('Deleted orphaned archive from the database: %s.'% item.name) + self.logger.debug( + "Deleted orphaned "\ + "archive from the database: "\ + "{}.".format(item.name)) except boto.exception.SDBResponseError as e: - raise ResponseException( - 'SimpleDB did not respond correctly to our inventory check.', + raise glacierexception.ResponseException( + "SimpleDB did not respond correctly "\ + "to our inventory check.", cause=self._decode_error_message(e.body), code=e.code) - # If refresh == True or no current inventory jobs either finished or - # in progress, we have to start a new job. Then request the job details - # through describejob to return. + # If refresh == True or no current inventory jobs either + # finished or in progress, we have to start a new job. + # Then request the job details through describejob to return. if refresh or not inventory_job: - self.logger.debug('No inventory jobs finished or running; starting a new job.') + self.logger.debug("No inventory jobs finished or "\ + "running; starting a new job.") job_data = {'Type': 'inventory-retrieval'} try: - new_job = self.glacierconn.initiate_job(vault_name, job_data) + new_job = self.glacierconn.initiate_job(vault_name, + job_data) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to create a new inventory retrieval job for vault %s.'% vault_name, + raise glacierexception.ResponseException( + "Failed to create a new inventory retrieval "\ + "job for vault {}.".format(vault_name), cause=self._decode_error_message(e.body), code=e.code) - inventory_job = self.describejob(vault_name, new_job['JobId']) + inventory_job = self.describejob(vault_name, + new_job['JobId']) return (inventory_job, inventory) @@ -1750,7 +1795,7 @@ def get_tree_hash(self, file_name): try: reader = open(file_name, 'rb') except IOError as e: - raise InputException( + raise glacierexception.InputException( "Could not access the file given: %s." % file_name, cause=e, code='FileError') @@ -1779,8 +1824,10 @@ def _init_events_for_vaults(self, vaults, topic): result = dict() result["Vault Name"] = vault_name - result["Request Id"] = \ - self._init_events_for_vault(vault_name, topic)[u'RequestId'] + request_id = self._init_events_for_vault(vault_name, topic) + request_id = request_id[u'RequestId'] + result["Request Id"] = request_id + results += [result] return results @@ -1823,8 +1870,8 @@ def sns_sync(self, sns_options, output): try: protocol, endpoint = method.split(',') except ValueError: - raise InputException( - ("If you specify method, you should use format " + raise glacierexception.InputException( + ("If you specify method, you should ""use format " "'protocol1,endpoint1;protocol2,endpoint2'."), cause='Wrong method format in configuration file.', code='SNSConfigurationError') @@ -1872,7 +1919,7 @@ def sns_subscribe(self, protocol, endpoint, topic, sns_options, vault_names=None if vault_names: vaults = vault_names.split(",") self._init_events_for_vaults(vaults, topic_arn) - + topic_arns = [topic_arn] if len(topic_arns): @@ -1884,7 +1931,9 @@ def sns_subscribe(self, protocol, endpoint, topic, sns_options, vault_names=None 'RequestId': result['ResponseMetadata']['RequestId']}] return results except boto.exception.BotoServerError as e: - raise ResponseException("Failed to subscribe to notifications for vault %s." % vault_name, + raise glacierexception.ResponseException("Failed to subscribe to "\ + "notifications for vault"\ + " %s." % vault_name, cause=self._decode_error_message(e.body), code=e.code) else: @@ -1894,7 +1943,7 @@ def sns_subscribe(self, protocol, endpoint, topic, sns_options, vault_names=None @sns_connect def sns_list_topics(self, sns_options): topics = self.sns_conn.get_all_topics()['ListTopicsResponse']['ListTopicsResult']['Topics'] - + results = [] for topic in topics: results += [{"Topic":topic['TopicArn'].split(":")[-1], "Topic ARN":topic['TopicArn']}] @@ -1937,16 +1986,15 @@ def sns_list_subscriptions(self, protocol, endpoint, topic, sns_options): @sns_connect def sns_unsubscribe(self, protocol, endpoint, topic, sns_options): if not protocol or not endpoint or not topic: - raise InputException( + raise glacierexception.InputException( ("You must specify at least one parameter that will " "by which subscriptions will be canceled."), cause='No parameters given for unsubscribe.', code='SNSParameterError') matching_subs = self.sns_list_subscriptions(protocol, - endpoint, - topic, - sns_options=sns_options) + endpoint, topic, + sns_options=sns_options) unsubscribed = [] for res in matching_subs: @@ -1957,8 +2005,9 @@ def sns_unsubscribe(self, protocol, endpoint, topic, sns_options): return unsubscribed def __init__(self, aws_access_key, aws_secret_key, region, - bookkeeping=False, no_bookkeeping=None, bookkeeping_domain_name=None, - sdb_access_key=None, sdb_secret_key=None, sdb_region=None, + bookkeeping=False, no_bookkeeping=None, + bookkeeping_domain_name=None, sdb_access_key=None, + sdb_secret_key=None, sdb_region=None, logfile=None, loglevel='WARNING', logtostdout=True): """ Constructor, sets up important variables and so for GlacierWrapper. diff --git a/glacier/modules/__init__.py b/glacier/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/glacier/constants.py b/glacier/modules/constants.py similarity index 81% rename from glacier/constants.py rename to glacier/modules/constants.py index 5fee23f..f6ea8fc 100644 --- a/glacier/constants.py +++ b/glacier/modules/constants.py @@ -1,4 +1,3 @@ - # constants definition CHUNK_SIZE = 1024 DEFAULT_PART_SIZE = 128 # in MB, power of 2. @@ -15,11 +14,11 @@ # For large files, the limits above could be surpassed. We also set a per-Gb # criteria that allows more errors for larger uploads. MAX_TOTAL_RETRY_PER_GB = 2 -HEADERS_OUTPUT_FORMAT = ["csv","json"] +HEADERS_OUTPUT_FORMAT = ["csv","json", "print"] TABLE_OUTPUT_FORMAT = ["csv","json", "print"] SYSTEM_WIDE_CONFIG_FILENAME = "/etc/glacier-cmd.conf" USER_CONFIG_FILENAME = "~/.glacier-cmd" -HELP_MESSAGE_CONFIG = u"(Required if you have not created a " +HELP_MESSAGE_CONFIG = "(Required if you have not created a "\ "~/.glacier-cmd or /etc/glacier-cmd.conf config file)" ERRORCODE = {'InternalError': 127, # Library internal error. 'UndefinedErrorCode': 126, # Undefined code. @@ -48,20 +47,31 @@ MAX_VAULT_NAME_LENGTH = 255 MAX_VAULT_DESCRIPTION_LENGTH = 1024 MAX_PARTS = 10000 -AVAILABLE_REGIONS = ('us-east-1', 'us-west-2', 'us-west-1', - 'eu-west-1', 'eu-central-1', 'sa-east-1', - 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2') +AVAILABLE_REGIONS = ('us-east-1', 'us-east-2', + 'us-west-2', 'us-west-1', + 'ca-central-1' + 'eu-west-1', 'eu-central-1', 'eu-west-2', + 'ap-south-1', + 'ap-northeast-1','ap-northeast-2', + 'ap-southeast-1', 'ap-southeast-2', + 'cn-north-1') AVAILABLE_REGIONS_MESSAGE = """\ Invalid region. Available regions for Amazon Glacier are: us-east-1 (US - Virginia) + us-east-2 (US - Ohio) us-west-1 (US - N. California) us-west-2 (US - Oregon) eu-west-1 (EU - Ireland) + eu-west-2 (EU - London) + ca-central-1 (CA - Canada) eu-central-1 (EU - Frankfurt) sa-east-1 (South America - Sao Paulo) + ap-south-1 (Mumbai) ap-northeast-1 (Asia-Pacific - Tokyo) + ap-northeast-2 (Asia-Pacific - Seul) ap-southeast-1 (Asia Pacific (Singapore) - ap-southeast-2 (Asia-Pacific - Sydney)\ + ap-southeast-2 (Asia-Pacific - Sydney) + cn-north-1 (China - Beijing) """ diff --git a/glacier/modules/decorators.py b/glacier/modules/decorators.py new file mode 100644 index 0000000..dc6ebd8 --- /dev/null +++ b/glacier/modules/decorators.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# import modules +from functools import wraps +import sys +# import our modules +from modules import glacierexception + +def handle_errors(fn): + """ + Decorator for exception handling. + """ + + @wraps(fn) + def wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + except glacierexception.GlacierException as e: + + # We are only interested in the error message in case + # it is a self-caused exception. + e.write(indentation='|| ', stack=False, message=True) + sys.exit(e.exitcode) + + return wrapper diff --git a/glacier/modules/glacier_utils.py b/glacier/modules/glacier_utils.py new file mode 100644 index 0000000..26bd2ea --- /dev/null +++ b/glacier/modules/glacier_utils.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# import modules +import csv +import json +import sys +from prettytable import PrettyTable + + +# import our modules +from modules.GlacierWrapper import GlacierWrapper +from modules import constants + +# function definition +def default_glacier_wrapper(args, **kwargs): + """ + Convenience function to call an instance of GlacierWrapper + with all required arguments. + """ + return GlacierWrapper(args.aws_access_key, + args.aws_secret_key, + args.region, + bookkeeping=args.bookkeeping, + no_bookkeeping=args.no_bookkeeping, + bookkeeping_domain_name=args.bookkeeping_domain_name, + sdb_access_key=args.sdb_access_key, + sdb_secret_key=args.sdb_secret_key, + sdb_region=args.sdb_region, + # sns_enable=args.sns_enable, + # sns_topic=args.sns_topic, + # sns_monitored_vaults=args.sns_monitored_vaults, + # sns_options=args.sns_options, + # config_object=args.config_object, + logfile=args.logfile, + loglevel=args.loglevel, + logtostdout=args.logtostdout) + +def output_msg(msg, output, success=True): + """ + In case of a single message output, e.g. nothing found. + + :param msg: a single message to output. + :type msg: str + :param success: whether the operation was a success or not. + :type success: boolean + :param output: output format + :type output: string + :raises: ValueError if output not in constants.TABLE_OUTPUT_FORMAT + :returns :nothing, prints the output or sys.exist with status 125 + """ + + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be{}, " + "got {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) + + if msg is not None: + if output == 'print': + print msg + + if output == 'csv': + csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) + csvwriter.writerow(msg) + + if output == 'json': + print json.dumps(msg) + + if not success: + sys.exit(125) + +def size_fmt(num, decimals = 1): + """ + Formats file sizes in human readable format. Anything bigger than + TB is returned is TB. Number of decimals is optional, + defaults to 1. + + :param num: value to be converted + :type num: int + :decimals: number of decimals to consider + :type decimals: int + :returns: formatted number + :returns type: string + """ + + fmt = "%%3.%sf %%s"% decimals + for x in ['bytes','KB','MB','GB']: + if num < 1024.0: + return fmt % (num, x) + + num /= 1024.0 + + return fmt % (num, 'TB') + + +def output_headers(headers, output): + """ + Prints a list of headers - single item output. + + :param headers: the output to be printed as {'header1':'data1',...} + :type headers: dict + :param output: selects output type, must be + listed in constants.HEADERS_OUTPUT_FORMAT + :type output: string + :raises: ValueError when output format is not supported + """ + + if output not in constants.HEADERS_OUTPUT_FORMAT: + raise ValueError("Output must be in {}, got" + ":{}".format(constants.HEADERS_OUTPUT_FORMAT, + output)) + rows = [(k, headers[k]) for k in headers.keys()] + if output == 'print': + table = PrettyTable(["Header", "Value"]) + for row in rows: + if len(str(row[1])) <= 138: + table.add_row(row) + + print table + + if output == 'csv': + csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) + for row in rows: + csvwriter.writerow(row) + + if output == 'json': + print json.dumps(headers) + +def output_table(results, output, keys=None, sort_key=None): + """ + Prettyprints results. Expects a list of identical dicts. + Use the dict keys as headers unless keys is given; + one line for each item. + + Expected format of data is a list of dicts: + [{'key1':'data1.1', 'key2':'data1.2', ... }, + {'key1':'data1.2', 'key2':'data2.2', ... }, + ...] + :param keys: dict of headers to be printed for each key: + {'key1':'header1', 'key2':'header2',...} + :type keys: dict + :param sort_key: the key to use for sorting the table. + :type sort_key: string + :raises : ValueError if output not in constants.TABLE_OUTPUT_FORMAT + :returns: nothing, prints the output on the console + """ + + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be{}, " + "got {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) + if output == 'print': + if len(results) == 0: + print 'No output!' + return + + headers = [keys[k] for k in keys.keys()] if keys else results[0].keys() + table = PrettyTable(headers) + for line in results: + table.add_row([line[k] if k in line else '' for k in (keys.keys() if keys else headers)]) + + if sort_key: + table.sortby = keys[sort_key] if keys else sort_key + + print table + + if output == 'csv': + csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) + keys = results[0].keys() + csvwriter.writerow(keys) + for row in results: + csvwriter.writerow([row[k] for k in keys]) + + if output == 'json': + print json.dumps(results) diff --git a/glacier/glaciercorecalls.py b/glacier/modules/glaciercorecalls.py similarity index 56% rename from glacier/glaciercorecalls.py rename to glacier/modules/glaciercorecalls.py index c9f2b4c..2f83cba 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/modules/glaciercorecalls.py @@ -12,32 +12,35 @@ writer.write(block of data) writer.close() # Get the id of the newly created archive - archive_id = writer.get_archive_id()from boto.connection import AWSAuthConnection + archive_id = writer.get_archive_id()from boto.connection + import AWSAuthConnection """ +# import modules + import urllib import hashlib import math import json import sys import time - import boto.glacier.layer1 -from glacierexception import * +# import our modules +from modules import constants +from modules import glacierexception # Placeholder, effectively renaming the class. class GlacierConnection(boto.glacier.layer1.Layer1): pass - def chunk_hashes(data): """ Break up the byte-string into 1MB chunks and return sha256 hashes for each. """ - chunk = 1024*1024 + chunk = constants.CHUNK_SIZE*constants.CHUNK_SIZE chunk_count = int(math.ceil(len(data)/float(chunk))) return [hashlib.sha256(data[i*chunk:(i+1)*chunk]).digest() for i in range(chunk_count)] @@ -72,55 +75,47 @@ class GlacierWriter(object): Presents a file-like object for writing to a Amazon Glacier Archive. The data is written using the multi-part upload API. """ - DEFAULT_PART_SIZE = 128 # in MB, power of 2. - # After every failed block upload we sleep (SLEEP_TIME * retries) seconds. - # The more retries we've made for one particular block, the longer we sleep - # before re-attempting to re-upload that block. - SLEEP_TIME = 300 - # How many retries we should make to upload a particular block. We will not - # give up unless we've made at LEAST this many attempts to upload a block. - BLOCK_RETRIES = 10 - # How many retries we should allow for the whole upload. We will not give up - # unless we've made at LEAST this many attempts to upload the archive. - TOTAL_RETRIES = 100 - # For large files, the limits above could be surpassed. We also set a per-Gb - # criteria that allows more errors for larger uploads. - MAX_TOTAL_RETRY_PER_GB = 2 + def __init__(self, connection, vault_name, - description=None, part_size_in_bytes=DEFAULT_PART_SIZE*1024*1024, + description=None, + part_size_in_bytes=constants.DEFAULT_PART_SIZE*1024*1024, uploadid=None, logger=None): self.part_size = part_size_in_bytes self.vault_name = vault_name self.connection = connection -## self.location = None self.logger = logger self.total_retries = 0 if uploadid: self.uploadid = uploadid else: - response = self.connection.initiate_multipart_upload(self.vault_name, - self.part_size, - description) + response = self.connection.initiate_multipart_upload( + self.vault_name, + self.part_size, + description) self.uploadid = response['UploadId'] self.uploaded_size = 0 self.tree_hashes = [] self.closed = False -## self.upload_url = response.getheader("location") def write(self, data): if self.closed: - raise CommunicationError( - "Tried to write to a GlacierWriter that is already closed.", + raise glacierexception.CommunicationError( + "Tried to write to a GlacierWriter that is "\ + "already closed.", code='InternalError') - if len(data) > self.part_size: - raise InputException ( - 'Block of data provided must be equal to or smaller than the set block size.', + len_data = len(data) + if len_data > self.part_size: + raise glacierexception.InputException ( + "Block of data provided({}) "\ + "must be equal " + "to or smaller than the set " + "block size({}).".format(len_data, self.part_size), code='InternalError') part_tree_hash = tree_hash(chunk_hashes(data)) @@ -128,8 +123,8 @@ def write(self, data): headers = { "x-amz-glacier-version": "2012-06-01", "Content-Range": "bytes %d-%d/*" % (self.uploaded_size, - (self.uploaded_size+len(data))-1), - "Content-Length": str(len(data)), + (self.uploaded_size+len_data)-1), + "Content-Length": str(len_data), "Content-Type": "application/octet-stream", "x-amz-sha256-tree-hash": bytes_to_hex(part_tree_hash), "x-amz-content-sha256": hashlib.sha256(data).hexdigest() @@ -140,30 +135,42 @@ def write(self, data): while True: try: - response = self.connection.upload_part(self.vault_name, - self.uploadid, - hashlib.sha256(data).hexdigest(), - bytes_to_hex(part_tree_hash), - (self.uploaded_size, self.uploaded_size+len(data)-1), - data) + response = self.connection.upload_part( + self.vault_name, + self.uploadid, + hashlib.sha256(data).hexdigest(), + bytes_to_hex(part_tree_hash), + (self.uploaded_size, self.uploaded_size+len_data-1), + data) response.read() break except Exception as e: - if '408' in e.message or e.code == "ServiceUnavailableException" or e.type == "Server": - uploaded_gb = self.uploaded_size / (1024 * 1024 * 1024) - if retries >= self.BLOCK_RETRIES and retries > math.log10(uploaded_gb) * 10: + if ('408' in e.message or + e.code == "ServiceUnavailableException" or + e.type == "Server"): + + uploaded_gb = self.uploaded_size / (1024 ** 1024) + if (retries >= constants.BLOCK_RETRIES and + retries > math.log10(uploaded_gb) * 10): + if self.logger: - self.logger.warning('Retries exhausted for this block.') + self.logger.warning("Retries exhausted "\ + "({}) for this "\ + "block.".format(constants.BLOCK_RETRIES)) raise e if uploaded_gb > 0: retry_per_gb = self.total_retries / uploaded_gb else: retry_per_gb = 0 - if self.total_retries >= self.TOTAL_RETRIES and retry_per_gb > self.MAX_TOTAL_RETRY_PER_GB: + if (self.total_retries >= constants.TOTAL_RETRIES and + retry_per_gb > constants.MAX_TOTAL_RETRY_PER_GB): + if self.logger: - self.logger.warning('Total retries exhausted.') + self.logger.warning("Total retries "\ + "exhausted"\ + "({}).".format(constants.TOTAL_RETRIES)) raise e retries += 1 @@ -172,16 +179,27 @@ def write(self, data): if self.logger: self.logger.warning(e.message) if sys.version_info < (2, 7, 0): - self.logger.warning('Total uploaded size = %d, block hash = %s' % (self.uploaded_size, bytes_to_hex(part_tree_hash))) + self.logger.warning("Total uploaded size " + "= {}, block hash = " + "{}".format(self.uploaded_size, + bytes_to_hex(part_tree_hash))) else: # Commify large numbers - self.logger.warning('Total uploaded size = {:,d}, block hash = {:}'.format(self.uploaded_size, bytes_to_hex(part_tree_hash))) - - self.logger.warning('Retries (this block, total) = %d/%d, %d/%d' % (retries, self.BLOCK_RETRIES, self.total_retries, self.TOTAL_RETRIES)) - self.logger.warning('Check the AWS status at: http://status.aws.amazon.com/') - self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (self.SLEEP_TIME, self.SLEEP_TIME / 60.0)) - - time.sleep(self.SLEEP_TIME * retries) + self.logger.warning("Total uploaded size " + "= {:,d}, block hash = " + "{:}".format(self.uploaded_size, + bytes_to_hex(part_tree_hash))) + + self.logger.warning("Retries (this block, " + "total) = {}/{}, {}/{}".format(retries, + constants.BLOCK_RETRIES, + self.total_retries, + constants.TOTAL_RETRIES)) + self.logger.warning("Check the AWS status " + "at: http://status.aws.amazon.com/") + self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (constants.SLEEP_TIME, constants.SLEEP_TIME / 60.0)) + + time.sleep(constants.SLEEP_TIME * retries) else: self.logger.warning(e.message) @@ -196,10 +214,11 @@ def close(self): return # Complete the multiplart glacier upload - response = self.connection.complete_multipart_upload(self.vault_name, - self.uploadid, - bytes_to_hex(tree_hash(self.tree_hashes)), - self.uploaded_size) + response = self.connection.complete_multipart_upload( + self.vault_name, + self.uploadid, + bytes_to_hex(tree_hash(self.tree_hashes)), + self.uploaded_size) self.archive_id = response['ArchiveId'] self.location = response['Location'] self.hash_sha256 = bytes_to_hex(tree_hash(self.tree_hashes)) diff --git a/glacier/glacierexception.py b/glacier/modules/glacierexception.py similarity index 71% rename from glacier/glacierexception.py rename to glacier/modules/glacierexception.py index 960c27e..43ba125 100644 --- a/glacier/glacierexception.py +++ b/glacier/modules/glacierexception.py @@ -1,7 +1,5 @@ -import traceback -import re -import sys -import logging +#!/usr/bin/env python +# encoding: utf-8 """ @@ -9,11 +7,22 @@ Note by wvmarle: This file contains the complete code from chained_exception.py plus the -error handling code from GlacierWrapper.py, allowing it to be used in other -modules like glaciercorecalls as well. +error handling code from GlacierWrapper.py, allowing it to be +used in other modules like glaciercorecalls as well. ********** """ + +# import modules +import traceback +import re +import sys +import logging + +# import our modules +from modules import constants + + class GlacierException(Exception): """ An extension of the built-in Exception class, this handles @@ -27,29 +36,8 @@ class GlacierException(Exception): TODO: describe usage. """ - ERRORCODE = {'InternalError': 127, # Library internal error. - 'UndefinedErrorCode': 126, # Undefined code. - 'NoResults': 125, # Operation yielded no results. - 'GlacierConnectionError': 1, # Can not connect to Glacier. - 'SdbConnectionError': 2, # Can not connect to SimpleDB. - 'CommandError': 3, # Command line is invalid. - 'VaultNameError': 4, # Invalid vault name. - 'DescriptionError': 5, # Invalid archive description. - 'IdError': 6, # Invalid upload/archive/job ID given. - 'RegionError': 7, # Invalid region given. - 'FileError': 8, # Error related to reading/writing a file. - 'ResumeError': 9, # Problem resuming a multipart upload. - 'NotReady': 10, # Requested download is not ready yet. - 'BookkeepingError': 11, # Bookkeeping not available. - 'SdbCommunicationError': 12, # Problem reading/writing SimpleDB data. - 'ResourceNotFoundException': 13, # Glacier can not find the requested resource. - 'InvalidParameterValueException': 14, # Parameter not accepted. - 'DownloadError': 15, # Downloading an archive failed. - 'SNSConnectionError': 126, # Can not connect to SNS - 'SNSConfigurationError': 127, # Problem with configuration file - 'SNSParameterError':128, # Problem with arguments passed to SNS - } - + + def __init__(self, message, code=None, cause=None): """ Constructor. Logs the error. @@ -62,10 +50,10 @@ def __init__(self, message, code=None, cause=None): :type cause: str """ self.logger = logging.getLogger(self.__class__.__name__) - self.exitcode = self.ERRORCODE[code] if code in self.ERRORCODE else 254 + self.exitcode = constants.ERRORCODE[code] if code in constants.ERRORCODE else 254 self.code = code if cause: - self.logger.error('ERROR: %s'% cause) + self.logger.error('ERROR:{}'.format(cause)) self.cause = cause if isinstance(cause, tuple) else (cause,) self.stack = traceback.format_stack()[:-2] @@ -78,18 +66,19 @@ def __init__(self, message, code=None, cause=None): self.stack = ( traceback.format_stack()[:-2] + traceback.format_tb(sys.exc_info()[2])) - # ^^^ let's hope the information is still there; caller must take - # care of this. - + # ^^^ let's hope the information is still there; + # caller must take care of this. + self.message = message self.logger.info(self.fetch(message=True)) self.logger.debug(self.fetch(stack=True)) if self.exitcode == 254: - self.logger.debug('Unknown error code: %s.'% code) + self.logger.debug('Unknown error code: {}'.format(code)) # Works as a generator to help get the stack trace and the cause # written out. - def causeTree(self, indentation=' ', alreadyMentionedTree=[], stack=False, message=False): + def causeTree(self, indentation=' ', alreadyMentionedTree=[], + stack=False, message=False): """ Returns a complete stack tree, an error message, or both. Returns a warning if neither stack or message are True. @@ -108,18 +97,19 @@ def causeTree(self, indentation=' ', alreadyMentionedTree=[], stack=False, mess ellipsed, "" if ellipsed == 1 else "s") ellipsed = False # marker for "given out" - + yield line if message: exc = self if self.message is None else self.message - for line in traceback.format_exception_only(exc.__class__, exc): + for line in traceback.format_exception_only(exc.__class__, + exc): yield line - + if self.cause: yield ("Caused by: %d exception%s\n" % (len(self.cause), "" if len(self.cause) == 1 else "s")) - + for causePart in self.cause: if hasattr(causePart,"causeTree"): for line in causePart.causeTree(indentation, self.stack): @@ -129,16 +119,19 @@ def causeTree(self, indentation=' ', alreadyMentionedTree=[], stack=False, mess yield re.sub(r'([^\n]*\n)', indentation + r'\1', line) if not message and not stack: - yield ('No output. Specify message=True and/or stack=True \ -to get output when calling this function.\n') + yield ("No output. Specify message=True and/or stack=True " + "to get output when calling this function.\n") - def write(self, stream=None, indentation=' ', message=False, stack=False): + def write(self, stream=None, indentation=' ', + message=False, stack=False): """ Writes the error details to sys.stderr or a stream. """ - + stream = sys.stderr if stream is None else stream - for line in self.causeTree(indentation, message=message, stack=stack): + for line in self.causeTree(indentation, + message=message, + stack=stack): stream.write(line) def fetch(self, indentation=' ', message=False, stack=False): @@ -146,7 +139,9 @@ def fetch(self, indentation=' ', message=False, stack=False): Fetches the error details and returns them as string. """ out = '' - for line in self.causeTree(indentation, message=message, stack=stack): + for line in self.causeTree(indentation, + message=message, + stack=stack): out += line return out @@ -156,7 +151,7 @@ class InputException(GlacierException): Exception that is raised when there is someting wrong with the user input. """ - + VaultNameError = 1 VaultDescriptionError = 2 def __init__(self, message, code=None, cause=None): @@ -165,7 +160,7 @@ def __init__(self, message, code=None, cause=None): :param message: the error message. :type message: str :param code: the error code. - :type code: + :type code: :param cause: explanation on what caused the error. :type cause: str """ @@ -176,7 +171,7 @@ class ConnectionException(GlacierException): Exception that is raised when there is something wrong with the connection. """ - + GlacierConnectionError = 1 SdbConnectionError = 2 def __init__(self, message, code=None, cause=None): @@ -185,7 +180,7 @@ def __init__(self, message, code=None, cause=None): :param message: the error message. :type message: str :param code: the error code. - :type code: + :type code: :param cause: explanation on what caused the error. :type cause: str """ @@ -202,7 +197,7 @@ def __init__(self, message, code=None, cause=None): :param message: the error message. :type message: str :param code: the error code. - :type code: + :type code: :param cause: explanation on what caused the error. :type cause: str """ diff --git a/glacier/modules/sns_utils.py b/glacier/modules/sns_utils.py new file mode 100644 index 0000000..56377e5 --- /dev/null +++ b/glacier/modules/sns_utils.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# import modules +from modules.glacier_utils import default_glacier_wrapper + +# function definition + +def snssync(args): + """ + If monitored_vaults is specified in configuration file, subscribe + vaults specificed in it to notifications, otherwiser + subscribe all vault. + """ + + glacier = default_glacier_wrapper(args) + response = glacier.sns_sync(sns_options=args.sns_options, + output=args.output) + output_table(response, args.output) + +def snssubscribe(args): + """ + Subscribe individual vaults to notifications by method + specified by user. + """ + + protocol = args.protocol + endpoint = args.endpoint + vault_names = args.vault + topic = args.topic + + glacier = default_glacier_wrapper(args) + response = glacier.sns_subscribe(protocol, endpoint, topic, + vault_names=vault_names, + sns_options=args.sns_options) + output_table(response, args.output) + +def snslistsubscriptions(args): + """ + List subscriptions. + """ + + protocol = args.protocol + endpoint = args.endpoint + topic = args.topic + + glacier = default_glacier_wrapper(args) + response = glacier.sns_list_subscriptions(protocol, + endpoint, topic, sns_options=args.sns_options) + output_table(response, args.output) + +def snslisttopics(args): + """ + List subscriptions. + """ + + glacier = default_glacier_wrapper(args) + response = glacier.sns_list_topics(sns_options=args.sns_options) + output_table(response, args.output) + +def snsunsubscribe(args): + """ + Unsubscribe individual vaults from notifications for + specified protocol, endpoint and vault. + """ + + protocol = args.protocol + endpoint = args.endpoint + topic = args.topic + + glacier = default_glacier_wrapper(args) + response = glacier.sns_unsubscribe(protocol, endpoint, + topic, sns_options=args.sns_options) + output_table(response, args.output)