From b9187db7a2ff2ddca2c97d016c3da64810ea3195 Mon Sep 17 00:00:00 2001 From: marzona Date: Sat, 10 Jun 2017 11:10:53 +0100 Subject: [PATCH 1/6] - added/updated docstrings - updated/fixed constants values. --- glacier/constants.py | 4 ++-- glacier/glacier.py | 50 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/glacier/constants.py b/glacier/constants.py index 5fee23f..20cb723 100644 --- a/glacier/constants.py +++ b/glacier/constants.py @@ -15,11 +15,11 @@ # For large files, the limits above could be surpassed. We also set a per-Gb # criteria that allows more errors for larger uploads. MAX_TOTAL_RETRY_PER_GB = 2 -HEADERS_OUTPUT_FORMAT = ["csv","json"] +HEADERS_OUTPUT_FORMAT = ["csv","json", "print"] TABLE_OUTPUT_FORMAT = ["csv","json", "print"] SYSTEM_WIDE_CONFIG_FILENAME = "/etc/glacier-cmd.conf" USER_CONFIG_FILENAME = "~/.glacier-cmd" -HELP_MESSAGE_CONFIG = u"(Required if you have not created a " +HELP_MESSAGE_CONFIG = "(Required if you have not created a "\ "~/.glacier-cmd or /etc/glacier-cmd.conf config file)" ERRORCODE = {'InternalError': 127, # Library internal error. 'UndefinedErrorCode': 126, # Undefined code. diff --git a/glacier/glacier.py b/glacier/glacier.py index d55c97a..9ea5a58 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -29,7 +29,16 @@ def output_headers(headers, output): :param headers: the output to be printed as {'header1':'data1',...} :type headers: dict + :param output: selects output type, must be + listed in constants.HEADERS_OUTPUT_FORMAT + :type output: string + :raises: ValueError when output format is not supported """ + + if output not in constants.HEADERS_OUTPUT_FORMAT: + raise ValueError("Output must be in {}, got" + ":{}".format(constants.HEADERS_OUTPUT_FORMAT, + output)) rows = [(k, headers[k]) for k in headers.keys()] if output == 'print': table = PrettyTable(["Header", "Value"]) @@ -39,11 +48,6 @@ def output_headers(headers, output): print table - if output not in constants.HEADERS_OUTPUT_FORMAT: - raise ValueError("Output format must be {}, got" - ":{}".format(constants.HEADERS_OUTPUT_FORMAT, - output)) - if output == 'csv': csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) for row in rows: @@ -62,10 +66,13 @@ def output_table(results, output, keys=None, sort_key=None): [{'key1':'data1.1', 'key2':'data1.2', ... }, {'key1':'data1.2', 'key2':'data2.2', ... }, ...] - keys: dict of headers to be printed for each key: + :param keys: dict of headers to be printed for each key: {'key1':'header1', 'key2':'header2',...} - - sort_key: the key to use for sorting the table. + :type keys: dict + :param sort_key: the key to use for sorting the table. + :type sort_key: string + :raises : ValueError if output not in constants.TABLE_OUTPUT_FORMAT + :returns: nothing, prints the output on the console """ if output not in constants.TABLE_OUTPUT_FORMAT: @@ -105,6 +112,10 @@ def output_msg(msg, output, success=True): :type msg: str :param success: whether the operation was a success or not. :type success: boolean + :param output: output format + :type output: string + :raises: ValueError if output not in constants.TABLE_OUTPUT_FORMAT + :returns :nothing, prints the output or sys.exist with status 125 """ if output not in constants.TABLE_OUTPUT_FORMAT: @@ -131,7 +142,15 @@ def size_fmt(num, decimals = 1): Formats file sizes in human readable format. Anything bigger than TB is returned is TB. Number of decimals is optional, defaults to 1. + + :param num: value to be converted + :type num: int + :decimals: number of decimals to consider + :type decimals: int + :returns: formatted number + :returns type: string """ + fmt = "%%3.%sf %%s"% decimals for x in ['bytes','KB','MB','GB']: if num < 1024.0: @@ -208,6 +227,7 @@ def rmvault(args): """ Remove a vault. """ + glacier = default_glacier_wrapper(args) response = glacier.rmvault(args.vault) output_headers(response, args.output) @@ -217,6 +237,7 @@ def describevault(args): """ Give the description of a vault. """ + glacier = default_glacier_wrapper(args) response = glacier.describevault(args.vault) headers = {'LastInventoryDate': "LastInventory", @@ -231,6 +252,7 @@ def listmultiparts(args): """ Give an overview of all multipart uploads that are not finished. """ + glacier = default_glacier_wrapper(args) response = glacier.listmultiparts(args.vault) if not response: @@ -244,6 +266,7 @@ def abortmultipart(args): """ Abort a multipart upload which is in progress. """ + glacier = default_glacier_wrapper(args) response = glacier.abortmultipart(args.vault, args.uploadId) output_headers(response, args.output) @@ -253,6 +276,7 @@ def listjobs(args): """ List all the active jobs for a vault. """ + glacier = default_glacier_wrapper(args) job_list = glacier.list_jobs(args.vault) if job_list == []: @@ -281,6 +305,7 @@ def download(args): """ Download an archive. """ + glacier = default_glacier_wrapper(args) response = glacier.download(args.vault, args.archive, args.partsize, out_file_name=args.outfile, @@ -368,6 +393,7 @@ def getarchive(args): """ Initiate an archive retrieval job. """ + glacier = default_glacier_wrapper(args) status, job, jobid = glacier.getarchive(args.vault, args.archive) output_headers(job, args.output) @@ -377,6 +403,7 @@ def rmarchive(args): """ Remove an archive from a vault. """ + glacier = default_glacier_wrapper(args) glacier.rmarchive(args.vault, args.archive) output_msg("Archive removed.", args.output, success=True) @@ -386,6 +413,7 @@ def search(args): """ Search the database for file name or description. """ + glacier = default_glacier_wrapper(args) response = glacier.search(vault=args.vault, region=args.region, @@ -398,6 +426,7 @@ def inventory(args): """ Fetch latest inventory (or start a retrieval job if not ready). """ + glacier = default_glacier_wrapper(args) output = args.output if sys.stdout.isatty() and output == 'print': @@ -439,6 +468,7 @@ def treehash(args): """ Calculates the tree hash of the given file(s). """ + glacier = default_glacier_wrapper(args) hash_results = [] for f in args.filename: @@ -467,6 +497,7 @@ def snssync(args): vaults specificed in it to notifications, otherwiser subscribe all vault. """ + glacier = default_glacier_wrapper(args) response = glacier.sns_sync(sns_options=args.sns_options, output=args.output) @@ -477,6 +508,7 @@ def snssubscribe(args): Subscribe individual vaults to notifications by method specified by user. """ + protocol = args.protocol endpoint = args.endpoint vault_names = args.vault @@ -492,6 +524,7 @@ def snslistsubscriptions(args): """ List subscriptions. """ + protocol = args.protocol endpoint = args.endpoint topic = args.topic @@ -512,6 +545,7 @@ def snsunsubscribe(args): Unsubscribe individual vaults from notifications for specified protocol, endpoint and vault. """ + protocol = args.protocol endpoint = args.endpoint topic = args.topic From a6ea7ea75e5602139eb6d16bbb4bd05fbfce1d4e Mon Sep 17 00:00:00 2001 From: marzona Date: Sun, 11 Jun 2017 17:22:24 +0100 Subject: [PATCH 2/6] Still working mainly on glacier.py - created a sns_utils.py file with sns related code - created a glacier_utils.py file with glacier helper functions - created modules folder with all the files except glacier.py - started to move all the constants into constants.py --- glacier/glacier.py | 259 +--------------- glacier/{ => modules}/GlacierWrapper.py | 342 ++++++++++++---------- glacier/modules/__init__.py | 0 glacier/{ => modules}/constants.py | 0 glacier/modules/decorators.py | 25 ++ glacier/modules/glacier_utils.py | 175 +++++++++++ glacier/{ => modules}/glaciercorecalls.py | 103 ++++--- glacier/{ => modules}/glacierexception.py | 93 +++--- glacier/modules/sns_utils.py | 68 +++++ 9 files changed, 568 insertions(+), 497 deletions(-) rename glacier/{ => modules}/GlacierWrapper.py (88%) create mode 100644 glacier/modules/__init__.py rename glacier/{ => modules}/constants.py (100%) create mode 100644 glacier/modules/decorators.py create mode 100644 glacier/modules/glacier_utils.py rename glacier/{ => modules}/glaciercorecalls.py (62%) rename glacier/{ => modules}/glacierexception.py (72%) create mode 100644 glacier/modules/sns_utils.py diff --git a/glacier/glacier.py b/glacier/glacier.py index 9ea5a58..8d7d3d9 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -6,6 +6,8 @@ :synopsis: Command line interface for amazon glacier """ +# import modules + import sys import os import ConfigParser @@ -13,192 +15,16 @@ import re import locale import glob -import csv -import json - -import glacierexception -import constants - -from prettytable import PrettyTable -from GlacierWrapper import GlacierWrapper -from functools import wraps - -def output_headers(headers, output): - """ - Prints a list of headers - single item output. - - :param headers: the output to be printed as {'header1':'data1',...} - :type headers: dict - :param output: selects output type, must be - listed in constants.HEADERS_OUTPUT_FORMAT - :type output: string - :raises: ValueError when output format is not supported - """ - - if output not in constants.HEADERS_OUTPUT_FORMAT: - raise ValueError("Output must be in {}, got" - ":{}".format(constants.HEADERS_OUTPUT_FORMAT, - output)) - rows = [(k, headers[k]) for k in headers.keys()] - if output == 'print': - table = PrettyTable(["Header", "Value"]) - for row in rows: - if len(str(row[1])) <= 138: - table.add_row(row) - - print table - - if output == 'csv': - csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - for row in rows: - csvwriter.writerow(row) - - if output == 'json': - print json.dumps(headers) - -def output_table(results, output, keys=None, sort_key=None): - """ - Prettyprints results. Expects a list of identical dicts. - Use the dict keys as headers unless keys is given; - one line for each item. - - Expected format of data is a list of dicts: - [{'key1':'data1.1', 'key2':'data1.2', ... }, - {'key1':'data1.2', 'key2':'data2.2', ... }, - ...] - :param keys: dict of headers to be printed for each key: - {'key1':'header1', 'key2':'header2',...} - :type keys: dict - :param sort_key: the key to use for sorting the table. - :type sort_key: string - :raises : ValueError if output not in constants.TABLE_OUTPUT_FORMAT - :returns: nothing, prints the output on the console - """ - - if output not in constants.TABLE_OUTPUT_FORMAT: - raise ValueError("Output format must be{}, " - "got {}".format(constants.TABLE_OUTPUT_FORMAT, - output)) - if output == 'print': - if len(results) == 0: - print 'No output!' - return - - headers = [keys[k] for k in keys.keys()] if keys else results[0].keys() - table = PrettyTable(headers) - for line in results: - table.add_row([line[k] if k in line else '' for k in (keys.keys() if keys else headers)]) - - if sort_key: - table.sortby = keys[sort_key] if keys else sort_key - - print table - - if output == 'csv': - csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - keys = results[0].keys() - csvwriter.writerow(keys) - for row in results: - csvwriter.writerow([row[k] for k in keys]) - - if output == 'json': - print json.dumps(results) - -def output_msg(msg, output, success=True): - """ - In case of a single message output, e.g. nothing found. - - :param msg: a single message to output. - :type msg: str - :param success: whether the operation was a success or not. - :type success: boolean - :param output: output format - :type output: string - :raises: ValueError if output not in constants.TABLE_OUTPUT_FORMAT - :returns :nothing, prints the output or sys.exist with status 125 - """ - - if output not in constants.TABLE_OUTPUT_FORMAT: - raise ValueError("Output format must be{}, " - "got {}".format(constants.TABLE_OUTPUT_FORMAT, - output)) - - if msg is not None: - if output == 'print': - print msg - - if output == 'csv': - csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - csvwriter.writerow(msg) - - if output == 'json': - print json.dumps(msg) - - if not success: - sys.exit(125) - -def size_fmt(num, decimals = 1): - """ - Formats file sizes in human readable format. Anything bigger than - TB is returned is TB. Number of decimals is optional, - defaults to 1. - - :param num: value to be converted - :type num: int - :decimals: number of decimals to consider - :type decimals: int - :returns: formatted number - :returns type: string - """ - - fmt = "%%3.%sf %%s"% decimals - for x in ['bytes','KB','MB','GB']: - if num < 1024.0: - return fmt % (num, x) - - num /= 1024.0 - - return fmt % (num, 'TB') - -def default_glacier_wrapper(args, **kwargs): - """ - Convenience function to call an instance of GlacierWrapper - with all required arguments. - """ - return GlacierWrapper(args.aws_access_key, - args.aws_secret_key, - args.region, - bookkeeping=args.bookkeeping, - no_bookkeeping=args.no_bookkeeping, - bookkeeping_domain_name=args.bookkeeping_domain_name, - sdb_access_key=args.sdb_access_key, - sdb_secret_key=args.sdb_secret_key, - sdb_region=args.sdb_region, - # sns_enable=args.sns_enable, - # sns_topic=args.sns_topic, - # sns_monitored_vaults=args.sns_monitored_vaults, - # sns_options=args.sns_options, - # config_object=args.config_object, - logfile=args.logfile, - loglevel=args.loglevel, - logtostdout=args.logtostdout) - -def handle_errors(fn): - """ - Decorator for exception handling. - """ - @wraps(fn) - def wrapper(*args, **kwargs): - try: - return fn(*args, **kwargs) - except glacierexception.GlacierException as e: - # We are only interested in the error message in case - # it is a self-caused exception. - e.write(indentation='|| ', stack=False, message=True) - sys.exit(e.exitcode) - - return wrapper +# import our modules +from modules.glacier_utils import (default_glacier_wrapper, size_fmt, + output_msg, output_headers, output_table) +from modules.sns_utils import (snssync, snssubscribe, + snslistsubscriptions, snslisttopics, snsunsubscribe) +from modules.decorators import handle_errors +from modules import glacierexception +from modules import constants +from modules.GlacierWrapper import GlacierWrapper @handle_errors def lsvault(args): @@ -491,70 +317,7 @@ def treehash(args): output_table(hash_results, args.output) -def snssync(args): - """ - If monitored_vaults is specified in configuration file, subscribe - vaults specificed in it to notifications, otherwiser - subscribe all vault. - """ - - glacier = default_glacier_wrapper(args) - response = glacier.sns_sync(sns_options=args.sns_options, - output=args.output) - output_table(response, args.output) - -def snssubscribe(args): - """ - Subscribe individual vaults to notifications by method - specified by user. - """ - - protocol = args.protocol - endpoint = args.endpoint - vault_names = args.vault - topic = args.topic - - glacier = default_glacier_wrapper(args) - response = glacier.sns_subscribe(protocol, endpoint, topic, - vault_names=vault_names, - sns_options=args.sns_options) - output_table(response, args.output) - -def snslistsubscriptions(args): - """ - List subscriptions. - """ - - protocol = args.protocol - endpoint = args.endpoint - topic = args.topic - glacier = default_glacier_wrapper(args) - response = glacier.sns_list_subscriptions(protocol, endpoint, - topic, - sns_options=args.sns_options) - output_table(response, args.output) - -def snslisttopics(args): - glacier = default_glacier_wrapper(args) - response = glacier.sns_list_topics(sns_options=args.sns_options) - output_table(response, args.output) - -def snsunsubscribe(args): - """ - Unsubscribe individual vaults from notifications for - specified protocol, endpoint and vault. - """ - - protocol = args.protocol - endpoint = args.endpoint - topic = args.topic - - glacier = default_glacier_wrapper(args) - response = glacier.sns_unsubscribe(protocol, endpoint, - topic, - sns_options=args.sns_options) - output_table(response, args.output) def main(): program_description = u""" diff --git a/glacier/GlacierWrapper.py b/glacier/modules/GlacierWrapper.py similarity index 88% rename from glacier/GlacierWrapper.py rename to glacier/modules/GlacierWrapper.py index 7053533..0060853 100755 --- a/glacier/GlacierWrapper.py +++ b/glacier/modules/GlacierWrapper.py @@ -2,10 +2,11 @@ """ .. module:: GlacierWrapper :platform: Unix, Windows - :synopsis: Wrapper for accessing Amazon Glacier, with Amazon SimpleDB + :synopsis: Wrapper for accessing Amazon Glacier, with Amazon SimpleDB support and other features. """ +# import modules import math import json import pytz @@ -16,7 +17,7 @@ import time import sys import traceback -import glaciercorecalls + import select import hashlib import fcntl @@ -25,6 +26,7 @@ import boto import boto.sdb + from boto import sns from functools import wraps @@ -32,9 +34,11 @@ from datetime import datetime from pprint import pformat -from glaciercorecalls import GlacierConnection, GlacierWriter +# import our modules + +from modules import constants +from modules.glaciercorecalls import GlacierConnection, GlacierWriter -from glacierexception import * class log_class_call(object): """ @@ -109,27 +113,6 @@ class GlacierWrapper(object): and other features. """ - VAULT_NAME_ALLOWED_CHARACTERS = "[a-zA-Z\.\-\_0-9]+" - ID_ALLOWED_CHARACTERS = "[a-zA-Z\-\_0-9]+" - MAX_VAULT_NAME_LENGTH = 255 - MAX_VAULT_DESCRIPTION_LENGTH = 1024 - MAX_PARTS = 10000 - AVAILABLE_REGIONS = ('us-east-1', 'us-west-2', 'us-west-1', - 'eu-west-1', 'eu-central-1', 'sa-east-1', - 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2') - AVAILABLE_REGIONS_MESSAGE = """\ -Invalid region. Available regions for Amazon Glacier are: -us-east-1 (US - Virginia) -us-west-1 (US - N. California) -us-west-2 (US - Oregon) -eu-west-1 (EU - Ireland) -eu-central-1 (EU - Frankfurt) -sa-east-1 (South America - Sao Paulo) -ap-northeast-1 (Asia-Pacific - Tokyo) -ap-southeast-1 (Asia Pacific (Singapore) -ap-southeast-2 (Asia-Pacific - Sydney)\ -""" - def setuplogging(self, logfile, loglevel, logtostdout): """ Set up the logging facility. @@ -337,10 +320,10 @@ def _check_vault_name(self, name): :raises: :py:exc:`glacier.glacierexception.InputException` """ - if len(name) > self.MAX_VAULT_NAME_LENGTH: + if len(name) > constants.MAX_VAULT_NAME_LENGTH: raise InputException( - u"Vault name can be at most %s characters long." % self.MAX_VAULT_NAME_LENGTH, - cause="Vault name more than %s characters long." % self.MAX_VAULT_NAME_LENGTH, + u"Vault name can be at most %s characters long." % constants.MAX_VAULT_NAME_LENGTH, + cause="Vault name more than %s characters long." % constants.MAX_VAULT_NAME_LENGTH, code="VaultNameError") if len(name) == 0: @@ -352,10 +335,11 @@ def _check_vault_name(self, name): # If the name starts with an illegal character, then result # m is None. In that case the expression becomes '0 != len(name)' # which of course is always True. - m = re.match(self.VAULT_NAME_ALLOWED_CHARACTERS, name) + m = re.match(constants.VAULT_NAME_ALLOWED_CHARACTERS, name) if (m.end() if m else 0) != len(name): raise InputException( - u"""Allowed characters are a-z, A-Z, 0-9, '_' (underscore), '-' (hyphen), and '.' (period)""", + u"allowed characters are a-z, A-Z, 0-9, '_' "\ + "(underscore), '-' (hyphen), and '.' (period)", cause='Illegal characters in the vault name.', code="VaultNameError") @@ -365,8 +349,8 @@ def _check_vault_name(self, name): 'Vault description is valid.') def _check_vault_description(self, description): """ - Checks whether a vault description is valid (at least one character, - not too long, no illegal characters). + Checks whether a vault description is valid (at least one + character, not too long, no illegal characters). :param description: Vault description :type description: str @@ -376,19 +360,19 @@ def _check_vault_description(self, description): :raises: :py:exc:`glacier.glacierexception.InputException` """ - if len(description) > self.MAX_VAULT_DESCRIPTION_LENGTH: + if len(description) > constants.MAX_VAULT_DESCRIPTION_LENGTH: raise InputException( - u"Description must be no more than %s characters."% self.MAX_VAULT_DESCRIPTION_LENGTH, - cause='Vault description contains more than %s characters.'% self.MAX_VAULT_DESCRIPTION_LENGTH, + u"Description must be no more than %s characters."% constants.MAX_VAULT_DESCRIPTION_LENGTH, + cause='Vault description contains more than %s characters.'% constants.MAX_VAULT_DESCRIPTION_LENGTH, code="VaultDescriptionError") for char in description: n = ord(char) if n < 32 or n > 126: raise InputException( - u"""The allowed characters are 7-bit ASCII without \ -control codes, specifically ASCII values 32-126 decimal \ -or 0x20-0x7E hexadecimal.""", + u"The allowed characters are 7-bit ASCII without "\ + "control codes, specifically ASCII values 32-126 "\ + "decimal or 0x20-0x7E hexadecimal.", cause="Invalid characters in the vault name.", code="VaultDescriptionError") @@ -405,7 +389,8 @@ def _check_id(self, amazon_id, id_type): :param amazon_id: id to be validated :type amazon_id: str - :param id_type: the case-sensity type of id (JobId, UploadId, ArchiveId). + :param id_type: the case-sensity type of id + (JobId, UploadId, ArchiveId). :type id_type: str :returns: True if valid, raises exception otherwise. @@ -423,7 +408,7 @@ def _check_id(self, amazon_id, id_type): cause='Incorrect length of the %s string.'% id_type, code="IdError") - m = re.match(self.ID_ALLOWED_CHARACTERS, amazon_id) + m = re.match(constants.ID_ALLOWED_CHARACTERS, amazon_id) if (m.end() if m else 0) != len(amazon_id): raise InputException(u"""\ This %s contains invalid characters. \ @@ -448,9 +433,9 @@ def _check_region(self, region): :raises: GlacierWrapper.InputException """ - if not region in self.AVAILABLE_REGIONS: + if not region in constants.AVAILABLE_REGIONS: raise InputException( - self.AVAILABLE_REGIONS_MESSAGE, + constants.AVAILABLE_REGIONS_MESSAGE, cause='Invalid region code: %s.' % region, code='RegionError') @@ -471,7 +456,7 @@ def _check_part_size(self, part_size, total_size): def _part_size_for_total_size(total_size): return self._next_power_of_2( int(math.ceil( - float(total_size) / (1024 * 1024 * self.MAX_PARTS) + float(total_size) / (1024 * 1024 * constants.MAX_PARTS) ))) if part_size < 0: @@ -482,15 +467,15 @@ def _part_size_for_total_size(total_size): else: ps = self._next_power_of_2(part_size) if not ps == part_size: - self.logger.warning("""\ -Part size in MB must be a power of 2, \ -e.g. 1, 2, 4, 8 MB; automatically increased part size from %s to %s.\ -""" % (part_size, ps)) - + self.logger.warning("Part size in MB must be a "\ + "power of 2, "\ + "e.g. 1, 2, 4, 8 MB; "\ + "automatically increased part "\ + "size from %s to %s." % (part_size, ps)) part_size = ps # Check if user specified value is big enough, and adjust if needed. - if total_size > part_size * 1024 * 1024 * self.MAX_PARTS: + if total_size > part_size * 1024 * 1024 * constants.MAX_PARTS: part_size = _part_size_for_total_size(total_size) self.logger.warning("Part size given is too small; \ using %s MB parts to upload." % part_size) @@ -539,7 +524,9 @@ def _progress(self, msg): if sys.stdout.isatty(): # Get the current screen width. - cols = struct.unpack('hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, '1234'))[1] + cols = struct.unpack('hh', fcntl.ioctl(sys.stdout, + termios.TIOCGWINSZ, + '1234'))[1] # Make sure the message fits on a single line, strip if not, # and add spaces to fill the line if it's shorter (to erase @@ -699,7 +686,8 @@ def rmvault(self, vault_name): self.logger.debug('Deleted orphaned archive from the database: %s.' % item.name) except boto.exception.SDBResponseError as e: raise ResponseException( - 'SimpleDB did not respond correctly to our orphaned listings check.', + "SimpleDB did not respond correctly to "\ + "our orphaned listings check.", cause=self._decode_error_message(e.body), code=e.code) @@ -783,10 +771,11 @@ def list_jobs(self, vault_name, completed=None, job_list = [] while True: try: - response = self.glacierconn.list_jobs(vault_name, - completed=completed, - status_code=status_code, - marker=marker) + response = self.glacierconn.list_jobs( + vault_name, + completed=completed, + status_code=status_code, + marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: raise ResponseException( 'Failed to recieve the jobs list for vault %s.' % vault_name, @@ -855,12 +844,13 @@ def describejob(self, vault_name, job_id): "Multipart upload successfully aborted.") def abortmultipart(self, vault_name, upload_id): """ - Aborts an incomplete multipart upload, causing any uploaded data to be - removed from Amazon Glacier. + Aborts an incomplete multipart upload, causing any + uploaded data to be removed from Amazon Glacier. :param vault_name: Name of the vault. :type vault_name: str - :param upload_id: the UploadId of the multipart upload to be aborted. + :param upload_id: the UploadId of the multipart upload + to be aborted. :type upload_id: str :returns: server response. @@ -877,7 +867,9 @@ def abortmultipart(self, vault_name, upload_id): self._check_vault_name(vault_name) self._check_id(upload_id, "UploadId") try: - response = self.glacierconn.abort_multipart_upload(vault_name, upload_id) + response = self.glacierconn.abort_multipart_upload( + vault_name, + upload_id) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: raise ResponseException( 'Failed to abort multipart upload with id %s.' % upload_id, @@ -982,7 +974,11 @@ def upload(self, vault_name, file_name, description, region, if resume and stdin: raise InputException( - 'You must provide the UploadId to resume upload of streams from stdin.\nUse glacier-cmd listmultiparts to find the UploadId.', + "You must provide the UploadId to "\ + "resume upload of streams from "\ + "stdin.\nUse glacier-cmd "\ + "listmultiparts "\ + "to find the UploadId.", code='CommandError') # If file_name is given, try to use this file(s). @@ -1037,42 +1033,57 @@ def upload(self, vault_name, file_name, description, region, else: self.logger.info('Starting upload of %s to %s.\nDescription: %s'% (file_name if file_name else 'data from stdin', vault_name, description)) - # If user did not specify part_size, compute the optimal (i.e. lowest - # value to stay within the self.MAX_PARTS (10,000) block limit). + # If user did not specify part_size, compute the + # optimal (i.e. lowest value to stay within the + # constants.MAX_PARTS (10,000) block limit). part_size = self._check_part_size(part_size, total_size) part_size_in_bytes = part_size * 1024 * 1024 - # If we have an UploadId, check whether it is linked to a current - # job. If so, check whether uploaded data matches the input data and - # try to resume uploading. + # If we have an UploadId, check whether it is linked + # to a current job. If so, check whether uploaded data + # matches the input data and try to resume uploading. upload = None if uploadid: uploads = self.listmultiparts(vault_name) for upload in uploads: if uploadid == upload['MultipartUploadId']: - self.logger.debug('Found a matching upload id. Continuing upload resumption attempt.') + self.logger.debug("Found a matching upload id. "\ + "Continuing upload resumption "\ + "attempt.") self.logger.debug(upload) part_size_in_bytes = upload['PartSizeInBytes'] break else: raise InputException( - 'Can not resume upload of this data as no existing job with this uploadid could be found.', - code='IdError') + "Can not resume upload of "\ + "this data as no existing "\ + "job with this uploadid "\ + "could be found.", + code='IdError') # Initialise the writer task. - writer = GlacierWriter(self.glacierconn, vault_name, description=description, - part_size_in_bytes=part_size_in_bytes, uploadid=uploadid, logger=self.logger) + writer = GlacierWriter(self.glacierconn, vault_name, + description=description, + part_size_in_bytes=part_size_in_bytes, + uploadid=uploadid, logger=self.logger) if upload: marker = None while True: - # Fetch a list of already uploaded parts and their SHA hashes. + # Fetch a list of already uploaded parts and their + # SHA hashes. try: - response = self.glacierconn.list_parts(vault_name, uploadid, marker=marker) + response = self.glacierconn.list_parts( + vault_name, + uploadid, + marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: raise ResponseException( - 'Failed to get a list already uploaded parts for interrupted upload %s.'% uploadid, + "Failed to get a list "\ + "already uploaded parts "\ + "for interrupted "\ + "upload ".format(uploadid), cause=self._decode_error_message(e.body), code=e.code) @@ -1080,10 +1091,13 @@ def upload(self, vault_name, file_name, description, region, current_position = 0 stop = 0 # Process the parts list. - # For each part of data, take the matching data range from + # For each part of data, take the matching + # data range from # the local file, and compare hashes. - # If recieving data over stdin, the parts must be sequential - # and the first must start at 0. For file, we can use the seek() + # If recieving data over stdin, the parts + # must be sequential + # and the first must start at 0. For file, + # we can use the seek() # function to handle non-sequential parts. for part in list_parts_response['Parts']: start, stop = (int(p) for p in part['RangeInBytes'].split('-')) @@ -1091,18 +1105,20 @@ def upload(self, vault_name, file_name, description, region, if not start == current_position: if stdin or is_pipe: raise InputException( - 'Cannot verify non-sequential upload data from stdin or pipe.', + "Cannot verify non-sequential upload "\ + "data from stdin or pipe.", code='ResumeError') if reader: reader.seek(start) if mmapped_file and stop > mmapped_file.size: raise InputException( - 'File does not match uploaded data; please check your uploadid and try again.', - cause='File is smaller than uploaded data.', + constants.UPLOAD_DATA_ERROR_MSG, + cause="File is smaller than uploaded data.", code='ResumeError') - # Try to read the chunk of data, and take the hash if we + # Try to read the chunk of data, and take + # the hash if we # have received anything. # If no data or hash mismatch, stop checking raise an # exception. @@ -1115,13 +1131,13 @@ def upload(self, vault_name, file_name, description, region, writer.tree_hashes.append(data_hash) else: raise InputException( - 'Received data does not match uploaded data; please check your uploadid and try again.', - cause='SHA256 hash mismatch.', - code='ResumeError') + constants.UPLOAD_DATA_ERROR_MSG, + cause="SHA256 hash mismatch.", + code="ResumeError") else: raise InputException( - 'Received data does not match uploaded data; please check your uploadid and try again.', + constants.UPLOAD_DATA_ERROR_MSG, cause='No or not enough data to match.', code='ResumeError') @@ -1145,8 +1161,8 @@ def upload(self, vault_name, file_name, description, region, self._progress(msg) - # Finished checking; log this and print the final status update - # before resuming the upload. + # Finished checking; log this and print the final + # status update before resuming the upload. self.logger.info('Already uploaded: %s. Continuing from there.'% self._size_fmt(stop)) if total_size > 0: msg = 'Checked %s of %s (%s%%). Check done; resuming upload.' \ @@ -1231,12 +1247,15 @@ def upload(self, vault_name, file_name, description, region, location = writer.get_location() if self.bookkeeping: - self.logger.info('Writing upload information into the bookkeeping database.') + self.logger.info("Writing upload information into "\ + "the bookkeeping database.") - # Use the alternative name as given by --name if we have it. + # Use the alternative name as given by + # --name if we have it. file_name = alternative_name if alternative_name else file_name - # If still no name this is an stdin job, so set name accordingly. + # If still no name this is an stdin job, + # so set name accordingly. file_name = file_name if file_name else 'Data from stdin.' file_attrs = { 'region': region, @@ -1250,11 +1269,6 @@ def upload(self, vault_name, file_name, description, region, 'size': writer.uploaded_size } -## if file_name: -## file_attrs['filename'] = file_name -## elif stdin: -## file_attrs['filename'] = 'data from stdin' - self.sdb_domain.put_attributes(file_attrs['filename'], file_attrs) return (archive_id, sha256hash) @@ -1280,7 +1294,8 @@ def getarchive(self, vault_name, archive_id): - return tuple ("ready", job, jobId). - :param vault: Vault name from where we want to retrieve the archive. + :param vault: Vault name from where we want to + retrieve the archive. :type vault: str :param archive: ArchiveID of archive to be retrieved. :type archive: str @@ -1313,7 +1328,8 @@ def getarchive(self, vault_name, archive_id): response = self.glacierconn.initiate_job(vault_name, job_data) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: raise ResponseException( - 'Failed to initiate an archive retrieval job for archive %s in vault %s.'% (archive_id, vault_name), + "Failed to initiate an archive retrieval "\ + "job for archive %s in vault %s."% (archive_id, vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -1343,24 +1359,30 @@ def download(self, vault_name, archive_id, part_size, download_job = job if not job['Completed']: raise CommunicationException( - "Archive retrieval request not completed yet. Please try again later.", + "Archive retrieval request not completed "\ + "yet. Please try again later.", code='NotReady') - self.logger.debug('Archive retrieval completed; archive is available for download now.') + self.logger.debug("Archive retrieval completed; "\ + "archive is available for "\ + "download now.") break else: raise InputException( - "Requested archive not available. Please make sure \ -your archive ID is correct, and start a retrieval job using \ -'getarchive' if necessary.", + "Requested archive not available. Please make sure "\ + "your archive ID is correct, and start a retrieval "\ + "job using getarchive' if necessary.", code='IdError') - # Check whether we can access the file the archive has to be written to. + # Check whether we can access the file the archive + # has to be written to. out_file = None if out_file_name: if os.path.isfile(out_file_name) and not overwrite: raise InputException( - "File exists already, aborting. Use the overwrite flag to overwrite existing file.", + "File exists already, aborting. "\ + "Use the overwrite flag to overwrite "\ + "existing file.", code="FileError") try: out_file = open(out_file_name, 'w') @@ -1379,20 +1401,24 @@ def download(self, vault_name, archive_id, part_size, # Log our pending action. if out_file: - self.logger.debug('Starting download of archive to file %s.'% out_file_name) + self.logger.debug("Starting download of archive "\ + "to file %s."% out_file_name) else: - self.logger.debug('Starting download of archive to stdout.') + self.logger.debug("Starting download of archive to stdout.") # Download the data, one part at a time. while downloaded_size < total_size: # Read a part of data. from_bytes = downloaded_size - to_bytes = min(downloaded_size + part_size_in_bytes, total_size) + to_bytes = min(downloaded_size + part_size_in_bytes, + total_size) try: - response = self.glacierconn.get_job_output(vault_name, - download_job['JobId'], - byte_range=(from_bytes, to_bytes-1)) + response = self.glacierconn.get_job_output( + vault_name, + download_job['JobId'], + byte_range=(from_bytes, to_bytes-1)) + data = response.read() except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: raise ResponseException( @@ -1460,7 +1486,8 @@ def download(self, vault_name, archive_id, part_size, @sdb_connect @log_class_call("Searching for archive.", "Search done.") - def search(self, vault=None, region=None, file_name=None, search_term=None): + def search(self, vault=None, region=None, + file_name=None, search_term=None): """ Searches for archives using SimpleDB @@ -1495,17 +1522,6 @@ def search(self, vault=None, region=None, file_name=None, search_term=None): if region: self._check_region(region) -## if file_name and ('"' in file_name or "'" in file_name): -## raise InputException( -## 'Quotes like \' and \" are not allowed in search terms.', -## cause='Invalid search term %s: contains quotes.'% file_name) -## -## -## if search_term and ('"' in search_term or "'" in search_term): -## raise InputException( -## 'Quotes like \' and \" are not allowed in search terms.', -## cause='Invalid search term %s: contains quotes.'% search_term) - self.logger.debug('Search terms: vault %s, region %s, file name %s, search term %s'% (vault, region, file_name, search_term)) search_params = [] @@ -1654,20 +1670,24 @@ def inventory(self, vault_name, refresh): # If inventory retrieval is complete, process it. if inventory_done: - self.logger.debug('Fetching results of finished inventory retrieval.') + self.logger.debug("Fetching results of finished "\ + "inventory retrieval.") response = self.glacierconn.get_job_output(vault_name, inventory_job['JobId']) inventory = response.copy() archives = [] # If bookkeeping is enabled, update cache. - # Add all inventory information to the database, then check - # for any archives listed in the database for that vault and + # Add all inventory information to the database, + # then check for any archives listed + # in the database for that vault and # remove those. if self.bookkeeping and len(inventory['ArchiveList']) > 0: - self.logger.debug('Updating the bookkeeping with the latest inventory.') + self.logger.debug("Updating the bookkeeping "\ + "with the latest inventory.") items = {} - # Add items to the inventory, 25 at a time (maximum batch). + # Add items to the inventory, 25 at a time + # (maximum batch). for item in inventory['ArchiveList']: items[item['ArchiveId']] = { 'vault': vault_name, @@ -1680,12 +1700,15 @@ def inventory(self, vault_name, refresh): } archives.append(item['ArchiveId']) if len(items) == 25: - self.logger.debug('Writing batch of 25 inventory items to the bookkeeping db.') + self.logger.debug( + "Writing batch of 25 inventory "\ + "items to the bookkeeping db.") try: self.sdb_domain.batch_put_attributes(items) except boto.exception.SDBResponseError as e: raise CommunicationException( - "Cannot update inventory cache, Amazon SimpleDB is not happy.", + "Cannot update inventory cache, "\ + "Amazon SimpleDB is not happy.", cause=e, code="SdbWriteError") items = {} @@ -1693,46 +1716,57 @@ def inventory(self, vault_name, refresh): # Add the remaining batch of items, if any, to the # database. if items: - self.logger.debug('Writing final batch of %s inventory items to the bookkeeping db.'% len(items)) + self.logger.debug("Writing final batch of %s "\ + "inventory items to the"\ + " bookkeeping db."% len(items)) try: self.sdb_domain.batch_put_attributes(items) except boto.exception.SDBResponseError as e: raise CommunicationException( - "Cannot update inventory cache, Amazon SimpleDB is not happy.", + "Cannot update inventory cache, "\ + "Amazon SimpleDB is not happy.", cause=e, code="SdbWriteError") - # Get the inventory from the database for this vault, - # and delete any orphaned items. + # Get the inventory from the database for this + # vault, and delete any orphaned items. query = "select * from `%s` where vault='%s'" % (self.bookkeeping_domain_name, vault_name) result = self.sdb_domain.select(query) try: for item in result: if not item.name in archives: self.sdb_domain.delete_item(item) - self.logger.debug('Deleted orphaned archive from the database: %s.'% item.name) + self.logger.debug( + "Deleted orphaned "\ + "archive from the database: "\ + "{}.".format(item.name)) except boto.exception.SDBResponseError as e: raise ResponseException( - 'SimpleDB did not respond correctly to our inventory check.', + "SimpleDB did not respond correctly "\ + "to our inventory check.", cause=self._decode_error_message(e.body), code=e.code) - # If refresh == True or no current inventory jobs either finished or - # in progress, we have to start a new job. Then request the job details - # through describejob to return. + # If refresh == True or no current inventory jobs either + # finished or in progress, we have to start a new job. + # Then request the job details through describejob to return. if refresh or not inventory_job: - self.logger.debug('No inventory jobs finished or running; starting a new job.') + self.logger.debug("No inventory jobs finished or "\ + "running; starting a new job.") job_data = {'Type': 'inventory-retrieval'} try: - new_job = self.glacierconn.initiate_job(vault_name, job_data) + new_job = self.glacierconn.initiate_job(vault_name, + job_data) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: raise ResponseException( - 'Failed to create a new inventory retrieval job for vault %s.'% vault_name, + "Failed to create a new inventory retrieval "\ + "job for vault {}.".format(vault_name), cause=self._decode_error_message(e.body), code=e.code) - inventory_job = self.describejob(vault_name, new_job['JobId']) + inventory_job = self.describejob(vault_name, + new_job['JobId']) return (inventory_job, inventory) @@ -1779,8 +1813,10 @@ def _init_events_for_vaults(self, vaults, topic): result = dict() result["Vault Name"] = vault_name - result["Request Id"] = \ - self._init_events_for_vault(vault_name, topic)[u'RequestId'] + request_id = self._init_events_for_vault(vault_name, topic) + request_id = request_id[u'RequestId'] + result["Request Id"] = request_id + results += [result] return results @@ -1824,7 +1860,7 @@ def sns_sync(self, sns_options, output): protocol, endpoint = method.split(',') except ValueError: raise InputException( - ("If you specify method, you should use format " + ("If you specify method, you should ""use format " "'protocol1,endpoint1;protocol2,endpoint2'."), cause='Wrong method format in configuration file.', code='SNSConfigurationError') @@ -1872,7 +1908,7 @@ def sns_subscribe(self, protocol, endpoint, topic, sns_options, vault_names=None if vault_names: vaults = vault_names.split(",") self._init_events_for_vaults(vaults, topic_arn) - + topic_arns = [topic_arn] if len(topic_arns): @@ -1884,7 +1920,9 @@ def sns_subscribe(self, protocol, endpoint, topic, sns_options, vault_names=None 'RequestId': result['ResponseMetadata']['RequestId']}] return results except boto.exception.BotoServerError as e: - raise ResponseException("Failed to subscribe to notifications for vault %s." % vault_name, + raise ResponseException("Failed to subscribe to "\ + "notifications for vault"\ + " %s." % vault_name, cause=self._decode_error_message(e.body), code=e.code) else: @@ -1894,7 +1932,7 @@ def sns_subscribe(self, protocol, endpoint, topic, sns_options, vault_names=None @sns_connect def sns_list_topics(self, sns_options): topics = self.sns_conn.get_all_topics()['ListTopicsResponse']['ListTopicsResult']['Topics'] - + results = [] for topic in topics: results += [{"Topic":topic['TopicArn'].split(":")[-1], "Topic ARN":topic['TopicArn']}] @@ -1944,9 +1982,8 @@ def sns_unsubscribe(self, protocol, endpoint, topic, sns_options): code='SNSParameterError') matching_subs = self.sns_list_subscriptions(protocol, - endpoint, - topic, - sns_options=sns_options) + endpoint, topic, + sns_options=sns_options) unsubscribed = [] for res in matching_subs: @@ -1957,8 +1994,9 @@ def sns_unsubscribe(self, protocol, endpoint, topic, sns_options): return unsubscribed def __init__(self, aws_access_key, aws_secret_key, region, - bookkeeping=False, no_bookkeeping=None, bookkeeping_domain_name=None, - sdb_access_key=None, sdb_secret_key=None, sdb_region=None, + bookkeeping=False, no_bookkeeping=None, + bookkeeping_domain_name=None, sdb_access_key=None, + sdb_secret_key=None, sdb_region=None, logfile=None, loglevel='WARNING', logtostdout=True): """ Constructor, sets up important variables and so for GlacierWrapper. diff --git a/glacier/modules/__init__.py b/glacier/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/glacier/constants.py b/glacier/modules/constants.py similarity index 100% rename from glacier/constants.py rename to glacier/modules/constants.py diff --git a/glacier/modules/decorators.py b/glacier/modules/decorators.py new file mode 100644 index 0000000..c20969f --- /dev/null +++ b/glacier/modules/decorators.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# import modules +from functools import wraps + +# import our modules +from modules import glacierexception + +def handle_errors(fn): + """ + Decorator for exception handling. + """ + @wraps(fn) + def wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + except glacierexception.GlacierException as e: + + # We are only interested in the error message in case + # it is a self-caused exception. + e.write(indentation='|| ', stack=False, message=True) + sys.exit(e.exitcode) + + return wrapper diff --git a/glacier/modules/glacier_utils.py b/glacier/modules/glacier_utils.py new file mode 100644 index 0000000..26bd2ea --- /dev/null +++ b/glacier/modules/glacier_utils.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# import modules +import csv +import json +import sys +from prettytable import PrettyTable + + +# import our modules +from modules.GlacierWrapper import GlacierWrapper +from modules import constants + +# function definition +def default_glacier_wrapper(args, **kwargs): + """ + Convenience function to call an instance of GlacierWrapper + with all required arguments. + """ + return GlacierWrapper(args.aws_access_key, + args.aws_secret_key, + args.region, + bookkeeping=args.bookkeeping, + no_bookkeeping=args.no_bookkeeping, + bookkeeping_domain_name=args.bookkeeping_domain_name, + sdb_access_key=args.sdb_access_key, + sdb_secret_key=args.sdb_secret_key, + sdb_region=args.sdb_region, + # sns_enable=args.sns_enable, + # sns_topic=args.sns_topic, + # sns_monitored_vaults=args.sns_monitored_vaults, + # sns_options=args.sns_options, + # config_object=args.config_object, + logfile=args.logfile, + loglevel=args.loglevel, + logtostdout=args.logtostdout) + +def output_msg(msg, output, success=True): + """ + In case of a single message output, e.g. nothing found. + + :param msg: a single message to output. + :type msg: str + :param success: whether the operation was a success or not. + :type success: boolean + :param output: output format + :type output: string + :raises: ValueError if output not in constants.TABLE_OUTPUT_FORMAT + :returns :nothing, prints the output or sys.exist with status 125 + """ + + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be{}, " + "got {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) + + if msg is not None: + if output == 'print': + print msg + + if output == 'csv': + csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) + csvwriter.writerow(msg) + + if output == 'json': + print json.dumps(msg) + + if not success: + sys.exit(125) + +def size_fmt(num, decimals = 1): + """ + Formats file sizes in human readable format. Anything bigger than + TB is returned is TB. Number of decimals is optional, + defaults to 1. + + :param num: value to be converted + :type num: int + :decimals: number of decimals to consider + :type decimals: int + :returns: formatted number + :returns type: string + """ + + fmt = "%%3.%sf %%s"% decimals + for x in ['bytes','KB','MB','GB']: + if num < 1024.0: + return fmt % (num, x) + + num /= 1024.0 + + return fmt % (num, 'TB') + + +def output_headers(headers, output): + """ + Prints a list of headers - single item output. + + :param headers: the output to be printed as {'header1':'data1',...} + :type headers: dict + :param output: selects output type, must be + listed in constants.HEADERS_OUTPUT_FORMAT + :type output: string + :raises: ValueError when output format is not supported + """ + + if output not in constants.HEADERS_OUTPUT_FORMAT: + raise ValueError("Output must be in {}, got" + ":{}".format(constants.HEADERS_OUTPUT_FORMAT, + output)) + rows = [(k, headers[k]) for k in headers.keys()] + if output == 'print': + table = PrettyTable(["Header", "Value"]) + for row in rows: + if len(str(row[1])) <= 138: + table.add_row(row) + + print table + + if output == 'csv': + csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) + for row in rows: + csvwriter.writerow(row) + + if output == 'json': + print json.dumps(headers) + +def output_table(results, output, keys=None, sort_key=None): + """ + Prettyprints results. Expects a list of identical dicts. + Use the dict keys as headers unless keys is given; + one line for each item. + + Expected format of data is a list of dicts: + [{'key1':'data1.1', 'key2':'data1.2', ... }, + {'key1':'data1.2', 'key2':'data2.2', ... }, + ...] + :param keys: dict of headers to be printed for each key: + {'key1':'header1', 'key2':'header2',...} + :type keys: dict + :param sort_key: the key to use for sorting the table. + :type sort_key: string + :raises : ValueError if output not in constants.TABLE_OUTPUT_FORMAT + :returns: nothing, prints the output on the console + """ + + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be{}, " + "got {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) + if output == 'print': + if len(results) == 0: + print 'No output!' + return + + headers = [keys[k] for k in keys.keys()] if keys else results[0].keys() + table = PrettyTable(headers) + for line in results: + table.add_row([line[k] if k in line else '' for k in (keys.keys() if keys else headers)]) + + if sort_key: + table.sortby = keys[sort_key] if keys else sort_key + + print table + + if output == 'csv': + csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) + keys = results[0].keys() + csvwriter.writerow(keys) + for row in results: + csvwriter.writerow([row[k] for k in keys]) + + if output == 'json': + print json.dumps(results) diff --git a/glacier/glaciercorecalls.py b/glacier/modules/glaciercorecalls.py similarity index 62% rename from glacier/glaciercorecalls.py rename to glacier/modules/glaciercorecalls.py index c9f2b4c..3b224df 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/modules/glaciercorecalls.py @@ -12,19 +12,23 @@ writer.write(block of data) writer.close() # Get the id of the newly created archive - archive_id = writer.get_archive_id()from boto.connection import AWSAuthConnection + archive_id = writer.get_archive_id()from boto.connection + import AWSAuthConnection """ +# import modules + import urllib import hashlib import math import json import sys import time - import boto.glacier.layer1 -from glacierexception import * +# import our modules +from modules import constants +from modules.glacierexception import * # Placeholder, effectively renaming the class. class GlacierConnection(boto.glacier.layer1.Layer1): @@ -37,7 +41,7 @@ def chunk_hashes(data): Break up the byte-string into 1MB chunks and return sha256 hashes for each. """ - chunk = 1024*1024 + chunk = constants.CHUNK_SIZE*constants.CHUNK_SIZE chunk_count = int(math.ceil(len(data)/float(chunk))) return [hashlib.sha256(data[i*chunk:(i+1)*chunk]).digest() for i in range(chunk_count)] @@ -72,55 +76,47 @@ class GlacierWriter(object): Presents a file-like object for writing to a Amazon Glacier Archive. The data is written using the multi-part upload API. """ - DEFAULT_PART_SIZE = 128 # in MB, power of 2. - # After every failed block upload we sleep (SLEEP_TIME * retries) seconds. - # The more retries we've made for one particular block, the longer we sleep - # before re-attempting to re-upload that block. - SLEEP_TIME = 300 - # How many retries we should make to upload a particular block. We will not - # give up unless we've made at LEAST this many attempts to upload a block. - BLOCK_RETRIES = 10 - # How many retries we should allow for the whole upload. We will not give up - # unless we've made at LEAST this many attempts to upload the archive. - TOTAL_RETRIES = 100 - # For large files, the limits above could be surpassed. We also set a per-Gb - # criteria that allows more errors for larger uploads. - MAX_TOTAL_RETRY_PER_GB = 2 + def __init__(self, connection, vault_name, - description=None, part_size_in_bytes=DEFAULT_PART_SIZE*1024*1024, + description=None, + part_size_in_bytes=constants.DEFAULT_PART_SIZE*1024*1024, uploadid=None, logger=None): self.part_size = part_size_in_bytes self.vault_name = vault_name self.connection = connection -## self.location = None self.logger = logger self.total_retries = 0 if uploadid: self.uploadid = uploadid else: - response = self.connection.initiate_multipart_upload(self.vault_name, - self.part_size, - description) + response = self.connection.initiate_multipart_upload( + self.vault_name, + self.part_size, + description) self.uploadid = response['UploadId'] self.uploaded_size = 0 self.tree_hashes = [] self.closed = False -## self.upload_url = response.getheader("location") def write(self, data): if self.closed: raise CommunicationError( - "Tried to write to a GlacierWriter that is already closed.", + "Tried to write to a GlacierWriter that is "\ + "already closed.", code='InternalError') - if len(data) > self.part_size: + len_data = len(data) + if len_data > self.part_size: raise InputException ( - 'Block of data provided must be equal to or smaller than the set block size.', + "Block of data provided({}) "\ + "must be equal " + "to or smaller than the set " + "block size({}).".format(len_data, self.part_size), code='InternalError') part_tree_hash = tree_hash(chunk_hashes(data)) @@ -128,8 +124,8 @@ def write(self, data): headers = { "x-amz-glacier-version": "2012-06-01", "Content-Range": "bytes %d-%d/*" % (self.uploaded_size, - (self.uploaded_size+len(data))-1), - "Content-Length": str(len(data)), + (self.uploaded_size+len_data)-1), + "Content-Length": str(len_data), "Content-Type": "application/octet-stream", "x-amz-sha256-tree-hash": bytes_to_hex(part_tree_hash), "x-amz-content-sha256": hashlib.sha256(data).hexdigest() @@ -140,30 +136,40 @@ def write(self, data): while True: try: - response = self.connection.upload_part(self.vault_name, - self.uploadid, - hashlib.sha256(data).hexdigest(), - bytes_to_hex(part_tree_hash), - (self.uploaded_size, self.uploaded_size+len(data)-1), - data) + response = self.connection.upload_part( + self.vault_name, + self.uploadid, + hashlib.sha256(data).hexdigest(), + bytes_to_hex(part_tree_hash), + (self.uploaded_size, self.uploaded_size+len_data-1), + data) response.read() break except Exception as e: - if '408' in e.message or e.code == "ServiceUnavailableException" or e.type == "Server": - uploaded_gb = self.uploaded_size / (1024 * 1024 * 1024) - if retries >= self.BLOCK_RETRIES and retries > math.log10(uploaded_gb) * 10: + if ('408' in e.message or + e.code == "ServiceUnavailableException" or + e.type == "Server"): + + uploaded_gb = self.uploaded_size / (1024 ** 1024) + if (retries >= constants.BLOCK_RETRIES and + retries > math.log10(uploaded_gb) * 10): + if self.logger: - self.logger.warning('Retries exhausted for this block.') + self.logger.warning("Retries exhausted "\ + "({}) for this block.".format(constants.BLOCK_RETRIES)) raise e if uploaded_gb > 0: retry_per_gb = self.total_retries / uploaded_gb else: retry_per_gb = 0 - if self.total_retries >= self.TOTAL_RETRIES and retry_per_gb > self.MAX_TOTAL_RETRY_PER_GB: + if (self.total_retries >= constants.TOTAL_RETRIES and + retry_per_gb > constants.MAX_TOTAL_RETRY_PER_GB): + if self.logger: - self.logger.warning('Total retries exhausted.') + self.logger.warning("Total retries "\ + "exhausted({}).".format(constants.TOTAL_RETRIES)) raise e retries += 1 @@ -177,11 +183,11 @@ def write(self, data): # Commify large numbers self.logger.warning('Total uploaded size = {:,d}, block hash = {:}'.format(self.uploaded_size, bytes_to_hex(part_tree_hash))) - self.logger.warning('Retries (this block, total) = %d/%d, %d/%d' % (retries, self.BLOCK_RETRIES, self.total_retries, self.TOTAL_RETRIES)) + self.logger.warning('Retries (this block, total) = %d/%d, %d/%d' % (retries, constants.BLOCK_RETRIES, self.total_retries, constants.TOTAL_RETRIES)) self.logger.warning('Check the AWS status at: http://status.aws.amazon.com/') - self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (self.SLEEP_TIME, self.SLEEP_TIME / 60.0)) + self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (constants.SLEEP_TIME, constants.SLEEP_TIME / 60.0)) - time.sleep(self.SLEEP_TIME * retries) + time.sleep(constants.SLEEP_TIME * retries) else: self.logger.warning(e.message) @@ -196,10 +202,11 @@ def close(self): return # Complete the multiplart glacier upload - response = self.connection.complete_multipart_upload(self.vault_name, - self.uploadid, - bytes_to_hex(tree_hash(self.tree_hashes)), - self.uploaded_size) + response = self.connection.complete_multipart_upload( + self.vault_name, + self.uploadid, + bytes_to_hex(tree_hash(self.tree_hashes)), + self.uploaded_size) self.archive_id = response['ArchiveId'] self.location = response['Location'] self.hash_sha256 = bytes_to_hex(tree_hash(self.tree_hashes)) diff --git a/glacier/glacierexception.py b/glacier/modules/glacierexception.py similarity index 72% rename from glacier/glacierexception.py rename to glacier/modules/glacierexception.py index 960c27e..a4d0e9f 100644 --- a/glacier/glacierexception.py +++ b/glacier/modules/glacierexception.py @@ -1,7 +1,5 @@ -import traceback -import re -import sys -import logging +#!/usr/bin/env python +# encoding: utf-8 """ @@ -9,11 +7,22 @@ Note by wvmarle: This file contains the complete code from chained_exception.py plus the -error handling code from GlacierWrapper.py, allowing it to be used in other -modules like glaciercorecalls as well. +error handling code from GlacierWrapper.py, allowing it to be +used in other modules like glaciercorecalls as well. ********** """ + +# import modules +import traceback +import re +import sys +import logging + +# import our modules +from modules import constants + + class GlacierException(Exception): """ An extension of the built-in Exception class, this handles @@ -27,29 +36,8 @@ class GlacierException(Exception): TODO: describe usage. """ - ERRORCODE = {'InternalError': 127, # Library internal error. - 'UndefinedErrorCode': 126, # Undefined code. - 'NoResults': 125, # Operation yielded no results. - 'GlacierConnectionError': 1, # Can not connect to Glacier. - 'SdbConnectionError': 2, # Can not connect to SimpleDB. - 'CommandError': 3, # Command line is invalid. - 'VaultNameError': 4, # Invalid vault name. - 'DescriptionError': 5, # Invalid archive description. - 'IdError': 6, # Invalid upload/archive/job ID given. - 'RegionError': 7, # Invalid region given. - 'FileError': 8, # Error related to reading/writing a file. - 'ResumeError': 9, # Problem resuming a multipart upload. - 'NotReady': 10, # Requested download is not ready yet. - 'BookkeepingError': 11, # Bookkeeping not available. - 'SdbCommunicationError': 12, # Problem reading/writing SimpleDB data. - 'ResourceNotFoundException': 13, # Glacier can not find the requested resource. - 'InvalidParameterValueException': 14, # Parameter not accepted. - 'DownloadError': 15, # Downloading an archive failed. - 'SNSConnectionError': 126, # Can not connect to SNS - 'SNSConfigurationError': 127, # Problem with configuration file - 'SNSParameterError':128, # Problem with arguments passed to SNS - } - + + def __init__(self, message, code=None, cause=None): """ Constructor. Logs the error. @@ -62,7 +50,7 @@ def __init__(self, message, code=None, cause=None): :type cause: str """ self.logger = logging.getLogger(self.__class__.__name__) - self.exitcode = self.ERRORCODE[code] if code in self.ERRORCODE else 254 + self.exitcode = constants.ERRORCODE[code] if code in constants.ERRORCODE else 254 self.code = code if cause: self.logger.error('ERROR: %s'% cause) @@ -78,9 +66,9 @@ def __init__(self, message, code=None, cause=None): self.stack = ( traceback.format_stack()[:-2] + traceback.format_tb(sys.exc_info()[2])) - # ^^^ let's hope the information is still there; caller must take - # care of this. - + # ^^^ let's hope the information is still there; + # caller must take care of this. + self.message = message self.logger.info(self.fetch(message=True)) self.logger.debug(self.fetch(stack=True)) @@ -89,7 +77,8 @@ def __init__(self, message, code=None, cause=None): # Works as a generator to help get the stack trace and the cause # written out. - def causeTree(self, indentation=' ', alreadyMentionedTree=[], stack=False, message=False): + def causeTree(self, indentation=' ', alreadyMentionedTree=[], + stack=False, message=False): """ Returns a complete stack tree, an error message, or both. Returns a warning if neither stack or message are True. @@ -108,18 +97,19 @@ def causeTree(self, indentation=' ', alreadyMentionedTree=[], stack=False, mess ellipsed, "" if ellipsed == 1 else "s") ellipsed = False # marker for "given out" - + yield line if message: exc = self if self.message is None else self.message - for line in traceback.format_exception_only(exc.__class__, exc): + for line in traceback.format_exception_only(exc.__class__, + exc): yield line - + if self.cause: yield ("Caused by: %d exception%s\n" % (len(self.cause), "" if len(self.cause) == 1 else "s")) - + for causePart in self.cause: if hasattr(causePart,"causeTree"): for line in causePart.causeTree(indentation, self.stack): @@ -129,16 +119,19 @@ def causeTree(self, indentation=' ', alreadyMentionedTree=[], stack=False, mess yield re.sub(r'([^\n]*\n)', indentation + r'\1', line) if not message and not stack: - yield ('No output. Specify message=True and/or stack=True \ -to get output when calling this function.\n') + yield ("No output. Specify message=True and/or stack=True " + "to get output when calling this function.\n") - def write(self, stream=None, indentation=' ', message=False, stack=False): + def write(self, stream=None, indentation=' ', + message=False, stack=False): """ Writes the error details to sys.stderr or a stream. """ - + stream = sys.stderr if stream is None else stream - for line in self.causeTree(indentation, message=message, stack=stack): + for line in self.causeTree(indentation, + message=message, + stack=stack): stream.write(line) def fetch(self, indentation=' ', message=False, stack=False): @@ -146,7 +139,9 @@ def fetch(self, indentation=' ', message=False, stack=False): Fetches the error details and returns them as string. """ out = '' - for line in self.causeTree(indentation, message=message, stack=stack): + for line in self.causeTree(indentation, + message=message, + tack=stack): out += line return out @@ -156,7 +151,7 @@ class InputException(GlacierException): Exception that is raised when there is someting wrong with the user input. """ - + VaultNameError = 1 VaultDescriptionError = 2 def __init__(self, message, code=None, cause=None): @@ -165,7 +160,7 @@ def __init__(self, message, code=None, cause=None): :param message: the error message. :type message: str :param code: the error code. - :type code: + :type code: :param cause: explanation on what caused the error. :type cause: str """ @@ -176,7 +171,7 @@ class ConnectionException(GlacierException): Exception that is raised when there is something wrong with the connection. """ - + GlacierConnectionError = 1 SdbConnectionError = 2 def __init__(self, message, code=None, cause=None): @@ -185,7 +180,7 @@ def __init__(self, message, code=None, cause=None): :param message: the error message. :type message: str :param code: the error code. - :type code: + :type code: :param cause: explanation on what caused the error. :type cause: str """ @@ -202,7 +197,7 @@ def __init__(self, message, code=None, cause=None): :param message: the error message. :type message: str :param code: the error code. - :type code: + :type code: :param cause: explanation on what caused the error. :type cause: str """ diff --git a/glacier/modules/sns_utils.py b/glacier/modules/sns_utils.py new file mode 100644 index 0000000..15f1832 --- /dev/null +++ b/glacier/modules/sns_utils.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# import modules +def snssync(args): + """ + If monitored_vaults is specified in configuration file, subscribe + vaults specificed in it to notifications, otherwiser + subscribe all vault. + """ + + glacier = default_glacier_wrapper(args) + response = glacier.sns_sync(sns_options=args.sns_options, + output=args.output) + output_table(response, args.output) + +def snssubscribe(args): + """ + Subscribe individual vaults to notifications by method + specified by user. + """ + + protocol = args.protocol + endpoint = args.endpoint + vault_names = args.vault + topic = args.topic + + glacier = default_glacier_wrapper(args) + response = glacier.sns_subscribe(protocol, endpoint, topic, + vault_names=vault_names, + sns_options=args.sns_options) + output_table(response, args.output) + +def snslistsubscriptions(args): + """ + List subscriptions. + """ + + protocol = args.protocol + endpoint = args.endpoint + topic = args.topic + + glacier = default_glacier_wrapper(args) + response = glacier.sns_list_subscriptions(protocol, endpoint, + topic, + sns_options=args.sns_options) + output_table(response, args.output) + +def snslisttopics(args): + glacier = default_glacier_wrapper(args) + response = glacier.sns_list_topics(sns_options=args.sns_options) + output_table(response, args.output) + +def snsunsubscribe(args): + """ + Unsubscribe individual vaults from notifications for + specified protocol, endpoint and vault. + """ + + protocol = args.protocol + endpoint = args.endpoint + topic = args.topic + + glacier = default_glacier_wrapper(args) + response = glacier.sns_unsubscribe(protocol, endpoint, + topic, + sns_options=args.sns_options) + output_table(response, args.output) From 273f677965d31784a010f288a10e49c340804512 Mon Sep 17 00:00:00 2001 From: marzona Date: Tue, 4 Jul 2017 15:41:13 +0100 Subject: [PATCH 3/6] - code maintenance: removing %s, reducing import scope, split code in modules/code re-org, adding docstirngs - moved constants into constants file - removed import * and similar --- glacier/glacier.py | 5 +- glacier/modules/GlacierWrapper.py | 192 +++++++++++++++------------- glacier/modules/constants.py | 1 - glacier/modules/decorators.py | 1 + glacier/modules/glaciercorecalls.py | 34 +++-- glacier/modules/glacierexception.py | 4 +- glacier/modules/sns_utils.py | 16 ++- 7 files changed, 142 insertions(+), 111 deletions(-) diff --git a/glacier/glacier.py b/glacier/glacier.py index 8d7d3d9..ab12425 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -22,10 +22,11 @@ from modules.sns_utils import (snssync, snssubscribe, snslistsubscriptions, snslisttopics, snsunsubscribe) from modules.decorators import handle_errors -from modules import glacierexception -from modules import constants +from modules import glacierexception, constants from modules.GlacierWrapper import GlacierWrapper +# function definition + @handle_errors def lsvault(args): """ diff --git a/glacier/modules/GlacierWrapper.py b/glacier/modules/GlacierWrapper.py index 0060853..645447e 100755 --- a/glacier/modules/GlacierWrapper.py +++ b/glacier/modules/GlacierWrapper.py @@ -38,7 +38,7 @@ from modules import constants from modules.glaciercorecalls import GlacierConnection, GlacierWriter - +from modules import glacierexception class log_class_call(object): """ @@ -212,7 +212,7 @@ def glacier_connect_wrap(*args, **kwargs): self.aws_secret_key, region_name=self.region) except boto.exception.AWSConnectionError as e: - raise ConnectionException( + raise glacierexception.ConnectionException( "Cannot connect to Amazon Glacier.", cause=e.cause, code="GlacierConnectionError") @@ -244,7 +244,7 @@ def sdb_connect_wrap(*args, **kwargs): # we need to glaciercorecalls? if not self.bookkeeping_domain_name: - raise InputException( + raise glacierexception.InputException( '''\ Bookkeeping enabled but no Amazon SimpleDB domain given. Provide a domain in either the config file or via the @@ -267,8 +267,9 @@ def sdb_connect_wrap(*args, **kwargs): aws_secret_access_key=self.sdb_secret_key) domain_name = self.bookkeeping_domain_name self.sdb_domain = self.sdb_conn.create_domain(domain_name) - except (boto.exception.AWSConnectionError, boto.exception.SDBResponseError) as e: - raise ConnectionException( + except (boto.exception.AWSConnectionError, + boto.exception.SDBResponseError) as e: + raise glacierexception.ConnectionException( "Cannot connect to Amazon SimpleDB.", cause=e, code="SdbConnectionError") @@ -299,7 +300,7 @@ def sns_connect_wrap(*args, **kwargs): aws_secret_access_key=self.aws_secret_key, region_name=self.region) except boto.exception.AWSConnectionError as e: - raise ConnectionException( + raise glacierexception.ConnectionException( "Cannot connect to Amazon SNS.", cause=e.cause, code="SNSConnectionError") @@ -321,13 +322,15 @@ def _check_vault_name(self, name): """ if len(name) > constants.MAX_VAULT_NAME_LENGTH: - raise InputException( - u"Vault name can be at most %s characters long." % constants.MAX_VAULT_NAME_LENGTH, - cause="Vault name more than %s characters long." % constants.MAX_VAULT_NAME_LENGTH, + raise glacierexception.InputException( + "Vault name can be at most {} characters "\ + "long.".format(onstants.MAX_VAULT_NAME_LENGTH), + cause="Vault name more than {} characters "\ + "long.".format(constants.MAX_VAULT_NAME_LENGTH), code="VaultNameError") if len(name) == 0: - raise InputException( + raise glacierexception.InputException( u"Vault name has to be at least 1 character long.", cause='Vault name has to be at least 1 character long.', code="VaultNameError") @@ -337,7 +340,7 @@ def _check_vault_name(self, name): # which of course is always True. m = re.match(constants.VAULT_NAME_ALLOWED_CHARACTERS, name) if (m.end() if m else 0) != len(name): - raise InputException( + raise glacierexception.InputException( u"allowed characters are a-z, A-Z, 0-9, '_' "\ "(underscore), '-' (hyphen), and '.' (period)", cause='Illegal characters in the vault name.', @@ -361,15 +364,17 @@ def _check_vault_description(self, description): """ if len(description) > constants.MAX_VAULT_DESCRIPTION_LENGTH: - raise InputException( - u"Description must be no more than %s characters."% constants.MAX_VAULT_DESCRIPTION_LENGTH, - cause='Vault description contains more than %s characters.'% constants.MAX_VAULT_DESCRIPTION_LENGTH, + raise glacierexception.InputException( + "Description must be no more than {} characters"\ + ".".format(constants.MAX_VAULT_DESCRIPTION_LENGTH), + cause='Vault description contains more than {} "\ + "characters.'.format(constants.MAX_VAULT_DESCRIPTION_LENGTH), code="VaultDescriptionError") for char in description: n = ord(char) if n < 32 or n > 126: - raise InputException( + raise glacierexception.InputException( u"The allowed characters are 7-bit ASCII without "\ "control codes, specifically ASCII values 32-126 "\ "decimal or 0x20-0x7E hexadecimal.", @@ -401,20 +406,23 @@ def _check_id(self, amazon_id, id_type): length = {'JobId': 92, 'UploadId': 92, 'ArchiveId': 138} - self.logger.debug('Checking a %s.' % id_type) + self.logger.debug('Checking a {}.'.format(id_type)) if len(amazon_id) != length[id_type]: - raise InputException( - 'A %s must be %s characters long. This ID is %s characters.'% (id_type, length[id_type], len(amazon_id)), - cause='Incorrect length of the %s string.'% id_type, + raise glacierexception.InputException( + 'A {} must be {} characters long. This ID is {} "\ + "characters.'.format(id_type, + length[id_type], + len(amazon_id)), + cause='Incorrect length of the {} string.'.format(id_type), code="IdError") m = re.match(constants.ID_ALLOWED_CHARACTERS, amazon_id) if (m.end() if m else 0) != len(amazon_id): - raise InputException(u"""\ -This %s contains invalid characters. \ + raise glacierexception.InputException(u"""\ +This {} contains invalid characters. \ Allowed characters are a-z, A-Z, 0-9, '_' (underscore) and '-' (hyphen)\ -""" % id_type, - cause='Illegal characters in the %s string.' % id_type, +""".format(id_type), + cause='Illegal characters in the {} string.'.format(id_type), code="IdError") return True @@ -434,9 +442,9 @@ def _check_region(self, region): """ if not region in constants.AVAILABLE_REGIONS: - raise InputException( + raise glacierexception.InputException( constants.AVAILABLE_REGIONS_MESSAGE, - cause='Invalid region code: %s.' % region, + cause='Invalid region code: {}.'.format(region), code='RegionError') return True @@ -471,14 +479,14 @@ def _part_size_for_total_size(total_size): "power of 2, "\ "e.g. 1, 2, 4, 8 MB; "\ "automatically increased part "\ - "size from %s to %s." % (part_size, ps)) + "size from {} to {}.".format(part_size, ps)) part_size = ps # Check if user specified value is big enough, and adjust if needed. if total_size > part_size * 1024 * 1024 * constants.MAX_PARTS: part_size = _part_size_for_total_size(total_size) self.logger.warning("Part size given is too small; \ -using %s MB parts to upload." % part_size) +using {} MB parts to upload.".format(part_size)) return part_size @@ -602,7 +610,7 @@ def lsvault(self, limit=None): try: response = self.glacierconn.list_vaults(marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( 'Failed to recieve vault list.', cause=self._decode_error_message(e.body), code=e.code) @@ -637,8 +645,8 @@ def mkvault(self, vault_name): try: response = self.glacierconn.create_vault(vault_name) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to create vault with name %s.' % vault_name, + raise glacierexception.ResponseException( + 'Failed to create vault with name {}.'.format(vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -670,22 +678,25 @@ def rmvault(self, vault_name): try: response = self.glacierconn.delete_vault(vault_name) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to remove vault with name %s.' % vault_name, + raise glacierexception.ResponseException( + 'Failed to remove vault with name {}.'.format(vault_name), cause=self._decode_error_message(e.body), code=e.code) # Check for orphaned entries in the bookkeeping database, and # remove them. if self.bookkeeping: - query = "select * from `%s` where vault='%s'" % (self.bookkeeping_domain_name, vault_name) + query = "select * from `{}` where "\ + "vault='{}'".format(self.bookkeeping_domain_name, + vault_name) result = self.sdb_domain.select(query) try: for item in result: self.sdb_domain.delete_item(item) - self.logger.debug('Deleted orphaned archive from the database: %s.' % item.name) + self.logger.debug("Deleted orphaned archive from "\ + "the database: {}.".format(item.name)) except boto.exception.SDBResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( "SimpleDB did not respond correctly to "\ "our orphaned listings check.", cause=self._decode_error_message(e.body), @@ -722,8 +733,8 @@ def describevault(self, vault_name): try: response = self.glacierconn.describe_vault(vault_name) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to get description of vault with name %s.' % vault_name, + raise glacierexception.ResponseException( + 'Failed to get description of vault with name {}.'.format(vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -777,8 +788,9 @@ def list_jobs(self, vault_name, completed=None, status_code=status_code, marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to recieve the jobs list for vault %s.' % vault_name, + raise glacierexception.ResponseException( + "Failed to recieve the jobs list for vault "\ + "{}.".format(vault_name), cause=self._decode_error_message(e.body), code=e.code) job_list += response.copy()['JobList'] @@ -832,8 +844,8 @@ def describejob(self, vault_name, job_id): try: response = self.glacierconn.describe_job(vault_name, job_id) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to get description of job with job id %s.' % job_id, + raise glacierexception.ResponseException( + 'Failed to get description of job with job id {}.'.format(job_id), cause=self._decode_error_message(e.body), code=e.code) @@ -871,8 +883,8 @@ def abortmultipart(self, vault_name, upload_id): vault_name, upload_id) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to abort multipart upload with id %s.' % upload_id, + raise glacierexception.ResponseException( + 'Failed to abort multipart upload with id {}.'.format(upload_id), cause=self._decode_error_message(e.body), code=e.code) @@ -911,8 +923,8 @@ def listmultiparts(self, vault_name, limit=None): response = self.glacierconn.list_multipart_uploads(vault_name, marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to get a list of multipart uploads for vault %s.' % vault_name, + raise glacierexception.ResponseException( + 'Failed to get a list of multipart uploads for vault {}.'.format(vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -973,7 +985,7 @@ def upload(self, vault_name, file_name, description, region, self._check_id(uploadid, 'UploadId') if resume and stdin: - raise InputException( + raise glacierexception.InputException( "You must provide the UploadId to "\ "resume upload of streams from "\ "stdin.\nUse glacier-cmd "\ @@ -994,7 +1006,7 @@ def upload(self, vault_name, file_name, description, region, if not stdin and not is_pipe: if not file_name: - raise InputException( + raise glacierexception.InputException( "No file name given for upload.", code='CommandError') @@ -1003,8 +1015,8 @@ def upload(self, vault_name, file_name, description, region, mmapped_file = mmap(f) total_size = os.path.getsize(file_name) except IOError as e: - raise InputException( - "Could not access file: %s."% file_name, + raise glacierexception.InputException( + "Could not access file: {}.".format(file_name), cause=e, code='FileError') @@ -1013,25 +1025,25 @@ def upload(self, vault_name, file_name, description, region, reader = open(file_name, 'rb') total_size = 0 except IOError: - raise InputException( - "Could not access pipe: %s."% file_name, + raise glacierexception.InputException( + "Could not access pipe: {}.".format(file_name), cause = e, code = 'FileError') elif select.select([sys.stdin,],[],[],0.0)[0]: reader = sys.stdin total_size = 0 else: - raise InputException( + raise glacierexception.InputException( "There is nothing to upload.", code='CommandError') # Log the kind of upload we're going to do. if uploadid: - self.logger.info('Attempting resumption of upload of %s to %s.'% (file_name if file_name else 'data from stdin', vault_name)) + self.logger.info('Attempting resumption of upload of %s to {}.'.format(file_name if file_name else 'data from stdin', vault_name)) elif resume: - self.logger.info('Attempting resumption of upload of %s to %s.'% (file_name, vault_name)) + self.logger.info('Attempting resumption of upload of {} to {}.'.format(file_name, vault_name)) else: - self.logger.info('Starting upload of %s to %s.\nDescription: %s'% (file_name if file_name else 'data from stdin', vault_name, description)) + self.logger.info('Starting upload of {} to {}.\nDescription: {}'.format(file_name if file_name else 'data from stdin', vault_name, description)) # If user did not specify part_size, compute the # optimal (i.e. lowest value to stay within the @@ -1054,7 +1066,7 @@ def upload(self, vault_name, file_name, description, region, part_size_in_bytes = upload['PartSizeInBytes'] break else: - raise InputException( + raise glacierexception.InputException( "Can not resume upload of "\ "this data as no existing "\ "job with this uploadid "\ @@ -1079,7 +1091,7 @@ def upload(self, vault_name, file_name, description, region, uploadid, marker=marker) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( "Failed to get a list "\ "already uploaded parts "\ "for interrupted "\ @@ -1104,7 +1116,7 @@ def upload(self, vault_name, file_name, description, region, stop += 1 if not start == current_position: if stdin or is_pipe: - raise InputException( + raise glacierexception.InputException( "Cannot verify non-sequential upload "\ "data from stdin or pipe.", code='ResumeError') @@ -1112,7 +1124,7 @@ def upload(self, vault_name, file_name, description, region, reader.seek(start) if mmapped_file and stop > mmapped_file.size: - raise InputException( + raise glacierexception.InputException( constants.UPLOAD_DATA_ERROR_MSG, cause="File is smaller than uploaded data.", code='ResumeError') @@ -1127,16 +1139,16 @@ def upload(self, vault_name, file_name, description, region, if data: data_hash = glaciercorecalls.tree_hash(glaciercorecalls.chunk_hashes(data)) if glaciercorecalls.bytes_to_hex(data_hash) == part['SHA256TreeHash']: - self.logger.debug('Part %s hash matches.'% part['RangeInBytes']) + self.logger.debug('Part {} hash matches.'.format(part['RangeInBytes'])) writer.tree_hashes.append(data_hash) else: - raise InputException( + raise glacierexception.InputException( constants.UPLOAD_DATA_ERROR_MSG, cause="SHA256 hash mismatch.", code="ResumeError") else: - raise InputException( + raise glacierexception.InputException( constants.UPLOAD_DATA_ERROR_MSG, cause='No or not enough data to match.', code='ResumeError') @@ -1156,22 +1168,22 @@ def upload(self, vault_name, file_name, description, region, self._size_fmt(total_size), self._bold(str(int(100 * writer.uploaded_size/total_size)))) else: - msg = 'Checked %s.' \ - % (self._size_fmt(writer.uploaded_size)) + msg = "Checked "\ + "{}.".format(self._size_fmt(writer.uploaded_size)) self._progress(msg) # Finished checking; log this and print the final # status update before resuming the upload. - self.logger.info('Already uploaded: %s. Continuing from there.'% self._size_fmt(stop)) + self.logger.info("Already uploaded: {}"\ + ". Continuing from there.".format(self._size_fmt(stop))) if total_size > 0: msg = 'Checked %s of %s (%s%%). Check done; resuming upload.' \ % (self._size_fmt(writer.uploaded_size), self._size_fmt(total_size), self._bold(str(int(100 * writer.uploaded_size/total_size)))) else: - msg = 'Checked %s. Check done; resuming upload.' \ - % (self._size_fmt(writer.uploaded_size)) + msg = "Checked {}. Check done; resuming upload.".format(self._size_fmt(writer.uploaded_size)) self._progress(msg) @@ -1327,9 +1339,9 @@ def getarchive(self, vault_name, archive_id): try: response = self.glacierconn.initiate_job(vault_name, job_data) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( "Failed to initiate an archive retrieval "\ - "job for archive %s in vault %s."% (archive_id, vault_name), + "job for archive {} in vault {}.".format(archive_id, vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -1358,7 +1370,7 @@ def download(self, vault_name, archive_id, part_size, if job['ArchiveId'] == archive_id: download_job = job if not job['Completed']: - raise CommunicationException( + raise glacierexception.CommunicationException( "Archive retrieval request not completed "\ "yet. Please try again later.", code='NotReady') @@ -1368,7 +1380,7 @@ def download(self, vault_name, archive_id, part_size, break else: - raise InputException( + raise glacierexception.InputException( "Requested archive not available. Please make sure "\ "your archive ID is correct, and start a retrieval "\ "job using getarchive' if necessary.", @@ -1379,7 +1391,7 @@ def download(self, vault_name, archive_id, part_size, out_file = None if out_file_name: if os.path.isfile(out_file_name) and not overwrite: - raise InputException( + raise glacierexception.InputException( "File exists already, aborting. "\ "Use the overwrite flag to overwrite "\ "existing file.", @@ -1387,7 +1399,7 @@ def download(self, vault_name, archive_id, part_size, try: out_file = open(out_file_name, 'w') except IOError as e: - raise InputException( + raise glacierexception.InputException( "Cannot access the ouput file.", cause=e, code='FileError') @@ -1402,7 +1414,7 @@ def download(self, vault_name, archive_id, part_size, # Log our pending action. if out_file: self.logger.debug("Starting download of archive "\ - "to file %s."% out_file_name) + "to file {}.".format(out_file_name)) else: self.logger.debug("Starting download of archive to stdout.") @@ -1421,8 +1433,8 @@ def download(self, vault_name, archive_id, part_size, data = response.read() except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( - 'Failed to download archive %s.'% archive_id, + raise glacierexception.ResponseException( + 'Failed to download archive {}.'.format(archive_id), cause=self._decode_error_message(e.body), code=e.code) @@ -1432,7 +1444,7 @@ def download(self, vault_name, archive_id, part_size, try: out_file.write(response.read()) except IOError as e: - raise InputException( + raise glacierexception.InputException( "Cannot write data to the specified file.", cause=e, code='FileError') @@ -1469,7 +1481,7 @@ def download(self, vault_name, archive_id, part_size, if out_file: out_file.close() if glaciercorecalls.bytes_to_hex(glaciercorecalls.tree_hash(hash_list)) != download_job['SHA256TreeHash']: - raise CommunicationException( + raise glacierexception.CommunicationException( "Downloaded data hash mismatch", code="DownloadError", cause=None) @@ -1511,7 +1523,7 @@ def search(self, vault=None, region=None, # Sanity checking. if not self.bookkeeping: - raise InputException( + raise glacierexception.InputException( "You must enable bookkeeping to be able to do searches.", cause='Bookkeeping not enabled.', code='BookkeepingError') @@ -1555,7 +1567,7 @@ def search(self, vault=None, region=None, if item.has_key('archive_id'): items.append(item) except boto.exception.SDBResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( 'SimpleDB did not like your query with parameters %s.'% search_params, cause=self._decode_error_message(e.body), code=e.code) @@ -1583,7 +1595,7 @@ def rmarchive(self, vault_name, archive_id): try: self.glacierconn.delete_archive(vault_name, archive_id) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( 'Failed to remove archive %s from vault %s.'% (archive_id, vault_name), cause=self._decode_error_message(e.body), code=e.code) @@ -1595,7 +1607,7 @@ def rmarchive(self, vault_name, archive_id): if item: self.sdb_domain.delete_item(item) except boto.exception.SDBResponseError as e: - raise CommunicationException( + raise glacierexception.CommunicationException( "Cannot delete item from Amazon SimpleDB.", code="SdbWriteError", cause=e) @@ -1706,7 +1718,7 @@ def inventory(self, vault_name, refresh): try: self.sdb_domain.batch_put_attributes(items) except boto.exception.SDBResponseError as e: - raise CommunicationException( + raise glacierexception.CommunicationException( "Cannot update inventory cache, "\ "Amazon SimpleDB is not happy.", cause=e, @@ -1722,7 +1734,7 @@ def inventory(self, vault_name, refresh): try: self.sdb_domain.batch_put_attributes(items) except boto.exception.SDBResponseError as e: - raise CommunicationException( + raise glacierexception.CommunicationException( "Cannot update inventory cache, "\ "Amazon SimpleDB is not happy.", cause=e, @@ -1742,7 +1754,7 @@ def inventory(self, vault_name, refresh): "{}.".format(item.name)) except boto.exception.SDBResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( "SimpleDB did not respond correctly "\ "to our inventory check.", cause=self._decode_error_message(e.body), @@ -1759,7 +1771,7 @@ def inventory(self, vault_name, refresh): new_job = self.glacierconn.initiate_job(vault_name, job_data) except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: - raise ResponseException( + raise glacierexception.ResponseException( "Failed to create a new inventory retrieval "\ "job for vault {}.".format(vault_name), cause=self._decode_error_message(e.body), @@ -1784,7 +1796,7 @@ def get_tree_hash(self, file_name): try: reader = open(file_name, 'rb') except IOError as e: - raise InputException( + raise glacierexception.InputException( "Could not access the file given: %s." % file_name, cause=e, code='FileError') @@ -1859,7 +1871,7 @@ def sns_sync(self, sns_options, output): try: protocol, endpoint = method.split(',') except ValueError: - raise InputException( + raise glacierexception.InputException( ("If you specify method, you should ""use format " "'protocol1,endpoint1;protocol2,endpoint2'."), cause='Wrong method format in configuration file.', @@ -1920,7 +1932,7 @@ def sns_subscribe(self, protocol, endpoint, topic, sns_options, vault_names=None 'RequestId': result['ResponseMetadata']['RequestId']}] return results except boto.exception.BotoServerError as e: - raise ResponseException("Failed to subscribe to "\ + raise glacierexception.ResponseException("Failed to subscribe to "\ "notifications for vault"\ " %s." % vault_name, cause=self._decode_error_message(e.body), @@ -1975,7 +1987,7 @@ def sns_list_subscriptions(self, protocol, endpoint, topic, sns_options): @sns_connect def sns_unsubscribe(self, protocol, endpoint, topic, sns_options): if not protocol or not endpoint or not topic: - raise InputException( + raise glacierexception.InputException( ("You must specify at least one parameter that will " "by which subscriptions will be canceled."), cause='No parameters given for unsubscribe.', diff --git a/glacier/modules/constants.py b/glacier/modules/constants.py index 20cb723..9cd4ee3 100644 --- a/glacier/modules/constants.py +++ b/glacier/modules/constants.py @@ -1,4 +1,3 @@ - # constants definition CHUNK_SIZE = 1024 DEFAULT_PART_SIZE = 128 # in MB, power of 2. diff --git a/glacier/modules/decorators.py b/glacier/modules/decorators.py index c20969f..0aac0ef 100644 --- a/glacier/modules/decorators.py +++ b/glacier/modules/decorators.py @@ -11,6 +11,7 @@ def handle_errors(fn): """ Decorator for exception handling. """ + @wraps(fn) def wrapper(*args, **kwargs): try: diff --git a/glacier/modules/glaciercorecalls.py b/glacier/modules/glaciercorecalls.py index 3b224df..2f83cba 100755 --- a/glacier/modules/glaciercorecalls.py +++ b/glacier/modules/glaciercorecalls.py @@ -28,14 +28,13 @@ # import our modules from modules import constants -from modules.glacierexception import * +from modules import glacierexception # Placeholder, effectively renaming the class. class GlacierConnection(boto.glacier.layer1.Layer1): pass - def chunk_hashes(data): """ Break up the byte-string into 1MB chunks and return sha256 hashes @@ -105,14 +104,14 @@ def __init__(self, connection, vault_name, def write(self, data): if self.closed: - raise CommunicationError( + raise glacierexception.CommunicationError( "Tried to write to a GlacierWriter that is "\ "already closed.", code='InternalError') len_data = len(data) if len_data > self.part_size: - raise InputException ( + raise glacierexception.InputException ( "Block of data provided({}) "\ "must be equal " "to or smaller than the set " @@ -157,7 +156,8 @@ def write(self, data): if self.logger: self.logger.warning("Retries exhausted "\ - "({}) for this block.".format(constants.BLOCK_RETRIES)) + "({}) for this "\ + "block.".format(constants.BLOCK_RETRIES)) raise e if uploaded_gb > 0: @@ -169,7 +169,8 @@ def write(self, data): if self.logger: self.logger.warning("Total retries "\ - "exhausted({}).".format(constants.TOTAL_RETRIES)) + "exhausted"\ + "({}).".format(constants.TOTAL_RETRIES)) raise e retries += 1 @@ -178,13 +179,24 @@ def write(self, data): if self.logger: self.logger.warning(e.message) if sys.version_info < (2, 7, 0): - self.logger.warning('Total uploaded size = %d, block hash = %s' % (self.uploaded_size, bytes_to_hex(part_tree_hash))) + self.logger.warning("Total uploaded size " + "= {}, block hash = " + "{}".format(self.uploaded_size, + bytes_to_hex(part_tree_hash))) else: # Commify large numbers - self.logger.warning('Total uploaded size = {:,d}, block hash = {:}'.format(self.uploaded_size, bytes_to_hex(part_tree_hash))) - - self.logger.warning('Retries (this block, total) = %d/%d, %d/%d' % (retries, constants.BLOCK_RETRIES, self.total_retries, constants.TOTAL_RETRIES)) - self.logger.warning('Check the AWS status at: http://status.aws.amazon.com/') + self.logger.warning("Total uploaded size " + "= {:,d}, block hash = " + "{:}".format(self.uploaded_size, + bytes_to_hex(part_tree_hash))) + + self.logger.warning("Retries (this block, " + "total) = {}/{}, {}/{}".format(retries, + constants.BLOCK_RETRIES, + self.total_retries, + constants.TOTAL_RETRIES)) + self.logger.warning("Check the AWS status " + "at: http://status.aws.amazon.com/") self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (constants.SLEEP_TIME, constants.SLEEP_TIME / 60.0)) time.sleep(constants.SLEEP_TIME * retries) diff --git a/glacier/modules/glacierexception.py b/glacier/modules/glacierexception.py index a4d0e9f..8e79e59 100644 --- a/glacier/modules/glacierexception.py +++ b/glacier/modules/glacierexception.py @@ -53,7 +53,7 @@ def __init__(self, message, code=None, cause=None): self.exitcode = constants.ERRORCODE[code] if code in constants.ERRORCODE else 254 self.code = code if cause: - self.logger.error('ERROR: %s'% cause) + self.logger.error('ERROR:{}'.format(cause)) self.cause = cause if isinstance(cause, tuple) else (cause,) self.stack = traceback.format_stack()[:-2] @@ -73,7 +73,7 @@ def __init__(self, message, code=None, cause=None): self.logger.info(self.fetch(message=True)) self.logger.debug(self.fetch(stack=True)) if self.exitcode == 254: - self.logger.debug('Unknown error code: %s.'% code) + self.logger.debug('Unknown error code: {}'.format(code)) # Works as a generator to help get the stack trace and the cause # written out. diff --git a/glacier/modules/sns_utils.py b/glacier/modules/sns_utils.py index 15f1832..56377e5 100644 --- a/glacier/modules/sns_utils.py +++ b/glacier/modules/sns_utils.py @@ -2,6 +2,10 @@ # -*- coding: utf-8 -*- # import modules +from modules.glacier_utils import default_glacier_wrapper + +# function definition + def snssync(args): """ If monitored_vaults is specified in configuration file, subscribe @@ -41,12 +45,15 @@ def snslistsubscriptions(args): topic = args.topic glacier = default_glacier_wrapper(args) - response = glacier.sns_list_subscriptions(protocol, endpoint, - topic, - sns_options=args.sns_options) + response = glacier.sns_list_subscriptions(protocol, + endpoint, topic, sns_options=args.sns_options) output_table(response, args.output) def snslisttopics(args): + """ + List subscriptions. + """ + glacier = default_glacier_wrapper(args) response = glacier.sns_list_topics(sns_options=args.sns_options) output_table(response, args.output) @@ -63,6 +70,5 @@ def snsunsubscribe(args): glacier = default_glacier_wrapper(args) response = glacier.sns_unsubscribe(protocol, endpoint, - topic, - sns_options=args.sns_options) + topic, sns_options=args.sns_options) output_table(response, args.output) From 7c7a08f98849d6a657e80322fc424559c577e5d0 Mon Sep 17 00:00:00 2001 From: marzona Date: Tue, 4 Jul 2017 18:54:04 +0100 Subject: [PATCH 4/6] - fixed import modules - updated region lists (removed wrong one, adding others new) - other minor updates --- glacier/modules/constants.py | 15 ++++++++++++--- glacier/modules/decorators.py | 2 +- glacier/modules/glacierexception.py | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/glacier/modules/constants.py b/glacier/modules/constants.py index 9cd4ee3..8feadb5 100644 --- a/glacier/modules/constants.py +++ b/glacier/modules/constants.py @@ -47,18 +47,27 @@ MAX_VAULT_NAME_LENGTH = 255 MAX_VAULT_DESCRIPTION_LENGTH = 1024 MAX_PARTS = 10000 -AVAILABLE_REGIONS = ('us-east-1', 'us-west-2', 'us-west-1', - 'eu-west-1', 'eu-central-1', 'sa-east-1', - 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2') +AVAILABLE_REGIONS = ('us-east-1', 'us-east-2', + 'us-west-2', 'us-west-1', + 'ca-central-1' + 'eu-west-1', 'eu-central-1', 'eu-west-2', + 'ap-south-1', + 'ap-northeast-1','ap-northeast-2', + 'ap-southeast-1', 'ap-southeast-2') AVAILABLE_REGIONS_MESSAGE = """\ Invalid region. Available regions for Amazon Glacier are: us-east-1 (US - Virginia) + us-east-2 (US - Ohio) us-west-1 (US - N. California) us-west-2 (US - Oregon) eu-west-1 (EU - Ireland) + eu-west-2 (EU - London) + ca-central-1 (CA - Canada) eu-central-1 (EU - Frankfurt) sa-east-1 (South America - Sao Paulo) + ap-south-1 (Mumbai) ap-northeast-1 (Asia-Pacific - Tokyo) + ap-northeast-2 (Asia-Pacific - Seul) ap-southeast-1 (Asia Pacific (Singapore) ap-southeast-2 (Asia-Pacific - Sydney)\ """ diff --git a/glacier/modules/decorators.py b/glacier/modules/decorators.py index 0aac0ef..dc6ebd8 100644 --- a/glacier/modules/decorators.py +++ b/glacier/modules/decorators.py @@ -3,7 +3,7 @@ # import modules from functools import wraps - +import sys # import our modules from modules import glacierexception diff --git a/glacier/modules/glacierexception.py b/glacier/modules/glacierexception.py index 8e79e59..43ba125 100644 --- a/glacier/modules/glacierexception.py +++ b/glacier/modules/glacierexception.py @@ -141,7 +141,7 @@ def fetch(self, indentation=' ', message=False, stack=False): out = '' for line in self.causeTree(indentation, message=message, - tack=stack): + stack=stack): out += line return out From d44c44e7d29e5807d4eb7674fd844247e453c47f Mon Sep 17 00:00:00 2001 From: marzona Date: Tue, 4 Jul 2017 19:02:57 +0100 Subject: [PATCH 5/6] addedn cn-north-1 region --- glacier/modules/constants.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/glacier/modules/constants.py b/glacier/modules/constants.py index 8feadb5..f6ea8fc 100644 --- a/glacier/modules/constants.py +++ b/glacier/modules/constants.py @@ -53,7 +53,8 @@ 'eu-west-1', 'eu-central-1', 'eu-west-2', 'ap-south-1', 'ap-northeast-1','ap-northeast-2', - 'ap-southeast-1', 'ap-southeast-2') + 'ap-southeast-1', 'ap-southeast-2', + 'cn-north-1') AVAILABLE_REGIONS_MESSAGE = """\ Invalid region. Available regions for Amazon Glacier are: us-east-1 (US - Virginia) @@ -69,7 +70,8 @@ ap-northeast-1 (Asia-Pacific - Tokyo) ap-northeast-2 (Asia-Pacific - Seul) ap-southeast-1 (Asia Pacific (Singapore) - ap-southeast-2 (Asia-Pacific - Sydney)\ + ap-southeast-2 (Asia-Pacific - Sydney) + cn-north-1 (China - Beijing) """ From 1b6908c83e505f7f577aa212df14fb6113b22cdd Mon Sep 17 00:00:00 2001 From: marzona Date: Tue, 4 Jul 2017 19:24:54 +0100 Subject: [PATCH 6/6] - fixed import issue causing a module not to be avaiable --- glacier/modules/GlacierWrapper.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/glacier/modules/GlacierWrapper.py b/glacier/modules/GlacierWrapper.py index 645447e..2968732 100755 --- a/glacier/modules/GlacierWrapper.py +++ b/glacier/modules/GlacierWrapper.py @@ -37,7 +37,7 @@ # import our modules from modules import constants -from modules.glaciercorecalls import GlacierConnection, GlacierWriter +from modules import glaciercorecalls from modules import glacierexception class log_class_call(object): @@ -208,9 +208,8 @@ def glacier_connect_wrap(*args, **kwargs): self.aws_access_key, self.aws_secret_key, self.region) - self.glacierconn = GlacierConnection(self.aws_access_key, - self.aws_secret_key, - region_name=self.region) + self.glacierconn = glaciercorecalls.GlacierConnection(self.aws_access_key, + self.aws_secret_key, region_name=self.region) except boto.exception.AWSConnectionError as e: raise glacierexception.ConnectionException( "Cannot connect to Amazon Glacier.", @@ -471,7 +470,7 @@ def _part_size_for_total_size(total_size): if total_size > 0: part_size = _part_size_for_total_size(total_size) else: - part_size = GlacierWriter.DEFAULT_PART_SIZE + part_size = glaciercorecalls.GlacierWriter.DEFAULT_PART_SIZE else: ps = self._next_power_of_2(part_size) if not ps == part_size: @@ -1074,7 +1073,7 @@ def upload(self, vault_name, file_name, description, region, code='IdError') # Initialise the writer task. - writer = GlacierWriter(self.glacierconn, vault_name, + writer = glaciercorecalls.GlacierWriter(self.glacierconn, vault_name, description=description, part_size_in_bytes=part_size_in_bytes, uploadid=uploadid, logger=self.logger)