Skip to content

please merge #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Marzona wants to merge 6 commits into
base: master
Choose a base branch
from
Open

please merge #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b9187db
- added/updated docstrings
Marzona Jun 10, 2017
a6ea7ea
Still working mainly on glacier.py
Marzona Jun 11, 2017
273f677
- code maintenance: removing %s, reducing import scope, split code in…
Marzona Jul 4, 2017
7c7a08f
- fixed import modules
Marzona Jul 4, 2017
d44c44e
addedn cn-north-1 region
Marzona Jul 4, 2017
1b6908c
- fixed import issue causing a module not to be avaiable
Marzona Jul 4, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
246 changes: 22 additions & 224 deletions glacier/glacier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,180 +6,26 @@
:synopsis: Command line interface for amazon glacier
"""

# import modules

import sys
import os
import ConfigParser
import argparse
import re
import locale
import glob
import csv
import json

import glacierexception
import constants

from prettytable import PrettyTable
from GlacierWrapper import GlacierWrapper
from functools import wraps

def output_headers(headers, output):
"""
Prints a list of headers - single item output.

:param headers: the output to be printed as {'header1':'data1',...}
:type headers: dict
"""
rows = [(k, headers[k]) for k in headers.keys()]
if output == 'print':
table = PrettyTable(["Header", "Value"])
for row in rows:
if len(str(row[1])) <= 138:
table.add_row(row)

print table

if output not in constants.HEADERS_OUTPUT_FORMAT:
raise ValueError("Output format must be {}, got"
":{}".format(constants.HEADERS_OUTPUT_FORMAT,
output))

if output == 'csv':
csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)
for row in rows:
csvwriter.writerow(row)

if output == 'json':
print json.dumps(headers)

def output_table(results, output, keys=None, sort_key=None):
"""
Prettyprints results. Expects a list of identical dicts.
Use the dict keys as headers unless keys is given;
one line for each item.

Expected format of data is a list of dicts:
[{'key1':'data1.1', 'key2':'data1.2', ... },
{'key1':'data1.2', 'key2':'data2.2', ... },
...]
keys: dict of headers to be printed for each key:
{'key1':'header1', 'key2':'header2',...}

sort_key: the key to use for sorting the table.
"""

if output not in constants.TABLE_OUTPUT_FORMAT:
raise ValueError("Output format must be{}, "
"got {}".format(constants.TABLE_OUTPUT_FORMAT,
output))
if output == 'print':
if len(results) == 0:
print 'No output!'
return

headers = [keys[k] for k in keys.keys()] if keys else results[0].keys()
table = PrettyTable(headers)
for line in results:
table.add_row([line[k] if k in line else '' for k in (keys.keys() if keys else headers)])

if sort_key:
table.sortby = keys[sort_key] if keys else sort_key

print table

if output == 'csv':
csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)
keys = results[0].keys()
csvwriter.writerow(keys)
for row in results:
csvwriter.writerow([row[k] for k in keys])

if output == 'json':
print json.dumps(results)

def output_msg(msg, output, success=True):
"""
In case of a single message output, e.g. nothing found.

:param msg: a single message to output.
:type msg: str
:param success: whether the operation was a success or not.
:type success: boolean
"""

if output not in constants.TABLE_OUTPUT_FORMAT:
raise ValueError("Output format must be{}, "
"got {}".format(constants.TABLE_OUTPUT_FORMAT,
output))

if msg is not None:
if output == 'print':
print msg

if output == 'csv':
csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)
csvwriter.writerow(msg)

if output == 'json':
print json.dumps(msg)

if not success:
sys.exit(125)

def size_fmt(num, decimals = 1):
"""
Formats file sizes in human readable format. Anything bigger than
TB is returned is TB. Number of decimals is optional,
defaults to 1.
"""
fmt = "%%3.%sf %%s"% decimals
for x in ['bytes','KB','MB','GB']:
if num < 1024.0:
return fmt % (num, x)

num /= 1024.0

return fmt % (num, 'TB')

def default_glacier_wrapper(args, **kwargs):
"""
Convenience function to call an instance of GlacierWrapper
with all required arguments.
"""
return GlacierWrapper(args.aws_access_key,
args.aws_secret_key,
args.region,
bookkeeping=args.bookkeeping,
no_bookkeeping=args.no_bookkeeping,
bookkeeping_domain_name=args.bookkeeping_domain_name,
sdb_access_key=args.sdb_access_key,
sdb_secret_key=args.sdb_secret_key,
sdb_region=args.sdb_region,
# sns_enable=args.sns_enable,
# sns_topic=args.sns_topic,
# sns_monitored_vaults=args.sns_monitored_vaults,
# sns_options=args.sns_options,
# config_object=args.config_object,
logfile=args.logfile,
loglevel=args.loglevel,
logtostdout=args.logtostdout)

def handle_errors(fn):
"""
Decorator for exception handling.
"""
@wraps(fn)
def wrapper(*args, **kwargs):
try:
return fn(*args, **kwargs)
except glacierexception.GlacierException as e:
# import our modules
from modules.glacier_utils import (default_glacier_wrapper, size_fmt,
output_msg, output_headers, output_table)
from modules.sns_utils import (snssync, snssubscribe,
snslistsubscriptions, snslisttopics, snsunsubscribe)
from modules.decorators import handle_errors
from modules import glacierexception, constants
from modules.GlacierWrapper import GlacierWrapper

# We are only interested in the error message in case
# it is a self-caused exception.
e.write(indentation='|| ', stack=False, message=True)
sys.exit(e.exitcode)

return wrapper
# function definition

@handle_errors
def lsvault(args):
Expand Down Expand Up @@ -208,6 +54,7 @@ def rmvault(args):
"""
Remove a vault.
"""

glacier = default_glacier_wrapper(args)
response = glacier.rmvault(args.vault)
output_headers(response, args.output)
Expand All @@ -217,6 +64,7 @@ def describevault(args):
"""
Give the description of a vault.
"""

glacier = default_glacier_wrapper(args)
response = glacier.describevault(args.vault)
headers = {'LastInventoryDate': "LastInventory",
Expand All @@ -231,6 +79,7 @@ def listmultiparts(args):
"""
Give an overview of all multipart uploads that are not finished.
"""

glacier = default_glacier_wrapper(args)
response = glacier.listmultiparts(args.vault)
if not response:
Expand All @@ -244,6 +93,7 @@ def abortmultipart(args):
"""
Abort a multipart upload which is in progress.
"""

glacier = default_glacier_wrapper(args)
response = glacier.abortmultipart(args.vault, args.uploadId)
output_headers(response, args.output)
Expand All @@ -253,6 +103,7 @@ def listjobs(args):
"""
List all the active jobs for a vault.
"""

glacier = default_glacier_wrapper(args)
job_list = glacier.list_jobs(args.vault)
if job_list == []:
Expand Down Expand Up @@ -281,6 +132,7 @@ def download(args):
"""
Download an archive.
"""

glacier = default_glacier_wrapper(args)
response = glacier.download(args.vault, args.archive, args.partsize,
out_file_name=args.outfile,
Expand Down Expand Up @@ -368,6 +220,7 @@ def getarchive(args):
"""
Initiate an archive retrieval job.
"""

glacier = default_glacier_wrapper(args)
status, job, jobid = glacier.getarchive(args.vault, args.archive)
output_headers(job, args.output)
Expand All @@ -377,6 +230,7 @@ def rmarchive(args):
"""
Remove an archive from a vault.
"""

glacier = default_glacier_wrapper(args)
glacier.rmarchive(args.vault, args.archive)
output_msg("Archive removed.", args.output, success=True)
Expand All @@ -386,6 +240,7 @@ def search(args):
"""
Search the database for file name or description.
"""

glacier = default_glacier_wrapper(args)
response = glacier.search(vault=args.vault,
region=args.region,
Expand All @@ -398,6 +253,7 @@ def inventory(args):
"""
Fetch latest inventory (or start a retrieval job if not ready).
"""

glacier = default_glacier_wrapper(args)
output = args.output
if sys.stdout.isatty() and output == 'print':
Expand Down Expand Up @@ -439,6 +295,7 @@ def treehash(args):
"""
Calculates the tree hash of the given file(s).
"""

glacier = default_glacier_wrapper(args)
hash_results = []
for f in args.filename:
Expand All @@ -461,66 +318,7 @@ def treehash(args):

output_table(hash_results, args.output)

def snssync(args):
"""
If monitored_vaults is specified in configuration file, subscribe
vaults specificed in it to notifications, otherwiser
subscribe all vault.
"""
glacier = default_glacier_wrapper(args)
response = glacier.sns_sync(sns_options=args.sns_options,
output=args.output)
output_table(response, args.output)

def snssubscribe(args):
"""
Subscribe individual vaults to notifications by method
specified by user.
"""
protocol = args.protocol
endpoint = args.endpoint
vault_names = args.vault
topic = args.topic

glacier = default_glacier_wrapper(args)
response = glacier.sns_subscribe(protocol, endpoint, topic,
vault_names=vault_names,
sns_options=args.sns_options)
output_table(response, args.output)

def snslistsubscriptions(args):
"""
List subscriptions.
"""
protocol = args.protocol
endpoint = args.endpoint
topic = args.topic

glacier = default_glacier_wrapper(args)
response = glacier.sns_list_subscriptions(protocol, endpoint,
topic,
sns_options=args.sns_options)
output_table(response, args.output)

def snslisttopics(args):
glacier = default_glacier_wrapper(args)
response = glacier.sns_list_topics(sns_options=args.sns_options)
output_table(response, args.output)

def snsunsubscribe(args):
"""
Unsubscribe individual vaults from notifications for
specified protocol, endpoint and vault.
"""
protocol = args.protocol
endpoint = args.endpoint
topic = args.topic

glacier = default_glacier_wrapper(args)
response = glacier.sns_unsubscribe(protocol, endpoint,
topic,
sns_options=args.sns_options)
output_table(response, args.output)

def main():
program_description = u"""
Expand Down
Loading