From 3a01ce47591735a144c14f9d253cd63d6d6a192e Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 18 Dec 2018 12:32:40 -0500 Subject: [PATCH 1/6] Linting. --- packagemega/__init__.py | 2 +- packagemega/cli/cli.py | 8 +++--- packagemega/constructed_file.py | 39 +++++++++++------------------ packagemega/custom_errors.py | 14 +++++++++++ packagemega/file.py | 42 +++++++++++++++++++++++++++++++ packagemega/mini_language.py | 36 ++++++++++++++------------- packagemega/repo.py | 42 +++++++++++++++++-------------- packagemega/source_file.py | 44 ++++++++++++--------------------- 8 files changed, 133 insertions(+), 94 deletions(-) create mode 100644 packagemega/file.py diff --git a/packagemega/__init__.py b/packagemega/__init__.py index d4b1989..bce431c 100644 --- a/packagemega/__init__.py +++ b/packagemega/__init__.py @@ -2,4 +2,4 @@ from .repo import * from .base_recipe import * from .source_file import * -from .constructed_file import * \ No newline at end of file +from .constructed_file import * diff --git a/packagemega/cli/cli.py b/packagemega/cli/cli.py index c315ec5..408cd5a 100644 --- a/packagemega/cli/cli.py +++ b/packagemega/cli/cli.py @@ -12,7 +12,7 @@ version = {} version_path = os.path.join(os.path.dirname(__file__), '../version.py') with open(version_path) as version_file: - exec(version_file.read(), version) + exec(version_file.read(), version) # pylint: disable=exec-used @click.group() @@ -25,7 +25,7 @@ def main(): def tableStatus(tblName, statusMap): - '''Check if records in a table are valid and print a report to stdout.''' + """Check if records in a table are valid and print a report to stdout.""" sys.stdout.write('\n{} {}... '.format(len(statusMap), tblName)) allGood = True for name, status in statusMap.items(): @@ -36,8 +36,8 @@ def tableStatus(tblName, statusMap): sys.stdout.write('all good.') -@main.command() -def status(): +@main.command(name='status') +def pm_status(): repo = Repo.loadRepo() sys.stdout.write('Checking status') for tblName, statusMap in repo.dbStatus().items(): diff --git a/packagemega/constructed_file.py b/packagemega/constructed_file.py index 430d5d4..650b4c6 100644 --- a/packagemega/constructed_file.py +++ b/packagemega/constructed_file.py @@ -1,32 +1,21 @@ + +import os.path from gimme_input import UserInput, BoolUserInput + from .custom_errors import UnresolvableFileError -import os.path +from .file import PMFile -class ConstructedFile: - def __init__(self, repo, filename, *args): - self.filename = filename - self.hook = None - if len(args) > 0: - self.hook = args[0] - self._filepath = None - self.repo = repo +class ConstructedFile(PMFile): - def _askUserForFile(self): - _filepath = None - msg = 'Is {} already on this system?'.format(self.filename) - if self.hook is None or BoolUserInput(msg, False).resolve(): - msg = 'Please indicate where {} is stored'.format(self.filename) - _filepath = UserInput(msg).resolve() - return _filepath + def __init__(self, *args, hook=None, **kwargs): + super().__init__(*args, **kwargs) + self.hook = hook - def resolve(self): - actualFile = self._askUserForFile() - if actualFile is None and self.hook is not None: - actualFile = self.hook() - if actualFile is None: - raise UnresolvableFileError() - self._filepath = os.path.abspath(actualFile) + def _resolver(self): + """Return constructor hook.""" + return self.hook - def filepath(self): - return self._filepath + def _resolve_actual_file(self): + """Resolve file by calling hook.""" + return self.hook() diff --git a/packagemega/custom_errors.py b/packagemega/custom_errors.py index 17d0487..d2a1107 100644 --- a/packagemega/custom_errors.py +++ b/packagemega/custom_errors.py @@ -1,7 +1,21 @@ +"""Custom PackageMega errors and exceptions.""" class UnresolvableOperandError(Exception): pass + +class UnresolvableOperandLevel(Exception): + pass + + class UnresolvableFileError(Exception): pass + + +class RecipeNotFoundError(Exception): + pass + + +class InvalidRecipeURI(Exception): + pass diff --git a/packagemega/file.py b/packagemega/file.py new file mode 100644 index 0000000..ab75666 --- /dev/null +++ b/packagemega/file.py @@ -0,0 +1,42 @@ +"""Base PackageMega file class.""" + +from gimme_input import UserInput, BoolUserInput +from subprocess import check_output +import os.path +from .custom_errors import UnresolvableFileError + + +class PMFile: + + def __init__(self, repo, filename, *args, **kwargs): + super().__init__(*args, **kwargs) + self.repo = repo + self.filename = filename + self._filepath = None + + def _askUserForFile(self): + _filepath = None + msg = 'Is {} already on this system?'.format(self.filename) + if self._resolver() is None or BoolUserInput(msg, False).resolve(): + msg = 'Please indicate where {} is stored'.format(self.filename) + _filepath = UserInput(msg).resolve() + return _filepath + + def _resolver(self): + """Return subclass-specific file resolver.""" + raise NotImplementedError() + + def _resolve_actual_file(self): + """Resolve file in subclass-specific way.""" + raise NotImplementedError() + + def resolve(self): + actualFile = self._askUserForFile() + if actualFile is None and self._resolver() is not None: + actualFile = self._resolve_actual_file() + if actualFile is None: + raise UnresolvableFileError() + self._filepath = os.path.abspath(actualFile) + + def filepath(self): + return self._filepath diff --git a/packagemega/mini_language.py b/packagemega/mini_language.py index 5e4ecbb..122edf0 100644 --- a/packagemega/mini_language.py +++ b/packagemega/mini_language.py @@ -1,5 +1,7 @@ + + import os.path -from .custom_errors import UnresolvableOperandError +from .custom_errors import UnresolvableOperandError, UnresolvableOperandLevel def _filePrefix(fs): @@ -48,30 +50,31 @@ def _fileDir(fs): def _processFullOperand(db, operand, subops): - ''' + """ Returns a filepath based on .. should also accept 2 special commands for : prefix and dir which return a shared or fail if that does not exist - ''' + """ fs = {} for r in db.results(): if r.name != '.'.join(subops[:2]): continue - for k, f in r.files(): + for _, f in r.files(): fs[f.name] = f.filepath() + if subops[2] == 'prefix': return _filePrefix(fs) - elif subops[2] == 'dir': + if subops[2] == 'dir': return _fileDir(fs) - else: + + try: + return fs[subops[2]] + except KeyError: try: - return fs[subops[2]] + return fs[operand] except KeyError: - try: - return fs[operand] - except KeyError: - raise UnresolvableOperandError(operand) + raise UnresolvableOperandError(operand) def processOperand(repo, operand, stringify=False): @@ -80,12 +83,9 @@ def processOperand(repo, operand, stringify=False): db = repo.database(subops[0]) if oplevel == 1: - if stringify: - return db.tree() - else: - return db + return db.tree() if stringify else db - elif oplevel == 2: + if oplevel == 2: rs = {r.name: r for r in db.results()} out = [] for k, v in rs.items(): @@ -95,5 +95,7 @@ def processOperand(repo, operand, stringify=False): out = '\n'.join([el.tree() for el in out]) return out - elif oplevel == 3: + if oplevel == 3: return _processFullOperand(db, operand, subops) + + raise UnresolvableOperandLevel(oplevel) diff --git a/packagemega/repo.py b/packagemega/repo.py index 13d1d13..f604716 100644 --- a/packagemega/repo.py +++ b/packagemega/repo.py @@ -1,14 +1,15 @@ +"""PackageMega repository class.""" + +import inspect import os.path -import datasuper as ds from os import listdir, symlink from shutil import copyfile -import sys -import inspect from subprocess import call +import sys +import datasuper as ds -class RecipeNotFoundError(Exception): - pass +from .custom_errors import RecipeNotFoundError, InvalidRecipeURI class Repo: @@ -20,10 +21,13 @@ def __init__(self, abspath): self.stagingDir = os.path.join(self.abspath, 'staging') def addRecipes(self, uri, dev=False): - if os.path.exists(self.uri): - return self.addFromLocal() - elif 'git' in self.uri: - return self.addFromGithub() + if os.path.exists(uri): + return self.addFromLocal(uri) + + if 'git' in uri: + return self.addFromGithub(uri) + + raise InvalidRecipeURI(uri) def addFromLocal(self, uri, dev=False): if uri[-9:] == 'recipe.py': @@ -47,9 +51,9 @@ def addFromLocal(self, uri, dev=False): return out def addFromGithub(self, uri): - hname = self.uri.split('/')[-1].split('.')[0] + hname = uri.split('/')[-1].split('.')[0] dest = os.path.join(self.stagingDir, hname) - cmd = 'git clone {} {}'.format(self.uri, dest) + cmd = 'git clone {} {}'.format(uri, dest) call(cmd, shell=True) self.addFromLocal(dest) @@ -59,8 +63,8 @@ def _recipeName(self, recipeFilename): if r[-1] in ['-', '_', '.']: r = r[:-1] return os.path.basename(r) - else: - assert False and '{} is not a recipe'.format(recipeFilename) + + raise InvalidRecipeURI('{} is not a recipe'.format(recipeFilename)) def allRecipes(self): out = set() @@ -97,6 +101,8 @@ def _loadRecipe(self, recipeName): if name == cname: return c() + raise RecipeNotFoundError(recipeName) + def _getClassName(self, fname): recipeStr = open(fname).read() cname = None @@ -162,10 +168,9 @@ def saveFiles(self, recipe, subName, *filepaths, **kwFilepaths): sample = ds.getOrMakeSample(dsr, recipe.name(), 'db') sample.addResult(result).save(modify=True) - def dsRepo(self): return ds.Repo.loadRepo(self.abspath) - + @staticmethod def loadRepo(): try: @@ -175,7 +180,7 @@ def loadRepo(): targetDir = os.path.join(targetDir, Repo.repoDirName) p = os.path.abspath(targetDir) try: - dsRepo = ds.Repo.loadRepo(p) + _ = ds.Repo.loadRepo(p) except FileNotFoundError: Repo._initRepo() return Repo.loadRepo() @@ -202,7 +207,6 @@ def _initRepo(): def dictify(el): if type(el) == list: return {i: sub for i, sub in enumerate(el)} - elif type(el) == dict: + if type(el) == dict: return el - else: - return {0: el} + return {0: el} diff --git a/packagemega/source_file.py b/packagemega/source_file.py index 368efd5..9ed36ef 100644 --- a/packagemega/source_file.py +++ b/packagemega/source_file.py @@ -1,18 +1,17 @@ -from gimme_input import UserInput, BoolUserInput -from subprocess import check_output + import os.path +from subprocess import check_output +from gimme_input import UserInput, BoolUserInput + from .custom_errors import UnresolvableFileError +from .file import PMFile -class SourceFile: +class SourceFile(PMFile): - def __init__(self, repo, filename, *args): - self.filename = filename - self.url = None - if len(args) > 0: - self.url = args[0] - self._filepath = None - self.repo = repo + def __init__(self, *args, url=None, **kwargs): + super().__init__(*args, **kwargs) + self.url = url def _downloadFile(self): targetPath = os.path.join(self.repo.downloadDir(), self.filename) @@ -20,21 +19,10 @@ def _downloadFile(self): check_output(cmd, shell=True) return targetPath - def _askUserForFile(self): - _filepath = None - msg = 'Is {} already on this system?'.format(self.filename) - if self.url is None or BoolUserInput(msg, False).resolve(): - msg = 'Please indicate where {} is stored'.format(self.filename) - _filepath = UserInput(msg).resolve() - return _filepath - - def resolve(self): - actualFile = self._askUserForFile() - if actualFile is None and self.url is not None: - actualFile = self._downloadFile() - if actualFile is None: - raise UnresolvableFileError() - self._filepath = os.path.abspath(actualFile) - - def filepath(self): - return self._filepath + def _resolver(self): + """Return file url.""" + return self.url + + def _resolve_actual_file(self): + """Resolve file by downloading it.""" + return self._downloadFile() From a25cc9716a951a185a74ff118ce437ed57e01755 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 18 Dec 2018 13:27:48 -0500 Subject: [PATCH 2/6] Add docstrings. --- packagemega/__init__.py | 1 + packagemega/base_recipe.py | 5 +++++ packagemega/cli/__init__.py | 2 ++ packagemega/cli/cli.py | 10 ++++++++-- packagemega/constructed_file.py | 3 +++ packagemega/custom_errors.py | 10 +++++----- packagemega/file.py | 5 +++++ packagemega/mini_language.py | 7 ++++--- packagemega/repo.py | 16 ++++++++++++++++ packagemega/source_file.py | 4 ++++ 10 files changed, 53 insertions(+), 10 deletions(-) diff --git a/packagemega/__init__.py b/packagemega/__init__.py index bce431c..0154b08 100644 --- a/packagemega/__init__.py +++ b/packagemega/__init__.py @@ -1,3 +1,4 @@ +"""Export all PackageMega components.""" from .repo import * from .base_recipe import * diff --git a/packagemega/base_recipe.py b/packagemega/base_recipe.py index 481088d..3d85ff2 100644 --- a/packagemega/base_recipe.py +++ b/packagemega/base_recipe.py @@ -1,10 +1,15 @@ +"""Base PackageMega recipe class.""" + from .repo import Repo class BaseRecipe: + """Base PackageMega recipe class.""" def __init__(self): + """Initialize recipe.""" self.repo = Repo.loadRepo() def makeRecipe(self): + """Create the recipe.""" raise NotImplementedError() diff --git a/packagemega/cli/__init__.py b/packagemega/cli/__init__.py index a5bd848..93eeae0 100644 --- a/packagemega/cli/__init__.py +++ b/packagemega/cli/__init__.py @@ -1 +1,3 @@ +"""Comand Line Interface for PackageMega modules.""" + from .cli import * diff --git a/packagemega/cli/cli.py b/packagemega/cli/cli.py index 408cd5a..7eac627 100644 --- a/packagemega/cli/cli.py +++ b/packagemega/cli/cli.py @@ -18,7 +18,7 @@ @click.group() @click.version_option(version['__version__']) def main(): - pass + """Enter PackageMega CLI.""" ############################################################################### @@ -38,6 +38,7 @@ def tableStatus(tblName, statusMap): @main.command(name='status') def pm_status(): + """Print PackageMega repository status report to stdout.""" repo = Repo.loadRepo() sys.stdout.write('Checking status') for tblName, statusMap in repo.dbStatus().items(): @@ -52,6 +53,7 @@ def pm_status(): help='Install recipe with symlinks') @click.argument('uri') def add(dev, uri): + """Add URI to PackageMega repository.""" repo = Repo.loadRepo() repo.addFromLocal(uri, dev=dev) @@ -61,6 +63,7 @@ def add(dev, uri): @main.command() @click.argument('name') def install(name): + """Install PackageMega recipe.""" repo = Repo.loadRepo() repo.makeRecipe(name) @@ -69,11 +72,12 @@ def install(name): @main.group() def view(): - pass + """Enter group of PackageMega view commands.""" @main.command(name='recipe') def viewRecipes(): + """View all recipes present in PackageMega repository.""" repo = Repo.loadRepo() for recipe in repo.allRecipes(): print(recipe) @@ -82,6 +86,7 @@ def viewRecipes(): def printAllDatabases(repo): + """Print the name of all databased present in repository.""" for db in repo.allDatabases(): print(db.name) @@ -89,6 +94,7 @@ def printAllDatabases(repo): @main.command(name='database') @click.argument('operands', nargs=-1) def viewDatabase(operands): + """Print database names filtered by operand arguments.""" repo = Repo.loadRepo() if len(operands) == 0: printAllDatabases(repo) diff --git a/packagemega/constructed_file.py b/packagemega/constructed_file.py index 650b4c6..6574d8c 100644 --- a/packagemega/constructed_file.py +++ b/packagemega/constructed_file.py @@ -1,3 +1,4 @@ +"""File subclass sourced from a constructor hook.""" import os.path from gimme_input import UserInput, BoolUserInput @@ -7,8 +8,10 @@ class ConstructedFile(PMFile): + """File subclass sourced from a constructor hook.""" def __init__(self, *args, hook=None, **kwargs): + """Initialize by storing the constructor hook.""" super().__init__(*args, **kwargs) self.hook = hook diff --git a/packagemega/custom_errors.py b/packagemega/custom_errors.py index d2a1107..1115cf8 100644 --- a/packagemega/custom_errors.py +++ b/packagemega/custom_errors.py @@ -2,20 +2,20 @@ class UnresolvableOperandError(Exception): - pass + """Raised when database operand cannot be resolved.""" class UnresolvableOperandLevel(Exception): - pass + """Raised when too many operand levels are supplied.""" class UnresolvableFileError(Exception): - pass + """Raised when a file source cannot be resolved.""" class RecipeNotFoundError(Exception): - pass + """Raised when a missing recipe is requested.""" class InvalidRecipeURI(Exception): - pass + """Raised when an invalid recipe URI is supplied.""" diff --git a/packagemega/file.py b/packagemega/file.py index ab75666..c60e7b9 100644 --- a/packagemega/file.py +++ b/packagemega/file.py @@ -7,14 +7,17 @@ class PMFile: + """Base PackageMega file class.""" def __init__(self, repo, filename, *args, **kwargs): + """Initialze PMFile with repo and filename this file is to resolve to.""" super().__init__(*args, **kwargs) self.repo = repo self.filename = filename self._filepath = None def _askUserForFile(self): + """Prompt user for existing file path.""" _filepath = None msg = 'Is {} already on this system?'.format(self.filename) if self._resolver() is None or BoolUserInput(msg, False).resolve(): @@ -31,6 +34,7 @@ def _resolve_actual_file(self): raise NotImplementedError() def resolve(self): + """Create file from subclass-specific resolver.""" actualFile = self._askUserForFile() if actualFile is None and self._resolver() is not None: actualFile = self._resolve_actual_file() @@ -39,4 +43,5 @@ def resolve(self): self._filepath = os.path.abspath(actualFile) def filepath(self): + """Expose private _filepath as read-only.""" return self._filepath diff --git a/packagemega/mini_language.py b/packagemega/mini_language.py index 122edf0..9fc5afe 100644 --- a/packagemega/mini_language.py +++ b/packagemega/mini_language.py @@ -1,4 +1,4 @@ - +"""Filepath support for PackageMega naming convention.""" import os.path from .custom_errors import UnresolvableOperandError, UnresolvableOperandLevel @@ -51,10 +51,10 @@ def _fileDir(fs): def _processFullOperand(db, operand, subops): """ - Returns a filepath based on .. + Return a filepath based on ... should also accept 2 special commands for : prefix and dir - which return a shared or fail if that does not exist + which return a shared or fail if that does not exist. """ fs = {} for r in db.results(): @@ -78,6 +78,7 @@ def _processFullOperand(db, operand, subops): def processOperand(repo, operand, stringify=False): + """Process a full PackageMega operand.""" subops = operand.split('.') oplevel = len(subops) db = repo.database(subops[0]) diff --git a/packagemega/repo.py b/packagemega/repo.py index f604716..a9df501 100644 --- a/packagemega/repo.py +++ b/packagemega/repo.py @@ -13,14 +13,18 @@ class Repo: + """PackageMega repository class.""" + repoDirName = '.package_mega' def __init__(self, abspath): + """Initialize with repository path.""" self.abspath = abspath self.recipeDir = os.path.join(self.abspath, 'recipes') self.stagingDir = os.path.join(self.abspath, 'staging') def addRecipes(self, uri, dev=False): + """Add recipe(s) from URI.""" if os.path.exists(uri): return self.addFromLocal(uri) @@ -30,6 +34,7 @@ def addRecipes(self, uri, dev=False): raise InvalidRecipeURI(uri) def addFromLocal(self, uri, dev=False): + """Add recipe from local filepath.""" if uri[-9:] == 'recipe.py': fs = [uri] else: @@ -51,6 +56,7 @@ def addFromLocal(self, uri, dev=False): return out def addFromGithub(self, uri): + """Add recipe from GitHub URI.""" hname = uri.split('/')[-1].split('.')[0] dest = os.path.join(self.stagingDir, hname) cmd = 'git clone {} {}'.format(uri, dest) @@ -67,6 +73,7 @@ def _recipeName(self, recipeFilename): raise InvalidRecipeURI('{} is not a recipe'.format(recipeFilename)) def allRecipes(self): + """Return set containing names of all recipes in repository.""" out = set() for recipe in listdir(self.recipeDir): try: @@ -77,6 +84,7 @@ def allRecipes(self): return out def makeRecipe(self, recipeName): + """Create a recipe from the recipe name.""" # check if we have the recipe # if not throw an error if recipeName not in self.allRecipes(): @@ -115,6 +123,7 @@ def _getClassName(self, fname): return cname def downloadDir(self): + """Get directory for storing PackageMega downloads.""" try: return os.environ['PACKAGE_MEGA_DOWNLOADS'] except KeyError: @@ -128,18 +137,22 @@ def downloadDir(self): return defaultDatabaseDir def allDatabases(self): + """Return list of all databases present in the repository.""" out = [] for database in self.dsRepo().sampleTable.getAll(): out.append(database) return out def database(self, databaseName): + """Get database by name.""" return self.dsRepo().sampleTable.get(databaseName) def dbStatus(self): + """Get status of underlying DataSuper repository.""" return self.dsRepo().checkStatus() def saveFiles(self, recipe, subName, *filepaths, **kwFilepaths): + """Save a group of files to a recipe using the subName suffix.""" fs = {} for i, filepath in enumerate(filepaths): fs[i] = filepath @@ -169,10 +182,12 @@ def saveFiles(self, recipe, subName, *filepaths, **kwFilepaths): sample.addResult(result).save(modify=True) def dsRepo(self): + """Return the underlying DataSuper repository.""" return ds.Repo.loadRepo(self.abspath) @staticmethod def loadRepo(): + """Load the system-wide PackageMega repository.""" try: targetDir = os.environ['PACKAGE_MEGA_HOME'] except KeyError: @@ -205,6 +220,7 @@ def _initRepo(): def dictify(el): + """Transform element of any type into dictionary.""" if type(el) == list: return {i: sub for i, sub in enumerate(el)} if type(el) == dict: diff --git a/packagemega/source_file.py b/packagemega/source_file.py index 9ed36ef..8af0c26 100644 --- a/packagemega/source_file.py +++ b/packagemega/source_file.py @@ -1,3 +1,4 @@ +"""File subclass sourced from a URI.""" import os.path from subprocess import check_output @@ -8,12 +9,15 @@ class SourceFile(PMFile): + """File subclass sourced from a URI.""" def __init__(self, *args, url=None, **kwargs): + """Initialize by storing the URI.""" super().__init__(*args, **kwargs) self.url = url def _downloadFile(self): + """Download the file.""" targetPath = os.path.join(self.repo.downloadDir(), self.filename) cmd = 'wget {} -O {}'.format(self.url, targetPath) check_output(cmd, shell=True) From bc33b23d0b309d60b15675afc0f86425ed2f8a56 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 18 Dec 2018 13:28:51 -0500 Subject: [PATCH 3/6] Re-enable flake8 and pylint steps in CI. --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a8f9bfa..675c58a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,9 +15,7 @@ jobs: - run: name: Lint code (non-fatal) command: | - set +e tox -e flake8,pylint - exit 0 - run: name: Run tox From f62b54d257619491fdd98c97348f59227edd8cc1 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 18 Dec 2018 13:46:53 -0500 Subject: [PATCH 4/6] Fix flake8 complaints. --- packagemega/__init__.py | 10 ++++++---- packagemega/cli/__init__.py | 2 +- packagemega/constructed_file.py | 4 ---- packagemega/file.py | 4 ++-- packagemega/repo.py | 3 ++- packagemega/source_file.py | 2 -- tests/test_repo.py | 2 -- 7 files changed, 11 insertions(+), 16 deletions(-) diff --git a/packagemega/__init__.py b/packagemega/__init__.py index 0154b08..319ef10 100644 --- a/packagemega/__init__.py +++ b/packagemega/__init__.py @@ -1,6 +1,8 @@ """Export all PackageMega components.""" -from .repo import * -from .base_recipe import * -from .source_file import * -from .constructed_file import * +# flake8: noqa + +from .repo import Repo +from .base_recipe import BaseRecipe +from .source_file import SourceFile +from .constructed_file import ConstructedFile diff --git a/packagemega/cli/__init__.py b/packagemega/cli/__init__.py index 93eeae0..37f0dc8 100644 --- a/packagemega/cli/__init__.py +++ b/packagemega/cli/__init__.py @@ -1,3 +1,3 @@ """Comand Line Interface for PackageMega modules.""" -from .cli import * +from .cli import main # noqa: F401 diff --git a/packagemega/constructed_file.py b/packagemega/constructed_file.py index 6574d8c..13c5803 100644 --- a/packagemega/constructed_file.py +++ b/packagemega/constructed_file.py @@ -1,9 +1,5 @@ """File subclass sourced from a constructor hook.""" -import os.path -from gimme_input import UserInput, BoolUserInput - -from .custom_errors import UnresolvableFileError from .file import PMFile diff --git a/packagemega/file.py b/packagemega/file.py index c60e7b9..c0e52b0 100644 --- a/packagemega/file.py +++ b/packagemega/file.py @@ -1,8 +1,8 @@ """Base PackageMega file class.""" -from gimme_input import UserInput, BoolUserInput -from subprocess import check_output import os.path +from gimme_input import UserInput, BoolUserInput + from .custom_errors import UnresolvableFileError diff --git a/packagemega/repo.py b/packagemega/repo.py index a9df501..4a73a09 100644 --- a/packagemega/repo.py +++ b/packagemega/repo.py @@ -195,7 +195,8 @@ def loadRepo(): targetDir = os.path.join(targetDir, Repo.repoDirName) p = os.path.abspath(targetDir) try: - _ = ds.Repo.loadRepo(p) + # Ignore the return value + ds.Repo.loadRepo(p) except FileNotFoundError: Repo._initRepo() return Repo.loadRepo() diff --git a/packagemega/source_file.py b/packagemega/source_file.py index 8af0c26..bf21c84 100644 --- a/packagemega/source_file.py +++ b/packagemega/source_file.py @@ -2,9 +2,7 @@ import os.path from subprocess import check_output -from gimme_input import UserInput, BoolUserInput -from .custom_errors import UnresolvableFileError from .file import PMFile diff --git a/tests/test_repo.py b/tests/test_repo.py index 407d5da..413d559 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,7 +1,5 @@ """Basic test suite for PackgeMega.""" -import os - from packagemega import Repo From 17dbb1646d43c1881d2198ac2ba5b6129901e253 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 18 Dec 2018 14:31:29 -0500 Subject: [PATCH 5/6] Renable all pylint rules. --- .pylintrc | 6 +- packagemega/base_recipe.py | 6 +- packagemega/cli/cli.py | 54 ++++----- packagemega/file.py | 12 +- packagemega/mini_language.py | 56 ++++----- packagemega/repo.py | 220 +++++++++++++++++------------------ packagemega/source_file.py | 10 +- 7 files changed, 181 insertions(+), 183 deletions(-) diff --git a/.pylintrc b/.pylintrc index 564216b..615e8a6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,8 +1,5 @@ [MASTER] -# Temporary disable list -disable=invalid-name,unused-wildcard-import,wildcard-import,unused-import,wrong-import-order,missing-docstring,no-self-use,unused-argument,unidiomatic-typecheck,too-many-branches,too-many-arguments,too-many-locals,len-as-condition,line-too-long,too-many-instance-attributes,too-few-public-methods - # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code @@ -395,7 +392,8 @@ good-names=i, k, ex, Run, - _ + _, + db # Include a hint for the correct naming format with invalid-name include-naming-hint=no diff --git a/packagemega/base_recipe.py b/packagemega/base_recipe.py index 3d85ff2..db018ca 100644 --- a/packagemega/base_recipe.py +++ b/packagemega/base_recipe.py @@ -3,13 +3,13 @@ from .repo import Repo -class BaseRecipe: +class BaseRecipe: # pylint: disable=too-few-public-methods """Base PackageMega recipe class.""" def __init__(self): """Initialize recipe.""" - self.repo = Repo.loadRepo() + self.repo = Repo.load_repo() - def makeRecipe(self): + def make_recipe(self): """Create the recipe.""" raise NotImplementedError() diff --git a/packagemega/cli/cli.py b/packagemega/cli/cli.py index 7eac627..7388530 100644 --- a/packagemega/cli/cli.py +++ b/packagemega/cli/cli.py @@ -5,12 +5,12 @@ import click from packagemega import Repo -from packagemega.mini_language import processOperand +from packagemega.mini_language import process_operand from packagemega.custom_errors import UnresolvableOperandError -version = {} -version_path = os.path.join(os.path.dirname(__file__), '../version.py') +version = {} # pylint: disable=invalid-name +version_path = os.path.join(os.path.dirname(__file__), '../version.py') # pylint: disable=invalid-name with open(version_path) as version_file: exec(version_file.read(), version) # pylint: disable=exec-used @@ -24,25 +24,25 @@ def main(): ############################################################################### -def tableStatus(tblName, statusMap): +def table_status(tbl_name, status_map): """Check if records in a table are valid and print a report to stdout.""" - sys.stdout.write('\n{} {}... '.format(len(statusMap), tblName)) - allGood = True - for name, status in statusMap.items(): + sys.stdout.write('\n{} {}... '.format(len(status_map), tbl_name)) + all_good = True + for name, status in status_map.items(): if not status: - allGood = False + all_good = False sys.stdout.write('\n - {} failed'.format(name)) - if allGood: + if all_good: sys.stdout.write('all good.') @main.command(name='status') def pm_status(): """Print PackageMega repository status report to stdout.""" - repo = Repo.loadRepo() + repo = Repo.load_repo() sys.stdout.write('Checking status') - for tblName, statusMap in repo.dbStatus().items(): - tableStatus(tblName, statusMap) + for tbl_name, status_map in repo.db_status().items(): + table_status(tbl_name, status_map) sys.stdout.write('\nDone\n') ############################################################################### @@ -54,8 +54,8 @@ def pm_status(): @click.argument('uri') def add(dev, uri): """Add URI to PackageMega repository.""" - repo = Repo.loadRepo() - repo.addFromLocal(uri, dev=dev) + repo = Repo.load_repo() + repo.add_from_local(uri, dev=dev) ############################################################################### @@ -64,8 +64,8 @@ def add(dev, uri): @click.argument('name') def install(name): """Install PackageMega recipe.""" - repo = Repo.loadRepo() - repo.makeRecipe(name) + repo = Repo.load_repo() + repo.make_recipe(name) ############################################################################### @@ -76,32 +76,32 @@ def view(): @main.command(name='recipe') -def viewRecipes(): +def view_recipes(): """View all recipes present in PackageMega repository.""" - repo = Repo.loadRepo() - for recipe in repo.allRecipes(): + repo = Repo.load_repo() + for recipe in repo.all_recipes(): print(recipe) ############################################################################### -def printAllDatabases(repo): +def print_all_databases(repo): """Print the name of all databased present in repository.""" - for db in repo.allDatabases(): + for db in repo.all_databases(): print(db.name) @main.command(name='database') @click.argument('operands', nargs=-1) -def viewDatabase(operands): +def view_database(operands): """Print database names filtered by operand arguments.""" - repo = Repo.loadRepo() - if len(operands) == 0: - printAllDatabases(repo) + repo = Repo.load_repo() + if not operands: + print_all_databases(repo) for operand in operands: try: - el = processOperand(repo, operand, stringify=True) - print(el) + element = process_operand(repo, operand, stringify=True) + print(element) except UnresolvableOperandError: print('{} could not be resolved.'.format(operand), file=sys.stderr) diff --git a/packagemega/file.py b/packagemega/file.py index c0e52b0..2c81355 100644 --- a/packagemega/file.py +++ b/packagemega/file.py @@ -16,7 +16,7 @@ def __init__(self, repo, filename, *args, **kwargs): self.filename = filename self._filepath = None - def _askUserForFile(self): + def _ask_user_for_file(self): """Prompt user for existing file path.""" _filepath = None msg = 'Is {} already on this system?'.format(self.filename) @@ -35,12 +35,12 @@ def _resolve_actual_file(self): def resolve(self): """Create file from subclass-specific resolver.""" - actualFile = self._askUserForFile() - if actualFile is None and self._resolver() is not None: - actualFile = self._resolve_actual_file() - if actualFile is None: + actual_file = self._ask_user_for_file() + if actual_file is None and self._resolver() is not None: + actual_file = self._resolve_actual_file() + if actual_file is None: raise UnresolvableFileError() - self._filepath = os.path.abspath(actualFile) + self._filepath = os.path.abspath(actual_file) def filepath(self): """Expose private _filepath as read-only.""" diff --git a/packagemega/mini_language.py b/packagemega/mini_language.py index 9fc5afe..cd21e19 100644 --- a/packagemega/mini_language.py +++ b/packagemega/mini_language.py @@ -4,18 +4,18 @@ from .custom_errors import UnresolvableOperandError, UnresolvableOperandLevel -def _filePrefix(fs): +def _file_prefix(file_sources): out = '' - ls = [len(fpath) for fpath in fs.values()] - for i in range(min(ls)): - cs = [fpath[i] for fpath in fs.values()] + path_lengths = [len(fpath) for fpath in file_sources.values()] + for i in range(min(path_lengths)): + components = [fpath[i] for fpath in file_sources.values()] consensus = True - for j in range(len(cs) - 1): - if cs[j] != cs[j + 1]: + for j in range(len(components) - 1): + if components[j] != components[j + 1]: consensus = False break if consensus: - out += cs[0] + out += components[0] else: break if out[-1] == '.': @@ -23,11 +23,11 @@ def _filePrefix(fs): return out -def _fileDir(fs): - if len(fs) == 1: - return os.path.dirname(fs.values()[0]) +def _file_dir(file_sources): + if len(file_sources) == 1: + return os.path.dirname(file_sources.values()[0]) sections = [] - for path in fs.values(): + for path in file_sources.values(): for i, section in enumerate(path.split('/')): while len(sections) <= i: sections.append([]) @@ -49,35 +49,35 @@ def _fileDir(fs): return fdir -def _processFullOperand(db, operand, subops): +def _process_full_operand(db, operand, subops): """ Return a filepath based on ... should also accept 2 special commands for : prefix and dir which return a shared or fail if that does not exist. """ - fs = {} - for r in db.results(): - if r.name != '.'.join(subops[:2]): + file_sources = {} + for result in db.results(): + if result.name != '.'.join(subops[:2]): continue - for _, f in r.files(): - fs[f.name] = f.filepath() + for _, result_file in result.files(): + file_sources[result_file.name] = result_file.filepath() if subops[2] == 'prefix': - return _filePrefix(fs) + return _file_prefix(file_sources) if subops[2] == 'dir': - return _fileDir(fs) + return _file_dir(file_sources) try: - return fs[subops[2]] + return file_sources[subops[2]] except KeyError: try: - return fs[operand] + return file_sources[operand] except KeyError: raise UnresolvableOperandError(operand) -def processOperand(repo, operand, stringify=False): +def process_operand(repo, operand, stringify=False): """Process a full PackageMega operand.""" subops = operand.split('.') oplevel = len(subops) @@ -87,16 +87,16 @@ def processOperand(repo, operand, stringify=False): return db.tree() if stringify else db if oplevel == 2: - rs = {r.name: r for r in db.results()} + results = {result.name: result for result in db.results()} out = [] - for k, v in rs.items(): - if str(k) == operand: - out.append(v) + for key, value in results.items(): + if str(key) == operand: + out.append(value) if stringify: - out = '\n'.join([el.tree() for el in out]) + out = '\n'.join([element.tree() for element in out]) return out if oplevel == 3: - return _processFullOperand(db, operand, subops) + return _process_full_operand(db, operand, subops) raise UnresolvableOperandLevel(oplevel) diff --git a/packagemega/repo.py b/packagemega/repo.py index 4a73a09..7e2b1c0 100644 --- a/packagemega/repo.py +++ b/packagemega/repo.py @@ -15,106 +15,106 @@ class Repo: """PackageMega repository class.""" - repoDirName = '.package_mega' + repo_dir_name = '.package_mega' def __init__(self, abspath): """Initialize with repository path.""" self.abspath = abspath - self.recipeDir = os.path.join(self.abspath, 'recipes') - self.stagingDir = os.path.join(self.abspath, 'staging') + self.recipe_dir = os.path.join(self.abspath, 'recipes') + self.staging_dir = os.path.join(self.abspath, 'staging') - def addRecipes(self, uri, dev=False): + def add_recipes(self, uri, dev=False): """Add recipe(s) from URI.""" if os.path.exists(uri): - return self.addFromLocal(uri) + return self.add_from_local(uri) if 'git' in uri: - return self.addFromGithub(uri) + return self.add_from_github(uri) raise InvalidRecipeURI(uri) - def addFromLocal(self, uri, dev=False): + def add_from_local(self, uri, dev=False): """Add recipe from local filepath.""" if uri[-9:] == 'recipe.py': - fs = [uri] + file_sources = [uri] else: - fs = [os.path.join(uri, f) for f in listdir(uri)] + file_sources = [os.path.join(uri, dir_file) for dir_file in listdir(uri)] out = [] - for f in fs: - if f[-9:] == 'recipe.py': - absP = os.path.abspath(f) - target = os.path.basename(f) - target = os.path.join(self.recipeDir, target) + for file_source in file_sources: + if file_source[-9:] == 'recipe.py': + abs_path = os.path.abspath(file_source) + target = os.path.basename(file_source) + target = os.path.join(self.recipe_dir, target) if dev: try: - symlink(absP, target) + symlink(abs_path, target) except FileExistsError: pass else: - copyfile(absP, target) - out.append(self._recipeName(f)) + copyfile(abs_path, target) + out.append(self._recipe_name(file_source)) return out - def addFromGithub(self, uri): + def add_from_github(self, uri): """Add recipe from GitHub URI.""" hname = uri.split('/')[-1].split('.')[0] - dest = os.path.join(self.stagingDir, hname) + dest = os.path.join(self.staging_dir, hname) cmd = 'git clone {} {}'.format(uri, dest) call(cmd, shell=True) - self.addFromLocal(dest) + self.add_from_local(dest) - def _recipeName(self, recipeFilename): - if recipeFilename[-9:] == 'recipe.py': - r = recipeFilename[:-9] - if r[-1] in ['-', '_', '.']: - r = r[:-1] - return os.path.basename(r) + def _recipe_name(self, recipe_filename): # pylint: disable=no-self-use + if recipe_filename[-9:] == 'recipe.py': + recipe_prefix = recipe_filename[:-9] + if recipe_prefix[-1] in ['-', '_', '.']: + recipe_prefix = recipe_prefix[:-1] + return os.path.basename(recipe_prefix) - raise InvalidRecipeURI('{} is not a recipe'.format(recipeFilename)) + raise InvalidRecipeURI('{} is not a recipe'.format(recipe_filename)) - def allRecipes(self): + def all_recipes(self): """Return set containing names of all recipes in repository.""" out = set() - for recipe in listdir(self.recipeDir): + for recipe in listdir(self.recipe_dir): try: - r = self._recipeName(recipe) - out.add(r) + recipe_name = self._recipe_name(recipe) + out.add(recipe_name) except AssertionError: pass return out - def makeRecipe(self, recipeName): + def make_recipe(self, recipe_name): """Create a recipe from the recipe name.""" # check if we have the recipe # if not throw an error - if recipeName not in self.allRecipes(): - raise RecipeNotFoundError(recipeName) + if recipe_name not in self.all_recipes(): + raise RecipeNotFoundError(recipe_name) # else run it - recipe = self._loadRecipe(recipeName) - recipe.makeRecipe() + recipe = self._load_recipe(recipe_name) + recipe.make_recipe() - def _loadRecipe(self, recipeName): + def _load_recipe(self, recipe_name): # (sort of hacky) - for f in listdir(self.recipeDir): - if f[: len(recipeName)] == recipeName: - fname = os.path.join(self.recipeDir, f) + for recipe_file in listdir(self.recipe_dir): + if recipe_file[: len(recipe_name)] == recipe_name: + fname = os.path.join(self.recipe_dir, recipe_file) break - cname = self._getClassName(fname) - importName = os.path.basename(fname)[:-3] + cname = self._get_class_name(fname) + import_name = os.path.basename(fname)[:-3] sys.path.append(os.path.dirname(fname)) - __import__(importName) - classes = inspect.getmembers(sys.modules[importName], inspect.isclass) - for name, c in classes: + __import__(import_name) + classes = inspect.getmembers(sys.modules[import_name], inspect.isclass) + for name, class_module in classes: if name == cname: - return c() + return class_module() - raise RecipeNotFoundError(recipeName) + raise RecipeNotFoundError(recipe_name) - def _getClassName(self, fname): - recipeStr = open(fname).read() + def _get_class_name(self, fname): # pylint: disable=no-self-use + recipe_str = open(fname).read() cname = None - for line in recipeStr.split('\n'): + for line in recipe_str.split('\n'): if 'class' in line: cname = line.split()[1] cname = cname.split(':')[0] @@ -122,108 +122,108 @@ def _getClassName(self, fname): break return cname - def downloadDir(self): + def download_dir(self): """Get directory for storing PackageMega downloads.""" try: return os.environ['PACKAGE_MEGA_DOWNLOADS'] except KeyError: try: - defaultDatabaseDir = os.path.join(self.abspath, - 'database_dir_location.txt') - defaultDatabaseDir = open(defaultDatabaseDir).read() - return defaultDatabaseDir + default_db_dir = os.path.join(self.abspath, + 'database_dir_location.txt') + default_db_dir = open(default_db_dir).read() + return default_db_dir except FileNotFoundError: - defaultDatabaseDir = os.path.join(self.abspath, 'databases') - return defaultDatabaseDir + default_db_dir = os.path.join(self.abspath, 'databases') + return default_db_dir - def allDatabases(self): + def all_databases(self): """Return list of all databases present in the repository.""" out = [] - for database in self.dsRepo().sampleTable.getAll(): + for database in self.ds_repo().sampleTable.getAll(): out.append(database) return out - def database(self, databaseName): + def database(self, database_name): """Get database by name.""" - return self.dsRepo().sampleTable.get(databaseName) + return self.ds_repo().sampleTable.get(database_name) - def dbStatus(self): + def db_status(self): """Get status of underlying DataSuper repository.""" - return self.dsRepo().checkStatus() + return self.ds_repo().checkStatus() - def saveFiles(self, recipe, subName, *filepaths, **kwFilepaths): - """Save a group of files to a recipe using the subName suffix.""" - fs = {} + def save_files(self, recipe, sub_name, *filepaths, **kw_filepaths): # pylint: disable=too-many-locals + """Save a group of files to a recipe using the sub_name suffix.""" + file_sources = {} for i, filepath in enumerate(filepaths): - fs[i] = filepath - for k, v in kwFilepaths.items(): - fs[k] = v + file_sources[i] = filepath + for key, value in kw_filepaths.items(): + file_sources[key] = value - with self.dsRepo() as dsr: - schema = recipe.resultSchema()[subName] + with self.ds_repo() as dsr: + schema = recipe.resultSchema()[sub_name] - fileRecs = [] + file_recs = [] ftypes = dictify(schema) - for key, fpath in fs.items(): - fname = '{}.{}.{}'.format(recipe.name(), subName, key) + for key, fpath in file_sources.items(): + fname = '{}.{}.{}'.format(recipe.name(), sub_name, key) print('{} {}'.format(fname, fpath)) ftype = ftypes[key] dsr.addFileType(ftype) ds.makeFile(dsr, fname, fpath, ftype, modify=True) - fileRecs.append(fname) + file_recs.append(fname) - schemaName = '{}::{}'.format(recipe.name(), subName) - dsr.addResultSchema(schemaName, schema) - rname = '{}.{}'.format(recipe.name(), subName) - result = ds.getOrMakeResult(dsr, rname, schemaName, fileRecs) + schema_name = '{}::{}'.format(recipe.name(), sub_name) + dsr.addResultSchema(schema_name, schema) + rname = '{}.{}'.format(recipe.name(), sub_name) + result = ds.getOrMakeResult(dsr, rname, schema_name, file_recs) dsr.addSampleType('db') sample = ds.getOrMakeSample(dsr, recipe.name(), 'db') sample.addResult(result).save(modify=True) - def dsRepo(self): + def ds_repo(self): """Return the underlying DataSuper repository.""" return ds.Repo.loadRepo(self.abspath) @staticmethod - def loadRepo(): + def load_repo(): """Load the system-wide PackageMega repository.""" try: - targetDir = os.environ['PACKAGE_MEGA_HOME'] + target_dir = os.environ['PACKAGE_MEGA_HOME'] except KeyError: - targetDir = os.environ['HOME'] - targetDir = os.path.join(targetDir, Repo.repoDirName) - p = os.path.abspath(targetDir) + target_dir = os.environ['HOME'] + target_dir = os.path.join(target_dir, Repo.repo_dir_name) + target_dir_path = os.path.abspath(target_dir) try: # Ignore the return value - ds.Repo.loadRepo(p) + ds.Repo.loadRepo(target_dir_path) except FileNotFoundError: - Repo._initRepo() - return Repo.loadRepo() - return Repo(p) + Repo._init_repo() + return Repo.load_repo() + return Repo(target_dir_path) @staticmethod - def _initRepo(): + def _init_repo(): try: - targetDir = os.environ['PACKAGE_MEGA_HOME'] + target_dir = os.environ['PACKAGE_MEGA_HOME'] except KeyError: - targetDir = os.environ['HOME'] - p = os.path.abspath(targetDir) - p = os.path.join(p, Repo.repoDirName) - os.makedirs(p) - ds.Repo.initRepo(targetDir=p) - r = os.path.join(p, 'recipes') - os.makedirs(r) - s = os.path.join(p, 'staging') - os.makedirs(s) - d = os.path.join(p, 'databases') - os.makedirs(d) - - -def dictify(el): + target_dir = os.environ['HOME'] + target_dir_path = os.path.abspath(target_dir) + target_dir_path = os.path.join(target_dir_path, Repo.repo_dir_name) + os.makedirs(target_dir_path) + ds.Repo.initRepo(targetDir=target_dir_path) + recipes_dir = os.path.join(target_dir_path, 'recipes') + os.makedirs(recipes_dir) + staging_dir = os.path.join(target_dir_path, 'staging') + os.makedirs(staging_dir) + databases_dir = os.path.join(target_dir_path, 'databases') + os.makedirs(databases_dir) + + +def dictify(element): """Transform element of any type into dictionary.""" - if type(el) == list: - return {i: sub for i, sub in enumerate(el)} - if type(el) == dict: - return el - return {0: el} + if isinstance(element, list): + return {i: sub for i, sub in enumerate(element)} + if isinstance(element, dict): + return element + return {0: element} diff --git a/packagemega/source_file.py b/packagemega/source_file.py index bf21c84..632a8b8 100644 --- a/packagemega/source_file.py +++ b/packagemega/source_file.py @@ -14,12 +14,12 @@ def __init__(self, *args, url=None, **kwargs): super().__init__(*args, **kwargs) self.url = url - def _downloadFile(self): + def _download_file(self): """Download the file.""" - targetPath = os.path.join(self.repo.downloadDir(), self.filename) - cmd = 'wget {} -O {}'.format(self.url, targetPath) + target_path = os.path.join(self.repo.download_dir(), self.filename) + cmd = 'wget {} -O {}'.format(self.url, target_path) check_output(cmd, shell=True) - return targetPath + return target_path def _resolver(self): """Return file url.""" @@ -27,4 +27,4 @@ def _resolver(self): def _resolve_actual_file(self): """Resolve file by downloading it.""" - return self._downloadFile() + return self._download_file() From 3bd17f4d342c9a1d380683a398e2881fbf3834e9 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Tue, 18 Dec 2018 14:33:35 -0500 Subject: [PATCH 6/6] Update lint CI step name. [skip ci] --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 675c58a..de33bf6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: - checkout - run: - name: Lint code (non-fatal) + name: Lint code command: | tox -e flake8,pylint