From 9b9d60b32549cc024f4beb3d8111e6597de798d9 Mon Sep 17 00:00:00 2001 From: mhidas Date: Thu, 14 Jan 2021 17:14:43 +1100 Subject: [PATCH 1/3] update MooringsProductsHandler to use new aodncore API for remote file downloads and file_collection additions --- aodndata/moorings/products_handler.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/aodndata/moorings/products_handler.py b/aodndata/moorings/products_handler.py index 3acb167b..e67c9f99 100644 --- a/aodndata/moorings/products_handler.py +++ b/aodndata/moorings/products_handler.py @@ -201,7 +201,7 @@ def _get_input_files(self): # Download input files to local cache. self.logger.info("Downloading {n} input files".format(n=len(self.input_file_collection))) - self.input_file_collection.download(self._upload_store_runner.broker, self.temp_dir) + self.state_query.download(self.input_file_collection, self.temp_dir) # TODO: Replace temp_dir above with cache_dir? def _get_old_product_files(self): @@ -242,9 +242,8 @@ def _log_excluded_files(self, errors): def _add_to_collection(self, product_url): """Add a new product file to the file_collection to be harvested and uploaded.""" - product_file = PipelineFile(product_url, file_update_callback=self._file_update_callback) - product_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD - self.file_collection.add(product_file) + product_file = PipelineFile(product_url, publish_type = PipelineFilePublishType.HARVEST_UPLOAD) + self.add_pipelinefile(product_file) def _input_list_for_variables(self, *variables): """Return a list of input files containing any of the given variables""" @@ -357,10 +356,10 @@ def _cleanup_previous_version(self, product_filename): if os.path.basename(old_product_url) != product_filename: # Add the previous version as a "late deletion". It will be deleted during the handler's `publish` # step after (and only if) all new files have been successfully published. - old_file = PipelineFile(old_product_url, dest_path=old_product_url, is_deletion=True, - late_deletion=True, file_update_callback=self._file_update_callback) - old_file.publish_type = PipelineFilePublishType.DELETE_UNHARVEST - self.file_collection.add(old_file) + old_file = PipelineFile(old_product_url, dest_path=old_product_url, + is_deletion=True, late_deletion=True, + publish_type=PipelineFilePublishType.DELETE_UNHARVEST) + self.add_pipelinefile(old_file) def preprocess(self): """If the input is a manifest file, collect available input files and From 611f70a0b67d59b38ad79e3be5d15761004786ac Mon Sep 17 00:00:00 2001 From: mhidas Date: Thu, 14 Jan 2021 17:34:52 +1100 Subject: [PATCH 2/3] update moorings handlers to use new aodncore API --- aodndata/moorings/handlers.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aodndata/moorings/handlers.py b/aodndata/moorings/handlers.py index 7639eb80..e4d77198 100644 --- a/aodndata/moorings/handlers.py +++ b/aodndata/moorings/handlers.py @@ -59,9 +59,8 @@ def preprocess(self): self.logger.info("Burst-processing {f.name}".format(f=f)) product_path = create_burst_average_netcdf(f.src_path, self.products_dir) - product_file = PipelineFile(product_path, file_update_callback=self._file_update_callback) - product_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD - self.file_collection.add(product_file) + product_file = PipelineFile(product_path, publish_type=PipelineFilePublishType.HARVEST_UPLOAD) + self.add_pipelinefile(product_file) def postprocess(self): """Set error_cleanup_regexes so that if the same file was uploaded previously and failed, it can now be @@ -116,6 +115,6 @@ def process(self): self.file_collection.set_publish_types(PipelineFilePublishType.NO_ACTION) self.input_file_object.publish_type = PipelineFilePublishType.HARVEST_UPLOAD - self.file_collection.add(self.input_file_object) + self.add_pipelinefile(self.input_file_object) dest_path = DwmFileClassifier.dest_path From f7f60e417de6ecc5e56a66461e68efb5d0accfe0 Mon Sep 17 00:00:00 2001 From: Leigh Gordon Date: Mon, 18 Jan 2021 10:55:42 +1100 Subject: [PATCH 3/3] Update test.yml Add fail-fast: false --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4917f2d..8bf13aeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: [ '3.5', '3.6' ] stage: [ 'build', 'rc', 'production']