diff --git a/changes/36.canada.bugfix b/changes/36.canada.bugfix new file mode 100644 index 00000000..79fb5297 --- /dev/null +++ b/changes/36.canada.bugfix @@ -0,0 +1 @@ +Added `--resource-id` option to the `xloader submit` command. Fixed various TypeErrors and missing metadata in the CLI methods. diff --git a/changes/36.canada.changes b/changes/36.canada.changes new file mode 100644 index 00000000..3d7e6e45 --- /dev/null +++ b/changes/36.canada.changes @@ -0,0 +1 @@ +Now only stores the relative URIs in the database for XLoader Resource fields `original_url` (only for Uploads). No longer stores the `ckan_url` value in the database. diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 81273537..7d902ee7 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -6,6 +6,11 @@ import json import logging +# (canada fork only): non-qualified res_url for lang domain support +# TODO: upstream contrib!! +from urllib.parse import urlparse +from ckan.plugins.toolkit import request + import ckan.lib.jobs as rq_jobs import ckan.lib.navl.dictization_functions from ckan.logic import side_effect_free @@ -134,15 +139,26 @@ def xloader_submit(context, data_dict): task ) + # (canada fork only): non-qualified res_url for lang domain support + # only for upload types. + # TODO: upstream contrib!! + original_url = resource_dict.get('url') + if resource_dict.get('url_type') == 'upload': + original_url_parts = urlparse(original_url) + if original_url_parts.netloc: + original_url = original_url_parts._replace(scheme='', netloc='').geturl() + data = { 'job_type': 'xloader_to_datastore', 'metadata': { 'ignore_hash': data_dict.get('ignore_hash', False), - 'ckan_url': config['ckan.site_url'], + # (canada fork only): do not store site_url + 'ckan_url': None, 'resource_id': res_id, 'set_url_type': data_dict.get('set_url_type', False), 'task_created': task['last_updated'], - 'original_url': resource_dict.get('url'), + # (canada fork only): non-qualified res_url for lang domain support + 'original_url': original_url, } } diff --git a/ckanext/xloader/cli.py b/ckanext/xloader/cli.py index b876579a..1929a867 100644 --- a/ckanext/xloader/cli.py +++ b/ckanext/xloader/cli.py @@ -6,7 +6,7 @@ # (canada fork only): add db init from ckanext.xloader.db import init # (canada fork only): ckan.plugins.toolkit -from ckan.plugins.toolkit import config +from ckan.plugins.toolkit import config, get_action, ObjectNotFound # Click commands for CKAN 2.9 and above @@ -27,29 +27,52 @@ def status(): @xloader.command() -@click.argument(u'dataset-spec') +# (canada fork only): support resource IDs +# TODO: upstream contrib?? +@click.argument(u'dataset-spec', required=False) @click.option('-y', is_flag=True, default=False, help='Always answer yes to questions') @click.option('--dry-run', is_flag=True, default=False, help='Don\'t actually submit any resources') @click.option('--queue', help='Queue name for asynchronous processing, unused if executing immediately') @click.option('--sync', is_flag=True, default=False, help='Execute immediately instead of enqueueing for asynchronous processing') -def submit(dataset_spec, y, dry_run, queue, sync): +# (canada fork only): support resource IDs +# TODO: upstream contrib?? +@click.option('-r', '--resource-id', type=click.STRING, help='A CKAN Resource ID.', required=False) +def submit(dataset_spec, y, dry_run, queue, sync, resource_id): """ xloader submit [options] """ cmd = XloaderCmd(dry_run) - if dataset_spec == 'all': + # (canada fork only): support resource IDs + # TODO: upstream contrib?? + if resource_id: + cmd._setup_xloader_logger() + try: + res_dict = get_action('resource_show')( + {'ignore_auth': True}, {'id': resource_id}) + user = get_action('get_site_user')( + {'ignore_auth': True}, {}) + except ObjectNotFound: + click.echo('Resource %s not found') + raise click.Abort() + cmd._submit_resource(res_dict, user=user, sync=sync, queue=queue) + elif dataset_spec == 'all': cmd._setup_xloader_logger() cmd._submit_all(sync=sync, queue=queue) elif dataset_spec == 'all-existing': _confirm_or_abort(y, dry_run) cmd._setup_xloader_logger() cmd._submit_all_existing(sync=sync, queue=queue) - else: + elif dataset_spec: pkg_name_or_id = dataset_spec cmd._setup_xloader_logger() cmd._submit_package(pkg_name_or_id, sync=sync, queue=queue) + else: + # (canada fork only): support resource IDs + # TODO: upstream contrib?? + click.echo('No or --resource-id supplied.') + raise click.Abort() if cmd.error_occured: print('Finished but saw errors - see above for details') diff --git a/ckanext/xloader/command.py b/ckanext/xloader/command.py index 36551b37..8fc93d80 100644 --- a/ckanext/xloader/command.py +++ b/ckanext/xloader/command.py @@ -1,5 +1,10 @@ # -*- coding: utf-8 -*- +# (canada fork only): fix TypeErrors +# TODO: upstream contrib?? +import datetime +from urllib.parse import urlparse + import sys import logging # (canada fork only): ckan.plugins.toolkit @@ -115,13 +120,37 @@ def _submit_resource(self, resource, user, indent=0, sync=False, queue=None): 'ignore_hash': True, } if sync: - data_dict['ckan_url'] = tk.config.get('ckan.site_url') + # (canada fork only): do not store site_url + data_dict['ckan_url'] = None input_dict = { 'metadata': data_dict, 'api_key': 'TODO' } + # (canada fork only): non-qualified res_url for lang domain support + # only for upload types. + # TODO: upstream contrib!! + original_url = resource.get('url') + if resource.get('url_type') == 'upload': + original_url_parts = urlparse(original_url) + if original_url_parts.netloc: + original_url = original_url_parts._replace(scheme='', netloc='').geturl() + # (canada fork only): fix TypeErrors + # TODO: upstream contrib?? + job_dict = { + 'status': 'complete', + 'metadata': { + 'ignore_hash': True, + # (canada fork only): do not store site_url + 'ckan_url': None, + 'resource_id': resource['id'], + 'set_url_type': data_dict.get('set_url_type', False), + 'task_created': str(datetime.datetime.utcnow()), + # (canada fork only): non-qualified res_url for lang domain support + 'original_url': original_url, + } + } logger = logging.getLogger('ckanext.xloader.cli') - xloader_data_into_datastore_(input_dict, None, logger) + xloader_data_into_datastore_(input_dict, job_dict, logger) else: if queue: data_dict['queue'] = queue