From ac64c626cb95e374b7f45df26b3d74442e74cfbd Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 8 Jun 2026 10:59:15 -0400 Subject: [PATCH 1/5] feat(dev): non relative uris; - Do not store relative URIs in the database. --- ckanext/xloader/action.py | 19 +++++++++++++++++-- ckanext/xloader/command.py | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 81273537..5d0f6f9c 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -6,6 +6,10 @@ import json import logging +# (canada fork only): non-qualified res_url for lang domain support +from urllib.parse import urlparse +from ckan.plugins.toolkit import request + import ckan.lib.jobs as rq_jobs import ckan.lib.navl.dictization_functions from ckan.logic import side_effect_free @@ -134,15 +138,26 @@ def xloader_submit(context, data_dict): task ) + # (canada fork only): non-qualified res_url for lang domain support + # only for upload types. + original_url = resource_dict.get('url') + if resource_dict.get('url_type') == 'upload': + original_url_parts = urlparse(original_url) + if original_url_parts.netloc: + original_url = original_url.replace('%s://' % str(original_url_parts.scheme), '') + original_url = original_url.replace(str(original_url_parts.netloc), '') + data = { 'job_type': 'xloader_to_datastore', 'metadata': { 'ignore_hash': data_dict.get('ignore_hash', False), - 'ckan_url': config['ckan.site_url'], + # (canada fork only): do not store site_url + 'ckan_url': None, 'resource_id': res_id, 'set_url_type': data_dict.get('set_url_type', False), 'task_created': task['last_updated'], - 'original_url': resource_dict.get('url'), + # (canada fork only): non-qualified res_url for lang domain support + 'original_url': original_url, } } diff --git a/ckanext/xloader/command.py b/ckanext/xloader/command.py index 36551b37..ad3546fc 100644 --- a/ckanext/xloader/command.py +++ b/ckanext/xloader/command.py @@ -115,7 +115,8 @@ def _submit_resource(self, resource, user, indent=0, sync=False, queue=None): 'ignore_hash': True, } if sync: - data_dict['ckan_url'] = tk.config.get('ckan.site_url') + # (canada fork only): do not store site_url + data_dict['ckan_url'] = None input_dict = { 'metadata': data_dict, 'api_key': 'TODO' From 5c3ba3951a98cdf3877ffc195cf3fb9bce4477e3 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 8 Jun 2026 11:05:48 -0400 Subject: [PATCH 2/5] feat(misc): changelog; - Added change log file. --- changes/36.canada.changes | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/36.canada.changes diff --git a/changes/36.canada.changes b/changes/36.canada.changes new file mode 100644 index 00000000..3d7e6e45 --- /dev/null +++ b/changes/36.canada.changes @@ -0,0 +1 @@ +Now only stores the relative URIs in the database for XLoader Resource fields `original_url` (only for Uploads). No longer stores the `ckan_url` value in the database. From 48fbf66c8f6ac50e8cfcf125968e91785749c32b Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 15 Jun 2026 13:48:58 -0400 Subject: [PATCH 3/5] fix(dev): better syntax; - Use `urlparse._replace` instead of string replacements. --- ckanext/xloader/action.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 5d0f6f9c..176a8b41 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -7,7 +7,8 @@ import logging # (canada fork only): non-qualified res_url for lang domain support -from urllib.parse import urlparse +# TODO: upstream contrib!! +from urllib.parse import urlparse, urlunparse from ckan.plugins.toolkit import request import ckan.lib.jobs as rq_jobs @@ -140,12 +141,12 @@ def xloader_submit(context, data_dict): # (canada fork only): non-qualified res_url for lang domain support # only for upload types. + # TODO: upstream contrib!! original_url = resource_dict.get('url') if resource_dict.get('url_type') == 'upload': original_url_parts = urlparse(original_url) if original_url_parts.netloc: - original_url = original_url.replace('%s://' % str(original_url_parts.scheme), '') - original_url = original_url.replace(str(original_url_parts.netloc), '') + original_url = original_url_parts._replace(scheme='', netloc='').geturl() data = { 'job_type': 'xloader_to_datastore', From 861b3616c6d9c2736c4dd1c8873437006d68a3c7 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 15 Jun 2026 13:52:04 -0400 Subject: [PATCH 4/5] fix(dev): better syntax; - Use `urlparse._replace` instead of string replacements. --- ckanext/xloader/action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 176a8b41..7d902ee7 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -8,7 +8,7 @@ # (canada fork only): non-qualified res_url for lang domain support # TODO: upstream contrib!! -from urllib.parse import urlparse, urlunparse +from urllib.parse import urlparse from ckan.plugins.toolkit import request import ckan.lib.jobs as rq_jobs From 84473d39b580badc8b12d2ecf5d7cfb3fd2e62d1 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Wed, 24 Jun 2026 15:16:19 -0400 Subject: [PATCH 5/5] feat(cli): resource IDs; - Added resource ID option to xloader submit command. - Fixed TypeErrors and missing metadata in the cli methods. --- changes/36.canada.bugfix | 1 + ckanext/xloader/cli.py | 33 ++++++++++++++++++++++++++++----- ckanext/xloader/command.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 changes/36.canada.bugfix diff --git a/changes/36.canada.bugfix b/changes/36.canada.bugfix new file mode 100644 index 00000000..79fb5297 --- /dev/null +++ b/changes/36.canada.bugfix @@ -0,0 +1 @@ +Added `--resource-id` option to the `xloader submit` command. Fixed various TypeErrors and missing metadata in the CLI methods. diff --git a/ckanext/xloader/cli.py b/ckanext/xloader/cli.py index b876579a..1929a867 100644 --- a/ckanext/xloader/cli.py +++ b/ckanext/xloader/cli.py @@ -6,7 +6,7 @@ # (canada fork only): add db init from ckanext.xloader.db import init # (canada fork only): ckan.plugins.toolkit -from ckan.plugins.toolkit import config +from ckan.plugins.toolkit import config, get_action, ObjectNotFound # Click commands for CKAN 2.9 and above @@ -27,29 +27,52 @@ def status(): @xloader.command() -@click.argument(u'dataset-spec') +# (canada fork only): support resource IDs +# TODO: upstream contrib?? +@click.argument(u'dataset-spec', required=False) @click.option('-y', is_flag=True, default=False, help='Always answer yes to questions') @click.option('--dry-run', is_flag=True, default=False, help='Don\'t actually submit any resources') @click.option('--queue', help='Queue name for asynchronous processing, unused if executing immediately') @click.option('--sync', is_flag=True, default=False, help='Execute immediately instead of enqueueing for asynchronous processing') -def submit(dataset_spec, y, dry_run, queue, sync): +# (canada fork only): support resource IDs +# TODO: upstream contrib?? +@click.option('-r', '--resource-id', type=click.STRING, help='A CKAN Resource ID.', required=False) +def submit(dataset_spec, y, dry_run, queue, sync, resource_id): """ xloader submit [options] """ cmd = XloaderCmd(dry_run) - if dataset_spec == 'all': + # (canada fork only): support resource IDs + # TODO: upstream contrib?? + if resource_id: + cmd._setup_xloader_logger() + try: + res_dict = get_action('resource_show')( + {'ignore_auth': True}, {'id': resource_id}) + user = get_action('get_site_user')( + {'ignore_auth': True}, {}) + except ObjectNotFound: + click.echo('Resource %s not found') + raise click.Abort() + cmd._submit_resource(res_dict, user=user, sync=sync, queue=queue) + elif dataset_spec == 'all': cmd._setup_xloader_logger() cmd._submit_all(sync=sync, queue=queue) elif dataset_spec == 'all-existing': _confirm_or_abort(y, dry_run) cmd._setup_xloader_logger() cmd._submit_all_existing(sync=sync, queue=queue) - else: + elif dataset_spec: pkg_name_or_id = dataset_spec cmd._setup_xloader_logger() cmd._submit_package(pkg_name_or_id, sync=sync, queue=queue) + else: + # (canada fork only): support resource IDs + # TODO: upstream contrib?? + click.echo('No or --resource-id supplied.') + raise click.Abort() if cmd.error_occured: print('Finished but saw errors - see above for details') diff --git a/ckanext/xloader/command.py b/ckanext/xloader/command.py index ad3546fc..8fc93d80 100644 --- a/ckanext/xloader/command.py +++ b/ckanext/xloader/command.py @@ -1,5 +1,10 @@ # -*- coding: utf-8 -*- +# (canada fork only): fix TypeErrors +# TODO: upstream contrib?? +import datetime +from urllib.parse import urlparse + import sys import logging # (canada fork only): ckan.plugins.toolkit @@ -121,8 +126,31 @@ def _submit_resource(self, resource, user, indent=0, sync=False, queue=None): 'metadata': data_dict, 'api_key': 'TODO' } + # (canada fork only): non-qualified res_url for lang domain support + # only for upload types. + # TODO: upstream contrib!! + original_url = resource.get('url') + if resource.get('url_type') == 'upload': + original_url_parts = urlparse(original_url) + if original_url_parts.netloc: + original_url = original_url_parts._replace(scheme='', netloc='').geturl() + # (canada fork only): fix TypeErrors + # TODO: upstream contrib?? + job_dict = { + 'status': 'complete', + 'metadata': { + 'ignore_hash': True, + # (canada fork only): do not store site_url + 'ckan_url': None, + 'resource_id': resource['id'], + 'set_url_type': data_dict.get('set_url_type', False), + 'task_created': str(datetime.datetime.utcnow()), + # (canada fork only): non-qualified res_url for lang domain support + 'original_url': original_url, + } + } logger = logging.getLogger('ckanext.xloader.cli') - xloader_data_into_datastore_(input_dict, None, logger) + xloader_data_into_datastore_(input_dict, job_dict, logger) else: if queue: data_dict['queue'] = queue