diff --git a/changes/9404.feature b/changes/9404.feature new file mode 100644 index 00000000000..7b379e61226 --- /dev/null +++ b/changes/9404.feature @@ -0,0 +1,2 @@ +new datastore_sequence_create, datastore_sequence_delete, datastore_sequence_next +actions for sysadmins to generate unique identifiers. diff --git a/ckanext/datastore/backend/__init__.py b/ckanext/datastore/backend/__init__.py index 447e2e37bfa..5590c007dfa 100644 --- a/ckanext/datastore/backend/__init__.py +++ b/ckanext/datastore/backend/__init__.py @@ -243,3 +243,28 @@ def drop_function(self, *args: Any, **kwargs: Any) -> Any: def resource_plugin_data(self, resource_id: str) -> dict[str, Any]: raise NotImplementedError() + + def calculate_record_count(self, resource_id: str) -> None: + """Called after updating a table to calculate and cache the + new number of records for future calls to search. + """ + + def clear_table_stats(self, resource_id: str) -> None: + """Remove number or records and any other stats cached for + a resource. + """ + + def create_sequence(self, name: str, if_not_exists: bool) -> None: + """Called by `datastore_sequence_create` action. + """ + raise NotImplementedError() + + def drop_sequence(self, name: str, if_exists: bool) -> None: + """Called by `datastore_sequence_delete` action. + """ + raise NotImplementedError() + + def sequence_nextval(self, name: str) -> int: + """Called by `datastore_sequence_next` action. + """ + raise NotImplementedError() diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index b278b9b56c0..c40e65cb77f 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -11,6 +11,7 @@ from sqlalchemy.engine.base import Engine from sqlalchemy.dialects.postgresql import REGCLASS +from sqlalchemy.schema import CreateSequence, DropSequence from ckan.types import Context, ErrorDict import copy import logging @@ -2798,6 +2799,16 @@ def calculate_record_count(self, resource_id: str): except DatabaseError as err: raise DatastoreException(err) + def create_sequence(self, *args: Any, **kwargs: Any) -> None: + return create_sequence(*args, **kwargs) + + def drop_sequence(self, *args: Any, **kwargs: Any) -> None: + return drop_sequence(*args, **kwargs) + + def sequence_nextval(self, *args: Any, **kwargs: Any) -> int: + return sequence_nextval(*args, **kwargs) + + def create_function(name: str, arguments: Iterable[dict[str, Any]], rettype: Any, definition: str, or_replace: bool): @@ -2847,6 +2858,40 @@ def drop_function(name: str, if_exists: bool): raise ValidationError({u'name': [_programming_error_summary(pe)]}) +def create_sequence(name: str, if_not_exists: bool) -> None: + # sqlalchemy 1.4 CreateSequence doesn't implement if_not_exists + sql = ''' + CREATE SEQUENCE {if_not_exists} {name}; + '''.format( + if_not_exists='IF NOT EXISTS' if if_not_exists else '', + name=identifier(name), + ) + try: + _write_engine_execute(sql) + except ProgrammingError as pe: + raise ValidationError({'name': [_programming_error_summary(pe)]}) + + +def drop_sequence(name: str, if_exists: bool) -> None: + try: + with get_write_engine().begin() as conn: + seq = sa.Sequence(name) + conn.execute(DropSequence(seq, if_exists=if_exists)) + except ProgrammingError as pe: + raise ValidationError({'name': [_programming_error_summary(pe)]}) + + +def sequence_nextval(name: str) -> int: + try: + with get_write_engine().begin() as conn: + seq = sa.Sequence(name) + num = conn.execute(sa.select(seq.next_value())).scalar() + assert isinstance(num, int) + return num + except ProgrammingError as pe: + raise ValidationError({'name': [_programming_error_summary(pe)]}) + + def _write_engine_execute(sql: str): with get_write_engine().begin() as conn: conn.execute(sa.text(sql)) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 0b82c985eb5..270f86ce5ca 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -791,3 +791,51 @@ def datastore_function_delete(context: Context, data_dict: dict[str, Any]): p.toolkit.check_access('datastore_function_delete', context, data_dict) backend = DatastoreBackend.get_active_backend() backend.drop_function(data_dict['name'], data_dict['if_exists']) + + +@logic.validate(dsschema.datastore_sequence_create_schema) +def datastore_sequence_create(context: Context, data_dict: dict[str, Any]): + ''' + Create a sequence for use with trigger functions or + datastore_sequence_next + + :param name: sequence name + :type name: string + :param if_not_exists: True to skip existing sequences + (default: False) + ''' + p.toolkit.check_access('datastore_sequence_create', context, data_dict) + backend = DatastoreBackend.get_active_backend() + backend.create_sequence( + name=data_dict['name'], + if_not_exists=data_dict['if_not_exists']) + + +@logic.validate(dsschema.datastore_sequence_delete_schema) +def datastore_sequence_delete(context: Context, data_dict: dict[str, Any]): + ''' + Delete a sequence + + :param name: sequence name + :type name: string + :param if_exists: True to skip missing sequences + (default: False) + ''' + p.toolkit.check_access('datastore_sequence_delete', context, data_dict) + backend = DatastoreBackend.get_active_backend() + backend.drop_sequence( + name=data_dict['name'], if_exists=data_dict['if_exists']) + + +@logic.validate(dsschema.datastore_sequence_next_schema) +def datastore_sequence_next(context: Context, data_dict: dict[str, Any]): + ''' + Return the next value and advance a sequence + + :param name: sequence name + :type name: string + :rtype: int + ''' + p.toolkit.check_access('datastore_sequence_next', context, data_dict) + backend = DatastoreBackend.get_active_backend() + return backend.sequence_nextval(name=data_dict['name']) diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py index feb123ab723..7123441d4f9 100644 --- a/ckanext/datastore/logic/auth.py +++ b/ckanext/datastore/logic/auth.py @@ -97,3 +97,20 @@ def datastore_run_triggers( def datastore_analyze(context: Context, data_dict: DataDict) -> AuthResult: return {'success': False} + +def datastore_sequence_create( + context: Context, data_dict: DataDict) -> AuthResult: + '''sysadmin-only: sequences are shared across datastore''' + return {'success': False} + + +def datastore_sequence_delete( + context: Context, data_dict: DataDict) -> AuthResult: + '''sysadmin-only: sequences are shared across datastore''' + return {'success': False} + + +def datastore_sequence_next( + context: Context, data_dict: DataDict) -> AuthResult: + '''sysadmin-only: sequences are shared across datastore''' + return {'success': False} diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 3e67fe5b448..2021c1fad1d 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -251,3 +251,23 @@ def datastore_analyze_schema() -> Schema: return { 'resource_id': [unicode_safe, resource_id_exists], } + + +def datastore_sequence_create_schema() -> Schema: + return { + 'name': [unicode_only, not_empty], + 'if_not_exists': [default(False), boolean_validator], + } + + +def datastore_sequence_delete_schema() -> Schema: + return { + 'name': [unicode_only, not_empty], + 'if_exists': [default(False), boolean_validator], + } + + +def datastore_sequence_next_schema() -> Schema: + return { + 'name': [unicode_only, not_empty], + } diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 4757f843632..0a43017aa4e 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -104,6 +104,9 @@ def get_actions(self) -> dict[str, Action]: 'datastore_function_create': action.datastore_function_create, 'datastore_function_delete': action.datastore_function_delete, 'datastore_run_triggers': action.datastore_run_triggers, + 'datastore_sequence_create': action.datastore_sequence_create, + 'datastore_sequence_delete': action.datastore_sequence_delete, + 'datastore_sequence_next': action.datastore_sequence_next, } if getattr(self.backend, 'enable_sql_search', False): # Only enable search_sql if the config/backend does not disable it @@ -127,6 +130,9 @@ def get_auth_functions(self) -> dict[str, AuthFunction]: 'datastore_function_create': auth.datastore_function_create, 'datastore_function_delete': auth.datastore_function_delete, 'datastore_run_triggers': auth.datastore_run_triggers, + 'datastore_sequence_create': auth.datastore_sequence_create, + 'datastore_sequence_delete': auth.datastore_sequence_delete, + 'datastore_sequence_next': auth.datastore_sequence_next, } # IResourceController diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 3cee2edfa13..140145b93ea 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -4,6 +4,8 @@ import pytest import sqlalchemy.orm as orm +from faker import Faker + import ckan.lib.create_test_data as ctd import ckan.model as model import ckan.plugins as p @@ -18,6 +20,8 @@ when_was_last_analyze, ) +fake = Faker() + @pytest.mark.usefixtures("with_request_context") class TestDatastoreCreateNewTests(object): @@ -1447,3 +1451,39 @@ def test_upsert_trigger_exception(self, app): assert error.value.error_dict == { u"records": [u'"BEANS"? Yeeeeccch!'] } + + +@pytest.mark.ckan_config("ckan.plugins", "datastore") +@pytest.mark.usefixtures("with_plugins", "with_request_context") +def test_sequences(): + seqname = fake.unique.user_name() + helpers.call_action("datastore_sequence_create", name=seqname) + assert helpers.call_action("datastore_sequence_next", name=seqname) == 1 + assert helpers.call_action("datastore_sequence_next", name=seqname) == 2 + helpers.call_action("datastore_sequence_delete", name=seqname) + + with pytest.raises(ValidationError) as error: + helpers.call_action("datastore_sequence_next", name=seqname) + assert error.value.error_dict == { + "name": [f'relation "{seqname}" does not exist'] + } + + helpers.call_action("datastore_sequence_create", name=seqname) + with pytest.raises(ValidationError) as error: + helpers.call_action("datastore_sequence_create", name=seqname) + assert error.value.error_dict == { + "name": [f'relation "{seqname}" already exists'] + } + helpers.call_action( + "datastore_sequence_create", name=seqname, if_not_exists=True, + ) + + helpers.call_action("datastore_sequence_delete", name=seqname) + with pytest.raises(ValidationError) as error: + helpers.call_action("datastore_sequence_delete", name=seqname) + assert error.value.error_dict == { + "name": [f'sequence "{seqname}" does not exist'] + } + helpers.call_action( + "datastore_sequence_delete", name=seqname, if_exists=True, + )