datastore sequence API feature backport

changes/9404.feature

@@ -0,0 +1,2 @@
+new datastore_sequence_create, datastore_sequence_delete, datastore_sequence_next
+actions for sysadmins to generate unique identifiers.

ckanext/datastore/backend/__init__.py

@@ -243,3 +243,28 @@ def drop_function(self, *args: Any, **kwargs: Any) -> Any:
 
     def resource_plugin_data(self, resource_id: str) -> dict[str, Any]:
         raise NotImplementedError()
+
+    def calculate_record_count(self, resource_id: str) -> None:
+        """Called after updating a table to calculate and cache the
+        new number of records for future calls to search.
+        """
+
+    def clear_table_stats(self, resource_id: str) -> None:
+        """Remove number or records and any other stats cached for
+        a resource.
+        """
+
+    def create_sequence(self, name: str, if_not_exists: bool) -> None:
+        """Called by `datastore_sequence_create` action.
+        """
+        raise NotImplementedError()
+
+    def drop_sequence(self, name: str, if_exists: bool) -> None:
+        """Called by `datastore_sequence_delete` action.
+        """
+        raise NotImplementedError()
+
+    def sequence_nextval(self, name: str) -> int:
+        """Called by `datastore_sequence_next` action.
+        """
+        raise NotImplementedError()

ckanext/datastore/backend/postgres.py

@@ -11,6 +11,7 @@
 
 from sqlalchemy.engine.base import Engine
 from sqlalchemy.dialects.postgresql import REGCLASS
+from sqlalchemy.schema import CreateSequence, DropSequence
 from ckan.types import Context, ErrorDict
 import copy
 import logging
@@ -2798,6 +2799,16 @@ def calculate_record_count(self, resource_id: str):
             except DatabaseError as err:
                 raise DatastoreException(err)
 
+    def create_sequence(self, *args: Any, **kwargs: Any) -> None:
+        return create_sequence(*args, **kwargs)
+
+    def drop_sequence(self, *args: Any, **kwargs: Any) -> None:
+        return drop_sequence(*args, **kwargs)
+
+    def sequence_nextval(self, *args: Any, **kwargs: Any) -> int:
+        return sequence_nextval(*args, **kwargs)
+
+
 
 def create_function(name: str, arguments: Iterable[dict[str, Any]],
                     rettype: Any, definition: str, or_replace: bool):
@@ -2847,6 +2858,40 @@ def drop_function(name: str, if_exists: bool):
         raise ValidationError({u'name': [_programming_error_summary(pe)]})
 
 
+def create_sequence(name: str, if_not_exists: bool) -> None:
+    # sqlalchemy 1.4 CreateSequence doesn't implement if_not_exists
+    sql = '''
+        CREATE SEQUENCE {if_not_exists} {name};
+    '''.format(
+        if_not_exists='IF NOT EXISTS' if if_not_exists else '',
+        name=identifier(name),
+    )
+    try:
+        _write_engine_execute(sql)
+    except ProgrammingError as pe:
+        raise ValidationError({'name': [_programming_error_summary(pe)]})
+
+
+def drop_sequence(name: str, if_exists: bool) -> None:
+    try:
+        with get_write_engine().begin() as conn:
+            seq = sa.Sequence(name)
+            conn.execute(DropSequence(seq, if_exists=if_exists))
+    except ProgrammingError as pe:
+        raise ValidationError({'name': [_programming_error_summary(pe)]})
+
+
+def sequence_nextval(name: str) -> int:
+    try:
+        with get_write_engine().begin() as conn:
+            seq = sa.Sequence(name)
+            num = conn.execute(sa.select(seq.next_value())).scalar()
+            assert isinstance(num, int)
+            return num
+    except ProgrammingError as pe:
+        raise ValidationError({'name': [_programming_error_summary(pe)]})
+
+
 def _write_engine_execute(sql: str):
     with get_write_engine().begin() as conn:
         conn.execute(sa.text(sql))

ckanext/datastore/logic/action.py

@@ -791,3 +791,51 @@ def datastore_function_delete(context: Context, data_dict: dict[str, Any]):
     p.toolkit.check_access('datastore_function_delete', context, data_dict)
     backend = DatastoreBackend.get_active_backend()
     backend.drop_function(data_dict['name'], data_dict['if_exists'])
+
+
+@logic.validate(dsschema.datastore_sequence_create_schema)
+def datastore_sequence_create(context: Context, data_dict: dict[str, Any]):
+    '''
+    Create a sequence for use with trigger functions or
+    datastore_sequence_next
+
+    :param name: sequence name
+    :type name: string
+    :param if_not_exists: True to skip existing sequences
+        (default: False)
+    '''
+    p.toolkit.check_access('datastore_sequence_create', context, data_dict)
+    backend = DatastoreBackend.get_active_backend()
+    backend.create_sequence(
+        name=data_dict['name'],
+        if_not_exists=data_dict['if_not_exists'])
+
+
+@logic.validate(dsschema.datastore_sequence_delete_schema)
+def datastore_sequence_delete(context: Context, data_dict: dict[str, Any]):
+    '''
+    Delete a sequence
+
+    :param name: sequence name
+    :type name: string
+    :param if_exists: True to skip missing sequences
+        (default: False)
+    '''
+    p.toolkit.check_access('datastore_sequence_delete', context, data_dict)
+    backend = DatastoreBackend.get_active_backend()
+    backend.drop_sequence(
+        name=data_dict['name'], if_exists=data_dict['if_exists'])
+
+
+@logic.validate(dsschema.datastore_sequence_next_schema)
+def datastore_sequence_next(context: Context, data_dict: dict[str, Any]):
+    '''
+    Return the next value and advance a sequence
+
+    :param name: sequence name
+    :type name: string
+    :rtype: int
+    '''
+    p.toolkit.check_access('datastore_sequence_next', context, data_dict)
+    backend = DatastoreBackend.get_active_backend()
+    return backend.sequence_nextval(name=data_dict['name'])

ckanext/datastore/logic/auth.py

@@ -97,3 +97,20 @@ def datastore_run_triggers(
 
 def datastore_analyze(context: Context, data_dict: DataDict) -> AuthResult:
     return {'success': False}
+
+def datastore_sequence_create(
+        context: Context, data_dict: DataDict) -> AuthResult:
+    '''sysadmin-only: sequences are shared across datastore'''
+    return {'success': False}
+
+
+def datastore_sequence_delete(
+        context: Context, data_dict: DataDict) -> AuthResult:
+    '''sysadmin-only: sequences are shared across datastore'''
+    return {'success': False}
+
+
+def datastore_sequence_next(
+        context: Context, data_dict: DataDict) -> AuthResult:
+    '''sysadmin-only: sequences are shared across datastore'''
+    return {'success': False}

ckanext/datastore/logic/schema.py

@@ -251,3 +251,23 @@ def datastore_analyze_schema() -> Schema:
     return {
         'resource_id': [unicode_safe, resource_id_exists],
     }
+
+
+def datastore_sequence_create_schema() -> Schema:
+    return {
+        'name': [unicode_only, not_empty],
+        'if_not_exists': [default(False), boolean_validator],
+    }
+
+
+def datastore_sequence_delete_schema() -> Schema:
+    return {
+        'name': [unicode_only, not_empty],
+        'if_exists': [default(False), boolean_validator],
+    }
+
+
+def datastore_sequence_next_schema() -> Schema:
+    return {
+        'name': [unicode_only, not_empty],
+    }

ckanext/datastore/plugin.py

@@ -104,6 +104,9 @@ def get_actions(self) -> dict[str, Action]:
             'datastore_function_create': action.datastore_function_create,
             'datastore_function_delete': action.datastore_function_delete,
             'datastore_run_triggers': action.datastore_run_triggers,
+            'datastore_sequence_create': action.datastore_sequence_create,
+            'datastore_sequence_delete': action.datastore_sequence_delete,
+            'datastore_sequence_next': action.datastore_sequence_next,
         }
         if getattr(self.backend, 'enable_sql_search', False):
             # Only enable search_sql if the config/backend does not disable it
@@ -127,6 +130,9 @@ def get_auth_functions(self) -> dict[str, AuthFunction]:
             'datastore_function_create': auth.datastore_function_create,
             'datastore_function_delete': auth.datastore_function_delete,
             'datastore_run_triggers': auth.datastore_run_triggers,
+            'datastore_sequence_create': auth.datastore_sequence_create,
+            'datastore_sequence_delete': auth.datastore_sequence_delete,
+            'datastore_sequence_next': auth.datastore_sequence_next,
         }
 
     # IResourceController

ckanext/datastore/tests/test_create.py

@@ -4,6 +4,8 @@
 import pytest
 import sqlalchemy.orm as orm
 
+from faker import Faker
+
 import ckan.lib.create_test_data as ctd
 import ckan.model as model
 import ckan.plugins as p
@@ -18,6 +20,8 @@
     when_was_last_analyze,
 )
 
+fake = Faker()
+
 
 @pytest.mark.usefixtures("with_request_context")
 class TestDatastoreCreateNewTests(object):
@@ -1447,3 +1451,39 @@ def test_upsert_trigger_exception(self, app):
             assert error.value.error_dict == {
                 u"records": [u'"BEANS"? Yeeeeccch!']
             }
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore")
+@pytest.mark.usefixtures("with_plugins", "with_request_context")
+def test_sequences():
+    seqname = fake.unique.user_name()
+    helpers.call_action("datastore_sequence_create", name=seqname)
+    assert helpers.call_action("datastore_sequence_next", name=seqname) == 1
+    assert helpers.call_action("datastore_sequence_next", name=seqname) == 2
+    helpers.call_action("datastore_sequence_delete", name=seqname)
+
+    with pytest.raises(ValidationError) as error:
+        helpers.call_action("datastore_sequence_next", name=seqname)
+    assert error.value.error_dict == {
+        "name": [f'relation "{seqname}" does not exist']
+    }
+
+    helpers.call_action("datastore_sequence_create", name=seqname)
+    with pytest.raises(ValidationError) as error:
+        helpers.call_action("datastore_sequence_create", name=seqname)
+    assert error.value.error_dict == {
+        "name": [f'relation "{seqname}" already exists']
+    }
+    helpers.call_action(
+        "datastore_sequence_create", name=seqname, if_not_exists=True,
+    )
+
+    helpers.call_action("datastore_sequence_delete", name=seqname)
+    with pytest.raises(ValidationError) as error:
+        helpers.call_action("datastore_sequence_delete", name=seqname)
+    assert error.value.error_dict == {
+        "name": [f'sequence "{seqname}" does not exist']
+    }
+    helpers.call_action(
+        "datastore_sequence_delete", name=seqname, if_exists=True,
+    )