Migration lep

cds_migrator_kit/errors.py

@@ -36,7 +36,7 @@ def __init__(
         self.exc = exc
         self.message = message
         self.priority = priority
-        super(CDSMigrationException, self).__init__(*args)
+        super(CDSMigrationException, self).__init__(message, *args)
 
 
 class RecordModelMissing(CDSMigrationException):

cds_migrator_kit/rdm/README.md

@@ -6,6 +6,8 @@
 
 https://gitlab.cern.ch/cds-team/production_scripts/-/blob/master/cds-rdm/migration/dump_users.py?ref_type=heads
 
+
+
 ## Dump a subset of records on legacy
 
 on webnode: `cds-migration-01`

cds_migrator_kit/rdm/cli.py

@@ -59,8 +59,18 @@ def migration():
     "--keep-logs",
     is_flag=True,
 )
+@click.option(
+    "--workers",
+    type=int,
+    default=None,
+    help=(
+        "Number of threads for parallel record transformation. "
+        "Defaults to sequential (no threads). "
+        "Can also be set per-collection in streams.yaml under transform.workers."
+    ),
+)
 @with_appcontext
-def run(collection, dry_run=False, keep_logs=False):
+def run(collection, dry_run=False, keep_logs=False, workers=None):
     """Run."""
     stream_config = current_app.config["CDS_MIGRATOR_KIT_STREAM_CONFIG"]
     runner = Runner(
@@ -70,7 +80,9 @@ def run(collection, dry_run=False, keep_logs=False):
         dry_run=dry_run,
         collection=collection,
         keep_logs=keep_logs,
+        workers=workers,
     )
+
     runner.run()
 
 

cds_migrator_kit/rdm/migration_config.py

@@ -7,6 +7,8 @@
 https://inveniordm.docs.cern.ch/reference/configuration/.
 """
 
+from pathlib import Path
+
 import json
 import os
 from datetime import datetime, timedelta
@@ -86,6 +88,7 @@ def _(x):  # needed to avoid start time failure with lazy strings
 # See https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/
 
 SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://cds-rdm:cds-rdm@localhost/cds-rdm"
+SQLALCHEMY_ENGINE_OPTIONS = {"connect_args": {"options": "-c timezone=UTC"}}
 
 # Invenio-App
 # ===========
@@ -342,9 +345,7 @@ def _(x):  # needed to avoid start time failure with lazy strings
 RDM_NAMESPACES = {**NAMESPACES}
 RDM_CUSTOM_FIELDS = CUSTOM_FIELDS
 
-import cds_migrator_kit
-
-base_path = os.path.dirname(os.path.realpath(cds_migrator_kit.__file__))
+base_path = Path(__file__).resolve().parent.parent
 logs_dir = os.path.join(base_path, "tmp/logs/")
 CDS_MIGRATOR_KIT_LOGS_PATH = (
     os.environ.get("INVENIO_CDS_MIGRATOR_KIT_LOGS_PATH") or logs_dir
@@ -385,6 +386,7 @@ def _(x):  # needed to avoid start time failure with lazy strings
             "validator": schemes.is_indico,
             "datacite": "INDICO",
         },
+        "hal": {"label": "HAL", "validator": schemes.is_hal, "datacite": "HAL"},
     },
     # keep internal identifiers' schemes for internal record relations
     **RDM_RECORDS_IDENTIFIERS_SCHEMES,
@@ -464,6 +466,10 @@ def resolve_record_pid(pid):
 
 CDS_MIGRATOR_KIT_ENV = "local"
 
+CDS_MIGRATOR_KIT_VOCABULARIES_DIR = None
+"""Absolute path to the vocabularies directory. Defaults to
+{instance_path}/app_data/vocabularies when None."""
+
 CDS_ACCESS_GROUP_MAPPINGS = {
     "SSO": ["cern-accounts-primary"],
     "ITDepRestrFile": ["it-dep"],