feature: single workfloww worker

.github/workflows/publish_asr_worker.yml

@@ -35,6 +35,26 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           tag: ${{ github.ref_name }}
 
+  publish-asr-worker-to-pypi:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/datashare-asr-worker
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.10.8"
+      - name: Install Python 3.11
+        run: uv python install 3.11
+      - name: Build
+        run: cd workers/asr-worker && uv build
+      - name: Publish
+        run: cd workers/asr-worker && uv publish
+
   publish-io-worker:
     runs-on: ubuntu-latest
     steps:
@@ -128,6 +148,22 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
+  trigger-workflows-worker-publication:
+    needs:
+      - publish-asr-worker-to-pypi
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Trigger workflow worker release
+        run: |
+          worker_version="${worker_version/asr-worker-/}"
+          gh workflow run .github/workflows/publish_workflows_worker.yml -f worker=$worker -f worker_version="$worker_version" --ref ${{ github.ref_name }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          worker_version: ${{ github.ref_name }}
+          worker: asr-worker
+
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false

.github/workflows/publish_translation_worker.yml

@@ -20,6 +20,26 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           tag: ${{ github.ref_name }}
 
+  publish-translation-worker-to-pypi:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/datashare-translation-worker
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.10.8"
+      - name: Install Python 3.11
+        run: uv python install 3.11
+      - name: Build
+        run: cd workers/translation-worker && uv build
+      - name: Publish
+        run: cd workers/translation-worker && uv publish
+
   publish-io-worker:
     runs-on: ubuntu-latest
     steps:
@@ -82,6 +102,21 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
+  trigger-workflows-worker-publication:
+    needs:
+      - publish-translation-worker-to-pypi
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Trigger workflow worker release
+        run: |
+          worker_version="${worker_version/translation-worker-/}"
+          gh workflow run .github/workflows/publish_workflows_worker.yml -f worker=$worker -f worker_version="$worker_version" --ref ${{ github.ref_name }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          worker_version: ${{ github.ref_name }}
+          worker: translation-worker
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false

.github/workflows/publish_worker_template.yml

@@ -17,26 +17,6 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           tag: ${{ github.ref_name }}
 
-  publish-datashare-python-to-pypi:
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-    environment:
-      name: pypi
-      url: https://pypi.org/p/datashare-worker-template
-    steps:
-      - uses: actions/checkout@v6
-      - name: Install uv
-        uses: astral-sh/setup-uv@v7
-        with:
-          version: "0.10.8"
-      - name: Install Python 3.11
-        run: uv python install 3.11
-      - name: Build
-        run: cd worker-template && uv build
-      - name: Publish
-        run: cd worker-template && uv publish
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false

.github/workflows/publish_workflows_worker.yml

@@ -0,0 +1,96 @@
+name: Publish workflows-worker
+
+on:
+  workflow_dispatch:
+    inputs:
+      worker:
+        description: 'Worker app to update'
+        required: true
+        type: choice
+        options:
+          - asr-worker
+          - translation-worker
+      worker_version:
+        description: 'New version of the worker app'
+        required: true
+        type: string
+
+env:
+  WORKER: ${{ inputs.worker }}
+  WORKER_VERSION: ${{ inputs.worker_version }}
+  PYTHON_VERSION: 3.12
+  ASTRAL_VERSION: 0.11.6
+
+jobs:
+  bump-workflows-worker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.worker }}-${{ inputs.version }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: ${{ env.ASTRAL_VERSION }}
+          python-version: ${{ env.PYTHON_VERSION }}
+          enable-cache: true
+          working-directory: workers/workflows-worker
+      - name: Bump workflows worker and
+        id: bump-workflows-worker
+        run: |
+          cd workers/workflows-worker
+          WORKFLOWS_WORKER_VERSION_INFO=workflows-worker-$(run --frozen main.py datashare-"$WORKER" "$WORKER_VERSION")
+          echo workflows_worker_version_info="$WORKFLOWS_WORKER_VERSION_INFO" >> "$GITHUB_OUTPUT"
+          echo WORKFLOWS_WORKER_VERSION_INFO="$WORKFLOWS_WORKER_VERSION_INFO" >> "$GITHUB_ENV"
+          ./bump_workflow.sh "$WORKER" "$WORKER_VERSION"
+      - name: Create a workflows worker tags
+        run: |
+          git tag "$WORKFLOWS_WORKER_VERSION"
+          git push --tags
+          gh release create "$WORKFLOWS_WORKER_VERSION" --generate-notes
+        env:
+          WORKFLOWS_WORKER_VERSION: ${{ fromJSON(steps.bump-workflows-worker.outputs.workflows_worker_version_info).next }}
+      - name: Create GH release
+        run: gh release create "$WORKFLOWS_WORKER_VERSION" --generate-notes
+
+  publish-workflows-worker:
+    runs-on: ubuntu-latest
+    needs: bump-workflows-worker
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.bump-workflows-worker.outputs.workflows_worker_version }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: icij/datashare-workflows-worker
+          tags: |
+            type=match,pattern=workflows-worker-(.*),group=1
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          # You'll need to set these secrets
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v7
+        with:
+          target: io-worker
+          context: "{{defaultContext}}:workers/workflows-worker"
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false

.github/workflows/test_workflows_worker.yml

@@ -0,0 +1,29 @@
+name: Test workflows-worker
+
+on:
+  push:
+    branches: [ 'main' ]
+  pull_request:
+    paths:
+      - 'workers/workflows-worker/**'
+
+env:
+  PYTHON_VERSION: 3.12
+  ASTRAL_VERSION: 0.11.6
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/ruff-action@v3
+        with:
+          args: "--version" # skips test by displaying the version
+      - name: Check formatting
+        run: ruff format --config qa/ruff.toml --check workers/workflows-worker
+      - name: Lint test
+        run: ruff check --config qa/ruff.toml workers/workflows-worker
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true

Makefile

@@ -2,6 +2,7 @@ lock-all:
 	./scripts/lock-dist.sh worker-template
 	./scripts/lock-dist.sh asr-worker
 	./scripts/lock-dist.sh translation-worker
+	./scripts/lock-dist.sh workflows-worker
 
 lock-dist:
 	./scripts/lock-dist.sh ${project}

datashare-python/datashare_python/cli/worker.py

@@ -77,10 +77,16 @@ async def list_activities(
 async def start(
     queue: Annotated[str, typer.Option("--queue", "-q", help=_WORKER_QUEUE_HELP)],
     workflows: Annotated[
-        list[str] | None, typer.Option(help=_START_WORKER_WORKFLOWS_HELP)
+        list[str] | None,
+        typer.Option(
+            "--workflows", "--workflow", "-w", help=_START_WORKER_WORKFLOWS_HELP
+        ),
     ] = None,
     activities: Annotated[
-        list[str] | None, typer.Option(help=_START_WORKER_ACTIVITIES_HELP)
+        list[str] | None,
+        typer.Option(
+            "--activities", "--activity", "-a", help=_START_WORKER_ACTIVITIES_HELP
+        ),
     ] = None,
     dependencies: Annotated[
         str | None, typer.Option(help=_START_WORKER_DEPS_HELP)

workers/asr-worker/.dockerignore

@@ -0,0 +1,6 @@
+*
+!asr_worker
+!entrypoints
+!pyproject.toml
+!uv.dist.lock
+!uv.lock

workers/asr-worker/asr_worker/dependencies.py

@@ -31,7 +31,7 @@ def set_multiprocessing_start_method(**_) -> Generator[None, None, None]:
 
 
 REGISTRY = {
-    "inference": [set_worker_config, set_multiprocessing_start_method],
-    "io": [set_worker_config, set_es_client],
-    "cpu": [set_worker_config],
+    "asr.inference": [set_worker_config, set_multiprocessing_start_method],
+    "asr.io": [set_worker_config, set_es_client],
+    "asr.cpu": [set_worker_config],
 }

workers/asr-worker/entrypoints/cpu_worker.sh

@@ -2,7 +2,7 @@
 set -e
 
 uv run --no-sync datashare-python worker start \
-    --dependencies cpu \
+    --dependencies asr.cpu \
     --queue asr.cpu \
-    --activities asr.transcription.preprocess \
-    --activities asr.transcription.postprocess
+    --activity asr.transcription.preprocess \
+    --activity asr.transcription.postprocess

workers/asr-worker/entrypoints/inference_gpu_worker.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
 uv run --no-sync datashare-python worker start \
-    --dependencies inference \
+    --dependencies asr.inference \
     --queue asr.inference.gpu \
-    --activities asr.transcription.infer
+    --activity asr.transcription.infer

workers/asr-worker/entrypoints/io_worker.sh

@@ -2,7 +2,7 @@
 set -e
 
 uv run --no-sync datashare-python worker start \
-    --dependencies io \
+    --dependencies asr.io \
     --queue asr.io \
-    --activities asr.transcription.search-audios \
-    --workflows asr.transcription
+    --activity asr.transcription.search-audios \
+    --workflow asr.transcription

workers/asr-worker/pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-name = "asr-worker"
+name = "datashare-asr-worker"
 dynamic = ["version"]
 description = "ASR Temporal task"
 authors = [
@@ -72,6 +72,8 @@ dev = [
 
 [tool.uv-dynamic-versioning]
 fallback-version = "0.0.0"
+pattern-prefix = "asr-worker-"
+pattern = "default-unprefixed"
 
 [[tool.uv.index]]
 name = "pytorch-cpu"

workers/asr-worker/tests/test_workflows.py

@@ -48,7 +48,7 @@ async def io_bound_worker(
     client = test_temporal_client_session
     worker_id = f"worker-{uuid.uuid4()}"
     task_queue = TaskQueues.IO
-    dependencies = REGISTRY["io"]
+    dependencies = REGISTRY["asr.io"]
     activities = ASRActivities(client, event_loop)
     worker_ctx = worker_context(
         worker_id,
@@ -75,7 +75,7 @@ async def cpu_bound_worker(
     activities = ASRActivities(client, event_loop)
     worker_id = f"worker-{uuid.uuid4()}"
     task_queue = TaskQueues.CPU
-    dependencies = REGISTRY["cpu"]
+    dependencies = REGISTRY["asr.cpu"]
     worker_ctx = worker_context(
         worker_id,
         worker_config=test_worker_config,
@@ -100,7 +100,7 @@ async def gpu_inference_worker(
     activities = ASRActivities(client, event_loop)
     worker_id = f"worker-{uuid.uuid4()}"
     task_queue = TaskQueues.INFERENCE_GPU
-    dependencies = REGISTRY["inference"]
+    dependencies = REGISTRY["asr.inference"]
     worker_ctx = worker_context(
         worker_id,
         worker_config=test_worker_config,