From 2d43a39acb4337f8326360a7b0c53ee2d49a6e2d Mon Sep 17 00:00:00 2001 From: FYWinds Date: Wed, 8 Apr 2026 22:23:52 -0400 Subject: [PATCH 01/16] feat: enhance deployment configuration and versioning with WCS_VERSION_SUFFIX --- README.md | 286 +++++++++++++++++++++---- app/schemas/constants.py | 5 +- deploy/overlays/dev/kustomization.yaml | 10 + 3 files changed, 254 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 2109cf3..ff526fc 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,240 @@ -## Deploying - -First create the necessary secrets for database and admin token access. Replace the placeholders with your actual values. -```bash -kubectl -n wynnsource-dev create secret generic wynnsource-secrets \ - --from-literal=WCS_ADMIN_TOKEN='' -``` -Or you can use sealed secrets for better security. - -Then use this as an example to deploy the application using ArgoCD. -```yaml -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: wynnsource-dev - namespace: argocd -spec: - project: default - source: - repoURL: git@github.com:WynnSource/WynnSourceServer.git - targetRevision: dev - path: deploy - destination: - server: https://kubernetes.default.svc - namespace: wynnsource-dev - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true -``` -You have to add your ingress configuration as well, -also make sure there is the `X-Real-IP` header from the ingress controller for proper client IP logging and rate limiting. - -## License - -`Copyright (C) <2026> ` - -This project is licensed under the GNU Affero General Public License v3.0 (AGPL-v3) with the following [**Exception**](#exception-for-generated-client-code). - -### Exception for Generated Client Code -As a special exception to the AGPL-v3, the copyright holders of this library give you permission to generate, use, distribute, and license the client libraries (SDKs) generated from this project's API specifications (e.g., OpenAPI/Swagger documents, Protocol Buffers, GraphQL schemas) under any license of your choice, including proprietary licenses. This exception does not apply to the backend logic itself. - -### Reason -We've considered the implications of the AGPL-v3 and have decided to apply it to the backend logic of this project to ensure that any modifications to the server-side code are shared with the community. However, we recognize that client libraries generated from our API specifications may be used in a wide variety of applications, including minecraft mods, which may not be compatible with the AGPL-v3. By granting this exception, we aim to encourage the use of our API and allow developers to create client libraries without worrying about licensing issues, while still ensuring that contributions to the server-side code are shared with the community. \ No newline at end of file +
+ +# WynnSourceServer + +The server component of [WynnSource](https://github.com/WynnSource) — a Wynncraft crowdsourcing mod. + +[![Python 3.12+](https://img.shields.io/badge/python-3.12+-3776AB?logo=python&logoColor=white)](https://www.python.org/) [![FastAPI](https://img.shields.io/badge/FastAPI-009688?logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/) [![Protobuf](https://img.shields.io/badge/Protocol%20Buffers-4285F4?logo=google&logoColor=white)](https://protobuf.dev/) [![License: AGPL-3.0](https://img.shields.io/badge/License-AGPL--3.0-blue.svg)](https://www.gnu.org/licenses/agpl-3.0) + +
+ +--- + +## Prerequisites + +- Python 3.12+ +- [uv](https://docs.astral.sh/uv/) — Python package manager +- [buf](https://buf.build/) — Protocol Buffers toolchain +- A PostgreSQL database +- A Redis instance + +## Development Setup + +```bash +# Clone with submodules +git clone --recurse-submodules https://github.com/WynnSource/WynnSourceServer.git +cd WynnSourceServer + +# Generate protobuf code +buf generate --template buf.gen.yaml + +# Install dependencies +uv sync + +# Run database migrations +uv run alembic upgrade head + +# Start the development server +uv run fastapi dev +``` + +### Environment Variables + +| Variable | Description | Default | +|---|---|---| +| `POSTGRES_HOST` | PostgreSQL host | `localhost` | +| `POSTGRES_PORT` | PostgreSQL port | `5432` | +| `POSTGRES_USER` | PostgreSQL user | `postgres` | +| `POSTGRES_PASSWORD` | PostgreSQL password | `postgres` | +| `POSTGRES_DB` | PostgreSQL database name | `wcs_db` | +| `REDIS_HOST` | Redis host | `localhost` | +| `REDIS_PORT` | Redis port | `6379` | +| `WCS_ADMIN_TOKEN` | Admin API token | None | +| `LEVEL` | Log level | `DEBUG` | +| `BETA_ALLOWED_VERSIONS` | Comma-separated allowed mod versions | `` | + +## Deploying to Kubernetes + +WynnSourceServer uses [Kustomize](https://kustomize.io/) overlays for deployment and [ArgoCD](https://argo-cd.readthedocs.io/) with [Image Updater](https://argocd-image-updater.readthedocs.io/) for GitOps. + +### Directory Structure + +``` +deploy/ + base/ # Shared manifests + deployment.yaml # App deployment with health probes + service.yaml # ClusterIP service on port 8000 + postgres.yaml # CNPG PostgreSQL cluster + redis.yaml # Redis instance (via Redis Operator) + migration-job.yaml # Alembic migration (ArgoCD sync hook) + kustomization.yaml + overlays/ + dev/ # Dev environment (namespace: wynnsource-dev) + kustomization.yaml + prod/ # Prod environment (namespace: wynnsource, replicas: 2) + kustomization.yaml +``` + +### Cluster Prerequisites + +The following operators must be installed in the cluster: + +- [CloudNativePG](https://cloudnative-pg.io/) — PostgreSQL operator +- [Redis Operator](https://github.com/OT-CONTAINER-KIT/redis-operator) — Redis operator +- [ArgoCD Image Updater](https://argocd-image-updater.readthedocs.io/) — Automatic image updates + +### Step 1: Create Secrets + +Create the application secrets in the target namespace. Using [Sealed Secrets](https://sealed-secrets.netlify.app/) is recommended. + +```bash +kubectl -n create secret generic wynnsource-secrets \ + --from-literal=WCS_ADMIN_TOKEN='' +``` + +### Step 2: Create ConfigMap + +```bash +kubectl -n create configmap wynnsource-config \ + --from-literal=LEVEL='INFO' \ + --from-literal=BETA_ALLOWED_VERSIONS='0.2.6, 0.2.7' +``` + +### Step 3: Create ArgoCD Application + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: wynnsource + namespace: argocd + annotations: + argocd-image-updater.argoproj.io/image-list: app=ghcr.io/wynnsource/wynnsource-server + argocd-image-updater.argoproj.io/app.update-strategy: newest-build + argocd-image-updater.argoproj.io/app.allow-tags: "regexp:^dev-" + argocd-image-updater.argoproj.io/app.ignore-tags: "latest,dev-latest" +spec: + project: default + source: + repoURL: https://github.com/WynnSource/WynnSourceServer.git + targetRevision: dev # or master for production + path: deploy/overlays/dev # or deploy/overlays/prod + destination: + server: https://kubernetes.default.svc + namespace: wynnsource-dev # or wynnsource + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true +``` + +### Step 4: Configure Ingress + +Add an IngressRoute (or Ingress) pointing to the `wynnsource-server` service on port 8000. Make sure the `X-Real-IP` header is forwarded from the ingress controller for proper client IP logging and rate limiting. + +## Using the Schema (Protobuf) + +WynnSourceServer uses [Protocol Buffers](https://protobuf.dev/) to define item encoding schemas. The `.proto` definitions live in the [`schema`](https://github.com/WynnSource/schema) submodule under `schema/proto/wynnsource/`. + +### Generating Python Code + +#### With buf (recommended) + +Install [buf](https://buf.build/docs/installation), then from the repository root: + +```bash +buf generate --template buf.gen.yaml +``` + +This generates Python protobuf modules into `generated/wynnsource/` based on: + +```yaml +# buf.gen.yaml +version: v2 +plugins: + - remote: buf.build/protocolbuffers/python:v33.5 + out: generated + - remote: buf.build/protocolbuffers/pyi:v33.5 + out: generated +inputs: + - directory: schema/proto +``` + +#### With protoc + +```bash +protoc \ + --proto_path=schema/proto \ + --python_out=generated \ + --pyi_out=generated \ + schema/proto/wynnsource/**/*.proto +``` + +### Using Generated Code + +Install the `protobuf` runtime, then import the generated modules: + +```bash +pip install protobuf +``` + +```python +from wynnsource.item.gear_pb2 import IdentifiedGear +from wynnsource.item.consumable_pb2 import Consumable +from wynnsource.common.enums_pb2 import Rarity + +# Create an item +gear = IdentifiedGear() +gear.name = "Cataclysm" + +# Serialize to bytes +data = gear.SerializeToString() + +# Deserialize from bytes +parsed = IdentifiedGear() +parsed.ParseFromString(data) +``` + +### Using as a Workspace Package + +In this repository, the generated code is a [uv workspace](https://docs.astral.sh/uv/concepts/workspaces/) package called `wynnsource-proto`. After running `buf generate` and `uv sync`, you can import directly: + +```python +from wynnsource.item.wynn_source_item_pb2 import WynnSourceItem +``` + +### Mappings + +The schema repository includes JSON mapping files: + +- `schema/mapping/identification.json` — Identification ID mappings +- `schema/mapping/shiny.json` — Shiny stat mappings + +### Generating for Other Languages + +buf supports many languages. See the [buf plugin registry](https://buf.build/plugins) for available plugins. Example for TypeScript: + +```yaml +# buf.gen.yaml +version: v2 +plugins: + - remote: buf.build/connectrpc/es + out: gen/ts +inputs: + - directory: schema/proto +``` + +## License + +This project is licensed under the [GNU Affero General Public License v3.0 (AGPL-3.0)](https://www.gnu.org/licenses/agpl-3.0) with the following exception. + +### Exception for Generated Client Code + +As a special exception, the copyright holders grant permission to generate, use, distribute, and license client libraries and SDKs produced from this project's API specifications (including OpenAPI documents, Protocol Buffer definitions, and GraphQL schemas) under any license of your choice, including proprietary licenses. This exception applies solely to the generated client code — the server-side source code remains subject to the AGPL-3.0 in full. + +### Rationale + +The AGPL-3.0 ensures that improvements to the server are shared with the community. However, client libraries derived from our API specifications are commonly embedded in Minecraft mods and other applications whose licenses may be incompatible with the AGPL. This exception removes that friction: developers can freely build and ship clients against our API without licensing concerns, while contributions to the backend itself continue to benefit everyone. diff --git a/app/schemas/constants.py b/app/schemas/constants.py index 33c0d61..55aaf39 100644 --- a/app/schemas/constants.py +++ b/app/schemas/constants.py @@ -1,4 +1,5 @@ import enum +import os import tomllib @@ -19,7 +20,9 @@ def get_media_type(cls, media_type: str) -> str: pyproject = tomllib.load(f) __NAME__ = "WynnSource Server" -__VERSION__: str = pyproject["project"]["version"] +_base_version: str = pyproject["project"]["version"] +_version_suffix = os.environ.get("WCS_VERSION_SUFFIX", "") +__VERSION__: str = f"{_base_version}{_version_suffix}" if _version_suffix else _base_version __DESCRIPTION__: str = pyproject["project"]["description"] __REVISION__ = 2 diff --git a/deploy/overlays/dev/kustomization.yaml b/deploy/overlays/dev/kustomization.yaml index 67a22ab..e159fbd 100644 --- a/deploy/overlays/dev/kustomization.yaml +++ b/deploy/overlays/dev/kustomization.yaml @@ -6,3 +6,13 @@ resources: images: - name: ghcr.io/wynnsource/wynnsource-server newTag: dev-latest +patches: + - target: + kind: Deployment + name: wynnsource-server + patch: | + - op: add + path: /spec/template/spec/containers/0/env + value: + - name: WCS_VERSION_SUFFIX + value: "-beta" From 669e9bcceb12c14535616cf4750fde5f54bcdc0b Mon Sep 17 00:00:00 2001 From: FYWinds Date: Wed, 8 Apr 2026 22:33:53 -0400 Subject: [PATCH 02/16] feat: add labels for instance identification in kustomization files --- deploy/overlays/dev/kustomization.yaml | 4 ++++ deploy/overlays/prod/kustomization.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/deploy/overlays/dev/kustomization.yaml b/deploy/overlays/dev/kustomization.yaml index e159fbd..1920e05 100644 --- a/deploy/overlays/dev/kustomization.yaml +++ b/deploy/overlays/dev/kustomization.yaml @@ -6,6 +6,10 @@ resources: images: - name: ghcr.io/wynnsource/wynnsource-server newTag: dev-latest +labels: + - pairs: + app.kubernetes.io/instance: wynnsource-dev + includeSelectors: false patches: - target: kind: Deployment diff --git a/deploy/overlays/prod/kustomization.yaml b/deploy/overlays/prod/kustomization.yaml index 6602c1a..602396f 100644 --- a/deploy/overlays/prod/kustomization.yaml +++ b/deploy/overlays/prod/kustomization.yaml @@ -6,6 +6,10 @@ resources: images: - name: ghcr.io/wynnsource/wynnsource-server newTag: latest +labels: + - pairs: + app.kubernetes.io/instance: wynnsource-prod + includeSelectors: false patches: - target: kind: Deployment From 0beb3f746e532bbf56f13b82cf34a2486efe4ec5 Mon Sep 17 00:00:00 2001 From: FYWinds Date: Wed, 8 Apr 2026 22:49:34 -0400 Subject: [PATCH 03/16] fix: correct patch path for WCS_VERSION_SUFFIX in kustomization.yaml --- deploy/overlays/dev/kustomization.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/overlays/dev/kustomization.yaml b/deploy/overlays/dev/kustomization.yaml index 1920e05..870e322 100644 --- a/deploy/overlays/dev/kustomization.yaml +++ b/deploy/overlays/dev/kustomization.yaml @@ -16,7 +16,7 @@ patches: name: wynnsource-server patch: | - op: add - path: /spec/template/spec/containers/0/env + path: /spec/template/spec/containers/0/env/- value: - - name: WCS_VERSION_SUFFIX - value: "-beta" + name: WCS_VERSION_SUFFIX + value: "-beta" From cdf08a9ad1f81c56ea1e185b7d304dc6a85e06b6 Mon Sep 17 00:00:00 2001 From: FYWinds Date: Thu, 9 Apr 2026 09:42:19 -0400 Subject: [PATCH 04/16] fix: updated gambit region and etc. --- app/module/raid/config.py | 10 +++++----- app/module/raid/router.py | 8 ++------ app/module/raid/schema.py | 4 ---- app/module/raid/service.py | 11 +++-------- 4 files changed, 10 insertions(+), 23 deletions(-) diff --git a/app/module/raid/config.py b/app/module/raid/config.py index 49c8cc5..cda786e 100644 --- a/app/module/raid/config.py +++ b/app/module/raid/config.py @@ -6,6 +6,7 @@ GAMBIT_COUNT = 4 GAMBIT_SEPARATOR = "|" +GAMBIT_REGION = "global" FUZZY_WINDOW = timedelta(minutes=30) CONSENSUS_THRESHOLD = 0.6 @@ -19,17 +20,16 @@ class GambitRotation: def get_gambit_rotation(time: datetime, shift: int = 0) -> GambitRotation: """Get the daily gambit rotation window for the given timestamp. - Gambits rotate daily at EST/EDT midnight (00:00 America/New_York). + Gambits rotate daily at EST/EDT noon (12:00 America/New_York). """ if time.tzinfo is None: raise ValueError("The 'time' parameter must be timezone-aware.") local_time = time.astimezone(SERVER_TZ) - # Today's reset at midnight EST - today_reset = datetime.combine( - local_time.date(), datetime.min.time(), tzinfo=SERVER_TZ - ) + # Today's reset at noon EST + today_reset = datetime.combine(local_time.date(), datetime.min.time(), tzinfo=SERVER_TZ) + today_reset += timedelta(hours=12) if local_time < today_reset: today_reset -= timedelta(days=1) diff --git a/app/module/raid/router.py b/app/module/raid/router.py index d6d8cff..490303e 100644 --- a/app/module/raid/router.py +++ b/app/module/raid/router.py @@ -8,7 +8,6 @@ from app.core.rate_limiter import ip_based_key_func, user_based_key_func from app.core.router import DocedAPIRoute from app.core.security.auth import UserDep -from app.module.pool.schema import RaidRegion from app.schemas.enums import ApiTag from app.schemas.response import EMPTY_RESPONSE, EmptyResponse, WCSResponse @@ -47,11 +46,10 @@ async def submit_gambit_data(data: list[GambitSubmissionSchema], user: UserDep) return EMPTY_RESPONSE -@RaidRouter.get("/gambit/{region}", summary="Get Current Gambit Consensus") +@RaidRouter.get("/gambit", summary="Get Current Gambit Consensus") @metadata.rate_limit(limit=10, period=60, key_func=ip_based_key_func) @metadata.cached(expire=120) async def get_gambit_by_region( - region: RaidRegion, session: SessionDep, ) -> WCSResponse[GambitConsensusResponse]: """ @@ -59,11 +57,10 @@ async def get_gambit_by_region( """ try: rotation = get_gambit_rotation(datetime.datetime.now(tz=datetime.UTC)) - result = await get_gambit_consensus(session, region, rotation.start) + result = await get_gambit_consensus(session, rotation.start) if result is None: data = GambitConsensusResponse( - region=region, rotation_start=rotation.start, rotation_end=rotation.end, gambits=[], @@ -72,7 +69,6 @@ async def get_gambit_by_region( else: pairs, confidence = result data = GambitConsensusResponse( - region=region, rotation_start=rotation.start, rotation_end=rotation.end, gambits=[ diff --git a/app/module/raid/schema.py b/app/module/raid/schema.py index d954d07..6e893fb 100644 --- a/app/module/raid/schema.py +++ b/app/module/raid/schema.py @@ -2,8 +2,6 @@ from pydantic import BaseModel, Field, model_validator -from app.module.pool.schema import RaidRegion - from .config import GAMBIT_COUNT @@ -13,7 +11,6 @@ class GambitEntry(BaseModel): class GambitSubmissionSchema(BaseModel): - region: RaidRegion client_timestamp: datetime mod_version: str gambits: list[GambitEntry] = Field( @@ -37,7 +34,6 @@ class GambitConsensusEntry(BaseModel): class GambitConsensusResponse(BaseModel): - region: str rotation_start: datetime rotation_end: datetime gambits: list[GambitConsensusEntry] diff --git a/app/module/raid/service.py b/app/module/raid/service.py index c546a82..1ca0d01 100644 --- a/app/module/raid/service.py +++ b/app/module/raid/service.py @@ -7,10 +7,9 @@ from app.core.db import get_session from app.core.scheduler import SCHEDULER from app.core.security.model import User -from app.module.pool.schema import RaidRegion from app.module.pool.service import calculate_submission_weight -from .config import FUZZY_WINDOW, GAMBIT_COUNT, GAMBIT_SEPARATOR, get_gambit_rotation +from .config import FUZZY_WINDOW, GAMBIT_COUNT, GAMBIT_REGION, GAMBIT_SEPARATOR, get_gambit_rotation from .model import GambitRepository, GambitSubmission, GambitSubmissionRepository from .schema import GambitSubmissionSchema @@ -26,7 +25,7 @@ async def submit_gambit_data(session: AsyncSession, data: GambitSubmissionSchema rotation = get_gambit_rotation(data.client_timestamp) gambit = await gambit_repo.get_or_create_gambit( - region=data.region.value, + region=GAMBIT_REGION, rotation=rotation, ) @@ -104,21 +103,17 @@ async def compute_gambit_consensus(): slot_confidences.append(best_weight / total_slot_weight if total_slot_weight > 0 else 0.0) gambit.consensus_data = consensus_data - gambit.confidence = round( - sum(slot_confidences) / len(slot_confidences) if slot_confidences else 0.0, 4 - ) + gambit.confidence = round(sum(slot_confidences) / len(slot_confidences) if slot_confidences else 0.0, 4) gambit.needs_recalc = False async def get_gambit_consensus( session: AsyncSession, - region: RaidRegion, rotation_start: datetime.datetime, ) -> tuple[list[tuple[str, str]], float] | None: """Returns ([(name, description), ...], confidence) or None if no data.""" gambit_repo = GambitRepository(session) gambits = await gambit_repo.list_gambits( - region=region.value, rotation_start=rotation_start, ) From d0416988a2f48a6df71b4559e9aedcc41fb06b6e Mon Sep 17 00:00:00 2001 From: FYWinds Date: Thu, 9 Apr 2026 09:43:44 -0400 Subject: [PATCH 05/16] chore: bump version --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f8c60e0..80bbfb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "wynnsource_server" -version = "0.1.0" +version = "0.1.1" description = "The server component of WynnSource - a Wynncraft Crowdsourcing Mod." readme = "README.md" requires-python = ">=3.12, <4.0" diff --git a/uv.lock b/uv.lock index 1059a2d..ae0fe70 100644 --- a/uv.lock +++ b/uv.lock @@ -1604,7 +1604,7 @@ requires-dist = [ [[package]] name = "wynnsource-server" -version = "0.1.0" +version = "0.1.1" source = { virtual = "." } dependencies = [ { name = "alembic" }, From fe5ea480d151bc388ce69e11cdd8b63038fcf6a7 Mon Sep 17 00:00:00 2001 From: FYWinds Date: Thu, 9 Apr 2026 10:26:25 -0400 Subject: [PATCH 06/16] fix: gambit not updating --- app/module/raid/service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/module/raid/service.py b/app/module/raid/service.py index 1ca0d01..87a2b8a 100644 --- a/app/module/raid/service.py +++ b/app/module/raid/service.py @@ -64,6 +64,7 @@ async def compute_gambit_consensus(): gambit_repo = GambitRepository(session) rotation = get_gambit_rotation(datetime.datetime.now(tz=datetime.UTC)) active_gambits = await gambit_repo.list_gambits( + region=GAMBIT_REGION, rotation_start=rotation.start, needs_recalc=True, ) @@ -114,6 +115,7 @@ async def get_gambit_consensus( """Returns ([(name, description), ...], confidence) or None if no data.""" gambit_repo = GambitRepository(session) gambits = await gambit_repo.list_gambits( + region=GAMBIT_REGION, rotation_start=rotation_start, ) From 3907c40e010e28a481840d311938a6d1f3f14cff Mon Sep 17 00:00:00 2001 From: FYWinds Date: Thu, 9 Apr 2026 10:31:51 -0400 Subject: [PATCH 07/16] chore: bump version --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 80bbfb5..81043e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "wynnsource_server" -version = "0.1.1" +version = "0.1.2" description = "The server component of WynnSource - a Wynncraft Crowdsourcing Mod." readme = "README.md" requires-python = ">=3.12, <4.0" diff --git a/uv.lock b/uv.lock index ae0fe70..c98b57a 100644 --- a/uv.lock +++ b/uv.lock @@ -1604,7 +1604,7 @@ requires-dist = [ [[package]] name = "wynnsource-server" -version = "0.1.1" +version = "0.1.2" source = { virtual = "." } dependencies = [ { name = "alembic" }, From 0ad31214a22c19ba2d442677964e018015afe41d Mon Sep 17 00:00:00 2001 From: FYWinds Date: Thu, 9 Apr 2026 10:37:26 -0400 Subject: [PATCH 08/16] fix: container retention [skip ci] --- .github/workflows/ghcr-cleanup.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ghcr-cleanup.yaml b/.github/workflows/ghcr-cleanup.yaml index 64d1e47..79aaa7c 100644 --- a/.github/workflows/ghcr-cleanup.yaml +++ b/.github/workflows/ghcr-cleanup.yaml @@ -12,7 +12,7 @@ jobs: packages: write steps: - name: Clean up dev images - uses: snok/container-retention-policy@v3 + uses: snok/container-retention-policy@v3.0.0 with: account: ${{ github.repository_owner }} token: ${{ secrets.GITHUB_TOKEN }} @@ -24,7 +24,7 @@ jobs: image-tags: "!latest" - name: Clean up untagged manifests - uses: snok/container-retention-policy@v3 + uses: snok/container-retention-policy@v3.0.0 with: account: ${{ github.repository_owner }} token: ${{ secrets.GITHUB_TOKEN }} From a1e983cc2d6b315e3e9bf2aa6c8089d45e27927f Mon Sep 17 00:00:00 2001 From: FYWinds Date: Thu, 9 Apr 2026 10:39:46 -0400 Subject: [PATCH 09/16] fix: container retention [skip ci] --- .github/workflows/ghcr-cleanup.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ghcr-cleanup.yaml b/.github/workflows/ghcr-cleanup.yaml index 79aaa7c..e1e8d56 100644 --- a/.github/workflows/ghcr-cleanup.yaml +++ b/.github/workflows/ghcr-cleanup.yaml @@ -17,11 +17,10 @@ jobs: account: ${{ github.repository_owner }} token: ${{ secrets.GITHUB_TOKEN }} image-names: wynnsource-server - cut-off: 7 days ago UTC + cut-off: 1w keep-n-most-recent: 5 - filter-tags: "dev-*" - skip-tags: "dev-latest" - image-tags: "!latest" + tag-selection: tagged + image-tags: "dev-* !dev-latest !latest" - name: Clean up untagged manifests uses: snok/container-retention-policy@v3.0.0 @@ -29,5 +28,5 @@ jobs: account: ${{ github.repository_owner }} token: ${{ secrets.GITHUB_TOKEN }} image-names: wynnsource-server - cut-off: 1 day ago UTC - untagged-only: true + cut-off: 1d + tag-selection: untagged From 42026ebbb7fbbede9763d13a93595b51daf4f33d Mon Sep 17 00:00:00 2001 From: FYWinds Date: Sat, 11 Apr 2026 10:43:05 -0400 Subject: [PATCH 10/16] fix: optimized pool rotation calculation --- app/module/pool/config.py | 28 +++++++++++++--------------- app/module/raid/config.py | 16 ++++++---------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/app/module/pool/config.py b/app/module/pool/config.py index 28cfa79..79b4eb6 100644 --- a/app/module/pool/config.py +++ b/app/module/pool/config.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from datetime import date, datetime, timedelta +from datetime import UTC, date, datetime, timedelta from zoneinfo import ZoneInfo from app.core.score import Tier @@ -18,14 +18,16 @@ class PoolRotation: @dataclass class PoolConfig: weekday: int - winter_hour: int - summer_hour: int + winter_utc_hour: int + summer_utc_hour: int - def _get_exact_reset_time(self, d: date) -> datetime: + def _is_dst(self, d: date) -> bool: noon = datetime.combine(d, datetime.min.time().replace(hour=12), tzinfo=SERVER_TZ) - is_dst = bool(noon.dst()) - hour = self.summer_hour if is_dst else self.winter_hour - return datetime.combine(d, datetime.min.time().replace(hour=hour), tzinfo=SERVER_TZ) + return bool(noon.dst()) + + def _get_exact_reset_time(self, d: date) -> datetime: + hour = self.summer_utc_hour if self._is_dst(d) else self.winter_utc_hour + return datetime(d.year, d.month, d.day, hour, tzinfo=UTC) def get_rotation(self, time: datetime, shift: int = 0) -> PoolRotation: if time.tzinfo is None: @@ -38,7 +40,7 @@ def get_rotation(self, time: datetime, shift: int = 0) -> PoolRotation: current_reset = self._get_exact_reset_time(candidate_date) - if local_time < current_reset: + if time < current_reset: candidate_date -= timedelta(days=7) current_reset = self._get_exact_reset_time(candidate_date) @@ -52,13 +54,9 @@ def get_rotation(self, time: datetime, shift: int = 0) -> PoolRotation: POOL_REFRESH_CONFIG = { - PoolType.LR_ITEM: PoolConfig(weekday=4, winter_hour=15, summer_hour=14), - PoolType.RAID_ASPECT: PoolConfig(weekday=4, winter_hour=14, summer_hour=13), - PoolType.RAID_ITEM: PoolConfig( - weekday=4, - winter_hour=15, - summer_hour=14, - ), + PoolType.LR_ITEM: PoolConfig(weekday=4, winter_utc_hour=19, summer_utc_hour=18), + PoolType.RAID_ASPECT: PoolConfig(weekday=4, winter_utc_hour=18, summer_utc_hour=17), + PoolType.RAID_ITEM: PoolConfig(weekday=4, winter_utc_hour=18, summer_utc_hour=17), } FUZZY_WINDOW = timedelta(minutes=90) diff --git a/app/module/raid/config.py b/app/module/raid/config.py index cda786e..b1706de 100644 --- a/app/module/raid/config.py +++ b/app/module/raid/config.py @@ -1,12 +1,10 @@ from dataclasses import dataclass -from datetime import datetime, timedelta -from zoneinfo import ZoneInfo - -SERVER_TZ = ZoneInfo("America/New_York") +from datetime import UTC, datetime, timedelta GAMBIT_COUNT = 4 GAMBIT_SEPARATOR = "|" GAMBIT_REGION = "global" +GAMBIT_RESET_UTC_HOUR = 17 FUZZY_WINDOW = timedelta(minutes=30) CONSENSUS_THRESHOLD = 0.6 @@ -20,18 +18,16 @@ class GambitRotation: def get_gambit_rotation(time: datetime, shift: int = 0) -> GambitRotation: """Get the daily gambit rotation window for the given timestamp. - Gambits rotate daily at EST/EDT noon (12:00 America/New_York). + Gambits rotate daily at 17:00 UTC (12:00 EST / 13:00 EDT). """ if time.tzinfo is None: raise ValueError("The 'time' parameter must be timezone-aware.") - local_time = time.astimezone(SERVER_TZ) + utc_time = time.astimezone(UTC) - # Today's reset at noon EST - today_reset = datetime.combine(local_time.date(), datetime.min.time(), tzinfo=SERVER_TZ) - today_reset += timedelta(hours=12) + today_reset = datetime(utc_time.year, utc_time.month, utc_time.day, GAMBIT_RESET_UTC_HOUR, tzinfo=UTC) - if local_time < today_reset: + if utc_time < today_reset: today_reset -= timedelta(days=1) if shift != 0: From 5fde035a7137f52001a8926a2c89fb4ed948d743 Mon Sep 17 00:00:00 2001 From: FYWinds Date: Sat, 11 Apr 2026 11:10:33 -0400 Subject: [PATCH 11/16] feat: added info in recalc --- app/module/pool/router.py | 7 +++---- app/module/pool/service.py | 10 +++++++--- app/module/raid/router.py | 6 +++--- app/module/raid/service.py | 4 +++- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/app/module/pool/router.py b/app/module/pool/router.py index 30ac427..a2edcdf 100644 --- a/app/module/pool/router.py +++ b/app/module/pool/router.py @@ -101,11 +101,10 @@ async def get_pools_by_type_and_region( @PoolRouter.get("/pools/recalc", summary="Force Recalculate Pool Consensus") @metadata.permission("pool.recalc") -async def recalculate_pools() -> EmptyResponse: +async def recalculate_pools() -> WCSResponse[dict]: """ Force recalculate pool consensus. This is useful for admins to fix any issues with the consensus calculation. """ - - await compute_pool_consensus() - return EMPTY_RESPONSE + recalculated = await compute_pool_consensus() + return WCSResponse.from_message(f"Recalculated {recalculated} pool(s)") diff --git a/app/module/pool/service.py b/app/module/pool/service.py index dff1055..9ceba8d 100644 --- a/app/module/pool/service.py +++ b/app/module/pool/service.py @@ -105,12 +105,14 @@ def calculate_submission_weight(user: User, fuzzy: bool = False) -> float: misfire_grace_time=60, coalesce=True, # Coalesce multiple missed executions into one ) -async def compute_pool_consensus(): +async def compute_pool_consensus() -> int: + total = 0 for pool_type in PoolType: - await compute_pool_consensus_for_pool(pool_type) + total += await compute_pool_consensus_for_pool(pool_type) + return total -async def compute_pool_consensus_for_pool(pool_type: PoolType): +async def compute_pool_consensus_for_pool(pool_type: PoolType) -> int: async with get_session() as session: # Step 1: Fetch all active pools that need consensus computation poolRepo = PoolRepository(session) @@ -171,6 +173,8 @@ async def compute_pool_consensus_for_pool(pool_type: PoolType): pool.confidence = round(confidence, 4) pool.needs_recalc = False + return len(active_pools) + type ConsensusByPage = dict[int, tuple[list[bytes], float]] diff --git a/app/module/raid/router.py b/app/module/raid/router.py index 490303e..3af4d02 100644 --- a/app/module/raid/router.py +++ b/app/module/raid/router.py @@ -21,12 +21,12 @@ @RaidRouter.get("/gambit/recalc", summary="Force Recalculate Gambit Consensus") @metadata.permission("raid.recalc") -async def recalculate_gambits() -> EmptyResponse: +async def recalculate_gambits() -> WCSResponse[dict]: """ Force recalculate gambit consensus. """ - await compute_gambit_consensus() - return EMPTY_RESPONSE + recalculated = await compute_gambit_consensus() + return WCSResponse.from_message(f"Recalculated {recalculated} gambit(s)") @RaidRouter.post("/gambit/submit", summary="Submit Gambit Data") diff --git a/app/module/raid/service.py b/app/module/raid/service.py index 87a2b8a..c38053b 100644 --- a/app/module/raid/service.py +++ b/app/module/raid/service.py @@ -59,7 +59,7 @@ async def submit_gambit_data(session: AsyncSession, data: GambitSubmissionSchema misfire_grace_time=60, coalesce=True, ) -async def compute_gambit_consensus(): +async def compute_gambit_consensus() -> int: async with get_session() as session: gambit_repo = GambitRepository(session) rotation = get_gambit_rotation(datetime.datetime.now(tz=datetime.UTC)) @@ -107,6 +107,8 @@ async def compute_gambit_consensus(): gambit.confidence = round(sum(slot_confidences) / len(slot_confidences) if slot_confidences else 0.0, 4) gambit.needs_recalc = False + return len(active_gambits) + async def get_gambit_consensus( session: AsyncSession, From 81d4d36cb399ec1f4f4d590b9622a3d00236084e Mon Sep 17 00:00:00 2001 From: FYWinds Date: Sat, 11 Apr 2026 11:13:24 -0400 Subject: [PATCH 12/16] fix: forget to bump version --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 81043e2..be9d2f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "wynnsource_server" -version = "0.1.2" +version = "0.1.4" description = "The server component of WynnSource - a Wynncraft Crowdsourcing Mod." readme = "README.md" requires-python = ">=3.12, <4.0" diff --git a/uv.lock b/uv.lock index c98b57a..171f9f3 100644 --- a/uv.lock +++ b/uv.lock @@ -1604,7 +1604,7 @@ requires-dist = [ [[package]] name = "wynnsource-server" -version = "0.1.2" +version = "0.1.4" source = { virtual = "." } dependencies = [ { name = "alembic" }, From 4503faafe6c385ecc688628d0f360b792030a9ba Mon Sep 17 00:00:00 2001 From: FYWinds Date: Sun, 12 Apr 2026 01:31:15 -0400 Subject: [PATCH 13/16] fix: update ghcr cleanup action --- .github/workflows/ghcr-cleanup.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ghcr-cleanup.yaml b/.github/workflows/ghcr-cleanup.yaml index e1e8d56..3851910 100644 --- a/.github/workflows/ghcr-cleanup.yaml +++ b/.github/workflows/ghcr-cleanup.yaml @@ -12,7 +12,7 @@ jobs: packages: write steps: - name: Clean up dev images - uses: snok/container-retention-policy@v3.0.0 + uses: snok/container-retention-policy@v3.0.1 with: account: ${{ github.repository_owner }} token: ${{ secrets.GITHUB_TOKEN }} @@ -23,7 +23,7 @@ jobs: image-tags: "dev-* !dev-latest !latest" - name: Clean up untagged manifests - uses: snok/container-retention-policy@v3.0.0 + uses: snok/container-retention-policy@v3.0.1 with: account: ${{ github.repository_owner }} token: ${{ secrets.GITHUB_TOKEN }} From 8fdd57ed7a4cfeaac76d178e5484de20e241e615 Mon Sep 17 00:00:00 2001 From: FYWinds Date: Tue, 14 Apr 2026 13:41:33 -0400 Subject: [PATCH 14/16] feat: integrate Sentry for error tracking and monitoring --- app/config/__init__.py | 2 + app/config/sentry.py | 19 +++++++++ app/core/scheduler.py | 11 +++++ app/core/sentry.py | 53 +++++++++++++++++++++++++ app/main.py | 3 ++ deploy/base/deployment.yaml | 6 +++ deploy/overlays/dev/kustomization.yaml | 15 +++++++ deploy/overlays/prod/kustomization.yaml | 15 +++++++ pyproject.toml | 3 +- uv.lock | 4 +- 10 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 app/config/sentry.py create mode 100644 app/core/sentry.py diff --git a/app/config/__init__.py b/app/config/__init__.py index 6e4043c..8b931ce 100644 --- a/app/config/__init__.py +++ b/app/config/__init__.py @@ -1,11 +1,13 @@ from .admin import ADMIN_CONFIG as ADMIN_CONFIG from .db import DB_CONFIG as DB_CONFIG from .log import LOG_CONFIG as LOG_CONFIG +from .sentry import SENTRY_CONFIG as SENTRY_CONFIG from .user import USER_CONFIG as USER_CONFIG __all__ = [ "ADMIN_CONFIG", "DB_CONFIG", "LOG_CONFIG", + "SENTRY_CONFIG", "USER_CONFIG", ] diff --git a/app/config/sentry.py b/app/config/sentry.py new file mode 100644 index 0000000..b21b6fd --- /dev/null +++ b/app/config/sentry.py @@ -0,0 +1,19 @@ +from pydantic import Field +from pydantic_settings import BaseSettings + + +class SentryConfig(BaseSettings): + """ + Sentry configuration. + """ + + dsn: str | None = Field(alias="WCS_SENTRY_DSN", default=None) + environment: str = Field(alias="WCS_SENTRY_ENVIRONMENT", default="local") + traces_sample_rate: float = Field(alias="WCS_SENTRY_TRACES_SAMPLE_RATE", default=0.1) + profiles_sample_rate: float = Field(alias="WCS_SENTRY_PROFILES_SAMPLE_RATE", default=0.1) + + +SENTRY_CONFIG = SentryConfig() +__all__ = [ + "SENTRY_CONFIG", +] diff --git a/app/core/scheduler.py b/app/core/scheduler.py index 1d1b266..ab47c1a 100644 --- a/app/core/scheduler.py +++ b/app/core/scheduler.py @@ -1,6 +1,17 @@ +from apscheduler.events import EVENT_JOB_ERROR, JobExecutionEvent from apscheduler.schedulers.asyncio import AsyncIOScheduler +from app.core.sentry import sentry_enabled + SCHEDULER = AsyncIOScheduler() +if sentry_enabled(): + import sentry_sdk as sentry + + def sentry_listener(event: JobExecutionEvent): + sentry.capture_exception(event.exception) + + SCHEDULER.add_listener(sentry_listener, EVENT_JOB_ERROR) + __all__ = ["SCHEDULER"] diff --git a/app/core/sentry.py b/app/core/sentry.py new file mode 100644 index 0000000..b5ca004 --- /dev/null +++ b/app/core/sentry.py @@ -0,0 +1,53 @@ +import logging + +import sentry_sdk as sentry +from fastapi import HTTPException +from sentry_sdk.integrations.asyncio import AsyncioIntegration +from sentry_sdk.integrations.loguru import LoguruIntegration +from sentry_sdk.integrations.redis import RedisIntegration +from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration +from sentry_sdk.types import Event, Hint + +from app.config import SENTRY_CONFIG +from app.schemas.constants import __VERSION__ + + +def sentry_enabled() -> bool: + return SENTRY_CONFIG.dsn is not None + + +def filter_http_exception(event: Event, hint: Hint) -> Event | None: + _exc_type, exc_value, _traceback = hint.get("exc_info", (None, None, None)) + + if isinstance(exc_value, HTTPException): + if exc_value.status_code < 500: + return None + + return event + + +def init_sentry() -> bool: + if not SENTRY_CONFIG.dsn: + return False + sentry.init( + dsn=SENTRY_CONFIG.dsn, + environment=SENTRY_CONFIG.environment, + release=__VERSION__, + traces_sample_rate=SENTRY_CONFIG.traces_sample_rate, + profiles_sample_rate=SENTRY_CONFIG.profiles_sample_rate, + send_default_pii=False, + before_send=filter_http_exception, + integrations=[ + AsyncioIntegration(), + LoguruIntegration(event_level=logging.CRITICAL), + SqlalchemyIntegration(), + RedisIntegration(), + ], + ) + return True + + +__all__ = [ + "init_sentry", + "sentry_enabled", +] diff --git a/app/main.py b/app/main.py index 9bf2bfd..345398e 100644 --- a/app/main.py +++ b/app/main.py @@ -9,6 +9,7 @@ from app.core.db import RedisClient, close_db, init_db from app.core.openapi import custom_openapi from app.core.scheduler import SCHEDULER +from app.core.sentry import init_sentry from app.module.api.exception_handler import ( generic_exception_handler, http_exception_handler, @@ -18,6 +19,8 @@ from app.schemas.constants import __DESCRIPTION__, __NAME__, __VERSION__ from app.schemas.response import STATUS_RESPONSE, StatusResponse +init_sentry() + @asynccontextmanager async def lifespan(app: FastAPI): diff --git a/deploy/base/deployment.yaml b/deploy/base/deployment.yaml index 7c76d8b..53a18c6 100644 --- a/deploy/base/deployment.yaml +++ b/deploy/base/deployment.yaml @@ -59,6 +59,12 @@ spec: value: wynnsource-redis - name: REDIS_PORT value: "6379" + - name: WCS_SENTRY_DSN + valueFrom: + secretKeyRef: + name: wynnsource-secrets + key: sentry-dsn + optional: true livenessProbe: httpGet: { path: /healthz, port: 8000 } initialDelaySeconds: 10 diff --git a/deploy/overlays/dev/kustomization.yaml b/deploy/overlays/dev/kustomization.yaml index 870e322..194e18a 100644 --- a/deploy/overlays/dev/kustomization.yaml +++ b/deploy/overlays/dev/kustomization.yaml @@ -20,3 +20,18 @@ patches: value: name: WCS_VERSION_SUFFIX value: "-beta" + - op: add + path: /spec/template/spec/containers/0/env/- + value: + name: WCS_SENTRY_ENVIRONMENT + value: development + - op: add + path: /spec/template/spec/containers/0/env/- + value: + name: WCS_SENTRY_TRACES_SAMPLE_RATE + value: "0.5" + - op: add + path: /spec/template/spec/containers/0/env/- + value: + name: WCS_SENTRY_PROFILES_SAMPLE_RATE + value: "0.5" diff --git a/deploy/overlays/prod/kustomization.yaml b/deploy/overlays/prod/kustomization.yaml index 602396f..0f46406 100644 --- a/deploy/overlays/prod/kustomization.yaml +++ b/deploy/overlays/prod/kustomization.yaml @@ -18,3 +18,18 @@ patches: - op: replace path: /spec/replicas value: 2 + - op: add + path: /spec/template/spec/containers/0/env/- + value: + name: WCS_SENTRY_ENVIRONMENT + value: production + - op: add + path: /spec/template/spec/containers/0/env/- + value: + name: WCS_SENTRY_TRACES_SAMPLE_RATE + value: "0.1" + - op: add + path: /spec/template/spec/containers/0/env/- + value: + name: WCS_SENTRY_PROFILES_SAMPLE_RATE + value: "0.1" diff --git a/pyproject.toml b/pyproject.toml index be9d2f4..1decf6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "wynnsource_server" -version = "0.1.4" +version = "0.1.5" description = "The server component of WynnSource - a Wynncraft Crowdsourcing Mod." readme = "README.md" requires-python = ">=3.12, <4.0" @@ -25,6 +25,7 @@ dependencies = [ "pydantic-settings>=2.10.1", "redis>=7.1.1", "scalar-fastapi>=1.6.2", + "sentry-sdk>=2.53.0", "sqlalchemy>=2.0.40", "wynnsource-proto", ] diff --git a/uv.lock b/uv.lock index 171f9f3..117d8f2 100644 --- a/uv.lock +++ b/uv.lock @@ -1604,7 +1604,7 @@ requires-dist = [ [[package]] name = "wynnsource-server" -version = "0.1.4" +version = "0.1.5" source = { virtual = "." } dependencies = [ { name = "alembic" }, @@ -1618,6 +1618,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "redis" }, { name = "scalar-fastapi" }, + { name = "sentry-sdk" }, { name = "sqlalchemy" }, { name = "wynnsource-proto" }, ] @@ -1646,6 +1647,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.10.1" }, { name = "redis", specifier = ">=7.1.1" }, { name = "scalar-fastapi", specifier = ">=1.6.2" }, + { name = "sentry-sdk", specifier = ">=2.53.0" }, { name = "sqlalchemy", specifier = ">=2.0.40" }, { name = "wynnsource-proto", editable = "generated" }, ] From d7405ea58169413c0067a124c6d0c44498a3f72f Mon Sep 17 00:00:00 2001 From: FYWinds Date: Fri, 17 Apr 2026 17:07:36 -0400 Subject: [PATCH 15/16] feat: enable sentry logs --- app/core/sentry.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/core/sentry.py b/app/core/sentry.py index b5ca004..dbfb8dd 100644 --- a/app/core/sentry.py +++ b/app/core/sentry.py @@ -3,6 +3,7 @@ import sentry_sdk as sentry from fastapi import HTTPException from sentry_sdk.integrations.asyncio import AsyncioIntegration +from sentry_sdk.integrations.logging import LoggingIntegration from sentry_sdk.integrations.loguru import LoguruIntegration from sentry_sdk.integrations.redis import RedisIntegration from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration @@ -43,6 +44,8 @@ def init_sentry() -> bool: SqlalchemyIntegration(), RedisIntegration(), ], + disabled_integrations=[LoggingIntegration()], + enable_logs=True, ) return True From e9eaaeee5e3a8187db66754e294fbe33b7f956f1 Mon Sep 17 00:00:00 2001 From: FYWinds Date: Fri, 17 Apr 2026 17:08:13 -0400 Subject: [PATCH 16/16] feat: create separate jobs on pool rotation for burst --- app/module/pool/service.py | 63 ++++++++++++++++++++++++++++++++------ app/module/raid/service.py | 2 +- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/app/module/pool/service.py b/app/module/pool/service.py index 9ceba8d..2a2291d 100644 --- a/app/module/pool/service.py +++ b/app/module/pool/service.py @@ -2,6 +2,7 @@ import datetime from collections import defaultdict +from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.interval import IntervalTrigger from sqlalchemy.ext.asyncio import AsyncSession @@ -112,15 +113,63 @@ async def compute_pool_consensus() -> int: return total +BOOST_INTERVAL = datetime.timedelta(minutes=2) +BOOST_LEAD = datetime.timedelta(minutes=30) # reset 前多久开始 +BOOST_TAIL = datetime.timedelta(minutes=30) # reset 后多久结束 + + +def _boost_job_id(pool_type: PoolType) -> str: + return f"compute_pool_consensus_boost:{pool_type.value}" + + +@SCHEDULER.scheduled_job( + CronTrigger(hour=0, minute=5), + id="schedule_pool_boosts", + misfire_grace_time=600, + coalesce=True, +) +async def schedule_pool_boosts() -> None: + for pool_type in PoolType: + await _schedule_boost_for_pool(pool_type) + + +async def _schedule_boost_for_pool(pool_type: PoolType) -> None: + config = POOL_REFRESH_CONFIG[pool_type] + now = datetime.datetime.now(datetime.UTC) + rotation = config.get_rotation(now) + + prev_tail_end = rotation.start + BOOST_TAIL + if now < prev_tail_end: + reset_at = rotation.start + else: + reset_at = rotation.end + + boost_start = reset_at - BOOST_LEAD + boost_end = reset_at + BOOST_TAIL + + SCHEDULER.add_job( + compute_pool_consensus_for_pool, + args=[pool_type], + trigger=IntervalTrigger( + minutes=int(BOOST_INTERVAL.total_seconds() // 60), + start_date=max(boost_start, now), + end_date=boost_end, + ), + id=_boost_job_id(pool_type), + replace_existing=True, + coalesce=True, + misfire_grace_time=60, + max_instances=1, + ) + + async def compute_pool_consensus_for_pool(pool_type: PoolType) -> int: async with get_session() as session: # Step 1: Fetch all active pools that need consensus computation poolRepo = PoolRepository(session) active_pools = await poolRepo.list_pools( pool_type=pool_type, - rotation_start=POOL_REFRESH_CONFIG[pool_type] - .get_rotation(datetime.datetime.now(tz=datetime.UTC)) - .start, + rotation_start=POOL_REFRESH_CONFIG[pool_type].get_rotation(datetime.datetime.now(tz=datetime.UTC)).start, needs_recalc=True, ) # The active pools should only differ in (region, page) @@ -166,9 +215,7 @@ async def compute_pool_consensus_for_pool(pool_type: PoolType) -> int: pool.consensus_data = consensus_items confidence = ( - sum(consensus_weights) / (highest_weight * len(consensus_weights)) - if consensus_weights - else 0.0 + sum(consensus_weights) / (highest_weight * len(consensus_weights)) if consensus_weights else 0.0 ) pool.confidence = round(confidence, 4) pool.needs_recalc = False @@ -191,9 +238,7 @@ async def get_pool_consensus( poolRepo = PoolRepository(session) - pool = await poolRepo.list_pools( - pool_type=pool_type, region=region, rotation_start=rotation_start, order_by="page" - ) + pool = await poolRepo.list_pools(pool_type=pool_type, region=region, rotation_start=rotation_start, order_by="page") if not pool: return {} diff --git a/app/module/raid/service.py b/app/module/raid/service.py index c38053b..b3df519 100644 --- a/app/module/raid/service.py +++ b/app/module/raid/service.py @@ -54,7 +54,7 @@ async def submit_gambit_data(session: AsyncSession, data: GambitSubmissionSchema @SCHEDULER.scheduled_job( - IntervalTrigger(minutes=20), + IntervalTrigger(minutes=5), id="compute_gambit_consensus", misfire_grace_time=60, coalesce=True,