diff --git a/.env-sample b/.env-sample index d3f74701..b670719d 100644 --- a/.env-sample +++ b/.env-sample @@ -208,6 +208,12 @@ TORRENT_DISABLED_STREAM_NAME=[INFO] Comet # Stremio stream name shown when torre TORRENT_DISABLED_STREAM_DESCRIPTION=Direct torrent playback is disabled on this server. Please configure a debrid provider. # Description shown to users in Stremio TORRENT_DISABLED_STREAM_URL=https://comet.fast # Optional URL included in the placeholder stream response +# ============================== # +# Stream Result Limits # +# ============================== # +MAX_RESULTS_PER_RESOLUTION_CAP=0 # If >0, overrides client maxResultsPerResolution with this cap +MAX_STREAM_RESULTS_TOTAL=0 # If >0, hard limit on total streams returned per request + # ============================== # # Content Filtering # # ============================== # diff --git a/CHANGELOG.md b/CHANGELOG.md index 8853f0ae..86019070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ * add `GUNICORN_PRELOAD_APP` setting to control whether workers inherit a preloaded app or initialize independently * add `DATABASE_STARTUP_CLEANUP_INTERVAL` to throttle heavy startup cleanup sweeps across workers * add `DISABLE_TORRENT_STREAMS` toggle with customizable placeholder stream metadata +* expose Prometheus `/metrics` endpoint tracking stream requests per debrid service +* add server-side stream result caps via `MAX_RESULTS_PER_RESOLUTION_CAP` and `MAX_STREAM_RESULTS_TOTAL` ## [2.31.0](https://github.com/g0ldyy/comet/compare/v2.30.0...v2.31.0) (2025-12-08) diff --git a/PR_SUMMARY.md b/PR_SUMMARY.md new file mode 100644 index 00000000..4ccd1a96 --- /dev/null +++ b/PR_SUMMARY.md @@ -0,0 +1,8 @@ +# Branch Summary + +- Added replica-aware database routing (`comet/core/db_router.py`, `comet/core/database.py`, `comet/core/models.py`), letting PostgreSQL deployments list `DATABASE_READ_REPLICA_URLS` so reads can go to replicas while writes remain on the primary, plus a force-primary escape hatch inside maintenance queries. +- Reduced startup thrash by gating the heavy cleanup sweep behind `DATABASE_STARTUP_CLEANUP_INTERVAL` and persisting anime mapping data in `anime_mapping_cache` tables so most boots load instantly from the DB instead of redownloading (`comet/core/database.py`, `comet/services/anime.py`, env knobs `ANIME_MAPPING_SOURCE`/`ANIME_MAPPING_REFRESH_INTERVAL`). +- Hardened torrent ingestion and ranking: `TorrentManager` now funnels inserts through an `INSERT ... ON CONFLICT DO UPDATE` helper to quiet duplicate-key races, and ranking/logging paths gained small cleanups (`comet/services/torrent_manager.py`, `comet/services/lock.py`, logging tweaks in `comet/core/log_levels.py`). +- Added an env-toggle to skip Gunicorn preload (`GUNICORN_PRELOAD_APP`) so large deployments can trade startup cost vs. schema-read requirements (`comet/main.py`, `.env-sample`). +- Introduced `DISABLE_TORRENT_STREAMS` plus customizable placeholder metadata so operators can block magnet-only usage and show a friendly Stremio message instead (`comet/api/endpoints/stream.py`, `comet/core/models.py`, `.env-sample`). +- Updated documentation/config samples and changelog to capture the new settings and behavior (`.env-sample`, `CHANGELOG.md`). diff --git a/comet/api/app.py b/comet/api/app.py index f7ebd488..78d38022 100644 --- a/comet/api/app.py +++ b/comet/api/app.py @@ -10,6 +10,7 @@ from starlette.requests import Request from comet.api.endpoints import admin, base, config, manifest, playback +from comet.api.endpoints import metrics as metrics_router from comet.api.endpoints import stream as streams_router from comet.background_scraper.worker import background_scraper from comet.core.database import (cleanup_expired_locks, @@ -122,4 +123,5 @@ async def lifespan(app: FastAPI): app.include_router(manifest.router) app.include_router(admin.router) app.include_router(playback.router) +app.include_router(metrics_router.router) app.include_router(streams_router.streams) diff --git a/comet/api/endpoints/config.py b/comet/api/endpoints/config.py index 06b87ece..2727f113 100644 --- a/comet/api/endpoints/config.py +++ b/comet/api/endpoints/config.py @@ -19,5 +19,6 @@ async def configure(request: Request): else "", "webConfig": web_config, "proxyDebridStream": settings.PROXY_DEBRID_STREAM, + "disableTorrentStreams": settings.DISABLE_TORRENT_STREAMS, }, ) diff --git a/comet/api/endpoints/metrics.py b/comet/api/endpoints/metrics.py new file mode 100644 index 00000000..7f858a05 --- /dev/null +++ b/comet/api/endpoints/metrics.py @@ -0,0 +1,10 @@ +from fastapi import APIRouter + +from comet.core.metrics import prom_response + +router = APIRouter() + + +@router.get("/metrics") +async def metrics_endpoint(): + return prom_response() diff --git a/comet/api/endpoints/stream.py b/comet/api/endpoints/stream.py index 2a4e880f..6ab06d9a 100644 --- a/comet/api/endpoints/stream.py +++ b/comet/api/endpoints/stream.py @@ -7,6 +7,8 @@ from comet.core.config_validation import config_check from comet.core.logger import logger +from comet.core.metrics import (record_non_debrid_stream_request, + record_stream_request) from comet.core.models import database, settings, trackers from comet.debrid.manager import get_debrid_extension from comet.metadata.manager import MetadataScraper @@ -140,6 +142,21 @@ async def stream( ] } + per_resolution_cap = settings.MAX_RESULTS_PER_RESOLUTION_CAP or 0 + if per_resolution_cap > 0: + client_value = config.get("maxResultsPerResolution") or 0 + if client_value == 0 or client_value > per_resolution_cap: + logger.log( + "SCRAPER", + f"Clamping maxResultsPerResolution from {client_value} to {per_resolution_cap}", + ) + config["maxResultsPerResolution"] = per_resolution_cap + + if config.get("debridService") == "torrent": + record_non_debrid_stream_request() + + record_stream_request(config.get("debridService")) + if settings.DISABLE_TORRENT_STREAMS and config["debridService"] == "torrent": placeholder_stream = { "name": settings.TORRENT_DISABLED_STREAM_NAME or "[INFO] Comet", @@ -360,6 +377,25 @@ async def stream( await scrape_lock.release() lock_acquired = False + if debrid_service != "torrent": + is_valid_key = await debrid_service_instance.validate_credentials( + session, media_id, media_only_id + ) + if not is_valid_key: + logger.log( + "SCRAPER", + f"❌ Invalid API key for {debrid_service}; refusing to serve streams", + ) + return { + "streams": [ + { + "name": "[⚠️] Comet", + "description": f"Invalid or unauthorized API key for {debrid_service}. Please update your configuration.", + "url": "https://comet.fast", + } + ] + } + await debrid_service_instance.check_existing_availability( torrent_manager.torrents, season, episode ) @@ -432,7 +468,15 @@ async def stream( result_episode = episode if episode is not None else "n" torrents = torrent_manager.torrents + max_stream_results = settings.MAX_STREAM_RESULTS_TOTAL or 0 + streams_emitted = 0 for info_hash in torrent_manager.ranked_torrents: + if max_stream_results > 0 and streams_emitted >= max_stream_results: + logger.log( + "SCRAPER", + f"🔪 Truncated streams at {max_stream_results} entries (caps applied)", + ) + break torrent = torrents[info_hash] rtn_data = torrent["parsed"] @@ -480,4 +524,6 @@ async def stream( else: non_cached_results.append(the_stream) + streams_emitted += 1 + return {"streams": cached_results + non_cached_results} diff --git a/comet/core/database.py b/comet/core/database.py index 253e542c..144a907c 100644 --- a/comet/core/database.py +++ b/comet/core/database.py @@ -706,7 +706,6 @@ async def _run_startup_cleanup(): ON CONFLICT (id) DO UPDATE SET last_startup_cleanup = :timestamp """, {"timestamp": current_time}, - force_primary=True, ) finally: if lock_acquired: diff --git a/comet/core/metrics.py b/comet/core/metrics.py new file mode 100644 index 00000000..90c6fd59 --- /dev/null +++ b/comet/core/metrics.py @@ -0,0 +1,36 @@ +from fastapi import Response +from prometheus_client import (CONTENT_TYPE_LATEST, CollectorRegistry, Counter, + generate_latest) + +# Dedicated registry so we only expose Comet-specific metrics +_registry = CollectorRegistry() + +_stream_requests_total = Counter( + "comet_stream_requests_total", + "Total number of stream requests grouped by debrid service", + ["debrid_service"], + registry=_registry, +) + +_non_debrid_stream_requests_total = Counter( + "comet_stream_requests_non_debrid_total", + "Total number of stream requests that do not require a user API key", + registry=_registry, +) + + +def record_stream_request(debrid_service: str | None): + """Increment the stream request counter for the provided service.""" + label = (debrid_service or "unknown").lower() + _stream_requests_total.labels(debrid_service=label).inc() + + +def record_non_debrid_stream_request(): + """Increment the counter tracking torrent (non-API) stream requests.""" + _non_debrid_stream_requests_total.inc() + + +def prom_response() -> Response: + """Return a Response containing the current Prometheus metrics payload.""" + payload = generate_latest(_registry) + return Response(content=payload, media_type=CONTENT_TYPE_LATEST) diff --git a/comet/core/models.py b/comet/core/models.py index 4fd20a19..9158d99c 100644 --- a/comet/core/models.py +++ b/comet/core/models.py @@ -118,6 +118,8 @@ class AppSettings(BaseSettings): "Direct torrent playback is disabled on this server." ) TORRENT_DISABLED_STREAM_URL: Optional[str] = "https://comet.fast" + MAX_RESULTS_PER_RESOLUTION_CAP: Optional[int] = 0 + MAX_STREAM_RESULTS_TOTAL: Optional[int] = 0 REMOVE_ADULT_CONTENT: Optional[bool] = False BACKGROUND_SCRAPER_ENABLED: Optional[bool] = False BACKGROUND_SCRAPER_CONCURRENT_WORKERS: Optional[int] = 1 @@ -182,6 +184,20 @@ def normalize_urls(cls, v): return [url.rstrip("/") for url in v] return v + @field_validator( + "MAX_RESULTS_PER_RESOLUTION_CAP", + "MAX_STREAM_RESULTS_TOTAL", + mode="before", + ) + def non_negative_int(cls, v): + if v is None: + return 0 + try: + value = int(v) + except (TypeError, ValueError): + return 0 + return value if value >= 0 else 0 + def is_scraper_enabled(self, scraper_setting: Union[bool, str], context: str): if isinstance(scraper_setting, bool): return scraper_setting diff --git a/comet/metadata/manager.py b/comet/metadata/manager.py index 46c15a74..24b75c96 100644 --- a/comet/metadata/manager.py +++ b/comet/metadata/manager.py @@ -83,9 +83,15 @@ async def cache_metadata(self, media_id: str, metadata: dict, aliases: dict): ) def normalize_metadata(self, metadata: dict, season: int, episode: int): - title, year, year_end = metadata + if not metadata: + return None + + try: + title, year, year_end = metadata + except Exception: + return None - if title is None: # metadata retrieving failed + if title is None: return None return { diff --git a/comet/services/debrid.py b/comet/services/debrid.py index 7c048102..38d67f78 100644 --- a/comet/services/debrid.py +++ b/comet/services/debrid.py @@ -3,7 +3,8 @@ import orjson from RTN import ParsedData -from comet.debrid.manager import retrieve_debrid_availability +from comet.core.logger import logger +from comet.debrid.manager import get_debrid, retrieve_debrid_availability from comet.services.debrid_cache import (cache_availability, get_cached_availability) @@ -14,6 +15,30 @@ def __init__(self, debrid_service: str, debrid_api_key: str, ip: str): self.debrid_api_key = debrid_api_key self.ip = ip + async def validate_credentials( + self, session, media_id: str, media_only_id: str + ) -> bool: + if self.debrid_service == "torrent": + return True + + try: + client = get_debrid( + session, + media_id, + media_only_id, + self.debrid_service, + self.debrid_api_key, + self.ip, + ) + if client is None: + return False + return await client.check_premium() + except Exception as e: + logger.warning( + f"Failed to validate credentials for {self.debrid_service}: {e}" + ) + return False + async def get_and_cache_availability( self, session, diff --git a/comet/services/torrent_manager.py b/comet/services/torrent_manager.py index c1809735..638e8645 100644 --- a/comet/services/torrent_manager.py +++ b/comet/services/torrent_manager.py @@ -465,11 +465,23 @@ async def _check_batch_size(self): timestamp = EXCLUDED.timestamp """ -POSTGRES_CONFLICT_TARGETS = { - "series": "(media_id, info_hash, season, episode) WHERE season IS NOT NULL AND episode IS NOT NULL", - "season_only": "(media_id, info_hash, season) WHERE season IS NOT NULL AND episode IS NULL", - "episode_only": "(media_id, info_hash, episode) WHERE season IS NULL AND episode IS NOT NULL", - "none": "(media_id, info_hash) WHERE season IS NULL AND episode IS NULL", +POSTGRES_CONFLICT_MAP = { + "series": { + "target": "(media_id, info_hash, season, episode)", + "predicate": "WHERE season IS NOT NULL AND episode IS NOT NULL", + }, + "season_only": { + "target": "(media_id, info_hash, season)", + "predicate": "WHERE season IS NOT NULL AND episode IS NULL", + }, + "episode_only": { + "target": "(media_id, info_hash, episode)", + "predicate": "WHERE season IS NULL AND episode IS NOT NULL", + }, + "none": { + "target": "(media_id, info_hash)", + "predicate": "WHERE season IS NULL AND episode IS NULL", + }, } _POSTGRES_UPSERT_CACHE: dict[str, str] = {} @@ -490,11 +502,11 @@ def _get_torrent_upsert_query(conflict_key: str) -> str: return SQLITE_UPSERT_QUERY if settings.DATABASE_TYPE == "postgresql": - target = POSTGRES_CONFLICT_TARGETS[conflict_key] + conflict = POSTGRES_CONFLICT_MAP[conflict_key] if conflict_key not in _POSTGRES_UPSERT_CACHE: _POSTGRES_UPSERT_CACHE[conflict_key] = ( TORRENT_INSERT_TEMPLATE - + f" ON CONFLICT {target} " + + f" ON CONFLICT {conflict['target']} {conflict['predicate']} " + POSTGRES_UPDATE_SET ) return _POSTGRES_UPSERT_CACHE[conflict_key] diff --git a/comet/templates/index.html b/comet/templates/index.html index 0dee2263..829887b5 100644 --- a/comet/templates/index.html +++ b/comet/templates/index.html @@ -410,8 +410,10 @@
- + + {% if not disableTorrentStreams %} Torrent + {% endif %} TorBox Debrider EasyDebrid diff --git a/comet/utils/parsing.py b/comet/utils/parsing.py index a7020b45..fc7dc576 100644 --- a/comet/utils/parsing.py +++ b/comet/utils/parsing.py @@ -49,18 +49,31 @@ def default_dump(obj): return obj.model_dump() +def _safe_int(value, fallback=None): + try: + if value is None: + return fallback + text = str(value).strip() + if text == "": + return fallback + return int(text) + except (TypeError, ValueError): + return fallback + + def parse_media_id(media_type: str, media_id: str): if "kitsu" in media_id: info = media_id.split(":") - - if len(info) > 2: - return info[1], 1, int(info[2]) - else: - return info[1], 1, None + identifier = info[1] if len(info) > 1 else media_id + episode = _safe_int(info[2] if len(info) > 2 else None) + return identifier, 1, episode if media_type == "series": info = media_id.split(":") - return info[0], int(info[1]), int(info[2]) + identifier = info[0] + season = _safe_int(info[1] if len(info) > 1 else None, fallback=1) + episode = _safe_int(info[2] if len(info) > 2 else None) + return identifier, season, episode return media_id, None, None diff --git a/deployment/grafana/comet-stream-requests.json b/deployment/grafana/comet-stream-requests.json new file mode 100644 index 00000000..904cc6d9 --- /dev/null +++ b/deployment/grafana/comet-stream-requests.json @@ -0,0 +1,169 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(comet_stream_requests_total[5m])) by (debrid_service)", + "legendFormat": "{{debrid_service}}", + "range": true, + "refId": "A" + } + ], + "title": "Stream Requests per Debrid Service (rate)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(increase(comet_stream_requests_total[1h])) by (debrid_service)", + "format": "time_series", + "legendFormat": "{{debrid_service}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests per Debrid Service (last 1h)", + "type": "bargauge" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "comet", + "streams" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Comet Stream Requests", + "uid": "comet-stream-requests", + "version": 1, + "weekStart": "" +} diff --git a/pyproject.toml b/pyproject.toml index 2443313a..4f9ab896 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,4 +25,5 @@ dependencies = [ "python-multipart", "rank-torrent-name", "uvicorn", + "prometheus-client", ]