diff --git a/.config b/.config index ae0e1e2..3f7fff8 100644 --- a/.config +++ b/.config @@ -8,4 +8,10 @@ MAX_PROPOSED_LEN=10 # image PERMITTED_IMAGE_EXTS="jpeg,jpg,png" -IMAGE_SIZE_STUB=100 \ No newline at end of file +IMAGE_SIZE_STUB=100 + +# Cache configuration + +#redis + +REDIS_PASSWORD="REDIS_PASSWORD" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7d4eb98..cbdef97 100644 --- a/.gitignore +++ b/.gitignore @@ -196,6 +196,9 @@ cython_debug/ .idea/ .vscode/ + +# Dev + Makefile project_dump.txt @@ -208,7 +211,11 @@ project_dump.txt cache/data -tags.txt +# MacOS + +.DS_Store + + diff --git a/audiotagloader/app.py b/audiotagloader/app.py index 641b877..db245f0 100644 --- a/audiotagloader/app.py +++ b/audiotagloader/app.py @@ -1,4 +1,3 @@ -from pathlib import Path import discogs_client # type: ignore from .config import DISCOGS_TOKEN, MAX_PROPOSED_LEN, IMAGE_SIZE_STUB @@ -9,11 +8,14 @@ from .output import track_tags_to_output +from .cache import cache + class App: def __init__(self): self._client = discogs_client.Client("Fetcher/1.0", user_token=DISCOGS_TOKEN) + @cache def _get_artists_by_name(self, name: str) -> list[Artist]: res = [] @@ -25,6 +27,7 @@ def _get_artists_by_name(self, name: str) -> list[Artist]: return res + @cache def _get_albums_by_artist(self, artist: Artist) -> list[Album]: releases = self._client.search( type="master", format="album", artist=artist.name @@ -45,13 +48,14 @@ def _get_albums_by_artist(self, artist: Artist) -> list[Album]: year=master.data.get("year", 0), genres=master.data.get("genre", None), styles=master.data.get("style", None), - thumb=Path(master.data.get("thumb", "")), + thumb=master.data.get("thumb", ""), artist=artist.name, ), ) return target_albums + @cache def _get_cover_image(self, album_id: int) -> Image: master = self._client.master(album_id) @@ -60,14 +64,14 @@ def _get_cover_image(self, album_id: int) -> Image: for image in images: if image.get("type", "") == "primary": return Image( - url=Path(image.get("resource_url", "")), # type: ignore + url=image.get("resource_url", ""), width=int(image.get("width", IMAGE_SIZE_STUB)), height=int(image.get("height", IMAGE_SIZE_STUB)), ) if len(images) > 0: return Image( - url=Path(images[0].get("resource_url", "")), # type: ignore + url=images[0].get("resource_url", ""), width=int(images[0].get("width", IMAGE_SIZE_STUB)), height=int(images[0].get("height", IMAGE_SIZE_STUB)), ) @@ -75,6 +79,7 @@ def _get_cover_image(self, album_id: int) -> Image: return Image() return Image() + @cache def _get_tracklist(self, album_id: int) -> Tracklist: master = self._client.master(album_id) diff --git a/audiotagloader/cache.py b/audiotagloader/cache.py new file mode 100644 index 0000000..a488114 --- /dev/null +++ b/audiotagloader/cache.py @@ -0,0 +1,69 @@ +import hashlib +import json +from typing import Any, Callable, get_args +from functools import wraps + +from pydantic import BaseModel + +from .redis import redis_client + + +def hash_key(func_name: str, args: tuple, kwargs: dict) -> str: + key_dump = { + "func": func_name, + "args": args[1:], + "kwargs": kwargs, + } + + print(json.dumps(key_dump, indent=4, sort_keys=True, default=str)) + + key_string = json.dumps(key_dump, sort_keys=True, default=str) + key_hash = hashlib.md5(key_string.encode()).hexdigest() + + return f"cache:{func_name}:{key_hash}" + + +def cache(func: Callable) -> Callable: + @wraps(func) + def wrapper(*args, **kwargs) -> Any: + return_type: type = func.__annotations__.get("return") # type: ignore + + key = hash_key(func.__name__, args, kwargs) + print("HASH KEY:", key) + + try: + data = redis_client.get(key) + + if data is not None and issubclass(return_type, BaseModel): + return return_type.model_validate_json(data.decode()) + elif data is not None: + arg_type = get_args(return_type) + current_type = arg_type[0] + print("load from redis") + return [ + current_type.model_validate(item) + for item in json.loads(data.decode()) + ] + except Exception as e: + raise e + + res = func(*args, **kwargs) + print("load from api") + + try: + serialized = "null" + if res is not None and issubclass(return_type, BaseModel): + serialized = res.model_dump_json() + elif res is not None: + arg_type = get_args(return_type) + current_type = arg_type[0] + serialized = json.dumps([item.model_dump() for item in res]) + + redis_client.set(key, serialized.encode()) + print("dump to redis") + except Exception as e: + raise e + + return res + + return wrapper diff --git a/audiotagloader/config.py b/audiotagloader/config.py index 0202b46..4b8284f 100644 --- a/audiotagloader/config.py +++ b/audiotagloader/config.py @@ -9,3 +9,5 @@ PERMITTED_IMAGE_EXTS = os.environ["PERMITTED_IMAGE_EXTS"].split(",") IMAGE_SIZE_STUB = int(os.environ["IMAGE_SIZE_STUB"]) + +REDIS_PASSWORD = os.environ["REDIS_PASSWORD"] diff --git a/audiotagloader/models.py b/audiotagloader/models.py index fbc3133..060809b 100644 --- a/audiotagloader/models.py +++ b/audiotagloader/models.py @@ -1,5 +1,4 @@ from __future__ import annotations -from pathlib import Path from pydantic import BaseModel, Field, field_validator from .config import PERMITTED_IMAGE_EXTS, IMAGE_SIZE_STUB @@ -30,7 +29,7 @@ class Album(BaseModel): year: int = Field(default=0) genres: list[str] = Field(default_factory=lambda: list()) styles: list[str] = Field(default_factory=lambda: list()) - thumb: Path = Field(default_factory=lambda: Path()) + thumb: str = Field(default="") artist: str = Field(default="") @field_validator("genres", "styles", mode="before") @@ -43,16 +42,14 @@ def normlize_list(cls, value: list[str] | None) -> list[str]: class Image(BaseModel): - url: Path = Field(default_factory=lambda: Path()) + url: str = Field(default="") width: int = Field(default=IMAGE_SIZE_STUB) height: int = Field(default=IMAGE_SIZE_STUB) @field_validator("url", mode="before") @classmethod - def check_extension(cls, url: Path) -> Path: - if url and not any( - str(url).lower().endswith(ext) for ext in PERMITTED_IMAGE_EXTS - ): + def check_extension(cls, url: str) -> str: + if url and not any(url.lower().endswith(ext) for ext in PERMITTED_IMAGE_EXTS): raise ValueError(f"image url extension sould be in: {PERMITTED_IMAGE_EXTS}") return url diff --git a/audiotagloader/redis.py b/audiotagloader/redis.py new file mode 100644 index 0000000..320a6ff --- /dev/null +++ b/audiotagloader/redis.py @@ -0,0 +1,31 @@ +import redis +from .config import REDIS_PASSWORD + + +class RedisClient: + def __init__( + self, + host: str = "127.0.0.1", + port: int = 6379, + password: str = REDIS_PASSWORD, + decode_responses: bool = False, + ): + self._client = redis.Redis( + host=host, + port=port, + password=password, + decode_responses=decode_responses, + db=0, + socket_connect_timeout=5, + ) + + print("REDIS PING:", self._client.ping()) + + def get(self, key: str) -> bytes | None: + return self._client.get(key) # type: ignore + + def set(self, key: str, value: bytes) -> bool: + return self._client.set(key, value) # type: ignore + + +redis_client = RedisClient() diff --git a/cache/redis.conf b/cache/redis.conf index a48fdfc..906cfb7 100644 --- a/cache/redis.conf +++ b/cache/redis.conf @@ -1,6 +1,5 @@ -bind 127.0.0.1 +bind 0.0.0.0 port 6379 -protected-mode yes save 60 1 diff --git a/poetry.lock b/poetry.lock index 49c173f..666ccb9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -546,6 +546,26 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "redis" +version = "7.2.0" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "redis-7.2.0-py3-none-any.whl", hash = "sha256:01f591f8598e483f1842d429e8ae3a820804566f1c73dca1b80e23af9fba0497"}, + {file = "redis-7.2.0.tar.gz", hash = "sha256:4dd5bf4bd4ae80510267f14185a15cba2a38666b941aff68cccf0256b51c1f26"}, +] + +[package.extras] +circuit-breaker = ["pybreaker (>=1.4.0)"] +hiredis = ["hiredis (>=3.2.0)"] +jwt = ["pyjwt (>=2.9.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (>=20.0.1)", "requests (>=2.31.0)"] +otel = ["opentelemetry-api (>=1.39.1)", "opentelemetry-exporter-otlp-proto-http (>=1.39.1)", "opentelemetry-sdk (>=1.39.1)"] +xxhash = ["xxhash (>=3.6.0,<3.7.0)"] + [[package]] name = "requests" version = "2.32.5" @@ -656,4 +676,4 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "564f60fbfa0fd3e9ebddef4dad2c81f93ee8eaa6b875a26c886653433d226829" +content-hash = "f98b0ef0aca889731e1a35ca5fbdd1d701d928e3193be0a47f67c8ab4b726ceb" diff --git a/pyproject.toml b/pyproject.toml index b249ace..b2dda27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "discogs-client (>=2.3.0,<3.0.0)", "python-dotenv (>=1.2.1,<2.0.0)", "pydantic (>=2.12.5,<3.0.0)", + "redis (>=7.2.0,<8.0.0)", ] [tool.poetry]