From 3b74333b96725a89adcbf4a39577828d51f3795f Mon Sep 17 00:00:00 2001 From: codey Date: Fri, 12 Sep 2025 22:39:59 -0400 Subject: [PATCH] misc --- .gitignore | 3 ++ lyric_search/sources/genius.py | 6 +-- utils/rip_background.py | 14 ++++--- utils/sr_wrapper.py | 75 ++++++++++++++++++++++++++-------- utils/test.conf | 35 ++++++++++++++++ 5 files changed, 108 insertions(+), 25 deletions(-) create mode 100644 utils/test.conf diff --git a/.gitignore b/.gitignore index de92564..911bc58 100644 --- a/.gitignore +++ b/.gitignore @@ -25,5 +25,8 @@ endpoints/auth.py endpoints/radio2 endpoints/radio2/** hash_password.py +up.py +job_review.py +check_missing.py **/auth/* .gitignore diff --git a/lyric_search/sources/genius.py b/lyric_search/sources/genius.py index b51768f..81391fc 100644 --- a/lyric_search/sources/genius.py +++ b/lyric_search/sources/genius.py @@ -45,8 +45,8 @@ class Genius: Optional[LyricsResult]: The result, if found - None otherwise. """ try: - artist: str = artist.strip().lower() - song: str = song.strip().lower() + artist = artist.strip().lower() + song = song.strip().lower() time_start: float = time.time() logging.info("Searching %s - %s on %s", artist, song, self.label) search_term: str = f"{artist}%20{song}" @@ -56,7 +56,6 @@ class Genius: f"{self.genius_search_url}{search_term}", timeout=self.timeout, headers=self.headers, - verify_ssl=False, proxy=private.GENIUS_PROXY, ) as request: request.raise_for_status() @@ -113,7 +112,6 @@ class Genius: scrape_url, timeout=self.timeout, headers=self.headers, - verify_ssl=False, proxy=private.GENIUS_PROXY, ) as scrape_request: scrape_request.raise_for_status() diff --git a/utils/rip_background.py b/utils/rip_background.py index d0c42c9..4321f5c 100644 --- a/utils/rip_background.py +++ b/utils/rip_background.py @@ -19,8 +19,8 @@ from utils.sr_wrapper import SRUtil # ---------- Config ---------- ROOT_DIR = Path("/storage/music2") MAX_RETRIES = 5 -THROTTLE_MIN = 1.7 -THROTTLE_MAX = 10.0 +THROTTLE_MIN = 1.0 +THROTTLE_MAX = 3.5 HEADERS = { "User-Agent": ( @@ -259,7 +259,6 @@ def bulk_download(track_list: list, quality: str = "FLAC"): await asyncio.sleep(random.uniform(THROTTLE_MIN, THROTTLE_MAX)) finally: try: - await session.close() if tmp_file and tmp_file.exists(): tmp_file.unlink() except Exception: @@ -304,8 +303,13 @@ def bulk_download(track_list: list, quality: str = "FLAC"): top_artist = "Unknown Artist" combined_artist = sanitize_filename(top_artist) - short_id = uuid.uuid4().hex[:8] - staged_tarball = staging_root / f"{combined_artist}_{short_id}.tar.gz" + staged_tarball = staging_root / f"{combined_artist}.tar.gz" + # Ensure uniqueness (Windows-style padding) within the parent folder + counter = 1 + base_name = staged_tarball.stem + while staged_tarball.exists(): + counter += 1 + staged_tarball = staging_root / f"{base_name} ({counter}).tar.gz" final_tarball = ROOT_DIR / "completed" / quality / staged_tarball.name final_tarball.parent.mkdir(parents=True, exist_ok=True) diff --git a/utils/sr_wrapper.py b/utils/sr_wrapper.py index 3bf2ce3..8054c14 100644 --- a/utils/sr_wrapper.py +++ b/utils/sr_wrapper.py @@ -3,9 +3,11 @@ from uuid import uuid4 from urllib.parse import urlparse import hashlib import logging +import random import asyncio import os import aiohttp +import time from streamrip.client import TidalClient # type: ignore from streamrip.config import Config as StreamripConfig # type: ignore from dotenv import load_dotenv @@ -44,9 +46,24 @@ class SRUtil: ) self.streamrip_config self.streamrip_client = TidalClient(self.streamrip_config) + self.MAX_CONCURRENT_METADATA_REQUESTS = 2 + self.METADATA_RATE_LIMIT = 1.25 + self.METADATA_SEMAPHORE = asyncio.Semaphore(self.MAX_CONCURRENT_METADATA_REQUESTS) + self.LAST_METADATA_REQUEST = 0 self.MAX_METADATA_RETRIES = 5 + self.METADATA_ALBUM_CACHE: dict[str, dict] = {} self.RETRY_DELAY = 1.0 # seconds between retries + async def rate_limited_request(self, func, *args, **kwargs): + async with self.METADATA_SEMAPHORE: + now = time.time() + elapsed = now - self.LAST_METADATA_REQUEST + if elapsed < self.METADATA_RATE_LIMIT: + await asyncio.sleep(self.METADATA_RATE_LIMIT - elapsed) + result = await func(*args, **kwargs) + self.last_request_time = time.time() + return result + def dedupe_by_key(self, key: str, entries: list[dict]) -> list[dict]: deduped = {} for entry in entries: @@ -62,12 +79,14 @@ class SRUtil: return f"{m}:{s:02}" def combine_album_track_metadata( - self, album_json: dict[str, Any], track_json: dict[str, Any] - ) -> dict[str, Any]: + self, album_json: dict | None, track_json: dict + ) -> dict: """ Combine album-level and track-level metadata into a unified tag dictionary. - If track_json comes from album_json['tracks'], it will override album-level values where relevant. + Track-level metadata overrides album-level where relevant. """ + album_json = album_json or {} + # Album-level combined = { "album": album_json.get("title"), @@ -99,17 +118,18 @@ class SRUtil: "peak": track_json.get("peak"), "lyrics": track_json.get("lyrics"), "track_copyright": track_json.get("copyright"), - "cover_id": track_json.get("album", {}).get( - "cover", album_json.get("cover") + "cover_id": track_json.get("album", {}).get("cover") or album_json.get("cover"), + "cover_url": ( + f"https://resources.tidal.com/images/{track_json.get('album', {}).get('cover', album_json.get('cover'))}/1280x1280.jpg" + if (track_json.get("album", {}).get("cover") or album_json.get("cover")) + else None ), - "cover_url": f"https://resources.tidal.com/images/{track_json.get('album', {}).get('cover', album_json.get('cover'))}/1280x1280.jpg" - if (track_json.get("album", {}).get("cover") or album_json.get("cover")) - else None, } ) return combined + def combine_album_with_all_tracks( self, album_json: dict[str, Any] ) -> list[dict[str, Any]]: @@ -282,22 +302,40 @@ class SRUtil: async def get_metadata_by_track_id(self, track_id: int) -> Optional[dict]: """ - Fetch track + album metadata with retries. + Fetch track + album metadata with retries, caching album data. Returns combined metadata dict or None after exhausting retries. """ for attempt in range(1, self.MAX_METADATA_RETRIES + 1): try: await self.streamrip_client.login() - metadata = await self.streamrip_client.get_metadata( - str(track_id), "track" + + # Track metadata + metadata = await self.rate_limited_request( + self.streamrip_client.get_metadata, str(track_id), "track" ) + album_id = metadata.get("album", {}).get("id") - album_metadata = await self.streamrip_client.get_metadata( - album_id, "album" - ) + album_metadata = None + + if album_id: + # Check cache first + if album_id in self.METADATA_ALBUM_CACHE: + album_metadata = self.METADATA_ALBUM_CACHE[album_id] + else: + album_metadata = await self.rate_limited_request( + self.streamrip_client.get_metadata, album_id, "album" + ) + if not album_metadata: + return None + self.METADATA_ALBUM_CACHE[album_id] = album_metadata + + # Combine track + album metadata + if not album_metadata: + return None combined_metadata: dict = self.combine_album_track_metadata( album_metadata, metadata ) + logging.info( "Combined metadata for track ID %s (attempt %d): %s", track_id, @@ -305,16 +343,20 @@ class SRUtil: combined_metadata, ) return combined_metadata + except Exception as e: + # Exponential backoff with jitter for 429 or other errors + delay = self.RETRY_DELAY * (2 ** (attempt - 1)) + random.uniform(0, 0.5) logging.warning( - "Metadata fetch failed for track %s (attempt %d/%d): %s", + "Metadata fetch failed for track %s (attempt %d/%d): %s. Retrying in %.2fs", track_id, attempt, self.MAX_METADATA_RETRIES, str(e), + delay, ) if attempt < self.MAX_METADATA_RETRIES: - await asyncio.sleep(self.RETRY_DELAY) + await asyncio.sleep(delay) else: logging.error( "Metadata fetch failed permanently for track %s after %d attempts", @@ -323,6 +365,7 @@ class SRUtil: ) return None + async def download(self, track_id: int, quality: str = "LOSSLESS") -> bool | str: """Download track Args: diff --git a/utils/test.conf b/utils/test.conf new file mode 100644 index 0000000..f7479c3 --- /dev/null +++ b/utils/test.conf @@ -0,0 +1,35 @@ +# ----------------------- +# /m/m2/ PHP handler +location ~ ^/m/m2/(.+\.php)$ { + alias /storage/music2/completed/; + include fastcgi_params; + fastcgi_pass unix:/run/php/php8.2-fpm.sock; + fastcgi_param SCRIPT_FILENAME /storage/music2/completed/$1; + fastcgi_param DOCUMENT_ROOT /storage/music2/completed; + fastcgi_param SCRIPT_NAME /m/m2/$1; +} + +# /m/m2/ static files +location /m/m2/ { + alias /storage/music2/completed/; + index index.php; + try_files $uri $uri/ /index.php$is_args$args; +} + +# ----------------------- +# /m/ PHP handler +location ~ ^/m/(.+\.php)$ { + root /var/www/codey.lol/new/public; + include fastcgi_params; + fastcgi_pass unix:/run/php/php8.2-fpm.sock; + fastcgi_param SCRIPT_FILENAME $document_root/$1; + fastcgi_param DOCUMENT_ROOT $document_root; + fastcgi_param SCRIPT_NAME /m/$1; +} + +# /m/ static files +location /m/ { + root /var/www/codey.lol/new/public; + index index.php; + try_files $uri $uri/ /m/index.php$is_args$args; +}