codey/api
1
0
Fork 0
Code Issues Pull Requests Activity

various/stale

Browse Source
This commit is contained in:
codey
2026-01-25 13:14:00 -05:00
parent 10ccf8c8eb
commit 97fd7dd67d
14 changed files with 501 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
utils/rip_background.py
View File

@@ -9,7 +9,6 @@ import subprocess
import shutil
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse, unquote
import aiohttp
from datetime import datetime, timezone
from mediafile import MediaFile, Image, ImageType # type: ignore[import]
@@ -20,9 +19,9 @@ import re
# ---------- Config ----------
ROOT_DIR = Path("/storage/music2")
MAX_RETRIES = 5
THROTTLE_MIN = 1.0
THROTTLE_MAX = 3.5
MAX_RETRIES = 4
THROTTLE_MIN = 0.0
THROTTLE_MAX = 0.0
DISCORD_WEBHOOK = os.getenv("TRIP_WEBHOOK_URI", "").strip()
HEADERS = {
@@ -36,10 +35,7 @@ HEADERS = {
"Connection": "keep-alive",
}
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
# Logging is configured in base.py - don't override here
load_dotenv()
@@ -288,8 +284,8 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
all_artists = set()
(ROOT_DIR / "completed").mkdir(parents=True, exist_ok=True)
# Ensure aiohttp session is properly closed
async with aiohttp.ClientSession(headers=HEADERS) as session:
session = aiohttp.ClientSession(headers=HEADERS)
try:
print(f"DEBUG: Starting process_tracks with {len(track_list)} tracks")
# Set up a one-time rate-limit callback to notify on the first 429 seen by SRUtil
@@ -314,13 +310,57 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
print(f"DEBUG: Processing track {i + 1}/{total}: {track_id}")
track_info = {
"track_id": str(track_id),
"title": None,
"artist": None,
"status": "Pending",
"file_path": None,
"filename": None,
"error": None,
"attempts": 0,
}
attempt = 0
# Fetch metadata FIRST to check if track is available before attempting download
md = None
try:
print(f"DEBUG: Fetching metadata for track {track_id}")
md = await sr.get_metadata_by_track_id(track_id) or {}
print(f"DEBUG: Metadata fetched: {bool(md)}")
# Check if track is streamable
if md and not md.get("streamable", True):
print(f"TRACK {track_id}: Not streamable, skipping")
track_info["status"] = "Failed"
track_info["error"] = "Track not streamable"
track_info["title"] = md.get("title") or f"Track {track_id}"
track_info["artist"] = md.get("artist") or "Unknown Artist"
per_track_meta.append(track_info)
if job:
job.meta["tracks"] = per_track_meta
job.meta["progress"] = int(((i + 1) / total) * 100)
job.save_meta()
continue # Skip to next track
except MetadataFetchError as me:
# Permanent metadata failure — mark failed and skip
print(f"TRACK {track_id}: Metadata fetch failed permanently: {me}")
track_info["status"] = "Failed"
track_info["error"] = str(me)
track_info["title"] = f"Track {track_id}"
track_info["artist"] = "Unknown Artist"
per_track_meta.append(track_info)
if job:
job.meta["tracks"] = per_track_meta
job.meta["progress"] = int(((i + 1) / total) * 100)
job.save_meta()
continue # Skip to next track
except Exception as meta_err:
# Non-permanent error - will retry during download attempts
print(
f"TRACK {track_id}: Metadata prefetch failed (will retry): {meta_err}"
)
md = None
while attempt < MAX_RETRIES:
tmp_file = None
attempt += 1
@@ -367,21 +407,13 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
f"Download completed but no file created: {tmp_file}"
)
print(f"DEBUG: Fetching metadata for track {track_id}")
# Metadata fetch
try:
md = await sr.get_metadata_by_track_id(track_id) or {}
print(f"DEBUG: Metadata fetched: {bool(md)}")
except MetadataFetchError as me:
# Permanent metadata failure — mark failed and break
track_info["status"] = "Failed"
track_info["error"] = str(me)
per_track_meta.append(track_info)
if job:
job.meta["tracks"] = per_track_meta
job.meta["progress"] = int(((i + 1) / total) * 100)
job.save_meta()
break
# If we didn't get metadata earlier, try again now
if not md:
print(f"DEBUG: Re-fetching metadata for track {track_id}")
try:
md = await sr.get_metadata_by_track_id(track_id) or {}
except Exception:
md = {}
artist_raw = md.get("artist") or "Unknown Artist"
album_raw = md.get("album") or "Unknown Album"
@@ -391,6 +423,10 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
album = sanitize_filename(album_raw)
title = sanitize_filename(title_raw)
# Populate track_info fields so job meta contains the user-visible data
track_info["title"] = title
track_info["artist"] = artist
print(f"TRACK {track_id}: Processing '{title}' by {artist}")
all_artists.add(artist)
@@ -400,7 +436,7 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
# Move to final location
print(f"TRACK {track_id}: Moving to final location...")
tmp_file.rename(final_file)
shutil.move(str(tmp_file), str(final_file))
print(f"TRACK {track_id}: File moved successfully")
# Fetch cover art
@@ -507,6 +543,10 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
tmp_file = None
track_info["status"] = "Success"
track_info["file_path"] = str(final_file)
try:
track_info["filename"] = final_file.name
except Exception:
track_info["filename"] = None
track_info["error"] = None
all_final_files.append(final_file)
@@ -514,6 +554,9 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
f"TRACK {track_id}: SUCCESS! Progress: {((i + 1) / total) * 100:.0f}%"
)
# Throttle after successful download to avoid hitting server too quickly
await asyncio.sleep(random.uniform(THROTTLE_MIN, THROTTLE_MAX))
if job:
job.meta["progress"] = int(((i + 1) / total) * 100)
job.meta["tracks"] = per_track_meta + [track_info]
@@ -523,9 +566,34 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
except aiohttp.ClientResponseError as e:
msg = f"Track {track_id} attempt {attempt} ClientResponseError: {e}"
send_log_to_discord(msg, "WARNING", target)
# If 429, backoff as before. If 5xx, recreate session and refresh Tidal client.
if getattr(e, "status", None) == 429:
wait_time = min(60, 2**attempt)
await asyncio.sleep(wait_time)
elif 500 <= getattr(e, "status", 0) < 600:
# Recreate local aiohttp session on 5xx errors
try:
await session.close()
except Exception:
pass
session = aiohttp.ClientSession(headers=HEADERS)
# Also force a fresh Tidal login in case the upstream session is stale
try:
await sr._force_fresh_login()
send_log_to_discord(
f"Refreshed Tidal session after 5xx error on track {track_id}",
"WARNING",
target,
)
except Exception as login_err:
send_log_to_discord(
f"Failed to refresh Tidal session: {login_err}",
"ERROR",
target,
)
await asyncio.sleep(
random.uniform(THROTTLE_MIN, THROTTLE_MAX)
)
else:
await asyncio.sleep(
random.uniform(THROTTLE_MIN, THROTTLE_MAX)
@@ -533,10 +601,74 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
except Exception as e:
tb = traceback.format_exc()
err_str = str(e).lower()
is_no_stream_url = (
isinstance(e, RuntimeError) and str(e) == "No stream URL"
)
if is_no_stream_url:
# Check if this is a 5xx error from the server (may appear in error message)
is_5xx_error = any(
code in err_str for code in ("500", "502", "503", "504")
)
# Check for permanent failures that should NOT be retried
is_not_found = any(
phrase in err_str
for phrase in (
"track not found",
"not found",
"404",
"does not exist",
"no longer available",
"asset is not ready",
)
)
if is_not_found:
# Permanent failure - do not retry
msg = (
f"Track {track_id} not found/unavailable, skipping: {e}"
)
print(msg)
send_log_to_discord(msg, "WARNING", target)
track_info["status"] = "Failed"
track_info["error"] = str(e)
break # Exit retry loop immediately
elif is_5xx_error:
msg = (
f"Track {track_id} attempt {attempt} server error: {e}"
)
send_log_to_discord(msg, "WARNING", target)
track_info["error"] = err_str
# Recreate local aiohttp session
try:
await session.close()
except Exception:
pass
session = aiohttp.ClientSession(headers=HEADERS)
# Force a fresh Tidal login
try:
await sr._force_fresh_login()
send_log_to_discord(
f"Refreshed Tidal session after 5xx error on track {track_id}",
"WARNING",
target,
)
except Exception as login_err:
send_log_to_discord(
f"Failed to refresh Tidal session: {login_err}",
"ERROR",
target,
)
if attempt >= MAX_RETRIES:
track_info["status"] = "Failed"
send_log_to_discord(
f"Track {track_id} failed after {attempt} attempts (5xx)",
"ERROR",
target,
)
await asyncio.sleep(
random.uniform(THROTTLE_MIN, THROTTLE_MAX)
)
elif is_no_stream_url:
if attempt == 1 or attempt == MAX_RETRIES:
msg = f"Track {track_id} attempt {attempt} failed: {e}\n{tb}"
send_log_to_discord(msg, "ERROR", target)
@@ -575,8 +707,22 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
except Exception:
pass
# Ensure placeholders and filename for the job metadata
track_info["title"] = track_info.get("title") or f"Track {track_id}"
track_info["artist"] = track_info.get("artist") or "Unknown Artist"
if track_info.get("file_path") and not track_info.get("filename"):
try:
track_info["filename"] = Path(track_info["file_path"]).name
except Exception:
track_info["filename"] = None
per_track_meta.append(track_info)
finally:
try:
await session.close()
except Exception:
pass
if not all_final_files:
if job:
job.meta["tarball"] = None
@@ -624,7 +770,7 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
counter += 1
staged_tarball = staging_root / f"{base_name} ({counter}).tar.gz"
final_dir = ROOT_DIR / "completed" / quality
final_dir = Path("/storage/music/TRIP")
final_dir.mkdir(parents=True, exist_ok=True)
# Ensure we don't overwrite an existing final tarball. Preserve `.tar.gz` style.
final_tarball = ensure_unique_filename_in_dir(final_dir, staged_tarball.name)
@@ -677,6 +823,14 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
os.remove(f)
except Exception:
pass
except Exception as e:
send_log_to_discord(f"Tar creation failed: {e}", "ERROR", target)
if job:
job.meta["status"] = "compress_failed"
job.save_meta()
# Do not proceed further if tarball creation failed
await asyncio.sleep(0.1)
return []
if not staged_tarball.exists():
send_log_to_discord(
@@ -711,6 +865,9 @@ def bulk_download(track_list: list, quality: str = "FLAC"):
color=0x00FF00,
)
# Always log the final tarball path for debugging
logging.info("Job %s finished, tarball: %s", job_id, final_tarball)
return [str(final_tarball)]
loop = asyncio.new_event_loop()