diff --git a/endpoints/rip.py b/endpoints/rip.py index eed1d47..5e358b3 100644 --- a/endpoints/rip.py +++ b/endpoints/rip.py @@ -11,6 +11,7 @@ from utils.rip_background import bulk_download from lyric_search.sources import private from pydantic import BaseModel + class ValidBulkFetchRequest(BaseModel): track_ids: list[int] @@ -119,7 +120,13 @@ class RIP(FastAPI): } ) track_ids = data.track_ids - job = self.task_queue.enqueue(bulk_download, track_ids) + job = self.task_queue.enqueue( + bulk_download, + args=(track_ids,), + timeout=3600, + failure_ttl=86400, + result_ttl=86400, + ) self.redis_conn.lpush("enqueued_job_ids", job.id) return JSONResponse( content={ @@ -150,7 +157,7 @@ class RIP(FastAPI): """List all jobs in the queue (queued + finished, if result_ttl allows)""" jobs_info = [] - # 1️⃣ Jobs still in the queue (pending) + # Jobs still in the queue (pending) for job in self.task_queue.jobs: jobs_info.append( { @@ -162,7 +169,7 @@ class RIP(FastAPI): } ) - # 2️⃣ Started/running jobs tracked via enqueued_job_ids + # Started/running jobs tracked via enqueued_job_ids job_ids = self.redis_conn.lrange("enqueued_job_ids", 0, -1) for jid_bytes in job_ids: # type: ignore jid = jid_bytes.decode() diff --git a/utils/rip_background.py b/utils/rip_background.py index dd266c7..9da15ed 100644 --- a/utils/rip_background.py +++ b/utils/rip_background.py @@ -73,7 +73,7 @@ def bulk_download(track_list: list): while attempt < MAX_RETRIES: attempt += 1 try: - # 1️⃣ Get track URL + # Get track URL url = await sr.get_stream_url_by_track_id(track_id) if not url: logging.critical( @@ -84,7 +84,7 @@ def bulk_download(track_list: list): ) continue - # 2️⃣ Download file (chunked) + # Download file (chunked) parsed = urlparse(url) ext = Path(unquote(parsed.path)).suffix or ".mp3" tmp_file = Path(f"/tmp/{track_id}{ext}") @@ -95,7 +95,7 @@ def bulk_download(track_list: list): async for chunk in resp.content.iter_chunked(64 * 1024): f.write(chunk) - # 3️⃣ Extract metadata + # Extract metadata metadata = await sr.get_metadata_by_track_id(track_id) if not metadata: logging.critical( @@ -109,13 +109,13 @@ def bulk_download(track_list: list): logging.critical("Got metadata: %s/%s/%s", artist, album, title) - # 4️⃣ Organize path + # Organize path final_dir = ROOT_DIR / artist / album final_dir.mkdir(parents=True, exist_ok=True) final_file = final_dir / f"{title}{ext}" tmp_file.rename(final_file) - # 5️⃣ Track per-track info + # Track per-track info track_info.update( {"status": "success", "file_path": str(final_file)} ) @@ -134,17 +134,17 @@ def bulk_download(track_list: list): random.uniform(THROTTLE_MIN, THROTTLE_MAX) ) - # 6️⃣ Update RQ job meta + # Update RQ job meta per_track_meta.append(track_info) if job: job.meta["progress"] = int((i + 1) / total * 100) job.meta["tracks"] = per_track_meta job.save_meta() - # 7️⃣ Throttle between downloads + # Throttle between downloads await asyncio.sleep(random.uniform(THROTTLE_MIN, THROTTLE_MAX)) - # 8️⃣ Create per-artist tarballs + # Create per-artist tarballs tarballs = [] for artist, files in artist_files.items(): short_id = uuid.uuid4().hex[:8]