From be0ef08f3d6375acbf43d41a1dbedf30b63957a2 Mon Sep 17 00:00:00 2001 From: codey Date: Sun, 19 Jan 2025 07:01:07 -0500 Subject: [PATCH] docstring stuff --- base.py | 17 ++++++------ endpoints/lyric_search.py | 21 ++++++++++----- lyric_search_new/constructors.py | 14 +++++----- lyric_search_new/sources/aggregate.py | 14 ++++++++-- lyric_search_new/sources/cache.py | 36 ++++++++++++++++--------- lyric_search_new/sources/genius.py | 8 ++++-- lyric_search_new/sources/lrclib.py | 8 ++++-- lyric_search_new/sources/redis_cache.py | 25 +++++++++++------ lyric_search_new/utils.py | 27 ++++++++++++++----- 9 files changed, 118 insertions(+), 52 deletions(-) diff --git a/base.py b/base.py index 7ea4b7e..dbc4a46 100644 --- a/base.py +++ b/base.py @@ -3,10 +3,10 @@ import importlib import logging import asyncio - from typing import Any from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from lyric_search_new.sources import redis_cache logger = logging.getLogger() @@ -87,13 +87,14 @@ xc_endpoints = importlib.import_module("endpoints.xc").XC(app, util, constants, # Below: Karma endpoint(s) karma_endpoints = importlib.import_module("endpoints.karma").Karma(app, util, constants, glob_state) -# @app.on_event("startup") -# @repeat_every(seconds=10) -# async def cah_tasks() -> None: -# return await cah_endpoints.periodicals() - +""" +End Actionable Routes +""" """ -End Actionable Routes -""" \ No newline at end of file +Startup +""" +redis_cache = redis_cache.RedisCache() +asyncio.get_event_loop().create_task( + redis_cache.create_index()) \ No newline at end of file diff --git a/endpoints/lyric_search.py b/endpoints/lyric_search.py index 870058e..1f06b00 100644 --- a/endpoints/lyric_search.py +++ b/endpoints/lyric_search.py @@ -6,6 +6,7 @@ import traceback import logging import os import urllib.parse +from typing import Optional import regex import aiohttp import aiosqlite as sqlite3 @@ -186,11 +187,22 @@ class LyricSearch(FastAPI): if data.src.upper() not in self.acceptable_request_sources: raise HTTPException(detail="Invalid request", status_code=500) + + search_artist: Optional[str] = data.a + search_song: Optional[str] = data.s + + + + if search_artist and search_song: + search_artist = self.constants.DOUBLE_SPACE_REGEX.sub(" ", search_artist.strip()) + search_song = self.constants.DOUBLE_SPACE_REGEX.sub(" ", search_song.strip()) + search_artist = urllib.parse.unquote(search_artist) + search_song = urllib.parse.unquote(search_song) excluded_sources = data.excluded_sources aggregate_search = aggregate.Aggregate(exclude_methods=excluded_sources) plain_lyrics = not data.lrc - result = await aggregate_search.search(data.a, data.s, plain_lyrics) + result = await aggregate_search.search(search_artist, search_song, plain_lyrics) if not result: return { @@ -201,6 +213,7 @@ class LyricSearch(FastAPI): result = result.todict() if data.sub and not data.lrc: + seeked_found_line = None lyric_lines = result['lyrics'].strip().split(" / ") for i, line in enumerate(lyric_lines): line = regex.sub(r'\u2064', '', line.strip()) @@ -223,12 +236,10 @@ class LyricSearch(FastAPI): result['lyrics'] = regex.sub(r'(\s/\s|\n)', '
', result['lyrics']).strip() else: # Swap lyrics key for 'lrc' - logging.info("lyrics: %s, type: %s", - result['lyrics'], type(result['lyrics'])) result['lrc'] = result['lyrics'] result.pop('lyrics') - if "cached" in result['src']: + if "cache" in result['src']: result['from_cache'] = True """ @@ -333,8 +344,6 @@ class LyricSearch(FastAPI): if not reg_helper: continue reg_helper = reg_helper[0] - logging.debug("Reg helper: %s for line: %s; len: %s", - reg_helper, line, len(reg_helper)) _timetag = reg_helper[0] if not reg_helper[1].strip(): _words = "♪" diff --git a/lyric_search_new/constructors.py b/lyric_search_new/constructors.py index 1e49ddf..1400ec0 100644 --- a/lyric_search_new/constructors.py +++ b/lyric_search_new/constructors.py @@ -4,12 +4,14 @@ from dataclasses import dataclass, asdict @dataclass class LyricsResult: - """Class for returned Lyrics Results - @artist: returned artist - @song: returned song - @src: source result was fetched from - @lyrics: str if plain lyrics, list for lrc - @time: time taken to retrieve lyrics from source + """ + Class for returned Lyrics Results + Attributes: + artist (str): returned artist + song (str): returned song + src (str): source result was fetched from + lyrics (str|list): str if plain lyrics, list for lrc + time (float): time taken to retrieve lyrics from source """ artist: str song: str diff --git a/lyric_search_new/sources/aggregate.py b/lyric_search_new/sources/aggregate.py index 2f2ec00..b6acee0 100644 --- a/lyric_search_new/sources/aggregate.py +++ b/lyric_search_new/sources/aggregate.py @@ -14,7 +14,9 @@ logger = logging.getLogger() logger.setLevel(logging.INFO) class Aggregate: - """Aggregate all source methods""" + """ + Aggregate all source methods + """ def __init__(self, exclude_methods=None): if not exclude_methods: @@ -22,7 +24,15 @@ class Aggregate: self.exclude_methods = exclude_methods async def search(self, artist: str, song: str, plain: bool = True) -> Optional[LyricsResult]: - """Aggregate Search""" + """ + Aggregate Search + Args: + artist (str): Artist to search + song (str): Song to search + plain (bool): Search for plain lyrics (lrc otherwise) + Returns: + LyricsResult|None: The result, if found - None otherwise. + """ if not plain: logging.info("LRCs requested, limiting search to LRCLib") self.exclude_methods = ["genius", "cache"] diff --git a/lyric_search_new/sources/cache.py b/lyric_search_new/sources/cache.py index c550107..021b8a6 100644 --- a/lyric_search_new/sources/cache.py +++ b/lyric_search_new/sources/cache.py @@ -7,7 +7,6 @@ import regex import logging import sys import traceback -import asyncio sys.path.insert(1,'..') sys.path.insert(1,'.') from typing import Optional, Any @@ -29,9 +28,6 @@ class Cache: "cached_lyrics.db") self.redis_cache = redis_cache.RedisCache() - asyncio.get_event_loop().create_task( - self.redis_cache.create_index()) - self.cache_pre_query: str = "pragma journal_mode = WAL; pragma synchronous = normal;\ pragma temp_store = memory; pragma mmap_size = 30000000000;" self.sqlite_exts: list[str] = ['/usr/local/lib/python3.11/dist-packages/spellfix1.cpython-311-x86_64-linux-gnu.so'] @@ -39,7 +35,16 @@ class Cache: def get_matched(self, matched_candidate: tuple, confidence: int, sqlite_rows: list[sqlite3.Row] = None, redis_results: Any = None) -> Optional[LyricsResult]: - """Get Matched Result""" + """ + Get Matched Result + Args: + matched_candidate (tuple): the correctly matched candidate returned by matcher.best_match + confidence (int): % confidence + sqlite_rows (list[sqlite3.Row]|None): List of returned rows from SQLite DB, or None if Redis + redis_results (Any): List of Redis returned data, or None if SQLite + Returns: + LyricsResult|None: The result, if found - None otherwise. + """ matched_id: int = matched_candidate[0] if redis_results: for res in redis_results: @@ -68,7 +73,10 @@ class Cache: async def check_existence(self, artistsong: str) -> Optional[bool]: """ Check whether lyrics are already stored for track - @artistsong: artist and song in artist\nsong format + Args: + artistsong (str): artist and song in artist\\nsong format + Returns: + bool: Whether track was found in cache """ logging.debug("Checking whether %s is already stored", artistsong.replace("\n", " - ")) @@ -88,8 +96,11 @@ class Cache: async def store(self, lyr_result: LyricsResult) -> None: """ - store - @lyr_result: the returned lyrics to cache + Store lyrics to cache + Args: + lyr_result: the returned lyrics to cache + Returns: + None """ logging.info("Storing %s", @@ -127,11 +138,12 @@ class Cache: # pylint: disable=unused-argument async def search(self, artist: str, song: str, **kwargs) -> Optional[LyricsResult]: """ - search - @artist: the artist to search - @song: the song to search + Cache Search + Args: + artist: the artist to search + song: the song to search Returns: - - LyricsResult corresponding to nearest match found (if found), **None** otherwise + LyricsResult|None: The result, if found - None otherwise. """ try: # pylint: enable=unused-argument diff --git a/lyric_search_new/sources/genius.py b/lyric_search_new/sources/genius.py index 78532df..a07ac39 100644 --- a/lyric_search_new/sources/genius.py +++ b/lyric_search_new/sources/genius.py @@ -40,8 +40,12 @@ class Genius: # pylint: disable=unused-argument async def search(self, artist: str, song: str, **kwargs) -> Optional[LyricsResult]: """ - @artist: the artist to search - @song: the song to search + Genius Search + Args: + artist (str): the artist to search + song (str): the song to search + Returns: + LyricsResult|None: The result, if found - None otherwise. """ try: # pylint: enable=unused-argument diff --git a/lyric_search_new/sources/lrclib.py b/lyric_search_new/sources/lrclib.py index 00f80d2..0d832dd 100644 --- a/lyric_search_new/sources/lrclib.py +++ b/lyric_search_new/sources/lrclib.py @@ -34,8 +34,12 @@ class LRCLib: async def search(self, artist: str, song: str, plain: bool = True) -> Optional[LyricsResult]: """ - @artist: the artist to search - @song: the song to search + LRCLib Search + Args: + artist (str): the artist to search + song (str): the song to search + Returns: + LyricsResult|None: The result, if found - None otherwise. """ try: artist: str = artist.strip().lower() diff --git a/lyric_search_new/sources/redis_cache.py b/lyric_search_new/sources/redis_cache.py index 8dd64ae..69d9263 100644 --- a/lyric_search_new/sources/redis_cache.py +++ b/lyric_search_new/sources/redis_cache.py @@ -1,10 +1,14 @@ #!/usr/bin/env python3.12 -# pylint: disable=bare-except, broad-exception-caught +# pylint: disable=bare-except, broad-exception-caught, wrong-import-order +# pylint: disable=wrong-import-position import logging import traceback import json +import sys +sys.path.insert(1,'..') +from lyric_search_new import notifier import redis.asyncio as redis from redis.commands.search.query import Query from redis.commands.search.indexDefinition import IndexDefinition, IndexType @@ -12,6 +16,7 @@ from redis.commands.search.field import TextField from . import private + logger = logging.getLogger() log_level = logging.getLevelName(logger.level) @@ -27,6 +32,8 @@ class RedisCache: def __init__(self): self.redis_client = redis.Redis(password=private.REDIS_PW) + self.notifier = notifier.DiscordNotifier() + self.notify_warnings = True async def create_index(self): """Create Index""" @@ -41,8 +48,8 @@ class RedisCache: schema, definition=IndexDefinition(prefix=["lyrics:"], index_type=IndexType.JSON)) if str(result) != "OK": raise RedisException(f"Redis: Failed to create index: {result}") - except: - pass + except Exception as e: + await self.notifier.send(f"ERROR @ {__file__}", f"Failed to create idx: {str(e)}") async def search(self, **kwargs): """Search Redis Cache @@ -62,13 +69,13 @@ class RedisCache: raise RedisException("Lyric search not yet implemented") if not is_random_search: - artist = artist.replace("-", "") - song = song.replace("-", "") + artist = artist.replace("-", " ") + song = song.replace("-", " ") search_res = await self.redis_client.ft().search( Query(f"@artist:{artist} @song:{song}" )) search_res_out = [(result['id'].split(":", - maxsplit=1)[1][:-1], dict(json.loads(result['json']))) + maxsplit=1)[1], dict(json.loads(result['json']))) for result in search_res.docs] else: random_redis_key = await self.redis_client.randomkey() @@ -77,8 +84,10 @@ class RedisCache: search_res = await self.redis_client.json().get(random_redis_key) search_res_out = [(out_id, search_res)] - + if not search_res_out and self.notify_warnings: + await self.notifier.send("WARNING", f"Redis cache miss for: \n## *{artist} - {song}*") return search_res_out - except: + except Exception as e: + await self.notifier.send(f"ERROR @ {__file__}", str(e)) traceback.print_exc() \ No newline at end of file diff --git a/lyric_search_new/utils.py b/lyric_search_new/utils.py index 9262f9e..37279d3 100644 --- a/lyric_search_new/utils.py +++ b/lyric_search_new/utils.py @@ -62,6 +62,10 @@ class TrackMatcher: """ Normalize string for comparison by removing special characters, extra spaces, and converting to lowercase. + Args: + text (str): The text to normalize + Returns: + str: Normalized text """ # Remove special characters and convert to lowercase text = regex.sub(r'[^\w\s-]', '', text).lower() @@ -72,6 +76,11 @@ class TrackMatcher: def _calculate_token_similarity(self, str1: str, str2: str) -> float: """ Calculate similarity based on matching tokens (words). + Args: + str1 (str): string 1 to compare + str2 (str): string 2 to compare + Returns: + float: The token similarity score """ tokens1 = set(str1.split()) tokens2 = set(str2.split()) @@ -94,8 +103,12 @@ class DataUtils: def scrub_lyrics(self, lyrics: str) -> str: - """Regex Chain - @lyrics: The lyrics (str) to scrub + """ + Lyric Scrub Regex Chain + Args: + lyrics (str): The lyrics to scrub + Returns: + str: Regex scrubbed lyrics """ lyrics = regex.sub(r'(\[.*?\])(\s){0,}(\:){0,1}', '', lyrics) lyrics = regex.sub(r'(\d?)(Embed\b)', '', lyrics, flags=regex.IGNORECASE) @@ -104,8 +117,12 @@ class DataUtils: return lyrics def create_lrc_object(self, lrc_str: str) -> list[dict]: - """Create LRC Object - @lrc_str: The raw LRCLib syncedLyrics (str) + """ + Create LRC Object + Args: + lrc_str (str): The raw LRCLib syncedLyrics + Returns: + list[dict]: LRC Object comprised of timestamps/lyrics """ lrc_out: list = [] for line in lrc_str.split("\n"): @@ -128,6 +145,4 @@ class DataUtils: "timeTag": _timetag, "words": _words, }) - logging.info("util: returning %s, type: %s", - lrc_out, type(lrc_out)) return lrc_out \ No newline at end of file