codey/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup

Browse Source
This commit is contained in:
codey
2025-02-15 21:09:33 -05:00
parent 60416c493f
commit 39d1ddaffa
22 changed files with 509 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
lyric_search/constructors.py
View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3.12
from dataclasses import dataclass, asdict
from dataclasses import dataclass
from typing import Union
@dataclass
class LyricsResult:
@ -10,12 +11,12 @@ class LyricsResult:
artist (str): returned artist
song (str): returned song
src (str): source result was fetched from
lyrics (str|list): str if plain lyrics, list for lrc
lyrics (Union[str, list]): str if plain lyrics, list for lrc
time (float): time taken to retrieve lyrics from source
"""
artist: str
song: str
src: str
lyrics: str|list
lyrics: Union[str, list]
confidence: int
time: float = 0.00

6
lyric_search/sources/aggregate.py
View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3.12
# pylint: disable=wrong-import-order, wrong-import-position
from typing import Optional
from lyric_search.constructors import LyricsResult
@ -22,7 +21,8 @@ class Aggregate:
self.redis_cache = redis_cache.RedisCache()
self.notifier = notifier.DiscordNotifier()
async def search(self, artist: str, song: str, plain: bool = True) -> Optional[LyricsResult]:
async def search(self, artist: str, song: str,
plain: Optional[bool] = True) -> Optional[LyricsResult]:
"""
Aggregate Search
Args:
@ -30,7 +30,7 @@ class Aggregate:
song (str): Song to search
plain (bool): Search for plain lyrics (lrc otherwise)
Returns:
LyricsResult|None: The result, if found - None otherwise.
Optional[LyricsResult]: The result, if found - None otherwise.
"""
if not plain:
logging.info("LRCs requested, limiting search to LRCLib")

25
lyric_search/sources/cache.py
View File

@ -9,7 +9,7 @@ import sys
import traceback
sys.path.insert(1,'..')
sys.path.insert(1,'.')
from typing import Optional, Any
from typing import Optional, Union, LiteralString
import aiosqlite as sqlite3
from . import redis_cache
from lyric_search import utils, notifier
@ -23,7 +23,7 @@ log_level = logging.getLevelName(logger.level)
class Cache:
"""Cache Search Module"""
def __init__(self) -> None:
self.cache_db: str = os.path.join("/", "usr", "local", "share",
self.cache_db: Union[str, LiteralString] = os.path.join("/", "usr", "local", "share",
"sqlite_dbs", "cached_lyrics.db")
self.redis_cache = redis_cache.RedisCache()
self.notifier = notifier.DiscordNotifier()
@ -34,16 +34,17 @@ class Cache:
self.label: str = "Cache"
def get_matched(self, matched_candidate: tuple, confidence: int,
sqlite_rows: list[sqlite3.Row] = None, redis_results: Any = None) -> Optional[LyricsResult]:
sqlite_rows: Optional[list[sqlite3.Row]] = None,
redis_results: Optional[list] = None) -> Optional[LyricsResult]:
"""
Get Matched Result
Args:
matched_candidate (tuple): the correctly matched candidate returned by matcher.best_match
confidence (int): % confidence
sqlite_rows (list[sqlite3.Row]|None): List of returned rows from SQLite DB, or None if Redis
sqlite_rows (Optional[list[sqlite3.Row]]): List of returned rows from SQLite DB, or None if Redis
redis_results (Any): List of Redis returned data, or None if SQLite
Returns:
LyricsResult|None: The result, if found - None otherwise.
Optional[LyricsResult]: The result, if found - None otherwise.
"""
matched_id: int = matched_candidate[0]
if redis_results:
@ -60,7 +61,7 @@ class Cache:
else:
for row in sqlite_rows:
if row[0] == matched_id:
(_id, artist, song, lyrics, original_src, _confidence) = row
(_id, artist, song, lyrics, original_src) = row[:-1]
return LyricsResult(
artist=artist,
song=song,
@ -119,7 +120,8 @@ class Cache:
await self.notifier.send(f"ERROR @ {__file__.rsplit("/", maxsplit=1)[-1]}",
f"cache::store >> {str(e)}")
async def sqlite_rowcount(self, where: Optional[str] = None, params: Optional[tuple] = None) -> int:
async def sqlite_rowcount(self, where: Optional[str] = None,
params: Optional[tuple] = None) -> int:
"""
Get rowcount for cached_lyrics DB
Args:
@ -217,7 +219,7 @@ class Cache:
artist: the artist to search
song: the song to search
Returns:
LyricsResult|None: The result, if found - None otherwise.
Optional[LyricsResult]: The result, if found - None otherwise.
"""
try:
# pylint: enable=unused-argument
@ -253,7 +255,7 @@ class Cache:
result_tracks.append((key, f"{track['artist']} - {track['song']}"))
if not random_search:
best_match: tuple|None = matcher.find_best_match(input_track=input_track,
best_match: Optional[tuple] = matcher.find_best_match(input_track=input_track,
candidate_tracks=result_tracks)
else:
best_match = (result_tracks[0], 100)
@ -298,7 +300,7 @@ class Cache:
(_id, _artist, _song, _lyrics, _src, _confidence) = track
result_tracks.append((_id, f"{_artist} - {_song}"))
if not random_search:
best_match: tuple|None = matcher.find_best_match(input_track=input_track,
best_match: Optional[tuple] = matcher.find_best_match(input_track=input_track,
candidate_tracks=result_tracks)
else:
best_match = (result_tracks[0], 100)
@ -315,5 +317,4 @@ class Cache:
await self.redis_cache.increment_found_count(self.label)
return matched
except:
traceback.print_exc()
return
traceback.print_exc()

26
lyric_search/sources/genius.py
View File

@ -23,7 +23,9 @@ class InvalidResponseException(Exception):
"""
class Genius:
"""Genius Search Module"""
"""
Genius Search Module
"""
def __init__(self) -> None:
self.label: str = "Genius"
self.genius_url: str = private.GENIUS_URL
@ -36,14 +38,15 @@ class Genius:
self.redis_cache = redis_cache.RedisCache()
# pylint: disable=unused-argument
async def search(self, artist: str, song: str, **kwargs) -> Optional[LyricsResult]:
async def search(self, artist: str, song: str,
**kwargs) -> Optional[LyricsResult]:
"""
Genius Search
Args:
artist (str): the artist to search
song (str): the song to search
Returns:
LyricsResult|None: The result, if found - None otherwise.
Optional[LyricsResult]: The result, if found - None otherwise.
"""
try:
# pylint: enable=unused-argument
@ -59,7 +62,10 @@ class Genius:
timeout=self.timeout,
headers=self.headers) as request:
request.raise_for_status()
text: str|None = await request.text()
text: Optional[str] = await request.text()
if not text:
raise InvalidResponseException("No search response.")
if len(text) < 100:
raise InvalidResponseException("Search response text was invalid (len < 100 chars.)")
@ -94,14 +100,17 @@ class Genius:
timeout=self.timeout,
headers=self.headers) as scrape_request:
scrape_request.raise_for_status()
scrape_text: str|None = await scrape_request.text()
scrape_text: Optional[str] = await scrape_request.text()
if not scrape_text:
raise InvalidResponseException("No scrape response.")
if len(scrape_text) < 100:
raise InvalidResponseException("Scrape response was invalid (len < 100 chars.)")
html = BeautifulSoup(htm.unescape(scrape_text).replace('<br/>', '\n'), "html.parser")
divs: ResultSet|None = html.find_all("div", {"data-lyrics-container": "true"})
divs: Optional[ResultSet] = html.find_all("div", {"data-lyrics-container": "true"})
if not divs:
return
@ -124,8 +133,5 @@ class Genius:
await self.redis_cache.increment_found_count(self.label)
await self.cache.store(matched)
return matched
except:
# if log_level == "DEBUG":
traceback.print_exc()
return
traceback.print_exc()

18
lyric_search/sources/lrclib.py
View File

@ -32,14 +32,15 @@ class LRCLib:
self.cache = cache.Cache()
self.redis_cache = redis_cache.RedisCache()
async def search(self, artist: str, song: str, plain: bool = True) -> Optional[LyricsResult]:
async def search(self, artist: str, song: str,
plain: Optional[bool] = True) -> Optional[LyricsResult]:
"""
LRCLib Search
Args:
artist (str): the artist to search
song (str): the song to search
Returns:
LyricsResult|None: The result, if found - None otherwise.
Optional[LyricsResult]: The result, if found - None otherwise.
"""
try:
artist: str = artist.strip().lower()
@ -61,12 +62,16 @@ class LRCLib:
timeout=self.timeout,
headers=self.headers) as request:
request.raise_for_status()
text: str|None = await request.text()
text: Optional[str] = await request.text()
if not text:
raise InvalidResponseException("No search response.")
if len(text) < 100:
raise InvalidResponseException("Search response text was invalid (len < 100 chars.)")
search_data: dict|None = await request.json()
search_data: Optional[dict] = await request.json()
if not isinstance(search_data, dict):
raise InvalidResponseException("No JSON search data.")
# logging.info("Search Data:\n%s", search_data)
@ -125,5 +130,4 @@ class LRCLib:
await self.cache.store(matched)
return matched
except:
traceback.print_exc()
return
traceback.print_exc()

47
lyric_search/sources/redis_cache.py
View File

@ -1,7 +1,4 @@
#!/usr/bin/env python3.12
# pylint: disable=bare-except, broad-exception-caught, wrong-import-order
# pylint: disable=wrong-import-position
import logging
import traceback
@ -9,7 +6,9 @@ import json
import time
import sys
import regex
from regex import Pattern
import asyncio
from typing import Union, Optional
sys.path.insert(1,'..')
from lyric_search import notifier
from lyric_search.constructors import LyricsResult
@ -20,10 +19,6 @@ from redis.commands.search.field import TextField, TagField
from redis.commands.json.path import Path
from . import private
logger = logging.getLogger()
log_level = logging.getLevelName(logger.level)
@ -41,14 +36,15 @@ class RedisCache:
self.redis_client = redis.Redis(password=private.REDIS_PW)
self.notifier = notifier.DiscordNotifier()
self.notify_warnings = True
self.regexes = [
self.regexes: list[Pattern] = [
regex.compile(r'\-'),
regex.compile(r'[^a-zA-Z0-9\s]'),
]
try:
asyncio.get_event_loop().create_task(self.create_index())
except:
pass
except Exception as e:
logging.debug("Failed to create redis create_index task: %s",
str(e))
async def create_index(self) -> None:
"""Create Index"""
@ -64,10 +60,11 @@ class RedisCache:
if str(result) != "OK":
raise RedisException(f"Redis: Failed to create index: {result}")
except Exception as e:
pass
# await self.notifier.send(f"ERROR @ {__file__.rsplit("/", maxsplit=1)[-1]}", f"Failed to create idx: {str(e)}")
logging.debug("Failed to create redis index: %s",
str(e))
def sanitize_input(self, artist: str, song: str, fuzzy: bool = False) -> tuple[str, str]:
def sanitize_input(self, artist: str, song: str,
fuzzy: Optional[bool] = False) -> tuple[str, str]:
"""
Sanitize artist/song input (convert to redis matchable fuzzy query)
Args:
@ -121,7 +118,9 @@ class RedisCache:
traceback.print_exc()
async def search(self, **kwargs) -> list[tuple]:
async def search(self, artist: Optional[str] = None,
song: Optional[str] = None,
lyrics: Optional[str] = None) -> list[tuple]:
"""
Search Redis Cache
Args:
@ -133,9 +132,6 @@ class RedisCache:
"""
try:
artist = kwargs.get('artist', '')
song = kwargs.get('song', '')
lyrics = kwargs.get('lyrics')
fuzzy_artist = None
fuzzy_song = None
is_random_search = artist == "!" and song == "!"
@ -148,10 +144,10 @@ class RedisCache:
logging.debug("Redis: Searching normally first")
(artist, song) = self.sanitize_input(artist, song)
logging.debug("Seeking: %s - %s", artist, song)
search_res = await self.redis_client.ft().search(Query(
search_res: Union[dict, list] = await self.redis_client.ft().search(Query(
f"@artist:{artist} @song:{song}"
))
search_res_out = [(result['id'].split(":",
search_res_out: list[tuple] = [(result['id'].split(":",
maxsplit=1)[1], dict(json.loads(result['json'])))
for result in search_res.docs]
if not search_res_out:
@ -167,8 +163,8 @@ class RedisCache:
for result in search_res.docs]
else:
random_redis_key = await self.redis_client.randomkey()
out_id = str(random_redis_key).split(":",
random_redis_key: str = await self.redis_client.randomkey()
out_id: str = str(random_redis_key).split(":",
maxsplit=1)[1][:-1]
search_res = await self.redis_client.json().get(random_redis_key)
search_res_out = [(out_id, search_res)]
@ -179,7 +175,8 @@ class RedisCache:
except Exception as e:
traceback.print_exc()
# await self.notifier.send(f"ERROR @ {__file__.rsplit("/", maxsplit=1)[-1]}", f"{str(e)}\nSearch was: {artist} - {song}; fuzzy: {fuzzy_artist} - {fuzzy_song}")
async def redis_store(self, sqlite_id: int, lyr_result: LyricsResult) -> None:
async def redis_store(self, sqlite_id: int,
lyr_result: LyricsResult) -> None:
"""
Store lyrics to redis cache
Args:
@ -191,7 +188,7 @@ class RedisCache:
try:
(search_artist, search_song) = self.sanitize_input(lyr_result.artist,
lyr_result.song)
redis_mapping = {
redis_mapping: dict = {
'id': sqlite_id,
'src': lyr_result.src,
'date_retrieved': time.time(),
@ -206,8 +203,8 @@ class RedisCache:
'tags': '(none)',
'liked': 0,
}
newkey = f"lyrics:000{sqlite_id}"
jsonset = await self.redis_client.json().set(newkey, Path.root_path(),
newkey: str = f"lyrics:000{sqlite_id}"
jsonset: bool = await self.redis_client.json().set(newkey, Path.root_path(),
redis_mapping)
if not jsonset:
raise RedisException(f"Failed to store {lyr_result.artist} - {lyr_result.song} (SQLite id: {sqlite_id}) to redis:\n{jsonset}")

32
lyric_search/utils.py
View File

@ -1,9 +1,10 @@
#!/usr/bin/env python3.12
from difflib import SequenceMatcher
from typing import List, Optional, Tuple
from typing import List, Optional, Union, Any
import logging
import regex
from regex import Pattern
class TrackMatcher:
"""Track Matcher"""
@ -17,7 +18,7 @@ class TrackMatcher:
"""
self.threshold = threshold
def find_best_match(self, input_track: str, candidate_tracks: List[tuple[int|str, str]]) -> Optional[Tuple[str, float]]:
def find_best_match(self, input_track: str, candidate_tracks: List[tuple[int|str, str]]) -> Optional[tuple]:
"""
Find the best matching track from the candidate list.
@ -26,7 +27,7 @@ class TrackMatcher:
candidate_tracks (List[tuple[int|str, str]]): List of candidate tracks
Returns:
Optional[Tuple[int, str, float]]: Tuple of (best matching track, similarity score)
Optional[tuple[int, str, float]]: Tuple of (best matching track, similarity score)
or None if no good match found
"""
@ -38,7 +39,7 @@ class TrackMatcher:
input_track = self._normalize_string(input_track)
best_match = None
best_score = 0
best_score: float = 0.0
for candidate in candidate_tracks:
normalized_candidate = self._normalize_string(candidate[1])
@ -56,7 +57,10 @@ class TrackMatcher:
best_match = candidate
# Return the match only if it meets the threshold
return (best_match, round(best_score * 100)) if best_score >= self.threshold else None
if best_score >= self.threshold:
return None
match: tuple = (best_match, round(best_score * 100))
return match
def _normalize_string(self, text: str) -> str:
"""
@ -98,10 +102,14 @@ class DataUtils:
Data Utils
"""
def __init__(self):
def __init__(self) -> None:
self.lrc_regex = regex.compile(r'\[([0-9]{2}:[0-9]{2})\.[0-9]{1,3}\](\s(.*)){0,}')
self.scrub_regex_1: Pattern = regex.compile(r'(\[.*?\])(\s){0,}(\:){0,1}')
self.scrub_regex_2: Pattern = regex.compile(r'(\d?)(Embed\b)',
flags=regex.IGNORECASe)
self.scrub_regex_3: Pattern = regex.compile(r'\n{2}')
self.scrub_regex_4: Pattern = regex.compile(r'[0-9]\b$')
def scrub_lyrics(self, lyrics: str) -> str:
"""
Lyric Scrub Regex Chain
@ -110,10 +118,10 @@ class DataUtils:
Returns:
str: Regex scrubbed lyrics
"""
lyrics = regex.sub(r'(\[.*?\])(\s){0,}(\:){0,1}', '', lyrics)
lyrics = regex.sub(r'(\d?)(Embed\b)', '', lyrics, flags=regex.IGNORECASE)
lyrics = regex.sub(r'\n{2}', '\n', lyrics) # Gaps between verses
lyrics = regex.sub(r'[0-9]\b$', '', lyrics)
lyrics = self.scrub_regex_1.sub('', lyrics)
lyrics = self.scrub_regex_2.sub('', lyrics, flags=regex.IGNORECASE)
lyrics = self.scrub_regex_3.sub('\n', lyrics) # Gaps between verses
lyrics = self.scrub_regex_3.sub('', lyrics)
return lyrics
def create_lrc_object(self, lrc_str: str) -> list[dict]: