codey/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docstring stuff

Browse Source
This commit is contained in:
codey 2025-01-19 07:01:07 -05:00
parent 151643c5dc
commit be0ef08f3d
9 changed files with 118 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
base.py
View File

@ -3,10 +3,10 @@
import importlib
import logging
import asyncio
from typing import Any
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from lyric_search_new.sources import redis_cache
logger = logging.getLogger()
@ -87,13 +87,14 @@ xc_endpoints = importlib.import_module("endpoints.xc").XC(app, util, constants,
# Below: Karma endpoint(s)
karma_endpoints = importlib.import_module("endpoints.karma").Karma(app, util, constants, glob_state)
# @app.on_event("startup")
# @repeat_every(seconds=10)
# async def cah_tasks() -> None:
# return await cah_endpoints.periodicals()
"""
End Actionable Routes
"""
"""
Startup
"""
redis_cache = redis_cache.RedisCache()
asyncio.get_event_loop().create_task(
redis_cache.create_index())

21
endpoints/lyric_search.py
View File

@ -6,6 +6,7 @@ import traceback
import logging
import os
import urllib.parse
from typing import Optional
import regex
import aiohttp
import aiosqlite as sqlite3
@ -187,10 +188,21 @@ class LyricSearch(FastAPI):
if data.src.upper() not in self.acceptable_request_sources:
raise HTTPException(detail="Invalid request", status_code=500)
search_artist: Optional[str] = data.a
search_song: Optional[str] = data.s
if search_artist and search_song:
search_artist = self.constants.DOUBLE_SPACE_REGEX.sub(" ", search_artist.strip())
search_song = self.constants.DOUBLE_SPACE_REGEX.sub(" ", search_song.strip())
search_artist = urllib.parse.unquote(search_artist)
search_song = urllib.parse.unquote(search_song)
excluded_sources = data.excluded_sources
aggregate_search = aggregate.Aggregate(exclude_methods=excluded_sources)
plain_lyrics = not data.lrc
result = await aggregate_search.search(data.a, data.s, plain_lyrics)
result = await aggregate_search.search(search_artist, search_song, plain_lyrics)
if not result:
return {
@ -201,6 +213,7 @@ class LyricSearch(FastAPI):
result = result.todict()
if data.sub and not data.lrc:
seeked_found_line = None
lyric_lines = result['lyrics'].strip().split(" / ")
for i, line in enumerate(lyric_lines):
line = regex.sub(r'\u2064', '', line.strip())
@ -223,12 +236,10 @@ class LyricSearch(FastAPI):
result['lyrics'] = regex.sub(r'(\s/\s|\n)', '<br>', result['lyrics']).strip()
else:
# Swap lyrics key for 'lrc'
logging.info("lyrics: %s, type: %s",
result['lyrics'], type(result['lyrics']))
result['lrc'] = result['lyrics']
result.pop('lyrics')
if "cached" in result['src']:
if "cache" in result['src']:
result['from_cache'] = True
"""
@ -333,8 +344,6 @@ class LyricSearch(FastAPI):
if not reg_helper:
continue
reg_helper = reg_helper[0]
logging.debug("Reg helper: %s for line: %s; len: %s",
reg_helper, line, len(reg_helper))
_timetag = reg_helper[0]
if not reg_helper[1].strip():
_words = "♪"

14
lyric_search_new/constructors.py
View File

@ -4,12 +4,14 @@ from dataclasses import dataclass, asdict
@dataclass
class LyricsResult:
"""Class for returned Lyrics Results
@artist: returned artist
@song: returned song
@src: source result was fetched from
@lyrics: str if plain lyrics, list for lrc
@time: time taken to retrieve lyrics from source
"""
Class for returned Lyrics Results
Attributes:
artist (str): returned artist
song (str): returned song
src (str): source result was fetched from
lyrics (str|list): str if plain lyrics, list for lrc
time (float): time taken to retrieve lyrics from source
"""
artist: str
song: str

14
lyric_search_new/sources/aggregate.py
View File

@ -14,7 +14,9 @@ logger = logging.getLogger()
logger.setLevel(logging.INFO)
class Aggregate:
"""Aggregate all source methods"""
"""
Aggregate all source methods
"""
def __init__(self, exclude_methods=None):
if not exclude_methods:
@ -22,7 +24,15 @@ class Aggregate:
self.exclude_methods = exclude_methods
async def search(self, artist: str, song: str, plain: bool = True) -> Optional[LyricsResult]:
"""Aggregate Search"""
"""
Aggregate Search
Args:
artist (str): Artist to search
song (str): Song to search
plain (bool): Search for plain lyrics (lrc otherwise)
Returns:
LyricsResult|None: The result, if found - None otherwise.
"""
if not plain:
logging.info("LRCs requested, limiting search to LRCLib")
self.exclude_methods = ["genius", "cache"]

36
lyric_search_new/sources/cache.py
View File

@ -7,7 +7,6 @@ import regex
import logging
import sys
import traceback
import asyncio
sys.path.insert(1,'..')
sys.path.insert(1,'.')
from typing import Optional, Any
@ -29,9 +28,6 @@ class Cache:
"cached_lyrics.db")
self.redis_cache = redis_cache.RedisCache()
asyncio.get_event_loop().create_task(
self.redis_cache.create_index())
self.cache_pre_query: str = "pragma journal_mode = WAL; pragma synchronous = normal;\
pragma temp_store = memory; pragma mmap_size = 30000000000;"
self.sqlite_exts: list[str] = ['/usr/local/lib/python3.11/dist-packages/spellfix1.cpython-311-x86_64-linux-gnu.so']
@ -39,7 +35,16 @@ class Cache:
def get_matched(self, matched_candidate: tuple, confidence: int,
sqlite_rows: list[sqlite3.Row] = None, redis_results: Any = None) -> Optional[LyricsResult]:
"""Get Matched Result"""
"""
Get Matched Result
Args:
matched_candidate (tuple): the correctly matched candidate returned by matcher.best_match
confidence (int): % confidence
sqlite_rows (list[sqlite3.Row]|None): List of returned rows from SQLite DB, or None if Redis
redis_results (Any): List of Redis returned data, or None if SQLite
Returns:
LyricsResult|None: The result, if found - None otherwise.
"""
matched_id: int = matched_candidate[0]
if redis_results:
for res in redis_results:
@ -68,7 +73,10 @@ class Cache:
async def check_existence(self, artistsong: str) -> Optional[bool]:
"""
Check whether lyrics are already stored for track
@artistsong: artist and song in artist\nsong format
Args:
artistsong (str): artist and song in artist\\nsong format
Returns:
bool: Whether track was found in cache
"""
logging.debug("Checking whether %s is already stored",
artistsong.replace("\n", " - "))
@ -88,8 +96,11 @@ class Cache:
async def store(self, lyr_result: LyricsResult) -> None:
"""
store
@lyr_result: the returned lyrics to cache
Store lyrics to cache
Args:
lyr_result: the returned lyrics to cache
Returns:
None
"""
logging.info("Storing %s",
@ -127,11 +138,12 @@ class Cache:
# pylint: disable=unused-argument
async def search(self, artist: str, song: str, **kwargs) -> Optional[LyricsResult]:
"""
search
@artist: the artist to search
@song: the song to search
Cache Search
Args:
artist: the artist to search
song: the song to search
Returns:
- LyricsResult corresponding to nearest match found (if found), **None** otherwise
LyricsResult|None: The result, if found - None otherwise.
"""
try:
# pylint: enable=unused-argument

8
lyric_search_new/sources/genius.py
View File

@ -40,8 +40,12 @@ class Genius:
# pylint: disable=unused-argument
async def search(self, artist: str, song: str, **kwargs) -> Optional[LyricsResult]:
"""
@artist: the artist to search
@song: the song to search
Genius Search
Args:
artist (str): the artist to search
song (str): the song to search
Returns:
LyricsResult|None: The result, if found - None otherwise.
"""
try:
# pylint: enable=unused-argument

8
lyric_search_new/sources/lrclib.py
View File

@ -34,8 +34,12 @@ class LRCLib:
async def search(self, artist: str, song: str, plain: bool = True) -> Optional[LyricsResult]:
"""
@artist: the artist to search
@song: the song to search
LRCLib Search
Args:
artist (str): the artist to search
song (str): the song to search
Returns:
LyricsResult|None: The result, if found - None otherwise.
"""
try:
artist: str = artist.strip().lower()

25
lyric_search_new/sources/redis_cache.py
View File

@ -1,10 +1,14 @@
#!/usr/bin/env python3.12
# pylint: disable=bare-except, broad-exception-caught
# pylint: disable=bare-except, broad-exception-caught, wrong-import-order
# pylint: disable=wrong-import-position
import logging
import traceback
import json
import sys
sys.path.insert(1,'..')
from lyric_search_new import notifier
import redis.asyncio as redis
from redis.commands.search.query import Query
from redis.commands.search.indexDefinition import IndexDefinition, IndexType
@ -12,6 +16,7 @@ from redis.commands.search.field import TextField
from . import private
logger = logging.getLogger()
log_level = logging.getLevelName(logger.level)
@ -27,6 +32,8 @@ class RedisCache:
def __init__(self):
self.redis_client = redis.Redis(password=private.REDIS_PW)
self.notifier = notifier.DiscordNotifier()
self.notify_warnings = True
async def create_index(self):
"""Create Index"""
@ -41,8 +48,8 @@ class RedisCache:
schema, definition=IndexDefinition(prefix=["lyrics:"], index_type=IndexType.JSON))
if str(result) != "OK":
raise RedisException(f"Redis: Failed to create index: {result}")
except:
pass
except Exception as e:
await self.notifier.send(f"ERROR @ {__file__}", f"Failed to create idx: {str(e)}")
async def search(self, **kwargs):
"""Search Redis Cache
@ -62,13 +69,13 @@ class RedisCache:
raise RedisException("Lyric search not yet implemented")
if not is_random_search:
artist = artist.replace("-", "")
song = song.replace("-", "")
artist = artist.replace("-", " ")
song = song.replace("-", " ")
search_res = await self.redis_client.ft().search(
Query(f"@artist:{artist} @song:{song}"
))
search_res_out = [(result['id'].split(":",
maxsplit=1)[1][:-1], dict(json.loads(result['json'])))
maxsplit=1)[1], dict(json.loads(result['json'])))
for result in search_res.docs]
else:
random_redis_key = await self.redis_client.randomkey()
@ -77,8 +84,10 @@ class RedisCache:
search_res = await self.redis_client.json().get(random_redis_key)
search_res_out = [(out_id, search_res)]
if not search_res_out and self.notify_warnings:
await self.notifier.send("WARNING", f"Redis cache miss for: \n## *{artist} - {song}*")
return search_res_out
except:
except Exception as e:
await self.notifier.send(f"ERROR @ {__file__}", str(e))
traceback.print_exc()

27
lyric_search_new/utils.py
View File

@ -62,6 +62,10 @@ class TrackMatcher:
"""
Normalize string for comparison by removing special characters,
extra spaces, and converting to lowercase.
Args:
text (str): The text to normalize
Returns:
str: Normalized text
"""
# Remove special characters and convert to lowercase
text = regex.sub(r'[^\w\s-]', '', text).lower()
@ -72,6 +76,11 @@ class TrackMatcher:
def _calculate_token_similarity(self, str1: str, str2: str) -> float:
"""
Calculate similarity based on matching tokens (words).
Args:
str1 (str): string 1 to compare
str2 (str): string 2 to compare
Returns:
float: The token similarity score
"""
tokens1 = set(str1.split())
tokens2 = set(str2.split())
@ -94,8 +103,12 @@ class DataUtils:
def scrub_lyrics(self, lyrics: str) -> str:
"""Regex Chain
@lyrics: The lyrics (str) to scrub
"""
Lyric Scrub Regex Chain
Args:
lyrics (str): The lyrics to scrub
Returns:
str: Regex scrubbed lyrics
"""
lyrics = regex.sub(r'(\[.*?\])(\s){0,}(\:){0,1}', '', lyrics)
lyrics = regex.sub(r'(\d?)(Embed\b)', '', lyrics, flags=regex.IGNORECASE)
@ -104,8 +117,12 @@ class DataUtils:
return lyrics
def create_lrc_object(self, lrc_str: str) -> list[dict]:
"""Create LRC Object
@lrc_str: The raw LRCLib syncedLyrics (str)
"""
Create LRC Object
Args:
lrc_str (str): The raw LRCLib syncedLyrics
Returns:
list[dict]: LRC Object comprised of timestamps/lyrics
"""
lrc_out: list = []
for line in lrc_str.split("\n"):
@ -128,6 +145,4 @@ class DataUtils:
"timeTag": _timetag,
"words": _words,
})
logging.info("util: returning %s, type: %s",
lrc_out, type(lrc_out))
return lrc_out