codey/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

radio_util: open tracks SQLite DB in readonly mode; black: reformat files

Browse Source
This commit is contained in:
codey
2025-04-17 07:28:05 -04:00
parent 96add377df
commit 6c88c23a4d
25 changed files with 1913 additions and 1340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

239
lyric_search/sources/cache.py
View File

@ -4,8 +4,9 @@ import regex
import logging
import sys
import traceback
sys.path.insert(1,'..')
sys.path.insert(1,'.')
sys.path.insert(1, "..")
sys.path.insert(1, ".")
from typing import Optional, Union, LiteralString
import aiosqlite as sqlite3
from . import redis_cache
@ -15,27 +16,38 @@ from lyric_search.constructors import LyricsResult
logger = logging.getLogger()
log_level = logging.getLevelName(logger.level)
class Cache:
"""Cache Search Module"""
def __init__(self) -> None:
self.cache_db: Union[str, LiteralString] = os.path.join("/", "usr", "local", "share",
"sqlite_dbs", "cached_lyrics.db")
self.cache_db: Union[str, LiteralString] = os.path.join(
"/", "usr", "local", "share", "sqlite_dbs", "cached_lyrics.db"
)
self.redis_cache = redis_cache.RedisCache()
self.notifier = notifier.DiscordNotifier()
self.cache_pre_query: str = "pragma journal_mode = WAL; pragma synchronous = normal;\
self.cache_pre_query: str = (
"pragma journal_mode = WAL; pragma synchronous = normal;\
pragma temp_store = memory; pragma mmap_size = 30000000000;"
self.sqlite_exts: list[str] = ['/home/api/api/solibs/spellfix1.cpython-311-x86_64-linux-gnu.so']
)
self.sqlite_exts: list[str] = [
"/home/api/api/solibs/spellfix1.cpython-311-x86_64-linux-gnu.so"
]
self.label: str = "Cache"
def get_matched(self, matched_candidate: tuple, confidence: int,
sqlite_rows: Optional[list[sqlite3.Row]] = None,
redis_results: Optional[list] = None) -> Optional[LyricsResult]:
def get_matched(
self,
matched_candidate: tuple,
confidence: int,
sqlite_rows: Optional[list[sqlite3.Row]] = None,
redis_results: Optional[list] = None,
) -> Optional[LyricsResult]:
"""
Get Matched Result
Args:
matched_candidate (tuple): the correctly matched candidate returned by matcher.best_match
confidence (int): % confidence
confidence (int): % confidence
sqlite_rows (Optional[list[sqlite3.Row]]): List of returned rows from SQLite DB, or None if Redis
redis_results (Any): List of Redis returned data, or None if SQLite
Returns:
@ -47,11 +59,11 @@ class Cache:
(key, row) = res
if key == matched_id:
return LyricsResult(
artist=row['artist'],
song=row['song'],
lyrics=row['lyrics'],
artist=row["artist"],
song=row["song"],
lyrics=row["lyrics"],
src=f"{row['src']} (redis cache, id: {key})",
confidence=row['confidence']
confidence=row["confidence"],
)
else:
for row in sqlite_rows:
@ -62,9 +74,10 @@ class Cache:
song=song,
lyrics=lyrics,
src=f"{original_src} (cached, id: {_id})",
confidence=confidence)
confidence=confidence,
)
return None
async def check_existence(self, artistsong: str) -> Optional[bool]:
"""
Check whether lyrics are already stored for track
@ -73,10 +86,13 @@ class Cache:
Returns:
bool: Whether track was found in cache
"""
logging.debug("Checking whether %s is already stored",
artistsong.replace("\n", " - "))
check_query: str = 'SELECT id, artist, song FROM lyrics WHERE editdist3((lower(artist) || " " || lower(song)), (? || " " || ?))\
logging.debug(
"Checking whether %s is already stored", artistsong.replace("\n", " - ")
)
check_query: str = (
'SELECT id, artist, song FROM lyrics WHERE editdist3((lower(artist) || " " || lower(song)), (? || " " || ?))\
<= 410 ORDER BY editdist3((lower(artist) || " " || lower(song)), ?) ASC LIMIT 1'
)
artistsong_split = artistsong.split("\n", maxsplit=1)
artist = artistsong_split[0].lower()
song = artistsong_split[1].lower()
@ -84,39 +100,45 @@ class Cache:
async with sqlite3.connect(self.cache_db, timeout=2) as db_conn:
await db_conn.enable_load_extension(True)
for ext in self.sqlite_exts:
await db_conn.load_extension(ext)
async with await db_conn.executescript(self.cache_pre_query) as _db_cursor:
await db_conn.load_extension(ext)
async with await db_conn.executescript(self.cache_pre_query) as _db_cursor:
async with await db_conn.execute(check_query, params) as db_cursor:
result = await db_cursor.fetchone()
if result:
logging.debug("%s is already stored.",
artistsong.replace("\n", " - "))
logging.debug(
"%s is already stored.", artistsong.replace("\n", " - ")
)
return True
logging.debug("%s cleared to be stored.",
artistsong)
logging.debug("%s cleared to be stored.", artistsong)
return False
async def store(self, lyr_result: LyricsResult) -> None:
"""
Store lyrics (SQLite, then Redis)
Args:
lyr_result (LyricsResult): the returned lyrics to cache
Returns: None
Returns: None
"""
try:
sqlite_insert_id = await self.sqlite_store(lyr_result)
if sqlite_insert_id:
await self.redis_cache.redis_store(sqlite_insert_id, lyr_result)
except Exception as e:
traceback.print_exc()
logging.error("ERROR @ %s: %s",
__file__.rsplit("/", maxsplit=1)[-1], f"cache::store >> {str(e)}")
await self.notifier.send(f"ERROR @ {__file__.rsplit("/", maxsplit=1)[-1]}",
f"cache::store >> `{str(e)}`")
async def sqlite_rowcount(self, where: Optional[str] = None,
params: Optional[tuple] = None) -> int:
logging.error(
"ERROR @ %s: %s",
__file__.rsplit("/", maxsplit=1)[-1],
f"cache::store >> {str(e)}",
)
await self.notifier.send(
f"ERROR @ {__file__.rsplit("/", maxsplit=1)[-1]}",
f"cache::store >> `{str(e)}`",
)
async def sqlite_rowcount(
self, where: Optional[str] = None, params: Optional[tuple] = None
) -> int:
"""
Get rowcount for cached_lyrics DB
Args:
@ -130,8 +152,8 @@ class Cache:
query = f"SELECT count(id) AS rowcount FROM lyrics {where}".strip()
async with await db_conn.execute(query, params) as db_cursor:
result = await db_cursor.fetchone()
return result['rowcount']
return result["rowcount"]
async def sqlite_distinct(self, column: str) -> int:
"""
Get count of distinct values for a column
@ -145,8 +167,8 @@ class Cache:
query = f"SELECT COUNT(DISTINCT {column}) as distinct_items FROM lyrics"
async with await db_conn.execute(query) as db_cursor:
result = await db_cursor.fetchone()
return result['distinct_items']
return result["distinct_items"]
async def sqlite_lyrics_length(self) -> int:
"""
Get total length of text stored for lyrics
@ -160,9 +182,8 @@ class Cache:
query = "SELECT SUM(LENGTH(lyrics)) as lyrics_len FROM lyrics"
async with await db_conn.execute(query) as db_cursor:
result = await db_cursor.fetchone()
return result['lyrics_len']
return result["lyrics_len"]
async def sqlite_store(self, lyr_result: LyricsResult) -> int:
"""
Store lyrics to SQLite Cache
@ -172,30 +193,42 @@ class Cache:
int: the inserted row id
"""
logging.info("Storing %s",
f"{lyr_result.artist} - {lyr_result.song}")
logging.info("Storing %s", f"{lyr_result.artist} - {lyr_result.song}")
if lyr_result.src.lower() == "cache":
logging.info("Skipping cache storage - returned LyricsResult originated from cache")
logging.info(
"Skipping cache storage - returned LyricsResult originated from cache"
)
return
artistsong = f"{lyr_result.artist}\n{lyr_result.song}"
if await self.check_existence(artistsong):
logging.info("Skipping cache storage - %s is already stored.",
artistsong.replace("\n", " - "))
logging.info(
"Skipping cache storage - %s is already stored.",
artistsong.replace("\n", " - "),
)
return
try:
lyrics = regex.sub(r'(<br>|\n|\r\n)', ' / ', lyr_result.lyrics.strip())
lyrics = regex.sub(r'\s{2,}', ' ', lyrics)
lyrics = regex.sub(r"(<br>|\n|\r\n)", " / ", lyr_result.lyrics.strip())
lyrics = regex.sub(r"\s{2,}", " ", lyrics)
insert_query = "INSERT INTO lyrics (src, date_retrieved, artist, song, artistsong, confidence, lyrics)\
VALUES(?, ?, ?, ?, ?, ?, ?)"
params = (lyr_result.src, time.time(), lyr_result.artist,
lyr_result.song, artistsong, lyr_result.confidence, lyrics)
params = (
lyr_result.src,
time.time(),
lyr_result.artist,
lyr_result.song,
artistsong,
lyr_result.confidence,
lyrics,
)
async with sqlite3.connect(self.cache_db, timeout=2) as db_conn:
async with await db_conn.executescript(self.cache_pre_query) as _db_cursor:
async with await db_conn.executescript(
self.cache_pre_query
) as _db_cursor:
async with await db_conn.execute(insert_query, params) as _cursor:
await db_conn.commit()
logging.info("Stored %s to SQLite!", artistsong.replace("\n", " - "))
@ -203,7 +236,7 @@ class Cache:
except:
logging.critical("Cache storage error!")
traceback.print_exc()
async def search(self, artist: str, song: str, **kwargs) -> Optional[LyricsResult]:
"""
Cache Search
@ -214,8 +247,8 @@ class Cache:
Optional[LyricsResult]: The result, if found - None otherwise.
"""
try:
artist: str = artist.strip().lower()
song: str = song.strip().lower()
artist: str = artist.strip().lower()
song: str = song.strip().lower()
input_track: str = f"{artist} - {song}"
search_query = None
search_params: Optional[tuple] = None
@ -225,87 +258,105 @@ class Cache:
if artist == "!" and song == "!":
random_search = True
search_query: str = 'SELECT id, artist, song, lyrics, src, confidence\
FROM lyrics ORDER BY RANDOM() LIMIT 1'
search_query: str = (
"SELECT id, artist, song, lyrics, src, confidence\
FROM lyrics ORDER BY RANDOM() LIMIT 1"
)
logging.info("Searching %s - %s on %s", artist, song, self.label)
logging.info("Searching %s - %s on %s",
artist, song, self.label)
"""Check Redis First"""
logging.debug("Checking redis cache for %s...",
f"{artist} - {song}")
logging.debug("Checking redis cache for %s...", f"{artist} - {song}")
try:
redis_result = await self.redis_cache.search(artist=artist,
song=song)
redis_result = await self.redis_cache.search(artist=artist, song=song)
if redis_result:
result_tracks: list = []
for returned in redis_result:
(key, track) = returned
result_tracks.append((key, f"{track['artist']} - {track['song']}"))
result_tracks.append(
(key, f"{track['artist']} - {track['song']}")
)
if not random_search:
best_match: Optional[tuple] = matcher.find_best_match(input_track=input_track,
candidate_tracks=result_tracks)
best_match: Optional[tuple] = matcher.find_best_match(
input_track=input_track, candidate_tracks=result_tracks
)
else:
best_match = (result_tracks[0], 100)
if best_match:
(candidate, confidence) = best_match
matched = self.get_matched(redis_results=redis_result, matched_candidate=candidate,
confidence=confidence)
matched = self.get_matched(
redis_results=redis_result,
matched_candidate=candidate,
confidence=confidence,
)
if matched and confidence >= 90:
time_end: float = time.time()
time_diff: float = time_end - time_start
matched.confidence = confidence
matched.time = time_diff
logging.info("Found %s on redis cache, skipping SQLite...",
f"{artist} - {song}")
await self.redis_cache.increment_found_count(self.label)
logging.info(
"Found %s on redis cache, skipping SQLite...",
f"{artist} - {song}",
)
await self.redis_cache.increment_found_count(self.label)
return matched
except:
pass
"""SQLite: Fallback"""
async with sqlite3.connect(self.cache_db, timeout=2) as db_conn:
await db_conn.enable_load_extension(True)
for ext in self.sqlite_exts:
await db_conn.load_extension(ext)
async with await db_conn.executescript(self.cache_pre_query) as _db_cursor:
async with await db_conn.executescript(
self.cache_pre_query
) as _db_cursor:
if not random_search:
search_query: str = 'SELECT id, artist, song, lyrics, src, confidence FROM lyrics\
search_query: str = (
'SELECT id, artist, song, lyrics, src, confidence FROM lyrics\
WHERE editdist3((lower(artist) || " " || lower(song)), (? || " " || ?))\
<= 410 ORDER BY editdist3((lower(artist) || " " || lower(song)), ?) ASC LIMIT 10'
search_params: tuple = (artist.strip(), song.strip(),
f"{artist.strip()} {song.strip()}")
async with await _db_cursor.execute(search_query, search_params) as db_cursor:
)
search_params: tuple = (
artist.strip(),
song.strip(),
f"{artist.strip()} {song.strip()}",
)
async with await _db_cursor.execute(
search_query, search_params
) as db_cursor:
results: list = await db_cursor.fetchall()
result_tracks: list = []
for track in results:
(_id, _artist, _song, _lyrics, _src, _confidence) = track
result_tracks.append((_id, f"{_artist} - {_song}"))
if not random_search:
best_match: Optional[tuple] = matcher.find_best_match(input_track=input_track,
candidate_tracks=result_tracks)
best_match: Optional[tuple] = matcher.find_best_match(
input_track=input_track, candidate_tracks=result_tracks
)
else:
best_match = (result_tracks[0], 100)
if not best_match or confidence < 90:
return None
(candidate, confidence) = best_match
logging.info("Result found on %s", self.label)
matched = self.get_matched(sqlite_rows=results,
matched_candidate=candidate,
confidence=confidence)
matched = self.get_matched(
sqlite_rows=results,
matched_candidate=candidate,
confidence=confidence,
)
time_end: float = time.time()
time_diff: float = time_end - time_start
matched.time = time_diff
await self.redis_cache.increment_found_count(self.label)
return matched
except:
traceback.print_exc()
traceback.print_exc()