diff --git a/lyric_search_new/sources/cache.py b/lyric_search_new/sources/cache.py index 33ceb77..c3e7eed 100644 --- a/lyric_search_new/sources/cache.py +++ b/lyric_search_new/sources/cache.py @@ -7,13 +7,17 @@ import regex import logging import sys import traceback +import asyncio sys.path.insert(1,'..') sys.path.insert(1,'.') -from typing import Optional +from typing import Optional, Any import aiosqlite as sqlite3 +from . import redis_cache from lyric_search_new import utils from lyric_search_new.constructors import LyricsResult + + logger = logging.getLogger() log_level = logging.getLevelName(logger.level) @@ -23,24 +27,41 @@ class Cache: self.cache_db: str = os.path.join("/", "var", "lib", "singerdbs", "cached_lyrics.db") + self.redis_cache = redis_cache.RedisCache() + + asyncio.get_event_loop().create_task( + self.redis_cache.create_index()) self.cache_pre_query: str = "pragma journal_mode = WAL; pragma synchronous = normal;\ pragma temp_store = memory; pragma mmap_size = 30000000000;" self.sqlite_exts: list[str] = ['/usr/local/lib/python3.11/dist-packages/spellfix1.cpython-311-x86_64-linux-gnu.so'] self.label: str = "Cache" - def get_matched(self, sqlite_rows: list[sqlite3.Row], matched_candidate: tuple, confidence: int) -> Optional[LyricsResult]: + def get_matched(self, matched_candidate: tuple, confidence: int, + sqlite_rows: list[sqlite3.Row] = None, redis_results: Any = None) -> Optional[LyricsResult]: """Get Matched Result""" matched_id: int = matched_candidate[0] - for row in sqlite_rows: - if row[0] == matched_id: - (_id, artist, song, lyrics, original_src, _confidence) = row - return LyricsResult( - artist=artist, - song=song, - lyrics=lyrics, - src=f"{original_src} (cached, id: {_id})", - confidence=confidence) + if redis_results: + logging.info("Matched id: %s", matched_id) + for row in redis_results: + if row['id'] == matched_id: + return LyricsResult( + artist=row['artist'], + song=row['song'], + lyrics=row['lyrics'], + src=f"{row['src']} (redis, id: {row['id']})", + confidence=row['confidence'] + ) + else: + for row in sqlite_rows: + if row[0] == matched_id: + (_id, artist, song, lyrics, original_src, _confidence) = row + return LyricsResult( + artist=artist, + song=song, + lyrics=lyrics, + src=f"{original_src} (cached, id: {_id})", + confidence=confidence) return None async def check_existence(self, artistsong: str) -> Optional[bool]: @@ -115,12 +136,44 @@ class Cache: # pylint: enable=unused-argument artist: str = artist.strip().lower() song: str = song.strip().lower() + input_track: str = f"{artist} - {song}" search_params: Optional[tuple] = None random_search: bool = False time_start: float = time.time() + matcher = utils.TrackMatcher() logging.info("Searching %s - %s on %s", - artist, song, self.label) + artist, song, self.label) + + """Check Redis First""" + + logging.info("Checking redis cache for %s...", + f"{artist} - {song}") + redis_result = await self.redis_cache.search(artist=artist, + song=song) + + if redis_result: + result_tracks: list = [] + for track in redis_result: + result_tracks.append((track['id'], f"{track['artist']} - {track['song']}")) + + best_match: tuple|None = matcher.find_best_match(input_track=input_track, + candidate_tracks=result_tracks) + (candidate, confidence) = best_match + matched = self.get_matched(redis_results=redis_result, matched_candidate=candidate, + confidence=confidence) + time_end: float = time.time() + time_diff: float = time_end - time_start + matched.confidence = confidence + matched.time = time_diff + + if matched: + logging.info("Found %s on redis cache, skipping SQLite...", + f"{artist} - {song}") + return matched + + """SQLite: Fallback""" + async with sqlite3.connect(self.cache_db, timeout=2) as db_conn: await db_conn.enable_load_extension(True) for ext in self.sqlite_exts: @@ -142,8 +195,6 @@ class Cache: for track in results: (_id, _artist, _song, _lyrics, _src, _confidence) = track result_tracks.append((_id, f"{_artist} - {_song}")) - input_track: str = f"{artist} - {song}" - matcher = utils.TrackMatcher() if not random_search: best_match: tuple|None = matcher.find_best_match(input_track=input_track, candidate_tracks=result_tracks) @@ -161,6 +212,5 @@ class Cache: matched.time = time_diff return matched except: - if log_level == "DEBUG": - traceback.print_exc() + traceback.print_exc() return \ No newline at end of file diff --git a/lyric_search_new/sources/redis_cache.py b/lyric_search_new/sources/redis_cache.py new file mode 100644 index 0000000..4884a61 --- /dev/null +++ b/lyric_search_new/sources/redis_cache.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3.12 +# pylint: disable=bare-except, broad-exception-caught + + +import logging +import traceback +import json +import redis.asyncio as redis +from redis.commands.json.path import Path +from redis.commands.search.query import NumericFilter, Query +from redis.commands.search.indexDefinition import IndexDefinition, IndexType +from redis.commands.search.field import TextField, NumericField, TagField +from . import private + + +logger = logging.getLogger() +log_level = logging.getLevelName(logger.level) + +class RedisException(Exception): + """ + Redis Exception + """ + +class RedisCache: + """ + Redis Cache Methods + """ + + def __init__(self): + self.redis_pw = private.REDIS_PW + self.redis_client = redis.Redis(password=self.redis_pw) + + async def create_index(self): + """Create Index""" + try: + schema = ( + TextField("$.artist", as_name="artist"), + TextField("$.song", as_name="song"), + TextField("$.src", as_name="src"), + TextField("$.lyrics", as_name="lyrics") + ) + result = await self.redis_client.ft().create_index(schema, definition=IndexDefinition(prefix=["lyrics:"], index_type=IndexType.JSON)) + if str(result) != "OK": + raise RedisException(f"Redis: Failed to create index: {result}") + except Exception as e: + pass + + async def search(self, **kwargs): + """Search Redis Cache + @artist: artist to search + @song: song to search + @lyrics: lyrics to search (optional, used in place of artist/song if provided) + """ + + try: + artist = kwargs.get('artist') + song = kwargs.get('song') + lyrics = kwargs.get('lyrics') + + if lyrics: + # to code later + raise RedisException("Lyric search not yet implemented") + + search_res = await self.redis_client.ft().search( + Query(f"@artist:{artist} @song:{song}" + )) + search_res_out = [dict(json.loads(result['json'])) + for result in search_res.docs] + + return search_res_out + except: + traceback.print_exc() + \ No newline at end of file