trialing redis cache

lyric_search_new/sources/cache.py

@@ -7,13 +7,17 @@ import regex
 import logging
 import sys
 import traceback
+import asyncio
 sys.path.insert(1,'..')
 sys.path.insert(1,'.')
-from typing import Optional
+from typing import Optional, Any
 import aiosqlite as sqlite3
+from . import redis_cache
 from lyric_search_new import utils
 from lyric_search_new.constructors import LyricsResult
 
+
+
 logger = logging.getLogger()
 log_level = logging.getLevelName(logger.level)
 
@@ -23,24 +27,41 @@ class Cache:
         self.cache_db: str = os.path.join("/", "var",
                                      "lib", "singerdbs",
                                      "cached_lyrics.db")
+        self.redis_cache = redis_cache.RedisCache()
+        
+        asyncio.get_event_loop().create_task(
+            self.redis_cache.create_index())
         
         self.cache_pre_query: str = "pragma journal_mode = WAL; pragma synchronous = normal;\
             pragma temp_store = memory; pragma mmap_size = 30000000000;"
         self.sqlite_exts: list[str] = ['/usr/local/lib/python3.11/dist-packages/spellfix1.cpython-311-x86_64-linux-gnu.so']
         self.label: str = "Cache"
 
-    def get_matched(self, sqlite_rows: list[sqlite3.Row], matched_candidate: tuple, confidence: int) -> Optional[LyricsResult]:
+    def get_matched(self, matched_candidate: tuple, confidence: int,
+                    sqlite_rows: list[sqlite3.Row] = None, redis_results: Any = None) -> Optional[LyricsResult]:
         """Get Matched Result"""
         matched_id: int = matched_candidate[0]
-        for row in sqlite_rows:
+        if redis_results:
-            if row[0] == matched_id:
+            logging.info("Matched id: %s", matched_id)
-                (_id, artist, song, lyrics, original_src, _confidence) = row
+            for row in redis_results:
-                return LyricsResult(
+                if row['id'] == matched_id:
-                    artist=artist,
+                    return LyricsResult(
-                    song=song,
+                        artist=row['artist'],
-                    lyrics=lyrics,
+                        song=row['song'],
-                    src=f"{original_src} (cached, id: {_id})",
+                        lyrics=row['lyrics'],
-                    confidence=confidence)
+                        src=f"{row['src']} (redis, id: {row['id']})",
+                        confidence=row['confidence']
+                    )
+        else:
+            for row in sqlite_rows:
+                if row[0] == matched_id:
+                    (_id, artist, song, lyrics, original_src, _confidence) = row
+                    return LyricsResult(
+                        artist=artist,
+                        song=song,
+                        lyrics=lyrics,
+                        src=f"{original_src} (cached, id: {_id})",
+                        confidence=confidence)
         return None
     
     async def check_existence(self, artistsong: str) -> Optional[bool]:
@@ -115,12 +136,44 @@ class Cache:
             # pylint: enable=unused-argument
             artist: str = artist.strip().lower() 
             song: str  = song.strip().lower()
+            input_track: str = f"{artist} - {song}"
             search_params: Optional[tuple] = None
             random_search: bool = False
             time_start: float = time.time()
+            matcher = utils.TrackMatcher()
 
             logging.info("Searching %s - %s on %s",
                          artist, song, self.label) 
+            
+            """Check Redis First"""
+
+            logging.info("Checking redis cache for %s...",
+                    f"{artist} - {song}")
+            redis_result = await self.redis_cache.search(artist=artist,
+                                                         song=song)
+            
+            if redis_result:
+                result_tracks: list = []
+                for track in redis_result:
+                    result_tracks.append((track['id'], f"{track['artist']} - {track['song']}"))                
+                
+                best_match: tuple|None = matcher.find_best_match(input_track=input_track,
+                                                                 candidate_tracks=result_tracks)
+                (candidate, confidence) = best_match
+                matched = self.get_matched(redis_results=redis_result, matched_candidate=candidate,
+                                          confidence=confidence)                  
+                time_end: float = time.time()
+                time_diff: float = time_end - time_start
+                matched.confidence = confidence
+                matched.time = time_diff
+
+                if matched:
+                    logging.info("Found %s on redis cache, skipping SQLite...",
+                             f"{artist} - {song}")                  
+                    return matched
+            
+            """SQLite: Fallback"""
+                                   
             async with sqlite3.connect(self.cache_db, timeout=2) as db_conn:
                 await db_conn.enable_load_extension(True)
                 for ext in self.sqlite_exts:
@@ -142,8 +195,6 @@ class Cache:
                         for track in results:
                             (_id, _artist, _song, _lyrics, _src, _confidence) = track
                             result_tracks.append((_id, f"{_artist} - {_song}"))
-                        input_track: str = f"{artist} - {song}"
-                        matcher = utils.TrackMatcher()
                         if not random_search:
                             best_match: tuple|None = matcher.find_best_match(input_track=input_track,
                                                             candidate_tracks=result_tracks)
@@ -161,6 +212,5 @@ class Cache:
                         matched.time = time_diff
                         return matched
         except:
-            if log_level == "DEBUG":
+            traceback.print_exc()
-                traceback.print_exc()
             return

lyric_search_new/sources/redis_cache.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3.12
+# pylint: disable=bare-except, broad-exception-caught
+
+
+import logging
+import traceback
+import json
+import redis.asyncio as redis
+from redis.commands.json.path import Path
+from redis.commands.search.query import NumericFilter, Query
+from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.field import TextField, NumericField, TagField
+from . import private
+
+
+logger = logging.getLogger()
+log_level = logging.getLevelName(logger.level)
+
+class RedisException(Exception):
+    """
+    Redis Exception
+    """
+
+class RedisCache:
+    """
+    Redis Cache Methods
+    """
+
+    def __init__(self):
+        self.redis_pw = private.REDIS_PW
+        self.redis_client = redis.Redis(password=self.redis_pw)
+    
+    async def create_index(self):
+        """Create Index"""
+        try:
+            schema = (
+            TextField("$.artist", as_name="artist"),
+            TextField("$.song", as_name="song"),
+            TextField("$.src", as_name="src"),
+            TextField("$.lyrics", as_name="lyrics")
+        )    
+            result = await self.redis_client.ft().create_index(schema, definition=IndexDefinition(prefix=["lyrics:"], index_type=IndexType.JSON))
+            if str(result) != "OK":
+                raise RedisException(f"Redis: Failed to create index: {result}")
+        except Exception as e:
+            pass
+
+    async def search(self, **kwargs):
+        """Search Redis Cache
+        @artist: artist to search
+        @song: song to search
+        @lyrics: lyrics to search (optional, used in place of artist/song if provided)
+        """
+
+        try:
+            artist = kwargs.get('artist')
+            song = kwargs.get('song')
+            lyrics = kwargs.get('lyrics')
+
+            if lyrics:
+                # to code later
+                raise RedisException("Lyric search not yet implemented")
+            
+            search_res = await self.redis_client.ft().search(
+                Query(f"@artist:{artist} @song:{song}"
+        )) 
+            search_res_out = [dict(json.loads(result['json']))
+                              for result in search_res.docs]
+            
+            return search_res_out
+        except:
+            traceback.print_exc()
+