trialing redis cache
This commit is contained in:
parent
eb2e61a2f5
commit
22ca3b9adc
@ -7,13 +7,17 @@ import regex
|
||||
import logging
|
||||
import sys
|
||||
import traceback
|
||||
import asyncio
|
||||
sys.path.insert(1,'..')
|
||||
sys.path.insert(1,'.')
|
||||
from typing import Optional
|
||||
from typing import Optional, Any
|
||||
import aiosqlite as sqlite3
|
||||
from . import redis_cache
|
||||
from lyric_search_new import utils
|
||||
from lyric_search_new.constructors import LyricsResult
|
||||
|
||||
|
||||
|
||||
logger = logging.getLogger()
|
||||
log_level = logging.getLevelName(logger.level)
|
||||
|
||||
@ -23,24 +27,41 @@ class Cache:
|
||||
self.cache_db: str = os.path.join("/", "var",
|
||||
"lib", "singerdbs",
|
||||
"cached_lyrics.db")
|
||||
self.redis_cache = redis_cache.RedisCache()
|
||||
|
||||
asyncio.get_event_loop().create_task(
|
||||
self.redis_cache.create_index())
|
||||
|
||||
self.cache_pre_query: str = "pragma journal_mode = WAL; pragma synchronous = normal;\
|
||||
pragma temp_store = memory; pragma mmap_size = 30000000000;"
|
||||
self.sqlite_exts: list[str] = ['/usr/local/lib/python3.11/dist-packages/spellfix1.cpython-311-x86_64-linux-gnu.so']
|
||||
self.label: str = "Cache"
|
||||
|
||||
def get_matched(self, sqlite_rows: list[sqlite3.Row], matched_candidate: tuple, confidence: int) -> Optional[LyricsResult]:
|
||||
def get_matched(self, matched_candidate: tuple, confidence: int,
|
||||
sqlite_rows: list[sqlite3.Row] = None, redis_results: Any = None) -> Optional[LyricsResult]:
|
||||
"""Get Matched Result"""
|
||||
matched_id: int = matched_candidate[0]
|
||||
for row in sqlite_rows:
|
||||
if row[0] == matched_id:
|
||||
(_id, artist, song, lyrics, original_src, _confidence) = row
|
||||
return LyricsResult(
|
||||
artist=artist,
|
||||
song=song,
|
||||
lyrics=lyrics,
|
||||
src=f"{original_src} (cached, id: {_id})",
|
||||
confidence=confidence)
|
||||
if redis_results:
|
||||
logging.info("Matched id: %s", matched_id)
|
||||
for row in redis_results:
|
||||
if row['id'] == matched_id:
|
||||
return LyricsResult(
|
||||
artist=row['artist'],
|
||||
song=row['song'],
|
||||
lyrics=row['lyrics'],
|
||||
src=f"{row['src']} (redis, id: {row['id']})",
|
||||
confidence=row['confidence']
|
||||
)
|
||||
else:
|
||||
for row in sqlite_rows:
|
||||
if row[0] == matched_id:
|
||||
(_id, artist, song, lyrics, original_src, _confidence) = row
|
||||
return LyricsResult(
|
||||
artist=artist,
|
||||
song=song,
|
||||
lyrics=lyrics,
|
||||
src=f"{original_src} (cached, id: {_id})",
|
||||
confidence=confidence)
|
||||
return None
|
||||
|
||||
async def check_existence(self, artistsong: str) -> Optional[bool]:
|
||||
@ -115,12 +136,44 @@ class Cache:
|
||||
# pylint: enable=unused-argument
|
||||
artist: str = artist.strip().lower()
|
||||
song: str = song.strip().lower()
|
||||
input_track: str = f"{artist} - {song}"
|
||||
search_params: Optional[tuple] = None
|
||||
random_search: bool = False
|
||||
time_start: float = time.time()
|
||||
matcher = utils.TrackMatcher()
|
||||
|
||||
logging.info("Searching %s - %s on %s",
|
||||
artist, song, self.label)
|
||||
artist, song, self.label)
|
||||
|
||||
"""Check Redis First"""
|
||||
|
||||
logging.info("Checking redis cache for %s...",
|
||||
f"{artist} - {song}")
|
||||
redis_result = await self.redis_cache.search(artist=artist,
|
||||
song=song)
|
||||
|
||||
if redis_result:
|
||||
result_tracks: list = []
|
||||
for track in redis_result:
|
||||
result_tracks.append((track['id'], f"{track['artist']} - {track['song']}"))
|
||||
|
||||
best_match: tuple|None = matcher.find_best_match(input_track=input_track,
|
||||
candidate_tracks=result_tracks)
|
||||
(candidate, confidence) = best_match
|
||||
matched = self.get_matched(redis_results=redis_result, matched_candidate=candidate,
|
||||
confidence=confidence)
|
||||
time_end: float = time.time()
|
||||
time_diff: float = time_end - time_start
|
||||
matched.confidence = confidence
|
||||
matched.time = time_diff
|
||||
|
||||
if matched:
|
||||
logging.info("Found %s on redis cache, skipping SQLite...",
|
||||
f"{artist} - {song}")
|
||||
return matched
|
||||
|
||||
"""SQLite: Fallback"""
|
||||
|
||||
async with sqlite3.connect(self.cache_db, timeout=2) as db_conn:
|
||||
await db_conn.enable_load_extension(True)
|
||||
for ext in self.sqlite_exts:
|
||||
@ -142,8 +195,6 @@ class Cache:
|
||||
for track in results:
|
||||
(_id, _artist, _song, _lyrics, _src, _confidence) = track
|
||||
result_tracks.append((_id, f"{_artist} - {_song}"))
|
||||
input_track: str = f"{artist} - {song}"
|
||||
matcher = utils.TrackMatcher()
|
||||
if not random_search:
|
||||
best_match: tuple|None = matcher.find_best_match(input_track=input_track,
|
||||
candidate_tracks=result_tracks)
|
||||
@ -161,6 +212,5 @@ class Cache:
|
||||
matched.time = time_diff
|
||||
return matched
|
||||
except:
|
||||
if log_level == "DEBUG":
|
||||
traceback.print_exc()
|
||||
traceback.print_exc()
|
||||
return
|
||||
73
lyric_search_new/sources/redis_cache.py
Normal file
73
lyric_search_new/sources/redis_cache.py
Normal file
@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3.12
|
||||
# pylint: disable=bare-except, broad-exception-caught
|
||||
|
||||
|
||||
import logging
|
||||
import traceback
|
||||
import json
|
||||
import redis.asyncio as redis
|
||||
from redis.commands.json.path import Path
|
||||
from redis.commands.search.query import NumericFilter, Query
|
||||
from redis.commands.search.indexDefinition import IndexDefinition, IndexType
|
||||
from redis.commands.search.field import TextField, NumericField, TagField
|
||||
from . import private
|
||||
|
||||
|
||||
logger = logging.getLogger()
|
||||
log_level = logging.getLevelName(logger.level)
|
||||
|
||||
class RedisException(Exception):
|
||||
"""
|
||||
Redis Exception
|
||||
"""
|
||||
|
||||
class RedisCache:
|
||||
"""
|
||||
Redis Cache Methods
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.redis_pw = private.REDIS_PW
|
||||
self.redis_client = redis.Redis(password=self.redis_pw)
|
||||
|
||||
async def create_index(self):
|
||||
"""Create Index"""
|
||||
try:
|
||||
schema = (
|
||||
TextField("$.artist", as_name="artist"),
|
||||
TextField("$.song", as_name="song"),
|
||||
TextField("$.src", as_name="src"),
|
||||
TextField("$.lyrics", as_name="lyrics")
|
||||
)
|
||||
result = await self.redis_client.ft().create_index(schema, definition=IndexDefinition(prefix=["lyrics:"], index_type=IndexType.JSON))
|
||||
if str(result) != "OK":
|
||||
raise RedisException(f"Redis: Failed to create index: {result}")
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
async def search(self, **kwargs):
|
||||
"""Search Redis Cache
|
||||
@artist: artist to search
|
||||
@song: song to search
|
||||
@lyrics: lyrics to search (optional, used in place of artist/song if provided)
|
||||
"""
|
||||
|
||||
try:
|
||||
artist = kwargs.get('artist')
|
||||
song = kwargs.get('song')
|
||||
lyrics = kwargs.get('lyrics')
|
||||
|
||||
if lyrics:
|
||||
# to code later
|
||||
raise RedisException("Lyric search not yet implemented")
|
||||
|
||||
search_res = await self.redis_client.ft().search(
|
||||
Query(f"@artist:{artist} @song:{song}"
|
||||
))
|
||||
search_res_out = [dict(json.loads(result['json']))
|
||||
for result in search_res.docs]
|
||||
|
||||
return search_res_out
|
||||
except:
|
||||
traceback.print_exc()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user