codey/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

radio_util: open tracks SQLite DB in readonly mode; black: reformat files

Browse Source
This commit is contained in:
codey
2025-04-17 07:28:05 -04:00
parent 96add377df
commit 6c88c23a4d
25 changed files with 1913 additions and 1340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
lyric_search/sources/redis_cache.py
View File

@ -7,24 +7,27 @@ import regex
from regex import Pattern
import asyncio
from typing import Union, Optional
sys.path.insert(1,'..')
sys.path.insert(1, "..")
from lyric_search import notifier
from lyric_search.constructors import LyricsResult
import redis.asyncio as redis
from redis.commands.search.query import Query # type: ignore
from redis.commands.search.indexDefinition import IndexDefinition, IndexType # type: ignore
from redis.commands.search.field import TextField, TagField # type: ignore
from redis.commands.json.path import Path # type: ignore
from redis.commands.search.query import Query # type: ignore
from redis.commands.search.indexDefinition import IndexDefinition, IndexType # type: ignore
from redis.commands.search.field import TextField, TagField # type: ignore
from redis.commands.json.path import Path # type: ignore
from . import private
logger = logging.getLogger()
log_level = logging.getLevelName(logger.level)
class RedisException(Exception):
"""
Redis Exception
"""
class RedisCache:
"""
Redis Cache Methods
@ -35,34 +38,37 @@ class RedisCache:
self.notifier = notifier.DiscordNotifier()
self.notify_warnings = False
self.regexes: list[Pattern] = [
regex.compile(r'\-'),
regex.compile(r'[^a-zA-Z0-9\s]'),
regex.compile(r"\-"),
regex.compile(r"[^a-zA-Z0-9\s]"),
]
try:
asyncio.get_event_loop().create_task(self.create_index())
except Exception as e:
logging.debug("Failed to create redis create_index task: %s",
str(e))
logging.debug("Failed to create redis create_index task: %s", str(e))
async def create_index(self) -> None:
"""Create Index"""
try:
schema = (
TextField("$.search_artist", as_name="artist"),
TextField("$.search_song", as_name="song"),
TextField("$.src", as_name="src"),
TextField("$.lyrics", as_name="lyrics")
)
TextField("$.search_artist", as_name="artist"),
TextField("$.search_song", as_name="song"),
TextField("$.src", as_name="src"),
TextField("$.lyrics", as_name="lyrics"),
)
result = await self.redis_client.ft().create_index(
schema, definition=IndexDefinition(prefix=["lyrics:"], index_type=IndexType.JSON))
schema,
definition=IndexDefinition(
prefix=["lyrics:"], index_type=IndexType.JSON
),
)
if str(result) != "OK":
raise RedisException(f"Redis: Failed to create index: {result}")
except Exception as e:
logging.debug("Failed to create redis index: %s",
str(e))
def sanitize_input(self, artist: str, song: str,
fuzzy: Optional[bool] = False) -> tuple[str, str]:
logging.debug("Failed to create redis index: %s", str(e))
def sanitize_input(
self, artist: str, song: str, fuzzy: Optional[bool] = False
) -> tuple[str, str]:
"""
Sanitize artist/song input (convert to redis matchable fuzzy query)
Args:
@ -77,10 +83,12 @@ class RedisCache:
song = self.regexes[0].sub("", song)
song = self.regexes[1].sub("", song).strip()
if fuzzy:
artist = " ".join([f"(%{artist_word}%)" for artist_word in artist.split(" ")])
artist = " ".join(
[f"(%{artist_word}%)" for artist_word in artist.split(" ")]
)
song = " ".join([f"(%{song_word}%)" for song_word in song.split(" ")])
return (artist, song)
async def increment_found_count(self, src: str) -> None:
"""
Increment the found count for a source
@ -94,13 +102,13 @@ class RedisCache:
await self.redis_client.incr(f"returned:{src}")
except Exception as e:
file: str = __file__.rsplit("/", maxsplit=1)[-1]
await self.notifier.send(f"ERROR @ {file}", str(e))
await self.notifier.send(f"ERROR @ {file}", str(e))
traceback.print_exc()
async def get_found_counts(self) -> Optional[dict]:
"""
Get found counts for all sources (and failed count)
Returns:
dict: In the form {'source': count, 'source2': count, ...}
"""
@ -109,18 +117,20 @@ class RedisCache:
counts: dict[str, int] = {}
for src in sources:
src_found_count = await self.redis_client.get(f"returned:{src}")
counts[src] = int(src_found_count) # Redis returns bytes
counts[src] = int(src_found_count) # Redis returns bytes
return counts
except Exception as e:
file: str = __file__.rsplit("/", maxsplit=1)[-1]
await self.notifier.send(f"ERROR @ {file}", str(e))
await self.notifier.send(f"ERROR @ {file}", str(e))
traceback.print_exc()
return None
async def search(self, artist: Optional[str] = None,
song: Optional[str] = None,
lyrics: Optional[str] = None) -> Optional[list[tuple]]:
async def search(
self,
artist: Optional[str] = None,
song: Optional[str] = None,
lyrics: Optional[str] = None,
) -> Optional[list[tuple]]:
"""
Search Redis Cache
Args:
@ -133,57 +143,72 @@ class RedisCache:
try:
fuzzy_artist = None
fuzzy_song = None
fuzzy_song = None
is_random_search = artist == "!" and song == "!"
if lyrics:
# to code later
raise RedisException("Lyric search not yet implemented")
if not is_random_search:
logging.debug("Redis: Searching normally first")
if not artist or not song:
logging.info("redis_cache:: search failed: No artist or song provided.")
logging.info(
"redis_cache:: search failed: No artist or song provided."
)
return None
(artist, song) = self.sanitize_input(artist, song)
logging.debug("Seeking: %s - %s", artist, song)
search_res: Union[dict, list] = await self.redis_client.ft().search(Query( # type: ignore
f"@artist:{artist} @song:{song}"
))
search_res_out: list[tuple] = [(result['id'].split(":",
maxsplit=1)[1], dict(json.loads(result['json'])))
for result in search_res.docs] # type: ignore
search_res: Union[dict, list] = await self.redis_client.ft().search(
Query(f"@artist:{artist} @song:{song}") # type: ignore
)
search_res_out: list[tuple] = [
(
result["id"].split(":", maxsplit=1)[1],
dict(json.loads(result["json"])),
)
for result in search_res.docs
] # type: ignore
if not search_res_out:
logging.debug("Redis: Normal search failed, trying with fuzzy search")
logging.debug(
"Redis: Normal search failed, trying with fuzzy search"
)
short_artist = " ".join(artist.split(" ")[0:5])
short_song = " ".join(song.split(" ")[0:5])
(fuzzy_artist, fuzzy_song) = self.sanitize_input(artist=short_artist.strip(),
song=short_song.strip(), fuzzy=True)
search_res = await self.redis_client.ft().search(Query( # type: ignore
f"@artist:{fuzzy_artist} @song:{fuzzy_song}"
))
search_res_out = [(result['id'].split(":",
maxsplit=1)[1], dict(json.loads(result['json'])))
for result in search_res.docs] # type: ignore
short_song = " ".join(song.split(" ")[0:5])
(fuzzy_artist, fuzzy_song) = self.sanitize_input(
artist=short_artist.strip(), song=short_song.strip(), fuzzy=True
)
search_res = await self.redis_client.ft().search(
Query( # type: ignore
f"@artist:{fuzzy_artist} @song:{fuzzy_song}"
)
)
search_res_out = [
(
result["id"].split(":", maxsplit=1)[1],
dict(json.loads(result["json"])),
)
for result in search_res.docs
] # type: ignore
else:
random_redis_key: str = await self.redis_client.randomkey()
out_id: str = str(random_redis_key).split(":",
maxsplit=1)[1][:-1]
out_id: str = str(random_redis_key).split(":", maxsplit=1)[1][:-1]
search_res = await self.redis_client.json().get(random_redis_key)
search_res_out = [(out_id, search_res)]
if not search_res_out and self.notify_warnings:
await self.notifier.send("WARNING", f"Redis cache miss for: `{artist} - {song}`")
await self.notifier.send(
"WARNING", f"Redis cache miss for: `{artist} - {song}`"
)
return search_res_out
except Exception as e:
traceback.print_exc()
# await self.notifier.send(f"ERROR @ {__file__.rsplit("/", maxsplit=1)[-1]}", f"{str(e)}\nSearch was: {artist} - {song}; fuzzy: {fuzzy_artist} - {fuzzy_song}")
# await self.notifier.send(f"ERROR @ {__file__.rsplit("/", maxsplit=1)[-1]}", f"{str(e)}\nSearch was: {artist} - {song}; fuzzy: {fuzzy_artist} - {fuzzy_song}")
return None
async def redis_store(self, sqlite_id: int,
lyr_result: LyricsResult) -> None:
async def redis_store(self, sqlite_id: int, lyr_result: LyricsResult) -> None:
"""
Store lyrics to redis cache
Args:
@ -193,34 +218,47 @@ class RedisCache:
None
"""
try:
(search_artist, search_song) = self.sanitize_input(lyr_result.artist,
lyr_result.song)
(search_artist, search_song) = self.sanitize_input(
lyr_result.artist, lyr_result.song
)
redis_mapping: dict = {
'id': sqlite_id,
'src': lyr_result.src,
'date_retrieved': time.time(),
'artist': lyr_result.artist,
'search_artist': search_artist,
'search_song': search_song,
'search_artistsong': f'{search_artist}\n{search_song}',
'song': lyr_result.song,
'artistsong': f"{lyr_result.artist}\n{lyr_result.song}",
'confidence': lyr_result.confidence,
'lyrics': lyr_result.lyrics,
'tags': '(none)',
'liked': 0,
}
"id": sqlite_id,
"src": lyr_result.src,
"date_retrieved": time.time(),
"artist": lyr_result.artist,
"search_artist": search_artist,
"search_song": search_song,
"search_artistsong": f"{search_artist}\n{search_song}",
"song": lyr_result.song,
"artistsong": f"{lyr_result.artist}\n{lyr_result.song}",
"confidence": lyr_result.confidence,
"lyrics": lyr_result.lyrics,
"tags": "(none)",
"liked": 0,
}
newkey: str = f"lyrics:000{sqlite_id}"
jsonset: bool = await self.redis_client.json().set(newkey, Path.root_path(),
redis_mapping)
jsonset: bool = await self.redis_client.json().set(
newkey, Path.root_path(), redis_mapping
)
if not jsonset:
raise RedisException(f"Failed to store {lyr_result.artist} - {lyr_result.song} (SQLite id: {sqlite_id}) to redis:\n{jsonset}")
logging.info("Stored %s - %s (related SQLite Row ID: %s) to %s",
lyr_result.artist, lyr_result.song, sqlite_id, newkey)
await self.notifier.send("INFO",
f"Stored `{lyr_result.artist} - {lyr_result.song}` (related SQLite Row ID: `{sqlite_id}`) to redis: `{newkey}`")
raise RedisException(
f"Failed to store {lyr_result.artist} - {lyr_result.song} (SQLite id: {sqlite_id}) to redis:\n{jsonset}"
)
logging.info(
"Stored %s - %s (related SQLite Row ID: %s) to %s",
lyr_result.artist,
lyr_result.song,
sqlite_id,
newkey,
)
await self.notifier.send(
"INFO",
f"Stored `{lyr_result.artist} - {lyr_result.song}` (related SQLite Row ID: `{sqlite_id}`) to redis: `{newkey}`",
)
except Exception as e:
file: str = __file__.rsplit("/", maxsplit=1)[-1]
await self.notifier.send(f"ERROR @ {file}",
f"Failed to store `{lyr_result.artist} - {lyr_result.song}`\
(SQLite id: `{sqlite_id}`) to Redis:\n`{str(e)}`")
await self.notifier.send(
f"ERROR @ {file}",
f"Failed to store `{lyr_result.artist} - {lyr_result.song}`\
(SQLite id: `{sqlite_id}`) to Redis:\n`{str(e)}`",
)