codey/api
1
0
Fork 0
Code Issues Pull Requests Activity

- Implemented a new script to upsert data from SQLite dumps into PostgreSQL asynchronously.

Browse Source 
- Added detailed reporting for each table processed, including success and error statuses.
- Integrated Discord notifications for various stages of the upsert process.
- Included functionality to fetch the latest SQLite dump from a specified URL.
- Added error handling and state management for the upsert process.

Resolves #34
This commit is contained in:
codey
2026-01-25 22:06:24 -05:00
parent 97fd7dd67d
commit 277804d212
8 changed files with 2136 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
lyric_search/sources/cache.py
View File

@@ -91,8 +91,10 @@ class Cache:
logging.debug(
"Checking whether %s is already stored", artistsong.replace("\n", " - ")
)
check_query: str = 'SELECT id, artist, song FROM lyrics WHERE editdist3((lower(artist) || " " || lower(song)), (? || " " || ?))\
check_query: str = (
'SELECT id, artist, song FROM lyrics WHERE editdist3((lower(artist) || " " || lower(song)), (? || " " || ?))\
<= 410 ORDER BY editdist3((lower(artist) || " " || lower(song)), ?) ASC LIMIT 1'
)
artistsong_split = artistsong.split("\n", maxsplit=1)
artist = artistsong_split[0].lower()
song = artistsong_split[1].lower()
@@ -213,10 +215,8 @@ class Cache:
lyrics = regex.sub(r"(<br>|\n|\r\n)", " / ", lyr_result.lyrics.strip())
lyrics = regex.sub(r"\s{2,}", " ", lyrics)
insert_query = (
"INSERT INTO lyrics (src, date_retrieved, artist, song, artistsong, confidence, lyrics)\
insert_query = "INSERT INTO lyrics (src, date_retrieved, artist, song, artistsong, confidence, lyrics)\
VALUES(?, ?, ?, ?, ?, ?, ?)"
)
params = (
lyr_result.src,
time.time(),
@@ -260,8 +260,10 @@ class Cache:
if artist == "!" and song == "!":
random_search = True
search_query: str = "SELECT id, artist, song, lyrics, src, confidence\
search_query: str = (
"SELECT id, artist, song, lyrics, src, confidence\
FROM lyrics ORDER BY RANDOM() LIMIT 1"
)
logging.info("Searching %s - %s on %s", artist, song, self.label)
@@ -320,9 +322,11 @@ class Cache:
self.cache_pre_query
) as _db_cursor:
if not random_search:
search_query: str = 'SELECT id, artist, song, lyrics, src, confidence FROM lyrics\
search_query: str = (
'SELECT id, artist, song, lyrics, src, confidence FROM lyrics\
WHERE editdist3((lower(artist) || " " || lower(song)), (? || " " || ?))\
<= 410 ORDER BY editdist3((lower(artist) || " " || lower(song)), ?) ASC LIMIT 10'
)
search_params: tuple = (
artist.strip(),
song.strip(),