changes include: allow GET endpoints, redirect to codey.lol instead of returning status 422, clean junk from ai endpoint, add misc endpoint, add methods to lyric_search_new.sources.cache, other misc/cleanup
This commit is contained in:
parent
e95ef3b088
commit
38dbddd297
22
base.py
22
base.py
@ -4,7 +4,7 @@ import importlib
|
||||
import logging
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from lyric_search_new.sources import redis_cache
|
||||
|
||||
@ -34,7 +34,7 @@ origins = [
|
||||
app.add_middleware(CORSMiddleware,
|
||||
allow_origins=origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["POST"],
|
||||
allow_methods=["POST", "GET", "HEAD"],
|
||||
allow_headers=["*"])
|
||||
|
||||
|
||||
@ -49,9 +49,18 @@ Blacklisted routes
|
||||
def disallow_get():
|
||||
return util.get_blocked_response()
|
||||
|
||||
@app.get("/{any:path}")
|
||||
def disallow_get_any(var: Any = None): # pylint: disable=unused-argument
|
||||
return util.get_blocked_response()
|
||||
@app.get("/{path}")
|
||||
def disallow_get_any(request: Request, var: Any = None): # pylint: disable=unused-argument
|
||||
path = request.path_params['path']
|
||||
if not (
|
||||
isinstance(path, str)
|
||||
and
|
||||
path.split("/", maxsplit=1) == "widget"
|
||||
):
|
||||
return util.get_blocked_response()
|
||||
else:
|
||||
logging.info("OK, %s",
|
||||
path)
|
||||
|
||||
@app.post("/")
|
||||
def disallow_base_post():
|
||||
@ -79,9 +88,10 @@ lastfm_endpoints = importlib.import_module("endpoints.lastfm").LastFM(app, util,
|
||||
yt_endpoints = importlib.import_module("endpoints.yt").YT(app, util, constants, glob_state)
|
||||
# Below: XC endpoint(s)
|
||||
xc_endpoints = importlib.import_module("endpoints.xc").XC(app, util, constants, glob_state)
|
||||
|
||||
# Below: Karma endpoint(s)
|
||||
karma_endpoints = importlib.import_module("endpoints.karma").Karma(app, util, constants, glob_state)
|
||||
# Below: Misc endpoints
|
||||
misc_endpoints = importlib.import_module("endpoints.misc").Misc(app, util, constants, glob_state)
|
||||
|
||||
"""
|
||||
End Actionable Routes
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
import logging
|
||||
import traceback
|
||||
import regex
|
||||
import json
|
||||
|
||||
from aiohttp import ClientSession, ClientTimeout
|
||||
from fastapi import FastAPI, Request, HTTPException, BackgroundTasks
|
||||
@ -49,7 +50,8 @@ class AI(FastAPI):
|
||||
}
|
||||
|
||||
for endpoint, handler in self.endpoints.items():
|
||||
app.add_api_route(f"/{endpoint}/", handler, methods=["POST"])
|
||||
app.add_api_route(f"/{endpoint}/", handler, methods=["POST"] if not endpoint == "testy" else ["POST", "GET"])
|
||||
|
||||
|
||||
async def respond_via_webhook(self, data: ValidHookSongRequest, originalRequest: Request):
|
||||
"""Respond via Webhook"""
|
||||
|
||||
77
endpoints/misc.py
Normal file
77
endpoints/misc.py
Normal file
@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3.12
|
||||
# pylint: disable=bare-except, broad-exception-caught, invalid-name
|
||||
|
||||
import logging
|
||||
import traceback
|
||||
import time
|
||||
from fastapi import FastAPI
|
||||
import redis.asyncio as redis
|
||||
from redis.commands.search.query import Query
|
||||
from redis.commands.search.indexDefinition import IndexDefinition, IndexType
|
||||
from redis.commands.search.field import TextField, TagField
|
||||
from redis.commands.json.path import Path
|
||||
from lyric_search_new.sources import private, cache as LyricsCache
|
||||
|
||||
class Misc(FastAPI):
|
||||
"""Misc Endpoints"""
|
||||
def __init__(self, app: FastAPI, my_util, constants, glob_state): # pylint: disable=super-init-not-called
|
||||
self.app = app
|
||||
self.util = my_util
|
||||
self.constants = constants
|
||||
self.glob_state = glob_state
|
||||
self.lyr_cache = LyricsCache.Cache()
|
||||
self.redis_client = redis.Redis(password=private.REDIS_PW)
|
||||
self.endpoints = {
|
||||
"widget/redis": self.homepage_redis_widget,
|
||||
"widget/sqlite": self.homepage_sqlite_widget,
|
||||
}
|
||||
|
||||
for endpoint, handler in self.endpoints.items():
|
||||
app.add_api_route(f"/{endpoint}/", handler, methods=["GET"])
|
||||
|
||||
async def homepage_redis_widget(self) -> dict:
|
||||
"""
|
||||
/widget/redis/
|
||||
Homepage Widget Handler
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
dict
|
||||
"""
|
||||
|
||||
# Measure response time w/ test lyric search
|
||||
time_start: float = time.time() # Start time for response_time
|
||||
test_lyrics_result = await self.redis_client.ft().search("@artist: test @song: test")
|
||||
time_end: float = time.time()
|
||||
# End response time test
|
||||
|
||||
total_keys = await self.redis_client.dbsize()
|
||||
response_time: float = time_end - time_start
|
||||
(_, ci_keys) = await self.redis_client.scan(cursor=0, match="ci_session*", count=10000000)
|
||||
num_ci_keys = len(ci_keys)
|
||||
index_info = await self.redis_client.ft().info()
|
||||
indexed_lyrics = index_info.get('num_docs')
|
||||
return {
|
||||
'responseTime': round(response_time, 7),
|
||||
'storedKeys': total_keys,
|
||||
'indexedLyrics': indexed_lyrics,
|
||||
'sessions': num_ci_keys,
|
||||
}
|
||||
|
||||
async def homepage_sqlite_widget(self) -> dict:
|
||||
"""
|
||||
/widget/sqlite/
|
||||
Homepage Widget Handler
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
dict
|
||||
"""
|
||||
row_count = await self.lyr_cache.sqlite_rowcount()
|
||||
distinct_artists = await self.lyr_cache.sqlite_distinct("artist")
|
||||
lyrics_length = await self.lyr_cache.sqlite_lyrics_length()
|
||||
return {
|
||||
'storedRows': row_count,
|
||||
'distinctArtists': distinct_artists,
|
||||
'lyricsLength': lyrics_length,
|
||||
}
|
||||
@ -110,16 +110,13 @@ class LastFM:
|
||||
async with session.get(f"{self.api_base_url}artist.gettopalbums&artist={artist}&api_key={self.creds.get('key')}&autocorrect=1&format=json",
|
||||
timeout=ClientTimeout(connect=3, sock_read=8)) as request:
|
||||
assert request.status in [200, 204]
|
||||
# return request.text
|
||||
data = await request.json()
|
||||
data = data.get('topalbums').get('album')
|
||||
# print(f"Data:\n{data}")
|
||||
retObj = [
|
||||
{
|
||||
'title': item.get('name')
|
||||
} for item in data if not(item.get('name').lower() == "(null)") and int(item.get('playcount')) >= 50
|
||||
]
|
||||
# # print(f"Keys: {data[0].keys()}")
|
||||
return retObj
|
||||
except:
|
||||
traceback.print_exc()
|
||||
|
||||
@ -50,7 +50,6 @@ class Cache:
|
||||
for res in redis_results:
|
||||
(key, row) = res
|
||||
if key == matched_id:
|
||||
logging.info("Matched row: %s", row)
|
||||
return LyricsResult(
|
||||
artist=row['artist'],
|
||||
song=row['song'],
|
||||
@ -114,6 +113,53 @@ class Cache:
|
||||
await self.redis_cache.redis_store(sqlite_insert_id, lyr_result)
|
||||
|
||||
|
||||
async def sqlite_rowcount(self, where: Optional[str] = None, params: Optional[tuple] = None) -> int:
|
||||
"""
|
||||
Get rowcount for cached_lyrics DB
|
||||
Args:
|
||||
where (Optional[str]): WHERE ext for query if needed
|
||||
params (Optional[tuple]): Parameters to query, if where is specified
|
||||
Returns:
|
||||
int: Number of rows found
|
||||
"""
|
||||
async with sqlite3.connect(self.cache_db, timeout=2) as db_conn:
|
||||
db_conn.row_factory = sqlite3.Row
|
||||
query = f"SELECT count(id) AS rowcount FROM lyrics {where}".strip()
|
||||
async with await db_conn.execute(query, params) as db_cursor:
|
||||
result = await db_cursor.fetchone()
|
||||
return result['rowcount']
|
||||
|
||||
async def sqlite_distinct(self, column: str) -> int:
|
||||
"""
|
||||
Get count of distinct values for a column
|
||||
Args:
|
||||
column (str): The column to check
|
||||
Returns:
|
||||
int: Number of distinct values found
|
||||
"""
|
||||
async with sqlite3.connect(self.cache_db, timeout=2) as db_conn:
|
||||
db_conn.row_factory = sqlite3.Row
|
||||
query = f"SELECT COUNT(DISTINCT {column}) as distinct_items FROM lyrics"
|
||||
async with await db_conn.execute(query) as db_cursor:
|
||||
result = await db_cursor.fetchone()
|
||||
return result['distinct_items']
|
||||
|
||||
async def sqlite_lyrics_length(self) -> int:
|
||||
"""
|
||||
Get total length of text stored for lyrics
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
int: Total length of stored lyrics
|
||||
"""
|
||||
async with sqlite3.connect(self.cache_db, timeout=2) as db_conn:
|
||||
db_conn.row_factory = sqlite3.Row
|
||||
query = "SELECT SUM(LENGTH(lyrics)) as lyrics_len FROM lyrics"
|
||||
async with await db_conn.execute(query) as db_cursor:
|
||||
result = await db_cursor.fetchone()
|
||||
return result['lyrics_len']
|
||||
|
||||
|
||||
async def sqlite_store(self, lyr_result: LyricsResult) -> int:
|
||||
"""
|
||||
Store lyrics to SQLite Cache
|
||||
@ -188,7 +234,7 @@ class Cache:
|
||||
|
||||
"""Check Redis First"""
|
||||
|
||||
logging.info("Checking redis cache for %s...",
|
||||
logging.debug("Checking redis cache for %s...",
|
||||
f"{artist} - {song}")
|
||||
redis_result = await self.redis_cache.search(artist=artist,
|
||||
song=song)
|
||||
|
||||
@ -105,7 +105,7 @@ class RedisCache:
|
||||
if not is_random_search:
|
||||
logging.debug("Redis: Searching normally first")
|
||||
(artist, song) = self.sanitize_input(artist, song)
|
||||
logging.info("Seeking: %s - %s", artist, song)
|
||||
logging.debug("Seeking: %s - %s", artist, song)
|
||||
search_res = await self.redis_client.ft().search(Query(
|
||||
f"@artist:{artist} @song:{song}"
|
||||
))
|
||||
|
||||
13
util.py
13
util.py
@ -3,21 +3,20 @@
|
||||
import logging
|
||||
|
||||
from fastapi import FastAPI, Response, HTTPException
|
||||
from fastapi.responses import RedirectResponse
|
||||
|
||||
|
||||
class Utilities:
|
||||
"""API Utilities"""
|
||||
def __init__(self, app: FastAPI, constants):
|
||||
self.constants = constants
|
||||
self.blocked_response_status = 422
|
||||
self.blocked_response_content = None
|
||||
self.blocked_redirect_uri = "https://codey.lol"
|
||||
self.app = app
|
||||
|
||||
def get_blocked_response(self, path: str | None = None): # pylint: disable=unused-argument
|
||||
"""Get Blocked HTTP Response"""
|
||||
logging.error("Rejected request: Blocked")
|
||||
return Response(content=self.blocked_response_content,
|
||||
status_code=self.blocked_response_status)
|
||||
return RedirectResponse(url=self.blocked_redirect_uri)
|
||||
|
||||
def get_no_endpoint_found(self, path: str | None = None): # pylint: disable=unused-argument
|
||||
"""Get 404 Response"""
|
||||
@ -29,27 +28,21 @@ class Utilities:
|
||||
Accepts path as an argument to allow fine tuning access for each API key, not currently in use.
|
||||
"""
|
||||
|
||||
# print(f"Testing with path: {path}, key: {key}")
|
||||
|
||||
if not key or not key.startswith("Bearer "):
|
||||
return False
|
||||
|
||||
key = key.split("Bearer ", maxsplit=1)[1].strip()
|
||||
|
||||
if not key in self.constants.API_KEYS:
|
||||
# print("Auth failed")
|
||||
return False
|
||||
|
||||
if req_type == 2:
|
||||
if not key.startswith("PRV-"):
|
||||
# print("Auth failed - not a PRV key")
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
if path.lower().startswith("/xc/") and not key.startswith("XC-"):
|
||||
# print("Auth failed - not an XC Key")
|
||||
return False
|
||||
|
||||
# print("Auth succeeded")
|
||||
return True
|
||||
Loading…
x
Reference in New Issue
Block a user