JDM170/spotizerr-dev
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Change watchlist behaviour. It now updates progressively based on maxItemsPerRun and runs a batch on intervals determined by watchPollInterval

Browse Source
This commit is contained in:
Xoconoch
2025-08-21 11:16:34 -05:00
parent 96b26bac6a
commit 0456c25830
15 changed files with 963 additions and 1176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
routes/__init__.py
View File

@@ -1,36 +1,7 @@
import logging
import atexit
# Configure basic logging for the application if not already configured
# This is a good place for it if routes are a central part of your app structure.
logging.basicConfig(
level=logging.INFO, format="%(message)s"
)
# This remains safe to execute on import
logging.basicConfig(level=logging.INFO, format="%(message)s")
logger = logging.getLogger(__name__)
# Run DB migrations early so other modules see expected schemas
try:
from routes.migrations import run_migrations_if_needed
run_migrations_if_needed()
logger.info("Database migrations executed (if needed).")
except Exception as e:
logger.error(f"Database migration step failed: {e}", exc_info=True)
try:
from routes.utils.watch.manager import start_watch_manager, stop_watch_manager
# Start the playlist watch manager when the application/blueprint is initialized
start_watch_manager()
# Register the stop function to be called on application exit
atexit.register(stop_watch_manager)
logger.info("Playlist Watch Manager initialized and registered for shutdown.")
except ImportError as e:
logger.error(
f"Could not import or start Playlist Watch Manager: {e}. Playlist watching will be disabled."
)
except Exception as e:
logger.error(
f"An unexpected error occurred during Playlist Watch Manager setup: {e}",
exc_info=True,
)

190
routes/migrations/runner.py
View File

@@ -3,10 +3,7 @@ import sqlite3
from pathlib import Path
from typing import Optional
from .v3_0_6 import MigrationV3_0_6
from .v3_1_0 import MigrationV3_1_0
from .v3_1_1 import MigrationV3_1_1
from .v3_1_2 import MigrationV3_1_2
from .v3_2_0 import MigrationV3_2_0
logger = logging.getLogger(__name__)
@@ -41,7 +38,7 @@ CHILDREN_EXPECTED_COLUMNS: dict[str, str] = {
"metadata": "TEXT",
}
# 3.1.2 expected schemas for Watch DBs (kept here to avoid importing modules with side-effects)
# 3.2.0 expected schemas for Watch DBs (kept here to avoid importing modules with side-effects)
EXPECTED_WATCHED_PLAYLISTS_COLUMNS: dict[str, str] = {
"spotify_id": "TEXT PRIMARY KEY",
"name": "TEXT",
@@ -103,10 +100,7 @@ EXPECTED_ARTIST_ALBUMS_COLUMNS: dict[str, str] = {
"is_fully_downloaded_managed_by_app": "INTEGER DEFAULT 0",
}
m306 = MigrationV3_0_6()
m310 = MigrationV3_1_0()
m311 = MigrationV3_1_1()
m312 = MigrationV3_1_2()
m320 = MigrationV3_2_0()
def _safe_connect(path: Path) -> Optional[sqlite3.Connection]:
@@ -184,60 +178,53 @@ def _create_or_update_children_table(conn: sqlite3.Connection, table_name: str)
)
def _update_children_tables_for_history(conn: sqlite3.Connection) -> None:
# --- Helper to validate instance is at least 3.1.2 on history DB ---
def _history_children_tables(conn: sqlite3.Connection) -> list[str]:
tables: set[str] = set()
try:
try:
cur = conn.execute(
"SELECT DISTINCT children_table FROM download_history WHERE children_table IS NOT NULL AND TRIM(children_table) != ''"
)
for row in cur.fetchall():
table_name = row[0]
if not table_name:
continue
_create_or_update_children_table(conn, table_name)
except sqlite3.Error as e:
logger.warning(
f"Failed to scan referenced children tables from main history: {e}"
)
try:
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND (name LIKE 'album_%' OR name LIKE 'playlist_%') AND name != 'download_history'"
)
for row in cur.fetchall():
table_name = row[0]
_create_or_update_children_table(conn, table_name)
except sqlite3.Error as e:
logger.warning(f"Failed to scan legacy children tables in history DB: {e}")
logger.info("Children history tables migration ensured")
except Exception:
logger.error("Failed migrating children history tables", exc_info=True)
def _ensure_creds_filesystem() -> None:
try:
BLOBS_DIR.mkdir(parents=True, exist_ok=True)
if not SEARCH_JSON.exists():
SEARCH_JSON.write_text(
'{ "client_id": "", "client_secret": "" }\n', encoding="utf-8"
)
logger.info(f"Created default global Spotify creds file at {SEARCH_JSON}")
except Exception:
logger.error(
"Failed to ensure credentials filesystem (blobs/search.json)", exc_info=True
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND (name LIKE 'album_%' OR name LIKE 'playlist_%') AND name != 'download_history'"
)
for row in cur.fetchall():
if row and row[0]:
tables.add(row[0])
except sqlite3.Error as e:
logger.warning(f"Failed to scan sqlite_master for children tables: {e}")
try:
cur = conn.execute(
"SELECT DISTINCT children_table FROM download_history WHERE children_table IS NOT NULL AND TRIM(children_table) != ''"
)
for row in cur.fetchall():
t = row[0]
if t:
tables.add(t)
except sqlite3.Error as e:
logger.warning(f"Failed to scan download_history for children tables: {e}")
return sorted(tables)
def _apply_versioned_updates(
conn: sqlite3.Connection, c_base, u_base, post_update=None
) -> None:
if not c_base(conn):
u_base(conn)
if post_update:
post_update(conn)
def _is_history_at_least_3_2_0(conn: sqlite3.Connection) -> bool:
required_cols = {"service", "quality_format", "quality_bitrate"}
tables = _history_children_tables(conn)
if not tables:
# Nothing to migrate implies OK
return True
for t in tables:
try:
cur = conn.execute(f"PRAGMA table_info({t})")
cols = {row[1] for row in cur.fetchall()}
if not required_cols.issubset(cols):
return False
except sqlite3.OperationalError:
return False
return True
# --- 3.1.2 upgrade helpers for Watch DBs ---
# --- 3.2.0 verification helpers for Watch DBs ---
def _update_watch_playlists_db(conn: sqlite3.Connection) -> None:
@@ -298,10 +285,10 @@ def _update_watch_playlists_db(conn: sqlite3.Connection) -> None:
EXPECTED_PLAYLIST_TRACKS_COLUMNS,
f"playlist tracks ({table_name})",
)
logger.info("Upgraded watch playlists DB to 3.1.2 schema")
logger.info("Upgraded watch playlists DB to 3.2.0 base schema")
except Exception:
logger.error(
"Failed to upgrade watch playlists DB to 3.1.2 schema", exc_info=True
"Failed to upgrade watch playlists DB to 3.2.0 base schema", exc_info=True
)
@@ -361,10 +348,24 @@ def _update_watch_artists_db(conn: sqlite3.Connection) -> None:
EXPECTED_ARTIST_ALBUMS_COLUMNS,
f"artist albums ({table_name})",
)
logger.info("Upgraded watch artists DB to 3.1.2 schema")
logger.info("Upgraded watch artists DB to 3.2.0 base schema")
except Exception:
logger.error(
"Failed to upgrade watch artists DB to 3.1.2 schema", exc_info=True
"Failed to upgrade watch artists DB to 3.2.0 base schema", exc_info=True
)
def _ensure_creds_filesystem() -> None:
try:
BLOBS_DIR.mkdir(parents=True, exist_ok=True)
if not SEARCH_JSON.exists():
SEARCH_JSON.write_text(
'{ "client_id": "", "client_secret": "" }\n', encoding="utf-8"
)
logger.info(f"Created default global Spotify creds file at {SEARCH_JSON}")
except Exception:
logger.error(
"Failed to ensure credentials filesystem (blobs/search.json)", exc_info=True
)
@@ -374,75 +375,42 @@ def run_migrations_if_needed():
return
try:
# History DB
with _safe_connect(HISTORY_DB) as conn:
if conn:
_apply_versioned_updates(
conn,
m306.check_history,
m306.update_history,
post_update=_update_children_tables_for_history,
# Require instance to be at least 3.2.0 on history DB; otherwise abort
with _safe_connect(HISTORY_DB) as history_conn:
if history_conn and not _is_history_at_least_3_2_0(history_conn):
logger.error(
"Instance is not at schema version 3.2.0. Please upgrade to 3.2.0 before applying 3.2.1."
)
raise RuntimeError(
"Instance is not at schema version 3.2.0. Please upgrade to 3.2.0 before applying 3.2.1."
)
_apply_versioned_updates(conn, m311.check_history, m311.update_history)
_apply_versioned_updates(conn, m312.check_history, m312.update_history)
conn.commit()
# Watch playlists DB
with _safe_connect(PLAYLISTS_DB) as conn:
if conn:
_apply_versioned_updates(
conn,
m306.check_watch_playlists,
m306.update_watch_playlists,
)
_apply_versioned_updates(
conn,
m311.check_watch_playlists,
m311.update_watch_playlists,
)
_apply_versioned_updates(
conn,
m312.check_watch_playlists,
m312.update_watch_playlists,
)
_update_watch_playlists_db(conn)
# Apply 3.2.0 additions (batch progress columns)
if not m320.check_watch_playlists(conn):
m320.update_watch_playlists(conn)
conn.commit()
# Watch artists DB
# Watch artists DB (if exists)
if ARTISTS_DB.exists():
with _safe_connect(ARTISTS_DB) as conn:
if conn:
_apply_versioned_updates(
conn, m306.check_watch_artists, m306.update_watch_artists
)
_apply_versioned_updates(
conn, m310.check_watch_artists, m310.update_watch_artists
)
_apply_versioned_updates(
conn, m311.check_watch_artists, m311.update_watch_artists
)
_apply_versioned_updates(
conn, m312.check_watch_artists, m312.update_watch_artists
)
_update_watch_artists_db(conn)
if not m320.check_watch_artists(conn):
m320.update_watch_artists(conn)
conn.commit()
# Accounts DB
# Accounts DB (no changes for this migration path)
with _safe_connect(ACCOUNTS_DB) as conn:
if conn:
_apply_versioned_updates(
conn, m306.check_accounts, m306.update_accounts
)
_apply_versioned_updates(
conn, m311.check_accounts, m311.update_accounts
)
_apply_versioned_updates(
conn, m312.check_accounts, m312.update_accounts
)
conn.commit()
except Exception as e:
logger.error("Error during migration: %s", e, exc_info=True)
raise
else:
_ensure_creds_filesystem()
logger.info("Database migrations check completed")
logger.info("Database migrations check completed (3.2.0 -> 3.2.1 path)")

201
routes/migrations/v3_0_6.py
View File

@@ -1,201 +0,0 @@
import sqlite3
class MigrationV3_0_6:
HISTORY_SQL = """
CREATE TABLE IF NOT EXISTS download_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
download_type TEXT NOT NULL,
title TEXT NOT NULL,
artists TEXT,
timestamp REAL NOT NULL,
status TEXT NOT NULL,
service TEXT,
quality_format TEXT,
quality_bitrate TEXT,
total_tracks INTEGER,
successful_tracks INTEGER,
failed_tracks INTEGER,
skipped_tracks INTEGER,
children_table TEXT,
task_id TEXT,
external_ids TEXT,
metadata TEXT,
release_date TEXT,
genres TEXT,
images TEXT,
owner TEXT,
album_type TEXT,
duration_total_ms INTEGER,
explicit BOOLEAN
);
CREATE INDEX IF NOT EXISTS idx_download_history_timestamp ON download_history(timestamp);
CREATE INDEX IF NOT EXISTS idx_download_history_type_status ON download_history(download_type, status);
CREATE INDEX IF NOT EXISTS idx_download_history_task_id ON download_history(task_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_download_history_task_type_ids ON download_history(task_id, download_type, external_ids);
"""
WATCH_PLAYLISTS_SQL = """
CREATE TABLE IF NOT EXISTS watched_playlists (
spotify_id TEXT PRIMARY KEY,
name TEXT,
owner_id TEXT,
owner_name TEXT,
total_tracks INTEGER,
link TEXT,
snapshot_id TEXT,
last_checked INTEGER,
added_at INTEGER,
is_active INTEGER DEFAULT 1
);
"""
WATCH_ARTISTS_SQL = """
CREATE TABLE IF NOT EXISTS watched_artists (
spotify_id TEXT PRIMARY KEY,
name TEXT,
link TEXT,
total_albums_on_spotify INTEGER,
last_checked INTEGER,
added_at INTEGER,
is_active INTEGER DEFAULT 1,
genres TEXT,
popularity INTEGER,
image_url TEXT
);
"""
ACCOUNTS_SPOTIFY_SQL = """
CREATE TABLE IF NOT EXISTS spotify (
name TEXT PRIMARY KEY,
region TEXT,
created_at REAL,
updated_at REAL
);
"""
ACCOUNTS_DEEZER_SQL = """
CREATE TABLE IF NOT EXISTS deezer (
name TEXT PRIMARY KEY,
arl TEXT,
region TEXT,
created_at REAL,
updated_at REAL
);
"""
@staticmethod
def _table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
try:
cur = conn.execute(f"PRAGMA table_info({table})")
return {row[1] for row in cur.fetchall()}
except Exception:
return set()
# --- Checks ---
def check_history(self, conn: sqlite3.Connection) -> bool:
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='download_history'"
)
if not cur.fetchone():
return False
required = {
"id",
"download_type",
"title",
"artists",
"timestamp",
"status",
"service",
"quality_format",
"quality_bitrate",
"total_tracks",
"successful_tracks",
"failed_tracks",
"skipped_tracks",
"children_table",
"task_id",
"external_ids",
"metadata",
"release_date",
"genres",
"images",
"owner",
"album_type",
"duration_total_ms",
"explicit",
}
return required.issubset(self._table_columns(conn, "download_history"))
def check_watch_playlists(self, conn: sqlite3.Connection) -> bool:
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='watched_playlists'"
)
if not cur.fetchone():
return False
required = {
"spotify_id",
"name",
"owner_id",
"owner_name",
"total_tracks",
"link",
"snapshot_id",
"last_checked",
"added_at",
"is_active",
}
return required.issubset(self._table_columns(conn, "watched_playlists"))
def check_watch_artists(self, conn: sqlite3.Connection) -> bool:
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='watched_artists'"
)
if not cur.fetchone():
return False
required = {
"spotify_id",
"name",
"link",
"total_albums_on_spotify",
"last_checked",
"added_at",
"is_active",
"genres",
"popularity",
"image_url",
}
return required.issubset(self._table_columns(conn, "watched_artists"))
def check_accounts(self, conn: sqlite3.Connection) -> bool:
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='spotify'"
)
if not cur.fetchone():
return False
if not {"name", "region", "created_at", "updated_at"}.issubset(
self._table_columns(conn, "spotify")
):
return False
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='deezer'"
)
if not cur.fetchone():
return False
return {"name", "arl", "region", "created_at", "updated_at"}.issubset(
self._table_columns(conn, "deezer")
)
# --- Updates ---
def update_history(self, conn: sqlite3.Connection) -> None:
conn.executescript(self.HISTORY_SQL)
def update_watch_playlists(self, conn: sqlite3.Connection) -> None:
conn.executescript(self.WATCH_PLAYLISTS_SQL)
def update_watch_artists(self, conn: sqlite3.Connection) -> None:
conn.executescript(self.WATCH_ARTISTS_SQL)
def update_accounts(self, conn: sqlite3.Connection) -> None:
conn.executescript(self.ACCOUNTS_SPOTIFY_SQL)
conn.executescript(self.ACCOUNTS_DEEZER_SQL)

88
routes/migrations/v3_1_0.py
View File

@@ -1,88 +0,0 @@
import sqlite3
import logging
logger = logging.getLogger(__name__)
class MigrationV3_1_0:
ARTIST_ALBUMS_EXPECTED_COLUMNS: dict[str, str] = {
"album_spotify_id": "TEXT PRIMARY KEY",
"artist_spotify_id": "TEXT",
"name": "TEXT",
"album_group": "TEXT",
"album_type": "TEXT",
"release_date": "TEXT",
"release_date_precision": "TEXT",
"total_tracks": "INTEGER",
"link": "TEXT",
"image_url": "TEXT",
"added_to_db": "INTEGER",
"last_seen_on_spotify": "INTEGER",
"download_task_id": "TEXT",
"download_status": "INTEGER DEFAULT 0",
"is_fully_downloaded_managed_by_app": "INTEGER DEFAULT 0",
}
def _table_columns(self, conn: sqlite3.Connection, table: str) -> set[str]:
try:
cur = conn.execute(f"PRAGMA table_info({table})")
return {row[1] for row in cur.fetchall()}
except sqlite3.OperationalError:
return set()
def check_watch_artists(self, conn: sqlite3.Connection) -> bool:
"""Checks if the artist-specific tables have the new columns."""
try:
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'artist_%' LIMIT 1"
)
first_artist_table = cur.fetchone()
if not first_artist_table:
return True # No artist tables, so no migration needed
table_name = first_artist_table[0]
existing_columns = self._table_columns(conn, table_name)
required_columns = self.ARTIST_ALBUMS_EXPECTED_COLUMNS.keys()
return set(required_columns).issubset(existing_columns)
except Exception as e:
logger.error(f"Error checking artist watch DB schema: {e}")
return False
def update_watch_artists(self, conn: sqlite3.Connection) -> None:
"""Updates all artist-specific tables with new columns."""
try:
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'artist_%'"
)
artist_tables = cur.fetchall()
for row in artist_tables:
table_name = row[0]
existing_columns = self._table_columns(conn, table_name)
for col_name, col_type in self.ARTIST_ALBUMS_EXPECTED_COLUMNS.items():
if col_name in existing_columns:
continue
try:
# Remove constraints for ADD COLUMN
col_type_for_add = (
col_type.replace("PRIMARY KEY", "")
.replace("AUTOINCREMENT", "")
.replace("NOT NULL", "")
.strip()
)
conn.execute(
f'ALTER TABLE "{table_name}" ADD COLUMN {col_name} {col_type_for_add}'
)
logger.info(
f"Added column '{col_name}' to table '{table_name}' in artists.db."
)
except sqlite3.OperationalError as e:
logger.warning(
f"Could not add column '{col_name}' to table '{table_name}': {e}"
)
except Exception as e:
logger.error(f"Failed to update artist watch DB: {e}", exc_info=True)

42
routes/migrations/v3_1_1.py
View File

@@ -1,42 +0,0 @@
import sqlite3
class MigrationV3_1_1:
"""
Dummy migration for version 3.1.1 to 3.1.2.
No database schema changes were made between these versions.
This class serves as a placeholder to ensure the migration runner
is aware of this version and can proceed without errors.
"""
def check_history(self, conn: sqlite3.Connection) -> bool:
# No changes, so migration is not needed.
return True
def update_history(self, conn: sqlite3.Connection) -> None:
# No-op
pass
def check_watch_artists(self, conn: sqlite3.Connection) -> bool:
# No changes, so migration is not needed.
return True
def update_watch_artists(self, conn: sqlite3.Connection) -> None:
# No-op
pass
def check_watch_playlists(self, conn: sqlite3.Connection) -> bool:
# No changes, so migration is not needed.
return True
def update_watch_playlists(self, conn: sqlite3.Connection) -> None:
# No-op
pass
def check_accounts(self, conn: sqlite3.Connection) -> bool:
# No changes, so migration is not needed.
return True
def update_accounts(self, conn: sqlite3.Connection) -> None:
# No-op
pass

103
routes/migrations/v3_1_2.py
View File

@@ -1,103 +0,0 @@
import sqlite3
import logging
logger = logging.getLogger(__name__)
class MigrationV3_1_2:
"""
Migration for version 3.1.2.
Ensure history children tables (album_*/playlist_*) include service and quality columns.
"""
CHILDREN_EXTRA_COLUMNS: dict[str, str] = {
"service": "TEXT",
"quality_format": "TEXT",
"quality_bitrate": "TEXT",
}
def _table_columns(self, conn: sqlite3.Connection, table: str) -> set[str]:
try:
cur = conn.execute(f"PRAGMA table_info({table})")
return {row[1] for row in cur.fetchall()}
except sqlite3.OperationalError:
return set()
def _list_children_tables(self, conn: sqlite3.Connection) -> list[str]:
tables: set[str] = set()
try:
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND (name LIKE 'album_%' OR name LIKE 'playlist_%') AND name != 'download_history'"
)
for row in cur.fetchall():
if row and row[0]:
tables.add(row[0])
except sqlite3.Error as e:
logger.warning(f"Failed to scan sqlite_master for children tables: {e}")
try:
cur = conn.execute(
"SELECT DISTINCT children_table FROM download_history WHERE children_table IS NOT NULL AND TRIM(children_table) != ''"
)
for row in cur.fetchall():
t = row[0]
if t:
tables.add(t)
except sqlite3.Error as e:
logger.warning(f"Failed to scan download_history for children tables: {e}")
return sorted(tables)
def check_history(self, conn: sqlite3.Connection) -> bool:
tables = self._list_children_tables(conn)
if not tables:
# Nothing to migrate
return True
# If any table is missing any of the extra columns, migration is needed
for t in tables:
cols = self._table_columns(conn, t)
if not set(self.CHILDREN_EXTRA_COLUMNS.keys()).issubset(cols):
return False
return True
def update_history(self, conn: sqlite3.Connection) -> None:
tables = self._list_children_tables(conn)
for t in tables:
existing = self._table_columns(conn, t)
for col_name, col_type in self.CHILDREN_EXTRA_COLUMNS.items():
if col_name in existing:
continue
try:
conn.execute(f"ALTER TABLE {t} ADD COLUMN {col_name} {col_type}")
logger.info(
f"Added column '{col_name} {col_type}' to history children table '{t}'."
)
except sqlite3.OperationalError as e:
logger.warning(
f"Could not add column '{col_name}' to history children table '{t}': {e}"
)
def check_watch_artists(self, conn: sqlite3.Connection) -> bool:
# No changes for watch artists in 3.1.2
return True
def update_watch_artists(self, conn: sqlite3.Connection) -> None:
# No-op
pass
def check_watch_playlists(self, conn: sqlite3.Connection) -> bool:
# No changes for watch playlists in 3.1.2
return True
def update_watch_playlists(self, conn: sqlite3.Connection) -> None:
# No-op
pass
def check_accounts(self, conn: sqlite3.Connection) -> bool:
# No changes for accounts in 3.1.2
return True
def update_accounts(self, conn: sqlite3.Connection) -> None:
# No-op
pass

100
routes/migrations/v3_2_0.py Normal file
View File

@@ -0,0 +1,100 @@
import sqlite3
import logging
logger = logging.getLogger(__name__)
class MigrationV3_2_0:
"""
Migration for version 3.2.0 (upgrade path 3.2.0 -> 3.2.1).
- Adds per-item batch progress columns to Watch DBs to support page-by-interval processing.
- Enforces prerequisite: previous instance version must be 3.1.2 (validated by runner).
"""
# New columns to add to watched tables
PLAYLISTS_ADDED_COLUMNS: dict[str, str] = {
"batch_next_offset": "INTEGER DEFAULT 0",
"batch_processing_snapshot_id": "TEXT",
}
ARTISTS_ADDED_COLUMNS: dict[str, str] = {
"batch_next_offset": "INTEGER DEFAULT 0",
}
# --- No-op for history/accounts in 3.2.1 ---
def check_history(self, conn: sqlite3.Connection) -> bool:
return True
def update_history(self, conn: sqlite3.Connection) -> None:
pass
def check_accounts(self, conn: sqlite3.Connection) -> bool:
return True
def update_accounts(self, conn: sqlite3.Connection) -> None:
pass
# --- Watch: playlists ---
def check_watch_playlists(self, conn: sqlite3.Connection) -> bool:
try:
cur = conn.execute("PRAGMA table_info(watched_playlists)")
cols = {row[1] for row in cur.fetchall()}
return set(self.PLAYLISTS_ADDED_COLUMNS.keys()).issubset(cols)
except sqlite3.OperationalError:
# Table missing means not ready
return False
def update_watch_playlists(self, conn: sqlite3.Connection) -> None:
# Add new columns if missing
try:
cur = conn.execute("PRAGMA table_info(watched_playlists)")
existing = {row[1] for row in cur.fetchall()}
for col_name, col_type in self.PLAYLISTS_ADDED_COLUMNS.items():
if col_name in existing:
continue
try:
conn.execute(
f"ALTER TABLE watched_playlists ADD COLUMN {col_name} {col_type}"
)
logger.info(
f"Added column '{col_name} {col_type}' to watched_playlists for 3.2.1 batch progress."
)
except sqlite3.OperationalError as e:
logger.warning(
f"Could not add column '{col_name}' to watched_playlists: {e}"
)
except Exception:
logger.error("Failed to update watched_playlists for 3.2.1", exc_info=True)
# --- Watch: artists ---
def check_watch_artists(self, conn: sqlite3.Connection) -> bool:
try:
cur = conn.execute("PRAGMA table_info(watched_artists)")
cols = {row[1] for row in cur.fetchall()}
return set(self.ARTISTS_ADDED_COLUMNS.keys()).issubset(cols)
except sqlite3.OperationalError:
return False
def update_watch_artists(self, conn: sqlite3.Connection) -> None:
try:
cur = conn.execute("PRAGMA table_info(watched_artists)")
existing = {row[1] for row in cur.fetchall()}
for col_name, col_type in self.ARTISTS_ADDED_COLUMNS.items():
if col_name in existing:
continue
try:
conn.execute(
f"ALTER TABLE watched_artists ADD COLUMN {col_name} {col_type}"
)
logger.info(
f"Added column '{col_name} {col_type}' to watched_artists for 3.2.1 batch progress."
)
except sqlite3.OperationalError as e:
logger.warning(
f"Could not add column '{col_name}' to watched_artists: {e}"
)
except Exception:
logger.error("Failed to update watched_artists for 3.2.1", exc_info=True)

121
routes/utils/watch/db.py
View File

@@ -25,6 +25,9 @@ EXPECTED_WATCHED_PLAYLISTS_COLUMNS = {
"last_checked": "INTEGER",
"added_at": "INTEGER",
"is_active": "INTEGER DEFAULT 1",
# New: batch progress for per-interval page fetching
"batch_next_offset": "INTEGER DEFAULT 0",
"batch_processing_snapshot_id": "TEXT",
}
EXPECTED_PLAYLIST_TRACKS_COLUMNS = {
@@ -55,6 +58,8 @@ EXPECTED_WATCHED_ARTISTS_COLUMNS = {
"genres": "TEXT", # Comma-separated
"popularity": "INTEGER",
"image_url": "TEXT",
# New: batch progress for per-interval page fetching
"batch_next_offset": "INTEGER DEFAULT 0",
}
EXPECTED_ARTIST_ALBUMS_COLUMNS = {
@@ -439,6 +444,61 @@ def update_playlist_snapshot(
)
# --- New: per-playlist batch progress helpers ---
def get_playlist_batch_progress(playlist_spotify_id: str) -> tuple[int, str | None]:
"""Returns (batch_next_offset, batch_processing_snapshot_id) for a watched playlist."""
try:
with _get_playlists_db_connection() as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT batch_next_offset, batch_processing_snapshot_id FROM watched_playlists WHERE spotify_id = ?",
(playlist_spotify_id,),
)
row = cursor.fetchone()
if not row:
return 0, None
next_offset = (
row["batch_next_offset"] if "batch_next_offset" in row.keys() else 0
)
processing_snapshot = (
row["batch_processing_snapshot_id"]
if "batch_processing_snapshot_id" in row.keys()
else None
)
return int(next_offset or 0), processing_snapshot
except sqlite3.Error as e:
logger.error(
f"Error retrieving batch progress for playlist {playlist_spotify_id}: {e}",
exc_info=True,
)
return 0, None
def set_playlist_batch_progress(
playlist_spotify_id: str, next_offset: int, processing_snapshot_id: str | None
) -> None:
"""Updates batch_next_offset and batch_processing_snapshot_id for a watched playlist."""
try:
with _get_playlists_db_connection() as conn:
cursor = conn.cursor()
cursor.execute(
"""
UPDATE watched_playlists
SET batch_next_offset = ?, batch_processing_snapshot_id = ?
WHERE spotify_id = ?
""",
(int(next_offset or 0), processing_snapshot_id, playlist_spotify_id),
)
conn.commit()
except sqlite3.Error as e:
logger.error(
f"Error updating batch progress for playlist {playlist_spotify_id}: {e}",
exc_info=True,
)
def get_playlist_track_ids_from_db(playlist_spotify_id: str):
"""Retrieves all track Spotify IDs from a specific playlist's tracks table in playlists.db."""
table_name = f"playlist_{playlist_spotify_id.replace('-', '_')}"
@@ -773,7 +833,7 @@ def add_specific_tracks_to_playlist_table(
def remove_specific_tracks_from_playlist_table(
playlist_spotify_id: str, track_spotify_ids: list
):
"""Removes specific tracks from the playlist's local track table."""
"""Removes specific tracks from the playlist's local DB table."""
table_name = f"playlist_{playlist_spotify_id.replace('-', '_')}"
if not track_spotify_ids:
return 0
@@ -799,7 +859,7 @@ def remove_specific_tracks_from_playlist_table(
conn.commit()
deleted_count = cursor.rowcount
logger.info(
f"Manually removed {deleted_count} tracks from DB for playlist {playlist_spotify_id}."
f"Successfully removed {deleted_count} tracks locally for playlist {playlist_spotify_id}."
)
return deleted_count
except sqlite3.Error as e:
@@ -1164,6 +1224,53 @@ def update_artist_metadata_after_check(
)
# --- New: per-artist batch progress helpers ---
def get_artist_batch_next_offset(artist_spotify_id: str) -> int:
try:
with _get_artists_db_connection() as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT batch_next_offset FROM watched_artists WHERE spotify_id = ?",
(artist_spotify_id,),
)
row = cursor.fetchone()
if not row:
return 0
return (
int(row["batch_next_offset"])
if "batch_next_offset" in row.keys()
else 0
)
except sqlite3.Error as e:
logger.error(
f"Error retrieving batch_next_offset for artist {artist_spotify_id}: {e}",
exc_info=True,
)
return 0
def set_artist_batch_next_offset(artist_spotify_id: str, next_offset: int) -> None:
try:
with _get_artists_db_connection() as conn:
cursor = conn.cursor()
cursor.execute(
"""
UPDATE watched_artists
SET batch_next_offset = ?
WHERE spotify_id = ?
""",
(int(next_offset or 0), artist_spotify_id),
)
conn.commit()
except sqlite3.Error as e:
logger.error(
f"Error updating batch_next_offset for artist {artist_spotify_id}: {e}",
exc_info=True,
)
def get_artist_album_ids_from_db(artist_spotify_id: str):
"""Retrieves all album Spotify IDs from a specific artist's albums table in artists.db."""
table_name = f"artist_{artist_spotify_id.replace('-', '_')}"
@@ -1289,11 +1396,11 @@ def add_or_update_album_for_artist(
total_tracks,
link,
image_url,
current_time, # added_to_db
current_time, # last_seen_on_spotify
task_id, # download_task_id
download_status, # download_status
0, # is_fully_downloaded_managed_by_app
current_time, # added_to_db
current_time, # last_seen_on_spotify
task_id, # download_task_id
download_status, # download_status
0, # is_fully_downloaded_managed_by_app
)
cursor.execute(
f"""

911
routes/utils/watch/manager.py
View File

File diff suppressed because it is too large Load Diff