JDM170/spotizerr-dev
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve 3.0.6 migration

Browse Source
This commit is contained in:
Xoconoch
2025-08-17 12:49:47 -06:00
parent 2d61941acd
commit 2ffd64c047
5 changed files with 573 additions and 589 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
routes/migrations/runner.py
View File

@@ -4,7 +4,6 @@ from pathlib import Path
from typing import Optional
from .v3_0_6 import MigrationV3_0_6
from .v3_1_0 import MigrationV3_1_0
logger = logging.getLogger(__name__)
@@ -39,8 +38,69 @@ CHILDREN_EXPECTED_COLUMNS: dict[str, str] = {
"metadata": "TEXT",
}
# 3.1.2 expected schemas for Watch DBs (kept here to avoid importing modules with side-effects)
EXPECTED_WATCHED_PLAYLISTS_COLUMNS: dict[str, str] = {
"spotify_id": "TEXT PRIMARY KEY",
"name": "TEXT",
"owner_id": "TEXT",
"owner_name": "TEXT",
"total_tracks": "INTEGER",
"link": "TEXT",
"snapshot_id": "TEXT",
"last_checked": "INTEGER",
"added_at": "INTEGER",
"is_active": "INTEGER DEFAULT 1",
}
EXPECTED_PLAYLIST_TRACKS_COLUMNS: dict[str, str] = {
"spotify_track_id": "TEXT PRIMARY KEY",
"title": "TEXT",
"artist_names": "TEXT",
"album_name": "TEXT",
"album_artist_names": "TEXT",
"track_number": "INTEGER",
"album_spotify_id": "TEXT",
"duration_ms": "INTEGER",
"added_at_playlist": "TEXT",
"added_to_db": "INTEGER",
"is_present_in_spotify": "INTEGER DEFAULT 1",
"last_seen_in_spotify": "INTEGER",
"snapshot_id": "TEXT",
"final_path": "TEXT",
}
EXPECTED_WATCHED_ARTISTS_COLUMNS: dict[str, str] = {
"spotify_id": "TEXT PRIMARY KEY",
"name": "TEXT",
"link": "TEXT",
"total_albums_on_spotify": "INTEGER",
"last_checked": "INTEGER",
"added_at": "INTEGER",
"is_active": "INTEGER DEFAULT 1",
"genres": "TEXT",
"popularity": "INTEGER",
"image_url": "TEXT",
}
EXPECTED_ARTIST_ALBUMS_COLUMNS: dict[str, str] = {
"album_spotify_id": "TEXT PRIMARY KEY",
"artist_spotify_id": "TEXT",
"name": "TEXT",
"album_group": "TEXT",
"album_type": "TEXT",
"release_date": "TEXT",
"release_date_precision": "TEXT",
"total_tracks": "INTEGER",
"link": "TEXT",
"image_url": "TEXT",
"added_to_db": "INTEGER",
"last_seen_on_spotify": "INTEGER",
"download_task_id": "TEXT",
"download_status": "INTEGER DEFAULT 0",
"is_fully_downloaded_managed_by_app": "INTEGER DEFAULT 0",
}
m306 = MigrationV3_0_6()
m310 = MigrationV3_1_0()
def _safe_connect(path: Path) -> Optional[sqlite3.Connection]:
@@ -151,15 +211,118 @@ def _ensure_creds_filesystem() -> None:
logger.error("Failed to ensure credentials filesystem (blobs/search.json)", exc_info=True)
def _apply_versioned_updates(conn: sqlite3.Connection, c306, u306, c310, u310, post_update=None) -> None:
if not c306(conn):
u306(conn)
if not c310(conn):
u310(conn)
def _apply_versioned_updates(conn: sqlite3.Connection, c_base, u_base, post_update=None) -> None:
if not c_base(conn):
u_base(conn)
if post_update:
post_update(conn)
# --- 3.1.2 upgrade helpers for Watch DBs ---
def _update_watch_playlists_db(conn: sqlite3.Connection) -> None:
try:
# Ensure core watched_playlists table exists and has expected schema
conn.execute(
"""
CREATE TABLE IF NOT EXISTS watched_playlists (
spotify_id TEXT PRIMARY KEY,
name TEXT,
owner_id TEXT,
owner_name TEXT,
total_tracks INTEGER,
link TEXT,
snapshot_id TEXT,
last_checked INTEGER,
added_at INTEGER,
is_active INTEGER DEFAULT 1
)
"""
)
_ensure_table_schema(conn, "watched_playlists", EXPECTED_WATCHED_PLAYLISTS_COLUMNS, "watched playlists")
# Upgrade all dynamic playlist_ tables
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'playlist_%'")
for row in cur.fetchall():
table_name = row[0]
conn.execute(
f"""
CREATE TABLE IF NOT EXISTS {table_name} (
spotify_track_id TEXT PRIMARY KEY,
title TEXT,
artist_names TEXT,
album_name TEXT,
album_artist_names TEXT,
track_number INTEGER,
album_spotify_id TEXT,
duration_ms INTEGER,
added_at_playlist TEXT,
added_to_db INTEGER,
is_present_in_spotify INTEGER DEFAULT 1,
last_seen_in_spotify INTEGER,
snapshot_id TEXT,
final_path TEXT
)
"""
)
_ensure_table_schema(conn, table_name, EXPECTED_PLAYLIST_TRACKS_COLUMNS, f"playlist tracks ({table_name})")
logger.info("Upgraded watch playlists DB to 3.1.2 schema")
except Exception:
logger.error("Failed to upgrade watch playlists DB to 3.1.2 schema", exc_info=True)
def _update_watch_artists_db(conn: sqlite3.Connection) -> None:
try:
# Ensure core watched_artists table exists and has expected schema
conn.execute(
"""
CREATE TABLE IF NOT EXISTS watched_artists (
spotify_id TEXT PRIMARY KEY,
name TEXT,
link TEXT,
total_albums_on_spotify INTEGER,
last_checked INTEGER,
added_at INTEGER,
is_active INTEGER DEFAULT 1,
genres TEXT,
popularity INTEGER,
image_url TEXT
)
"""
)
_ensure_table_schema(conn, "watched_artists", EXPECTED_WATCHED_ARTISTS_COLUMNS, "watched artists")
# Upgrade all dynamic artist_ tables
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'artist_%'")
for row in cur.fetchall():
table_name = row[0]
conn.execute(
f"""
CREATE TABLE IF NOT EXISTS {table_name} (
album_spotify_id TEXT PRIMARY KEY,
artist_spotify_id TEXT,
name TEXT,
album_group TEXT,
album_type TEXT,
release_date TEXT,
release_date_precision TEXT,
total_tracks INTEGER,
link TEXT,
image_url TEXT,
added_to_db INTEGER,
last_seen_on_spotify INTEGER,
download_task_id TEXT,
download_status INTEGER DEFAULT 0,
is_fully_downloaded_managed_by_app INTEGER DEFAULT 0
)
"""
)
_ensure_table_schema(conn, table_name, EXPECTED_ARTIST_ALBUMS_COLUMNS, f"artist albums ({table_name})")
logger.info("Upgraded watch artists DB to 3.1.2 schema")
except Exception:
logger.error("Failed to upgrade watch artists DB to 3.1.2 schema", exc_info=True)
def run_migrations_if_needed() -> None:
try:
# History DB
@@ -170,8 +333,6 @@ def run_migrations_if_needed() -> None:
h_conn,
m306.check_history,
m306.update_history,
m310.check_history,
m310.update_history,
post_update=_update_children_tables_for_history,
)
h_conn.commit()
@@ -186,9 +347,8 @@ def run_migrations_if_needed() -> None:
p_conn,
m306.check_watch_playlists,
m306.update_watch_playlists,
m310.check_watch_playlists,
m310.update_watch_playlists,
)
_update_watch_playlists_db(p_conn)
p_conn.commit()
finally:
p_conn.close()
@@ -201,9 +361,8 @@ def run_migrations_if_needed() -> None:
a_conn,
m306.check_watch_artists,
m306.update_watch_artists,
m310.check_watch_artists,
m310.update_watch_artists,
)
_update_watch_artists_db(a_conn)
a_conn.commit()
finally:
a_conn.close()
@@ -216,8 +375,6 @@ def run_migrations_if_needed() -> None:
c_conn,
m306.check_accounts,
m306.update_accounts,
m310.check_accounts,
m310.update_accounts,
)
c_conn.commit()
finally:

331
routes/migrations/v3_1_0.py
View File

@@ -1,331 +0,0 @@
import sqlite3
class MigrationV3_1_0:
# --- Expected Schemas (3.1.0) ---
HISTORY_MAIN_REQUIRED = {
"id",
"download_type",
"title",
"artists",
"timestamp",
"status",
"service",
"quality_format",
"quality_bitrate",
"total_tracks",
"successful_tracks",
"failed_tracks",
"skipped_tracks",
"children_table",
"task_id",
"external_ids",
"metadata",
"release_date",
"genres",
"images",
"owner",
"album_type",
"duration_total_ms",
"explicit",
}
HISTORY_MAIN_SQL = """
CREATE TABLE IF NOT EXISTS download_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
download_type TEXT NOT NULL,
title TEXT NOT NULL,
artists TEXT,
timestamp REAL NOT NULL,
status TEXT NOT NULL,
service TEXT,
quality_format TEXT,
quality_bitrate TEXT,
total_tracks INTEGER,
successful_tracks INTEGER,
failed_tracks INTEGER,
skipped_tracks INTEGER,
children_table TEXT,
task_id TEXT,
external_ids TEXT,
metadata TEXT,
release_date TEXT,
genres TEXT,
images TEXT,
owner TEXT,
album_type TEXT,
duration_total_ms INTEGER,
explicit BOOLEAN
);
CREATE INDEX IF NOT EXISTS idx_download_history_timestamp ON download_history(timestamp);
CREATE INDEX IF NOT EXISTS idx_download_history_type_status ON download_history(download_type, status);
CREATE INDEX IF NOT EXISTS idx_download_history_task_id ON download_history(task_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_download_history_task_type_ids ON download_history(task_id, download_type, external_ids);
"""
# Children tables schema (album_% / playlist_%):
HISTORY_CHILDREN_EXPECTED = {
"id": "INTEGER PRIMARY KEY AUTOINCREMENT",
"title": "TEXT NOT NULL",
"artists": "TEXT",
"album_title": "TEXT",
"duration_ms": "INTEGER",
"track_number": "INTEGER",
"disc_number": "INTEGER",
"explicit": "BOOLEAN",
"status": "TEXT NOT NULL",
"external_ids": "TEXT",
"genres": "TEXT",
"isrc": "TEXT",
"timestamp": "REAL NOT NULL",
"position": "INTEGER",
"metadata": "TEXT",
}
WATCH_PLAYLISTS_REQUIRED = {
"spotify_id",
"name",
"owner_id",
"owner_name",
"total_tracks",
"link",
"snapshot_id",
"last_checked",
"added_at",
"is_active",
}
# Per-playlist tracks table expected columns
PLAYLIST_TRACKS_EXPECTED = {
"spotify_track_id": "TEXT PRIMARY KEY",
"title": "TEXT",
"artist_names": "TEXT",
"album_name": "TEXT",
"album_artist_names": "TEXT",
"track_number": "INTEGER",
"album_spotify_id": "TEXT",
"duration_ms": "INTEGER",
"added_at_playlist": "TEXT",
"added_to_db": "INTEGER",
"is_present_in_spotify": "INTEGER DEFAULT 1",
"last_seen_in_spotify": "INTEGER",
"snapshot_id": "TEXT",
"final_path": "TEXT",
}
WATCH_ARTISTS_REQUIRED = {
"spotify_id",
"name",
"link",
"total_albums_on_spotify",
"last_checked",
"added_at",
"is_active",
"genres",
"popularity",
"image_url",
}
ARTIST_ALBUMS_EXPECTED = {
"album_spotify_id": "TEXT PRIMARY KEY",
"artist_spotify_id": "TEXT",
"name": "TEXT",
"album_group": "TEXT",
"album_type": "TEXT",
"release_date": "TEXT",
"release_date_precision": "TEXT",
"total_tracks": "INTEGER",
"link": "TEXT",
"image_url": "TEXT",
"added_to_db": "INTEGER",
"last_seen_on_spotify": "INTEGER",
"download_task_id": "TEXT",
"download_status": "INTEGER DEFAULT 0",
"is_fully_downloaded_managed_by_app": "INTEGER DEFAULT 0",
}
ACCOUNTS_SPOTIFY_REQUIRED = {"name", "region", "created_at", "updated_at"}
ACCOUNTS_DEEZER_REQUIRED = {"name", "arl", "region", "created_at", "updated_at"}
ACCOUNTS_SPOTIFY_SQL = """
CREATE TABLE IF NOT EXISTS spotify (
name TEXT PRIMARY KEY,
region TEXT,
created_at REAL,
updated_at REAL
);
"""
ACCOUNTS_DEEZER_SQL = """
CREATE TABLE IF NOT EXISTS deezer (
name TEXT PRIMARY KEY,
arl TEXT,
region TEXT,
created_at REAL,
updated_at REAL
);
"""
@staticmethod
def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
try:
cur = conn.execute(f"PRAGMA table_info({table})")
return {row[1] for row in cur.fetchall()}
except Exception:
return set()
@staticmethod
def _ensure_table_schema(conn: sqlite3.Connection, table_name: str, expected: dict[str, str], desc: str) -> None:
cur = conn.execute(f"PRAGMA table_info({table_name})")
existing = {row[1] for row in cur.fetchall()}
for col, col_type in expected.items():
if col in existing:
continue
col_type_for_add = (
col_type.replace("PRIMARY KEY", "").replace("AUTOINCREMENT", "").replace("NOT NULL", "").strip()
)
try:
conn.execute(f"ALTER TABLE {table_name} ADD COLUMN {col} {col_type_for_add}")
except sqlite3.OperationalError:
pass
# --- Check methods ---
def check_history(self, conn: sqlite3.Connection) -> bool:
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='download_history'")
if not cur.fetchone():
return False
return self.HISTORY_MAIN_REQUIRED.issubset(self._columns(conn, "download_history"))
def check_watch_playlists(self, conn: sqlite3.Connection) -> bool:
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='watched_playlists'")
if not cur.fetchone():
return False
if not self.WATCH_PLAYLISTS_REQUIRED.issubset(self._columns(conn, "watched_playlists")):
return False
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'playlist_%'")
rows = cur.fetchall()
for (table_name,) in rows:
cols = self._columns(conn, table_name)
required_cols = set(self.PLAYLIST_TRACKS_EXPECTED.keys())
if not required_cols.issubset(cols):
return False
return True
def check_watch_artists(self, conn: sqlite3.Connection) -> bool:
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='watched_artists'")
if not cur.fetchone():
return False
if not self.WATCH_ARTISTS_REQUIRED.issubset(self._columns(conn, "watched_artists")):
return False
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'artist_%'")
rows = cur.fetchall()
for (table_name,) in rows:
cols = self._columns(conn, table_name)
required_cols = set(self.ARTIST_ALBUMS_EXPECTED.keys())
if not required_cols.issubset(cols):
return False
return True
def check_accounts(self, conn: sqlite3.Connection) -> bool:
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='spotify'")
if not cur.fetchone():
return False
if not self.ACCOUNTS_SPOTIFY_REQUIRED.issubset(self._columns(conn, "spotify")):
return False
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='deezer'")
if not cur.fetchone():
return False
if not self.ACCOUNTS_DEEZER_REQUIRED.issubset(self._columns(conn, "deezer")):
return False
return True
# --- Update methods ---
def update_history(self, conn: sqlite3.Connection) -> None:
conn.executescript(self.HISTORY_MAIN_SQL)
def update_watch_playlists(self, conn: sqlite3.Connection) -> None:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS watched_playlists (
spotify_id TEXT PRIMARY KEY,
name TEXT,
owner_id TEXT,
owner_name TEXT,
total_tracks INTEGER,
link TEXT,
snapshot_id TEXT,
last_checked INTEGER,
added_at INTEGER,
is_active INTEGER DEFAULT 1
)
"""
)
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'playlist_%'")
for (table_name,) in cur.fetchall():
conn.execute(
f"""
CREATE TABLE IF NOT EXISTS {table_name} (
spotify_track_id TEXT PRIMARY KEY,
title TEXT,
artist_names TEXT,
album_name TEXT,
album_artist_names TEXT,
track_number INTEGER,
album_spotify_id TEXT,
duration_ms INTEGER,
added_at_playlist TEXT,
added_to_db INTEGER,
is_present_in_spotify INTEGER DEFAULT 1,
last_seen_in_spotify INTEGER,
snapshot_id TEXT,
final_path TEXT
)
"""
)
self._ensure_table_schema(conn, table_name, self.PLAYLIST_TRACKS_EXPECTED, f"playlist tracks {table_name}")
def update_watch_artists(self, conn: sqlite3.Connection) -> None:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS watched_artists (
spotify_id TEXT PRIMARY KEY,
name TEXT,
link TEXT,
total_albums_on_spotify INTEGER,
last_checked INTEGER,
added_at INTEGER,
is_active INTEGER DEFAULT 1,
genres TEXT,
popularity INTEGER,
image_url TEXT
)
"""
)
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'artist_%'")
for (table_name,) in cur.fetchall():
conn.execute(
f"""
CREATE TABLE IF NOT EXISTS {table_name} (
album_spotify_id TEXT PRIMARY KEY,
artist_spotify_id TEXT,
name TEXT,
album_group TEXT,
album_type TEXT,
release_date TEXT,
release_date_precision TEXT,
total_tracks INTEGER,
link TEXT,
image_url TEXT,
added_to_db INTEGER,
last_seen_on_spotify INTEGER,
download_task_id TEXT,
download_status INTEGER DEFAULT 0,
is_fully_downloaded_managed_by_app INTEGER DEFAULT 0
)
"""
)
self._ensure_table_schema(conn, table_name, self.ARTIST_ALBUMS_EXPECTED, f"artist albums {table_name}")
def update_accounts(self, conn: sqlite3.Connection) -> None:
conn.executescript(self.ACCOUNTS_SPOTIFY_SQL)
conn.executescript(self.ACCOUNTS_DEEZER_SQL)