queue management refactor, embrace celery and redis
This commit is contained in:
architect.in.git
@@ -18,7 +18,8 @@ def download_album(
|
||||
pad_tracks=True,
|
||||
initial_retry_delay=5,
|
||||
retry_delay_increase=5,
|
||||
max_retries=3
|
||||
max_retries=3,
|
||||
progress_callback=None
|
||||
):
|
||||
try:
|
||||
# Load Spotify client credentials if available
|
||||
@@ -51,7 +52,8 @@ def download_album(
|
||||
dl = DeeLogin(
|
||||
arl=deezer_creds.get('arl', ''),
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
# Download using download_albumspo; pass real_time_dl accordingly and the custom formatting
|
||||
dl.download_albumspo(
|
||||
@@ -92,7 +94,8 @@ def download_album(
|
||||
spo = SpoLogin(
|
||||
credentials_path=spo_creds_path,
|
||||
spotify_client_id=fallback_client_id,
|
||||
spotify_client_secret=fallback_client_secret
|
||||
spotify_client_secret=fallback_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
spo.download_album(
|
||||
link_album=url,
|
||||
@@ -126,7 +129,8 @@ def download_album(
|
||||
spo = SpoLogin(
|
||||
credentials_path=credentials_path,
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
spo.download_album(
|
||||
link_album=url,
|
||||
@@ -156,7 +160,8 @@ def download_album(
|
||||
dl = DeeLogin(
|
||||
arl=creds.get('arl', ''),
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
dl.download_albumdee(
|
||||
link_album=url,
|
||||
|
||||
@@ -1,18 +1,22 @@
|
||||
import json
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
import os
|
||||
import logging
|
||||
from routes.utils.celery_queue_manager import download_queue_manager, get_config_params
|
||||
|
||||
from deezspot.easy_spoty import Spo
|
||||
from deezspot.libutils.utils import get_ids, link_is_valid
|
||||
from routes.utils.queue import download_queue_manager # Global download queue manager
|
||||
|
||||
# Configure logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def log_json(message_dict):
|
||||
"""Helper function to output a JSON-formatted log message."""
|
||||
print(json.dumps(message_dict))
|
||||
|
||||
|
||||
def get_artist_discography(url, main, album_type='album,single,compilation,appears_on'):
|
||||
def get_artist_discography(url, main, album_type='album,single,compilation,appears_on', progress_callback=None):
|
||||
"""
|
||||
Validate the URL, extract the artist ID, and retrieve the discography.
|
||||
"""
|
||||
@@ -59,94 +63,155 @@ def get_artist_discography(url, main, album_type='album,single,compilation,appea
|
||||
raise
|
||||
|
||||
|
||||
def download_artist_albums(service, url, main, fallback=None, quality=None,
|
||||
fall_quality=None, real_time=False,
|
||||
album_type='album,single,compilation,appears_on',
|
||||
custom_dir_format="%ar_album%/%album%/%copyright%",
|
||||
custom_track_format="%tracknum%. %music% - %artist%",
|
||||
pad_tracks=True,
|
||||
initial_retry_delay=5,
|
||||
retry_delay_increase=5,
|
||||
max_retries=3):
|
||||
def download_artist_albums(service, url, album_type="album,single,compilation", request_args=None, progress_callback=None):
|
||||
"""
|
||||
Retrieves the artist discography and, for each album with a valid Spotify URL,
|
||||
creates a download task that is queued via the global download queue. The queue
|
||||
creates a PRG file for each album download. This function returns a list of those
|
||||
album PRG filenames.
|
||||
Download albums from an artist.
|
||||
|
||||
Args:
|
||||
service (str): 'spotify' or 'deezer'
|
||||
url (str): URL of the artist
|
||||
album_type (str): Comma-separated list of album types to download (album,single,compilation,appears_on)
|
||||
request_args (dict): Original request arguments for additional parameters
|
||||
progress_callback (callable): Optional callback function for progress reporting
|
||||
|
||||
Returns:
|
||||
list: List of task IDs for the enqueued album downloads
|
||||
"""
|
||||
try:
|
||||
discography = get_artist_discography(url, main, album_type=album_type)
|
||||
except Exception as e:
|
||||
log_json({"status": "error", "message": f"Error retrieving artist discography: {e}"})
|
||||
raise
|
||||
|
||||
albums = discography.get('items', [])
|
||||
if not albums:
|
||||
log_json({"status": "done", "message": "No albums found for the artist."})
|
||||
return []
|
||||
|
||||
prg_files = []
|
||||
|
||||
for album in albums:
|
||||
try:
|
||||
album_url = album.get('external_urls', {}).get('spotify')
|
||||
if not album_url:
|
||||
log_json({
|
||||
"status": "warning",
|
||||
"message": f"No Spotify URL found for album '{album.get('name', 'Unknown Album')}'; skipping."
|
||||
logger.info(f"Starting artist albums download: {url} (service: {service}, album_types: {album_type})")
|
||||
|
||||
if request_args is None:
|
||||
request_args = {}
|
||||
|
||||
# Get config parameters
|
||||
config_params = get_config_params()
|
||||
|
||||
# Get the artist information first
|
||||
if service == 'spotify':
|
||||
from deezspot.spotloader import SpoLogin
|
||||
|
||||
# Get credentials
|
||||
spotify_profile = request_args.get('main', config_params['spotify'])
|
||||
credentials_path = os.path.abspath(os.path.join('./creds/spotify', spotify_profile, 'credentials.json'))
|
||||
|
||||
# Validate credentials
|
||||
if not os.path.isfile(credentials_path):
|
||||
raise ValueError(f"Invalid Spotify credentials path: {credentials_path}")
|
||||
|
||||
# Load Spotify client credentials if available
|
||||
spotify_client_id = None
|
||||
spotify_client_secret = None
|
||||
search_creds_path = Path(f'./creds/spotify/{spotify_profile}/search.json')
|
||||
if search_creds_path.exists():
|
||||
try:
|
||||
with open(search_creds_path, 'r') as f:
|
||||
search_creds = json.load(f)
|
||||
spotify_client_id = search_creds.get('client_id')
|
||||
spotify_client_secret = search_creds.get('client_secret')
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading Spotify search credentials: {e}")
|
||||
|
||||
# Initialize the Spotify client
|
||||
spo = SpoLogin(
|
||||
credentials_path=credentials_path,
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
|
||||
# Get artist information
|
||||
artist_info = spo.get_artist_info(url)
|
||||
artist_name = artist_info['name']
|
||||
artist_id = artist_info['id']
|
||||
|
||||
# Get the list of albums
|
||||
album_types = album_type.split(',')
|
||||
albums = []
|
||||
|
||||
for album_type_item in album_types:
|
||||
# Fetch albums of the specified type
|
||||
albums_of_type = spo.get_albums_by_artist(artist_id, album_type_item.strip())
|
||||
for album in albums_of_type:
|
||||
albums.append({
|
||||
'name': album['name'],
|
||||
'url': album['external_urls']['spotify'],
|
||||
'type': 'album',
|
||||
'artist': artist_name
|
||||
})
|
||||
continue
|
||||
|
||||
album_name = album.get('name', 'Unknown Album')
|
||||
artists = album.get('artists', [])
|
||||
# Extract artist names or use "Unknown" as a fallback.
|
||||
artists = [artist.get("name", "Unknown") for artist in artists]
|
||||
|
||||
# Prepare the download task dictionary.
|
||||
task = {
|
||||
"download_type": "album",
|
||||
"service": service,
|
||||
"url": album_url,
|
||||
"main": main,
|
||||
"fallback": fallback,
|
||||
"quality": quality,
|
||||
"fall_quality": fall_quality,
|
||||
"real_time": real_time,
|
||||
"custom_dir_format": custom_dir_format,
|
||||
"custom_track_format": custom_track_format,
|
||||
"pad_tracks": pad_tracks,
|
||||
"initial_retry_delay": initial_retry_delay,
|
||||
"retry_delay_increase": retry_delay_increase,
|
||||
"max_retries": max_retries,
|
||||
# Extra info for logging in the PRG file.
|
||||
"name": album_name,
|
||||
"type": "album",
|
||||
"artist": artists,
|
||||
"orig_request": {
|
||||
"type": "album",
|
||||
"name": album_name,
|
||||
"artist": artists
|
||||
}
|
||||
}
|
||||
|
||||
# Add the task to the global download queue.
|
||||
# The queue manager creates the album's PRG file and returns its filename.
|
||||
prg_filename = download_queue_manager.add_task(task)
|
||||
prg_files.append(prg_filename)
|
||||
|
||||
log_json({
|
||||
"status": "queued",
|
||||
"album": album_name,
|
||||
"artist": artists,
|
||||
"prg_file": prg_filename,
|
||||
"message": "Album queued for download."
|
||||
|
||||
elif service == 'deezer':
|
||||
from deezspot.deezloader import DeeLogin
|
||||
|
||||
# Get credentials
|
||||
deezer_profile = request_args.get('main', config_params['deezer'])
|
||||
credentials_path = os.path.abspath(os.path.join('./creds/deezer', deezer_profile, 'credentials.json'))
|
||||
|
||||
# Validate credentials
|
||||
if not os.path.isfile(credentials_path):
|
||||
raise ValueError(f"Invalid Deezer credentials path: {credentials_path}")
|
||||
|
||||
# For Deezer, we need to extract the ARL
|
||||
with open(credentials_path, 'r') as f:
|
||||
credentials = json.load(f)
|
||||
arl = credentials.get('arl')
|
||||
|
||||
if not arl:
|
||||
raise ValueError("No ARL found in Deezer credentials")
|
||||
|
||||
# Load Spotify client credentials if available for search purposes
|
||||
spotify_client_id = None
|
||||
spotify_client_secret = None
|
||||
search_creds_path = Path(f'./creds/spotify/{deezer_profile}/search.json')
|
||||
if search_creds_path.exists():
|
||||
try:
|
||||
with open(search_creds_path, 'r') as f:
|
||||
search_creds = json.load(f)
|
||||
spotify_client_id = search_creds.get('client_id')
|
||||
spotify_client_secret = search_creds.get('client_secret')
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading Spotify search credentials: {e}")
|
||||
|
||||
# Initialize the Deezer client
|
||||
dee = DeeLogin(
|
||||
arl=arl,
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
|
||||
# Get artist information
|
||||
artist_info = dee.get_artist_info(url)
|
||||
artist_name = artist_info['name']
|
||||
|
||||
# Get the list of albums (Deezer doesn't distinguish types like Spotify)
|
||||
albums_result = dee.get_artist_albums(url)
|
||||
albums = []
|
||||
|
||||
for album in albums_result:
|
||||
albums.append({
|
||||
'name': album['title'],
|
||||
'url': f"https://www.deezer.com/album/{album['id']}",
|
||||
'type': 'album',
|
||||
'artist': artist_name
|
||||
})
|
||||
|
||||
except Exception as album_error:
|
||||
log_json({
|
||||
"status": "error",
|
||||
"message": f"Error processing album '{album.get('name', 'Unknown')}': {album_error}"
|
||||
})
|
||||
traceback.print_exc()
|
||||
|
||||
return prg_files
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported service: {service}")
|
||||
|
||||
# Queue the album downloads
|
||||
album_task_ids = []
|
||||
|
||||
for album in albums:
|
||||
# Create a task for each album
|
||||
task_id = download_queue_manager.add_task({
|
||||
"download_type": "album",
|
||||
"service": service,
|
||||
"url": album['url'],
|
||||
"name": album['name'],
|
||||
"artist": album['artist'],
|
||||
"orig_request": request_args.copy() # Pass along original request args
|
||||
})
|
||||
|
||||
album_task_ids.append(task_id)
|
||||
logger.info(f"Queued album: {album['name']} by {album['artist']} (task ID: {task_id})")
|
||||
|
||||
return album_task_ids
|
||||
|
||||
122
routes/utils/celery_config.py
Normal file
122
routes/utils/celery_config.py
Normal file
@@ -0,0 +1,122 @@
|
||||
import os
|
||||
import json
|
||||
|
||||
# Load configuration from ./config/main.json and get the max_concurrent_dl value.
|
||||
CONFIG_PATH = './config/main.json'
|
||||
|
||||
try:
|
||||
with open(CONFIG_PATH, 'r') as f:
|
||||
config_data = json.load(f)
|
||||
MAX_CONCURRENT_DL = config_data.get("maxConcurrentDownloads", 3)
|
||||
MAX_RETRIES = config_data.get("maxRetries", 3)
|
||||
RETRY_DELAY = config_data.get("retryDelaySeconds", 5)
|
||||
RETRY_DELAY_INCREASE = config_data.get("retry_delay_increase", 5)
|
||||
except Exception as e:
|
||||
print(f"Error loading configuration: {e}")
|
||||
# Fallback to default values if there's an error reading the config.
|
||||
MAX_CONCURRENT_DL = 3
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY = 5
|
||||
RETRY_DELAY_INCREASE = 5
|
||||
|
||||
def get_config_params():
|
||||
"""
|
||||
Get common download parameters from the config file.
|
||||
This centralizes parameter retrieval and reduces redundancy in API calls.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing common parameters from config
|
||||
"""
|
||||
try:
|
||||
with open(CONFIG_PATH, 'r') as f:
|
||||
config = json.load(f)
|
||||
|
||||
return {
|
||||
'service': config.get('service', 'spotify'),
|
||||
'spotify': config.get('spotify', ''),
|
||||
'deezer': config.get('deezer', ''),
|
||||
'fallback': config.get('fallback', False),
|
||||
'spotifyQuality': config.get('spotifyQuality', 'NORMAL'),
|
||||
'deezerQuality': config.get('deezerQuality', 'MP3_128'),
|
||||
'realTime': config.get('realTime', False),
|
||||
'customDirFormat': config.get('customDirFormat', '%ar_album%/%album%'),
|
||||
'customTrackFormat': config.get('customTrackFormat', '%tracknum%. %music%'),
|
||||
'tracknum_padding': config.get('tracknum_padding', True),
|
||||
'maxRetries': config.get('maxRetries', 3),
|
||||
'retryDelaySeconds': config.get('retryDelaySeconds', 5),
|
||||
'retry_delay_increase': config.get('retry_delay_increase', 5)
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"Error reading config for parameters: {e}")
|
||||
# Return defaults if config read fails
|
||||
return {
|
||||
'service': 'spotify',
|
||||
'spotify': '',
|
||||
'deezer': '',
|
||||
'fallback': False,
|
||||
'spotifyQuality': 'NORMAL',
|
||||
'deezerQuality': 'MP3_128',
|
||||
'realTime': False,
|
||||
'customDirFormat': '%ar_album%/%album%',
|
||||
'customTrackFormat': '%tracknum%. %music%',
|
||||
'tracknum_padding': True,
|
||||
'maxRetries': 3,
|
||||
'retryDelaySeconds': 5,
|
||||
'retry_delay_increase': 5
|
||||
}
|
||||
|
||||
# Celery configuration
|
||||
REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/0')
|
||||
REDIS_BACKEND = os.environ.get('REDIS_BACKEND', 'redis://localhost:6379/0')
|
||||
|
||||
# Define task queues
|
||||
task_queues = {
|
||||
'default': {
|
||||
'exchange': 'default',
|
||||
'routing_key': 'default',
|
||||
},
|
||||
'downloads': {
|
||||
'exchange': 'downloads',
|
||||
'routing_key': 'downloads',
|
||||
}
|
||||
}
|
||||
|
||||
# Set default queue
|
||||
task_default_queue = 'downloads'
|
||||
task_default_exchange = 'downloads'
|
||||
task_default_routing_key = 'downloads'
|
||||
|
||||
# Celery task settings
|
||||
task_serializer = 'json'
|
||||
accept_content = ['json']
|
||||
result_serializer = 'json'
|
||||
enable_utc = True
|
||||
|
||||
# Configure worker concurrency based on MAX_CONCURRENT_DL
|
||||
worker_concurrency = MAX_CONCURRENT_DL
|
||||
|
||||
# Configure task rate limiting - these are per-minute limits
|
||||
task_annotations = {
|
||||
'routes.utils.celery_tasks.download_track': {
|
||||
'rate_limit': f'{MAX_CONCURRENT_DL}/m',
|
||||
},
|
||||
'routes.utils.celery_tasks.download_album': {
|
||||
'rate_limit': f'{MAX_CONCURRENT_DL}/m',
|
||||
},
|
||||
'routes.utils.celery_tasks.download_playlist': {
|
||||
'rate_limit': f'{MAX_CONCURRENT_DL}/m',
|
||||
}
|
||||
}
|
||||
|
||||
# Configure retry settings
|
||||
task_default_retry_delay = RETRY_DELAY # seconds
|
||||
task_max_retries = MAX_RETRIES
|
||||
|
||||
# Task result settings
|
||||
task_track_started = True
|
||||
result_expires = 60 * 60 * 24 * 7 # 7 days
|
||||
|
||||
# Configure visibility timeout for task messages
|
||||
broker_transport_options = {
|
||||
'visibility_timeout': 3600, # 1 hour
|
||||
}
|
||||
440
routes/utils/celery_queue_manager.py
Normal file
440
routes/utils/celery_queue_manager.py
Normal file
@@ -0,0 +1,440 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from routes.utils.celery_tasks import (
|
||||
celery_app,
|
||||
download_track,
|
||||
download_album,
|
||||
download_playlist,
|
||||
store_task_status,
|
||||
store_task_info,
|
||||
get_task_info,
|
||||
get_task_status,
|
||||
get_last_task_status,
|
||||
cancel_task as cancel_celery_task,
|
||||
retry_task as retry_celery_task,
|
||||
get_all_tasks,
|
||||
ProgressState
|
||||
)
|
||||
|
||||
# Configure logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Load configuration
|
||||
CONFIG_PATH = './config/main.json'
|
||||
try:
|
||||
with open(CONFIG_PATH, 'r') as f:
|
||||
config_data = json.load(f)
|
||||
MAX_CONCURRENT_DL = config_data.get("maxConcurrentDownloads", 3)
|
||||
except Exception as e:
|
||||
print(f"Error loading configuration: {e}")
|
||||
# Fallback default
|
||||
MAX_CONCURRENT_DL = 3
|
||||
|
||||
def get_config_params():
|
||||
"""
|
||||
Get common download parameters from the config file.
|
||||
This centralizes parameter retrieval and reduces redundancy in API calls.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing common parameters from config
|
||||
"""
|
||||
try:
|
||||
with open(CONFIG_PATH, 'r') as f:
|
||||
config = json.load(f)
|
||||
|
||||
return {
|
||||
'spotify': config.get('spotify', ''),
|
||||
'deezer': config.get('deezer', ''),
|
||||
'fallback': config.get('fallback', False),
|
||||
'spotifyQuality': config.get('spotifyQuality', 'NORMAL'),
|
||||
'deezerQuality': config.get('deezerQuality', 'MP3_128'),
|
||||
'realTime': config.get('realTime', False),
|
||||
'customDirFormat': config.get('customDirFormat', '%ar_album%/%album%'),
|
||||
'customTrackFormat': config.get('customTrackFormat', '%tracknum%. %music%'),
|
||||
'tracknum_padding': config.get('tracknum_padding', True),
|
||||
'maxRetries': config.get('maxRetries', 3),
|
||||
'retryDelaySeconds': config.get('retryDelaySeconds', 5),
|
||||
'retry_delay_increase': config.get('retry_delay_increase', 5)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading config for parameters: {e}")
|
||||
# Return defaults if config read fails
|
||||
return {
|
||||
'spotify': '',
|
||||
'deezer': '',
|
||||
'fallback': False,
|
||||
'spotifyQuality': 'NORMAL',
|
||||
'deezerQuality': 'MP3_128',
|
||||
'realTime': False,
|
||||
'customDirFormat': '%ar_album%/%album%',
|
||||
'customTrackFormat': '%tracknum%. %music%',
|
||||
'tracknum_padding': True,
|
||||
'maxRetries': 3,
|
||||
'retryDelaySeconds': 5,
|
||||
'retry_delay_increase': 5
|
||||
}
|
||||
|
||||
class CeleryDownloadQueueManager:
|
||||
"""
|
||||
Manages a queue of download tasks using Celery.
|
||||
This is a drop-in replacement for the previous DownloadQueueManager.
|
||||
|
||||
Instead of using file-based progress tracking, it uses Redis via Celery
|
||||
for task management and progress tracking.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the Celery-based download queue manager"""
|
||||
self.max_concurrent = MAX_CONCURRENT_DL
|
||||
self.paused = False
|
||||
print(f"Celery Download Queue Manager initialized with max_concurrent={self.max_concurrent}")
|
||||
|
||||
def add_task(self, task):
|
||||
"""
|
||||
Adds a new download task to the queue.
|
||||
|
||||
Args:
|
||||
task (dict): Dictionary containing task parameters
|
||||
|
||||
Returns:
|
||||
str: The task ID for status tracking
|
||||
"""
|
||||
try:
|
||||
download_type = task.get("download_type", "unknown")
|
||||
service = task.get("service", "")
|
||||
|
||||
# Get common parameters from config
|
||||
config_params = get_config_params()
|
||||
|
||||
# Use service from config instead of task
|
||||
service = config_params.get('service')
|
||||
|
||||
# Generate a unique task ID
|
||||
task_id = str(uuid.uuid4())
|
||||
|
||||
# Store the original request in task info
|
||||
original_request = task.get("orig_request", {}).copy()
|
||||
|
||||
# Add essential metadata for retry operations
|
||||
original_request["download_type"] = download_type
|
||||
|
||||
# Add type from download_type if not provided
|
||||
if "type" not in task:
|
||||
task["type"] = download_type
|
||||
|
||||
# Ensure key information is included
|
||||
for key in ["type", "name", "artist", "service", "url"]:
|
||||
if key in task and key not in original_request:
|
||||
original_request[key] = task[key]
|
||||
|
||||
# Add API endpoint information
|
||||
if "endpoint" not in original_request:
|
||||
original_request["endpoint"] = f"/api/{download_type}/download"
|
||||
|
||||
# Add explicit display information for the frontend
|
||||
original_request["display_title"] = task.get("name", original_request.get("name", "Unknown"))
|
||||
original_request["display_type"] = task.get("type", original_request.get("type", download_type))
|
||||
original_request["display_artist"] = task.get("artist", original_request.get("artist", ""))
|
||||
|
||||
# Build the complete task with config parameters
|
||||
complete_task = {
|
||||
"download_type": download_type,
|
||||
"type": task.get("type", download_type),
|
||||
"name": task.get("name", ""),
|
||||
"artist": task.get("artist", ""),
|
||||
"service": service,
|
||||
"url": task.get("url", ""),
|
||||
|
||||
# Use config values but allow override from request
|
||||
"main": original_request.get("main",
|
||||
config_params['spotify'] if service == 'spotify' else config_params['deezer']),
|
||||
|
||||
"fallback": original_request.get("fallback",
|
||||
config_params['spotify'] if config_params['fallback'] and service == 'spotify' else None),
|
||||
|
||||
"quality": original_request.get("quality",
|
||||
config_params['spotifyQuality'] if service == 'spotify' else config_params['deezerQuality']),
|
||||
|
||||
"fall_quality": original_request.get("fall_quality", config_params['spotifyQuality']),
|
||||
|
||||
# Parse boolean parameters from string values
|
||||
"real_time": self._parse_bool_param(original_request.get("real_time"), config_params['realTime']),
|
||||
|
||||
"custom_dir_format": original_request.get("custom_dir_format", config_params['customDirFormat']),
|
||||
"custom_track_format": original_request.get("custom_track_format", config_params['customTrackFormat']),
|
||||
|
||||
# Parse boolean parameters from string values
|
||||
"pad_tracks": self._parse_bool_param(original_request.get("tracknum_padding"), config_params['tracknum_padding']),
|
||||
|
||||
"retry_count": 0,
|
||||
"original_request": original_request,
|
||||
"created_at": time.time()
|
||||
}
|
||||
|
||||
# Store the task info in Redis for later retrieval
|
||||
store_task_info(task_id, complete_task)
|
||||
|
||||
# Store initial queued status
|
||||
store_task_status(task_id, {
|
||||
"status": ProgressState.QUEUED,
|
||||
"timestamp": time.time(),
|
||||
"type": complete_task["type"],
|
||||
"name": complete_task["name"],
|
||||
"artist": complete_task["artist"],
|
||||
"retry_count": 0,
|
||||
"queue_position": len(get_all_tasks()) + 1 # Approximate queue position
|
||||
})
|
||||
|
||||
# Launch the appropriate Celery task based on download_type
|
||||
celery_task = None
|
||||
|
||||
if download_type == "track":
|
||||
celery_task = download_track.apply_async(
|
||||
kwargs=complete_task,
|
||||
task_id=task_id,
|
||||
countdown=0 if not self.paused else 3600 # Delay task if paused
|
||||
)
|
||||
elif download_type == "album":
|
||||
celery_task = download_album.apply_async(
|
||||
kwargs=complete_task,
|
||||
task_id=task_id,
|
||||
countdown=0 if not self.paused else 3600
|
||||
)
|
||||
elif download_type == "playlist":
|
||||
celery_task = download_playlist.apply_async(
|
||||
kwargs=complete_task,
|
||||
task_id=task_id,
|
||||
countdown=0 if not self.paused else 3600
|
||||
)
|
||||
else:
|
||||
# Store error status for unknown download type
|
||||
store_task_status(task_id, {
|
||||
"status": ProgressState.ERROR,
|
||||
"message": f"Unsupported download type: {download_type}",
|
||||
"timestamp": time.time()
|
||||
})
|
||||
logger.error(f"Unsupported download type: {download_type}")
|
||||
return task_id # Still return the task_id so the error can be tracked
|
||||
|
||||
logger.info(f"Added {download_type} download task {task_id} to Celery queue")
|
||||
return task_id
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error adding task to Celery queue: {e}", exc_info=True)
|
||||
# Generate a task ID even for failed tasks so we can track the error
|
||||
error_task_id = str(uuid.uuid4())
|
||||
store_task_status(error_task_id, {
|
||||
"status": ProgressState.ERROR,
|
||||
"message": f"Error adding task to queue: {str(e)}",
|
||||
"timestamp": time.time(),
|
||||
"type": task.get("type", "unknown"),
|
||||
"name": task.get("name", "Unknown"),
|
||||
"artist": task.get("artist", "")
|
||||
})
|
||||
return error_task_id
|
||||
|
||||
def _parse_bool_param(self, param_value, default_value=False):
|
||||
"""Helper function to parse boolean parameters from string values"""
|
||||
if param_value is None:
|
||||
return default_value
|
||||
if isinstance(param_value, bool):
|
||||
return param_value
|
||||
if isinstance(param_value, str):
|
||||
return param_value.lower() in ['true', '1', 'yes', 'y', 'on']
|
||||
return bool(param_value)
|
||||
|
||||
def cancel_task(self, task_id):
|
||||
"""
|
||||
Cancels a task by its ID.
|
||||
|
||||
Args:
|
||||
task_id (str): The ID of the task to cancel
|
||||
|
||||
Returns:
|
||||
dict: Status information about the cancellation
|
||||
"""
|
||||
return cancel_celery_task(task_id)
|
||||
|
||||
def retry_task(self, task_id):
|
||||
"""
|
||||
Retry a failed task.
|
||||
|
||||
Args:
|
||||
task_id (str): The ID of the failed task to retry
|
||||
|
||||
Returns:
|
||||
dict: Status information about the retry
|
||||
"""
|
||||
return retry_celery_task(task_id)
|
||||
|
||||
def cancel_all_tasks(self):
|
||||
"""
|
||||
Cancel all currently queued and running tasks.
|
||||
|
||||
Returns:
|
||||
dict: Status information about the cancellation
|
||||
"""
|
||||
tasks = get_all_tasks()
|
||||
cancelled_count = 0
|
||||
|
||||
for task in tasks:
|
||||
task_id = task.get("task_id")
|
||||
status = task.get("status")
|
||||
|
||||
# Only cancel tasks that are not already completed or cancelled
|
||||
if status not in [ProgressState.COMPLETE, ProgressState.CANCELLED]:
|
||||
result = cancel_celery_task(task_id)
|
||||
if result.get("status") == "cancelled":
|
||||
cancelled_count += 1
|
||||
|
||||
return {
|
||||
"status": "all_cancelled",
|
||||
"cancelled_count": cancelled_count,
|
||||
"total_tasks": len(tasks)
|
||||
}
|
||||
|
||||
def get_queue_status(self):
|
||||
"""
|
||||
Get the current status of the queue.
|
||||
|
||||
Returns:
|
||||
dict: Status information about the queue
|
||||
"""
|
||||
tasks = get_all_tasks()
|
||||
|
||||
# Count tasks by status
|
||||
running_count = 0
|
||||
pending_count = 0
|
||||
failed_count = 0
|
||||
|
||||
running_tasks = []
|
||||
failed_tasks = []
|
||||
|
||||
for task in tasks:
|
||||
status = task.get("status")
|
||||
|
||||
if status == ProgressState.PROCESSING:
|
||||
running_count += 1
|
||||
running_tasks.append({
|
||||
"task_id": task.get("task_id"),
|
||||
"name": task.get("name", "Unknown"),
|
||||
"type": task.get("type", "unknown"),
|
||||
"download_type": task.get("download_type", "unknown")
|
||||
})
|
||||
elif status == ProgressState.QUEUED:
|
||||
pending_count += 1
|
||||
elif status == ProgressState.ERROR:
|
||||
failed_count += 1
|
||||
|
||||
# Get task info for retry information
|
||||
task_info = get_task_info(task.get("task_id"))
|
||||
last_status = get_last_task_status(task.get("task_id"))
|
||||
|
||||
retry_count = 0
|
||||
if last_status:
|
||||
retry_count = last_status.get("retry_count", 0)
|
||||
|
||||
failed_tasks.append({
|
||||
"task_id": task.get("task_id"),
|
||||
"name": task.get("name", "Unknown"),
|
||||
"type": task.get("type", "unknown"),
|
||||
"download_type": task.get("download_type", "unknown"),
|
||||
"retry_count": retry_count
|
||||
})
|
||||
|
||||
return {
|
||||
"running": running_count,
|
||||
"pending": pending_count,
|
||||
"failed": failed_count,
|
||||
"max_concurrent": self.max_concurrent,
|
||||
"paused": self.paused,
|
||||
"running_tasks": running_tasks,
|
||||
"failed_tasks": failed_tasks
|
||||
}
|
||||
|
||||
def pause(self):
|
||||
"""Pause processing of new tasks."""
|
||||
self.paused = True
|
||||
|
||||
# Get all queued tasks
|
||||
tasks = get_all_tasks()
|
||||
for task in tasks:
|
||||
if task.get("status") == ProgressState.QUEUED:
|
||||
# Update status to indicate the task is paused
|
||||
store_task_status(task.get("task_id"), {
|
||||
"status": ProgressState.QUEUED,
|
||||
"paused": True,
|
||||
"message": "Queue is paused, task will run when queue is resumed",
|
||||
"timestamp": time.time()
|
||||
})
|
||||
|
||||
logger.info("Download queue processing paused")
|
||||
return {"status": "paused"}
|
||||
|
||||
def resume(self):
|
||||
"""Resume processing of tasks."""
|
||||
self.paused = False
|
||||
|
||||
# Get all queued tasks
|
||||
tasks = get_all_tasks()
|
||||
for task in tasks:
|
||||
if task.get("status") == ProgressState.QUEUED:
|
||||
task_id = task.get("task_id")
|
||||
|
||||
# Get the task info
|
||||
task_info = get_task_info(task_id)
|
||||
if not task_info:
|
||||
continue
|
||||
|
||||
# Update status to indicate the task is no longer paused
|
||||
store_task_status(task_id, {
|
||||
"status": ProgressState.QUEUED,
|
||||
"paused": False,
|
||||
"message": "Queue resumed, task will run soon",
|
||||
"timestamp": time.time()
|
||||
})
|
||||
|
||||
# Reschedule the task to run immediately
|
||||
download_type = task_info.get("download_type", "unknown")
|
||||
|
||||
if download_type == "track":
|
||||
download_track.apply_async(
|
||||
kwargs=task_info,
|
||||
task_id=task_id
|
||||
)
|
||||
elif download_type == "album":
|
||||
download_album.apply_async(
|
||||
kwargs=task_info,
|
||||
task_id=task_id
|
||||
)
|
||||
elif download_type == "playlist":
|
||||
download_playlist.apply_async(
|
||||
kwargs=task_info,
|
||||
task_id=task_id
|
||||
)
|
||||
|
||||
logger.info("Download queue processing resumed")
|
||||
return {"status": "resumed"}
|
||||
|
||||
def start(self):
|
||||
"""Start the queue manager (no-op for Celery implementation)."""
|
||||
logger.info("Celery Download Queue Manager started")
|
||||
return {"status": "started"}
|
||||
|
||||
def stop(self):
|
||||
"""Stop the queue manager (graceful shutdown)."""
|
||||
logger.info("Celery Download Queue Manager stopping...")
|
||||
|
||||
# Cancel all tasks or just let them finish?
|
||||
# For now, we'll let them finish and just log the shutdown
|
||||
|
||||
logger.info("Celery Download Queue Manager stopped")
|
||||
return {"status": "stopped"}
|
||||
|
||||
# Create the global instance
|
||||
download_queue_manager = CeleryDownloadQueueManager()
|
||||
653
routes/utils/celery_tasks.py
Normal file
653
routes/utils/celery_tasks.py
Normal file
@@ -0,0 +1,653 @@
|
||||
import time
|
||||
import json
|
||||
import uuid
|
||||
import logging
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from celery import Celery, Task, states
|
||||
from celery.signals import task_prerun, task_postrun, task_failure, worker_ready
|
||||
from celery.exceptions import Retry
|
||||
|
||||
# Setup Redis and Celery
|
||||
from routes.utils.celery_config import REDIS_URL, REDIS_BACKEND, get_config_params
|
||||
|
||||
# Configure logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Initialize Celery app
|
||||
celery_app = Celery('download_tasks',
|
||||
broker=REDIS_URL,
|
||||
backend=REDIS_BACKEND)
|
||||
|
||||
# Load Celery config
|
||||
celery_app.config_from_object('routes.utils.celery_config')
|
||||
|
||||
# Create Redis connection for storing task data that's not part of the Celery result backend
|
||||
import redis
|
||||
redis_client = redis.Redis.from_url(REDIS_URL)
|
||||
|
||||
class ProgressState:
|
||||
"""Enum-like class for progress states"""
|
||||
QUEUED = "queued"
|
||||
PROCESSING = "processing"
|
||||
COMPLETE = "complete"
|
||||
ERROR = "error"
|
||||
RETRYING = "retrying"
|
||||
CANCELLED = "cancel"
|
||||
|
||||
def store_task_status(task_id, status_data):
|
||||
"""Store task status information in Redis"""
|
||||
# Add timestamp if not present
|
||||
if 'timestamp' not in status_data:
|
||||
status_data['timestamp'] = time.time()
|
||||
|
||||
# Convert to JSON and store in Redis
|
||||
try:
|
||||
redis_client.rpush(f"task:{task_id}:status", json.dumps(status_data))
|
||||
# Set expiry for the list to avoid filling up Redis with old data
|
||||
redis_client.expire(f"task:{task_id}:status", 60 * 60 * 24 * 7) # 7 days
|
||||
except Exception as e:
|
||||
logger.error(f"Error storing task status: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
def get_task_status(task_id):
|
||||
"""Get all task status updates from Redis"""
|
||||
try:
|
||||
status_list = redis_client.lrange(f"task:{task_id}:status", 0, -1)
|
||||
return [json.loads(s.decode('utf-8')) for s in status_list]
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting task status: {e}")
|
||||
return []
|
||||
|
||||
def get_last_task_status(task_id):
|
||||
"""Get the most recent task status update from Redis"""
|
||||
try:
|
||||
last_status = redis_client.lindex(f"task:{task_id}:status", -1)
|
||||
if last_status:
|
||||
return json.loads(last_status.decode('utf-8'))
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting last task status: {e}")
|
||||
return None
|
||||
|
||||
def store_task_info(task_id, task_info):
|
||||
"""Store task information in Redis"""
|
||||
try:
|
||||
redis_client.set(f"task:{task_id}:info", json.dumps(task_info))
|
||||
redis_client.expire(f"task:{task_id}:info", 60 * 60 * 24 * 7) # 7 days
|
||||
except Exception as e:
|
||||
logger.error(f"Error storing task info: {e}")
|
||||
|
||||
def get_task_info(task_id):
|
||||
"""Get task information from Redis"""
|
||||
try:
|
||||
task_info = redis_client.get(f"task:{task_id}:info")
|
||||
if task_info:
|
||||
return json.loads(task_info.decode('utf-8'))
|
||||
return {}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting task info: {e}")
|
||||
return {}
|
||||
|
||||
def cancel_task(task_id):
|
||||
"""Cancel a task by its ID"""
|
||||
try:
|
||||
# Mark the task as cancelled in Redis
|
||||
store_task_status(task_id, {
|
||||
"status": ProgressState.CANCELLED,
|
||||
"message": "Task cancelled by user",
|
||||
"timestamp": time.time()
|
||||
})
|
||||
|
||||
# Try to revoke the Celery task if it hasn't started yet
|
||||
celery_app.control.revoke(task_id, terminate=True, signal='SIGTERM')
|
||||
|
||||
return {"status": "cancelled", "task_id": task_id}
|
||||
except Exception as e:
|
||||
logger.error(f"Error cancelling task {task_id}: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def retry_task(task_id):
|
||||
"""Retry a failed task"""
|
||||
try:
|
||||
# Get task info
|
||||
task_info = get_task_info(task_id)
|
||||
if not task_info:
|
||||
return {"status": "error", "message": f"Task {task_id} not found"}
|
||||
|
||||
# Check if task has retry_count information
|
||||
last_status = get_last_task_status(task_id)
|
||||
if last_status and last_status.get("status") == "error":
|
||||
# Get current retry count
|
||||
retry_count = last_status.get("retry_count", 0)
|
||||
|
||||
# Get retry configuration from config
|
||||
config_params = get_config_params()
|
||||
max_retries = config_params.get('maxRetries', 3)
|
||||
initial_retry_delay = config_params.get('retryDelaySeconds', 5)
|
||||
retry_delay_increase = config_params.get('retry_delay_increase', 5)
|
||||
|
||||
# Check if we've exceeded max retries
|
||||
if retry_count >= max_retries:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Maximum retry attempts ({max_retries}) exceeded"
|
||||
}
|
||||
|
||||
# Calculate retry delay
|
||||
retry_delay = initial_retry_delay + (retry_count * retry_delay_increase)
|
||||
|
||||
# Create a new task_id for the retry
|
||||
new_task_id = f"{task_id}_retry{retry_count + 1}"
|
||||
|
||||
# Update task info for the retry
|
||||
task_info["retry_count"] = retry_count + 1
|
||||
task_info["retry_of"] = task_id
|
||||
|
||||
# Get the service and fallback configuration from config
|
||||
service = config_params.get("service")
|
||||
fallback_enabled = config_params.get("fallback", False)
|
||||
|
||||
# Update main, fallback, and quality parameters based on service and fallback setting
|
||||
if service == 'spotify':
|
||||
if fallback_enabled:
|
||||
# If fallback is enabled with Spotify service:
|
||||
# - main becomes the Deezer account
|
||||
# - fallback becomes the Spotify account
|
||||
task_info["main"] = config_params.get("deezer", "")
|
||||
task_info["fallback"] = config_params.get("spotify", "")
|
||||
task_info["quality"] = config_params.get("deezerQuality", "MP3_128")
|
||||
task_info["fall_quality"] = config_params.get("spotifyQuality", "NORMAL")
|
||||
else:
|
||||
# If fallback is disabled with Spotify service:
|
||||
# - main is the Spotify account
|
||||
# - no fallback
|
||||
task_info["main"] = config_params.get("spotify", "")
|
||||
task_info["fallback"] = None
|
||||
task_info["quality"] = config_params.get("spotifyQuality", "NORMAL")
|
||||
task_info["fall_quality"] = None
|
||||
elif service == 'deezer':
|
||||
# For Deezer service:
|
||||
# - main is the Deezer account
|
||||
# - no fallback (even if enabled in config)
|
||||
task_info["main"] = config_params.get("deezer", "")
|
||||
task_info["fallback"] = None
|
||||
task_info["quality"] = config_params.get("deezerQuality", "MP3_128")
|
||||
task_info["fall_quality"] = None
|
||||
else:
|
||||
# Default to Spotify if unknown service
|
||||
task_info["main"] = config_params.get("spotify", "")
|
||||
task_info["fallback"] = None
|
||||
task_info["quality"] = config_params.get("spotifyQuality", "NORMAL")
|
||||
task_info["fall_quality"] = None
|
||||
|
||||
# Ensure service comes from config for the retry
|
||||
task_info["service"] = service
|
||||
|
||||
# Update other config-derived parameters
|
||||
task_info["real_time"] = task_info.get("real_time", config_params.get("realTime", False))
|
||||
task_info["custom_dir_format"] = task_info.get("custom_dir_format", config_params.get("customDirFormat", "%ar_album%/%album%"))
|
||||
task_info["custom_track_format"] = task_info.get("custom_track_format", config_params.get("customTrackFormat", "%tracknum%. %music%"))
|
||||
task_info["pad_tracks"] = task_info.get("pad_tracks", config_params.get("tracknum_padding", True))
|
||||
|
||||
# Store the updated task info
|
||||
store_task_info(new_task_id, task_info)
|
||||
|
||||
# Create a queued status
|
||||
store_task_status(new_task_id, {
|
||||
"status": ProgressState.QUEUED,
|
||||
"type": task_info.get("type", "unknown"),
|
||||
"name": task_info.get("name", "Unknown"),
|
||||
"artist": task_info.get("artist", ""),
|
||||
"retry_count": retry_count + 1,
|
||||
"max_retries": max_retries,
|
||||
"retry_delay": retry_delay,
|
||||
"timestamp": time.time()
|
||||
})
|
||||
|
||||
# Launch the appropriate task based on download_type
|
||||
download_type = task_info.get("download_type", "unknown")
|
||||
task = None
|
||||
|
||||
if download_type == "track":
|
||||
task = download_track.apply_async(
|
||||
kwargs=task_info,
|
||||
task_id=new_task_id,
|
||||
queue='downloads'
|
||||
)
|
||||
elif download_type == "album":
|
||||
task = download_album.apply_async(
|
||||
kwargs=task_info,
|
||||
task_id=new_task_id,
|
||||
queue='downloads'
|
||||
)
|
||||
elif download_type == "playlist":
|
||||
task = download_playlist.apply_async(
|
||||
kwargs=task_info,
|
||||
task_id=new_task_id,
|
||||
queue='downloads'
|
||||
)
|
||||
else:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Unknown download type: {download_type}"
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "requeued",
|
||||
"task_id": new_task_id,
|
||||
"retry_count": retry_count + 1,
|
||||
"max_retries": max_retries,
|
||||
"retry_delay": retry_delay
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Task is not in a failed state"
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrying task {task_id}: {e}")
|
||||
traceback.print_exc()
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def get_all_tasks():
|
||||
"""Get all active task IDs"""
|
||||
try:
|
||||
# Get all keys matching the task info pattern
|
||||
task_keys = redis_client.keys("task:*:info")
|
||||
|
||||
# Extract task IDs from the keys
|
||||
task_ids = [key.decode('utf-8').split(':')[1] for key in task_keys]
|
||||
|
||||
# Get info for each task
|
||||
tasks = []
|
||||
for task_id in task_ids:
|
||||
task_info = get_task_info(task_id)
|
||||
last_status = get_last_task_status(task_id)
|
||||
|
||||
if task_info and last_status:
|
||||
tasks.append({
|
||||
"task_id": task_id,
|
||||
"type": task_info.get("type", "unknown"),
|
||||
"name": task_info.get("name", "Unknown"),
|
||||
"artist": task_info.get("artist", ""),
|
||||
"download_type": task_info.get("download_type", "unknown"),
|
||||
"status": last_status.get("status", "unknown"),
|
||||
"timestamp": last_status.get("timestamp", 0)
|
||||
})
|
||||
|
||||
return tasks
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting all tasks: {e}")
|
||||
return []
|
||||
|
||||
class ProgressTrackingTask(Task):
|
||||
"""Base task class that tracks progress through callbacks"""
|
||||
|
||||
def progress_callback(self, progress_data):
|
||||
"""
|
||||
Process progress data from deezspot library callbacks
|
||||
|
||||
Args:
|
||||
progress_data: Dictionary containing progress information
|
||||
"""
|
||||
task_id = self.request.id
|
||||
|
||||
# Add timestamp if not present
|
||||
if 'timestamp' not in progress_data:
|
||||
progress_data['timestamp'] = time.time()
|
||||
|
||||
# Map deezspot status to our progress state
|
||||
status = progress_data.get("status", "unknown")
|
||||
|
||||
# Store the progress update in Redis
|
||||
store_task_status(task_id, progress_data)
|
||||
|
||||
# Log the progress update
|
||||
logger.info(f"Task {task_id} progress: {progress_data}")
|
||||
|
||||
# Celery signal handlers
|
||||
@task_prerun.connect
|
||||
def task_prerun_handler(task_id=None, task=None, *args, **kwargs):
|
||||
"""Signal handler when a task begins running"""
|
||||
try:
|
||||
# Get task info from Redis
|
||||
task_info = get_task_info(task_id)
|
||||
|
||||
# Update task status to processing
|
||||
store_task_status(task_id, {
|
||||
"status": ProgressState.PROCESSING,
|
||||
"timestamp": time.time(),
|
||||
"type": task_info.get("type", "unknown"),
|
||||
"name": task_info.get("name", "Unknown"),
|
||||
"artist": task_info.get("artist", "")
|
||||
})
|
||||
|
||||
logger.info(f"Task {task_id} started processing: {task_info.get('name', 'Unknown')}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error in task_prerun_handler: {e}")
|
||||
|
||||
@task_postrun.connect
|
||||
def task_postrun_handler(task_id=None, task=None, retval=None, state=None, *args, **kwargs):
|
||||
"""Signal handler when a task finishes"""
|
||||
try:
|
||||
# Skip if task is already marked as complete or error in Redis
|
||||
last_status = get_last_task_status(task_id)
|
||||
if last_status and last_status.get("status") in [ProgressState.COMPLETE, ProgressState.ERROR]:
|
||||
return
|
||||
|
||||
# Get task info from Redis
|
||||
task_info = get_task_info(task_id)
|
||||
|
||||
# Update task status based on Celery task state
|
||||
if state == states.SUCCESS:
|
||||
store_task_status(task_id, {
|
||||
"status": ProgressState.COMPLETE,
|
||||
"timestamp": time.time(),
|
||||
"type": task_info.get("type", "unknown"),
|
||||
"name": task_info.get("name", "Unknown"),
|
||||
"artist": task_info.get("artist", ""),
|
||||
"message": "Download completed successfully."
|
||||
})
|
||||
logger.info(f"Task {task_id} completed successfully: {task_info.get('name', 'Unknown')}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error in task_postrun_handler: {e}")
|
||||
|
||||
@task_failure.connect
|
||||
def task_failure_handler(task_id=None, exception=None, traceback=None, *args, **kwargs):
|
||||
"""Signal handler when a task fails"""
|
||||
try:
|
||||
# Skip if Retry exception (will be handled by the retry mechanism)
|
||||
if isinstance(exception, Retry):
|
||||
return
|
||||
|
||||
# Get task info and last status from Redis
|
||||
task_info = get_task_info(task_id)
|
||||
last_status = get_last_task_status(task_id)
|
||||
|
||||
# Get retry count
|
||||
retry_count = 0
|
||||
if last_status:
|
||||
retry_count = last_status.get("retry_count", 0)
|
||||
|
||||
# Get retry configuration
|
||||
config_params = get_config_params()
|
||||
max_retries = config_params.get('maxRetries', 3)
|
||||
|
||||
# Check if we can retry
|
||||
can_retry = retry_count < max_retries
|
||||
|
||||
# Update task status to error
|
||||
store_task_status(task_id, {
|
||||
"status": ProgressState.ERROR,
|
||||
"timestamp": time.time(),
|
||||
"type": task_info.get("type", "unknown"),
|
||||
"name": task_info.get("name", "Unknown"),
|
||||
"artist": task_info.get("artist", ""),
|
||||
"error": str(exception),
|
||||
"traceback": str(traceback),
|
||||
"can_retry": can_retry,
|
||||
"retry_count": retry_count,
|
||||
"max_retries": max_retries,
|
||||
"message": f"Error: {str(exception)}"
|
||||
})
|
||||
|
||||
logger.error(f"Task {task_id} failed: {str(exception)}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error in task_failure_handler: {e}")
|
||||
|
||||
@worker_ready.connect
|
||||
def worker_ready_handler(**kwargs):
|
||||
"""Signal handler when a worker starts up"""
|
||||
logger.info("Celery worker ready and listening for tasks")
|
||||
|
||||
# Check Redis connection
|
||||
try:
|
||||
redis_client.ping()
|
||||
logger.info("Redis connection successful")
|
||||
except Exception as e:
|
||||
logger.error(f"Redis connection failed: {e}")
|
||||
|
||||
# Define the download tasks
|
||||
@celery_app.task(bind=True, base=ProgressTrackingTask, name="download_track", queue="downloads")
|
||||
def download_track(self, **task_data):
|
||||
"""
|
||||
Task to download a track
|
||||
|
||||
Args:
|
||||
**task_data: Dictionary containing all task parameters
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Processing track download task: {task_data.get('name', 'Unknown')}")
|
||||
from routes.utils.track import download_track as download_track_func
|
||||
|
||||
# Get config parameters including service
|
||||
config_params = get_config_params()
|
||||
|
||||
# Get the service from config
|
||||
service = config_params.get("service")
|
||||
|
||||
# Determine main, fallback, and quality parameters based on service and fallback setting
|
||||
fallback_enabled = config_params.get("fallback", False)
|
||||
|
||||
if service == 'spotify':
|
||||
if fallback_enabled:
|
||||
# If fallback is enabled with Spotify service:
|
||||
# - main becomes the Deezer account
|
||||
# - fallback becomes the Spotify account
|
||||
main = config_params.get("deezer", "")
|
||||
fallback = config_params.get("spotify", "")
|
||||
quality = config_params.get("deezerQuality", "MP3_128")
|
||||
fall_quality = config_params.get("spotifyQuality", "NORMAL")
|
||||
else:
|
||||
# If fallback is disabled with Spotify service:
|
||||
# - main is the Spotify account
|
||||
# - no fallback
|
||||
main = config_params.get("spotify", "")
|
||||
fallback = None
|
||||
quality = config_params.get("spotifyQuality", "NORMAL")
|
||||
fall_quality = None
|
||||
elif service == 'deezer':
|
||||
# For Deezer service:
|
||||
# - main is the Deezer account
|
||||
# - no fallback (even if enabled in config)
|
||||
main = config_params.get("deezer", "")
|
||||
fallback = None
|
||||
quality = config_params.get("deezerQuality", "MP3_128")
|
||||
fall_quality = None
|
||||
else:
|
||||
# Default to Spotify if unknown service
|
||||
main = config_params.get("spotify", "")
|
||||
fallback = None
|
||||
quality = config_params.get("spotifyQuality", "NORMAL")
|
||||
fall_quality = None
|
||||
|
||||
# Get remaining parameters from task_data or config
|
||||
url = task_data.get("url", "")
|
||||
real_time = task_data.get("real_time", config_params.get("realTime", False))
|
||||
custom_dir_format = task_data.get("custom_dir_format", config_params.get("customDirFormat", "%ar_album%/%album%"))
|
||||
custom_track_format = task_data.get("custom_track_format", config_params.get("customTrackFormat", "%tracknum%. %music%"))
|
||||
pad_tracks = task_data.get("pad_tracks", config_params.get("tracknum_padding", True))
|
||||
|
||||
# Execute the download function with progress callback
|
||||
download_track_func(
|
||||
service=service,
|
||||
url=url,
|
||||
main=main,
|
||||
fallback=fallback,
|
||||
quality=quality,
|
||||
fall_quality=fall_quality,
|
||||
real_time=real_time,
|
||||
custom_dir_format=custom_dir_format,
|
||||
custom_track_format=custom_track_format,
|
||||
pad_tracks=pad_tracks,
|
||||
progress_callback=self.progress_callback # Pass the callback from our ProgressTrackingTask
|
||||
)
|
||||
|
||||
return {"status": "success", "message": "Track download completed"}
|
||||
except Exception as e:
|
||||
logger.error(f"Error in download_track task: {e}")
|
||||
traceback.print_exc()
|
||||
raise
|
||||
|
||||
@celery_app.task(bind=True, base=ProgressTrackingTask, name="download_album", queue="downloads")
|
||||
def download_album(self, **task_data):
|
||||
"""
|
||||
Task to download an album
|
||||
|
||||
Args:
|
||||
**task_data: Dictionary containing all task parameters
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Processing album download task: {task_data.get('name', 'Unknown')}")
|
||||
from routes.utils.album import download_album as download_album_func
|
||||
|
||||
# Get config parameters including service
|
||||
config_params = get_config_params()
|
||||
|
||||
# Get the service from config
|
||||
service = config_params.get("service")
|
||||
|
||||
# Determine main, fallback, and quality parameters based on service and fallback setting
|
||||
fallback_enabled = config_params.get("fallback", False)
|
||||
|
||||
if service == 'spotify':
|
||||
if fallback_enabled:
|
||||
# If fallback is enabled with Spotify service:
|
||||
# - main becomes the Deezer account
|
||||
# - fallback becomes the Spotify account
|
||||
main = config_params.get("deezer", "")
|
||||
fallback = config_params.get("spotify", "")
|
||||
quality = config_params.get("deezerQuality", "MP3_128")
|
||||
fall_quality = config_params.get("spotifyQuality", "NORMAL")
|
||||
else:
|
||||
# If fallback is disabled with Spotify service:
|
||||
# - main is the Spotify account
|
||||
# - no fallback
|
||||
main = config_params.get("spotify", "")
|
||||
fallback = None
|
||||
quality = config_params.get("spotifyQuality", "NORMAL")
|
||||
fall_quality = None
|
||||
elif service == 'deezer':
|
||||
# For Deezer service:
|
||||
# - main is the Deezer account
|
||||
# - no fallback (even if enabled in config)
|
||||
main = config_params.get("deezer", "")
|
||||
fallback = None
|
||||
quality = config_params.get("deezerQuality", "MP3_128")
|
||||
fall_quality = None
|
||||
else:
|
||||
# Default to Spotify if unknown service
|
||||
main = config_params.get("spotify", "")
|
||||
fallback = None
|
||||
quality = config_params.get("spotifyQuality", "NORMAL")
|
||||
fall_quality = None
|
||||
|
||||
# Get remaining parameters from task_data or config
|
||||
url = task_data.get("url", "")
|
||||
real_time = task_data.get("real_time", config_params.get("realTime", False))
|
||||
custom_dir_format = task_data.get("custom_dir_format", config_params.get("customDirFormat", "%ar_album%/%album%"))
|
||||
custom_track_format = task_data.get("custom_track_format", config_params.get("customTrackFormat", "%tracknum%. %music%"))
|
||||
pad_tracks = task_data.get("pad_tracks", config_params.get("tracknum_padding", True))
|
||||
|
||||
# Execute the download function with progress callback
|
||||
download_album_func(
|
||||
service=service,
|
||||
url=url,
|
||||
main=main,
|
||||
fallback=fallback,
|
||||
quality=quality,
|
||||
fall_quality=fall_quality,
|
||||
real_time=real_time,
|
||||
custom_dir_format=custom_dir_format,
|
||||
custom_track_format=custom_track_format,
|
||||
pad_tracks=pad_tracks,
|
||||
progress_callback=self.progress_callback # Pass the callback from our ProgressTrackingTask
|
||||
)
|
||||
|
||||
return {"status": "success", "message": "Album download completed"}
|
||||
except Exception as e:
|
||||
logger.error(f"Error in download_album task: {e}")
|
||||
traceback.print_exc()
|
||||
raise
|
||||
|
||||
@celery_app.task(bind=True, base=ProgressTrackingTask, name="download_playlist", queue="downloads")
|
||||
def download_playlist(self, **task_data):
|
||||
"""
|
||||
Task to download a playlist
|
||||
|
||||
Args:
|
||||
**task_data: Dictionary containing all task parameters
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Processing playlist download task: {task_data.get('name', 'Unknown')}")
|
||||
from routes.utils.playlist import download_playlist as download_playlist_func
|
||||
|
||||
# Get config parameters including service
|
||||
config_params = get_config_params()
|
||||
|
||||
# Get the service from config
|
||||
service = config_params.get("service")
|
||||
|
||||
# Determine main, fallback, and quality parameters based on service and fallback setting
|
||||
fallback_enabled = config_params.get("fallback", False)
|
||||
|
||||
if service == 'spotify':
|
||||
if fallback_enabled:
|
||||
# If fallback is enabled with Spotify service:
|
||||
# - main becomes the Deezer account
|
||||
# - fallback becomes the Spotify account
|
||||
main = config_params.get("deezer", "")
|
||||
fallback = config_params.get("spotify", "")
|
||||
quality = config_params.get("deezerQuality", "MP3_128")
|
||||
fall_quality = config_params.get("spotifyQuality", "NORMAL")
|
||||
else:
|
||||
# If fallback is disabled with Spotify service:
|
||||
# - main is the Spotify account
|
||||
# - no fallback
|
||||
main = config_params.get("spotify", "")
|
||||
fallback = None
|
||||
quality = config_params.get("spotifyQuality", "NORMAL")
|
||||
fall_quality = None
|
||||
elif service == 'deezer':
|
||||
# For Deezer service:
|
||||
# - main is the Deezer account
|
||||
# - no fallback (even if enabled in config)
|
||||
main = config_params.get("deezer", "")
|
||||
fallback = None
|
||||
quality = config_params.get("deezerQuality", "MP3_128")
|
||||
fall_quality = None
|
||||
else:
|
||||
# Default to Spotify if unknown service
|
||||
main = config_params.get("spotify", "")
|
||||
fallback = None
|
||||
quality = config_params.get("spotifyQuality", "NORMAL")
|
||||
fall_quality = None
|
||||
|
||||
# Get remaining parameters from task_data or config
|
||||
url = task_data.get("url", "")
|
||||
real_time = task_data.get("real_time", config_params.get("realTime", False))
|
||||
custom_dir_format = task_data.get("custom_dir_format", config_params.get("customDirFormat", "%ar_album%/%album%"))
|
||||
custom_track_format = task_data.get("custom_track_format", config_params.get("customTrackFormat", "%tracknum%. %music%"))
|
||||
pad_tracks = task_data.get("pad_tracks", config_params.get("tracknum_padding", True))
|
||||
|
||||
# Execute the download function with progress callback
|
||||
download_playlist_func(
|
||||
service=service,
|
||||
url=url,
|
||||
main=main,
|
||||
fallback=fallback,
|
||||
quality=quality,
|
||||
fall_quality=fall_quality,
|
||||
real_time=real_time,
|
||||
custom_dir_format=custom_dir_format,
|
||||
custom_track_format=custom_track_format,
|
||||
pad_tracks=pad_tracks,
|
||||
progress_callback=self.progress_callback # Pass the callback from our ProgressTrackingTask
|
||||
)
|
||||
|
||||
return {"status": "success", "message": "Playlist download completed"}
|
||||
except Exception as e:
|
||||
logger.error(f"Error in download_playlist task: {e}")
|
||||
traceback.print_exc()
|
||||
raise
|
||||
@@ -18,7 +18,8 @@ def download_playlist(
|
||||
pad_tracks=True,
|
||||
initial_retry_delay=5,
|
||||
retry_delay_increase=5,
|
||||
max_retries=3
|
||||
max_retries=3,
|
||||
progress_callback=None
|
||||
):
|
||||
try:
|
||||
# Load Spotify client credentials if available
|
||||
@@ -51,7 +52,8 @@ def download_playlist(
|
||||
dl = DeeLogin(
|
||||
arl=deezer_creds.get('arl', ''),
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
# Download using download_playlistspo; pass the custom formatting parameters.
|
||||
dl.download_playlistspo(
|
||||
@@ -92,7 +94,8 @@ def download_playlist(
|
||||
spo = SpoLogin(
|
||||
credentials_path=spo_creds_path,
|
||||
spotify_client_id=fallback_client_id,
|
||||
spotify_client_secret=fallback_client_secret
|
||||
spotify_client_secret=fallback_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
spo.download_playlist(
|
||||
link_playlist=url,
|
||||
@@ -126,7 +129,8 @@ def download_playlist(
|
||||
spo = SpoLogin(
|
||||
credentials_path=credentials_path,
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
spo.download_playlist(
|
||||
link_playlist=url,
|
||||
@@ -156,7 +160,8 @@ def download_playlist(
|
||||
dl = DeeLogin(
|
||||
arl=creds.get('arl', ''),
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
dl.download_playlistdee(
|
||||
link_playlist=url,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,7 +18,8 @@ def download_track(
|
||||
pad_tracks=True,
|
||||
initial_retry_delay=5,
|
||||
retry_delay_increase=5,
|
||||
max_retries=3
|
||||
max_retries=3,
|
||||
progress_callback=None
|
||||
):
|
||||
try:
|
||||
# Load Spotify client credentials if available
|
||||
@@ -49,7 +50,8 @@ def download_track(
|
||||
dl = DeeLogin(
|
||||
arl=deezer_creds.get('arl', ''),
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
dl.download_trackspo(
|
||||
link_track=url,
|
||||
@@ -86,7 +88,8 @@ def download_track(
|
||||
spo = SpoLogin(
|
||||
credentials_path=spo_creds_path,
|
||||
spotify_client_id=fallback_client_id,
|
||||
spotify_client_secret=fallback_client_secret
|
||||
spotify_client_secret=fallback_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
spo.download_track(
|
||||
link_track=url,
|
||||
@@ -113,7 +116,8 @@ def download_track(
|
||||
spo = SpoLogin(
|
||||
credentials_path=credentials_path,
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
spo.download_track(
|
||||
link_track=url,
|
||||
@@ -142,7 +146,8 @@ def download_track(
|
||||
dl = DeeLogin(
|
||||
arl=creds.get('arl', ''),
|
||||
spotify_client_id=spotify_client_id,
|
||||
spotify_client_secret=spotify_client_secret
|
||||
spotify_client_secret=spotify_client_secret,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
dl.download_trackdee(
|
||||
link_track=url,
|
||||
|
||||
Reference in New Issue
Block a user