From 8b40f5194e9f1318e5d764f2c1baec40e7c36293 Mon Sep 17 00:00:00 2001 From: Xoconoch Date: Tue, 19 Aug 2025 21:17:57 -0600 Subject: [PATCH] feat: Add real_time_multiplier. This param speeds up the download time by X when real_time_dl is set to True --- deezspot/easy_spoty.py | 87 +++++++++++++++++++++---- deezspot/libutils/others_settings.py | 2 + deezspot/models/download/preferences.py | 4 +- deezspot/spotloader/__download__.py | 55 ++++++++++++---- deezspot/spotloader/__init__.py | 18 ++++- deezspot/spotloader/spotify_settings.py | 4 ++ 6 files changed, 144 insertions(+), 26 deletions(-) diff --git a/deezspot/easy_spoty.py b/deezspot/easy_spoty.py index dd87807..dde84be 100644 --- a/deezspot/easy_spoty.py +++ b/deezspot/easy_spoty.py @@ -72,16 +72,55 @@ class Spo: @classmethod def __lazy(cls, results, api=None): - """Process paginated results""" + """Process paginated results and extend the initial page's items in-place.""" api = api or cls.__api - albums = results['items'] + if not results or 'items' not in results: + return results + items_ref = results['items'] - while results['next']: + while results.get('next'): results = api.next(results) - albums.extend(results['items']) + if results and 'items' in results: + items_ref.extend(results['items']) + else: + break return results + @classmethod + def __fetch_all_album_tracks(cls, album_id: str, api: Spotify) -> dict: + """ + Fetch all tracks for an album using album_tracks pagination. + Returns a dict shaped like Spotify's 'tracks' object with all items merged. + """ + all_items = [] + limit = 50 + offset = 0 + first_page = None + while True: + page = api.album_tracks(album_id, limit=limit, offset=offset) + if first_page is None: + first_page = dict(page) if page is not None else None + items = page.get('items', []) if page else [] + if not items: + break + all_items.extend(items) + offset += len(items) + if page.get('next') is None: + break + if first_page is None: + return {'items': [], 'total': 0, 'limit': limit, 'offset': 0} + # Build a consolidated tracks object + total_val = first_page.get('total', len(all_items)) + return { + 'items': all_items, + 'total': total_val, + 'limit': limit, + 'offset': 0, + 'next': None, + 'previous': None + } + @classmethod def get_track(cls, ids, client_id=None, client_secret=None): """ @@ -108,6 +147,7 @@ class Spo: def get_tracks(cls, ids: list, market: str = None, client_id=None, client_secret=None): """ Get information for multiple tracks by a list of IDs. + Handles chunking by 50 IDs per request and merges results while preserving order. Args: ids (list): A list of Spotify track IDs. @@ -116,23 +156,35 @@ class Spo: client_secret (str, optional): Optional custom Spotify client secret. Returns: - dict: A dictionary containing a list of track information. + dict: A dictionary containing a list of track information under key 'tracks'. """ + if not ids: + return {'tracks': []} + api = cls.__get_api(client_id, client_secret) + all_tracks = [] + chunk_size = 50 try: - tracks_json = api.tracks(ids, market=market) + for i in range(0, len(ids), chunk_size): + chunk = ids[i:i + chunk_size] + resp = api.tracks(chunk, market=market) if market else api.tracks(chunk) + # Spotify returns {'tracks': [...]} for each chunk + chunk_tracks = resp.get('tracks', []) if resp else [] + all_tracks.extend(chunk_tracks) except SpotifyException as error: if error.http_status in cls.__error_codes: # Create a string of the first few IDs for the error message ids_preview = ', '.join(ids[:3]) + ('...' if len(ids) > 3 else '') raise InvalidLink(f"one or more IDs in the list: [{ids_preview}]") + else: + raise - return tracks_json + return {'tracks': all_tracks} @classmethod def get_album(cls, ids, client_id=None, client_secret=None): """ - Get album information by ID. + Get album information by ID and include all tracks (paged if needed). Args: ids (str): Spotify album ID @@ -140,7 +192,7 @@ class Spo: client_secret (str, optional): Optional custom Spotify client secret Returns: - dict: Album information + dict: Album information with full 'tracks.items' """ api = cls.__get_api(client_id, client_secret) try: @@ -148,9 +200,22 @@ class Spo: except SpotifyException as error: if error.http_status in cls.__error_codes: raise InvalidLink(ids) + else: + raise - tracks = album_json['tracks'] - cls.__lazy(tracks, api) + # Replace/ensure tracks contains all items via dedicated pagination endpoint + try: + full_tracks_obj = cls.__fetch_all_album_tracks(ids, api) + if isinstance(album_json, dict): + album_json['tracks'] = full_tracks_obj + except Exception: + # Fallback to lazy-paging over embedded 'tracks' if available + try: + tracks = album_json.get('tracks') if isinstance(album_json, dict) else None + if tracks: + cls.__lazy(tracks, api) + except Exception: + pass return album_json diff --git a/deezspot/libutils/others_settings.py b/deezspot/libutils/others_settings.py index fd8f3fb..5697894 100644 --- a/deezspot/libutils/others_settings.py +++ b/deezspot/libutils/others_settings.py @@ -22,5 +22,7 @@ stock_recursive_download = False stock_not_interface = False stock_zip = False stock_real_time_dl = False +# New: default real-time multiplier (0-10). 1 means real-time, 0 disables pacing. +stock_real_time_multiplier = 1 stock_save_cover = False # Default for saving cover image stock_market = None diff --git a/deezspot/models/download/preferences.py b/deezspot/models/download/preferences.py index 377ba49..d52c07c 100644 --- a/deezspot/models/download/preferences.py +++ b/deezspot/models/download/preferences.py @@ -28,4 +28,6 @@ class Preferences: # New: optional Spotify trackObject to use when spotify_metadata is True self.spotify_track_obj = None # New: optional Spotify albumObject (from spotloader tracking_album) for album-level spotify_metadata - self.spotify_album_obj = None \ No newline at end of file + self.spotify_album_obj = None + # New: real-time throttling multiplier (0 disables pacing, 1=real-time, >1 speeds up up to 10x) + self.real_time_multiplier: int = 1 \ No newline at end of file diff --git a/deezspot/spotloader/__download__.py b/deezspot/spotloader/__download__.py index b7eaa47..beb4f87 100644 --- a/deezspot/spotloader/__download__.py +++ b/deezspot/spotloader/__download__.py @@ -491,7 +491,20 @@ class EASY_DW: # Real-time download path duration = self.__song_metadata_dict["duration"] if duration > 0: - rate_limit = total_size / duration + # Base rate limit in bytes per second to match real-time + base_rate_limit = total_size / duration + # Multiplier handling (0 disables pacing, 1=real-time, up to 10x) + m = getattr(self.__preferences, 'real_time_multiplier', 1) + try: + m = int(m) + except Exception: + m = 1 + if m < 0: + m = 0 + if m > 10: + m = 10 + pacing_enabled = m > 0 + rate_limit = base_rate_limit * m if pacing_enabled else None chunk_size = 4096 bytes_written = 0 start_time = time.time() @@ -514,7 +527,7 @@ class EASY_DW: if current_percentage > self._last_reported_percentage: self._last_reported_percentage = current_percentage - # Report real-time progress + # Report real-time progress report_track_realtime_progress( track_obj=track_obj, time_elapsed=int((current_time - start_time) * 1000), @@ -524,10 +537,12 @@ class EASY_DW: total_tracks=total_tracks_val ) - # Rate limiting (if needed) - expected_time = bytes_written / rate_limit - if expected_time > (time.time() - start_time): - time.sleep(expected_time - (time.time() - start_time)) + # Rate limiting (if pacing is enabled) + if pacing_enabled and rate_limit: + expected_time = bytes_written / rate_limit + elapsed = time.time() - start_time + if expected_time > elapsed: + time.sleep(expected_time - elapsed) else: # Non real-time download path data = c_stream.read(total_size) @@ -805,7 +820,20 @@ class EASY_DW: if self.__real_time_dl and self.__song_metadata_dict.get("duration") and self.__song_metadata_dict["duration"] > 0: # Restored Real-time download logic for episodes duration = self.__song_metadata_dict["duration"] - rate_limit = total_size / duration + # Base rate to match real-time + base_rate_limit = total_size / duration if duration > 0 else None + # Multiplier handling + m = getattr(self.__preferences, 'real_time_multiplier', 1) + try: + m = int(m) + except Exception: + m = 1 + if m < 0: + m = 0 + if m > 10: + m = 10 + pacing_enabled = (base_rate_limit is not None) and m > 0 + rate_limit = (base_rate_limit * m) if pacing_enabled else None chunk_size = 4096 bytes_written = 0 start_time = time.time() @@ -818,10 +846,11 @@ class EASY_DW: bytes_written += len(chunk) # Optional: Real-time progress reporting for episodes (can be added here if desired) # Matching the style of download_try, no specific progress report inside this loop for episodes by default. - expected_time = bytes_written / rate_limit - elapsed_time = time.time() - start_time - if expected_time > elapsed_time: - time.sleep(expected_time - elapsed_time) + if pacing_enabled and rate_limit: + expected_time = bytes_written / rate_limit + elapsed_time = time.time() - start_time + if expected_time > elapsed_time: + time.sleep(expected_time - elapsed_time) except Exception as e_realtime: # If any error occurs during real-time download, clean up if not c_stream.closed: @@ -1047,11 +1076,11 @@ class DW_ALBUM: logger.warning(f"Track '{song_tags.get('music')}' from album '{album.album_name}' failed to download. Reason: {track.error_message}") tracks.append(track) - + # Save album cover image if self.__preferences.save_cover and album.image and album_base_directory: save_cover_image(album.image, album_base_directory, "cover.jpg") - + if self.__make_zip: song_quality = tracks[0].quality if tracks and tracks[0].quality else 'HIGH' # Fallback quality zip_name = create_zip( diff --git a/deezspot/spotloader/__init__.py b/deezspot/spotloader/__init__.py index 13d3ef5..9a1b4f2 100644 --- a/deezspot/spotloader/__init__.py +++ b/deezspot/spotloader/__init__.py @@ -35,7 +35,8 @@ from deezspot.libutils.others_settings import ( stock_zip, stock_save_cover, stock_real_time_dl, - stock_market + stock_market, + stock_real_time_multiplier ) from deezspot.libutils.logging_utils import logger, ProgressReporter, report_progress @@ -88,6 +89,7 @@ class SpoLogin: recursive_download=stock_recursive_download, not_interface=stock_not_interface, real_time_dl=stock_real_time_dl, + real_time_multiplier: int = stock_real_time_multiplier, custom_dir_format=None, custom_track_format=None, pad_tracks=True, @@ -113,6 +115,7 @@ class SpoLogin: preferences = Preferences() preferences.real_time_dl = real_time_dl + preferences.real_time_multiplier = int(real_time_multiplier) if real_time_multiplier is not None else 1 preferences.link = link_track preferences.song_metadata = song_metadata preferences.quality_download = quality_download @@ -172,6 +175,7 @@ class SpoLogin: not_interface=stock_not_interface, make_zip=stock_zip, real_time_dl=stock_real_time_dl, + real_time_multiplier: int = stock_real_time_multiplier, custom_dir_format=None, custom_track_format=None, pad_tracks=True, @@ -199,6 +203,7 @@ class SpoLogin: preferences = Preferences() preferences.real_time_dl = real_time_dl + preferences.real_time_multiplier = int(real_time_multiplier) if real_time_multiplier is not None else 1 preferences.link = link_album preferences.song_metadata = song_metadata preferences.quality_download = quality_download @@ -246,6 +251,7 @@ class SpoLogin: not_interface=stock_not_interface, make_zip=stock_zip, real_time_dl=stock_real_time_dl, + real_time_multiplier: int = stock_real_time_multiplier, custom_dir_format=None, custom_track_format=None, pad_tracks=True, @@ -306,6 +312,7 @@ class SpoLogin: preferences = Preferences() preferences.real_time_dl = real_time_dl + preferences.real_time_multiplier = int(real_time_multiplier) if real_time_multiplier is not None else 1 preferences.link = link_playlist preferences.song_metadata = song_metadata_list preferences.quality_download = quality_download @@ -353,6 +360,7 @@ class SpoLogin: recursive_download=stock_recursive_download, not_interface=stock_not_interface, real_time_dl=stock_real_time_dl, + real_time_multiplier: int = stock_real_time_multiplier, custom_dir_format=None, custom_track_format=None, pad_tracks=True, @@ -380,6 +388,7 @@ class SpoLogin: preferences = Preferences() preferences.real_time_dl = real_time_dl + preferences.real_time_multiplier = int(real_time_multiplier) if real_time_multiplier is not None else 1 preferences.link = link_episode preferences.song_metadata = episode_metadata preferences.output_dir = output_dir @@ -427,6 +436,7 @@ class SpoLogin: not_interface=stock_not_interface, make_zip=stock_zip, real_time_dl=stock_real_time_dl, + real_time_multiplier: int = stock_real_time_multiplier, custom_dir_format=None, custom_track_format=None, pad_tracks=True, @@ -468,6 +478,7 @@ class SpoLogin: not_interface=not_interface, make_zip=make_zip, real_time_dl=real_time_dl, + real_time_multiplier=real_time_multiplier, custom_dir_format=custom_dir_format, custom_track_format=custom_track_format, pad_tracks=pad_tracks, @@ -496,6 +507,7 @@ class SpoLogin: not_interface=stock_not_interface, make_zip=stock_zip, real_time_dl=stock_real_time_dl, + real_time_multiplier: int = stock_real_time_multiplier, custom_dir_format=None, custom_track_format=None, pad_tracks=True, @@ -530,6 +542,7 @@ class SpoLogin: recursive_download=recursive_download, not_interface=not_interface, real_time_dl=real_time_dl, + real_time_multiplier=real_time_multiplier, custom_dir_format=custom_dir_format, custom_track_format=custom_track_format, pad_tracks=pad_tracks, @@ -557,6 +570,7 @@ class SpoLogin: not_interface=not_interface, make_zip=make_zip, real_time_dl=real_time_dl, + real_time_multiplier=real_time_multiplier, custom_dir_format=custom_dir_format, custom_track_format=custom_track_format, pad_tracks=pad_tracks, @@ -584,6 +598,7 @@ class SpoLogin: not_interface=not_interface, make_zip=make_zip, real_time_dl=real_time_dl, + real_time_multiplier=real_time_multiplier, custom_dir_format=custom_dir_format, custom_track_format=custom_track_format, pad_tracks=pad_tracks, @@ -610,6 +625,7 @@ class SpoLogin: recursive_download=recursive_download, not_interface=not_interface, real_time_dl=real_time_dl, + real_time_multiplier=real_time_multiplier, custom_dir_format=custom_dir_format, custom_track_format=custom_track_format, pad_tracks=pad_tracks, diff --git a/deezspot/spotloader/spotify_settings.py b/deezspot/spotloader/spotify_settings.py index dee2f69..dbe20ba 100644 --- a/deezspot/spotloader/spotify_settings.py +++ b/deezspot/spotloader/spotify_settings.py @@ -25,4 +25,8 @@ qualities = { } } +# Document the allowed bounds for real-time pacing speed-ups +REAL_TIME_MULTIPLIER_MIN = 0 +REAL_TIME_MULTIPLIER_MAX = 10 + stock_market = None