From ebe8a60ca5c01b7044906e7fe38826c1c203fd6c Mon Sep 17 00:00:00 2001 From: AdeHub Date: Sun, 16 Jun 2024 20:01:35 +1200 Subject: [PATCH] Soulseek tweaks -include soulseek results in overall search results -fixed windows paths not quite working on macOS -allow user search term -tighten search for self titled, Various Artists -postprocess by user/folder -bit more logging --- headphones/postprocessor.py | 19 ++++--- headphones/searcher.py | 81 +++++++++++++++++++++++---- headphones/soulseek.py | 109 +++++++++++++++++++++++++++++------- headphones/webserve.py | 2 +- 4 files changed, 169 insertions(+), 42 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 26cc91db..4ba268fa 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -41,21 +41,22 @@ def checkFolder(): with postprocessor_lock: myDB = db.DBConnection() snatched = myDB.select('SELECT * from snatched WHERE Status="Snatched"') - - # If soulseek is used, this part will get the status from the soulseek api and return completed and errored albums - completed_albums, errored_albums = set(), set() - if any(album['Kind'] == 'soulseek' for album in snatched): - completed_albums, errored_albums = soulseek.download_completed() - for album in snatched: if album['FolderName']: folder_name = album['FolderName'] single = False + + # Soulseek, check download complete or errored if album['Kind'] == 'soulseek': - if folder_name in errored_albums: + match = re.search(r'\{(.*?)\}(.*?)$', folder_name) # get soulseek user from folder_name + user_name = match.group(1) + folder_name = match.group(2) + completed, errored = soulseek.download_completed_album(user_name, folder_name) + if errored: # If the album had any tracks with errors in it, the whole download is considered faulty. Status will be reset to wanted. - logger.info(f"Album with folder '{folder_name}' had errors during download. Setting status to 'Wanted'.") + logger.info(f"Soulseek: Album with folder '{folder_name}' had errors during download. Setting status to 'Wanted'.") myDB.action('UPDATE albums SET Status="Wanted" WHERE AlbumID=? AND Status="Snatched"', (album['AlbumID'],)) + myDB.action('UPDATE snatched SET status = "Unprocessed" WHERE AlbumID=?', (album['AlbumID'],)) # Folder will be removed from configured complete and Incomplete directory complete_path = os.path.join(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR, folder_name) @@ -66,7 +67,7 @@ def checkFolder(): except Exception as e: pass continue - elif folder_name in completed_albums: + elif completed: download_dir = headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR else: continue diff --git a/headphones/searcher.py b/headphones/searcher.py index 254412d6..1ccd9337 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -299,11 +299,21 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): headphones.CONFIG.ORPHEUS or headphones.CONFIG.REDACTED) + BANDCAMP = 1 if (headphones.CONFIG.BANDCAMP and + headphones.CONFIG.BANDCAMP_DIR) else 0 + + SOULSEEK = 1 if (headphones.CONFIG.SOULSEEK and + headphones.CONFIG.SOULSEEK_API_URL and + headphones.CONFIG.SOULSEEK_API_KEY and + headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR and + headphones.CONFIG.SOULSEEK_INCOMPLETE_DOWNLOAD_DIR) else 0 + results = [] myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] + # NZBs if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: if NZB_PROVIDERS and NZB_DOWNLOADERS: results = searchNZB(album, new, losslessOnly, albumlength) @@ -311,9 +321,13 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): if not results and TORRENT_PROVIDERS: results = searchTorrent(album, new, losslessOnly, albumlength) - if not results and headphones.CONFIG.BANDCAMP: + if not results and BANDCAMP: results = searchBandcamp(album, new, albumlength) + if not results and SOULSEEK: + results = searchSoulseek(album, new, losslessOnly, albumlength) + + # Torrents elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download: if TORRENT_PROVIDERS: results = searchTorrent(album, new, losslessOnly, albumlength) @@ -321,17 +335,32 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: results = searchNZB(album, new, losslessOnly, albumlength) - if not results and headphones.CONFIG.BANDCAMP: - results = searchBandcamp(album, new, albumlength) + if not results and BANDCAMP: + results = searchBandcamp(album, new, albumlength) + if not results and SOULSEEK: + results = searchSoulseek(album, new, losslessOnly, albumlength) + + # Soulseek elif headphones.CONFIG.PREFER_TORRENTS == 2 and not choose_specific_download: results = searchSoulseek(album, new, losslessOnly, albumlength) + if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: + results = searchNZB(album, new, losslessOnly, albumlength) + + if not results and TORRENT_PROVIDERS: + results = searchTorrent(album, new, losslessOnly, albumlength) + + if not results and BANDCAMP: + results = searchBandcamp(album, new, albumlength) + else: + # No Preference nzb_results = [] torrent_results = [] bandcamp_results = [] + soulseek_results = [] if NZB_PROVIDERS and NZB_DOWNLOADERS: nzb_results = searchNZB(album, new, losslessOnly, @@ -341,10 +370,13 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): torrent_results = searchTorrent(album, new, losslessOnly, albumlength, choose_specific_download) - if headphones.CONFIG.BANDCAMP: + if BANDCAMP: bandcamp_results = searchBandcamp(album, new, albumlength) - results = nzb_results + torrent_results + bandcamp_results + if SOULSEEK: + soulseek_results = searchSoulseek(album, new, losslessOnly, albumlength) + + results = nzb_results + torrent_results + bandcamp_results + soulseek_results if choose_specific_download: return results @@ -876,8 +908,14 @@ def send_to_downloader(data, result, album): logger.info("Setting folder_name to: {}".format(folder_name)) elif kind == 'soulseek': - soulseek.download(user=result.user, filelist=result.files) - folder_name = result.folder + try: + soulseek.download(user=result.user, filelist=result.files) + folder_name = '{' + result.user + '}' + result.folder + logger.info(f"Soulseek folder name: {result.folder}") + except Exception as e: + logger.error(f"Soulseek error, check server logs: {e}") + return + else: folder_name = '%s - %s [%s]' % ( @@ -1953,12 +1991,33 @@ def searchSoulseek(album, new=False, losslessOnly=False, albumlength=None): num_tracks = get_album_track_count(album['AlbumID']) year = get_year_from_release_date(album['ReleaseDate']) - cleanalbum = unidecode(helpers.replace_all(album['AlbumTitle'], replacements)).strip() - cleanartist = unidecode(helpers.replace_all(album['ArtistName'], replacements)).strip() + cleanalbum = unidecode(replace_all(album['AlbumTitle'], replacements)).strip() + cleanartist = unidecode(replace_all(album['ArtistName'], replacements)).strip() - results = soulseek.search(artist=cleanartist, album=cleanalbum, year=year, losslessOnly=losslessOnly, num_tracks=num_tracks) + # If Preferred Bitrate and High Limit and Allow Lossless then get both lossy and lossless + if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE and headphones.CONFIG.PREFERRED_BITRATE_HIGH_BUFFER and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS: + allow_lossless = True + else: + allow_lossless = False - return results + if headphones.CONFIG.PREFERRED_QUALITY == 3 : + losslessOnly = True + elif headphones.CONFIG.PREFERRED_QUALITY == 1: + allow_lossless = True + + if album['SearchTerm']: + term = album['SearchTerm'] + else: + term = '' + + try: + results = soulseek.search(artist=cleanartist, album=cleanalbum, year=year, losslessOnly=losslessOnly, + allow_lossless=allow_lossless, num_tracks=num_tracks, user_search_term=term) + if not results: + logger.info("No valid results found from Soulseek") + return results + except Exception as e: + logger.error(f"Soulseek error, check server logs: {e}") def get_album_track_count(album_id): diff --git a/headphones/soulseek.py b/headphones/soulseek.py index 2802c35c..27bb3571 100644 --- a/headphones/soulseek.py +++ b/headphones/soulseek.py @@ -14,40 +14,50 @@ def initialize_soulseek_client(): return slskd_api.SlskdClient(host=host, api_key=api_key) # Search logic, calling search and processing fucntions -def search(artist, album, year, num_tracks, losslessOnly): +def search(artist, album, year, num_tracks, losslessOnly, allow_lossless, user_search_term): client = initialize_soulseek_client() + + # override search string with user provided search term if entered + if user_search_term: + artist = user_search_term + album = '' + year = '' # Stage 1: Search with artist, album, year, and num_tracks - results = execute_search(client, artist, album, year, losslessOnly) - processed_results = process_results(results, losslessOnly, num_tracks) - if processed_results: + logger.info(f"Searching Soulseek using term: {artist} {album} {year}") + results = execute_search(client, artist, album, year, losslessOnly, allow_lossless) + processed_results = process_results(results, losslessOnly, allow_lossless, num_tracks) + if processed_results or user_search_term or album.lower() == artist.lower(): return processed_results # Stage 2: If Stage 1 fails, search with artist, album, and num_tracks (excluding year) logger.info("Soulseek search stage 1 did not meet criteria. Retrying without year...") - results = execute_search(client, artist, album, None, losslessOnly) - processed_results = process_results(results, losslessOnly, num_tracks) - if processed_results: + results = execute_search(client, artist, album, None, losslessOnly, allow_lossless) + processed_results = process_results(results, losslessOnly, allow_lossless, num_tracks) + if processed_results or artist == "Various Artists": return processed_results # Stage 3: Final attempt, search only with artist and album logger.info("Soulseek search stage 2 did not meet criteria. Final attempt with only artist and album.") - results = execute_search(client, artist, album, None, losslessOnly) - processed_results = process_results(results, losslessOnly, num_tracks, ignore_track_count=True) - + results = execute_search(client, artist, album, None, losslessOnly, allow_lossless) + processed_results = process_results(results, losslessOnly, allow_lossless, num_tracks, ignore_track_count=True) + return processed_results -def execute_search(client, artist, album, year, losslessOnly): +def execute_search(client, artist, album, year, losslessOnly, allow_lossless): search_text = f"{artist} {album}" if year: search_text += f" {year}" + if losslessOnly: - search_text += ".flac" + search_text += " flac" + elif not allow_lossless: + search_text += " mp3" # Actual search search_response = client.searches.search_text(searchText=search_text, filterResponses=True) search_id = search_response.get('id') - + # Wait for search completion and return response while not client.searches.state(id=search_id).get('isComplete'): time.sleep(2) @@ -55,8 +65,15 @@ def execute_search(client, artist, album, year, losslessOnly): return client.searches.search_responses(id=search_id) # Processing the search result passed -def process_results(results, losslessOnly, num_tracks, ignore_track_count=False): - valid_extensions = {'.flac'} if losslessOnly else {'.mp3', '.flac'} +def process_results(results, losslessOnly, allow_lossless, num_tracks, ignore_track_count=False): + + if losslessOnly: + valid_extensions = {'.flac'} + elif allow_lossless: + valid_extensions = {'.mp3', '.flac'} + else: + valid_extensions = {'.mp3'} + albums = defaultdict(lambda: {'files': [], 'user': None, 'hasFreeUploadSlot': None, 'queueLength': None, 'uploadSpeed': None}) # Extract info from the api response and combine files at album level @@ -71,7 +88,8 @@ def process_results(results, losslessOnly, num_tracks, ignore_track_count=False) filename = file.get('filename') file_extension = os.path.splitext(filename)[1].lower() if file_extension in valid_extensions: - album_directory = os.path.dirname(filename) + #album_directory = os.path.dirname(filename) + album_directory = filename.rsplit('\\', 1)[0] albums[album_directory]['files'].append(file) # Update metadata only once per album_directory @@ -86,8 +104,9 @@ def process_results(results, losslessOnly, num_tracks, ignore_track_count=False) # Filter albums based on num_tracks, add bunch of useful info to the compiled album final_results = [] for directory, album_data in albums.items(): - if ignore_track_count or len(album_data['files']) == num_tracks: - album_title = os.path.basename(directory) + if ignore_track_count and len(album_data['files']) > 1 or len(album_data['files']) == num_tracks: + #album_title = os.path.basename(directory) + album_title = directory.rsplit('\\', 1)[1] total_size = sum(file.get('size', 0) for file in album_data['files']) final_results.append(Result( title=album_title, @@ -102,7 +121,8 @@ def process_results(results, losslessOnly, num_tracks, ignore_track_count=False) files=album_data['files'], kind='soulseek', url='http://thisisnot.needed', # URL is needed in other parts of the program. - folder=os.path.basename(directory) + #folder=os.path.basename(directory) + folder = album_title )) return final_results @@ -163,13 +183,13 @@ def download_completed(): username = file_data.get('username', '') success = client.transfers.cancel_download(username, file_id) if not success: - print(f"Failed to cancel download for file ID: {file_id}") + logger.debug(f"Soulseek failed to cancel download for file ID: {file_id}") # Clear completed/canceled/errored stuff from client downloads try: client.transfers.remove_completed_downloads() except Exception as e: - print(f"Failed to remove completed downloads: {e}") + logger.debug(f"Soulseek failed to remove completed downloads: {e}") # Identify completed albums completed_albums = {album for album, counts in album_completion_tracker.items() if counts['total'] == counts['completed']} @@ -178,6 +198,53 @@ def download_completed(): return completed_albums, errored_albums +def download_completed_album(username, foldername): + client = initialize_soulseek_client() + downloads = client.transfers.get_downloads(username) + + # Anything older than 24 hours will be canceled + cutoff_time = datetime.now() - timedelta(hours=24) + + total_count = 0 + completed_count = 0 + errored_count = 0 + file_ids = [] + + # Identify errored and completed album + directories = downloads.get('directories', []) + for directory in directories: + album_part = directory.get('directory', '').split('\\')[-1] + if album_part == foldername: + files = directory.get('files', []) + for file_data in files: + state = file_data.get('state', '') + requested_at_str = file_data.get('requestedAt', '1900-01-01 00:00:00') + requested_at = parse_datetime(requested_at_str) + + total_count += 1 + file_id = file_data.get('id', '') + file_ids.append(file_id) + + if 'Completed, Succeeded' in state: + completed_count += 1 + elif 'Completed, Errored' in state or requested_at < cutoff_time: + errored_count += 1 + break + + completed = True if completed_count == total_count else False + errored = True if errored_count else False + + # Cancel downloads for errored album + if errored: + for file_id in file_ids: + try: + success = client.transfers.cancel_download(username, file_id, remove=True) + except Exception as e: + logger.debug(f"Soulseek failed to cancel download for folder with file ID: {foldername} {file_id}") + + return completed, errored + + def parse_datetime(datetime_string): # Parse the datetime api response if '.' in datetime_string: diff --git a/headphones/webserve.py b/headphones/webserve.py index 005f4146..26a993a4 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1491,7 +1491,7 @@ class WebInterface(object): "songkick_enabled", "songkick_filter_enabled", "mpc_enabled", "email_enabled", "email_ssl", "email_tls", "email_onsnatch", "customauth", "idtag", "deluge_paused", - "join_enabled", "join_onsnatch", "use_bandcamp" + "join_enabled", "join_onsnatch", "use_bandcamp", "use_soulseek" ] for checked_config in checked_configs: if checked_config not in kwargs: