Merge remote-tracking branch 'el133/slskd-python-api_support' into develop

data/interfaces/default/config.html

@@ -482,7 +482,33 @@
                                     <label>Prefer</label>
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_0" value="0" ${config['prefer_torrents_0']}>NZBs
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_1" value="1" ${config['prefer_torrents_1']}>Torrents
-                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>No Preference
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>Soulseek
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_3" value="3" ${config['prefer_torrents_3']}>No Preference
+                                </div>
+                            </fieldset>
+                        </td>
+                        <td>
+                            <fieldset>
+                                <legend>Soulseek</legend>
+                                <div class="row">
+                                    <label>Soulseek API URL</label>
+                                    <input type="text" name="soulseek_api_url" value="${config['soulseek_api_url']}" size="50">
+                                </div>
+                                <div class="row">
+                                    <label>Soulseek API KEY</label>
+                                    <input type="text" name="soulseek_api_key" value="${config['soulseek_api_key']}" size="20">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Soulseek Download Dir:
+                                    </label>
+                                    <input type="text" name="soulseek_download_dir" value="${config['soulseek_download_dir']}" size="50">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Soulseek Incomplete Download Dir:
+                                    </label>
+                                    <input type="text" name="soulseek_incomplete_download_dir" value="${config['soulseek_incomplete_download_dir']}" size="50">
                                 </div>
                             </fieldset>
                         </td>
@@ -594,7 +620,6 @@
                                     </div>
                                 </div>
                             </fieldset>
-
                             <fieldset>
                                 <legend>Other</legend>
                                 <fieldset>
@@ -602,6 +627,11 @@
                                         <input id="use_bandcamp" type="checkbox" class="bigcheck" name="use_bandcamp" value="1" ${config['use_bandcamp']} /><label for="use_bandcamp"><span class="option">Bandcamp</span></label>
                                     </div>
                                 </fieldset>
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_soulseek" type="checkbox" class="bigcheck" name="use_soulseek" value="1" ${config['use_soulseek']} /><label for="use_soulseek"><span class="option">Soulseek</span></label>
+                                    </div>
+                                </fieldset>
                             </fieldset>
                         </td>
                         <td>

headphones/config.py

@@ -270,6 +270,11 @@ _CONFIG_DEFINITIONS = {
     'SONGKICK_ENABLED': (int, 'Songkick', 1),
     'SONGKICK_FILTER_ENABLED': (int, 'Songkick', 0),
     'SONGKICK_LOCATION': (str, 'Songkick', ''),
+    'SOULSEEK_API_URL': (str, 'Soulseek', ''),
+    'SOULSEEK_API_KEY': (str, 'Soulseek', ''),
+    'SOULSEEK_DOWNLOAD_DIR': (str, 'Soulseek', ''),
+    'SOULSEEK_INCOMPLETE_DOWNLOAD_DIR': (str, 'Soulseek', ''),
+    'SOULSEEK': (int, 'Soulseek', 0),
     'SUBSONIC_ENABLED': (int, 'Subsonic', 0),
     'SUBSONIC_HOST': (str, 'Subsonic', ''),
     'SUBSONIC_PASSWORD': (str, 'Subsonic', ''),

headphones/postprocessor.py

@@ -27,7 +27,7 @@ from beets import config as beetsconfig
 from beets import logging as beetslogging
 from mediafile import MediaFile, FileTypeError, UnreadableFileError
 from beetsplug import lyrics as beetslyrics
-from headphones import notifiers, utorrent, transmission, deluge, qbittorrent
+from headphones import notifiers, utorrent, transmission, deluge, qbittorrent, soulseek
 from headphones import db, albumart, librarysync
 from headphones import logger, helpers, mb, music_encoder
 from headphones import metadata
@@ -36,20 +36,44 @@ postprocessor_lock = threading.Lock()
 
 
 def checkFolder():
-    logger.debug("Checking download folder for completed downloads (only snatched ones).")
+    logger.info("Checking download folder for completed downloads (only snatched ones).")
 
     with postprocessor_lock:
         myDB = db.DBConnection()
         snatched = myDB.select('SELECT * from snatched WHERE Status="Snatched"')
 
+        # If soulseek is used, this part will get the status from the soulseek api and return completed and errored albums
+        completed_albums, errored_albums = set(), set()
+        if any(album['Kind'] == 'soulseek' for album in snatched):
+            completed_albums, errored_albums = soulseek.download_completed()
+
         for album in snatched:
             if album['FolderName']:
                 folder_name = album['FolderName']
                 single = False
-                if album['Kind'] == 'nzb':
-                    download_dir = headphones.CONFIG.DOWNLOAD_DIR
+                if album['Kind'] == 'soulseek':
+                    if folder_name in errored_albums:
+                        # If the album had any tracks with errors in it, the whole download is considered faulty. Status will be reset to wanted.
+                        logger.info(f"Album with folder '{folder_name}' had errors during download. Setting status to 'Wanted'.")
+                        myDB.action('UPDATE albums SET Status="Wanted" WHERE AlbumID=? AND Status="Snatched"', (album['AlbumID'],))
+                        
+                        # Folder will be removed from configured complete and Incomplete directory
+                        complete_path = os.path.join(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR, folder_name)
+                        incomplete_path = os.path.join(headphones.CONFIG.SOULSEEK_INCOMPLETE_DOWNLOAD_DIR, folder_name)
+                        for path in [complete_path, incomplete_path]:
+                            try:
+                                shutil.rmtree(path)
+                            except Exception as e:
+                                pass
+                        continue
+                    elif folder_name in completed_albums:
+                        download_dir = headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR
+                    else:
+                        continue
+                elif album['Kind'] == 'nzb':
+                    download_dir = headphones.CONFIG.DOWNLOAD_DIR   
                 elif album['Kind'] == 'bandcamp':
-                    download_dir = headphones.CONFIG.BANDCAMP_DIR
+                    download_dir = headphones.CONFIG.BANDCAMP_DIR 
                 else:
                     if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3:
                         download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY
@@ -1172,6 +1196,8 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
             download_dirs.append(dir)
         if headphones.CONFIG.DOWNLOAD_DIR and not dir:
             download_dirs.append(headphones.CONFIG.DOWNLOAD_DIR)
+        if headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR and not dir:
+            download_dirs.append(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR)
         if headphones.CONFIG.DOWNLOAD_TORRENT_DIR and not dir:
             download_dirs.append(
                 headphones.CONFIG.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, 'replace'))

headphones/searcher.py

@@ -56,6 +56,7 @@ from headphones import (
     notifiers, 
     qbittorrent, 
     rutracker, 
+    soulseek,
     transmission, 
     utorrent, 
     )
@@ -279,6 +280,8 @@ def strptime_musicbrainz(date_str):
 
 
 def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
+
+
     NZB_PROVIDERS = (headphones.CONFIG.HEADPHONES_INDEXER or
                      headphones.CONFIG.NEWZNAB or
                      headphones.CONFIG.NZBSORG or
@@ -319,7 +322,11 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
             results = searchNZB(album, new, losslessOnly, albumlength)
 
         if not results and headphones.CONFIG.BANDCAMP:
-            results = searchBandcamp(album, new, albumlength)
+            results = searchBandcamp(album, new, albumlength)    
+
+    elif headphones.CONFIG.PREFER_TORRENTS == 2 and not choose_specific_download:
+        results = searchSoulseek(album, new, losslessOnly, albumlength)
+
     else:
 
         nzb_results = None
@@ -363,6 +370,7 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
     (data, result) = preprocess(sorted_search_results)
 
     if data and result:
+        #print(f'going to send stuff to downloader. data: {data}, album: {album}')
         send_to_downloader(data, result, album)
 
 
@@ -868,11 +876,15 @@ def send_to_downloader(data, result, album):
             except Exception as e:
                 logger.error('Couldn\'t write NZB file: %s', e)
                 return
-
     elif kind == 'bandcamp':
         folder_name = bandcamp.download(album, result)
         logger.info("Setting folder_name to: {}".format(folder_name))
 
+
+    elif kind == 'soulseek':
+        soulseek.download(user=result.user, filelist=result.files)
+        folder_name = result.folder    
+        
     else:
         folder_name = '%s - %s [%s]' % (
             unidecode(album['ArtistName']).replace('/', '_'),
@@ -1929,14 +1941,49 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
     return results
 
 
+def searchSoulseek(album, new=False, losslessOnly=False, albumlength=None):
+    # Not using some of the input stuff for now or ever
+    replacements = {
+        '...': '',
+        ' & ': ' ',
+        ' = ': ' ',
+        '?': '',
+        '$': '',
+        ' + ': ' ',
+        '"': '',
+        ',': '',
+        '*': '',
+        '.': '',
+        ':': ''
+    }
+
+    num_tracks = get_album_track_count(album['AlbumID'])
+    year = get_year_from_release_date(album['ReleaseDate'])
+    cleanalbum = unidecode(helpers.replace_all(album['AlbumTitle'], replacements)).strip()
+    cleanartist = unidecode(helpers.replace_all(album['ArtistName'], replacements)).strip()
+
+    results = soulseek.search(artist=cleanartist, album=cleanalbum, year=year, losslessOnly=losslessOnly, num_tracks=num_tracks)
+
+    return results
+
+
+def get_album_track_count(album_id):
+    # Not sure if this should be considered a helper function.
+    myDB = db.DBConnection()
+    track_count = myDB.select('SELECT COUNT(*) as count FROM tracks WHERE AlbumID=?', [album_id])[0]['count']
+    return track_count
+
+
 # THIS IS KIND OF A MESS AND PROBABLY NEEDS TO BE CLEANED UP
 
 
 def preprocess(resultlist):
     for result in resultlist:
-
         headers = {'User-Agent': USER_AGENT}
 
+        if result.kind == 'soulseek':
+            return True, result
+        
         if result.kind == 'torrent':
 
             # rutracker always needs the torrent data

headphones/soulseek.py

@@ -0,0 +1,185 @@
+from collections import defaultdict, namedtuple
+import os
+import time
+import slskd_api
+import headphones
+from headphones import logger
+from datetime import datetime, timedelta
+
+Result = namedtuple('Result', ['title', 'size', 'user', 'provider', 'type', 'matches', 'bandwidth', 'hasFreeUploadSlot', 'queueLength', 'files', 'kind', 'url', 'folder'])
+
+def initialize_soulseek_client():
+    host = headphones.CONFIG.SOULSEEK_API_URL
+    api_key = headphones.CONFIG.SOULSEEK_API_KEY
+    return slskd_api.SlskdClient(host=host, api_key=api_key)
+
+    # Search logic, calling search and processing fucntions
+def search(artist, album, year, num_tracks, losslessOnly):
+    client = initialize_soulseek_client()
+    
+    # Stage 1: Search with artist, album, year, and num_tracks
+    results = execute_search(client, artist, album, year, losslessOnly)
+    processed_results = process_results(results, losslessOnly, num_tracks)
+    if processed_results:
+        return processed_results
+    
+    # Stage 2: If Stage 1 fails, search with artist, album, and num_tracks (excluding year)
+    logger.info("Soulseek search stage 1 did not meet criteria. Retrying without year...")
+    results = execute_search(client, artist, album, None, losslessOnly)
+    processed_results = process_results(results, losslessOnly, num_tracks)
+    if processed_results:
+        return processed_results
+    
+    # Stage 3: Final attempt, search only with artist and album
+    logger.info("Soulseek search stage 2 did not meet criteria. Final attempt with only artist and album.")
+    results = execute_search(client, artist, album, None, losslessOnly)
+    processed_results = process_results(results, losslessOnly, num_tracks, ignore_track_count=True)
+    
+    return processed_results
+
+def execute_search(client, artist, album, year, losslessOnly):
+    search_text = f"{artist} {album}"
+    if year:
+        search_text += f" {year}"
+    if losslessOnly:
+        search_text += ".flac"
+
+    # Actual search
+    search_response = client.searches.search_text(searchText=search_text, filterResponses=True)
+    search_id = search_response.get('id')
+    
+    # Wait for search completion and return response
+    while not client.searches.state(id=search_id).get('isComplete'):
+        time.sleep(2)
+    
+    return client.searches.search_responses(id=search_id)
+
+# Processing the search result passed
+def process_results(results, losslessOnly, num_tracks, ignore_track_count=False):
+    valid_extensions = {'.flac'} if losslessOnly else {'.mp3', '.flac'}
+    albums = defaultdict(lambda: {'files': [], 'user': None, 'hasFreeUploadSlot': None, 'queueLength': None, 'uploadSpeed': None})
+
+    # Extract info from the api response and combine files at album level
+    for result in results:
+        user = result.get('username')
+        hasFreeUploadSlot = result.get('hasFreeUploadSlot')
+        queueLength = result.get('queueLength')
+        uploadSpeed = result.get('uploadSpeed')
+
+        # Only handle .mp3 and .flac
+        for file in result.get('files', []):
+            filename = file.get('filename')
+            file_extension = os.path.splitext(filename)[1].lower()
+            if file_extension in valid_extensions:
+                album_directory = os.path.dirname(filename)
+                albums[album_directory]['files'].append(file)
+
+                # Update metadata only once per album_directory
+                if albums[album_directory]['user'] is None:
+                    albums[album_directory].update({
+                        'user': user,
+                        'hasFreeUploadSlot': hasFreeUploadSlot,
+                        'queueLength': queueLength,
+                        'uploadSpeed': uploadSpeed,
+                    })
+
+    # Filter albums based on num_tracks, add bunch of useful info to the compiled album
+    final_results = []
+    for directory, album_data in albums.items():
+        if ignore_track_count or len(album_data['files']) == num_tracks:
+            album_title = os.path.basename(directory)
+            total_size = sum(file.get('size', 0) for file in album_data['files'])
+            final_results.append(Result(
+                title=album_title,
+                size=int(total_size),
+                user=album_data['user'],
+                provider="soulseek",
+                type="soulseek",
+                matches=True,
+                bandwidth=album_data['uploadSpeed'],
+                hasFreeUploadSlot=album_data['hasFreeUploadSlot'],
+                queueLength=album_data['queueLength'],
+                files=album_data['files'],
+                kind='soulseek',
+                url='http://thisisnot.needed', # URL is needed in other parts of the program.
+                folder=os.path.basename(directory)
+            ))
+
+    return final_results
+
+
+def download(user, filelist):
+    client = initialize_soulseek_client()
+    client.transfers.enqueue(username=user, files=filelist)
+
+
+def download_completed():
+    client = initialize_soulseek_client()
+    all_downloads = client.transfers.get_all_downloads(includeRemoved=False)
+    album_completion_tracker = {}  # Tracks completion state of each album's songs
+    album_errored_tracker = {}  # Tracks albums with errored downloads
+
+    # Anything older than 24 hours will be canceled
+    cutoff_time = datetime.now() - timedelta(hours=24)
+
+    # Identify errored and completed albums
+    for download in all_downloads:
+        directories = download.get('directories', [])
+        for directory in directories:
+            album_part = directory.get('directory', '').split('\\')[-1]
+            files = directory.get('files', [])
+            for file_data in files:
+                state = file_data.get('state', '')
+                requested_at_str = file_data.get('requestedAt', '1900-01-01 00:00:00')
+                requested_at = parse_datetime(requested_at_str)
+
+                # Initialize or update album entry in trackers
+                if album_part not in album_completion_tracker:
+                    album_completion_tracker[album_part] = {'total': 0, 'completed': 0, 'errored': 0}
+                if album_part not in album_errored_tracker:
+                    album_errored_tracker[album_part] = False
+
+                album_completion_tracker[album_part]['total'] += 1
+
+                if 'Completed, Succeeded' in state:
+                    album_completion_tracker[album_part]['completed'] += 1
+                elif 'Completed, Errored' in state or requested_at < cutoff_time:
+                    album_completion_tracker[album_part]['errored'] += 1
+                    album_errored_tracker[album_part] = True  # Mark album as having errored downloads
+
+    # Identify errored albums
+    errored_albums = {album for album, errored in album_errored_tracker.items() if errored}
+
+    # Cancel downloads for errored albums
+    for download in all_downloads:
+        directories = download.get('directories', [])
+        for directory in directories:
+            album_part = directory.get('directory', '').split('\\')[-1]
+            files = directory.get('files', [])
+            for file_data in files:
+                if album_part in errored_albums:
+                    # Extract 'id' and 'username' for each file to cancel the download
+                    file_id = file_data.get('id', '')
+                    username = file_data.get('username', '')
+                    success = client.transfers.cancel_download(username, file_id)
+                    if not success:
+                        print(f"Failed to cancel download for file ID: {file_id}")
+
+    # Clear completed/canceled/errored stuff from client downloads
+    try:
+        client.transfers.remove_completed_downloads()
+    except Exception as e:
+        print(f"Failed to remove completed downloads: {e}")
+
+    # Identify completed albums
+    completed_albums = {album for album, counts in album_completion_tracker.items() if counts['total'] == counts['completed']}
+
+    # Return both completed and errored albums
+    return completed_albums, errored_albums
+
+
+def parse_datetime(datetime_string):
+    # Parse the datetime api response
+    if '.' in datetime_string:
+        datetime_string = datetime_string[:datetime_string.index('.')+7]
+    return datetime.strptime(datetime_string, '%Y-%m-%dT%H:%M:%S.%f')

headphones/webserve.py

@@ -1198,6 +1198,8 @@ class WebInterface(object):
             "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3),
             "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4),
             "download_dir": headphones.CONFIG.DOWNLOAD_DIR,
+            "soulseek_download_dir": headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR,
+            "soulseek_incomplete_download_dir": headphones.CONFIG.SOULSEEK_INCOMPLETE_DOWNLOAD_DIR,
             "use_blackhole": checked(headphones.CONFIG.BLACKHOLE),
             "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR,
             "usenet_retention": headphones.CONFIG.USENET_RETENTION,
@@ -1297,6 +1299,7 @@ class WebInterface(object):
             "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0),
             "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1),
             "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2),
+            "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3),
             "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0),
             "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1),
             "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2),
@@ -1416,7 +1419,10 @@ class WebInterface(object):
             "join_apikey": headphones.CONFIG.JOIN_APIKEY,
             "join_deviceid": headphones.CONFIG.JOIN_DEVICEID,
             "use_bandcamp": checked(headphones.CONFIG.BANDCAMP),
-            "bandcamp_dir": headphones.CONFIG.BANDCAMP_DIR
+            "bandcamp_dir": headphones.CONFIG.BANDCAMP_DIR,
+            'soulseek_api_url': headphones.CONFIG.SOULSEEK_API_URL,
+            'soulseek_api_key': headphones.CONFIG.SOULSEEK_API_KEY,
+            'use_soulseek': checked(headphones.CONFIG.SOULSEEK)
         }
 
         for k, v in config.items():