Merge pull request #3359 from hypsometric/gazelle_improvements

Gazelle improvements

data/interfaces/default/config.html

@@ -708,6 +708,10 @@
                                         <input id="use_redacted" type="checkbox" class="bigcheck" name="use_redacted" value="1" ${config['use_redacted']} /><label for="use_redacted"><span class="option">Redacted</span></label>
                                     </div>
                                     <div class="config">
+                                        <div class="row">
+                                            <label>Api Key</label>
+                                            <input type="text" name="redacted_apikey" value="${config['redacted_apikey']}" size="36">
+                                        </div>
                                         <div class="row">
                                             <label>Username</label>
                                             <input type="text" name="redacted_username" value="${config['redacted_username']}" size="36">

headphones/config.py

@@ -306,6 +306,7 @@ _CONFIG_DEFINITIONS = {
     'VERIFY_SSL_CERT': (bool_int, 'Advanced', 1),
     'WAIT_UNTIL_RELEASE_DATE': (int, 'General', 0),
     'REDACTED': (int, 'Redacted', 0),
+    'REDACTED_APIKEY': (str, 'Redacted', ''),
     'REDACTED_USERNAME': (str, 'Redacted', ''),
     'REDACTED_PASSWORD': (str, 'Redacted', ''),
     'REDACTED_RATIO': (str, 'Redacted', ''),

headphones/searcher.py

@@ -68,12 +68,9 @@ TORRENT_TO_MAGNET_SERVICES = [
     'https://www.seedpeer.me/torrent/%s'
 ]
 
-# Persistent Orpheus.network API object
-orpheusobj = None
 ruobj = None
-# Persistent RED API object
+# Persistent Orpheus.network and RED API objects
-redobj = None
+gazelleobjs = {}
-
 
 
 def fix_url(s, charset="utf-8"):
@@ -1330,7 +1327,7 @@ def verifyresult(title, artistterm, term, lossless):
                 dumbtoken = replace_all(token, dic)
                 if not has_token(title, dumbtoken):
                     logger.info(
-                        "Removed from results: %s (missing tokens: [%s, %s, %s])", 
+                        "Removed from results: %s (missing tokens: [%s, %s, %s])",
                         title, token, cleantoken, dumbtoken)
                     return False
 
@@ -1553,10 +1550,10 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                 if rulist:
                     resultlist.extend(rulist)
 
-    if headphones.CONFIG.ORPHEUS:
+    # RED, Orpheus.network and potentially other Gazelle API based trackers.
-        provider = "Orpheus.network"
-        providerurl = "https://orpheus.network/"
 
+    def _search_torrent_gazelle(provider, providerurl, username=None, password=None, apikey=None, try_use_fltoken=False):
+        global gazelleobjs
         bitrate = None
         bitrate_string = bitrate
 
@@ -1578,7 +1575,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                         bitrate_string = encoding_string
                 if bitrate_string not in gazelleencoding.ALL_ENCODINGS:
                     logger.info(
-                        "Your preferred bitrate is not one of the available Orpheus.network filters, so not using it as a search parameter.")
+                        f"Your preferred bitrate is not one of the available { provider } filters, so not using it as a search parameter.")
             maxsize = 10000000000
         elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless:  # Highest quality including lossless
             search_formats = [gazelleformat.FLAC, gazelleformat.MP3]
@@ -1587,64 +1584,75 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             search_formats = [gazelleformat.MP3]
             maxsize = 300000000
 
-        if not orpheusobj or not orpheusobj.logged_in():
+        gazelleobj = gazelleobjs.get(provider, None)
+        if not gazelleobj or not gazelleobj.logged_in():
             try:
-                logger.info("Attempting to log in to Orpheus.network...")
+                logger.info(f"Attempting to log in to {provider}...")
-                orpheusobj = gazelleapi.GazelleAPI(headphones.CONFIG.ORPHEUS_USERNAME,
+                if apikey:
-                                                headphones.CONFIG.ORPHEUS_PASSWORD,
+                    gazelleobj = gazelleapi.GazelleAPI(apikey=apikey,
-                                                headphones.CONFIG.ORPHEUS_URL)
+                                                        url=providerurl)
-                orpheusobj._login()
+                elif username and password:
+                    gazelleobj = gazelleapi.GazelleAPI(username=username,
+                                                        password=password,
+                                                        url=providerurl)
+                else:
+                    raise(f"Neither apikey nor username/password provided for provider {provider}.")
+                gazelleobj._login()
             except Exception as e:
-                orpheusobj = None
+                gazelleobj = None
-                logger.error("Orpheus.network credentials incorrect or site is down. Error: %s %s" % (
+                logger.error("%s credentials incorrect or site is down. Error: %s %s" % (
-                    e.__class__.__name__, str(e)))
+                    provider, e.__class__.__name__, str(e)))
+            gazelleobjs[provider] = gazelleobj
 
-        if orpheusobj and orpheusobj.logged_in():
+        if gazelleobj and gazelleobj.logged_in():
             logger.info("Searching %s..." % provider)
             all_torrents = []
 
             album_type = ""
 
             # Specify release types to filter by
-            if album['Type'] == 'Album':
+            gazelle_release_type_mapping = {
-                album_type = [gazellerelease_type.ALBUM]
+                'Album': [gazellerelease_type.ALBUM],
-            if album['Type'] == 'Soundtrack':
+                'Soundtrack': [gazellerelease_type.SOUNDTRACK],
-                album_type = [gazellerelease_type.SOUNDTRACK]
+                'EP': [gazellerelease_type.EP],
-            if album['Type'] == 'EP':
+                # No musicbrainz match for this type
-                album_type = [gazellerelease_type.EP]
+                # 'Anthology': [gazellerelease_type.ANTHOLOGY],
-            # No musicbrainz match for this type
+                'Compilation': [gazellerelease_type.COMPILATION],
-            # if album['Type'] == 'Anthology':
+                'DJ-mix': [gazellerelease_type.DJ_MIX],
-            #   album_type = [gazellerelease_type.ANTHOLOGY]
+                'Single': [gazellerelease_type.SINGLE],
-            if album['Type'] == 'Compilation':
+                'Live': [gazellerelease_type.LIVE_ALBUM],
-                album_type = [gazellerelease_type.COMPILATION]
+                'Remix': [gazellerelease_type.REMIX],
-            if album['Type'] == 'DJ-mix':
+                'Bootleg': [gazellerelease_type.BOOTLEG],
-                album_type = [gazellerelease_type.DJ_MIX]
+                'Interview': [gazellerelease_type.INTERVIEW],
-            if album['Type'] == 'Single':
+                'Mixtape/Street': [gazellerelease_type.MIXTAPE],
-                album_type = [gazellerelease_type.SINGLE]
+                'Other': [gazellerelease_type.UNKNOWN],
-            if album['Type'] == 'Live':
+            }
-                album_type = [gazellerelease_type.LIVE_ALBUM]
+
-            if album['Type'] == 'Remix':
+            album_type = gazelle_release_type_mapping.get(
-                album_type = [gazellerelease_type.REMIX]
+                album['Type'],
-            if album['Type'] == 'Bootleg':
+                gazellerelease_type.UNKNOWN
-                album_type = [gazellerelease_type.BOOTLEG]
+            )
-            if album['Type'] == 'Interview':
-                album_type = [gazellerelease_type.INTERVIEW]
-            if album['Type'] == 'Mixtape/Street':
-                album_type = [gazellerelease_type.MIXTAPE]
-            if album['Type'] == 'Other':
-                album_type = [gazellerelease_type.UNKNOWN]
 
             for search_format in search_formats:
                 if usersearchterm:
                     all_torrents.extend(
-                        orpheusobj.search_torrents(searchstr=usersearchterm, format=search_format,
+                        gazelleobj.search_torrents(
-                                                encoding=bitrate_string, releasetype=album_type)['results'])
+                            searchstr=usersearchterm,
+                            format=search_format,
+                            encoding=bitrate_string,
+                            releasetype=album_type
+                        )['results']
+                    )
                 else:
-                    all_torrents.extend(orpheusobj.search_torrents(artistname=semi_clean_artist_term,
+                    all_torrents.extend(
-                                                                groupname=semi_clean_album_term,
+                        gazelleobj.search_torrents(
-                                                                format=search_format,
+                            artistname=semi_clean_artist_term,
-                                                                encoding=bitrate_string,
+                            groupname=semi_clean_album_term,
-                                                                releasetype=album_type)['results'])
+                            format=search_format,
+                            encoding=bitrate_string,
+                            releasetype=album_type)
+                        ['results']
+                    )
 
             # filter on format, size, and num seeders
             logger.info("Filtering torrents by format, maximum size, and minimum seeders...")
@@ -1654,145 +1662,71 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             logger.info(
                 "Remaining torrents: %s" % ", ".join(repr(torrent) for torrent in match_torrents))
 
-            # sort by times d/l'd
+            # Sort by quality and seeders
             if not len(match_torrents):
                 logger.info("No results found from %s for %s after filtering" % (provider, term))
             elif len(match_torrents) > 1:
                 logger.info("Found %d matching releases from %s for %s - %s after filtering" %
                             (len(match_torrents), provider, artistterm, albumterm))
-                logger.info('Sorting torrents by number of seeders...')
-                match_torrents.sort(key=lambda x: int(x.seeders), reverse=True)
-                if gazelleformat.MP3 in search_formats:
-                    logger.info('Sorting torrents by seeders...')
-                    match_torrents.sort(key=lambda x: int(x.seeders), reverse=True)
                 if search_formats and None not in search_formats:
-                    match_torrents.sort(
+                    logger.info('Sorting torrents by format and number of seeders...')
-                        key=lambda x: int(search_formats.index(x.format)))  # prefer lossless
+                    match_torrents.sort(key=lambda x: (search_formats.index(x.format), -int(x.seeders)))
-                #                if bitrate:
+                else:
-                #                    match_torrents.sort(key=lambda x: re.match("mp3", x.getTorrentDetails(), flags=re.I), reverse=True)
+                    logger.info('Sorting torrents by number of seeders...')
-                #                    match_torrents.sort(key=lambda x: str(bitrate) in x.getTorrentFolderName(), reverse=True)
+                    match_torrents.sort(key=lambda x: int(x.seeders), reverse=True)
                 logger.info(
-                    "New order: %s" % ", ".join(repr(torrent) for torrent in match_torrents))
+                    "New order: %s" %
+                    ", ".join(repr(torrent) for torrent in match_torrents)
+                )
 
+            results = []
             for torrent in match_torrents:
                 if not torrent.file_path:
                     torrent.group.update_group_data()  # will load the file_path for the individual torrents
-                resultlist.append(
+
+                use_fltoken = try_use_fltoken and torrent.can_use_token
+
+                results.append(
                     Result(
                         torrent.file_path,
                         torrent.size,
-                        orpheusobj.generate_torrent_link(torrent.id),
+                        gazelleobj.generate_torrent_link(torrent.id, use_fltoken),
                         provider,
                         'torrent',
                         True
                     )
                 )
+            return results
+
+    if headphones.CONFIG.ORPHEUS:
+        provider = "Orpheus.network"
+        providerurl = "https://orpheus.network/"
+
+        resultlist.extend(
+            _search_torrent_gazelle(
+                provider,
+                providerurl,
+                username=headphones.CONFIG.ORPHEUS_USERNAME,
+                password=headphones.CONFIG.ORPHEUS_PASSWORD,
+                try_use_fltoken=False,
+            )
+        )
+
 
-    # Redacted - Using same logic as What.CD as it's also Gazelle, so should really make this into something reusable
     if headphones.CONFIG.REDACTED:
         provider = "Redacted"
-        providerurl = "https://redacted.ch"
+        providerurl = "https://redacted.sh"
 
-        bitrate = None
+        resultlist.extend(
-        bitrate_string = bitrate
+            _search_torrent_gazelle(
-
+                provider,
-        if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly:  # Lossless Only mode
+                providerurl,
-            search_formats = [gazelleformat.FLAC]
+                username=headphones.CONFIG.REDACTED_USERNAME,
-            maxsize = 10000000000
+                password=headphones.CONFIG.REDACTED_PASSWORD,
-        elif headphones.CONFIG.PREFERRED_QUALITY == 2:  # Preferred quality mode
+                apikey=headphones.CONFIG.REDACTED_APIKEY,
-            search_formats = [None]  # should return all
+                try_use_fltoken=headphones.CONFIG.REDACTED_USE_FLTOKEN,
-            bitrate = headphones.CONFIG.PREFERRED_BITRATE
+            )
-            if bitrate:
+        )
-                if 225 <= int(bitrate) < 256:
-                    bitrate = 'V0'
-                elif 200 <= int(bitrate) < 225:
-                    bitrate = 'V1'
-                elif 175 <= int(bitrate) < 200:
-                    bitrate = 'V2'
-                for encoding_string in gazelleencoding.ALL_ENCODINGS:
-                    if re.search(bitrate, encoding_string, flags=re.I):
-                        bitrate_string = encoding_string
-                if bitrate_string not in gazelleencoding.ALL_ENCODINGS:
-                    logger.info(
-                        "Your preferred bitrate is not one of the available RED filters, so not using it as a search parameter.")
-            maxsize = 10000000000
-        elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless:  # Highest quality including lossless
-            search_formats = [gazelleformat.FLAC, gazelleformat.MP3]
-            maxsize = 10000000000
-        else:  # Highest quality excluding lossless
-            search_formats = [gazelleformat.MP3]
-            maxsize = 300000000
-
-        if not redobj or not redobj.logged_in():
-            try:
-                logger.info("Attempting to log in to Redacted...")
-                redobj = gazelleapi.GazelleAPI(headphones.CONFIG.REDACTED_USERNAME,
-                                                headphones.CONFIG.REDACTED_PASSWORD,
-                                                providerurl)
-                redobj._login()
-            except Exception as e:
-                redobj = None
-                logger.error("Redacted credentials incorrect or site is down. Error: %s %s" % (
-                    e.__class__.__name__, str(e)))
-
-        if redobj and redobj.logged_in():
-            logger.info("Searching %s..." % provider)
-            all_torrents = []
-            for search_format in search_formats:
-                if usersearchterm:
-                    all_torrents.extend(
-                        redobj.search_torrents(searchstr=usersearchterm, format=search_format,
-                                                encoding=bitrate_string)['results'])
-                else:
-                    all_torrents.extend(redobj.search_torrents(artistname=semi_clean_artist_term,
-                                                                groupname=semi_clean_album_term,
-                                                                format=search_format,
-                                                                encoding=bitrate_string)['results'])
-
-            # filter on format, size, and num seeders
-            logger.info("Filtering torrents by format, maximum size, and minimum seeders...")
-            match_torrents = [t for t in all_torrents if
-                              t.size <= maxsize and t.seeders >= minimumseeders]
-
-            logger.info(
-                "Remaining torrents: %s" % ", ".join(repr(torrent) for torrent in match_torrents))
-
-            # sort by times d/l'd
-            if not len(match_torrents):
-                logger.info("No results found from %s for %s after filtering" % (provider, term))
-            elif len(match_torrents) > 1:
-                logger.info("Found %d matching releases from %s for %s - %s after filtering" %
-                            (len(match_torrents), provider, artistterm, albumterm))
-                logger.info(
-                    "Sorting torrents by times snatched and preferred bitrate %s..." % bitrate_string)
-                match_torrents.sort(key=lambda x: int(x.snatched), reverse=True)
-                if gazelleformat.MP3 in search_formats:
-                    # sort by size after rounding to nearest 10MB...hacky, but will favor highest quality
-                    match_torrents.sort(key=lambda x: int(10 * round(x.size / 1024. / 1024. / 10.)),
-                                        reverse=True)
-                if search_formats and None not in search_formats:
-                    match_torrents.sort(
-                        key=lambda x: int(search_formats.index(x.format)))  # prefer lossless
-                #                if bitrate:
-                #                    match_torrents.sort(key=lambda x: re.match("mp3", x.getTorrentDetails(), flags=re.I), reverse=True)
-                #                    match_torrents.sort(key=lambda x: str(bitrate) in x.getTorrentFolderName(), reverse=True)
-                logger.info(
-                    "New order: %s" % ", ".join(repr(torrent) for torrent in match_torrents))
-
-            for torrent in match_torrents:
-                if not torrent.file_path:
-                    torrent.group.update_group_data()  # will load the file_path for the individual torrents
-                use_token = headphones.CONFIG.REDACTED_USE_FLTOKEN and torrent.can_use_token
-                resultlist.append(
-                    Result(
-                        torrent.file_path,
-                        torrent.size,
-                        redobj.generate_torrent_link(torrent.id, use_token),
-                        provider,
-                        'torrent',
-                        True
-                    )
-                )
 
     # PIRATE BAY
 

headphones/webserve.py

@@ -1242,6 +1242,7 @@ class WebInterface(object):
             "orpheus_ratio": headphones.CONFIG.ORPHEUS_RATIO,
             "orpheus_url": headphones.CONFIG.ORPHEUS_URL,
             "use_redacted": checked(headphones.CONFIG.REDACTED),
+            "redacted_apikey": headphones.CONFIG.REDACTED_APIKEY,
             "redacted_username": headphones.CONFIG.REDACTED_USERNAME,
             "redacted_password": headphones.CONFIG.REDACTED_PASSWORD,
             "redacted_ratio": headphones.CONFIG.REDACTED_RATIO,

lib/pygazelle/api.py

@@ -41,11 +41,12 @@ class GazelleAPI(object):
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'}
 
 
-    def __init__(self, username=None, password=None, url=None):
+    def __init__(self, apikey=None, username=None, password=None, url=None):
         self.session = requests.session()
         self.session.headers = self.default_headers
         self.username = username
         self.password = password
+        self.apikey = apikey
         self.authkey = None
         self.passkey = None
         self.userid = None
@@ -94,14 +95,17 @@ class GazelleAPI(object):
 
         self.wait_for_rate_limit()
 
-        loginpage = self.site + 'login.php'
+        if self.apikey is not None:
-        data = {'username': self.username,
+            self.session.headers["Authorization"] = self.apikey
-                'password': self.password,
+        else:
-                'keeplogged': '1'}
+            loginpage = self.site + 'login.php'
-        r = self.session.post(loginpage, data=data, timeout=self.default_timeout, headers=self.default_headers)
+            data = {'username': self.username,
-        self.past_request_timestamps.append(time.time())
+                    'password': self.password,
-        if r.status_code != 200:
+                    'keeplogged': '1'}
-            raise LoginException("Login returned status code %s" % r.status_code)
+            r = self.session.post(loginpage, data=data, timeout=self.default_timeout, headers=self.default_headers)
+            self.past_request_timestamps.append(time.time())
+            if r.status_code != 200:
+                raise LoginException("Login returned status code %s" % r.status_code)
 
         try:
             accountinfo = self.request('index', autologin=False)