[youtube] simplify and de-duplicate client definitions (#577)

2024-11-10 09:07:58 +01:00 · 2021-07-31 01:15:04 +05:30 · 2021-07-31 01:15:04 +05:30 · 000c15a4ca
commit 000c15a4ca
parent 9275f62cf8
1 changed files with 157 additions and 249 deletions
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@ -67,6 +67,137 @@ def parse_qs(url):
    return compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)


+# any clients starting with _ cannot be explicity requested by the user
+INNERTUBE_CLIENTS = {
+    'web': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'WEB',
+                'clientVersion': '2.20210622.10.00',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 1
+    },
+    'web_embedded': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'WEB_EMBEDDED_PLAYER',
+                'clientVersion': '1.20210620.0.1',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 56
+    },
+    'web_music': {
+        'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
+        'INNERTUBE_HOST': 'music.youtube.com',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'WEB_REMIX',
+                'clientVersion': '1.20210621.00.00',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
+    },
+    'android': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'ANDROID',
+                'clientVersion': '16.20',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
+    },
+    'android_embedded': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'ANDROID_EMBEDDED_PLAYER',
+                'clientVersion': '16.20',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 55
+    },
+    'android_music': {
+        'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
+        'INNERTUBE_HOST': 'music.youtube.com',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'ANDROID_MUSIC',
+                'clientVersion': '4.32',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
+    },
+    'ios': {
+        'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'IOS',
+                'clientVersion': '16.20',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 5
+    },
+    'ios_embedded': {
+        'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'IOS_MESSAGES_EXTENSION',
+                'clientVersion': '16.20',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 66
+    },
+    'ios_music': {
+        'INNERTUBE_API_KEY': 'AIzaSyDK3iBpDP9nHVTk2qL73FLJICfOC3c51Og',
+        'INNERTUBE_HOST': 'music.youtube.com',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'IOS_MUSIC',
+                'clientVersion': '4.32',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 26
+    },
+    'mweb': {
+        'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'MWEB',
+                'clientVersion': '2.20210721.07.00',
+            }
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 2
+    },
+}
+
+
+def build_innertube_clients():
+    base_clients = ('android', 'web', 'ios', 'mweb')
+    priority = qualities(base_clients[::-1])
+
+    for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
+        ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM4DrUqRUYnGn3llEO78bcxq8')
+        ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
+        ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
+        ytcfg['priority'] = 10 * priority(client.split('_', 1)[0])
+
+        if client in base_clients:
+            INNERTUBE_CLIENTS[f'{client}_agegate'] = agegate_ytcfg = copy.deepcopy(ytcfg)
+            agegate_ytcfg['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
+            agegate_ytcfg['priority'] -= 1
+        elif client.endswith('_embedded'):
+            ytcfg['priority'] -= 2
+        else:
+            ytcfg['priority'] -= 3
+
+
+build_innertube_clients()
+
+
 class YoutubeBaseInfoExtractor(InfoExtractor):
    """Provide base functions for Youtube extractors"""
    _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
@ -312,250 +443,22 @@ def _real_initialize(self):
    _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
    _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'

-    _YT_DEFAULT_YTCFGS = {
-        'WEB': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'WEB',
-            'INNERTUBE_CLIENT_VERSION': '2.20210622.10.00',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'WEB',
-                    'clientVersion': '2.20210622.10.00',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 1
-        },
-        'WEB_AGEGATE': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'WEB',
-            'INNERTUBE_CLIENT_VERSION': '2.20210622.10.00',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'WEB',
-                    'clientVersion': '2.20210622.10.00',
-                    'clientScreen': 'EMBED',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 1
-        },
-        'WEB_REMIX': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'WEB_REMIX',
-            'INNERTUBE_CLIENT_VERSION': '1.20210621.00.00',
-            'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'WEB_REMIX',
-                    'clientVersion': '1.20210621.00.00',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 67
-        },
-        'WEB_EMBEDDED_PLAYER': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'WEB_EMBEDDED_PLAYER',
-            'INNERTUBE_CLIENT_VERSION': '1.20210620.0.1',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'WEB_EMBEDDED_PLAYER',
-                    'clientVersion': '1.20210620.0.1',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 56
-        },
-        'ANDROID': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'ANDROID',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'ANDROID',
-                    'clientVersion': '16.20',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 3
-        },
-        'ANDROID_AGEGATE': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'ANDROID',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'ANDROID',
-                    'clientVersion': '16.20',
-                    'clientScreen': 'EMBED',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 3
-        },
-        'ANDROID_EMBEDDED_PLAYER': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'ANDROID_EMBEDDED_PLAYER',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'ANDROID_EMBEDDED_PLAYER',
-                    'clientVersion': '16.20',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 55
-        },
-        'ANDROID_MUSIC': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'ANDROID_MUSIC',
-            'INNERTUBE_CLIENT_VERSION': '4.32',
-            'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'ANDROID_MUSIC',
-                    'clientVersion': '4.32',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 21
-        },
-        'IOS': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'IOS',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'IOS',
-                    'clientVersion': '16.20',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 5
-        },
-        'IOS_AGEGATE': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'IOS',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'IOS',
-                    'clientVersion': '16.20',
-                    'clientScreen': 'EMBED',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 5
-        },
-        'IOS_MUSIC': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'IOS_MUSIC',
-            'INNERTUBE_CLIENT_VERSION': '4.32',
-            'INNERTUBE_API_KEY': 'AIzaSyDK3iBpDP9nHVTk2qL73FLJICfOC3c51Og',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'IOS_MUSIC',
-                    'clientVersion': '4.32',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 26
-        },
-        'IOS_MESSAGES_EXTENSION': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'IOS_MESSAGES_EXTENSION',
-            'INNERTUBE_CLIENT_VERSION': '16.20',
-            'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'IOS_MESSAGES_EXTENSION',
-                    'clientVersion': '16.20',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 66
-        },
-        'MWEB': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'MWEB',
-            'INNERTUBE_CLIENT_VERSION': '2.20210721.07.00',
-            'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'MWEB',
-                    'clientVersion': '2.20210721.07.00',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 2
-        },
-        'MWEB_AGEGATE': {
-            'INNERTUBE_API_VERSION': 'v1',
-            'INNERTUBE_CLIENT_NAME': 'MWEB',
-            'INNERTUBE_CLIENT_VERSION': '2.20210721.07.00',
-            'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
-            'INNERTUBE_CONTEXT': {
-                'client': {
-                    'clientName': 'MWEB',
-                    'clientVersion': '2.20210721.07.00',
-                    'clientScreen': 'EMBED',
-                    'hl': 'en',
-                }
-            },
-            'INNERTUBE_CONTEXT_CLIENT_NAME': 2
-        },
-    }
+    def _get_default_ytcfg(self, client='web'):
+        return copy.deepcopy(INNERTUBE_CLIENTS[client])

-    _YT_DEFAULT_INNERTUBE_HOSTS = {
-        'DIRECT': 'youtubei.googleapis.com',
-        'WEB': 'www.youtube.com',
-        'WEB_REMIX': 'music.youtube.com',
-        'ANDROID_MUSIC': 'music.youtube.com'
-    }
+    def _get_innertube_host(self, client='web'):
+        return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']

-    # clients starting with _ cannot be explicity requested by the user
-    _YT_CLIENTS = {
-        'android': 'ANDROID',
-        'android_music': 'ANDROID_MUSIC',
-        'android_embedded': 'ANDROID_EMBEDDED_PLAYER',
-        'android_agegate': 'ANDROID_AGEGATE',
-        'ios': 'IOS',
-        'ios_music': 'IOS_MUSIC',
-        'ios_embedded': 'IOS_MESSAGES_EXTENSION',
-        'ios_agegate': 'IOS_AGEGATE',
-        'web': 'WEB',
-        'web_music': 'WEB_REMIX',
-        'web_embedded': 'WEB_EMBEDDED_PLAYER',
-        'web_agegate': 'WEB_AGEGATE',
-        'mweb': 'MWEB',
-        'mweb_agegate': 'MWEB_AGEGATE',
-    }
-
-    def _get_default_ytcfg(self, client='WEB'):
-        if client in self._YT_DEFAULT_YTCFGS:
-            return copy.deepcopy(self._YT_DEFAULT_YTCFGS[client])
-        self.write_debug(f'INNERTUBE default client {client} does not exist - falling back to WEB client.')
-        return copy.deepcopy(self._YT_DEFAULT_YTCFGS['WEB'])
-
-    def _get_innertube_host(self, client='WEB'):
-        return dict_get(self._YT_DEFAULT_INNERTUBE_HOSTS, (client, 'WEB'))
-
-    def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='WEB'):
+    def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
        # try_get but with fallback to default ytcfg client values when present
        _func = lambda y: try_get(y, getter, expected_type)
        return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))

-    def _extract_client_name(self, ytcfg, default_client='WEB'):
-        return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CLIENT_NAME'], compat_str, default_client)
+    def _extract_client_name(self, ytcfg, default_client='web'):
+        return (
+            try_get(ytcfg, lambda x: x['INNERTUBE_CLIENT_NAME'], compat_str)
+            or self._ytcfg_get_safe(
+                ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['clientName'], compat_str, default_client))

    @staticmethod
    def _extract_session_index(*data):
@ -564,13 +467,16 @@ def _extract_session_index(*data):
            if session_index is not None:
                return session_index

-    def _extract_client_version(self, ytcfg, default_client='WEB'):
-        return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str, default_client)
+    def _extract_client_version(self, ytcfg, default_client='web'):
+        return (
+            try_get(ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str)
+            or self._ytcfg_get_safe(
+                ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion'], compat_str, default_client))

-    def _extract_api_key(self, ytcfg=None, default_client='WEB'):
+    def _extract_api_key(self, ytcfg=None, default_client='web'):
        return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)

-    def _extract_context(self, ytcfg=None, default_client='WEB'):
+    def _extract_context(self, ytcfg=None, default_client='web'):
        _get_context = lambda y: try_get(y, lambda x: x['INNERTUBE_CONTEXT'], dict)
        context = _get_context(ytcfg)
        if context:
@ -612,7 +518,7 @@ def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):

    def _call_api(self, ep, query, video_id, fatal=True, headers=None,
                  note='Downloading API JSON', errnote='Unable to download API page',
-                  context=None, api_key=None, api_hostname=None, default_client='WEB'):
+                  context=None, api_key=None, api_hostname=None, default_client='web'):

        data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
        data.update(query)
@ -674,7 +580,7 @@ def extract_ytcfg(self, video_id, webpage):

    def generate_api_headers(
            self, ytcfg=None, identity_token=None, account_syncid=None,
-            visitor_data=None, api_hostname=None, default_client='WEB', session_index=None):
+            visitor_data=None, api_hostname=None, default_client='web', session_index=None):
        origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
        headers = {
            'X-YouTube-Client-Name': compat_str(
@ -819,7 +725,7 @@ def _get_text(data, *path_list, max_runs=None):

    def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
                          ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
-                          default_client='WEB'):
+                          default_client='web'):
        response = None
        last_error = None
        count = -1
@ -2452,20 +2358,22 @@ def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg,
        sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False)
        headers = self.generate_api_headers(
            player_ytcfg, identity_token, syncid,
-            default_client=self._YT_CLIENTS[client], session_index=session_index)
+            default_client=client, session_index=session_index)

        yt_query = {'videoId': video_id}
        yt_query.update(self._generate_player_context(sts))
        return self._extract_response(
            item_id=video_id, ep='player', query=yt_query,
            ytcfg=player_ytcfg, headers=headers, fatal=False,
-            default_client=self._YT_CLIENTS[client],
+            default_client=client,
            note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
        ) or None

    def _get_requested_clients(self, url, smuggled_data):
        requested_clients = []
-        allowed_clients = [client for client in self._YT_CLIENTS.keys() if client[:1] != '_']
+        allowed_clients = sorted(
+            [client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'],
+            key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
        for client in self._configuration_arg('player_client'):
            if client in allowed_clients:
                requested_clients.append(client)
@ -2516,7 +2424,7 @@ def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, pl

            if self._is_agegated(pr):
                client = f'{client}_agegate'
-                if client in self._YT_CLIENTS and client not in original_clients:
+                if client in INNERTUBE_CLIENTS and client not in original_clients:
                    clients.append(client)

        # Android player_response does not have microFormats which are needed for