From 96ade642faad7052b0b70171a2c0ac4c09819151 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sun, 26 Nov 2023 20:24:04 -0500 Subject: [PATCH 1/2] Channel: Render age restricted channels --- src/invidious/channels/about.cr | 170 +++++++++++++----------- src/invidious/playlists.cr | 7 + src/invidious/routes/api/v1/channels.cr | 89 ++++++++++--- src/invidious/routes/channels.cr | 74 ++++++++--- 4 files changed, 221 insertions(+), 119 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index edaf5c12..b3561fcd 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -15,7 +15,8 @@ record AboutChannel, allowed_regions : Array(String), tabs : Array(String), tags : Array(String), - verified : Bool + verified : Bool, + is_age_gated : Bool def get_about_info(ucid, locale) : AboutChannel begin @@ -45,46 +46,102 @@ def get_about_info(ucid, locale) : AboutChannel end tags = [] of String + tab_names = [] of String + total_views = 0_i64 + joined = Time.unix(0) - if auto_generated - author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s - author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s - author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - - # Raises a KeyError on failure. - banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? - - description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] - # some channels have the description in a simpleText - # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ - description_node = description_base_node.dig?("simpleText") || description_base_node - - tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") - .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + if ageGate = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") + description_node = nil + author = ageGate["channelTitle"].as_s + ucid = initdata["responseContext"]["serviceTrackingParams"][0]["params"][0]["value"].as_s + author_url = "https://www.youtube.com/channel/" + ucid + author_thumbnail = ageGate["avatar"]["thumbnails"][0]["url"].as_s + banner = nil + is_family_friendly = false + is_age_gated = true + tab_names = ["videos", "shorts", "streams"] + auto_generated = false else - author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s - author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s - author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) + if auto_generated + author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s + author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s + author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s + # Raises a KeyError on failure. + banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banner = banners.try &.[-1]?.try &.["url"].as_s? - # Raises a KeyError on failure. - banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") - banner = banners.try &.[-1]?.try &.["url"].as_s? + description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.dig?("simpleText") || description_base_node - # if banner.includes? "channels/c4/default_banner" - # banner = nil - # end + tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + else + author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s + author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s + author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s + author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) - description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? - tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s + + # Raises a KeyError on failure. + banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") + banner = banners.try &.[-1]?.try &.["url"].as_s? + + # if banner.includes? "channels/c4/default_banner" + # banner = nil + # end + + description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? + tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + end + + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? + # Get the name of the tabs available on this channel + tab_names = tabs_json.as_a.compact_map do |entry| + name = entry.dig?("tabRenderer", "title").try &.as_s.downcase + + # This is a small fix to not add extra code on the HTML side + # I.e, the URL for the "live" tab is .../streams, so use "streams" + # everywhere for the sake of simplicity + (name == "live") ? "streams" : name + end + + # Get the currently active tab ("About") + about_tab = extract_selected_tab(tabs_json) + + # Try to find the about metadata section + channel_about_meta = about_tab.dig?( + "content", + "sectionListRenderer", "contents", 0, + "itemSectionRenderer", "contents", 0, + "channelAboutFullMetadataRenderer" + ) + + if !channel_about_meta.nil? + total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + + # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. + joined = extract_text(channel_about_meta["joinedDateText"]?) + .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + # Normal Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + # For auto-generated channels, channel_about_meta only has + # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] + auto_generated = ( + (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || + channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" + ) + end + end end - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool - allowed_regions = initdata .dig?("microformat", "microformatDataRenderer", "availableCountries") .try &.as_a.map(&.as_s) || [] of String @@ -102,52 +159,6 @@ def get_about_info(ucid, locale) : AboutChannel end end - total_views = 0_i64 - joined = Time.unix(0) - - tab_names = [] of String - - if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? - # Get the name of the tabs available on this channel - tab_names = tabs_json.as_a.compact_map do |entry| - name = entry.dig?("tabRenderer", "title").try &.as_s.downcase - - # This is a small fix to not add extra code on the HTML side - # I.e, the URL for the "live" tab is .../streams, so use "streams" - # everywhere for the sake of simplicity - (name == "live") ? "streams" : name - end - - # Get the currently active tab ("About") - about_tab = extract_selected_tab(tabs_json) - - # Try to find the about metadata section - channel_about_meta = about_tab.dig?( - "content", - "sectionListRenderer", "contents", 0, - "itemSectionRenderer", "contents", 0, - "channelAboutFullMetadataRenderer" - ) - - if !channel_about_meta.nil? - total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 - - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = extract_text(channel_about_meta["joinedDateText"]?) - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # Normal Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has - # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - auto_generated = ( - (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ - extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || - channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" - ) - end - end - sub_count = 0 if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) @@ -177,6 +188,7 @@ def get_about_info(ucid, locale) : AboutChannel tabs: tab_names, tags: tags, verified: author_verified || false, + is_age_gated: is_age_gated || false, ) end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index a227f794..3e6eef95 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -46,8 +46,14 @@ struct PlaylistVideo XML.build { |xml| to_xml(xml) } end + def to_json(locale : String?, json : JSON::Builder) + to_json(json) + end + def to_json(json : JSON::Builder, index : Int32? = nil) json.object do + json.field "type", "video" + json.field "title", self.title json.field "videoId", self.id @@ -67,6 +73,7 @@ struct PlaylistVideo end json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 43a5c35b..2da76134 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -27,10 +27,21 @@ module Invidious::Routes::API::V1::Channels # Retrieve "sort by" setting from URL parameters sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" - begin - videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) + rescue ex + return error_json(500, ex) + end end JSON.build do |json| @@ -84,6 +95,7 @@ module Invidious::Routes::API::V1::Channels json.field "joined", channel.joined.to_unix json.field "autoGenerated", channel.auto_generated + json.field "ageGated", channel.is_age_gated json.field "isFamilyFriendly", channel.is_family_friendly json.field "description", html_to_content(channel.description_html) json.field "descriptionHtml", channel.description_html @@ -142,12 +154,23 @@ module Invidious::Routes::API::V1::Channels sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_videos( - channel, continuation: continuation, sort_by: sort_by - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -176,12 +199,23 @@ module Invidious::Routes::API::V1::Channels # Retrieve continuation from URL parameters continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -211,12 +245,23 @@ module Invidious::Routes::API::V1::Channels sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation, sort_by: sort_by - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 360af2cd..952098e0 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -36,12 +36,24 @@ module Invidious::Routes::Channels items = items.select(SearchPlaylist) items.each(&.author = "") else - sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_videos( - channel, continuation: continuation, sort_by: (sort_by || "newest") - ) + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_options = {"newest", "oldest", "popular"} + items, next_continuation = Channel::Tabs.get_videos( + channel, continuation: continuation, sort_by: (sort_by || "newest") + ) + end end selected_tab = Frontend::ChannelPage::TabsAvailable::Videos @@ -58,14 +70,27 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for shorts - sort_by = "" - sort_options = [] of String + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + # TODO: support sort option for shorts + sort_by = "" + sort_options = [] of String - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts templated "channel" @@ -81,13 +106,26 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" - sort_options = {"newest", "oldest", "popular"} + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation, sort_by: sort_by - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Streams templated "channel" From e31053e812517d8d097368ae8863404a4a563731 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sun, 19 May 2024 10:46:05 -0400 Subject: [PATCH 2/2] Use dig to get properties Co-authored-by: Samantaz Fox --- src/invidious/channels/about.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index b3561fcd..1380044a 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -53,9 +53,9 @@ def get_about_info(ucid, locale) : AboutChannel if ageGate = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") description_node = nil author = ageGate["channelTitle"].as_s - ucid = initdata["responseContext"]["serviceTrackingParams"][0]["params"][0]["value"].as_s - author_url = "https://www.youtube.com/channel/" + ucid - author_thumbnail = ageGate["avatar"]["thumbnails"][0]["url"].as_s + ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s + author_url = "https://www.youtube.com/channel/#{ucid}" + author_thumbnail = ageGate.dig("avatar", "thumbnails", 0, "url").as_s banner = nil is_family_friendly = false is_age_gated = true