From f9b8bc006f0375d4f7d24f2e671d0f8ab38059dd Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 7 Mar 2022 00:52:54 +0100 Subject: [PATCH] Create a search processors module --- src/invidious.cr | 1 + src/invidious/routes/api/v1/channels.cr | 2 +- src/invidious/search.cr | 47 ++------------------- src/invidious/search/processors.cr | 54 +++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 44 deletions(-) create mode 100644 src/invidious/search/processors.cr diff --git a/src/invidious.cr b/src/invidious.cr index a470c6b6..9f3d5d10 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -35,6 +35,7 @@ require "./invidious/frontend/*" require "./invidious/*" require "./invidious/channels/*" require "./invidious/user/*" +require "./invidious/search/*" require "./invidious/routes/**" require "./invidious/jobs/**" diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index c4d6643a..c4395353 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -262,7 +262,7 @@ module Invidious::Routes::API::V1::Channels page = env.params.query["page"]?.try &.to_i? page ||= 1 - search_results = channel_search(query, page, ucid) + search_results = Invidious::Search::Processors.channel(query, page, ucid) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/search.cr b/src/invidious/search.cr index ae106bf6..af854653 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -5,35 +5,6 @@ class ChannelSearchException < InfoException end end -def channel_search(query, page, channel) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{channel}") - - if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{channel}") - response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 - initial_data = extract_initial_data(response.body) - ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) - raise ChannelSearchException.new(channel) if !ucid - else - ucid = channel - end - - continuation = produce_channel_search_continuation(ucid, query, page) - response_json = YoutubeAPI.browse(continuation) - - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| - extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } - end - - return items -end - def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem) return [] of SearchItem if query.empty? @@ -175,11 +146,6 @@ def produce_channel_search_continuation(ucid, query, page) end def process_search_query(query, page, user, region) - if user - user = user.as(Invidious::User) - view_name = "subscriptions_#{sha256(user.email)}" - end - channel = nil content_type = "all" date = "" @@ -215,16 +181,11 @@ def process_search_query(query, page, user, region) search_query = (query.split(" ") - operators).join(" ") if channel - items = channel_search(search_query, page, channel) + items = Invidious::Search::Processors.channel(search_query, page, channel) elsif subscriptions - if view_name - items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM ( - SELECT *, - to_tsvector(#{view_name}.title) || - to_tsvector(#{view_name}.author) - as document - FROM #{view_name} - ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) + if user + user = user.as(Invidious::User) + items = Invidious::Search::Processors.subscriptions(query, page, user) else items = [] of ChannelVideo end diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr new file mode 100644 index 00000000..c5327f34 --- /dev/null +++ b/src/invidious/search/processors.cr @@ -0,0 +1,54 @@ +module Invidious::Search + module Processors + extend self + + # Search a youtube channel + # TODO: clean code, and rely more on YoutubeAPI + def channel(query, page, channel) : Array(SearchItem) + response = YT_POOL.client &.get("/channel/#{channel}") + + if response.status_code == 404 + response = YT_POOL.client &.get("/user/#{channel}") + response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 + initial_data = extract_initial_data(response.body) + ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) + raise ChannelSearchException.new(channel) if !ucid + else + ucid = channel + end + + continuation = produce_channel_search_continuation(ucid, query, page) + response_json = YoutubeAPI.browse(continuation) + + continuation_items = response_json["onResponseReceivedActions"]? + .try &.[0]["appendContinuationItemsAction"]["continuationItems"] + + return [] of SearchItem if !continuation_items + + items = [] of SearchItem + continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| + extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } + end + + return items + end + + # Search inside of user subscriptions + def subscriptions(query, page, user : Invidious::User) : Array(ChannelVideo) + view_name = "subscriptions_#{sha256(user.email)}" + + return PG_DB.query_all(" + SELECT id,title,published,updated,ucid,author,length_seconds + FROM ( + SELECT *, + to_tsvector(#{view_name}.title) || + to_tsvector(#{view_name}.author) + as document + FROM #{view_name} + ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", + query, (page - 1) * 20, + as: ChannelVideo + ) + end + end +end