From edfe1eefe83d4c520d1b4aaffbc59627914fefc9 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 25 Mar 2018 22:18:29 -0500 Subject: [PATCH] Add channel refresh --- src/helpers.cr | 9 +++++---- src/invidious.cr | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/helpers.cr b/src/helpers.cr index cd11baab..16e50ccd 100644 --- a/src/helpers.cr +++ b/src/helpers.cr @@ -489,11 +489,12 @@ def get_channel(id, client, db) if db.query_one?("SELECT EXISTS (SELECT true FROM channels WHERE id = $1)", id, as: Bool) channel = db.query_one("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) - if Time.now - channel.updated > 1.hours - db.exec("DELETE FROM channels * WHERE id = $1", id) + if Time.now - channel.updated > 1.minutes channel = fetch_channel(id, client) - args = arg_array(channel.to_a) - db.exec("INSERT INTO channels VALUES (#{args})", channel.to_a) + channel_array = channel.to_a[1..-1] + args = arg_array(channel_array) + + db.exec("UPDATE channels SET (rss,updated,author) = (#{args}) WHERE id = '#{channel.id}'", channel_array) end else channel = fetch_channel(id, client) diff --git a/src/invidious.cr b/src/invidious.cr index d78f8129..c9018507 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -27,6 +27,7 @@ CONFIG = Config.from_yaml(File.read("config/config.yml")) pool_size = CONFIG.pool_size threads = CONFIG.threads +channel_threads = 10 Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" @@ -122,6 +123,24 @@ threads.times do end end +channel_threads.times do |i| + spawn do + loop do + query = "SELECT id FROM channels ORDER BY updated \ + LIMIT (SELECT count(*)/#{channel_threads} FROM channels) \ + OFFSET (SELECT count(*)*#{i}/#{channel_threads} FROM channels)" + PG_DB.query(query) do |rs| + rs.each do + client = get_client(youtube_pool) + id = rs.read(String) + channel = get_channel(id, client, PG_DB) + youtube_pool << client + end + end + end + end +end + top_videos = [] of Video spawn do