From 9619d3f1bc7ad768a51935d31b4512d7aa0b79b3 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 16 Sep 2018 21:44:24 -0500 Subject: [PATCH] Fix channel refresh --- src/invidious/channels.cr | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index eabe8272..43b7edc1 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -48,6 +48,13 @@ def fetch_channel(ucid, client, db, pull_all_videos = true) end author = author.content + # Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + if author.ends_with?(" - Topic") || + {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author + auto_generated = true + end + if !pull_all_videos rss.xpath_nodes("//feed/entry").each do |entry| video_id = entry.xpath_node("videoid").not_nil!.content @@ -73,9 +80,25 @@ def fetch_channel(ucid, client, db, pull_all_videos = true) ids = [] of String loop do - videos = extract_playlist(ucid, page) + url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated) + response = client.get(url) + json = JSON.parse(response.body) + + if json["content_html"]? && !json["content_html"].as_s.empty? + document = XML.parse_html(json["content_html"].as_s) + nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) + else + break + end + + if auto_generated + videos = extract_videos(nodeset) + else + videos = extract_videos(nodeset, ucid) + end + + count = nodeset.size videos = videos.map { |video| ChannelVideo.new(video.id, video.title, video.published, Time.now, video.ucid, video.author) } - count = videos.size videos.each do |video| ids << video.id @@ -84,10 +107,10 @@ def fetch_channel(ucid, client, db, pull_all_videos = true) video_array = video.to_a args = arg_array(video_array) - db.exec("INSERT INTO channel_videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", video_array) + db.exec("INSERT INTO channel_videos VALUES (#{args}) ON CONFLICT (id) DO UPDATE SET published = $3", video_array) end - if count < 100 + if count < 30 break end