From d7ed7b90be1dde59323e3b21b1fb6f639384e31b Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 4 Aug 2018 11:04:13 -0500 Subject: [PATCH] Fix Reddit comments and add links to author channels/profiles --- src/invidious.cr | 5 +++-- src/invidious/helpers.cr | 28 +++++++++++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 5af868c5..088bf8b8 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -73,7 +73,7 @@ PG_URL = URI.new( PG_DB = DB.open PG_URL YT_URL = URI.parse("https://www.youtube.com") -REDDIT_URL = URI.parse("https://api.reddit.com") +REDDIT_URL = URI.parse("https://www.reddit.com") LOGIN_URL = URI.parse("https://accounts.google.com") crawl_threads.times do @@ -690,7 +690,7 @@ get "/api/v1/comments/:id" do |env| end elsif source == "reddit" client = make_client(REDDIT_URL) - headers = HTTP::Headers{"User-Agent" => "web:invidio.us:v0.1.0 (by /u/omarroth)"} + headers = HTTP::Headers{"User-Agent" => "web:invidio.us:v0.2.0 (by /u/omarroth)"} begin comments, reddit_thread = get_reddit_comments(id, client, headers) content_html = template_reddit_comments(comments) @@ -706,6 +706,7 @@ get "/api/v1/comments/:id" do |env| halt env, status_code: 404 end + env.response.content_type = "application/json" {"title" => reddit_thread.title, "permalink" => reddit_thread.permalink, "content_html" => content_html}.to_json diff --git a/src/invidious/helpers.cr b/src/invidious/helpers.cr index 0105ec0b..275f53a1 100644 --- a/src/invidious/helpers.cr +++ b/src/invidious/helpers.cr @@ -356,22 +356,22 @@ def search(query, page = 1) root = item.xpath_node(%q(div[contains(@class,"yt-lockup-video")]/div)) if !root next - end + end id = root.xpath_node(%q(.//div[contains(@class,"yt-lockup-thumbnail")]/a/@href)).not_nil!.content.lchop("/watch?v=") title = root.xpath_node(%q(.//div[@class="yt-lockup-content"]/h3/a)).not_nil!.content author = root.xpath_node(%q(.//div[@class="yt-lockup-content"]/div/a)).not_nil! - ucid = author["href"].rpartition("/")[-1] - author = author.content + ucid = author["href"].rpartition("/")[-1] + author = author.content published = root.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li[1])).not_nil!.content published = decode_date(published) video = ChannelVideo.new(id, title, published, Time.now, ucid, author) - videos << video - end + videos << video + end return videos end @@ -524,7 +524,7 @@ def get_reddit_comments(id, client, headers) thread = search_results.data.as(RedditListing).children.sort_by { |child| child.data.as(RedditLink).score }[-1] thread = thread.data.as(RedditLink) - result = client.get("/r/#{thread.subreddit}/comments/#{thread.id}?limit=100&sort=top", headers).body + result = client.get("/r/#{thread.subreddit}/comments/#{thread.id}.json?limit=100&sort=top", headers).body result = Array(RedditThing).from_json(result) elsif search_results.status_code == 302 result = client.get(search_results.headers["Location"], headers).body @@ -562,7 +562,9 @@ def template_youtube_comments(comments)

- [ - ] #{child["likeCount"]} #{child["author"]} + [ - ] + #{child["likeCount"]} + #{child["author"]}

#{child["content"]} @@ -606,7 +608,9 @@ def template_reddit_comments(root) content = <<-END_HTML

- [ - ] #{score} #{author} + [ - ] + #{score} + #{author}

#{body_html} @@ -721,7 +725,13 @@ def fill_links(html, scheme, host) end end - html = html.xpath_node(%q(//p[@id="eow-description"])).not_nil!.to_xml + if host == "www.youtube.com" + html = html.xpath_node(%q(//p[@id="eow-description"])).not_nil!.to_xml + else + html = html.to_xml(options: XML::SaveOptions::NO_DECL) + end + + html end def login_req(login_form, f_req)