From cc59de0c9310b31b002e836ffe055dfaf5cfafd7 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Tue, 22 Feb 2022 23:04:30 -0600 Subject: [PATCH 1/5] Extract live endpoints to route --- src/invidious.cr | 41 ++++-------------------------------- src/invidious/routes/live.cr | 34 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 src/invidious/routes/live.cr diff --git a/src/invidious.cr b/src/invidious.cr index d4878759..24f49930 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -399,6 +399,10 @@ Invidious::Routing.get "/s_p/:id/:name", Invidious::Routes::Images, :s_p_image Invidious::Routing.get "/yts/img/:name", Invidious::Routes::Images, :yts_image Invidious::Routing.get "/vi/:id/:name", Invidious::Routes::Images, :thumbnails +Invidious::Routing.get "/channel/:ucid/live", Invidious::Routes::Live, :check +Invidious::Routing.get "/user/:user/live", Invidious::Routes::Live, :check +Invidious::Routing.get "/c/:user/live", Invidious::Routes::Live, :check + # API routes (macro) define_v1_api_routes() @@ -406,43 +410,6 @@ define_v1_api_routes() define_api_manifest_routes() define_video_playback_routes() -# Channels - -{"/channel/:ucid/live", "/user/:user/live", "/c/:user/live"}.each do |route| - get route do |env| - locale = env.get("preferences").as(Preferences).locale - - # Appears to be a bug in routing, having several routes configured - # as `/a/:a`, `/b/:a`, `/c/:a` results in 404 - value = env.request.resource.split("/")[2] - body = "" - {"channel", "user", "c"}.each do |type| - response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1") - if response.status_code == 200 - body = response.body - end - end - - video_id = body.match(/'VIDEO_ID': "(?[a-zA-Z0-9_-]{11})"/).try &.["id"]? - if video_id - params = [] of String - env.params.query.each do |k, v| - params << "#{k}=#{v}" - end - params = params.join("&") - - url = "/watch?v=#{video_id}" - if !params.empty? - url += "&#{params}" - end - - env.redirect url - else - env.redirect "/channel/#{value}" - end - end -end - # Authenticated endpoints # The notification APIs can't be extracted yet diff --git a/src/invidious/routes/live.cr b/src/invidious/routes/live.cr new file mode 100644 index 00000000..e55111ce --- /dev/null +++ b/src/invidious/routes/live.cr @@ -0,0 +1,34 @@ +module Invidious::Routes::Live + def self.check(env) + locale = env.get("preferences").as(Preferences).locale + + # Appears to be a bug in routing, having several routes configured + # as `/a/:a`, `/b/:a`, `/c/:a` results in 404 + value = env.request.resource.split("/")[2] + body = "" + {"channel", "user", "c"}.each do |type| + response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1") + if response.status_code == 200 + body = response.body + end + end + + video_id = body.match(/'VIDEO_ID': "(?[a-zA-Z0-9_-]{11})"/).try &.["id"]? + if video_id + params = [] of String + env.params.query.each do |k, v| + params << "#{k}=#{v}" + end + params = params.join("&") + + url = "/watch?v=#{video_id}" + if !params.empty? + url += "&#{params}" + end + + env.redirect url + else + env.redirect "/channel/#{value}" + end + end +end From 3b1837a99b7abfcc3950605fa7e99f7e0c92ba4d Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Tue, 22 Feb 2022 23:20:09 -0600 Subject: [PATCH 2/5] Move remaining routes to new structure --- src/invidious.cr | 51 +++----------------- src/invidious/routes/api/v1/authenticated.cr | 18 +++++++ src/invidious/routes/captcha.cr | 8 +++ src/invidious/routes/playlists.cr | 11 +++++ 4 files changed, 44 insertions(+), 44 deletions(-) create mode 100644 src/invidious/routes/captcha.cr diff --git a/src/invidious.cr b/src/invidious.cr index 24f49930..dc055e59 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -154,8 +154,8 @@ if CONFIG.popular_enabled Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) end -connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32) -Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, CONFIG.database_url) +CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32) +Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url) Invidious::Jobs.start_all @@ -360,6 +360,7 @@ end Invidious::Routing.post "/playlist_ajax", Invidious::Routes::Playlists, :playlist_ajax Invidious::Routing.get "/playlist", Invidious::Routes::Playlists, :show Invidious::Routing.get "/mix", Invidious::Routes::Playlists, :mix + Invidious::Routing.get "/watch_videos", Invidious::Routes::Playlists, :watch_videos Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch Invidious::Routing.get "/results", Invidious::Routes::Search, :results @@ -390,6 +391,8 @@ end Invidious::Routing.post "/subscription_ajax", Invidious::Routes::Subscriptions, :toggle_subscription Invidious::Routing.get "/subscription_manager", Invidious::Routes::Subscriptions, :subscription_manager + + Invidious::Routing.get "/Captcha", Invidious::Routes::Captcha, :get {% end %} Invidious::Routing.get "/ggpht/*", Invidious::Routes::Images, :ggpht @@ -405,53 +408,13 @@ Invidious::Routing.get "/c/:user/live", Invidious::Routes::Live, :check # API routes (macro) define_v1_api_routes() +Invidious::Routing.get "/api/v1/auth/notifications", Invidious::Routes::API::V1::Authenticated, :notifications_get +Invidious::Routing.post "/api/v1/auth/notifications", Invidious::Routes::API::V1::Authenticated, :notifications_post # Video playback (macros) define_api_manifest_routes() define_video_playback_routes() -# Authenticated endpoints - -# The notification APIs can't be extracted yet -# due to the requirement of the `connection_channel` -# used by the `NotificationJob` - -get "/api/v1/auth/notifications" do |env| - env.response.content_type = "text/event-stream" - - topics = env.params.query["topics"]?.try &.split(",").uniq.first(1000) - topics ||= [] of String - - create_notification_stream(env, topics, connection_channel) -end - -post "/api/v1/auth/notifications" do |env| - env.response.content_type = "text/event-stream" - - topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) - topics ||= [] of String - - create_notification_stream(env, topics, connection_channel) -end - -get "/Captcha" do |env| - headers = HTTP::Headers{":authority" => "accounts.google.com"} - response = YT_POOL.client &.get(env.request.resource, headers) - env.response.headers["Content-Type"] = response.headers["Content-Type"] - response.body -end - -# Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos -get "/watch_videos" do |env| - response = YT_POOL.client &.get(env.request.resource) - if url = response.headers["Location"]? - url = URI.parse(url).request_target - next env.redirect url - end - - env.response.status_code = response.status_code -end - error 404 do |env| if md = env.request.path.match(/^\/(?([a-zA-Z0-9_-]{11})|(\w+))$/) item = md["id"] diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index c27853ca..6ced4edb 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -397,4 +397,22 @@ module Invidious::Routes::API::V1::Authenticated env.response.status_code = 204 end + + def self.notifications_get(env) + env.response.content_type = "text/event-stream" + + topics = env.params.query["topics"]?.try &.split(",").uniq.first(1000) + topics ||= [] of String + + create_notification_stream(env, topics, CONNECTION_CHANNEL) + end + + def self.notifications_post(env) + env.response.content_type = "text/event-stream" + + topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) + topics ||= [] of String + + create_notification_stream(env, topics, CONNECTION_CHANNEL) + end end diff --git a/src/invidious/routes/captcha.cr b/src/invidious/routes/captcha.cr new file mode 100644 index 00000000..a1d95a4f --- /dev/null +++ b/src/invidious/routes/captcha.cr @@ -0,0 +1,8 @@ +module Invidious::Routes::Captcha + def self.get(env) + headers = HTTP::Headers{":authority" => "accounts.google.com"} + response = YT_POOL.client &.get(env.request.resource, headers) + env.response.headers["Content-Type"] = response.headers["Content-Type"] + response.body + end +end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 1ed29e79..dbeb4f97 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -443,4 +443,15 @@ module Invidious::Routes::Playlists templated "mix" end + + # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos + def self.watch_videos(env) + response = YT_POOL.client &.get(env.request.resource) + if url = response.headers["Location"]? + url = URI.parse(url).request_target + return env.redirect url + end + + env.response.status_code = response.status_code + end end From 919413e2b90371d63d88c86305575c17cef6445d Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Wed, 23 Feb 2022 22:39:43 -0600 Subject: [PATCH 3/5] Move captcha endpoint into Login route --- src/invidious.cr | 2 -- src/invidious/routes/captcha.cr | 8 -------- src/invidious/routes/login.cr | 7 +++++++ src/invidious/routing.cr | 1 + 4 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 src/invidious/routes/captcha.cr diff --git a/src/invidious.cr b/src/invidious.cr index dc055e59..140a9f7b 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -391,8 +391,6 @@ end Invidious::Routing.post "/subscription_ajax", Invidious::Routes::Subscriptions, :toggle_subscription Invidious::Routing.get "/subscription_manager", Invidious::Routes::Subscriptions, :subscription_manager - - Invidious::Routing.get "/Captcha", Invidious::Routes::Captcha, :get {% end %} Invidious::Routing.get "/ggpht/*", Invidious::Routes::Images, :ggpht diff --git a/src/invidious/routes/captcha.cr b/src/invidious/routes/captcha.cr deleted file mode 100644 index a1d95a4f..00000000 --- a/src/invidious/routes/captcha.cr +++ /dev/null @@ -1,8 +0,0 @@ -module Invidious::Routes::Captcha - def self.get(env) - headers = HTTP::Headers{":authority" => "accounts.google.com"} - response = YT_POOL.client &.get(env.request.resource, headers) - env.response.headers["Content-Type"] = response.headers["Content-Type"] - response.body - end -end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 65b337d1..99fc13a2 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -481,4 +481,11 @@ module Invidious::Routes::Login env.redirect referer end + + def self.captcha(env) + headers = HTTP::Headers{":authority" => "accounts.google.com"} + response = YT_POOL.client &.get(env.request.resource, headers) + env.response.headers["Content-Type"] = response.headers["Content-Type"] + response.body + end end diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 5efe1bd8..d539d891 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -15,6 +15,7 @@ macro define_user_routes Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page Invidious::Routing.post "/login", Invidious::Routes::Login, :login Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout + Invidious::Routing.get "/Captcha", Invidious::Routes::Login, :captcha # User preferences Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show From e215a20a0ac3dd4a3141f842ec6dd90b54cb67c3 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Wed, 23 Feb 2022 22:41:34 -0600 Subject: [PATCH 4/5] Move live endpoints into Channels route --- src/invidious.cr | 7 +++---- src/invidious/routes/channels.cr | 33 +++++++++++++++++++++++++++++++ src/invidious/routes/live.cr | 34 -------------------------------- 3 files changed, 36 insertions(+), 38 deletions(-) delete mode 100644 src/invidious/routes/live.cr diff --git a/src/invidious.cr b/src/invidious.cr index 140a9f7b..e32c1086 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -324,6 +324,9 @@ end Invidious::Routing.get "/channel/:ucid/playlists", Invidious::Routes::Channels, :playlists Invidious::Routing.get "/channel/:ucid/community", Invidious::Routes::Channels, :community Invidious::Routing.get "/channel/:ucid/about", Invidious::Routes::Channels, :about + Invidious::Routing.get "/channel/:ucid/live", Invidious::Routes::Channels, :live + Invidious::Routing.get "/user/:user/live", Invidious::Routes::Channels, :live + Invidious::Routing.get "/c/:user/live", Invidious::Routes::Channels, :live ["", "/videos", "/playlists", "/community", "/about"].each do |path| # /c/LinusTechTips @@ -400,10 +403,6 @@ Invidious::Routing.get "/s_p/:id/:name", Invidious::Routes::Images, :s_p_image Invidious::Routing.get "/yts/img/:name", Invidious::Routes::Images, :yts_image Invidious::Routing.get "/vi/:id/:name", Invidious::Routes::Images, :thumbnails -Invidious::Routing.get "/channel/:ucid/live", Invidious::Routes::Live, :check -Invidious::Routing.get "/user/:user/live", Invidious::Routes::Live, :check -Invidious::Routing.get "/c/:user/live", Invidious::Routes::Live, :check - # API routes (macro) define_v1_api_routes() Invidious::Routing.get "/api/v1/auth/notifications", Invidious::Routes::API::V1::Authenticated, :notifications_get diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 6cb1e1f7..cd2e3323 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -147,6 +147,39 @@ module Invidious::Routes::Channels end end + def self.live(env) + locale = env.get("preferences").as(Preferences).locale + + # Appears to be a bug in routing, having several routes configured + # as `/a/:a`, `/b/:a`, `/c/:a` results in 404 + value = env.request.resource.split("/")[2] + body = "" + {"channel", "user", "c"}.each do |type| + response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1") + if response.status_code == 200 + body = response.body + end + end + + video_id = body.match(/'VIDEO_ID': "(?[a-zA-Z0-9_-]{11})"/).try &.["id"]? + if video_id + params = [] of String + env.params.query.each do |k, v| + params << "#{k}=#{v}" + end + params = params.join("&") + + url = "/watch?v=#{video_id}" + if !params.empty? + url += "&#{params}" + end + + env.redirect url + else + env.redirect "/channel/#{value}" + end + end + private def self.fetch_basic_information(env) locale = env.get("preferences").as(Preferences).locale diff --git a/src/invidious/routes/live.cr b/src/invidious/routes/live.cr deleted file mode 100644 index e55111ce..00000000 --- a/src/invidious/routes/live.cr +++ /dev/null @@ -1,34 +0,0 @@ -module Invidious::Routes::Live - def self.check(env) - locale = env.get("preferences").as(Preferences).locale - - # Appears to be a bug in routing, having several routes configured - # as `/a/:a`, `/b/:a`, `/c/:a` results in 404 - value = env.request.resource.split("/")[2] - body = "" - {"channel", "user", "c"}.each do |type| - response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1") - if response.status_code == 200 - body = response.body - end - end - - video_id = body.match(/'VIDEO_ID': "(?[a-zA-Z0-9_-]{11})"/).try &.["id"]? - if video_id - params = [] of String - env.params.query.each do |k, v| - params << "#{k}=#{v}" - end - params = params.join("&") - - url = "/watch?v=#{video_id}" - if !params.empty? - url += "&#{params}" - end - - env.redirect url - else - env.redirect "/channel/#{value}" - end - end -end From d5f43bae9222cd0ba8a418615272d3edd996a4a8 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Wed, 23 Feb 2022 22:45:07 -0600 Subject: [PATCH 5/5] Combine notifications endpoints and move them --- src/invidious.cr | 2 -- src/invidious/routes/api/v1/authenticated.cr | 14 +++----------- src/invidious/routing.cr | 3 +++ 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index e32c1086..db3921f6 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -405,8 +405,6 @@ Invidious::Routing.get "/vi/:id/:name", Invidious::Routes::Images, :thumbnails # API routes (macro) define_v1_api_routes() -Invidious::Routing.get "/api/v1/auth/notifications", Invidious::Routes::API::V1::Authenticated, :notifications_get -Invidious::Routing.post "/api/v1/auth/notifications", Invidious::Routes::API::V1::Authenticated, :notifications_post # Video playback (macros) define_api_manifest_routes() diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 6ced4edb..b559a01a 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -398,19 +398,11 @@ module Invidious::Routes::API::V1::Authenticated env.response.status_code = 204 end - def self.notifications_get(env) + def self.notifications(env) env.response.content_type = "text/event-stream" - topics = env.params.query["topics"]?.try &.split(",").uniq.first(1000) - topics ||= [] of String - - create_notification_stream(env, topics, CONNECTION_CHANNEL) - end - - def self.notifications_post(env) - env.response.content_type = "text/event-stream" - - topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) + raw_topics = env.params.body["topics"]? || env.params.query["topics"]? + topics = raw_topics.try &.split(",").uniq.first(1000) topics ||= [] of String create_notification_stream(env, topics, CONNECTION_CHANNEL) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index d539d891..bd72c577 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -96,6 +96,9 @@ macro define_v1_api_routes Invidious::Routing.post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token Invidious::Routing.post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token + Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + # Misc Invidious::Routing.get "/api/v1/stats", {{namespace}}::Misc, :stats Invidious::Routing.get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist