Add support for 'user' URLs in NewPipe import

This commit is contained in:
Omar Roth 2019-04-22 15:39:57 -05:00
parent 5567e2843d
commit 19ed5bf993
3 changed files with 23 additions and 4 deletions

View File

@ -1842,8 +1842,22 @@ post "/data_control" do |env|
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
when "import_newpipe_subscriptions" when "import_newpipe_subscriptions"
body = JSON.parse(body) body = JSON.parse(body)
user.subscriptions += body["subscriptions"].as_a.map do |channel| user.subscriptions += body["subscriptions"].as_a.compact_map do |channel|
channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
next match["channel"]
elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
client = make_client(YT_URL)
response = client.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
document = XML.parse_html(response.body)
canonical = document.xpath_node(%q(//link[@rel="canonical"]))
if canonical
ucid = canonical["href"].split("/")[-1]
next ucid
end
end
nil
end end
user.subscriptions.uniq! user.subscriptions.uniq!

View File

@ -51,8 +51,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma
final = [] of String final = [] of String
channels.size.times do channels.size.times do
ucid = finished_channel.receive if ucid = finished_channel.receive
if ucid
final << ucid final << ucid
end end
end end

View File

@ -63,6 +63,12 @@ def channel_search(query, page, channel)
canonical = document.xpath_node(%q(//link[@rel="canonical"])) canonical = document.xpath_node(%q(//link[@rel="canonical"]))
end end
if !canonical
response = client.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US")
document = XML.parse_html(response.body)
canonical = document.xpath_node(%q(//link[@rel="canonical"]))
end
if !canonical if !canonical
return 0, [] of SearchItem return 0, [] of SearchItem
end end