[^)]+)\)/).not_nil!["url"]
+ # Raises a KeyError on failure.
+ banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
+ banner = banners.try &.[-1]?.try &.["url"].as_s?
- if banner.includes? "channels/c4/default_banner"
- banner = nil
- end
+ # if banner.includes? "channels/c4/default_banner"
+ # banner = nil
+ # end
- description_html = about.xpath_node(%q(//div[contains(@class,"about-description")])).try &.to_s ||
- %()
+ description = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?.try &.as_s? || ""
+ description_html = HTML.escape(description).gsub("\n", "
- [ - ]
+ [ - ]
#{child.author} #{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}
@@ -522,6 +513,11 @@ def fill_links(html, scheme, host)
return html.to_xml(options: XML::SaveOptions::NO_DECL)
end
+def parse_content(content : JSON::Any) : String
+ content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s ||
+ content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s } || ""
+end
+
def content_to_comment_html(content)
comment_html = content.map do |run|
text = HTML.escape(run["text"].as_s)
@@ -556,7 +552,7 @@ def content_to_comment_html(content)
video_id = watch_endpoint["videoId"].as_s
if length_seconds
- text = %(#{text} )
+ text = %(#{text} )
else
text = %(#{text} )
end
@@ -609,6 +605,8 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top")
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
when "new", "newest"
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64
+ else # top
+ object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
end
continuation = object.try { |i| Protodec::Any.cast_json(object) }
diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr
index 456618cf..045b6701 100644
--- a/src/invidious/helpers/handlers.cr
+++ b/src/invidious/helpers/handlers.cr
@@ -61,7 +61,7 @@ class Kemal::ExceptionHandler
end
class FilteredCompressHandler < Kemal::Handler
- exclude ["/videoplayback", "/videoplayback/*", "/vi/*", "/ggpht/*", "/api/v1/auth/notifications"]
+ exclude ["/videoplayback", "/videoplayback/*", "/vi/*", "/sb/*", "/ggpht/*", "/api/v1/auth/notifications"]
exclude ["/api/v1/auth/notifications", "/data_control"], "POST"
def call(env)
@@ -74,10 +74,10 @@ class FilteredCompressHandler < Kemal::Handler
if request_headers.includes_word?("Accept-Encoding", "gzip")
env.response.headers["Content-Encoding"] = "gzip"
- env.response.output = Gzip::Writer.new(env.response.output, sync_close: true)
+ env.response.output = Compress::Gzip::Writer.new(env.response.output, sync_close: true)
elsif request_headers.includes_word?("Accept-Encoding", "deflate")
env.response.headers["Content-Encoding"] = "deflate"
- env.response.output = Flate::Writer.new(env.response.output, sync_close: true)
+ env.response.output = Compress::Deflate::Writer.new(env.response.output, sync_close: true)
end
call_next env
@@ -212,29 +212,3 @@ class DenyFrame < Kemal::Handler
call_next env
end
end
-
-# Temp fixes for https://github.com/crystal-lang/crystal/issues/7383
-class HTTP::UnknownLengthContent
- def read_byte
- ensure_send_continue
- if @io.is_a?(OpenSSL::SSL::Socket::Client)
- return if @io.as(OpenSSL::SSL::Socket::Client).@in_buffer_rem.empty?
- end
- @io.read_byte
- end
-end
-
-class HTTP::Client
- private def handle_response(response)
- if @socket.is_a?(OpenSSL::SSL::Socket::Client) && @host.ends_with?("googlevideo.com")
- close unless response.keep_alive? || @socket.as(OpenSSL::SSL::Socket::Client).@in_buffer_rem.empty?
-
- if @socket.as(OpenSSL::SSL::Socket::Client).@in_buffer_rem.empty?
- @socket = nil
- end
- else
- close unless response.keep_alive?
- end
- response
- end
-end
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr
index 2341d3be..62c24f3e 100644
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@@ -1,217 +1,100 @@
require "./macros"
struct Nonce
- db_mapping({
- nonce: String,
- expire: Time,
- })
+ include DB::Serializable
+
+ property nonce : String
+ property expire : Time
end
struct SessionId
- db_mapping({
- id: String,
- email: String,
- issued: String,
- })
+ include DB::Serializable
+
+ property id : String
+ property email : String
+ property issued : String
end
struct Annotation
- db_mapping({
- id: String,
- annotations: String,
- })
+ include DB::Serializable
+
+ property id : String
+ property annotations : String
end
struct ConfigPreferences
- module StringToArray
- def self.to_json(value : Array(String), json : JSON::Builder)
- json.array do
- value.each do |element|
- json.string element
- end
- end
- end
+ include YAML::Serializable
- def self.from_json(value : JSON::PullParser) : Array(String)
- begin
- result = [] of String
- value.read_array do
- result << HTML.escape(value.read_string[0, 100])
- end
- rescue ex
- result = [HTML.escape(value.read_string[0, 100]), ""]
- end
+ property annotations : Bool = false
+ property annotations_subscribed : Bool = false
+ property autoplay : Bool = false
+ property captions : Array(String) = ["", "", ""]
+ property comments : Array(String) = ["youtube", ""]
+ property continue : Bool = false
+ property continue_autoplay : Bool = true
+ property dark_mode : String = ""
+ property latest_only : Bool = false
+ property listen : Bool = false
+ property local : Bool = false
+ property locale : String = "en-US"
+ property max_results : Int32 = 40
+ property notifications_only : Bool = false
+ property player_style : String = "invidious"
+ property quality : String = "hd720"
+ property default_home : String = "Popular"
+ property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
+ property related_videos : Bool = true
+ property sort : String = "published"
+ property speed : Float32 = 1.0_f32
+ property thin_mode : Bool = false
+ property unseen_only : Bool = false
+ property video_loop : Bool = false
+ property volume : Int32 = 100
- result
- end
-
- def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
- yaml.sequence do
- value.each do |element|
- yaml.scalar element
- end
- end
- end
-
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
- begin
- unless node.is_a?(YAML::Nodes::Sequence)
- node.raise "Expected sequence, not #{node.class}"
- end
-
- result = [] of String
- node.nodes.each do |item|
- unless item.is_a?(YAML::Nodes::Scalar)
- node.raise "Expected scalar, not #{item.class}"
- end
-
- result << HTML.escape(item.value[0, 100])
- end
- rescue ex
- if node.is_a?(YAML::Nodes::Scalar)
- result = [HTML.escape(node.value[0, 100]), ""]
- else
- result = ["", ""]
- end
- end
-
- result
- end
+ def to_tuple
+ {% begin %}
+ {
+ {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
+ }
+ {% end %}
end
-
- module BoolToString
- def self.to_json(value : String, json : JSON::Builder)
- json.string value
- end
-
- def self.from_json(value : JSON::PullParser) : String
- begin
- result = value.read_string
-
- if result.empty?
- CONFIG.default_user_preferences.dark_mode
- else
- result
- end
- rescue ex
- if value.read_bool
- "dark"
- else
- "light"
- end
- end
- end
-
- def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
- yaml.scalar value
- end
-
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
- unless node.is_a?(YAML::Nodes::Scalar)
- node.raise "Expected scalar, not #{node.class}"
- end
-
- case node.value
- when "true"
- "dark"
- when "false"
- "light"
- when ""
- CONFIG.default_user_preferences.dark_mode
- else
- node.value
- end
- end
- end
-
- yaml_mapping({
- annotations: {type: Bool, default: false},
- annotations_subscribed: {type: Bool, default: false},
- autoplay: {type: Bool, default: false},
- captions: {type: Array(String), default: ["", "", ""], converter: StringToArray},
- comments: {type: Array(String), default: ["youtube", ""], converter: StringToArray},
- continue: {type: Bool, default: false},
- continue_autoplay: {type: Bool, default: true},
- dark_mode: {type: String, default: "", converter: BoolToString},
- latest_only: {type: Bool, default: false},
- listen: {type: Bool, default: false},
- local: {type: Bool, default: false},
- locale: {type: String, default: "en-US"},
- max_results: {type: Int32, default: 40},
- notifications_only: {type: Bool, default: false},
- player_style: {type: String, default: "invidious"},
- quality: {type: String, default: "hd720"},
- default_home: {type: String, default: "Popular"},
- feed_menu: {type: Array(String), default: ["Popular", "Trending", "Subscriptions", "Playlists"]},
- related_videos: {type: Bool, default: true},
- sort: {type: String, default: "published"},
- speed: {type: Float32, default: 1.0_f32},
- thin_mode: {type: Bool, default: false},
- unseen_only: {type: Bool, default: false},
- video_loop: {type: Bool, default: false},
- volume: {type: Int32, default: 100},
- })
end
struct Config
- module ConfigPreferencesConverter
- def self.to_yaml(value : Preferences, yaml : YAML::Nodes::Builder)
- value.to_yaml(yaml)
- end
+ include YAML::Serializable
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Preferences
- Preferences.new(*ConfigPreferences.new(ctx, node).to_tuple)
- end
- end
+ property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions)
+ property feed_threads : Int32 # Number of threads to use for updating feeds
+ property db : DBConfig # Database configuration
+ property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel
+ property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https://
+ property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
+ property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required
+ property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
+ property captcha_enabled : Bool = true
+ property login_enabled : Bool = true
+ property registration_enabled : Bool = true
+ property statistics_enabled : Bool = false
+ property admins : Array(String) = [] of String
+ property external_port : Int32? = nil
+ property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("")
+ property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs
+ property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc.
+ property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
+ property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc.
+ property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
+ property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
- module FamilyConverter
- def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
- case value
- when Socket::Family::UNSPEC
- yaml.scalar nil
- when Socket::Family::INET
- yaml.scalar "ipv4"
- when Socket::Family::INET6
- yaml.scalar "ipv6"
- end
- end
+ @[YAML::Field(converter: Preferences::FamilyConverter)]
+ property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
+ property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument)
+ property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument)
+ property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
+ property admin_email : String = "omarroth@protonmail.com" # Email for bug reports
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
- if node.is_a?(YAML::Nodes::Scalar)
- case node.value.downcase
- when "ipv4"
- Socket::Family::INET
- when "ipv6"
- Socket::Family::INET6
- else
- Socket::Family::UNSPEC
- end
- else
- node.raise "Expected scalar, not #{node.class}"
- end
- end
- end
-
- module StringToCookies
- def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
- (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
- end
-
- def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
- unless node.is_a?(YAML::Nodes::Scalar)
- node.raise "Expected scalar, not #{node.class}"
- end
-
- cookies = HTTP::Cookies.new
- node.value.split(";").each do |cookie|
- next if cookie.strip.empty?
- name, value = cookie.split("=", 2)
- cookies << HTTP::Cookie.new(name.strip, value.strip)
- end
-
- cookies
- end
- end
+ @[YAML::Field(converter: Preferences::StringToCookies)]
+ property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format
+ property captcha_key : String? = nil # Key for Anti-Captcha
def disabled?(option)
case disabled = CONFIG.disable_proxy
@@ -223,77 +106,20 @@ struct Config
else
return false
end
+ else
+ return false
end
end
-
- YAML.mapping({
- channel_threads: Int32, # Number of threads to use for crawling videos from channels (for updating subscriptions)
- feed_threads: Int32, # Number of threads to use for updating feeds
- db: DBConfig, # Database configuration
- full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel
- https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https://
- hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
- domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required
- use_pubsub_feeds: {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
- top_enabled: {type: Bool, default: true},
- captcha_enabled: {type: Bool, default: true},
- login_enabled: {type: Bool, default: true},
- registration_enabled: {type: Bool, default: true},
- statistics_enabled: {type: Bool, default: false},
- admins: {type: Array(String), default: [] of String},
- external_port: {type: Int32?, default: nil},
- default_user_preferences: {type: Preferences,
- default: Preferences.new(*ConfigPreferences.from_yaml("").to_tuple),
- converter: ConfigPreferencesConverter,
- },
- dmca_content: {type: Array(String), default: [] of String}, # For compliance with DMCA, disables download widget using list of video IDs
- check_tables: {type: Bool, default: false}, # Check table integrity, automatically try to add any missing columns, create tables, etc.
- cache_annotations: {type: Bool, default: false}, # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
- banner: {type: String?, default: nil}, # Optional banner to be displayed along top of page for announcements, etc.
- hsts: {type: Bool?, default: true}, # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
- disable_proxy: {type: Bool? | Array(String)?, default: false}, # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
- force_resolve: {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
- port: {type: Int32, default: 3000}, # Port to listen for connections (overrided by command line argument)
- host_binding: {type: String, default: "0.0.0.0"}, # Host to bind (overrided by command line argument)
- pool_size: {type: Int32, default: 100}, # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
- admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports
- cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format
- captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha
- })
end
struct DBConfig
- yaml_mapping({
- user: String,
- password: String,
- host: String,
- port: Int32,
- dbname: String,
- })
-end
+ include YAML::Serializable
-def rank_videos(db, n)
- top = [] of {Float64, String}
-
- db.query("SELECT id, wilson_score, published FROM videos WHERE views > 5000 ORDER BY published DESC LIMIT 1000") do |rs|
- rs.each do
- id = rs.read(String)
- wilson_score = rs.read(Float64)
- published = rs.read(Time)
-
- # Exponential decay, older videos tend to rank lower
- temperature = wilson_score * Math.exp(-0.000005*((Time.utc - published).total_minutes))
- top << {temperature, id}
- end
- end
-
- top.sort!
-
- # Make hottest come first
- top.reverse!
- top = top.map { |a, b| b }
-
- return top[0..n - 1]
+ property user : String
+ property password : String
+ property host : String
+ property port : Int32
+ property dbname : String
end
def login_req(f_req)
@@ -334,293 +160,179 @@ def html_to_content(description_html : String)
return description
end
-def extract_videos(nodeset, ucid = nil, author_name = nil)
- videos = extract_items(nodeset, ucid, author_name)
- videos.select { |item| item.is_a?(SearchVideo) }.map { |video| video.as(SearchVideo) }
+def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
+ extract_items(initial_data, author_fallback, author_id_fallback).select(&.is_a?(SearchVideo)).map(&.as(SearchVideo))
end
-def extract_items(nodeset, ucid = nil, author_name = nil)
- # TODO: Make this a 'common', so it makes more sense to be used here
+def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fallback : String? = nil)
+ if i = (item["videoRenderer"]? || item["gridVideoRenderer"]?)
+ video_id = i["videoId"].as_s
+ title = i["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || ""
+
+ author_info = i["ownerText"]?.try &.["runs"].as_a[0]?
+ author = author_info.try &.["text"].as_s || author_fallback || ""
+ author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || ""
+
+ published = i["publishedTimeText"]?.try &.["simpleText"]?.try { |t| decode_date(t.as_s) } || Time.local
+ view_count = i["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
+ description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
+ length_seconds = i["lengthText"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } ||
+ i["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?).try &.["thumbnailOverlayTimeStatusRenderer"]?
+ .try &.["text"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || 0
+
+ live_now = false
+ paid = false
+ premium = false
+
+ premiere_timestamp = i["upcomingEventData"]?.try &.["startTime"]?.try { |t| Time.unix(t.as_s.to_i64) }
+
+ i["badges"]?.try &.as_a.each do |badge|
+ b = badge["metadataBadgeRenderer"]
+ case b["label"].as_s
+ when "LIVE NOW"
+ live_now = true
+ when "New", "4K", "CC"
+ # TODO
+ when "Premium"
+ paid = true
+
+ # TODO: Potentially available as i["topStandaloneBadge"]["metadataBadgeRenderer"]
+ premium = true
+ else nil # Ignore
+ end
+ end
+
+ SearchVideo.new({
+ title: title,
+ id: video_id,
+ author: author,
+ ucid: author_id,
+ published: published,
+ views: view_count,
+ description_html: description_html,
+ length_seconds: length_seconds,
+ live_now: live_now,
+ paid: paid,
+ premium: premium,
+ premiere_timestamp: premiere_timestamp,
+ })
+ elsif i = item["channelRenderer"]?
+ author = i["title"]["simpleText"]?.try &.as_s || author_fallback || ""
+ author_id = i["channelId"]?.try &.as_s || author_id_fallback || ""
+
+ author_thumbnail = i["thumbnail"]["thumbnails"]?.try &.as_a[0]?.try { |u| "https:#{u["url"]}" } || ""
+ subscriber_count = i["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s.try { |s| short_text_to_number(s.split(" ")[0]) } || 0
+
+ auto_generated = false
+ auto_generated = true if !i["videoCountText"]?
+ video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
+ description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
+
+ SearchChannel.new({
+ author: author,
+ ucid: author_id,
+ author_thumbnail: author_thumbnail,
+ subscriber_count: subscriber_count,
+ video_count: video_count,
+ description_html: description_html,
+ auto_generated: auto_generated,
+ })
+ elsif i = item["gridPlaylistRenderer"]?
+ title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || ""
+ plid = i["playlistId"]?.try &.as_s || ""
+
+ video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
+ playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || ""
+
+ SearchPlaylist.new({
+ title: title,
+ id: plid,
+ author: author_fallback || "",
+ ucid: author_id_fallback || "",
+ video_count: video_count,
+ videos: [] of SearchPlaylistVideo,
+ thumbnail: playlist_thumbnail,
+ })
+ elsif i = item["playlistRenderer"]?
+ title = i["title"]["simpleText"]?.try &.as_s || ""
+ plid = i["playlistId"]?.try &.as_s || ""
+
+ video_count = i["videoCount"]?.try &.as_s.to_i || 0
+ playlist_thumbnail = i["thumbnails"].as_a[0]?.try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"].as_s || ""
+
+ author_info = i["shortBylineText"]?.try &.["runs"].as_a[0]?
+ author = author_info.try &.["text"].as_s || author_fallback || ""
+ author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || ""
+
+ videos = i["videos"]?.try &.as_a.map do |v|
+ v = v["childVideoRenderer"]
+ v_title = v["title"]["simpleText"]?.try &.as_s || ""
+ v_id = v["videoId"]?.try &.as_s || ""
+ v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0
+ SearchPlaylistVideo.new({
+ title: v_title,
+ id: v_id,
+ length_seconds: v_length_seconds,
+ })
+ end || [] of SearchPlaylistVideo
+
+ # TODO: i["publishedTimeText"]?
+
+ SearchPlaylist.new({
+ title: title,
+ id: plid,
+ author: author,
+ ucid: author_id,
+ video_count: video_count,
+ videos: videos,
+ thumbnail: playlist_thumbnail,
+ })
+ elsif i = item["radioRenderer"]? # Mix
+ # TODO
+ elsif i = item["showRenderer"]? # Show
+ # TODO
+ elsif i = item["shelfRenderer"]?
+ elsif i = item["horizontalCardListRenderer"]?
+ elsif i = item["searchPyvRenderer"]? # Ad
+ end
+end
+
+def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
items = [] of SearchItem
- nodeset.each do |node|
- anchor = node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
- if !anchor
- next
- end
- title = anchor.content.strip
- id = anchor["href"]
+ channel_v2_response = initial_data
+ .try &.["response"]?
+ .try &.["continuationContents"]?
+ .try &.["gridContinuation"]?
+ .try &.["items"]?
- if anchor["href"].starts_with? "https://www.googleadservices.com"
- next
- end
-
- author_id = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-byline")]/a)).try &.["href"].split("/")[-1] || ucid || ""
- author = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-byline")]/a)).try &.content.strip || author_name || ""
- description_html = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-description")])).try &.to_s || ""
-
- tile = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-tile")]))
- if !tile
- next
- end
-
- case tile["class"]
- when .includes? "yt-lockup-playlist"
- plid = HTTP::Params.parse(URI.parse(id).query.not_nil!)["list"]
-
- anchor = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-meta")]/a))
-
- if !anchor
- anchor = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/a))
- end
-
- video_count = node.xpath_node(%q(.//span[@class="formatted-video-count-label"]/b)) ||
- node.xpath_node(%q(.//span[@class="formatted-video-count-label"]))
- if video_count
- video_count = video_count.content
-
- if video_count == "50+"
- author = "YouTube"
- author_id = "UC-9-kyTW8ZkZNDHQJ6FgpwQ"
- end
-
- video_count = video_count.gsub(/\D/, "").to_i?
- end
- video_count ||= 0
-
- videos = [] of SearchPlaylistVideo
- node.xpath_nodes(%q(.//*[contains(@class, "yt-lockup-playlist-items")]/li)).each do |video|
- anchor = video.xpath_node(%q(.//a))
- if anchor
- video_title = anchor.content.strip
- id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"]
- end
- video_title ||= ""
- id ||= ""
-
- anchor = video.xpath_node(%q(.//span/span))
- if anchor
- length_seconds = decode_length_seconds(anchor.content)
- end
- length_seconds ||= 0
-
- videos << SearchPlaylistVideo.new(
- video_title,
- id,
- length_seconds
- )
- end
-
- playlist_thumbnail = node.xpath_node(%q(.//span/img)).try &.["data-thumb"]?
- playlist_thumbnail ||= node.xpath_node(%q(.//span/img)).try &.["src"]
-
- items << SearchPlaylist.new(
- title: title,
- id: plid,
- author: author,
- ucid: author_id,
- video_count: video_count,
- videos: videos,
- thumbnail: playlist_thumbnail
- )
- when .includes? "yt-lockup-channel"
- author = title.strip
-
- ucid = node.xpath_node(%q(.//button[contains(@class, "yt-uix-subscription-button")])).try &.["data-channel-external-id"]?
- ucid ||= id.split("/")[-1]
-
- author_thumbnail = node.xpath_node(%q(.//div/span/img)).try &.["data-thumb"]?
- author_thumbnail ||= node.xpath_node(%q(.//div/span/img)).try &.["src"]
- if author_thumbnail
- author_thumbnail = URI.parse(author_thumbnail)
- author_thumbnail.scheme = "https"
- author_thumbnail = author_thumbnail.to_s
- end
-
- author_thumbnail ||= ""
-
- subscriber_count = node.xpath_node(%q(.//span[contains(@class, "subscriber-count")]))
- .try &.["title"].try { |text| short_text_to_number(text) } || 0
-
- video_count = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li)).try &.content.split(" ")[0].gsub(/\D/, "").to_i?
-
- items << SearchChannel.new(
- author: author,
- ucid: ucid,
- author_thumbnail: author_thumbnail,
- subscriber_count: subscriber_count,
- video_count: video_count || 0,
- description_html: description_html,
- auto_generated: video_count ? false : true,
- )
- else
- id = id.lchop("/watch?v=")
-
- metadata = node.xpath_node(%q(.//div[contains(@class,"yt-lockup-meta")]/ul))
-
- published = metadata.try &.xpath_node(%q(.//li[contains(text(), " ago")])).try { |node| decode_date(node.content.sub(/^[a-zA-Z]+ /, "")) }
- published ||= metadata.try &.xpath_node(%q(.//span[@data-timestamp])).try { |node| Time.unix(node["data-timestamp"].to_i64) }
- published ||= Time.utc
-
- view_count = metadata.try &.xpath_node(%q(.//li[contains(text(), " views")])).try &.content.gsub(/\D/, "").to_i64?
- view_count ||= 0_i64
-
- length_seconds = node.xpath_node(%q(.//span[@class="video-time"])).try { |node| decode_length_seconds(node.content) }
- length_seconds ||= -1
-
- live_now = node.xpath_node(%q(.//span[contains(@class, "yt-badge-live")])) ? true : false
- premium = node.xpath_node(%q(.//span[text()="Premium"])) ? true : false
-
- if !premium || node.xpath_node(%q(.//span[contains(text(), "Free episode")]))
- paid = false
- else
- paid = true
- end
-
- premiere_timestamp = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/span[@class="localized-date"])).try &.["data-timestamp"]?.try &.to_i64
- if premiere_timestamp
- premiere_timestamp = Time.unix(premiere_timestamp)
- end
-
- items << SearchVideo.new(
- title: title,
- id: id,
- author: author,
- ucid: author_id,
- published: published,
- views: view_count,
- description_html: description_html,
- length_seconds: length_seconds,
- live_now: live_now,
- paid: paid,
- premium: premium,
- premiere_timestamp: premiere_timestamp
- )
- end
+ if channel_v2_response
+ channel_v2_response.try &.as_a.each { |item|
+ extract_item(item, author_fallback, author_id_fallback)
+ .try { |t| items << t }
+ }
+ else
+ initial_data.try { |t| t["contents"]? || t["response"]? }
+ .try { |t| t["twoColumnBrowseResultsRenderer"]?.try &.["tabs"].as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]?.try &.["tabRenderer"]["content"] ||
+ t["twoColumnSearchResultsRenderer"]?.try &.["primaryContents"] ||
+ t["continuationContents"]? }
+ .try { |t| t["sectionListRenderer"]? || t["sectionListContinuation"]? }
+ .try &.["contents"].as_a
+ .each { |c| c.try &.["itemSectionRenderer"]?.try &.["contents"].as_a
+ .try { |t| t[0]?.try &.["shelfRenderer"]?.try &.["content"]["expandedShelfContentsRenderer"]?.try &.["items"].as_a ||
+ t[0]?.try &.["gridRenderer"]?.try &.["items"].as_a || t }
+ .each { |item|
+ extract_item(item, author_fallback, author_id_fallback)
+ .try { |t| items << t }
+ } }
end
- return items
-end
-
-def extract_shelf_items(nodeset, ucid = nil, author_name = nil)
- items = [] of SearchPlaylist
-
- nodeset.each do |shelf|
- shelf_anchor = shelf.xpath_node(%q(.//h2[contains(@class, "branded-page-module-title")]))
- next if !shelf_anchor
-
- title = shelf_anchor.xpath_node(%q(.//span[contains(@class, "branded-page-module-title-text")])).try &.content.strip
- title ||= ""
-
- id = shelf_anchor.xpath_node(%q(.//a)).try &.["href"]
- next if !id
-
- shelf_is_playlist = false
- videos = [] of SearchPlaylistVideo
-
- shelf.xpath_nodes(%q(.//ul[contains(@class, "yt-uix-shelfslider-list") or contains(@class, "expanded-shelf-content-list")]/li)).each do |child_node|
- type = child_node.xpath_node(%q(./div))
- if !type
- next
- end
-
- case type["class"]
- when .includes? "yt-lockup-video"
- shelf_is_playlist = true
-
- anchor = child_node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
- if anchor
- video_title = anchor.content.strip
- video_id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"]
- end
- video_title ||= ""
- video_id ||= ""
-
- anchor = child_node.xpath_node(%q(.//span[@class="video-time"]))
- if anchor
- length_seconds = decode_length_seconds(anchor.content)
- end
- length_seconds ||= 0
-
- videos << SearchPlaylistVideo.new(
- video_title,
- video_id,
- length_seconds
- )
- when .includes? "yt-lockup-playlist"
- anchor = child_node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
- if anchor
- playlist_title = anchor.content.strip
- params = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)
- plid = params["list"]
- end
- playlist_title ||= ""
- plid ||= ""
-
- playlist_thumbnail = child_node.xpath_node(%q(.//span/img)).try &.["data-thumb"]?
- playlist_thumbnail ||= child_node.xpath_node(%q(.//span/img)).try &.["src"]
-
- video_count = child_node.xpath_node(%q(.//span[@class="formatted-video-count-label"]/b)) ||
- child_node.xpath_node(%q(.//span[@class="formatted-video-count-label"]))
- if video_count
- video_count = video_count.content.gsub(/\D/, "").to_i?
- end
- video_count ||= 50
-
- videos = [] of SearchPlaylistVideo
- child_node.xpath_nodes(%q(.//*[contains(@class, "yt-lockup-playlist-items")]/li)).each do |video|
- anchor = video.xpath_node(%q(.//a))
- if anchor
- video_title = anchor.content.strip
- id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"]
- end
- video_title ||= ""
- id ||= ""
-
- anchor = video.xpath_node(%q(.//span/span))
- if anchor
- length_seconds = decode_length_seconds(anchor.content)
- end
- length_seconds ||= 0
-
- videos << SearchPlaylistVideo.new(
- video_title,
- id,
- length_seconds
- )
- end
-
- items << SearchPlaylist.new(
- title: playlist_title,
- id: plid,
- author: author_name,
- ucid: ucid,
- video_count: video_count,
- videos: videos,
- thumbnail: playlist_thumbnail
- )
- end
- end
-
- if shelf_is_playlist
- plid = HTTP::Params.parse(URI.parse(id).query.not_nil!)["list"]
-
- items << SearchPlaylist.new(
- title: title,
- id: plid,
- author: author_name,
- ucid: ucid,
- video_count: videos.size,
- videos: videos,
- thumbnail: "https://i.ytimg.com/vi/#{videos[0].id}/mqdefault.jpg"
- )
- end
- end
-
- return items
+ items
end
def check_enum(db, logger, enum_name, struct_type = nil)
+ return # TODO
+
if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
logger.puts("CREATE TYPE #{enum_name}")
@@ -642,18 +354,14 @@ def check_table(db, logger, table_name, struct_type = nil)
end
end
- if !struct_type
- return
- end
+ return if !struct_type
- struct_array = struct_type.to_type_tuple
+ struct_array = struct_type.type_array
column_array = get_column_array(db, table_name)
column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?[\d\D]*?)\);/)
- .try &.["types"].split(",").map { |line| line.strip }
+ .try &.["types"].split(",").map { |line| line.strip }.reject &.starts_with?("CONSTRAINT")
- if !column_types
- return
- end
+ return if !column_types
struct_array.each_with_index do |name, i|
if name != column_array[i]?
@@ -704,6 +412,15 @@ def check_table(db, logger, table_name, struct_type = nil)
end
end
end
+
+ return if column_array.size <= struct_array.size
+
+ column_array.each do |column|
+ if !struct_array.includes? column
+ logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
+ db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
+ end
+ end
end
class PG::ResultSet
@@ -732,9 +449,7 @@ def cache_annotation(db, id, annotations)
body = XML.parse(annotations)
nodeset = body.xpath_nodes(%q(/document/annotations/annotation))
- if nodeset == 0
- return
- end
+ return if nodeset == 0
has_legacy_annotations = false
nodeset.each do |node|
@@ -744,13 +459,10 @@ def cache_annotation(db, id, annotations)
end
end
- if has_legacy_annotations
- # TODO: Update on conflict?
- db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations)
- end
+ db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) if has_legacy_annotations
end
-def create_notification_stream(env, config, kemal_config, decrypt_function, topics, connection_channel)
+def create_notification_stream(env, topics, connection_channel)
connection = Channel(PQ::Notification).new(8)
connection_channel.send({true, connection})
@@ -765,12 +477,12 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
loop do
time_span = [0, 0, 0, 0]
time_span[rand(4)] = rand(30) + 5
- published = Time.utc - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3])
+ published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])
video_id = TEST_IDS[rand(TEST_IDS.size)]
video = get_video(video_id, PG_DB)
video.published = published
- response = JSON.parse(video.to_json(locale, config, kemal_config, decrypt_function))
+ response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
@@ -804,7 +516,7 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
when .match(/UC[A-Za-z0-9_-]{22}/)
PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15",
topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video|
- response = JSON.parse(video.to_json(locale, config, Kemal.config))
+ response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
@@ -846,7 +558,7 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
video = get_video(video_id, PG_DB)
video.published = Time.unix(published)
- response = JSON.parse(video.to_json(locale, config, Kemal.config, decrypt_function))
+ response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
@@ -884,26 +596,46 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
end
end
-def extract_initial_data(body)
- initial_data = body.match(/window\["ytInitialData"\] = (?.*?);\n/).try &.["info"] || "{}"
+def extract_initial_data(body) : Hash(String, JSON::Any)
+ initial_data = body.match(/(window\["ytInitialData"\]|var\s+ytInitialData)\s*=\s*(?.*?);+\s*\n/).try &.["info"] || "{}"
if initial_data.starts_with?("JSON.parse(\"")
- return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s)
+ return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s).as_h
else
- return JSON.parse(initial_data)
+ return JSON.parse(initial_data).as_h
end
end
def proxy_file(response, env)
if response.headers.includes_word?("Content-Encoding", "gzip")
- Gzip::Writer.open(env.response) do |deflate|
- response.pipe(deflate)
+ Compress::Gzip::Writer.open(env.response) do |deflate|
+ IO.copy response.body_io, deflate
end
elsif response.headers.includes_word?("Content-Encoding", "deflate")
- Flate::Writer.open(env.response) do |deflate|
- response.pipe(deflate)
+ Compress::Deflate::Writer.open(env.response) do |deflate|
+ IO.copy response.body_io, deflate
end
else
- response.pipe(env.response)
+ IO.copy response.body_io, env.response
+ end
+end
+
+# See https://github.com/kemalcr/kemal/pull/576
+class HTTP::Server::Response::Output
+ def close
+ return if closed?
+
+ unless response.wrote_headers?
+ response.content_length = @out_count
+ end
+
+ ensure_headers_written
+
+ super
+
+ if @chunked
+ @io << "0\r\n\r\n"
+ @io.flush
+ end
end
end
diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr
index 4c9bb2d6..0faa2e32 100644
--- a/src/invidious/helpers/i18n.cr
+++ b/src/invidious/helpers/i18n.cr
@@ -24,6 +24,8 @@ def translate(locale : Hash(String, JSON::Any) | Nil, translation : String, text
if !locale[translation].as_s.empty?
translation = locale[translation].as_s
end
+ else
+ raise "Invalid translation #{translation}"
end
end
diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr
deleted file mode 100644
index f368d6df..00000000
--- a/src/invidious/helpers/jobs.cr
+++ /dev/null
@@ -1,370 +0,0 @@
-def refresh_channels(db, logger, config)
- max_channel = Channel(Int32).new
-
- spawn do
- max_threads = max_channel.receive
- active_threads = 0
- active_channel = Channel(Bool).new
-
- loop do
- db.query("SELECT id FROM channels ORDER BY updated") do |rs|
- rs.each do
- id = rs.read(String)
-
- if active_threads >= max_threads
- if active_channel.receive
- active_threads -= 1
- end
- end
-
- active_threads += 1
- spawn do
- begin
- channel = fetch_channel(id, db, config.full_refresh)
-
- db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
- rescue ex
- if ex.message == "Deleted or invalid channel"
- db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
- end
- logger.puts("#{id} : #{ex.message}")
- end
-
- active_channel.send(true)
- end
- end
- end
-
- sleep 1.minute
- Fiber.yield
- end
- end
-
- max_channel.send(config.channel_threads)
-end
-
-def refresh_feeds(db, logger, config)
- max_channel = Channel(Int32).new
- spawn do
- max_threads = max_channel.receive
- active_threads = 0
- active_channel = Channel(Bool).new
-
- loop do
- db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
- rs.each do
- email = rs.read(String)
- view_name = "subscriptions_#{sha256(email)}"
-
- if active_threads >= max_threads
- if active_channel.receive
- active_threads -= 1
- end
- end
-
- active_threads += 1
- spawn do
- begin
- # Drop outdated views
- column_array = get_column_array(db, view_name)
- ChannelVideo.to_type_tuple.each_with_index do |name, i|
- if name != column_array[i]?
- logger.puts("DROP MATERIALIZED VIEW #{view_name}")
- db.exec("DROP MATERIALIZED VIEW #{view_name}")
- raise "view does not exist"
- end
- end
-
- if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
- logger.puts("Materialized view #{view_name} is out-of-date, recreating...")
- db.exec("DROP MATERIALIZED VIEW #{view_name}")
- end
-
- db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
- rescue ex
- # Rename old views
- begin
- legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
-
- db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
- logger.puts("RENAME MATERIALIZED VIEW #{legacy_view_name}")
- db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
- rescue ex
- begin
- # While iterating through, we may have an email stored from a deleted account
- if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
- logger.puts("CREATE #{view_name}")
- db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}")
- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
- end
- rescue ex
- logger.puts("REFRESH #{email} : #{ex.message}")
- end
- end
- end
-
- active_channel.send(true)
- end
- end
- end
-
- sleep 5.seconds
- Fiber.yield
- end
- end
-
- max_channel.send(config.feed_threads)
-end
-
-def subscribe_to_feeds(db, logger, key, config)
- if config.use_pubsub_feeds
- case config.use_pubsub_feeds
- when Bool
- max_threads = config.use_pubsub_feeds.as(Bool).to_unsafe
- when Int32
- max_threads = config.use_pubsub_feeds.as(Int32)
- end
- max_channel = Channel(Int32).new
-
- spawn do
- max_threads = max_channel.receive
- active_threads = 0
- active_channel = Channel(Bool).new
-
- loop do
- db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
- rs.each do
- ucid = rs.read(String)
-
- if active_threads >= max_threads.as(Int32)
- if active_channel.receive
- active_threads -= 1
- end
- end
-
- active_threads += 1
-
- spawn do
- begin
- response = subscribe_pubsub(ucid, key, config)
-
- if response.status_code >= 400
- logger.puts("#{ucid} : #{response.body}")
- end
- rescue ex
- logger.puts("#{ucid} : #{ex.message}")
- end
-
- active_channel.send(true)
- end
- end
- end
-
- sleep 1.minute
- Fiber.yield
- end
- end
-
- max_channel.send(max_threads.as(Int32))
- end
-end
-
-def pull_top_videos(config, db)
- loop do
- begin
- top = rank_videos(db, 40)
- rescue ex
- sleep 1.minute
- Fiber.yield
-
- next
- end
-
- if top.size == 0
- sleep 1.minute
- Fiber.yield
-
- next
- end
-
- videos = [] of Video
-
- top.each do |id|
- begin
- videos << get_video(id, db)
- rescue ex
- next
- end
- end
-
- yield videos
-
- sleep 1.minute
- Fiber.yield
- end
-end
-
-def pull_popular_videos(db)
- loop do
- videos = db.query_all("SELECT DISTINCT ON (ucid) * FROM channel_videos WHERE ucid IN \
- (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d \
- GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) \
- ORDER BY ucid, published DESC", as: ChannelVideo).sort_by { |video| video.published }.reverse
-
- yield videos
-
- sleep 1.minute
- Fiber.yield
- end
-end
-
-def update_decrypt_function
- loop do
- begin
- decrypt_function = fetch_decrypt_function
- yield decrypt_function
- rescue ex
- next
- ensure
- sleep 1.minute
- Fiber.yield
- end
- end
-end
-
-def bypass_captcha(captcha_key, logger)
- loop do
- begin
- response = YT_POOL.client &.get("/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
- if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
- html = XML.parse_html(response.body)
- form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
- site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"]
-
- inputs = {} of String => String
- form.xpath_nodes(%(.//input[@name])).map do |node|
- inputs[node["name"]] = node["value"]
- end
-
- headers = response.cookies.add_request_headers(HTTP::Headers.new)
-
- response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
- "clientKey" => CONFIG.captcha_key,
- "task" => {
- "type" => "NoCaptchaTaskProxyless",
- # "type" => "NoCaptchaTask",
- "websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999",
- "websiteKey" => site_key,
- # "proxyType" => "http",
- # "proxyAddress" => CONFIG.proxy_address,
- # "proxyPort" => CONFIG.proxy_port,
- # "proxyLogin" => CONFIG.proxy_user,
- # "proxyPassword" => CONFIG.proxy_pass,
- # "userAgent" => "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
- },
- }.to_json).body)
-
- if response["error"]?
- raise response["error"].as_s
- end
-
- task_id = response["taskId"].as_i
-
- loop do
- sleep 10.seconds
-
- response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
- "clientKey" => CONFIG.captcha_key,
- "taskId" => task_id,
- }.to_json).body)
-
- if response["status"]?.try &.== "ready"
- break
- elsif response["errorId"]?.try &.as_i != 0
- raise response["errorDescription"].as_s
- end
- end
-
- inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
- response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
-
- yield response.cookies.select { |cookie| cookie.name != "PREF" }
- elsif response.headers["Location"]?.try &.includes?("/sorry/index")
- location = response.headers["Location"].try { |u| URI.parse(u) }
- client = QUIC::Client.new(location.host.not_nil!)
- response = client.get(location.full_path)
-
- html = XML.parse_html(response.body)
- form = html.xpath_node(%(//form[@action="index"])).not_nil!
- site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"]
-
- inputs = {} of String => String
- form.xpath_nodes(%(.//input[@name])).map do |node|
- inputs[node["name"]] = node["value"]
- end
-
- response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
- "clientKey" => CONFIG.captcha_key,
- "task" => {
- "type" => "NoCaptchaTaskProxyless",
- "websiteURL" => location.to_s,
- "websiteKey" => site_key,
- },
- }.to_json).body)
-
- if response["error"]?
- raise response["error"].as_s
- end
-
- task_id = response["taskId"].as_i
-
- loop do
- sleep 10.seconds
-
- response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
- "clientKey" => CONFIG.captcha_key,
- "taskId" => task_id,
- }.to_json).body)
-
- if response["status"]?.try &.== "ready"
- break
- elsif response["errorId"]?.try &.as_i != 0
- raise response["errorDescription"].as_s
- end
- end
-
- inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
- client.close
- client = QUIC::Client.new("www.google.com")
- response = client.post(location.full_path, form: inputs)
- headers = HTTP::Headers{
- "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
- }
- cookies = HTTP::Cookies.from_headers(headers)
-
- yield cookies
- end
- rescue ex
- logger.puts("Exception: #{ex.message}")
- ensure
- sleep 1.minute
- Fiber.yield
- end
- end
-end
-
-def find_working_proxies(regions)
- loop do
- regions.each do |region|
- proxies = get_proxies(region).first(20)
- proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} }
- # proxies = filter_proxies(proxies)
-
- yield region, proxies
- end
-
- sleep 1.minute
- Fiber.yield
- end
-end
diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr
index ddfb9f8e..8b74bc86 100644
--- a/src/invidious/helpers/macros.cr
+++ b/src/invidious/helpers/macros.cr
@@ -1,43 +1,51 @@
-macro db_mapping(mapping)
- def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
- end
+module DB::Serializable
+ macro included
+ {% verbatim do %}
+ macro finished
+ def self.type_array
+ \{{ @type.instance_vars
+ .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
+ .map { |name| name.stringify }
+ }}
+ end
- def to_a
- return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
- end
+ def initialize(tuple)
+ \{% for var in @type.instance_vars %}
+ \{% ann = var.annotation(::DB::Field) %}
+ \{% if ann && ann[:ignore] %}
+ \{% else %}
+ @\{{var.name}} = tuple[:\{{var.name.id}}]
+ \{% end %}
+ \{% end %}
+ end
- def self.to_type_tuple
- return { {{*mapping.keys.map { |id| "#{id}" }}} }
+ def to_a
+ \{{ @type.instance_vars
+ .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
+ .map { |name| name }
+ }}
+ end
+ end
+ {% end %}
end
-
- DB.mapping( {{mapping}} )
end
-macro json_mapping(mapping)
- def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
+module JSON::Serializable
+ macro included
+ {% verbatim do %}
+ macro finished
+ def initialize(tuple)
+ \{% for var in @type.instance_vars %}
+ \{% ann = var.annotation(::JSON::Field) %}
+ \{% if ann && ann[:ignore] %}
+ \{% else %}
+ @\{{var.name}} = tuple[:\{{var.name.id}}]
+ \{% end %}
+ \{% end %}
+ end
+ end
+ {% end %}
end
-
- def to_a
- return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
- end
-
- patched_json_mapping( {{mapping}} )
- YAML.mapping( {{mapping}} )
-end
-
-macro yaml_mapping(mapping)
- def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
- end
-
- def to_a
- return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
- end
-
- def to_tuple
- return { {{*mapping.keys.map { |id| "@#{id}".id }}} }
- end
-
- YAML.mapping({{mapping}})
end
macro templated(filename, template = "template")
diff --git a/src/invidious/helpers/patch_mapping.cr b/src/invidious/helpers/patch_mapping.cr
deleted file mode 100644
index 19bd8ca1..00000000
--- a/src/invidious/helpers/patch_mapping.cr
+++ /dev/null
@@ -1,166 +0,0 @@
-# Overloads https://github.com/crystal-lang/crystal/blob/0.28.0/src/json/from_json.cr#L24
-def Object.from_json(string_or_io, default) : self
- parser = JSON::PullParser.new(string_or_io)
- new parser, default
-end
-
-# Adds configurable 'default'
-macro patched_json_mapping(_properties_, strict = false)
- {% for key, value in _properties_ %}
- {% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %}
- {% end %}
-
- {% for key, value in _properties_ %}
- {% _properties_[key][:key_id] = key.id.gsub(/\?$/, "") %}
- {% end %}
-
- {% for key, value in _properties_ %}
- @{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
-
- {% if value[:setter] == nil ? true : value[:setter] %}
- def {{value[:key_id]}}=(_{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }})
- @{{value[:key_id]}} = _{{value[:key_id]}}
- end
- {% end %}
-
- {% if value[:getter] == nil ? true : value[:getter] %}
- def {{key.id}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
- @{{value[:key_id]}}
- end
- {% end %}
-
- {% if value[:presence] %}
- @{{value[:key_id]}}_present : Bool = false
-
- def {{value[:key_id]}}_present?
- @{{value[:key_id]}}_present
- end
- {% end %}
- {% end %}
-
- def initialize(%pull : ::JSON::PullParser, default = nil)
- {% for key, value in _properties_ %}
- %var{key.id} = nil
- %found{key.id} = false
- {% end %}
-
- %location = %pull.location
- begin
- %pull.read_begin_object
- rescue exc : ::JSON::ParseException
- raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc)
- end
- until %pull.kind.end_object?
- %key_location = %pull.location
- key = %pull.read_object_key
- case key
- {% for key, value in _properties_ %}
- when {{value[:key] || value[:key_id].stringify}}
- %found{key.id} = true
- begin
- %var{key.id} =
- {% if value[:nilable] || value[:default] != nil %} %pull.read_null_or { {% end %}
-
- {% if value[:root] %}
- %pull.on_key!({{value[:root]}}) do
- {% end %}
-
- {% if value[:converter] %}
- {{value[:converter]}}.from_json(%pull)
- {% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %}
- {{value[:type]}}.new(%pull)
- {% else %}
- ::Union({{value[:type]}}).new(%pull)
- {% end %}
-
- {% if value[:root] %}
- end
- {% end %}
-
- {% if value[:nilable] || value[:default] != nil %} } {% end %}
- rescue exc : ::JSON::ParseException
- raise ::JSON::MappingError.new(exc.message, self.class.to_s, {{value[:key] || value[:key_id].stringify}}, *%key_location, exc)
- end
- {% end %}
- else
- {% if strict %}
- raise ::JSON::MappingError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *%key_location, nil)
- {% else %}
- %pull.skip
- {% end %}
- end
- end
- %pull.read_next
-
- {% for key, value in _properties_ %}
- {% unless value[:nilable] || value[:default] != nil %}
- if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable?
- raise ::JSON::MappingError.new("Missing JSON attribute: {{(value[:key] || value[:key_id]).id}}", self.class.to_s, nil, *%location, nil)
- end
- {% end %}
-
- {% if value[:nilable] %}
- {% if value[:default] != nil %}
- @{{value[:key_id]}} = %found{key.id} ? %var{key.id} : (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}})
- {% else %}
- @{{value[:key_id]}} = %var{key.id}
- {% end %}
- {% elsif value[:default] != nil %}
- @{{value[:key_id]}} = %var{key.id}.nil? ? (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}}) : %var{key.id}
- {% else %}
- @{{value[:key_id]}} = (%var{key.id}).as({{value[:type]}})
- {% end %}
-
- {% if value[:presence] %}
- @{{value[:key_id]}}_present = %found{key.id}
- {% end %}
- {% end %}
- end
-
- def to_json(json : ::JSON::Builder)
- json.object do
- {% for key, value in _properties_ %}
- _{{value[:key_id]}} = @{{value[:key_id]}}
-
- {% unless value[:emit_null] %}
- unless _{{value[:key_id]}}.nil?
- {% end %}
-
- json.field({{value[:key] || value[:key_id].stringify}}) do
- {% if value[:root] %}
- {% if value[:emit_null] %}
- if _{{value[:key_id]}}.nil?
- nil.to_json(json)
- else
- {% end %}
-
- json.object do
- json.field({{value[:root]}}) do
- {% end %}
-
- {% if value[:converter] %}
- if _{{value[:key_id]}}
- {{ value[:converter] }}.to_json(_{{value[:key_id]}}, json)
- else
- nil.to_json(json)
- end
- {% else %}
- _{{value[:key_id]}}.to_json(json)
- {% end %}
-
- {% if value[:root] %}
- {% if value[:emit_null] %}
- end
- {% end %}
- end
- end
- {% end %}
- end
-
- {% unless value[:emit_null] %}
- end
- {% end %}
- {% end %}
- end
- end
-end
diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr
index 1d238576..f811500f 100644
--- a/src/invidious/helpers/signatures.cr
+++ b/src/invidious/helpers/signatures.cr
@@ -1,69 +1,53 @@
+alias SigProc = Proc(Array(String), Int32, Array(String))
+
def fetch_decrypt_function(id = "CvFH_6DNRCY")
- document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body
- url = document.match(/src="(?\/yts\/jsbin\/player_ias-.{9}\/en_US\/base.js)"/).not_nil!["url"]
+ document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
+ url = document.match(/src="(?\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
player = YT_POOL.client &.get(url).body
- function_name = player.match(/^(?[^=]+)=function\(a\){a=a\.split\(""\)/m).not_nil!["name"]
- function_body = player.match(/^#{Regex.escape(function_name)}=function\(a\){(?[^}]+)}/m).not_nil!["body"]
+ function_name = player.match(/^(?[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
+ function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?[^}]+)}/m).not_nil!["body"]
function_body = function_body.split(";")[1..-2]
var_name = function_body[0][0, 2]
var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?(.*?))};/).not_nil!["body"]
- operations = {} of String => String
+ operations = {} of String => SigProc
var_body.split("},").each do |operation|
op_name = operation.match(/^[^:]+/).not_nil![0]
op_body = operation.match(/\{[^}]+/).not_nil![0]
case op_body
when "{a.reverse()"
- operations[op_name] = "a"
+ operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse }
when "{a.splice(0,b)"
- operations[op_name] = "b"
+ operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
else
- operations[op_name] = "c"
+ operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
end
end
- decrypt_function = [] of {name: String, value: Int32}
+ decrypt_function = [] of {SigProc, Int32}
function_body.each do |function|
function = function.lchop(var_name).delete("[].")
op_name = function.match(/[^\(]+/).not_nil![0]
- value = function.match(/\(a,(?[\d]+)\)/).not_nil!["value"].to_i
+ value = function.match(/\(\w,(?[\d]+)\)/).not_nil!["value"].to_i
- decrypt_function << {name: operations[op_name], value: value}
+ decrypt_function << {operations[op_name], value}
end
return decrypt_function
end
-def decrypt_signature(fmt, code)
- if !fmt["s"]?
- return ""
+def decrypt_signature(fmt : Hash(String, JSON::Any))
+ return "" if !fmt["s"]? || !fmt["sp"]?
+
+ sp = fmt["sp"].as_s
+ sig = fmt["s"].as_s.split("")
+ DECRYPT_FUNCTION.each do |proc, value|
+ sig = proc.call(sig, value)
end
- a = fmt["s"]
- a = a.split("")
-
- code.each do |item|
- case item[:name]
- when "a"
- a.reverse!
- when "b"
- a.delete_at(0..(item[:value] - 1))
- when "c"
- a = splice(a, item[:value])
- end
- end
-
- signature = a.join("")
- return "&#{fmt["sp"]?}=#{signature}"
-end
-
-def splice(a, b)
- c = a[0]
- a[0] = a[b % a.size]
- a[b % a.size] = c
- return a
+ return "&#{sp}=#{sig.join("")}"
end
diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr
index 20d92b9c..be9d36ab 100644
--- a/src/invidious/helpers/static_file_handler.cr
+++ b/src/invidious/helpers/static_file_handler.cr
@@ -81,12 +81,12 @@ def send_file(env : HTTP::Server::Context, file_path : String, data : Slice(UInt
condition = config.is_a?(Hash) && config["gzip"]? == true && filesize > minsize && Kemal::Utils.zip_types(file_path)
if condition && request_headers.includes_word?("Accept-Encoding", "gzip")
env.response.headers["Content-Encoding"] = "gzip"
- Gzip::Writer.open(env.response) do |deflate|
+ Compress::Gzip::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
elsif condition && request_headers.includes_word?("Accept-Encoding", "deflate")
env.response.headers["Content-Encoding"] = "deflate"
- Flate::Writer.open(env.response) do |deflate|
+ Compress::Deflate::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
else
diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr
index 30f7d4f4..39aae367 100644
--- a/src/invidious/helpers/tokens.cr
+++ b/src/invidious/helpers/tokens.cr
@@ -1,3 +1,5 @@
+require "crypto/subtle"
+
def generate_token(email, scopes, expire, key, db)
session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}"
PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc)
@@ -41,15 +43,10 @@ def sign_token(key, hash)
string_to_sign = [] of String
hash.each do |key, value|
- if key == "signature"
- next
- end
+ next if key == "signature"
- if value.is_a?(JSON::Any)
- case value
- when .as_a?
- value = value.as_a.map { |item| item.as_s }
- end
+ if value.is_a?(JSON::Any) && value.as_a?
+ value = value.as_a.map { |i| i.as_s }
end
case value
@@ -76,14 +73,25 @@ def validate_request(token, session, request, key, db, locale = nil)
raise translate(locale, "Hidden field \"token\" is a required field")
end
- if token["signature"] != sign_token(key, token)
- raise translate(locale, "Invalid signature")
+ expire = token["expire"]?.try &.as_i
+ if expire.try &.< Time.utc.to_unix
+ raise translate(locale, "Token is expired, please try again")
end
if token["session"] != session
raise translate(locale, "Erroneous token")
end
+ scopes = token["scopes"].as_a.map { |v| v.as_s }
+ scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
+ if !scopes_include_scope(scopes, scope)
+ raise translate(locale, "Invalid scope")
+ end
+
+ if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token))
+ raise translate(locale, "Invalid signature")
+ end
+
if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time}))
if nonce[1] > Time.utc
db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0])
@@ -92,18 +100,6 @@ def validate_request(token, session, request, key, db, locale = nil)
end
end
- scopes = token["scopes"].as_a.map { |v| v.as_s }
- scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
-
- if !scopes_include_scope(scopes, scope)
- raise translate(locale, "Invalid scope")
- end
-
- expire = token["expire"]?.try &.as_i
- if expire.try &.< Time.utc.to_unix
- raise translate(locale, "Token is expired, please try again")
- end
-
return {scopes, expire, token["signature"].as_s}
end
diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr
index 6fcfa8d2..a51f15ce 100644
--- a/src/invidious/helpers/utils.cr
+++ b/src/invidious/helpers/utils.cr
@@ -2,13 +2,16 @@ require "lsquic"
require "pool/connection"
def add_yt_headers(request)
- request.headers["x-youtube-client-name"] ||= "1"
- request.headers["x-youtube-client-version"] ||= "1.20180719"
request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
request.headers["accept-language"] ||= "en-us,en;q=0.5"
- request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
+ return if request.resource.starts_with? "/sorry/index"
+ request.headers["x-youtube-client-name"] ||= "1"
+ request.headers["x-youtube-client-version"] ||= "2.20200609"
+ if !CONFIG.cookies.empty?
+ request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
+ end
end
struct QUICPool
@@ -77,7 +80,8 @@ def elapsed_text(elapsed)
end
def make_client(url : URI, region = nil)
- client = HTTPClient.new(url)
+ # TODO: Migrate any applicable endpoints to QUIC
+ client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
@@ -99,7 +103,7 @@ end
def decode_length_seconds(string)
length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i
length_seconds = [0] * (3 - length_seconds.size) + length_seconds
- length_seconds = Time::Span.new(length_seconds[0], length_seconds[1], length_seconds[2])
+ length_seconds = Time::Span.new hours: length_seconds[0], minutes: length_seconds[1], seconds: length_seconds[2]
length_seconds = length_seconds.total_seconds.to_i
return length_seconds
@@ -161,6 +165,7 @@ def decode_date(string : String)
return Time.utc
when "yesterday"
return Time.utc - 1.day
+ else nil # Continue
end
# String matches format "20 hours ago", "4 months ago"...
@@ -315,7 +320,7 @@ def get_referer(env, fallback = "/", unroll = true)
end
referer = referer.full_path
- referer = "/" + referer.lstrip("\/\\")
+ referer = "/" + referer.gsub(/[^\/?@&%=\-_.0-9a-zA-Z]/, "").lstrip("/\\")
if referer == env.request.path
referer = fallback
@@ -324,47 +329,10 @@ def get_referer(env, fallback = "/", unroll = true)
return referer
end
-struct VarInt
- def self.from_io(io : IO, format = IO::ByteFormat::NetworkEndian) : Int32
- result = 0_u32
- num_read = 0
-
- loop do
- byte = io.read_byte
- raise "Invalid VarInt" if !byte
- value = byte & 0x7f
-
- result |= value.to_u32 << (7 * num_read)
- num_read += 1
-
- break if byte & 0x80 == 0
- raise "Invalid VarInt" if num_read > 5
- end
-
- result.to_i32
- end
-
- def self.to_io(io : IO, value : Int32)
- io.write_byte 0x00 if value == 0x00
- value = value.to_u32
-
- while value != 0
- byte = (value & 0x7f).to_u8
- value >>= 7
-
- if value != 0
- byte |= 0x80
- end
-
- io.write_byte byte
- end
- end
-end
-
def sha256(text)
digest = OpenSSL::Digest.new("SHA256")
digest << text
- return digest.hexdigest
+ return digest.final.hexstring
end
def subscribe_pubsub(topic, key, config)
@@ -383,10 +351,8 @@ def subscribe_pubsub(topic, key, config)
nonce = Random::Secure.hex(4)
signature = "#{time}:#{nonce}"
- host_url = make_host_url(config, Kemal.config)
-
body = {
- "hub.callback" => "#{host_url}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
+ "hub.callback" => "#{HOST_URL}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
"hub.topic" => "https://www.youtube.com/xml/feeds/videos.xml?#{topic}",
"hub.verify" => "async",
"hub.mode" => "subscribe",
diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr
new file mode 100644
index 00000000..ec0cad64
--- /dev/null
+++ b/src/invidious/jobs.cr
@@ -0,0 +1,13 @@
+module Invidious::Jobs
+ JOBS = [] of BaseJob
+
+ def self.register(job : BaseJob)
+ JOBS << job
+ end
+
+ def self.start_all
+ JOBS.each do |job|
+ spawn { job.begin }
+ end
+ end
+end
diff --git a/src/invidious/jobs/base_job.cr b/src/invidious/jobs/base_job.cr
new file mode 100644
index 00000000..47e75864
--- /dev/null
+++ b/src/invidious/jobs/base_job.cr
@@ -0,0 +1,3 @@
+abstract class Invidious::Jobs::BaseJob
+ abstract def begin
+end
diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr
new file mode 100644
index 00000000..8b69e01a
--- /dev/null
+++ b/src/invidious/jobs/bypass_captcha_job.cr
@@ -0,0 +1,131 @@
+class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
+ private getter logger : Invidious::LogHandler
+ private getter config : Config
+
+ def initialize(@logger, @config)
+ end
+
+ def begin
+ loop do
+ begin
+ {"/watch?v=jNQXAC9IVRw&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: "UC4QobU6STFB0P71PMvOGN5A")}.each do |path|
+ response = YT_POOL.client &.get(path)
+ if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
+ html = XML.parse_html(response.body)
+ form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
+ site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
+ s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
+
+ inputs = {} of String => String
+ form.xpath_nodes(%(.//input[@name])).map do |node|
+ inputs[node["name"]] = node["value"]
+ end
+
+ headers = response.cookies.add_request_headers(HTTP::Headers.new)
+
+ response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
+ "clientKey" => config.captcha_key,
+ "task" => {
+ "type" => "NoCaptchaTaskProxyless",
+ "websiteURL" => "https://www.youtube.com#{path}",
+ "websiteKey" => site_key,
+ "recaptchaDataSValue" => s_value,
+ },
+ }.to_json).body)
+
+ raise response["error"].as_s if response["error"]?
+ task_id = response["taskId"].as_i
+
+ loop do
+ sleep 10.seconds
+
+ response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
+ "clientKey" => config.captcha_key,
+ "taskId" => task_id,
+ }.to_json).body)
+
+ if response["status"]?.try &.== "ready"
+ break
+ elsif response["errorId"]?.try &.as_i != 0
+ raise response["errorDescription"].as_s
+ end
+ end
+
+ inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
+ headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
+ response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
+
+ response.cookies
+ .select { |cookie| cookie.name != "PREF" }
+ .each { |cookie| config.cookies << cookie }
+
+ # Persist cookies between runs
+ File.write("config/config.yml", config.to_yaml)
+ elsif response.headers["Location"]?.try &.includes?("/sorry/index")
+ location = response.headers["Location"].try { |u| URI.parse(u) }
+ headers = HTTP::Headers{":authority" => location.host.not_nil!}
+ response = YT_POOL.client &.get(location.full_path, headers)
+
+ html = XML.parse_html(response.body)
+ form = html.xpath_node(%(//form[@action="index"])).not_nil!
+ site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
+ s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
+
+ inputs = {} of String => String
+ form.xpath_nodes(%(.//input[@name])).map do |node|
+ inputs[node["name"]] = node["value"]
+ end
+
+ captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com"))
+ captcha_client.family = config.force_resolve || Socket::Family::INET
+ response = JSON.parse(captcha_client.post("/createTask", body: {
+ "clientKey" => config.captcha_key,
+ "task" => {
+ "type" => "NoCaptchaTaskProxyless",
+ "websiteURL" => location.to_s,
+ "websiteKey" => site_key,
+ "recaptchaDataSValue" => s_value,
+ },
+ }.to_json).body)
+
+ raise response["error"].as_s if response["error"]?
+ task_id = response["taskId"].as_i
+
+ loop do
+ sleep 10.seconds
+
+ response = JSON.parse(captcha_client.post("/getTaskResult", body: {
+ "clientKey" => config.captcha_key,
+ "taskId" => task_id,
+ }.to_json).body)
+
+ if response["status"]?.try &.== "ready"
+ break
+ elsif response["errorId"]?.try &.as_i != 0
+ raise response["errorDescription"].as_s
+ end
+ end
+
+ inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
+ headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
+ response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
+ headers = HTTP::Headers{
+ "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
+ }
+ cookies = HTTP::Cookies.from_headers(headers)
+
+ cookies.each { |cookie| config.cookies << cookie }
+
+ # Persist cookies between runs
+ File.write("config/config.yml", config.to_yaml)
+ end
+ end
+ rescue ex
+ logger.puts("Exception: #{ex.message}")
+ ensure
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+ end
+end
diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr
new file mode 100644
index 00000000..2f525e08
--- /dev/null
+++ b/src/invidious/jobs/notification_job.cr
@@ -0,0 +1,24 @@
+class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
+ private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
+ private getter pg_url : URI
+
+ def initialize(@connection_channel, @pg_url)
+ end
+
+ def begin
+ connections = [] of Channel(PQ::Notification)
+
+ PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
+
+ loop do
+ action, connection = connection_channel.receive
+
+ case action
+ when true
+ connections << connection
+ when false
+ connections.delete(connection)
+ end
+ end
+ end
+end
diff --git a/src/invidious/jobs/pull_popular_videos_job.cr b/src/invidious/jobs/pull_popular_videos_job.cr
new file mode 100644
index 00000000..7a8ab84e
--- /dev/null
+++ b/src/invidious/jobs/pull_popular_videos_job.cr
@@ -0,0 +1,27 @@
+class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
+ QUERY = <<-SQL
+ SELECT DISTINCT ON (ucid) *
+ FROM channel_videos
+ WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d
+ GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40)
+ ORDER BY ucid, published DESC
+ SQL
+ POPULAR_VIDEOS = Atomic.new([] of ChannelVideo)
+ private getter db : DB::Database
+
+ def initialize(@db)
+ end
+
+ def begin
+ loop do
+ videos = db.query_all(QUERY, as: ChannelVideo)
+ .sort_by(&.published)
+ .reverse
+
+ POPULAR_VIDEOS.set(videos)
+
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+end
diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr
new file mode 100644
index 00000000..75fc474d
--- /dev/null
+++ b/src/invidious/jobs/refresh_channels_job.cr
@@ -0,0 +1,59 @@
+class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
+ private getter db : DB::Database
+ private getter logger : Invidious::LogHandler
+ private getter config : Config
+
+ def initialize(@db, @logger, @config)
+ end
+
+ def begin
+ max_threads = config.channel_threads
+ lim_threads = max_threads
+ active_threads = 0
+ active_channel = Channel(Bool).new
+ backoff = 1.seconds
+
+ loop do
+ db.query("SELECT id FROM channels ORDER BY updated") do |rs|
+ rs.each do
+ id = rs.read(String)
+
+ if active_threads >= lim_threads
+ if active_channel.receive
+ active_threads -= 1
+ end
+ end
+
+ active_threads += 1
+ spawn do
+ begin
+ channel = fetch_channel(id, db, config.full_refresh)
+
+ lim_threads = max_threads
+ db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
+ rescue ex
+ logger.puts("#{id} : #{ex.message}")
+ if ex.message == "Deleted or invalid channel"
+ db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
+ else
+ lim_threads = 1
+ logger.puts("#{id} : backing off for #{backoff}s")
+ sleep backoff
+ if backoff < 1.days
+ backoff += backoff
+ else
+ backoff = 1.days
+ end
+ end
+ end
+
+ active_channel.send(true)
+ end
+ end
+ end
+
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+end
diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr
new file mode 100644
index 00000000..eebdf0f3
--- /dev/null
+++ b/src/invidious/jobs/refresh_feeds_job.cr
@@ -0,0 +1,77 @@
+class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
+ private getter db : DB::Database
+ private getter logger : Invidious::LogHandler
+ private getter config : Config
+
+ def initialize(@db, @logger, @config)
+ end
+
+ def begin
+ max_threads = config.feed_threads
+ active_threads = 0
+ active_channel = Channel(Bool).new
+
+ loop do
+ db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
+ rs.each do
+ email = rs.read(String)
+ view_name = "subscriptions_#{sha256(email)}"
+
+ if active_threads >= max_threads
+ if active_channel.receive
+ active_threads -= 1
+ end
+ end
+
+ active_threads += 1
+ spawn do
+ begin
+ # Drop outdated views
+ column_array = get_column_array(db, view_name)
+ ChannelVideo.type_array.each_with_index do |name, i|
+ if name != column_array[i]?
+ logger.puts("DROP MATERIALIZED VIEW #{view_name}")
+ db.exec("DROP MATERIALIZED VIEW #{view_name}")
+ raise "view does not exist"
+ end
+ end
+
+ if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
+ logger.puts("Materialized view #{view_name} is out-of-date, recreating...")
+ db.exec("DROP MATERIALIZED VIEW #{view_name}")
+ end
+
+ db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
+ db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
+ rescue ex
+ # Rename old views
+ begin
+ legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
+
+ db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
+ logger.puts("RENAME MATERIALIZED VIEW #{legacy_view_name}")
+ db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
+ rescue ex
+ begin
+ # While iterating through, we may have an email stored from a deleted account
+ if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
+ logger.puts("CREATE #{view_name}")
+ db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}")
+ db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
+ end
+ rescue ex
+ logger.puts("REFRESH #{email} : #{ex.message}")
+ end
+ end
+ end
+
+ active_channel.send(true)
+ end
+ end
+ end
+
+ sleep 5.seconds
+ Fiber.yield
+ end
+ end
+end
diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr
new file mode 100644
index 00000000..021671be
--- /dev/null
+++ b/src/invidious/jobs/statistics_refresh_job.cr
@@ -0,0 +1,59 @@
+class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
+ STATISTICS = {
+ "version" => "2.0",
+ "software" => {
+ "name" => "invidious",
+ "version" => "",
+ "branch" => "",
+ },
+ "openRegistrations" => true,
+ "usage" => {
+ "users" => {
+ "total" => 0_i64,
+ "activeHalfyear" => 0_i64,
+ "activeMonth" => 0_i64,
+ },
+ },
+ "metadata" => {
+ "updatedAt" => Time.utc.to_unix,
+ "lastChannelRefreshedAt" => 0_i64,
+ },
+ }
+
+ private getter db : DB::Database
+ private getter config : Config
+
+ def initialize(@db, @config, @software_config : Hash(String, String))
+ end
+
+ def begin
+ load_initial_stats
+
+ loop do
+ refresh_stats
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+
+ # should only be called once at the very beginning
+ private def load_initial_stats
+ STATISTICS["software"] = {
+ "name" => @software_config["name"],
+ "version" => @software_config["version"],
+ "branch" => @software_config["branch"],
+ }
+ STATISTICS["openRegistration"] = config.registration_enabled
+ end
+
+ private def refresh_stats
+ users = STATISTICS.dig("usage", "users").as(Hash(String, Int64))
+ users["total"] = db.query_one("SELECT count(*) FROM users", as: Int64)
+ users["activeHalfyear"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64)
+ users["activeMonth"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64)
+ STATISTICS["metadata"] = {
+ "updatedAt" => Time.utc.to_unix,
+ "lastChannelRefreshedAt" => db.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64,
+ }
+ end
+end
diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr
new file mode 100644
index 00000000..3d3b2218
--- /dev/null
+++ b/src/invidious/jobs/subscribe_to_feeds_job.cr
@@ -0,0 +1,52 @@
+class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
+ private getter db : DB::Database
+ private getter logger : Invidious::LogHandler
+ private getter hmac_key : String
+ private getter config : Config
+
+ def initialize(@db, @logger, @config, @hmac_key)
+ end
+
+ def begin
+ max_threads = 1
+ if config.use_pubsub_feeds.is_a?(Int32)
+ max_threads = config.use_pubsub_feeds.as(Int32)
+ end
+
+ active_threads = 0
+ active_channel = Channel(Bool).new
+
+ loop do
+ db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
+ rs.each do
+ ucid = rs.read(String)
+
+ if active_threads >= max_threads.as(Int32)
+ if active_channel.receive
+ active_threads -= 1
+ end
+ end
+
+ active_threads += 1
+
+ spawn do
+ begin
+ response = subscribe_pubsub(ucid, hmac_key, config)
+
+ if response.status_code >= 400
+ logger.puts("#{ucid} : #{response.body}")
+ end
+ rescue ex
+ logger.puts("#{ucid} : #{ex.message}")
+ end
+
+ active_channel.send(true)
+ end
+ end
+ end
+
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+end
diff --git a/src/invidious/jobs/update_decrypt_function_job.cr b/src/invidious/jobs/update_decrypt_function_job.cr
new file mode 100644
index 00000000..5332c672
--- /dev/null
+++ b/src/invidious/jobs/update_decrypt_function_job.cr
@@ -0,0 +1,19 @@
+class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
+ DECRYPT_FUNCTION = [] of {SigProc, Int32}
+
+ def begin
+ loop do
+ begin
+ decrypt_function = fetch_decrypt_function
+ DECRYPT_FUNCTION.clear
+ decrypt_function.each { |df| DECRYPT_FUNCTION << df }
+ rescue ex
+ # TODO: Log error
+ next
+ ensure
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+ end
+end
diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr
index 04a37b87..c69eb0c4 100644
--- a/src/invidious/mixes.cr
+++ b/src/invidious/mixes.cr
@@ -1,32 +1,32 @@
struct MixVideo
- db_mapping({
- title: String,
- id: String,
- author: String,
- ucid: String,
- length_seconds: Int32,
- index: Int32,
- rdid: String,
- })
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property ucid : String
+ property length_seconds : Int32
+ property index : Int32
+ property rdid : String
end
struct Mix
- db_mapping({
- title: String,
- id: String,
- videos: Array(MixVideo),
- })
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property videos : Array(MixVideo)
end
def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
headers = HTTP::Headers.new
- headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
if cookies
headers = cookies.add_request_headers(headers)
end
- response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers)
+ video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
+ response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
initial_data = extract_initial_data(response.body)
if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?
@@ -49,23 +49,22 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
id = item["videoId"].as_s
title = item["title"]?.try &.["simpleText"].as_s
- if !title
- next
- end
+ next if !title
+
author = item["longBylineText"]["runs"][0]["text"].as_s
ucid = item["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s
length_seconds = decode_length_seconds(item["lengthText"]["simpleText"].as_s)
index = item["navigationEndpoint"]["watchEndpoint"]["index"].as_i
- videos << MixVideo.new(
- title,
- id,
- author,
- ucid,
- length_seconds,
- index,
- rdid
- )
+ videos << MixVideo.new({
+ title: title,
+ id: id,
+ author: author,
+ ucid: ucid,
+ length_seconds: length_seconds,
+ index: index,
+ rdid: rdid,
+ })
end
if !cookies
@@ -75,7 +74,11 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
videos.uniq! { |video| video.id }
videos = videos.first(50)
- return Mix.new(mix_title, rdid, videos)
+ return Mix.new({
+ title: mix_title,
+ id: rdid,
+ videos: videos,
+ })
end
def template_mix(mix)
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr
index 9c8afd3c..c984a12a 100644
--- a/src/invidious/playlists.cr
+++ b/src/invidious/playlists.cr
@@ -1,26 +1,38 @@
struct PlaylistVideo
- def to_xml(host_url, auto_generated, xml : XML::Builder)
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property ucid : String
+ property length_seconds : Int32
+ property published : Time
+ property plid : String
+ property index : Int64
+ property live_now : Bool
+
+ def to_xml(auto_generated, xml : XML::Builder)
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
- xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{self.id}")
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
- xml.element("a", href: "#{host_url}/watch?v=#{self.id}") do
- xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg")
+ xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do
+ xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
@@ -29,23 +41,23 @@ struct PlaylistVideo
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
- xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg",
+ xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
- def to_xml(host_url, auto_generated, xml : XML::Builder? = nil)
+ def to_xml(auto_generated, xml : XML::Builder? = nil)
if xml
- to_xml(host_url, auto_generated, xml)
+ to_xml(auto_generated, xml)
else
XML.build do |json|
- to_xml(host_url, auto_generated, xml)
+ to_xml(auto_generated, xml)
end
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder, index : Int32?)
+ def to_json(locale, json : JSON::Builder, index : Int32?)
json.object do
json.field "title", self.title
json.field "videoId", self.id
@@ -55,7 +67,7 @@ struct PlaylistVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
- generate_thumbnails(json, self.id, config, kemal_config)
+ generate_thumbnails(json, self.id)
end
if index
@@ -69,31 +81,32 @@ struct PlaylistVideo
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder? = nil, index : Int32? = nil)
+ def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil)
if json
- to_json(locale, config, kemal_config, json, index: index)
+ to_json(locale, json, index: index)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, json, index: index)
+ to_json(locale, json, index: index)
end
end
end
-
- db_mapping({
- title: String,
- id: String,
- author: String,
- ucid: String,
- length_seconds: Int32,
- published: Time,
- plid: String,
- index: Int64,
- live_now: Bool,
- })
end
struct Playlist
- def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil)
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property author_thumbnail : String
+ property ucid : String
+ property description : String
+ property video_count : Int32
+ property views : Int64
+ property updated : Time
+ property thumbnail : String?
+
+ def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
@@ -118,7 +131,7 @@ struct Playlist
end
end
- json.field "description", html_to_content(self.description_html)
+ json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
@@ -130,39 +143,30 @@ struct Playlist
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
- video.to_json(locale, config, Kemal.config, json)
+ video.to_json(locale, json)
end
end
end
end
end
- def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil)
+ def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
- to_json(offset, locale, config, kemal_config, json, continuation: continuation)
+ to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
- to_json(offset, locale, config, kemal_config, json, continuation: continuation)
+ to_json(offset, locale, json, continuation: continuation)
end
end
end
- db_mapping({
- title: String,
- id: String,
- author: String,
- author_thumbnail: String,
- ucid: String,
- description_html: String,
- video_count: Int32,
- views: Int64,
- updated: Time,
- thumbnail: String?,
- })
-
def privacy
PlaylistPrivacy::Public
end
+
+ def description_html
+ HTML.escape(self.description).gsub("\n", "]+>]+>\s*less\s*]+>\n<\/span><\/button>/, "")
- document = XML.parse_html(body)
+ initial_data = extract_initial_data(response.body)
+ playlist_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[0]["playlistSidebarPrimaryInfoRenderer"]?
- title = document.xpath_node(%q(//h1[@class="pl-header-title"]))
- if !title
- raise translate(locale, "Playlist does not exist.")
+ raise "Could not extract playlist info" if !playlist_info
+ title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || ""
+
+ desc_item = playlist_info["description"]?
+ description = desc_item.try &.["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || ""
+
+ thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]?
+ .try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s
+
+ views = 0_i64
+ updated = Time.utc
+ video_count = 0
+ playlist_info["stats"]?.try &.as_a.each do |stat|
+ text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
+ next if !text
+
+ if text.includes? "video"
+ video_count = text.gsub(/\D/, "").to_i? || 0
+ elsif text.includes? "view"
+ views = text.gsub(/\D/, "").to_i64? || 0_i64
+ else
+ updated = decode_date(text.lchop("Last updated on ").lchop("Updated "))
+ end
end
- title = title.content.strip(" \n")
- description_html = document.xpath_node(%q(//span[@class="pl-header-description-text"]/div/div[1])).try &.to_s ||
- document.xpath_node(%q(//span[@class="pl-header-description-text"])).try &.to_s || ""
+ author_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[1]["playlistSidebarSecondaryInfoRenderer"]?
+ .try &.["videoOwner"]["videoOwnerRenderer"]?
- playlist_thumbnail = document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["data-thumb"]? ||
- document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["src"]
+ raise "Could not extract author info" if !author_info
- # YouTube allows anonymous playlists, so most of this can be empty or optional
- anchor = document.xpath_node(%q(//ul[@class="pl-header-details"]))
- author = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.content
- author ||= ""
- author_thumbnail = document.xpath_node(%q(//img[@class="channel-header-profile-image"])).try &.["src"]
- author_thumbnail ||= ""
- ucid = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.["href"].split("/")[-1]
- ucid ||= ""
+ author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || ""
+ author = author_info["title"]["runs"][0]["text"]?.try &.as_s || ""
+ ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
- video_count = anchor.try &.xpath_node(%q(.//li[2])).try &.content.gsub(/\D/, "").to_i?
- video_count ||= 0
-
- views = anchor.try &.xpath_node(%q(.//li[3])).try &.content.gsub(/\D/, "").to_i64?
- views ||= 0_i64
-
- updated = anchor.try &.xpath_node(%q(.//li[4])).try &.content.lchop("Last updated on ").lchop("Updated ").try { |date| decode_date(date) }
- updated ||= Time.utc
-
- playlist = Playlist.new(
- title: title,
- id: plid,
- author: author,
+ return Playlist.new({
+ title: title,
+ id: plid,
+ author: author,
author_thumbnail: author_thumbnail,
- ucid: ucid,
- description_html: description_html,
- video_count: video_count,
- views: views,
- updated: updated,
- thumbnail: playlist_thumbnail,
- )
-
- return playlist
+ ucid: ucid,
+ description: description,
+ video_count: video_count,
+ views: views,
+ updated: updated,
+ thumbnail: thumbnail,
+ })
end
def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
@@ -437,35 +425,26 @@ end
def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil)
if continuation
- html = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
- html = XML.parse_html(html.body)
-
- index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i?.try &.- 1
- offset = index || offset
+ response = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en")
+ initial_data = extract_initial_data(response.body)
+ offset = initial_data["currentVideoEndpoint"]?.try &.["watchEndpoint"]?.try &.["index"]?.try &.as_i64 || offset
end
if video_count > 100
url = produce_playlist_url(plid, offset)
response = YT_POOL.client &.get(url)
- response = JSON.parse(response.body)
- if !response["content_html"]? || response["content_html"].as_s.empty?
- raise translate(locale, "Empty playlist")
- end
-
- document = XML.parse_html(response["content_html"].as_s)
- nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")]))
- videos = extract_playlist(plid, nodeset, offset)
+ initial_data = JSON.parse(response.body).as_a.find(&.as_h.["response"]?).try &.as_h
elsif offset > 100
return [] of PlaylistVideo
else # Extract first page of videos
- response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1")
- document = XML.parse_html(response.body)
- nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")]))
-
- videos = extract_playlist(plid, nodeset, 0)
+ response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en")
+ initial_data = extract_initial_data(response.body)
end
+ return [] of PlaylistVideo if !initial_data
+ videos = extract_playlist_videos(initial_data)
+
until videos.empty? || videos[0].index == offset
videos.shift
end
@@ -473,6 +452,45 @@ def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuat
return videos
end
+def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
+ videos = [] of PlaylistVideo
+
+ (initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"].as_a ||
+ initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a).try &.each do |item|
+ if i = item["playlistVideoRenderer"]?
+ video_id = i["navigationEndpoint"]["watchEndpoint"]["videoId"].as_s
+ plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s
+ index = i["navigationEndpoint"]["watchEndpoint"]["index"].as_i64
+
+ thumbnail = i["thumbnail"]["thumbnails"][0]["url"].as_s
+ title = i["title"].try { |t| t["simpleText"]? || t["runs"]?.try &.[0]["text"]? }.try &.as_s || ""
+ author = i["shortBylineText"]?.try &.["runs"][0]["text"].as_s || ""
+ ucid = i["shortBylineText"]?.try &.["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s || ""
+ length_seconds = i["lengthSeconds"]?.try &.as_s.to_i
+ live = false
+
+ if !length_seconds
+ live = true
+ length_seconds = 0
+ end
+
+ videos << PlaylistVideo.new({
+ title: title,
+ id: video_id,
+ author: author,
+ ucid: ucid,
+ length_seconds: length_seconds,
+ published: Time.utc,
+ plid: plid,
+ live_now: live,
+ index: index,
+ })
+ end
+ end
+
+ return videos
+end
+
def template_playlist(playlist)
html = <<-END_HTML
diff --git a/src/invidious/routes/base_route.cr b/src/invidious/routes/base_route.cr
new file mode 100644
index 00000000..c6e6667e
--- /dev/null
+++ b/src/invidious/routes/base_route.cr
@@ -0,0 +1,9 @@
+abstract class Invidious::Routes::BaseRoute
+ private getter config : Config
+ private getter logger : Invidious::LogHandler
+
+ def initialize(@config, @logger)
+ end
+
+ abstract def handle(env)
+end
diff --git a/src/invidious/routes/embed/index.cr b/src/invidious/routes/embed/index.cr
new file mode 100644
index 00000000..79c91d86
--- /dev/null
+++ b/src/invidious/routes/embed/index.cr
@@ -0,0 +1,27 @@
+class Invidious::Routes::Embed::Index < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+
+ if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
+ begin
+ playlist = get_playlist(PG_DB, plid, locale: locale)
+ offset = env.params.query["index"]?.try &.to_i? || 0
+ videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
+ rescue ex
+ error_message = ex.message
+ env.response.status_code = 500
+ return templated "error"
+ end
+
+ url = "/embed/#{videos[0].id}?#{env.params.query}"
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query}"
+ end
+ else
+ url = "/"
+ end
+
+ env.redirect url
+ end
+end
diff --git a/src/invidious/routes/embed/show.cr b/src/invidious/routes/embed/show.cr
new file mode 100644
index 00000000..23c2b86f
--- /dev/null
+++ b/src/invidious/routes/embed/show.cr
@@ -0,0 +1,174 @@
+class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+ id = env.params.url["id"]
+
+ plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
+ continuation = process_continuation(PG_DB, env.params.query, plid, id)
+
+ if md = env.params.query["playlist"]?
+ .try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/)
+ video_series = md[0].split(",")
+ env.params.query.delete("playlist")
+ end
+
+ preferences = env.get("preferences").as(Preferences)
+
+ if id.includes?("%20") || id.includes?("+") || env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
+ id = env.params.url["id"].gsub("%20", "").delete("+")
+
+ url = "/embed/#{id}"
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query.to_s.gsub("%20", "").delete("+")}"
+ end
+
+ return env.redirect url
+ end
+
+ # YouTube embed supports `videoseries` with either `list=PLID`
+ # or `playlist=VIDEO_ID,VIDEO_ID`
+ case id
+ when "videoseries"
+ url = ""
+
+ if plid
+ begin
+ playlist = get_playlist(PG_DB, plid, locale: locale)
+ offset = env.params.query["index"]?.try &.to_i? || 0
+ videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
+ rescue ex
+ error_message = ex.message
+ env.response.status_code = 500
+ return templated "error"
+ end
+
+ url = "/embed/#{videos[0].id}"
+ elsif video_series
+ url = "/embed/#{video_series.shift}"
+ env.params.query["playlist"] = video_series.join(",")
+ else
+ return env.redirect "/"
+ end
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query}"
+ end
+
+ return env.redirect url
+ when "live_stream"
+ response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
+ video_id = response.body.match(/"video_id":"(?[a-zA-Z0-9_-]{11})"/).try &.["video_id"]
+
+ env.params.query.delete_all("channel")
+
+ if !video_id || video_id == "live_stream"
+ error_message = "Video is unavailable."
+ return templated "error"
+ end
+
+ url = "/embed/#{video_id}"
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query}"
+ end
+
+ return env.redirect url
+ when id.size > 11
+ url = "/embed/#{id[0, 11]}"
+
+ if env.params.query.size > 0
+ url += "?#{env.params.query}"
+ end
+
+ return env.redirect url
+ else nil # Continue
+ end
+
+ params = process_video_params(env.params.query, preferences)
+
+ user = env.get?("user").try &.as(User)
+ if user
+ subscriptions = user.subscriptions
+ watched = user.watched
+ notifications = user.notifications
+ end
+ subscriptions ||= [] of String
+
+ begin
+ video = get_video(id, PG_DB, region: params.region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex
+ error_message = ex.message
+ env.response.status_code = 500
+ return templated "error"
+ end
+
+ if preferences.annotations_subscribed &&
+ subscriptions.includes?(video.ucid) &&
+ (env.params.query["iv_load_policy"]? || "1") == "1"
+ params.annotations = true
+ end
+
+ # if watched && !watched.includes? id
+ # PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
+ # end
+
+ if notifications && notifications.includes? id
+ PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
+ env.get("user").as(User).notifications.delete(id)
+ notifications.delete(id)
+ end
+
+ fmt_stream = video.fmt_stream
+ adaptive_fmts = video.adaptive_fmts
+
+ if params.local
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
+ end
+
+ video_streams = video.video_streams
+ audio_streams = video.audio_streams
+
+ if audio_streams.empty? && !video.live_now
+ if params.quality == "dash"
+ env.params.query.delete_all("quality")
+ return env.redirect "/embed/#{id}?#{env.params.query}"
+ elsif params.listen
+ env.params.query.delete_all("listen")
+ env.params.query["listen"] = "0"
+ return env.redirect "/embed/#{id}?#{env.params.query}"
+ end
+ end
+
+ captions = video.captions
+
+ preferred_captions = captions.select { |caption|
+ params.preferred_captions.includes?(caption.name.simpleText) ||
+ params.preferred_captions.includes?(caption.languageCode.split("-")[0])
+ }
+ preferred_captions.sort_by! { |caption|
+ (params.preferred_captions.index(caption.name.simpleText) ||
+ params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil!
+ }
+ captions = captions - preferred_captions
+
+ aspect_ratio = nil
+
+ thumbnail = "/vi/#{video.id}/maxres.jpg"
+
+ if params.raw
+ url = fmt_stream[0]["url"].as_s
+
+ fmt_stream.each do |fmt|
+ url = fmt["url"].as_s if fmt["quality"].as_s == params.quality
+ end
+
+ return env.redirect url
+ end
+
+ rendered "embed"
+ end
+end
diff --git a/src/invidious/routes/home.cr b/src/invidious/routes/home.cr
new file mode 100644
index 00000000..9b1bf61b
--- /dev/null
+++ b/src/invidious/routes/home.cr
@@ -0,0 +1,34 @@
+class Invidious::Routes::Home < Invidious::Routes::BaseRoute
+ def handle(env)
+ preferences = env.get("preferences").as(Preferences)
+ locale = LOCALES[preferences.locale]?
+ user = env.get? "user"
+
+ case preferences.default_home
+ when ""
+ templated "empty"
+ when "Popular"
+ templated "popular"
+ when "Trending"
+ env.redirect "/feed/trending"
+ when "Subscriptions"
+ if user
+ env.redirect "/feed/subscriptions"
+ else
+ templated "popular"
+ end
+ when "Playlists"
+ if user
+ env.redirect "/view_all_playlists"
+ else
+ templated "popular"
+ end
+ else
+ templated "empty"
+ end
+ end
+
+ private def popular_videos
+ Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
+ end
+end
diff --git a/src/invidious/routes/licenses.cr b/src/invidious/routes/licenses.cr
new file mode 100644
index 00000000..38fde7bb
--- /dev/null
+++ b/src/invidious/routes/licenses.cr
@@ -0,0 +1,6 @@
+class Invidious::Routes::Licenses < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+ rendered "licenses"
+ end
+end
diff --git a/src/invidious/routes/privacy.cr b/src/invidious/routes/privacy.cr
new file mode 100644
index 00000000..4565c94c
--- /dev/null
+++ b/src/invidious/routes/privacy.cr
@@ -0,0 +1,6 @@
+class Invidious::Routes::Privacy < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+ templated "privacy"
+ end
+end
diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr
new file mode 100644
index 00000000..4eee7793
--- /dev/null
+++ b/src/invidious/routes/watch.cr
@@ -0,0 +1,186 @@
+class Invidious::Routes::Watch < Invidious::Routes::BaseRoute
+ def handle(env)
+ locale = LOCALES[env.get("preferences").as(Preferences).locale]?
+ region = env.params.query["region"]?
+
+ if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
+ url = "/watch?" + env.params.query.to_s.gsub("%20", "").delete("+")
+ return env.redirect url
+ end
+
+ if env.params.query["v"]?
+ id = env.params.query["v"]
+
+ if env.params.query["v"].empty?
+ error_message = "Invalid parameters."
+ env.response.status_code = 400
+ return templated "error"
+ end
+
+ if id.size > 11
+ url = "/watch?v=#{id[0, 11]}"
+ env.params.query.delete_all("v")
+ if env.params.query.size > 0
+ url += "&#{env.params.query}"
+ end
+
+ return env.redirect url
+ end
+ else
+ return env.redirect "/"
+ end
+
+ plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
+ continuation = process_continuation(PG_DB, env.params.query, plid, id)
+
+ nojs = env.params.query["nojs"]?
+
+ nojs ||= "0"
+ nojs = nojs == "1"
+
+ preferences = env.get("preferences").as(Preferences)
+
+ user = env.get?("user").try &.as(User)
+ if user
+ subscriptions = user.subscriptions
+ watched = user.watched
+ notifications = user.notifications
+ end
+ subscriptions ||= [] of String
+
+ params = process_video_params(env.params.query, preferences)
+ env.params.query.delete_all("listen")
+
+ begin
+ video = get_video(id, PG_DB, region: params.region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex
+ error_message = ex.message
+ env.response.status_code = 500
+ logger.puts("#{id} : #{ex.message}")
+ return templated "error"
+ end
+
+ if preferences.annotations_subscribed &&
+ subscriptions.includes?(video.ucid) &&
+ (env.params.query["iv_load_policy"]? || "1") == "1"
+ params.annotations = true
+ end
+ env.params.query.delete_all("iv_load_policy")
+
+ if watched && !watched.includes? id
+ PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
+ end
+
+ if notifications && notifications.includes? id
+ PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
+ env.get("user").as(User).notifications.delete(id)
+ notifications.delete(id)
+ end
+
+ if nojs
+ if preferences
+ source = preferences.comments[0]
+ if source.empty?
+ source = preferences.comments[1]
+ end
+
+ if source == "youtube"
+ begin
+ comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ rescue ex
+ if preferences.comments[1] == "reddit"
+ comments, reddit_thread = fetch_reddit_comments(id)
+ comment_html = template_reddit_comments(comments, locale)
+
+ comment_html = fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = replace_links(comment_html)
+ end
+ end
+ elsif source == "reddit"
+ begin
+ comments, reddit_thread = fetch_reddit_comments(id)
+ comment_html = template_reddit_comments(comments, locale)
+
+ comment_html = fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = replace_links(comment_html)
+ rescue ex
+ if preferences.comments[1] == "youtube"
+ comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ end
+ end
+ end
+ else
+ comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ end
+
+ comment_html ||= ""
+ end
+
+ fmt_stream = video.fmt_stream
+ adaptive_fmts = video.adaptive_fmts
+
+ if params.local
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
+ end
+
+ video_streams = video.video_streams
+ audio_streams = video.audio_streams
+
+ # Older videos may not have audio sources available.
+ # We redirect here so they're not unplayable
+ if audio_streams.empty? && !video.live_now
+ if params.quality == "dash"
+ env.params.query.delete_all("quality")
+ env.params.query["quality"] = "medium"
+ return env.redirect "/watch?#{env.params.query}"
+ elsif params.listen
+ env.params.query.delete_all("listen")
+ env.params.query["listen"] = "0"
+ return env.redirect "/watch?#{env.params.query}"
+ end
+ end
+
+ captions = video.captions
+
+ preferred_captions = captions.select { |caption|
+ params.preferred_captions.includes?(caption.name.simpleText) ||
+ params.preferred_captions.includes?(caption.languageCode.split("-")[0])
+ }
+ preferred_captions.sort_by! { |caption|
+ (params.preferred_captions.index(caption.name.simpleText) ||
+ params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil!
+ }
+ captions = captions - preferred_captions
+
+ aspect_ratio = "16:9"
+
+ thumbnail = "/vi/#{video.id}/maxres.jpg"
+
+ if params.raw
+ if params.listen
+ url = audio_streams[0]["url"].as_s
+
+ audio_streams.each do |fmt|
+ if fmt["bitrate"].as_i == params.quality.rchop("k").to_i
+ url = fmt["url"].as_s
+ end
+ end
+ else
+ url = fmt_stream[0]["url"].as_s
+
+ fmt_stream.each do |fmt|
+ if fmt["quality"].as_s == params.quality
+ url = fmt["url"].as_s
+ end
+ end
+ end
+
+ return env.redirect url
+ end
+
+ templated "watch"
+ end
+end
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
new file mode 100644
index 00000000..c09dda38
--- /dev/null
+++ b/src/invidious/routing.cr
@@ -0,0 +1,8 @@
+module Invidious::Routing
+ macro get(path, controller)
+ get {{ path }} do |env|
+ controller_instance = {{ controller }}.new(config, logger)
+ controller_instance.handle(env)
+ end
+ end
+end
diff --git a/src/invidious/search.cr b/src/invidious/search.cr
index 92996f75..85fd024a 100644
--- a/src/invidious/search.cr
+++ b/src/invidious/search.cr
@@ -1,5 +1,20 @@
struct SearchVideo
- def to_xml(host_url, auto_generated, query_params, xml : XML::Builder)
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property ucid : String
+ property published : Time
+ property views : Int64
+ property description_html : String
+ property length_seconds : Int32
+ property live_now : Bool
+ property paid : Bool
+ property premium : Bool
+ property premiere_timestamp : Time?
+
+ def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
@@ -7,22 +22,22 @@ struct SearchVideo
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
- xml.element("link", rel: "alternate", href: "#{host_url}/watch?#{query_params}")
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
- xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
- xml.element("a", href: "#{host_url}/watch?#{query_params}") do
- xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg")
+ xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
+ xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
@@ -33,7 +48,7 @@ struct SearchVideo
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
- xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg",
+ xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text html_to_content(self.description_html) }
end
@@ -44,17 +59,17 @@ struct SearchVideo
end
end
- def to_xml(host_url, auto_generated, query_params, xml : XML::Builder | Nil = nil)
+ def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil)
if xml
- to_xml(host_url, auto_generated, query_params, xml)
+ to_xml(HOST_URL, auto_generated, query_params, xml)
else
XML.build do |json|
- to_xml(host_url, auto_generated, query_params, xml)
+ to_xml(HOST_URL, auto_generated, query_params, xml)
end
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder)
+ def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
@@ -65,7 +80,7 @@ struct SearchVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
- generate_thumbnails(json, self.id, config, kemal_config)
+ generate_thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
@@ -78,45 +93,49 @@ struct SearchVideo
json.field "liveNow", self.live_now
json.field "paid", self.paid
json.field "premium", self.premium
- end
- end
+ json.field "isUpcoming", self.is_upcoming
- def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
- if json
- to_json(locale, config, kemal_config, json)
- else
- JSON.build do |json|
- to_json(locale, config, kemal_config, json)
+ if self.premiere_timestamp
+ json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
end
end
- db_mapping({
- title: String,
- id: String,
- author: String,
- ucid: String,
- published: Time,
- views: Int64,
- description_html: String,
- length_seconds: Int32,
- live_now: Bool,
- paid: Bool,
- premium: Bool,
- premiere_timestamp: Time?,
- })
+ def to_json(locale, json : JSON::Builder | Nil = nil)
+ if json
+ to_json(locale, json)
+ else
+ JSON.build do |json|
+ to_json(locale, json)
+ end
+ end
+ end
+
+ def is_upcoming
+ premiere_timestamp ? true : false
+ end
end
struct SearchPlaylistVideo
- db_mapping({
- title: String,
- id: String,
- length_seconds: Int32,
- })
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property length_seconds : Int32
end
struct SearchPlaylist
- def to_json(locale, config, kemal_config, json : JSON::Builder)
+ include DB::Serializable
+
+ property title : String
+ property id : String
+ property author : String
+ property ucid : String
+ property video_count : Int32
+ property videos : Array(SearchPlaylistVideo)
+ property thumbnail : String?
+
+ def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "playlist"
json.field "title", self.title
@@ -137,7 +156,7 @@ struct SearchPlaylist
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
- generate_thumbnails(json, video.id, config, Kemal.config)
+ generate_thumbnails(json, video.id)
end
end
end
@@ -146,29 +165,29 @@ struct SearchPlaylist
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
+ def to_json(locale, json : JSON::Builder | Nil = nil)
if json
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
end
end
end
-
- db_mapping({
- title: String,
- id: String,
- author: String,
- ucid: String,
- video_count: Int32,
- videos: Array(SearchPlaylistVideo),
- thumbnail: String?,
- })
end
struct SearchChannel
- def to_json(locale, config, kemal_config, json : JSON::Builder)
+ include DB::Serializable
+
+ property author : String
+ property ucid : String
+ property author_thumbnail : String
+ property subscriber_count : Int32
+ property video_count : Int32
+ property description_html : String
+ property auto_generated : Bool
+
+ def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "channel"
json.field "author", self.author
@@ -198,85 +217,50 @@ struct SearchChannel
end
end
- def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
+ def to_json(locale, json : JSON::Builder | Nil = nil)
if json
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, json)
+ to_json(locale, json)
end
end
end
-
- db_mapping({
- author: String,
- ucid: String,
- author_thumbnail: String,
- subscriber_count: Int32,
- video_count: Int32,
- description_html: String,
- auto_generated: Bool,
- })
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist
def channel_search(query, page, channel)
- response = YT_POOL.client &.get("/channel/#{channel}?disable_polymer=1&hl=en&gl=US")
- document = XML.parse_html(response.body)
- canonical = document.xpath_node(%q(//link[@rel="canonical"]))
+ response = YT_POOL.client &.get("/channel/#{channel}?hl=en&gl=US")
+ response = YT_POOL.client &.get("/user/#{channel}?hl=en&gl=US") if response.headers["location"]?
+ response = YT_POOL.client &.get("/c/#{channel}?hl=en&gl=US") if response.headers["location"]?
- if !canonical
- response = YT_POOL.client &.get("/c/#{channel}?disable_polymer=1&hl=en&gl=US")
- document = XML.parse_html(response.body)
- canonical = document.xpath_node(%q(//link[@rel="canonical"]))
- end
+ ucid = response.body.match(/\\"channelId\\":\\"(?[^\\]+)\\"/).try &.["ucid"]?
- if !canonical
- response = YT_POOL.client &.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US")
- document = XML.parse_html(response.body)
- canonical = document.xpath_node(%q(//link[@rel="canonical"]))
- end
-
- if !canonical
- return 0, [] of SearchItem
- end
-
- ucid = canonical["href"].split("/")[-1]
+ return 0, [] of SearchItem if !ucid
url = produce_channel_search_url(ucid, query, page)
response = YT_POOL.client &.get(url)
- json = JSON.parse(response.body)
+ initial_data = JSON.parse(response.body).as_a.find &.["response"]?
+ return 0, [] of SearchItem if !initial_data
+ author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
+ items = extract_items(initial_data.as_h, author, ucid)
- if json["content_html"]? && !json["content_html"].as_s.empty?
- document = XML.parse_html(json["content_html"].as_s)
- nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-
- count = nodeset.size
- items = extract_items(nodeset)
- else
- count = 0
- items = [] of SearchItem
- end
-
- return count, items
+ return items.size, items
end
def search(query, page = 1, search_params = produce_search_params(content_type: "all"), region = nil)
- if query.empty?
- return {0, [] of SearchItem}
- end
+ return 0, [] of SearchItem if query.empty?
- html = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body)
- if html.empty?
- return {0, [] of SearchItem}
- end
+ body = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en").body)
+ return 0, [] of SearchItem if body.empty?
- html = XML.parse_html(html)
- nodeset = html.xpath_nodes(%q(//ol[@class="item-section"]/li))
- items = extract_items(nodeset)
+ initial_data = extract_initial_data(body)
+ items = extract_items(initial_data)
- return {nodeset.size, items}
+ # initial_data["estimatedResults"]?.try &.as_s.to_i64
+
+ return items.size, items
end
def produce_search_params(sort : String = "relevance", date : String = "", content_type : String = "",
@@ -310,6 +294,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
object["2:embedded"].as(Hash)["1:varint"] = 4_i64
when "year"
object["2:embedded"].as(Hash)["1:varint"] = 5_i64
+ else nil # Ignore
end
case content_type
@@ -334,6 +319,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
object["2:embedded"].as(Hash)["3:varint"] = 1_i64
when "long"
object["2:embedded"].as(Hash)["3:varint"] = 2_i64
+ else nil # Ignore
end
features.each do |feature|
@@ -358,6 +344,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
object["2:embedded"].as(Hash)["23:varint"] = 1_i64
when "hdr"
object["2:embedded"].as(Hash)["25:varint"] = 1_i64
+ else nil # Ignore
end
end
@@ -379,12 +366,9 @@ def produce_channel_search_url(ucid, query, page)
"2:string" => ucid,
"3:base64" => {
"2:string" => "search",
- "6:varint" => 2_i64,
"7:varint" => 1_i64,
- "12:varint" => 1_i64,
- "13:string" => "",
- "23:varint" => 0_i64,
"15:string" => "#{page}",
+ "23:varint" => 0_i64,
},
"11:string" => query,
},
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index 3a9c6935..8d078387 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -1,7 +1,4 @@
def fetch_trending(trending_type, region, locale)
- headers = HTTP::Headers.new
- headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
-
region ||= "US"
region = region.upcase
@@ -11,7 +8,7 @@ def fetch_trending(trending_type, region, locale)
if trending_type && trending_type != "Default"
trending_type = trending_type.downcase.capitalize
- response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en", headers).body
+ response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body
initial_data = extract_initial_data(response)
@@ -21,51 +18,28 @@ def fetch_trending(trending_type, region, locale)
if url
url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"]
url = url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
- url += "&disable_polymer=1&gl=#{region}&hl=en"
+ url = "#{url}&gl=#{region}&hl=en"
trending = YT_POOL.client &.get(url).body
plid = extract_plid(url)
else
- trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
+ trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body
end
else
- trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
+ trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body
end
- trending = XML.parse_html(trending)
- nodeset = trending.xpath_nodes(%q(//ul/li[@class="expanded-shelf-content-item-wrapper"]))
- trending = extract_videos(nodeset)
+ initial_data = extract_initial_data(trending)
+ trending = extract_videos(initial_data)
return {trending, plid}
end
def extract_plid(url)
- wrapper = HTTP::Params.parse(URI.parse(url).query.not_nil!)["bp"]
-
- wrapper = URI.decode_www_form(wrapper)
- wrapper = Base64.decode(wrapper)
-
- # 0xe2 0x02 0x2e
- wrapper += 3
-
- # 0x0a
- wrapper += 1
-
- # Looks like "/m/[a-z0-9]{5}", not sure what it does here
-
- item_size = wrapper[0]
- wrapper += 1
- item = wrapper[0, item_size]
- wrapper += item.size
-
- # 0x12
- wrapper += 1
-
- plid_size = wrapper[0]
- wrapper += 1
- plid = wrapper[0, plid_size]
- wrapper += plid.size
-
- plid = String.new(plid)
-
- return plid
+ return url.try { |i| URI.parse(i).query }
+ .try { |i| HTTP::Params.parse(i)["bp"] }
+ .try { |i| URI.decode_www_form(i) }
+ .try { |i| Base64.decode(i) }
+ .try { |i| IO::Memory.new(i) }
+ .try { |i| Protodec::Any.parse(i) }
+ .try &.["44:0:embedded"]?.try &.["2:1:string"]?.try &.as_s
end
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index afb100f2..46bf8865 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -4,6 +4,20 @@ require "crypto/bcrypt/password"
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
struct User
+ include DB::Serializable
+
+ property updated : Time
+ property notifications : Array(String)
+ property subscriptions : Array(String)
+ property email : String
+
+ @[DB::Field(converter: User::PreferencesConverter)]
+ property preferences : Preferences
+ property password : String?
+ property token : String
+ property watched : Array(String)
+ property feed_needs_update : Bool?
+
module PreferencesConverter
def self.from_rs(rs)
begin
@@ -13,31 +27,78 @@ struct User
end
end
end
-
- db_mapping({
- updated: Time,
- notifications: Array(String),
- subscriptions: Array(String),
- email: String,
- preferences: {
- type: Preferences,
- converter: PreferencesConverter,
- },
- password: String?,
- token: String,
- watched: Array(String),
- feed_needs_update: Bool?,
- })
end
struct Preferences
- module ProcessString
+ include JSON::Serializable
+ include YAML::Serializable
+
+ property annotations : Bool = CONFIG.default_user_preferences.annotations
+ property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
+ property autoplay : Bool = CONFIG.default_user_preferences.autoplay
+
+ @[JSON::Field(converter: Preferences::StringToArray)]
+ @[YAML::Field(converter: Preferences::StringToArray)]
+ property captions : Array(String) = CONFIG.default_user_preferences.captions
+
+ @[JSON::Field(converter: Preferences::StringToArray)]
+ @[YAML::Field(converter: Preferences::StringToArray)]
+ property comments : Array(String) = CONFIG.default_user_preferences.comments
+ property continue : Bool = CONFIG.default_user_preferences.continue
+ property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay
+
+ @[JSON::Field(converter: Preferences::BoolToString)]
+ @[YAML::Field(converter: Preferences::BoolToString)]
+ property dark_mode : String = CONFIG.default_user_preferences.dark_mode
+ property latest_only : Bool = CONFIG.default_user_preferences.latest_only
+ property listen : Bool = CONFIG.default_user_preferences.listen
+ property local : Bool = CONFIG.default_user_preferences.local
+
+ @[JSON::Field(converter: Preferences::ProcessString)]
+ property locale : String = CONFIG.default_user_preferences.locale
+
+ @[JSON::Field(converter: Preferences::ClampInt)]
+ property max_results : Int32 = CONFIG.default_user_preferences.max_results
+ property notifications_only : Bool = CONFIG.default_user_preferences.notifications_only
+
+ @[JSON::Field(converter: Preferences::ProcessString)]
+ property player_style : String = CONFIG.default_user_preferences.player_style
+
+ @[JSON::Field(converter: Preferences::ProcessString)]
+ property quality : String = CONFIG.default_user_preferences.quality
+ property default_home : String = CONFIG.default_user_preferences.default_home
+ property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu
+ property related_videos : Bool = CONFIG.default_user_preferences.related_videos
+
+ @[JSON::Field(converter: Preferences::ProcessString)]
+ property sort : String = CONFIG.default_user_preferences.sort
+ property speed : Float32 = CONFIG.default_user_preferences.speed
+ property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode
+ property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only
+ property video_loop : Bool = CONFIG.default_user_preferences.video_loop
+ property volume : Int32 = CONFIG.default_user_preferences.volume
+
+ module BoolToString
def self.to_json(value : String, json : JSON::Builder)
json.string value
end
def self.from_json(value : JSON::PullParser) : String
- HTML.escape(value.read_string[0, 100])
+ begin
+ result = value.read_string
+
+ if result.empty?
+ CONFIG.default_user_preferences.dark_mode
+ else
+ result
+ end
+ rescue ex
+ if value.read_bool
+ "dark"
+ else
+ "light"
+ end
+ end
end
def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
@@ -45,7 +106,20 @@ struct Preferences
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
- HTML.escape(node.value[0, 100])
+ unless node.is_a?(YAML::Nodes::Scalar)
+ node.raise "Expected scalar, not #{node.class}"
+ end
+
+ case node.value
+ when "true"
+ "dark"
+ when "false"
+ "light"
+ when ""
+ CONFIG.default_user_preferences.dark_mode
+ else
+ node.value
+ end
end
end
@@ -67,33 +141,130 @@ struct Preferences
end
end
- json_mapping({
- annotations: {type: Bool, default: CONFIG.default_user_preferences.annotations},
- annotations_subscribed: {type: Bool, default: CONFIG.default_user_preferences.annotations_subscribed},
- autoplay: {type: Bool, default: CONFIG.default_user_preferences.autoplay},
- captions: {type: Array(String), default: CONFIG.default_user_preferences.captions, converter: ConfigPreferences::StringToArray},
- comments: {type: Array(String), default: CONFIG.default_user_preferences.comments, converter: ConfigPreferences::StringToArray},
- continue: {type: Bool, default: CONFIG.default_user_preferences.continue},
- continue_autoplay: {type: Bool, default: CONFIG.default_user_preferences.continue_autoplay},
- dark_mode: {type: String, default: CONFIG.default_user_preferences.dark_mode, converter: ConfigPreferences::BoolToString},
- latest_only: {type: Bool, default: CONFIG.default_user_preferences.latest_only},
- listen: {type: Bool, default: CONFIG.default_user_preferences.listen},
- local: {type: Bool, default: CONFIG.default_user_preferences.local},
- locale: {type: String, default: CONFIG.default_user_preferences.locale, converter: ProcessString},
- max_results: {type: Int32, default: CONFIG.default_user_preferences.max_results, converter: ClampInt},
- notifications_only: {type: Bool, default: CONFIG.default_user_preferences.notifications_only},
- player_style: {type: String, default: CONFIG.default_user_preferences.player_style, converter: ProcessString},
- quality: {type: String, default: CONFIG.default_user_preferences.quality, converter: ProcessString},
- default_home: {type: String, default: CONFIG.default_user_preferences.default_home},
- feed_menu: {type: Array(String), default: CONFIG.default_user_preferences.feed_menu},
- related_videos: {type: Bool, default: CONFIG.default_user_preferences.related_videos},
- sort: {type: String, default: CONFIG.default_user_preferences.sort, converter: ProcessString},
- speed: {type: Float32, default: CONFIG.default_user_preferences.speed},
- thin_mode: {type: Bool, default: CONFIG.default_user_preferences.thin_mode},
- unseen_only: {type: Bool, default: CONFIG.default_user_preferences.unseen_only},
- video_loop: {type: Bool, default: CONFIG.default_user_preferences.video_loop},
- volume: {type: Int32, default: CONFIG.default_user_preferences.volume},
- })
+ module FamilyConverter
+ def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
+ case value
+ when Socket::Family::UNSPEC
+ yaml.scalar nil
+ when Socket::Family::INET
+ yaml.scalar "ipv4"
+ when Socket::Family::INET6
+ yaml.scalar "ipv6"
+ when Socket::Family::UNIX
+ raise "Invalid socket family #{value}"
+ end
+ end
+
+ def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
+ if node.is_a?(YAML::Nodes::Scalar)
+ case node.value.downcase
+ when "ipv4"
+ Socket::Family::INET
+ when "ipv6"
+ Socket::Family::INET6
+ else
+ Socket::Family::UNSPEC
+ end
+ else
+ node.raise "Expected scalar, not #{node.class}"
+ end
+ end
+ end
+
+ module ProcessString
+ def self.to_json(value : String, json : JSON::Builder)
+ json.string value
+ end
+
+ def self.from_json(value : JSON::PullParser) : String
+ HTML.escape(value.read_string[0, 100])
+ end
+
+ def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
+ yaml.scalar value
+ end
+
+ def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
+ HTML.escape(node.value[0, 100])
+ end
+ end
+
+ module StringToArray
+ def self.to_json(value : Array(String), json : JSON::Builder)
+ json.array do
+ value.each do |element|
+ json.string element
+ end
+ end
+ end
+
+ def self.from_json(value : JSON::PullParser) : Array(String)
+ begin
+ result = [] of String
+ value.read_array do
+ result << HTML.escape(value.read_string[0, 100])
+ end
+ rescue ex
+ result = [HTML.escape(value.read_string[0, 100]), ""]
+ end
+
+ result
+ end
+
+ def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
+ yaml.sequence do
+ value.each do |element|
+ yaml.scalar element
+ end
+ end
+ end
+
+ def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
+ begin
+ unless node.is_a?(YAML::Nodes::Sequence)
+ node.raise "Expected sequence, not #{node.class}"
+ end
+
+ result = [] of String
+ node.nodes.each do |item|
+ unless item.is_a?(YAML::Nodes::Scalar)
+ node.raise "Expected scalar, not #{item.class}"
+ end
+
+ result << HTML.escape(item.value[0, 100])
+ end
+ rescue ex
+ if node.is_a?(YAML::Nodes::Scalar)
+ result = [HTML.escape(node.value[0, 100]), ""]
+ else
+ result = ["", ""]
+ end
+ end
+
+ result
+ end
+ end
+
+ module StringToCookies
+ def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
+ (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
+ end
+
+ def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
+ unless node.is_a?(YAML::Nodes::Scalar)
+ node.raise "Expected scalar, not #{node.class}"
+ end
+
+ cookies = HTTP::Cookies.new
+ node.value.split(";").each do |cookie|
+ next if cookie.strip.empty?
+ name, value = cookie.split("=", 2)
+ cookies << HTTP::Cookie.new(name.strip, value.strip)
+ end
+
+ cookies
+ end
+ end
end
def get_user(sid, headers, db, refresh = true)
@@ -103,8 +274,7 @@ def get_user(sid, headers, db, refresh = true)
if refresh && Time.utc - user.updated > 1.minute
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
-
- user_array[4] = user_array[4].to_json
+ user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user_array)
db.exec("INSERT INTO users VALUES (#{args}) \
@@ -122,8 +292,7 @@ def get_user(sid, headers, db, refresh = true)
else
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
-
- user_array[4] = user_array[4].to_json
+ user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user.to_a)
db.exec("INSERT INTO users VALUES (#{args}) \
@@ -166,7 +335,17 @@ def fetch_user(sid, headers, db)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
- user = User.new(Time.utc, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true)
+ user = User.new({
+ updated: Time.utc,
+ notifications: [] of String,
+ subscriptions: channels,
+ email: email,
+ preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
+ password: nil,
+ token: token,
+ watched: [] of String,
+ feed_needs_update: true,
+ })
return user, sid
end
@@ -174,7 +353,17 @@ def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
- user = User.new(Time.utc, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true)
+ user = User.new({
+ updated: Time.utc,
+ notifications: [] of String,
+ subscriptions: [] of String,
+ email: email,
+ preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
+ password: password.to_s,
+ token: token,
+ watched: [] of String,
+ feed_needs_update: true,
+ })
return user, sid
end
@@ -267,7 +456,7 @@ def subscribe_ajax(channel_id, action, env_headers)
end
headers = cookies.add_request_headers(headers)
- if match = html.body.match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/)
+ if match = html.body.match(/'XSRF_TOKEN': "(?[^"]+)"/)
session_token = match["session_token"]
headers["content-type"] = "application/x-www-form-urlencoded"
@@ -281,48 +470,6 @@ def subscribe_ajax(channel_id, action, env_headers)
end
end
-# TODO: Playlist stub, sync with YouTube for Google accounts
-# def playlist_ajax(video_ids, source_playlist_id, name, privacy, action, env_headers)
-# headers = HTTP::Headers.new
-# headers["Cookie"] = env_headers["Cookie"]
-#
-# html = YT_POOL.client &.get("/view_all_playlists?disable_polymer=1", headers)
-#
-# cookies = HTTP::Cookies.from_headers(headers)
-# html.cookies.each do |cookie|
-# if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
-# if cookies[cookie.name]?
-# cookies[cookie.name] = cookie
-# else
-# cookies << cookie
-# end
-# end
-# end
-# headers = cookies.add_request_headers(headers)
-#
-# if match = html.body.match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/)
-# session_token = match["session_token"]
-#
-# headers["content-type"] = "application/x-www-form-urlencoded"
-#
-# post_req = {
-# video_ids: [] of String,
-# source_playlist_id: "",
-# n: name,
-# p: privacy,
-# session_token: session_token,
-# }
-# post_url = "/playlist_ajax?#{action}=1"
-#
-# response = client.post(post_url, headers, form: post_req)
-# if response.status_code == 200
-# return JSON.parse(response.body)["result"]["playlistId"].as_s
-# else
-# return nil
-# end
-# end
-# end
-
def get_subscription_feed(db, user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
@@ -350,6 +497,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
notifications.sort_by! { |video| video.author }
when "channel name - reverse"
notifications.sort_by! { |video| video.author }.reverse!
+ else nil # Ignore
end
else
if user.preferences.latest_only
@@ -398,6 +546,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
videos.sort_by! { |video| video.author }
when "channel name - reverse"
videos.sort_by! { |video| video.author }.reverse!
+ else nil # Ignore
end
notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, as: Array(String))
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 1c7599f8..8e314fe0 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -222,53 +222,73 @@ VIDEO_FORMATS = {
}
struct VideoPreferences
- json_mapping({
- annotations: Bool,
- autoplay: Bool,
- comments: Array(String),
- continue: Bool,
- continue_autoplay: Bool,
- controls: Bool,
- listen: Bool,
- local: Bool,
- preferred_captions: Array(String),
- player_style: String,
- quality: String,
- raw: Bool,
- region: String?,
- related_videos: Bool,
- speed: (Float32 | Float64),
- video_end: (Float64 | Int32),
- video_loop: Bool,
- video_start: (Float64 | Int32),
- volume: Int32,
- })
+ include JSON::Serializable
+
+ property annotations : Bool
+ property autoplay : Bool
+ property comments : Array(String)
+ property continue : Bool
+ property continue_autoplay : Bool
+ property controls : Bool
+ property listen : Bool
+ property local : Bool
+ property preferred_captions : Array(String)
+ property player_style : String
+ property quality : String
+ property raw : Bool
+ property region : String?
+ property related_videos : Bool
+ property speed : Float32 | Float64
+ property video_end : Float64 | Int32
+ property video_loop : Bool
+ property video_start : Float64 | Int32
+ property volume : Int32
end
struct Video
- property player_json : JSON::Any?
- property recommended_json : JSON::Any?
+ include DB::Serializable
- module HTTPParamConverter
+ property id : String
+
+ @[DB::Field(converter: Video::JSONConverter)]
+ property info : Hash(String, JSON::Any)
+ property updated : Time
+
+ @[DB::Field(ignore: true)]
+ property captions : Array(Caption)?
+
+ @[DB::Field(ignore: true)]
+ property adaptive_fmts : Array(Hash(String, JSON::Any))?
+
+ @[DB::Field(ignore: true)]
+ property fmt_stream : Array(Hash(String, JSON::Any))?
+
+ @[DB::Field(ignore: true)]
+ property description : String?
+
+ module JSONConverter
def self.from_rs(rs)
- HTTP::Params.parse(rs.read(String))
+ JSON.parse(rs.read(String)).as_h
end
end
- def to_json(locale, config, kemal_config, decrypt_function, json : JSON::Builder)
+ def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
+
+ json.field "error", info["reason"] if info["reason"]?
+
json.field "videoThumbnails" do
- generate_thumbnails(json, self.id, config, kemal_config)
+ generate_thumbnails(json, self.id)
end
json.field "storyboards" do
- generate_storyboards(json, self.id, self.storyboards, config, kemal_config)
+ generate_storyboards(json, self.id, self.storyboards)
end
- json.field "description", html_to_content(self.description_html)
+ json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
@@ -307,43 +327,39 @@ struct Video
json.field "lengthSeconds", self.length_seconds
json.field "allowRatings", self.allow_ratings
- json.field "rating", self.info["avg_rating"].to_f32
+ json.field "rating", self.average_rating
json.field "isListed", self.is_listed
json.field "liveNow", self.live_now
json.field "isUpcoming", self.is_upcoming
if self.premiere_timestamp
- json.field "premiereTimestamp", self.premiere_timestamp.not_nil!.to_unix
+ json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
- if player_response["streamingData"]?.try &.["hlsManifestUrl"]?
- host_url = make_host_url(config, kemal_config)
-
- hlsvp = player_response["streamingData"]["hlsManifestUrl"].as_s
- hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", host_url)
-
+ if hlsvp = self.hls_manifest_url
+ hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
json.field "hlsUrl", hlsvp
end
- json.field "dashUrl", "#{make_host_url(config, kemal_config)}/api/manifest/dash/id/#{id}"
+ json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}"
json.field "adaptiveFormats" do
json.array do
- self.adaptive_fmts(decrypt_function).each do |fmt|
+ self.adaptive_fmts.each do |fmt|
json.object do
- json.field "index", fmt["index"]
- json.field "bitrate", fmt["bitrate"]
- json.field "init", fmt["init"]
+ json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}"
+ json.field "bitrate", fmt["bitrate"].as_i.to_s
+ json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}"
json.field "url", fmt["url"]
- json.field "itag", fmt["itag"]
- json.field "type", fmt["type"]
- json.field "clen", fmt["clen"]
- json.field "lmt", fmt["lmt"]
- json.field "projectionType", fmt["projection_type"]
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
+ json.field "clen", fmt["contentLength"]
+ json.field "lmt", fmt["lastModified"]
+ json.field "projectionType", fmt["projectionType"]
fmt_info = itag_to_metadata?(fmt["itag"])
if fmt_info
- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.to_i || 30
+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
@@ -369,16 +385,16 @@ struct Video
json.field "formatStreams" do
json.array do
- self.fmt_stream(decrypt_function).each do |fmt|
+ self.fmt_stream.each do |fmt|
json.object do
json.field "url", fmt["url"]
- json.field "itag", fmt["itag"]
- json.field "type", fmt["type"]
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
json.field "quality", fmt["quality"]
fmt_info = itag_to_metadata?(fmt["itag"])
if fmt_info
- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.to_i || 30
+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
@@ -416,15 +432,13 @@ struct Video
json.field "recommendedVideos" do
json.array do
- self.info["rvs"]?.try &.split(",").each do |rv|
- rv = HTTP::Params.parse(rv)
-
+ self.related_videos.each do |rv|
if rv["id"]?
json.object do
json.field "videoId", rv["id"]
json.field "title", rv["title"]
json.field "videoThumbnails" do
- generate_thumbnails(json, rv["id"], config, kemal_config)
+ generate_thumbnails(json, rv["id"])
end
json.field "author", rv["author"]
@@ -437,7 +451,7 @@ struct Video
qualities.each do |quality|
json.object do
- json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
+ json.field "url", rv["author_thumbnail"]?.try &.gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
@@ -446,9 +460,9 @@ struct Video
end
end
- json.field "lengthSeconds", rv["length_seconds"].to_i
- json.field "viewCountText", rv["short_view_count_text"]
- json.field "viewCount", rv["view_count"]?.try &.to_i64
+ json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
+ json.field "viewCountText", rv["short_view_count_text"]?
+ json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
end
end
end
@@ -457,266 +471,164 @@ struct Video
end
end
- def to_json(locale, config, kemal_config, decrypt_function, json : JSON::Builder | Nil = nil)
+ def to_json(locale, json : JSON::Builder | Nil = nil)
if json
- to_json(locale, config, kemal_config, decrypt_function, json)
+ to_json(locale, json)
else
JSON.build do |json|
- to_json(locale, config, kemal_config, decrypt_function, json)
+ to_json(locale, json)
end
end
end
- # `description_html` is stored in DB as `description`, which can be
- # quite confusing. Since it currently isn't very practical to rename
- # it, we instead define a getter and setter here.
- def description_html
- self.description
+ def title
+ info["videoDetails"]["title"]?.try &.as_s || ""
end
- def description_html=(other : String)
- self.description = other
+ def ucid
+ info["videoDetails"]["channelId"]?.try &.as_s || ""
+ end
+
+ def author
+ info["videoDetails"]["author"]?.try &.as_s || ""
+ end
+
+ def length_seconds : Int32
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i ||
+ info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
+ end
+
+ def views : Int64
+ info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
+ end
+
+ def likes : Int64
+ info["likes"]?.try &.as_i64 || 0_i64
+ end
+
+ def dislikes : Int64
+ info["dislikes"]?.try &.as_i64 || 0_i64
+ end
+
+ def average_rating : Float64
+ # (likes / (likes + dislikes) * 4 + 1)
+ info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0
+ end
+
+ def published : Time
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location.local) } || Time.local
+ end
+
+ def published=(other : Time)
+ info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
+ end
+
+ def cookie
+ info["cookie"]?.try &.as_h.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
end
def allow_ratings
- allow_ratings = player_response["videoDetails"]?.try &.["allowRatings"]?.try &.as_bool
-
- if allow_ratings.nil?
- return true
- end
-
- return allow_ratings
+ r = info["videoDetails"]["allowRatings"]?.try &.as_bool
+ r.nil? ? false : r
end
def live_now
- live_now = player_response["videoDetails"]?.try &.["isLive"]?.try &.as_bool
-
- if live_now.nil?
- return false
- end
-
- return live_now
+ info["videoDetails"]["isLiveContent"]?.try &.as_bool || false
end
def is_listed
- is_listed = player_response["videoDetails"]?.try &.["isCrawlable"]?.try &.as_bool
-
- if is_listed.nil?
- return true
- end
-
- return is_listed
+ info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
end
def is_upcoming
- is_upcoming = player_response["videoDetails"]?.try &.["isUpcoming"]?.try &.as_bool
-
- if is_upcoming.nil?
- return false
- end
-
- return is_upcoming
+ info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
end
- def premiere_timestamp
- if self.is_upcoming
- premiere_timestamp = player_response["playabilityStatus"]?
- .try &.["liveStreamability"]?
- .try &.["liveStreamabilityRenderer"]?
- .try &.["offlineSlate"]?
- .try &.["liveStreamOfflineSlateRenderer"]?
- .try &.["scheduledStartTime"]?.try &.as_s.to_i64
- end
-
- if premiere_timestamp
- premiere_timestamp = Time.unix(premiere_timestamp)
- end
-
- return premiere_timestamp
+ def premiere_timestamp : Time?
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?
+ .try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) }
end
def keywords
- keywords = player_response["videoDetails"]?.try &.["keywords"]?.try &.as_a
- keywords ||= [] of String
-
- return keywords
+ info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
end
- def fmt_stream(decrypt_function)
- streams = [] of HTTP::Params
-
- if fmt_streams = player_response["streamingData"]?.try &.["formats"]?
- fmt_streams.as_a.each do |fmt_stream|
- if !fmt_stream.as_h?
- next
- end
-
- fmt = {} of String => String
-
- fmt["lmt"] = fmt_stream["lastModified"]?.try &.as_s || "0"
- fmt["projection_type"] = "1"
- fmt["type"] = fmt_stream["mimeType"].as_s
- fmt["clen"] = fmt_stream["contentLength"]?.try &.as_s || "0"
- fmt["bitrate"] = fmt_stream["bitrate"]?.try &.as_i.to_s || "0"
- fmt["itag"] = fmt_stream["itag"].as_i.to_s
- if fmt_stream["url"]?
- fmt["url"] = fmt_stream["url"].as_s
- end
- if fmt_stream["cipher"]?
- HTTP::Params.parse(fmt_stream["cipher"].as_s).each do |key, value|
- fmt[key] = value
- end
- end
- fmt["quality"] = fmt_stream["quality"].as_s
-
- if fmt_stream["width"]?
- fmt["size"] = "#{fmt_stream["width"]}x#{fmt_stream["height"]}"
- fmt["height"] = fmt_stream["height"].as_i.to_s
- end
-
- if fmt_stream["fps"]?
- fmt["fps"] = fmt_stream["fps"].as_i.to_s
- end
-
- if fmt_stream["qualityLabel"]?
- fmt["quality_label"] = fmt_stream["qualityLabel"].as_s
- end
-
- params = HTTP::Params.new
- fmt.each do |key, value|
- params[key] = value
- end
-
- streams << params
- end
-
- streams.sort_by! { |stream| stream["height"].to_i }.reverse!
- elsif fmt_stream = self.info["url_encoded_fmt_stream_map"]?
- fmt_stream.split(",").each do |string|
- if !string.empty?
- streams << HTTP::Params.parse(string)
- end
- end
- end
-
- streams.each { |s| s.add("label", "#{s["quality"]} - #{s["type"].split(";")[0].split("/")[1]}") }
- streams = streams.uniq { |s| s["label"] }
-
- if self.info["region"]?
- streams.each do |fmt|
- fmt["url"] += "®ion=" + self.info["region"]
- end
- end
-
- streams.each do |fmt|
- fmt["url"] += "&host=" + (URI.parse(fmt["url"]).host || "")
- fmt["url"] += decrypt_signature(fmt, decrypt_function)
- end
-
- return streams
+ def related_videos
+ info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
end
- def adaptive_fmts(decrypt_function)
- adaptive_fmts = [] of HTTP::Params
-
- if fmts = player_response["streamingData"]?.try &.["adaptiveFormats"]?
- fmts.as_a.each do |adaptive_fmt|
- next if !adaptive_fmt.as_h?
- fmt = {} of String => String
-
- if init = adaptive_fmt["initRange"]?
- fmt["init"] = "#{init["start"]}-#{init["end"]}"
- end
- fmt["init"] ||= "0-0"
-
- fmt["lmt"] = adaptive_fmt["lastModified"]?.try &.as_s || "0"
- fmt["projection_type"] = "1"
- fmt["type"] = adaptive_fmt["mimeType"].as_s
- fmt["clen"] = adaptive_fmt["contentLength"]?.try &.as_s || "0"
- fmt["bitrate"] = adaptive_fmt["bitrate"]?.try &.as_i.to_s || "0"
- fmt["itag"] = adaptive_fmt["itag"].as_i.to_s
- if adaptive_fmt["url"]?
- fmt["url"] = adaptive_fmt["url"].as_s
- end
- if adaptive_fmt["cipher"]?
- HTTP::Params.parse(adaptive_fmt["cipher"].as_s).each do |key, value|
- fmt[key] = value
- end
- end
- if index = adaptive_fmt["indexRange"]?
- fmt["index"] = "#{index["start"]}-#{index["end"]}"
- end
- fmt["index"] ||= "0-0"
-
- if adaptive_fmt["width"]?
- fmt["size"] = "#{adaptive_fmt["width"]}x#{adaptive_fmt["height"]}"
- end
-
- if adaptive_fmt["fps"]?
- fmt["fps"] = adaptive_fmt["fps"].as_i.to_s
- end
-
- if adaptive_fmt["qualityLabel"]?
- fmt["quality_label"] = adaptive_fmt["qualityLabel"].as_s
- end
-
- params = HTTP::Params.new
- fmt.each do |key, value|
- params[key] = value
- end
-
- adaptive_fmts << params
- end
- elsif fmts = self.info["adaptive_fmts"]?
- fmts.split(",") do |string|
- adaptive_fmts << HTTP::Params.parse(string)
- end
- end
-
- if self.info["region"]?
- adaptive_fmts.each do |fmt|
- fmt["url"] += "®ion=" + self.info["region"]
- end
- end
-
- adaptive_fmts.each do |fmt|
- fmt["url"] += "&host=" + (URI.parse(fmt["url"]).host || "")
- fmt["url"] += decrypt_signature(fmt, decrypt_function)
- end
-
- return adaptive_fmts
+ def allowed_regions
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?
+ .try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String
end
- def video_streams(adaptive_fmts)
- video_streams = adaptive_fmts.select { |s| s["type"].starts_with? "video" }
-
- return video_streams
+ def author_thumbnail : String
+ info["authorThumbnail"]?.try &.as_s || ""
end
- def audio_streams(adaptive_fmts)
- audio_streams = adaptive_fmts.select { |s| s["type"].starts_with? "audio" }
- audio_streams.sort_by! { |s| s["bitrate"].to_i }.reverse!
- audio_streams.each do |stream|
- stream["bitrate"] = (stream["bitrate"].to_f64/1000).to_i.to_s
+ def sub_count_text : String
+ info["subCountText"]?.try &.as_s || "-"
+ end
+
+ def fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
+
+ fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
end
-
- return audio_streams
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @fmt_stream = fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any)))
end
- def player_response
- @player_json = JSON.parse(@info["player_response"]) if !@player_json
- @player_json.not_nil!
+ def adaptive_fmts
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
+ fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
+ end
+ # See https://github.com/TeamNewPipe/NewPipe/issues/2415
+ # Some streams are segmented by URL `sq/` rather than index, for now we just filter them out
+ fmt_stream.reject! { |f| !f["indexRange"]? }
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @adaptive_fmts = fmt_stream
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
+ end
+
+ def video_streams
+ adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video")
+ end
+
+ def audio_streams
+ adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
end
def storyboards
- storyboards = player_response["storyboards"]?
+ storyboards = info["storyboards"]?
.try &.as_h
.try &.["playerStoryboardSpecRenderer"]?
.try &.["spec"]?
.try &.as_s.split("|")
if !storyboards
- if storyboard = player_response["storyboards"]?
+ if storyboard = info["storyboards"]?
.try &.as_h
.try &.["playerLiveStoryboardSpecRenderer"]?
.try &.["spec"]?
@@ -744,9 +656,7 @@ struct Video
storyboard_height: Int32,
storyboard_count: Int32)
- if !storyboards
- return items
- end
+ return items if !storyboards
url = URI.parse(storyboards.shift)
params = HTTP::Params.parse(url.query || "")
@@ -780,96 +690,102 @@ struct Video
end
def paid
- reason = player_response["playabilityStatus"]?.try &.["reason"]?
+ reason = info["playabilityStatus"]?.try &.["reason"]?
paid = reason == "This video requires payment to watch." ? true : false
-
- return paid
+ paid
end
def premium
- if info["premium"]?
- self.info["premium"] == "true"
- else
- false
- end
+ keywords.includes? "YouTube Red"
end
- def captions
- captions = [] of Caption
- if player_response["captions"]?
- caption_list = player_response["captions"]["playerCaptionsTracklistRenderer"]["captionTracks"]?.try &.as_a
- caption_list ||= [] of JSON::Any
-
- caption_list.each do |caption|
- caption = Caption.from_json(caption.to_json)
- caption.name.simpleText = caption.name.simpleText.split(" - ")[0]
- captions << caption
- end
+ def captions : Array(Caption)
+ return @captions.as(Array(Caption)) if @captions
+ captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption|
+ caption = Caption.from_json(caption.to_json)
+ caption.name.simpleText = caption.name.simpleText.split(" - ")[0]
+ caption
end
+ captions ||= [] of Caption
+ @captions = captions
+ return @captions.as(Array(Caption))
+ end
- return captions
+ def description
+ description = info["microformat"]?.try &.["playerMicroformatRenderer"]?
+ .try &.["description"]?.try &.["simpleText"]?.try &.as_s || ""
+ end
+
+ # TODO
+ def description=(value : String)
+ @description = value
+ end
+
+ def description_html
+ info["descriptionHtml"]?.try &.as_s || "
"
+ end
+
+ def description_html=(value : String)
+ info["descriptionHtml"] = JSON::Any.new(value)
end
def short_description
- short_description = self.description_html.gsub(/([1-9](\d+,?)*) views/).try &.["views"].gsub(/\D/, "") || "")
+ rv["id"] = r["videoId"]
+ JSON::Any.new(rv)
end
-def extract_recommended(recommended_videos)
- rvs = [] of HTTP::Params
+def extract_polymer_config(body)
+ params = {} of String => JSON::Any
+ player_response = body.match(/window\["ytInitialPlayerResponse"\]\s*=\s*(?.*?);\n/)
+ .try { |r| JSON.parse(r["info"]).as_h }
- recommended_videos.try &.each do |compact_renderer|
- if compact_renderer["compactRadioRenderer"]? || compact_renderer["compactPlaylistRenderer"]?
- # TODO
- elsif video_renderer = compact_renderer["compactVideoRenderer"]?
- recommended_video = HTTP::Params.new
- recommended_video["id"] = video_renderer["videoId"].as_s
- recommended_video["title"] = video_renderer["title"]["simpleText"].as_s
-
- next if !video_renderer["shortBylineText"]?
-
- recommended_video["author"] = video_renderer["shortBylineText"]["runs"].as_a[0]["text"].as_s
- recommended_video["ucid"] = video_renderer["shortBylineText"]["runs"].as_a[0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s
- recommended_video["author_thumbnail"] = video_renderer["channelThumbnail"]["thumbnails"][0]["url"].as_s
-
- if view_count = video_renderer["viewCountText"]?.try { |field| field["simpleText"]?.try &.as_s || field["runs"][0]?.try &.["text"].as_s }.try &.delete(", views watching").to_i64?.try &.to_s
- recommended_video["view_count"] = view_count
- recommended_video["short_view_count_text"] = "#{number_to_short_text(view_count.to_i64)} views"
- end
- recommended_video["length_seconds"] = decode_length_seconds(video_renderer["lengthText"]?.try &.["simpleText"]?.try &.as_s || "0:00").to_s
-
- rvs << recommended_video
- end
+ if body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
+ body.includes?("https://www.google.com/sorry/index")
+ params["reason"] = JSON::Any.new("Could not extract video info. Instance is likely blocked.")
+ elsif !player_response
+ params["reason"] = JSON::Any.new("Video unavailable.")
+ elsif player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK"
+ reason = player_response["playabilityStatus"]["errorScreen"]?.try &.["playerErrorMessageRenderer"]?.try &.["subreason"]?.try { |s| s["simpleText"]?.try &.as_s || s["runs"].as_a.map { |r| r["text"] }.join("") } ||
+ player_response["playabilityStatus"]["reason"].as_s
+ params["reason"] = JSON::Any.new(reason)
end
- rvs
-end
+ params["sessionToken"] = JSON::Any.new(body.match(/"XSRF_TOKEN":"(?[^"]+)"/).try &.["session_token"]?)
+ params["shortDescription"] = JSON::Any.new(body.match(/"og:description" content="(?[^"]+)"/).try &.["description"]?)
-def extract_polymer_config(body, html)
- params = HTTP::Params.new
+ return params if !player_response
- params["session_token"] = body.match(/"XSRF_TOKEN":"(?[A-Za-z0-9\_\-\=]+)"/).try &.["session_token"] || ""
-
- html_info = JSON.parse(body.match(/ytplayer\.config = (?.*?);ytplayer\.load/).try &.["info"] || "{}").try &.["args"]?.try &.as_h
-
- if html_info
- html_info.each do |key, value|
- params[key] = value.to_s
- end
+ {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f|
+ params[f] = player_response[f] if player_response[f]?
end
- initial_data = extract_initial_data(body)
+ yt_initial_data = body.match(/(window\["ytInitialData"\]|var\s+ytInitialData)\s*=\s*(?.*?);\s*\n/)
+ .try { |r| JSON.parse(r["info"]).as_h }
- primary_results = initial_data["contents"]?
- .try &.["twoColumnWatchNextResults"]?
- .try &.["results"]?
- .try &.["results"]?
- .try &.["contents"]?
-
- comment_continuation = primary_results.try &.as_a.select { |object| object["itemSectionRenderer"]? }[0]?
- .try &.["itemSectionRenderer"]?
- .try &.["continuations"]?
- .try &.[0]?
- .try &.["nextContinuationData"]?
-
- params["ctoken"] = comment_continuation.try &.["continuation"]?.try &.as_s || ""
- params["itct"] = comment_continuation.try &.["clickTrackingParams"]?.try &.as_s || ""
-
- rvs = initial_data["contents"]?
- .try &.["twoColumnWatchNextResults"]?
- .try &.["secondaryResults"]?
- .try &.["secondaryResults"]?
- .try &.["results"]?
- .try &.as_a
-
- params["rvs"] = extract_recommended(rvs).join(",")
-
- # TODO: Watching now
- params["views"] = primary_results.try &.as_a.select { |object| object["videoPrimaryInfoRenderer"]? }[0]?
- .try &.["videoPrimaryInfoRenderer"]?
- .try &.["viewCount"]?
- .try &.["videoViewCountRenderer"]?
- .try &.["viewCount"]?
- .try &.["simpleText"]?
- .try &.as_s.gsub(/\D/, "").to_i64.to_s || "0"
+ params["relatedVideos"] = yt_initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]?
+ .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r|
+ parse_related r
+ }.try { |a| JSON::Any.new(a) } || yt_initial_data.try &.["webWatchNextResponseExtensionData"]?.try &.["relatedVideoArgs"]?
+ .try &.as_s.split(",").map { |r|
+ r = HTTP::Params.parse(r).to_h
+ JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) }))
+ }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any)
+ primary_results = yt_initial_data.try &.["contents"]?.try &.["twoColumnWatchNextResults"]?.try &.["results"]?
+ .try &.["results"]?.try &.["contents"]?
sentiment_bar = primary_results.try &.as_a.select { |object| object["videoPrimaryInfoRenderer"]? }[0]?
.try &.["videoPrimaryInfoRenderer"]?
.try &.["sentimentBar"]?
@@ -1001,34 +860,13 @@ def extract_polymer_config(body, html)
.try &.["tooltip"]?
.try &.as_s
- likes, dislikes = sentiment_bar.try &.split(" / ").map { |a| a.delete(", ").to_i32 }[0, 2] || {0, 0}
+ likes, dislikes = sentiment_bar.try &.split(" / ", 2).map &.gsub(/\D/, "").to_i64 || {0_i64, 0_i64}
+ params["likes"] = JSON::Any.new(likes)
+ params["dislikes"] = JSON::Any.new(dislikes)
- params["likes"] = "#{likes}"
- params["dislikes"] = "#{dislikes}"
-
- published = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
- .try &.["videoSecondaryInfoRenderer"]?
- .try &.["dateText"]?
- .try &.["simpleText"]?
- .try &.as_s.split(" ")[-3..-1].join(" ")
-
- if published
- params["published"] = Time.parse(published, "%b %-d, %Y", Time::Location.local).to_unix.to_s
- else
- params["published"] = Time.utc(1990, 1, 1).to_unix.to_s
- end
-
- params["description_html"] = "
"
-
- description_html = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
- .try &.["videoSecondaryInfoRenderer"]?
- .try &.["description"]?
- .try &.["runs"]?
- .try &.as_a
-
- if description_html
- params["description_html"] = content_to_comment_html(description_html)
- end
+ params["descriptionHtml"] = JSON::Any.new(primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
+ .try &.["videoSecondaryInfoRenderer"]?.try &.["description"]?.try &.["runs"]?
+ .try &.as_a.try { |t| content_to_comment_html(t).gsub("\n", "
")
metadata = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
.try &.["videoSecondaryInfoRenderer"]?
@@ -1037,9 +875,8 @@ def extract_polymer_config(body, html)
.try &.["rows"]?
.try &.as_a
- params["genre"] = ""
- params["genre_ucid"] = ""
- params["license"] = ""
+ params["genre"] = params["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["category"]? || JSON::Any.new("")
+ params["genreUrl"] = JSON::Any.new(nil)
metadata.try &.each do |row|
title = row["metadataRowRenderer"]?.try &.["title"]?.try &.["simpleText"]?.try &.as_s
@@ -1051,219 +888,130 @@ def extract_polymer_config(body, html)
contents = contents.try &.["runs"]?
.try &.as_a[0]?
- params["genre"] = contents.try &.["text"]?
- .try &.as_s || ""
- params["genre_ucid"] = contents.try &.["navigationEndpoint"]?
- .try &.["browseEndpoint"]?
- .try &.["browseId"]?.try &.as_s || ""
+ params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "")
+ params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]?
+ .try &.["browseId"]?.try &.as_s || "")
elsif title.try &.== "License"
contents = contents.try &.["runs"]?
.try &.as_a[0]?
- params["license"] = contents.try &.["text"]?
- .try &.as_s || ""
+ params["license"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "")
elsif title.try &.== "Licensed to YouTube by"
- params["license"] = contents.try &.["simpleText"]?
- .try &.as_s || ""
+ params["license"] = JSON::Any.new(contents.try &.["simpleText"]?.try &.as_s || "")
end
end
author_info = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]?
- .try &.["videoSecondaryInfoRenderer"]?
- .try &.["owner"]?
- .try &.["videoOwnerRenderer"]?
+ .try &.["videoSecondaryInfoRenderer"]?.try &.["owner"]?.try &.["videoOwnerRenderer"]?
- params["author_thumbnail"] = author_info.try &.["thumbnail"]?
- .try &.["thumbnails"]?
- .try &.as_a[0]?
- .try &.["url"]?
- .try &.as_s || ""
+ params["authorThumbnail"] = JSON::Any.new(author_info.try &.["thumbnail"]?
+ .try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"]?
+ .try &.as_s || "")
- params["sub_count_text"] = author_info.try &.["subscriberCountText"]?
- .try &.["simpleText"]?
- .try &.as_s.gsub(/\D/, "") || "0"
+ params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]?
+ .try { |t| t["simpleText"]? || t["runs"]?.try &.[0]?.try &.["text"]? }.try &.as_s.split(" ", 2)[0] || "-")
- return params
+ initial_data = body.match(/ytplayer\.config\s*=\s*(?.*?);ytplayer\.web_player_context_config/)
+ .try { |r| JSON.parse(r["info"]) }.try &.["args"]["player_response"]?
+ .try &.as_s?.try &.try { |r| JSON.parse(r).as_h }
+
+ return params if !initial_data
+
+ {"playabilityStatus", "streamingData"}.each do |f|
+ params[f] = initial_data[f] if initial_data[f]?
+ end
+
+ params
end
-def extract_player_config(body, html)
- params = HTTP::Params.new
-
- if md = body.match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/)
- params["session_token"] = md["session_token"]
- end
-
- if md = body.match(/'RELATED_PLAYER_ARGS': (?.*?),\n/)
- recommended_json = JSON.parse(md["json"])
- rvs_params = recommended_json["rvs"].as_s.split(",").map { |params| HTTP::Params.parse(params) }
-
- if watch_next_response = recommended_json["watch_next_response"]?
- watch_next_json = JSON.parse(watch_next_response.as_s)
- rvs = watch_next_json["contents"]?
- .try &.["twoColumnWatchNextResults"]?
- .try &.["secondaryResults"]?
- .try &.["secondaryResults"]?
- .try &.["results"]?
- .try &.as_a
-
- rvs = extract_recommended(rvs).compact_map do |rv|
- if !rv["short_view_count_text"]?
- rv_params = rvs_params.select { |rv_params| rv_params["id"]? == (rv["id"]? || "") }[0]?
-
- if rv_params.try &.["short_view_count_text"]?
- rv["short_view_count_text"] = rv_params.not_nil!["short_view_count_text"]
- rv
- else
- nil
- end
- else
- rv
- end
+def get_video(id, db, refresh = true, region = nil, force_refresh = false)
+ if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region
+ # If record was last updated over 10 minutes ago, or video has since premiered,
+ # refresh (expire param in response lasts for 6 hours)
+ if (refresh &&
+ (Time.utc - video.updated > 10.minutes) ||
+ (video.premiere_timestamp.try &.< Time.utc)) ||
+ force_refresh
+ begin
+ video = fetch_video(id, region)
+ db.exec("UPDATE videos SET (id, info, updated) = ($1, $2, $3) WHERE id = $1", video.id, video.info.to_json, video.updated)
+ rescue ex
+ db.exec("DELETE FROM videos * WHERE id = $1", id)
+ raise ex
end
- params["rvs"] = (rvs.map &.to_s).join(",")
- end
- end
-
- html_info = body.match(/ytplayer\.config = (?.*?);ytplayer\.load/).try &.["info"]
-
- if html_info
- JSON.parse(html_info)["args"].as_h.each do |key, value|
- params[key] = value.to_s
end
else
- error_message = html.xpath_node(%q(//h1[@id="unavailable-message"]))
- if error_message
- params["reason"] = error_message.content.strip
- elsif body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
- body.includes?("https://www.google.com/sorry/index")
- params["reason"] = "Could not extract video info. Instance is likely blocked."
- else
- params["reason"] = "Video unavailable."
+ video = fetch_video(id, region)
+ if !region
+ db.exec("INSERT INTO videos VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", video.id, video.info.to_json, video.updated)
end
end
- return params
+ return video
end
def fetch_video(id, region)
- response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"))
+ response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999"))
if md = response.headers["location"]?.try &.match(/v=(?[a-zA-Z0-9_-]{11})/)
raise VideoRedirect.new(video_id: md["id"])
end
- html = XML.parse_html(response.body)
- info = extract_player_config(response.body, html)
- info["cookie"] = response.cookies.to_h.map { |name, cookie| "#{name}=#{cookie.value}" }.join("; ")
-
- allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).try &.["content"].split(",")
- if !allowed_regions || allowed_regions == [""]
- allowed_regions = [] of String
- end
+ info = extract_polymer_config(response.body)
+ info["cookie"] = JSON::Any.new(response.cookies.to_h.transform_values { |v| JSON::Any.new(v.value) })
+ allowed_regions = info["microformat"]?.try &.["playerMicroformatRenderer"]["availableCountries"]?.try &.as_a.map &.as_s || [] of String
# Check for region-blocks
- if info["reason"]? && info["reason"].includes?("your country")
+ if info["reason"]?.try &.as_s.includes?("your country")
bypass_regions = PROXY_LIST.keys & allowed_regions
if !bypass_regions.empty?
region = bypass_regions[rand(bypass_regions.size)]
- response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"))
+ response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999"))
- html = XML.parse_html(response.body)
- info = extract_player_config(response.body, html)
-
- info["region"] = region if region
- info["cookie"] = response.cookies.to_h.map { |name, cookie| "#{name}=#{cookie.value}" }.join("; ")
+ region_info = extract_polymer_config(response.body)
+ region_info["region"] = JSON::Any.new(region) if region
+ region_info["cookie"] = JSON::Any.new(response.cookies.to_h.transform_values { |v| JSON::Any.new(v.value) })
+ info = region_info if !region_info["reason"]?
end
end
# Try to pull streams from embed URL
if info["reason"]?
embed_page = YT_POOL.client &.get("/embed/#{id}").body
- sts = embed_page.match(/"sts"\s*:\s*(?\d+)/).try &.["sts"]?
- sts ||= ""
- embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&disable_polymer=1&sts=#{sts}").body)
+ sts = embed_page.match(/"sts"\s*:\s*(?\d+)/).try &.["sts"]? || ""
+ embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?html5=1&video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&sts=#{sts}").body)
- if !embed_info["reason"]?
- embed_info.each do |key, value|
- info[key] = value.to_s
+ if embed_info["player_response"]?
+ player_response = JSON.parse(embed_info["player_response"])
+ {"captions", "microformat", "playabilityStatus", "streamingData", "videoDetails", "storyboards"}.each do |f|
+ info[f] = player_response[f] if player_response[f]?
end
- else
- raise info["reason"]
end
+
+ initial_data = JSON.parse(embed_info["watch_next_response"]) if embed_info["watch_next_response"]?
+
+ info["relatedVideos"] = initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]?
+ .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r|
+ parse_related r
+ }.try { |a| JSON::Any.new(a) } || embed_info["rvs"]?.try &.split(",").map { |r|
+ r = HTTP::Params.parse(r).to_h
+ JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) }))
+ }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any)
end
- if info["reason"]? && !info["player_response"]?
- raise info["reason"]
- end
+ raise info["reason"]?.try &.as_s || "" if !info["videoDetails"]?
- player_json = JSON.parse(info["player_response"])
- if reason = player_json["playabilityStatus"]?.try &.["reason"]?.try &.as_s
- raise reason
- end
-
- title = player_json["videoDetails"]["title"].as_s
- author = player_json["videoDetails"]["author"]?.try &.as_s || ""
- ucid = player_json["videoDetails"]["channelId"]?.try &.as_s || ""
-
- info["premium"] = html.xpath_node(%q(.//span[text()="Premium"])) ? "true" : "false"
-
- views = html.xpath_node(%q(//meta[@itemprop="interactionCount"]))
- .try &.["content"].to_i64? || 0_i64
-
- likes = html.xpath_node(%q(//button[@title="I like this"]/span))
- .try &.content.delete(",").try &.to_i? || 0
-
- dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span))
- .try &.content.delete(",").try &.to_i? || 0
-
- avg_rating = (likes.to_f/(likes.to_f + dislikes.to_f) * 4 + 1)
- avg_rating = avg_rating.nan? ? 0.0 : avg_rating
- info["avg_rating"] = "#{avg_rating}"
-
- description_html = html.xpath_node(%q(//p[@id="eow-description"])).try &.to_xml(options: XML::SaveOptions::NO_DECL) || "
"
- wilson_score = ci_lower_bound(likes, likes + dislikes)
-
- published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).try &.["content"]
- published ||= Time.utc.to_s("%Y-%m-%d")
- published = Time.parse(published, "%Y-%m-%d", Time::Location.local)
-
- is_family_friendly = html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).try &.["content"] == "True"
- is_family_friendly ||= true
-
- genre = html.xpath_node(%q(//meta[@itemprop="genre"])).try &.["content"]
- genre ||= ""
-
- genre_url = html.xpath_node(%(//ul[contains(@class, "watch-info-tag-list")]/li/a[text()="#{genre}"])).try &.["href"]?
- genre_url ||= ""
-
- # YouTube provides invalid URLs for some genres, so we fix that here
- case genre
- when "Comedy"
- genre_url = "/channel/UCQZ43c4dAA9eXCQuXWu9aTw"
- when "Education"
- genre_url = "/channel/UCdxpofrI-dO6oYfsqHDHphw"
- when "Gaming"
- genre_url = "/channel/UCOpNcN46UbXVtpKMrmU4Abg"
- when "Movies"
- genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g"
- when "Nonprofits & Activism"
- genre_url = "/channel/UCfFyYRYslvuhwMDnx6KjUvw"
- when "Trailers"
- genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g"
- end
-
- license = html.xpath_node(%q(//h4[contains(text(),"License")]/parent::*/ul/li)).try &.content || ""
- sub_count_text = html.xpath_node(%q(//span[contains(@class, "subscriber-count")])).try &.["title"]? || "0"
- author_thumbnail = html.xpath_node(%(//span[@class="yt-thumb-clip"]/img)).try &.["data-thumb"]?.try &.gsub(/^\/\//, "https://") || ""
-
- video = Video.new(id, info, Time.utc, title, views, likes, dislikes, wilson_score, published, description_html,
- nil, author, ucid, allowed_regions, is_family_friendly, genre, genre_url, license, sub_count_text, author_thumbnail)
+ video = Video.new({
+ id: id,
+ info: info,
+ updated: Time.utc,
+ })
return video
end
-def itag_to_metadata?(itag : String)
- return VIDEO_FORMATS[itag]?
+def itag_to_metadata?(itag : JSON::Any)
+ return VIDEO_FORMATS[itag.to_s]?
end
def process_continuation(db, query, plid, id)
@@ -1365,34 +1113,34 @@ def process_video_params(query, preferences)
controls ||= 1
controls = controls >= 1
- params = VideoPreferences.new(
- annotations: annotations,
- autoplay: autoplay,
- comments: comments,
- continue: continue,
- continue_autoplay: continue_autoplay,
- controls: controls,
- listen: listen,
- local: local,
- player_style: player_style,
+ params = VideoPreferences.new({
+ annotations: annotations,
+ autoplay: autoplay,
+ comments: comments,
+ continue: continue,
+ continue_autoplay: continue_autoplay,
+ controls: controls,
+ listen: listen,
+ local: local,
+ player_style: player_style,
preferred_captions: preferred_captions,
- quality: quality,
- raw: raw,
- region: region,
- related_videos: related_videos,
- speed: speed,
- video_end: video_end,
- video_loop: video_loop,
- video_start: video_start,
- volume: volume,
- )
+ quality: quality,
+ raw: raw,
+ region: region,
+ related_videos: related_videos,
+ speed: speed,
+ video_end: video_end,
+ video_loop: video_loop,
+ video_start: video_start,
+ volume: volume,
+ })
return params
end
-def build_thumbnails(id, config, kemal_config)
+def build_thumbnails(id)
return {
- {name: "maxres", host: "#{make_host_url(config, kemal_config)}", url: "maxres", height: 720, width: 1280},
+ {name: "maxres", host: "#{HOST_URL}", url: "maxres", height: 720, width: 1280},
{name: "maxresdefault", host: "https://i.ytimg.com", url: "maxresdefault", height: 720, width: 1280},
{name: "sddefault", host: "https://i.ytimg.com", url: "sddefault", height: 480, width: 640},
{name: "high", host: "https://i.ytimg.com", url: "hqdefault", height: 360, width: 480},
@@ -1404,9 +1152,9 @@ def build_thumbnails(id, config, kemal_config)
}
end
-def generate_thumbnails(json, id, config, kemal_config)
+def generate_thumbnails(json, id)
json.array do
- build_thumbnails(id, config, kemal_config).each do |thumbnail|
+ build_thumbnails(id).each do |thumbnail|
json.object do
json.field "quality", thumbnail[:name]
json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
@@ -1417,7 +1165,7 @@ def generate_thumbnails(json, id, config, kemal_config)
end
end
-def generate_storyboards(json, id, storyboards, config, kemal_config)
+def generate_storyboards(json, id, storyboards)
json.array do
storyboards.each do |storyboard|
json.object do
diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr
index f1899faa..09eacbc8 100644
--- a/src/invidious/views/add_playlist_items.ecr
+++ b/src/invidious/views/add_playlist_items.ecr
@@ -20,12 +20,14 @@
-
-
+
<% videos.each_slice(4) do |slice| %>
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index b5eb46ea..caa0ad0e 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -28,7 +28,7 @@
-
<%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %>
+
<%= channel.description_html %>
@@ -92,7 +92,7 @@
<% if page > 1 %>
-
&sort_by=<%= sort_by %><% end %>">
+ &sort_by=<%= HTML.escape(sort_by) %><% end %>">
<%= translate(locale, "Previous page") %>
<% end %>
@@ -100,7 +100,7 @@
<% end %>
-
diff --git a/src/invidious/views/components/feed_menu.ecr b/src/invidious/views/components/feed_menu.ecr
index f72db2da..3dbeaf37 100644
--- a/src/invidious/views/components/feed_menu.ecr
+++ b/src/invidious/views/components/feed_menu.ecr
@@ -1,19 +1,11 @@
-
-
-
-
- <% feed_menu = env.get("preferences").as(Preferences).feed_menu.dup %>
- <% if !env.get?("user") %>
- <% feed_menu.reject! {|item| {"Subscriptions", "Playlists"}.includes? item} %>
- <% end %>
- <% feed_menu.each do |feed| %>
-
- <% end %>
-
-
-
+
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr
index 61ec3ce4..e4a60697 100644
--- a/src/invidious/views/components/item.ecr
+++ b/src/invidious/views/components/item.ecr
@@ -44,7 +44,7 @@
<% end %>
<% end %>
-
<%= item.title %>
+
<%= HTML.escape(item.title) %>
@@ -57,10 +57,10 @@
<% if plid = env.get?("remove_playlist_items") %>
-
<% end %>
-
<%= item.title %>
+
<%= HTML.escape(item.title) %>
@@ -85,7 +85,7 @@
- <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %>
+ <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
<% elsif Time.utc - item.published > 1.minute %>
<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
@@ -103,13 +103,12 @@
<% if env.get? "show_watched" %>
-
<% elsif plid = env.get? "add_playlist_items" %>
-
<% end %>
- <%= item.title %>
+
<%= HTML.escape(item.title) %>
<%= item.author %>
@@ -148,7 +147,7 @@
- <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %>
+ <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
<% elsif Time.utc - item.published > 1.minute %>
<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr
index ba6311cb..0e6664fa 100644
--- a/src/invidious/views/components/player.ecr
+++ b/src/invidious/views/components/player.ecr
@@ -1,28 +1,25 @@
-autoplay<% end %>
<% if params.video_loop %>loop<% end %>
<% if params.controls %>controls<% end %>>
- <% if hlsvp && !CONFIG.disabled?("livestreams") %>
-
+ <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %>
+
<% else %>
<% if params.listen %>
<% audio_streams.each_with_index do |fmt, i| %>
- <% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["bitrate"] %>k" selected="<%= i == 0 ? true : false %>">
+ <% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["bitrate"] %>k" selected="<%= i == 0 ? true : false %>">
<% end %>
<% else %>
<% if params.quality == "dash" %>
-
+
<% end %>
<% fmt_stream.each_with_index do |fmt, i| %>
<% if params.quality %>
- <% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["label"] %>" selected="<%= params.quality == fmt["label"].split(" - ")[0] %>">
+ <% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= params.quality == fmt["quality"] %>">
<% else %>
- <% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["label"] %>" selected="<%= i == 0 ? true : false %>">
+ <% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= i == 0 ? true : false %>">
<% end %>
<% end %>
<% end %>
@@ -39,12 +36,14 @@
<% end %>
-
diff --git a/src/invidious/views/components/player_sources.ecr b/src/invidious/views/components/player_sources.ecr
index d950e0da..d02f82d2 100644
--- a/src/invidious/views/components/player_sources.ecr
+++ b/src/invidious/views/components/player_sources.ecr
@@ -3,6 +3,8 @@
<%= HTML.escape(video.title) %> - Invidious
-
-
<%= rendered "components/player" %>
diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr
index 7d7ded2c..fe8c70b9 100644
--- a/src/invidious/views/history.ecr
+++ b/src/invidious/views/history.ecr
@@ -18,10 +18,12 @@
-
@@ -34,10 +36,10 @@ var watched_data = {
<% if !env.get("preferences").as(Preferences).thin_mode %>
-
diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr
index a32192b5..0c48be96 100644
--- a/src/invidious/views/playlists.ecr
+++ b/src/invidious/views/playlists.ecr
@@ -90,7 +90,7 @@
<% if continuation %>
-
&sort_by=<%= sort_by %><% end %>">
+ &sort_by=<%= HTML.escape(sort_by) %><% end %>">
<%= translate(locale, "Next page") %>
<% end %>
diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr
index 17e5804e..fb5bd44b 100644
--- a/src/invidious/views/preferences.ecr
+++ b/src/invidious/views/preferences.ecr
@@ -2,12 +2,6 @@
<%= translate(locale, "Preferences") %> - Invidious
<% end %>
-
-
-
- <%= translate(locale, "Top enabled: ") %>
-
-
<%= translate(locale, "CAPTCHA enabled: ") %>
checked<% end %>>
diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr
index d084bd31..bc13b7ea 100644
--- a/src/invidious/views/search.ecr
+++ b/src/invidious/views/search.ecr
@@ -2,6 +2,24 @@
<%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious
<% end %>
+
+
<% videos.each_slice(4) do |slice| %>
<% slice.each do |item| %>
diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/subscription_manager.ecr
index 43d14b37..6cddcd6c 100644
--- a/src/invidious/views/subscription_manager.ecr
+++ b/src/invidious/views/subscription_manager.ecr
@@ -37,9 +37,9 @@
-
@@ -52,32 +52,3 @@
<% end %>
<% end %>
-
-
diff --git a/src/invidious/views/subscriptions.ecr b/src/invidious/views/subscriptions.ecr
index ee31d241..af1d4fbc 100644
--- a/src/invidious/views/subscriptions.ecr
+++ b/src/invidious/views/subscriptions.ecr
@@ -45,10 +45,12 @@
-
diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr
index b7cf2dcb..61cf5c3a 100644
--- a/src/invidious/views/template.ecr
+++ b/src/invidious/views/template.ecr
@@ -111,7 +111,7 @@
-
@@ -44,32 +44,3 @@
<% end %>
<% end %>
-
-
diff --git a/src/invidious/views/top.ecr b/src/invidious/views/top.ecr
deleted file mode 100644
index f5db3aaa..00000000
--- a/src/invidious/views/top.ecr
+++ /dev/null
@@ -1,20 +0,0 @@
-<% content_for "header" do %>
-
- <% if env.get("preferences").as(Preferences).default_home != "Top" %>
- <%= translate(locale, "Top") %> - Invidious
- <% else %>
- Invidious
- <% end %>
-
-<% end %>
-
-<%= rendered "components/feed_menu" %>
-
-
- <% top_videos.each_slice(4) do |slice| %>
- <% slice.each do |item| %>
- <%= rendered "components/item" %>
- <% end %>
- <% end %>
-
diff --git a/src/invidious/views/view_all_playlists.ecr b/src/invidious/views/view_all_playlists.ecr
index 0fa7a325..5ec6aa31 100644
--- a/src/invidious/views/view_all_playlists.ecr
+++ b/src/invidious/views/view_all_playlists.ecr
@@ -6,7 +6,7 @@
-
<%= translate(locale, "`x` playlists", %(#{items.size} )) %>
+ <%= translate(locale, "`x` created playlists", %(#{items_created.size} )) %>
@@ -16,7 +16,21 @@
- <% items.each_slice(4) do |slice| %>
+ <% items_created.each_slice(4) do |slice| %>
+ <% slice.each do |item| %>
+ <%= rendered "components/item" %>
+ <% end %>
+ <% end %>
+
+
+
+
+
<%= translate(locale, "`x` saved playlists", %(#{items_saved.size} )) %>
+
+
+
+
+ <% items_saved.each_slice(4) do |slice| %>
<% slice.each do |item| %>
<%= rendered "components/item" %>
<% end %>
diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr
index df61abc5..9a1e6c32 100644
--- a/src/invidious/views/watch.ecr
+++ b/src/invidious/views/watch.ecr
@@ -3,47 +3,49 @@
">
-
+
-
+
-
-
+
+
-
+
-
-
-
+
+
+
<%= rendered "components/player_sources" %>
<%= HTML.escape(video.title) %> - Invidious
<% end %>
-
@@ -70,13 +72,13 @@ var video_data = {
<% end %>
- <% if !reason.empty? %>
+ <% if video.reason %>
- <%= reason %>
+ <%= video.reason %>
- <% elsif video.premiere_timestamp %>
+ <% elsif video.premiere_timestamp.try &.> Time.utc %>
- <%= translate(locale, "Premieres in `x`", recode_date((video.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
+ <%= video.premiere_timestamp.try { |t| translate(locale, "Premieres in `x`", recode_date((t - Time.utc).ago, locale)) } %>
<% end %>
@@ -84,10 +86,10 @@ var video_data = {
-
+
<%= translate(locale, "Watch on YouTube") %>
-
+
<% if params.annotations %>
<%= translate(locale, "Hide annotations") %>
@@ -99,26 +101,54 @@ var video_data = {
<% end %>
+ <% if user %>
+ <% playlists = PG_DB.query_all("SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%'", user.email, as: {String, String}) %>
+ <% if !playlists.empty? %>
+
+
+
+ <% end %>
+ <% end %>
+
<% if CONFIG.dmca_content.includes?(video.id) || CONFIG.disabled?("downloads") %>
- <%= translate(locale, "Download is disabled.") %>
+ <%= translate(locale, "Download is disabled.") %>
<% else %>
](screenshots/01_player.png?raw=true) | [
](screenshots/02_preferences.png?raw=true) | [
](screenshots/03_subscriptions.png?raw=true) |
| [
](screenshots/04_description.png?raw=true) | [
](screenshots/05_preferences.png?raw=true) | [
](screenshots/06_subscriptions.png?raw=true) |
-## Installation
+## Installation:
-See [Invidious-Updater](https://github.com/tmiland/Invidious-Updater) for a self-contained script that can automatically install and update Invidious.
+To manually compile invidious you need at least 2GB of RAM. If you have less you can setup SWAP to have a combined amount of 2 GB or use Docker instead.
+
+After installation take a look at the [Post-install steps](#post-install-configuration).
+
+### Automated installation:
+
+[Invidious-Updater](https://github.com/tmiland/Invidious-Updater) is a self-contained script that can automatically install and update Invidious.
### Docker:
@@ -58,7 +58,7 @@ See [Invidious-Updater](https://github.com/tmiland/Invidious-Updater) for a self
$ docker-compose up
```
-And visit `localhost:3000` in your browser.
+Then visit `localhost:3000` in your browser.
#### Rebuild cluster:
@@ -73,9 +73,11 @@ $ docker volume rm invidious_postgresdata
$ docker-compose build
```
+### Manual installation:
+
### Linux:
-#### Install dependencies
+#### Install the dependencies
```bash
# Arch Linux
@@ -88,23 +90,22 @@ $ curl -sSL https://dist.crystal-lang.org/apt/setup.sh | sudo bash
$ curl -sL "https://keybase.io/crystal/pgp_keys.asc" | sudo apt-key add -
$ echo "deb https://dist.crystal-lang.org/apt crystal main" | sudo tee /etc/apt/sources.list.d/crystal.list
$ sudo apt-get update
-$ sudo apt install crystal libssl-dev libxml2-dev libyaml-dev libgmp-dev libreadline-dev postgresql librsvg2-bin libsqlite3-dev
+$ sudo apt install crystal libssl-dev libxml2-dev libyaml-dev libgmp-dev libreadline-dev postgresql librsvg2-bin libsqlite3-dev zlib1g-dev
```
-#### Add invidious user and clone repository
+#### Add an Invidious user and clone the repository
```bash
$ useradd -m invidious
$ sudo -i -u invidious
-$ git clone https://github.com/omarroth/invidious
+$ git clone https://github.com/iv-org/invidious
$ exit
```
-#### Setup PostgresSQL
+#### Set up PostgresSQL
```bash
-$ sudo systemctl enable postgresql
-$ sudo systemctl start postgresql
+$ sudo systemctl enable --now postgresql
$ sudo -i -u postgres
$ psql -c "CREATE USER kemal WITH PASSWORD 'kemal';" # Change 'kemal' here to a stronger password, and update `password` in config/config.yml
$ createdb -O kemal invidious
@@ -115,10 +116,12 @@ $ psql invidious kemal < /home/invidious/invidious/config/sql/users.sql
$ psql invidious kemal < /home/invidious/invidious/config/sql/session_ids.sql
$ psql invidious kemal < /home/invidious/invidious/config/sql/nonces.sql
$ psql invidious kemal < /home/invidious/invidious/config/sql/annotations.sql
+$ psql invidious kemal < /home/invidious/invidious/config/sql/playlists.sql
+$ psql invidious kemal < /home/invidious/invidious/config/sql/playlist_videos.sql
$ exit
```
-#### Setup Invidious
+#### Set up Invidious
```bash
$ sudo -i -u invidious
@@ -130,23 +133,36 @@ $ ./invidious # stop with ctrl c
$ exit
```
-#### systemd service
+#### Systemd service:
```bash
$ sudo cp /home/invidious/invidious/invidious.service /etc/systemd/system/invidious.service
-$ sudo systemctl enable invidious.service
-$ sudo systemctl start invidious.service
+$ sudo systemctl enable --now invidious.service
```
-### OSX:
+#### Logrotate:
+
+```bash
+$ sudo echo "/home/invidious/invidious/invidious.log {
+rotate 4
+weekly
+notifempty
+missingok
+compress
+minsize 1048576
+}" | tee /etc/logrotate.d/invidious.logrotate
+$ sudo chmod 0644 /etc/logrotate.d/invidious.logrotate
+```
+
+### MacOS:
```bash
# Install dependencies
$ brew update
$ brew install shards crystal postgres imagemagick librsvg
-# Clone repository and setup postgres database
-$ git clone https://github.com/omarroth/invidious
+# Clone the repository and set up a PostgreSQL database
+$ git clone https://github.com/iv-org/invidious
$ cd invidious
$ brew services start postgresql
$ psql -c "CREATE ROLE kemal WITH PASSWORD 'kemal';" # Change 'kemal' here to a stronger password, and update `password` in config/config.yml
@@ -158,15 +174,30 @@ $ psql invidious kemal < config/sql/users.sql
$ psql invidious kemal < config/sql/session_ids.sql
$ psql invidious kemal < config/sql/nonces.sql
$ psql invidious kemal < config/sql/annotations.sql
+$ psql invidious kemal < config/sql/privacy.sql
+$ psql invidious kemal < config/sql/playlists.sql
+$ psql invidious kemal < config/sql/playlist_videos.sql
-# Setup Invidious
+# Set up Invidious
$ shards update && shards install
$ crystal build src/invidious.cr --release
```
+## Post-install configuration:
+
+Detailed configuration available in the [configuration guide](https://github.com/iv-org/invidious/wiki/Configuration).
+
+If you use a reverse proxy, you **must** configure invidious to properly serve request through it:
+
+`https_only: true` : if your are serving your instance via https, set it to true
+
+`domain: domain.ext`: if you are serving your instance via a domain name, set it here
+
+`external_port: 443`: if your are serving your instance via https, set it to 443
+
## Update Invidious
-You can see how to update Invidious [here](https://github.com/omarroth/invidious/wiki/Updating).
+Instructions are available in the [updating guide](https://github.com/iv-org/invidious/wiki/Updating).
## Usage:
@@ -197,39 +228,55 @@ $ ./sentry
## Documentation
-[Documentation](https://github.com/omarroth/invidious/wiki) can be found in the wiki.
+[Documentation](https://github.com/iv-org/invidious/wiki) can be found in the wiki.
## Extensions
-[Extensions](https://github.com/omarroth/invidious/wiki/Extensions) can be found in the wiki, as well as documentation for integrating it into other projects.
+[Extensions](https://github.com/iv-org/invidious/wiki/Extensions) can be found in the wiki, as well as documentation for integrating it into other projects.
## Made with Invidious
-- [FreeTube](https://github.com/FreeTubeApp/FreeTube): An Open Source YouTube app for privacy.
-- [CloudTube](https://cadence.moe/cloudtube/subscriptions): A JS-rich alternate YouTube player
-- [PeerTubeify](https://gitlab.com/Ealhad/peertubeify): On YouTube, displays a link to the same video on PeerTube, if it exists.
-- [MusicPiped](https://github.com/deep-gaurav/MusicPiped): A materialistic music player that streams music from YouTube.
+- [FreeTube](https://github.com/FreeTubeApp/FreeTube): A libre software YouTube app for privacy.
+- [CloudTube](https://cadence.moe/cloudtube/subscriptions): A JavaScript-rich alternate YouTube player
+- [PeerTubeify](https://gitlab.com/Cha_deL/peertubeify): On YouTube, displays a link to the same video on PeerTube, if it exists.
+- [MusicPiped](https://github.com/deep-gaurav/MusicPiped): A material design music player that streams music from YouTube.
+- [LapisTube](https://github.com/blubbll/lapis-tube): A fancy and advanced (experimental) YouTube front-end. Combined streams & custom YT features.
+- [HoloPlay](https://github.com/stephane-r/HoloPlay): Funny Android application connecting on Invidious API's with search, playlists and favoris.
## Contributing
-1. Fork it ( https://github.com/omarroth/invidious/fork )
+1. Fork it ( https://github.com/iv-org/invidious/fork )
2. Create your feature branch (git checkout -b my-new-feature)
3. Commit your changes (git commit -am 'Add some feature')
4. Push to the branch (git push origin my-new-feature)
-5. Create a new Pull Request
+5. Create a new pull request
+
+#### Translation
+
+- Log in with an account you have elsewhere, or register an account and start translating at [Hosted Weblate](https://hosted.weblate.org/engage/invidious/).
+
+## Donate:
+
+Liberapay: https://liberapay.com/iv-org/
## Contact
-Feel free to send an email to omarroth@protonmail.com or join our [Matrix Server](https://riot.im/app/#/room/#invidious:matrix.org), or #invidious on Freenode.
+Feel free to join our [Matrix room](https://matrix.to/#/#invidious:matrix.org), or #invidious on freenode. Both platforms are bridged together.
-You can also view release notes on the [releases](https://github.com/omarroth/invidious/releases) page or in the CHANGELOG.md included in the repository.
+## Liability
-## License
+We take no responsibility for the use of our tool, or external instances provided by third parties. We strongly recommend you abide by the valid official regulations in your country. Furthermore, we refuse liability for any inappropriate use of Invidious, such as illegal downloading. This tool is provided to you in the spirit of free, open software.
-[](http://www.gnu.org/licenses/agpl-3.0.en.html)
+You may view the LICENSE in which this software is provided to you [here](./LICENSE).
-Invidious is Free Software: You can use, study share and improve it at your
-will. Specifically you can redistribute and/or modify it under the terms of the
-[GNU Affero General Public License](https://www.gnu.org/licenses/agpl.html) as
-published by the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+> 16. Limitation of Liability.
+>
+> IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
diff --git a/TRANSLATION b/TRANSLATION
new file mode 100644
index 00000000..fa340d71
--- /dev/null
+++ b/TRANSLATION
@@ -0,0 +1 @@
+https://hosted.weblate.org/projects/invidious/
diff --git a/assets/css/default.css b/assets/css/default.css
index ea139b40..b7a77be6 100644
--- a/assets/css/default.css
+++ b/assets/css/default.css
@@ -60,6 +60,22 @@ body {
color: rgb(255, 0, 0);
}
+.feed-menu {
+ display: flex;
+ justify-content: center;
+ flex-wrap: wrap;
+}
+
+.feed-menu-item {
+ text-align: center;
+}
+
+@media screen and (max-width: 640px) {
+ .feed-menu-item {
+ flex: 0 0 40%;
+ }
+}
+
.h-box {
padding-left: 1em;
padding-right: 1em;
diff --git a/assets/css/embed.css b/assets/css/embed.css
new file mode 100644
index 00000000..12fefe58
--- /dev/null
+++ b/assets/css/embed.css
@@ -0,0 +1,10 @@
+#player {
+ position: fixed;
+ right: 0;
+ bottom: 0;
+ min-width: 100%;
+ min-height: 100%;
+ width: auto;
+ height: auto;
+ z-index: -100;
+}
diff --git a/assets/css/videojs-vtt-thumbnails-fix.css b/assets/css/videojs-vtt-thumbnails-fix.css
new file mode 100644
index 00000000..8b62cf0c
--- /dev/null
+++ b/assets/css/videojs-vtt-thumbnails-fix.css
@@ -0,0 +1,3 @@
+.video-js .vjs-vtt-thumbnail-display {
+ max-width: 158px;
+}
diff --git a/assets/js/community.js b/assets/js/community.js
index 754ec6d3..4077f1cd 100644
--- a/assets/js/community.js
+++ b/assets/js/community.js
@@ -1,3 +1,5 @@
+var community_data = JSON.parse(document.getElementById('community_data').innerHTML);
+
String.prototype.supplant = function (o) {
return this.replace(/{([^{}]*)}/g, function (a, b) {
var r = o[b];
diff --git a/assets/js/embed.js b/assets/js/embed.js
index 534c30ff..99d2fc53 100644
--- a/assets/js/embed.js
+++ b/assets/js/embed.js
@@ -1,3 +1,5 @@
+var video_data = JSON.parse(document.getElementById('video_data').innerHTML);
+
function get_playlist(plid, retries) {
if (retries == undefined) retries = 5;
diff --git a/assets/js/global.js b/assets/js/global.js
new file mode 100644
index 00000000..efb447fb
--- /dev/null
+++ b/assets/js/global.js
@@ -0,0 +1,3 @@
+// Disable Web Workers. Fixes Video.js CSP violation (created by `new Worker(objURL)`):
+// Refused to create a worker from 'blob:http://host/id' because it violates the following Content Security Policy directive: "worker-src 'self'".
+window.Worker = undefined;
diff --git a/assets/js/handlers.js b/assets/js/handlers.js
new file mode 100644
index 00000000..b3da8d9b
--- /dev/null
+++ b/assets/js/handlers.js
@@ -0,0 +1,144 @@
+'use strict';
+
+(function () {
+ var n2a = function (n) { return Array.prototype.slice.call(n); };
+
+ var video_player = document.getElementById('player_html5_api');
+ if (video_player) {
+ video_player.onmouseenter = function () { video_player['data-title'] = video_player['title']; video_player['title'] = ''; };
+ video_player.onmouseleave = function () { video_player['title'] = video_player['data-title']; video_player['data-title'] = ''; };
+ video_player.oncontextmenu = function () { video_player['title'] = video_player['data-title']; };
+ }
+
+ // For dynamically inserted elements
+ document.addEventListener('click', function (e) {
+ if (!e || !e.target) { return; }
+ e = e.target;
+ var handler_name = e.getAttribute('data-onclick');
+ switch (handler_name) {
+ case 'jump_to_time':
+ var time = e.getAttribute('data-jump-time');
+ player.currentTime(time);
+ break;
+ case 'get_youtube_replies':
+ var load_more = e.getAttribute('data-load-more') !== null;
+ get_youtube_replies(e, load_more);
+ break;
+ case 'toggle_parent':
+ toggle_parent(e);
+ break;
+ default:
+ break;
+ }
+ });
+
+ n2a(document.querySelectorAll('[data-mouse="switch_classes"]')).forEach(function (e) {
+ var classes = e.getAttribute('data-switch-classes').split(',');
+ var ec = classes[0];
+ var lc = classes[1];
+ var onoff = function (on, off) {
+ var cs = e.getAttribute('class');
+ cs = cs.split(off).join(on);
+ e.setAttribute('class', cs);
+ };
+ e.onmouseenter = function () { onoff(ec, lc); };
+ e.onmouseleave = function () { onoff(lc, ec); };
+ });
+
+ n2a(document.querySelectorAll('[data-onsubmit="return_false"]')).forEach(function (e) {
+ e.onsubmit = function () { return false; };
+ });
+
+ n2a(document.querySelectorAll('[data-onclick="mark_watched"]')).forEach(function (e) {
+ e.onclick = function () { mark_watched(e); };
+ });
+ n2a(document.querySelectorAll('[data-onclick="mark_unwatched"]')).forEach(function (e) {
+ e.onclick = function () { mark_unwatched(e); };
+ });
+ n2a(document.querySelectorAll('[data-onclick="add_playlist_video"]')).forEach(function (e) {
+ e.onclick = function () { add_playlist_video(e); };
+ });
+ n2a(document.querySelectorAll('[data-onclick="add_playlist_item"]')).forEach(function (e) {
+ e.onclick = function () { add_playlist_item(e); };
+ });
+ n2a(document.querySelectorAll('[data-onclick="remove_playlist_item"]')).forEach(function (e) {
+ e.onclick = function () { remove_playlist_item(e); };
+ });
+ n2a(document.querySelectorAll('[data-onclick="revoke_token"]')).forEach(function (e) {
+ e.onclick = function () { revoke_token(e); };
+ });
+ n2a(document.querySelectorAll('[data-onclick="remove_subscription"]')).forEach(function (e) {
+ e.onclick = function () { remove_subscription(e); };
+ });
+ n2a(document.querySelectorAll('[data-onclick="notification_requestPermission"]')).forEach(function (e) {
+ e.onclick = function () { Notification.requestPermission(); };
+ });
+
+ n2a(document.querySelectorAll('[data-onrange="update_volume_value"]')).forEach(function (e) {
+ var cb = function () { update_volume_value(e); }
+ e.oninput = cb;
+ e.onchange = cb;
+ });
+
+ function update_volume_value(element) {
+ document.getElementById('volume-value').innerText = element.value;
+ }
+
+ function revoke_token(target) {
+ var row = target.parentNode.parentNode.parentNode.parentNode.parentNode;
+ row.style.display = 'none';
+ var count = document.getElementById('count');
+ count.innerText = count.innerText - 1;
+
+ var referer = window.encodeURIComponent(document.location.href);
+ var url = '/token_ajax?action_revoke_token=1&redirect=false' +
+ '&referer=' + referer +
+ '&session=' + target.getAttribute('data-session');
+ var xhr = new XMLHttpRequest();
+ xhr.responseType = 'json';
+ xhr.timeout = 10000;
+ xhr.open('POST', url, true);
+ xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
+
+ xhr.onreadystatechange = function () {
+ if (xhr.readyState == 4) {
+ if (xhr.status != 200) {
+ count.innerText = parseInt(count.innerText) + 1;
+ row.style.display = '';
+ }
+ }
+ }
+
+ var csrf_token = target.parentNode.querySelector('input[name="csrf_token"]').value;
+ xhr.send('csrf_token=' + csrf_token);
+ }
+
+ function remove_subscription(target) {
+ var row = target.parentNode.parentNode.parentNode.parentNode.parentNode;
+ row.style.display = 'none';
+ var count = document.getElementById('count');
+ count.innerText = count.innerText - 1;
+
+ var referer = window.encodeURIComponent(document.location.href);
+ var url = '/subscription_ajax?action_remove_subscriptions=1&redirect=false' +
+ '&referer=' + referer +
+ '&c=' + target.getAttribute('data-ucid');
+ var xhr = new XMLHttpRequest();
+ xhr.responseType = 'json';
+ xhr.timeout = 10000;
+ xhr.open('POST', url, true);
+ xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
+
+ xhr.onreadystatechange = function () {
+ if (xhr.readyState == 4) {
+ if (xhr.status != 200) {
+ count.innerText = parseInt(count.innerText) + 1;
+ row.style.display = '';
+ }
+ }
+ }
+
+ var csrf_token = target.parentNode.querySelector('input[name="csrf_token"]').value;
+ xhr.send('csrf_token=' + csrf_token);
+ }
+})();
diff --git a/assets/js/notifications.js b/assets/js/notifications.js
index fcfc01e7..3d1ec1ed 100644
--- a/assets/js/notifications.js
+++ b/assets/js/notifications.js
@@ -1,3 +1,5 @@
+var notification_data = JSON.parse(document.getElementById('notification_data').innerHTML);
+
var notifications, delivered;
function get_subscriptions(callback, retries) {
diff --git a/assets/js/player.js b/assets/js/player.js
index e58af0cd..f79fbbf3 100644
--- a/assets/js/player.js
+++ b/assets/js/player.js
@@ -1,3 +1,6 @@
+var player_data = JSON.parse(document.getElementById('player_data').innerHTML);
+var video_data = JSON.parse(document.getElementById('video_data').innerHTML);
+
var options = {
preload: 'auto',
liveui: true,
@@ -35,7 +38,7 @@ var shareOptions = {
title: player_data.title,
description: player_data.description,
image: player_data.thumbnail,
- embedCode: ""
+ embedCode: ""
}
var player = videojs('player', options);
@@ -146,7 +149,8 @@ if (!video_data.params.listen && video_data.params.quality === 'dash') {
}
player.vttThumbnails({
- src: location.origin + '/api/v1/storyboards/' + video_data.id + '?height=90'
+ src: location.origin + '/api/v1/storyboards/' + video_data.id + '?height=90',
+ showTimestamp: true
});
// Enable annotations
@@ -228,11 +232,24 @@ function set_time_percent(percent) {
player.currentTime(newTime);
}
+function play() {
+ player.play();
+}
+
+function pause() {
+ player.pause();
+}
+
+function stop() {
+ player.pause();
+ player.currentTime(0);
+}
+
function toggle_play() {
if (player.paused()) {
- player.play();
+ play();
} else {
- player.pause();
+ pause();
}
}
@@ -338,9 +355,22 @@ window.addEventListener('keydown', e => {
switch (decoratedKey) {
case ' ':
case 'k':
+ case 'MediaPlayPause':
action = toggle_play;
break;
+ case 'MediaPlay':
+ action = play;
+ break;
+
+ case 'MediaPause':
+ action = pause;
+ break;
+
+ case 'MediaStop':
+ action = stop;
+ break;
+
case 'ArrowUp':
if (isPlayerFocused) {
action = increase_volume.bind(this, 0.1);
@@ -357,9 +387,11 @@ window.addEventListener('keydown', e => {
break;
case 'ArrowRight':
+ case 'MediaFastForward':
action = skip_seconds.bind(this, 5);
break;
case 'ArrowLeft':
+ case 'MediaTrackPrevious':
action = skip_seconds.bind(this, -5);
break;
case 'l':
@@ -391,9 +423,11 @@ window.addEventListener('keydown', e => {
break;
case 'N':
+ case 'MediaTrackNext':
action = next_video;
break;
case 'P':
+ case 'MediaTrackPrevious':
// TODO: Add support to play back previous video.
break;
diff --git a/assets/js/playlist_widget.js b/assets/js/playlist_widget.js
index 5d6ddf87..0ec27859 100644
--- a/assets/js/playlist_widget.js
+++ b/assets/js/playlist_widget.js
@@ -1,3 +1,29 @@
+var playlist_data = JSON.parse(document.getElementById('playlist_data').innerHTML);
+
+function add_playlist_video(target) {
+ var select = target.parentNode.children[0].children[1];
+ var option = select.children[select.selectedIndex];
+
+ var url = '/playlist_ajax?action_add_video=1&redirect=false' +
+ '&video_id=' + target.getAttribute('data-id') +
+ '&playlist_id=' + option.getAttribute('data-plid');
+ var xhr = new XMLHttpRequest();
+ xhr.responseType = 'json';
+ xhr.timeout = 10000;
+ xhr.open('POST', url, true);
+ xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
+
+ xhr.onreadystatechange = function () {
+ if (xhr.readyState == 4) {
+ if (xhr.status == 200) {
+ option.innerText = '✓' + option.innerText;
+ }
+ }
+ }
+
+ xhr.send('csrf_token=' + playlist_data.csrf_token);
+}
+
function add_playlist_item(target) {
var tile = target.parentNode.parentNode.parentNode.parentNode.parentNode;
tile.style.display = 'none';
diff --git a/assets/js/silvermine-videojs-quality-selector.min.js b/assets/js/silvermine-videojs-quality-selector.min.js
index e4869564..88621e8d 100644
--- a/assets/js/silvermine-videojs-quality-selector.min.js
+++ b/assets/js/silvermine-videojs-quality-selector.min.js
@@ -1,3 +1,4 @@
-/*! @silvermine/videojs-quality-selector 2019-09-26 v1.2.2-4-gc134430-dirty */
+/*! @silvermine/videojs-quality-selector 2020-03-02 v1.1.2-36-g64d620a-dirty */
-!function u(o,c,a){function l(e,n){if(!c[e]){if(!o[e]){var t="function"==typeof require&&require;if(!n&&t)return t(e,!0);if(s)return s(e,!0);var r=new Error("Cannot find module '"+e+"'");throw r.code="MODULE_NOT_FOUND",r}var i=c[e]={exports:{}};o[e][0].call(i.exports,function(n){return l(o[e][1][n]||n)},i,i.exports,u,o,c,a)}return c[e].exports}for(var s="function"==typeof require&&require,n=0;n
<% if plid = env.get?("remove_playlist_items") %>
- 
-