nihilist/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious synced 2024-11-27 21:43:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactor continuation protocol buffers

Browse Source
This commit is contained in:
Omar Roth 2019-07-20 20:18:08 -05:00
parent e736626953
commit f18d8229c0
No known key found for this signature in database
GPG Key ID: B8254FB7EC3D37F2
6 changed files with 298 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
spec/helpers_spec.cr
View File

File diff suppressed because one or more lines are too long

246
src/invidious/channels.cr
View File

@ -441,53 +441,57 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "
switch = 0x00
end
meta = IO::Memory.new
meta.write(Bytes[0x12, 0x06])
meta.print("videos")
data = IO::Memory.new
data.write_byte 0x12
data.write_byte 0x06
data.print "videos"
meta.write(Bytes[0x30, 0x02])
meta.write(Bytes[0x38, 0x01])
meta.write(Bytes[0x60, 0x01])
meta.write(Bytes[0x6a, 0x00])
meta.write(Bytes[0xb8, 0x01, 0x00])
data.write Bytes[0x30, 0x02]
data.write Bytes[0x38, 0x01]
data.write Bytes[0x60, 0x01]
data.write Bytes[0x6a, 0x00]
data.write Bytes[0xb8, 0x01, 0x00]
meta.write(Bytes[0x20, switch])
meta.write(Bytes[0x7a, page.size])
meta.print(page)
data.write Bytes[0x20, switch]
data.write_byte 0x7a
VarInt.to_io(data, page.bytesize)
data.print page
case sort_by
when "newest"
# Empty tags can be omitted
# meta.write(Bytes[0x18,0x00])
when "popular"
meta.write(Bytes[0x18, 0x01])
data.write Bytes[0x18, 0x01]
when "oldest"
meta.write(Bytes[0x18, 0x02])
data.write Bytes[0x18, 0x02]
end
meta.rewind
meta = Base64.urlsafe_encode(meta.to_slice)
meta = URI.escape(meta)
data = Base64.urlsafe_encode(data)
cursor = URI.escape(data)
continuation = IO::Memory.new
continuation.write(Bytes[0x12, ucid.size])
continuation.print(ucid)
data = IO::Memory.new
continuation.write(Bytes[0x1a, meta.size])
continuation.print(meta)
data.write_byte 0x12
VarInt.to_io(data, ucid.bytesize)
data.print ucid
continuation.rewind
continuation = continuation.gets_to_end
data.write_byte 0x1a
VarInt.to_io(data, cursor.bytesize)
data.print cursor
wrapper = IO::Memory.new
wrapper.write(Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02, continuation.size])
wrapper.print(continuation)
wrapper.rewind
data.rewind
wrapper = Base64.urlsafe_encode(wrapper.to_slice)
wrapper = URI.escape(wrapper)
buffer = IO::Memory.new
buffer.write Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02]
VarInt.to_io(buffer, data.bytesize)
url = "/browse_ajax?continuation=#{wrapper}&gl=US&hl=en"
IO.copy data, buffer
continuation = Base64.urlsafe_encode(buffer)
continuation = URI.escape(continuation)
url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
return url
end
@ -497,117 +501,108 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
cursor = Base64.urlsafe_encode(cursor, false)
end
meta = IO::Memory.new
data = IO::Memory.new
if auto_generated
meta.write(Bytes[0x08, 0x0a])
data.write Bytes[0x08, 0x0a]
end
meta.write(Bytes[0x12, 0x09])
meta.print("playlists")
data.write Bytes[0x12, 0x09]
data.print "playlists"
if auto_generated
meta.write(Bytes[0x20, 0x32])
data.write Bytes[0x20, 0x32]
else
# TODO: Look at 0x01, 0x00
case sort
when "oldest", "oldest_created"
meta.write(Bytes[0x18, 0x02])
data.write Bytes[0x18, 0x02]
when "newest", "newest_created"
meta.write(Bytes[0x18, 0x03])
data.write Bytes[0x18, 0x03]
when "last", "last_added"
meta.write(Bytes[0x18, 0x04])
data.write Bytes[0x18, 0x04]
end
meta.write(Bytes[0x20, 0x01])
data.write Bytes[0x20, 0x01]
end
meta.write(Bytes[0x30, 0x02])
meta.write(Bytes[0x38, 0x01])
meta.write(Bytes[0x60, 0x01])
meta.write(Bytes[0x6a, 0x00])
data.write Bytes[0x30, 0x02]
data.write Bytes[0x38, 0x01]
data.write Bytes[0x60, 0x01]
data.write Bytes[0x6a, 0x00]
meta.write(Bytes[0x7a, cursor.size])
meta.print(cursor)
data.write_byte 0x7a
VarInt.to_io(data, cursor.bytesize)
data.print cursor
meta.write(Bytes[0xb8, 0x01, 0x00])
data.write Bytes[0xb8, 0x01, 0x00]
meta.rewind
meta = Base64.urlsafe_encode(meta.to_slice)
meta = URI.escape(meta)
data.rewind
data = Base64.urlsafe_encode(data)
continuation = URI.escape(data)
continuation = IO::Memory.new
continuation.write(Bytes[0x12, ucid.size])
continuation.print(ucid)
data = IO::Memory.new
continuation.write(Bytes[0x1a])
continuation.write(write_var_int(meta.size))
continuation.print(meta)
data.write_byte 0x12
VarInt.to_io(data, ucid.bytesize)
data.print ucid
continuation.rewind
continuation = continuation.gets_to_end
data.write_byte 0x1a
VarInt.to_io(data, continuation.bytesize)
data.print continuation
wrapper = IO::Memory.new
wrapper.write(Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02])
wrapper.write(write_var_int(continuation.size))
wrapper.print(continuation)
wrapper.rewind
data.rewind
wrapper = Base64.urlsafe_encode(wrapper.to_slice)
wrapper = URI.escape(wrapper)
buffer = IO::Memory.new
buffer.write Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02]
VarInt.to_io(buffer, data.bytesize)
url = "/browse_ajax?continuation=#{wrapper}&gl=US&hl=en"
IO.copy data, buffer
continuation = Base64.urlsafe_encode(buffer)
continuation = URI.escape(continuation)
url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
return url
end
def extract_channel_playlists_cursor(url, auto_generated)
wrapper = HTTP::Params.parse(URI.parse(url).query.not_nil!)["continuation"]
continuation = HTTP::Params.parse(URI.parse(url).query.not_nil!)["continuation"]
wrapper = URI.unescape(wrapper)
wrapper = Base64.decode(wrapper)
continuation = URI.unescape(continuation)
data = IO::Memory.new(Base64.decode(continuation))
# 0xe2 0xa9 0x85 0xb2 0x02
wrapper += 5
data.pos += 5
continuation_size = read_var_int(wrapper[0, 4])
wrapper += write_var_int(continuation_size).size
continuation = wrapper[0, continuation_size]
continuation = Bytes.new(data.read_bytes(VarInt))
data.read continuation
data = IO::Memory.new(continuation)
# 0x12
continuation += 1
ucid_size = continuation[0]
continuation += 1
ucid = continuation[0, ucid_size]
continuation += ucid_size
data.read_byte # => 0x12
ucid = Bytes.new(data.read_bytes(VarInt))
data.read ucid
# 0x1a
continuation += 1
meta_size = read_var_int(continuation[0, 4])
continuation += write_var_int(meta_size).size
meta = continuation[0, meta_size]
continuation += meta_size
data.read_byte # => 0x1a
inner_continuation = Bytes.new(data.read_bytes(VarInt))
data.read inner_continuation
meta = String.new(meta)
meta = URI.unescape(meta)
meta = Base64.decode(meta)
continuation = String.new(inner_continuation)
continuation = URI.unescape(continuation)
data = IO::Memory.new(Base64.decode(continuation))
# 0x12 0x09 playlists
meta += 11
data.pos += 11
until meta[0] == 0x7a
tag = read_var_int(meta[0, 4])
meta += write_var_int(tag).size
value = meta[0]
meta += 1
until data.peek[0] == 0x7a
key = data.read_bytes(VarInt)
value = data.read_bytes(VarInt)
end
# 0x7a
meta += 1
cursor_size = meta[0]
meta += 1
cursor = meta[0, cursor_size]
data.pos += 1 # => 0x7a
cursor = Bytes.new(data.read_bytes(VarInt))
data.read cursor
cursor = String.new(cursor)
if !auto_generated
@ -874,20 +869,26 @@ end
def produce_channel_community_continuation(ucid, cursor)
cursor = URI.escape(cursor)
continuation = IO::Memory.new
continuation.write(Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02])
continuation.write(write_var_int(3 + ucid.size + write_var_int(cursor.size).size + cursor.size))
data = IO::Memory.new
continuation.write(Bytes[0x12, ucid.size])
continuation.print(ucid)
data.write_byte 0x12
VarInt.to_io(data, ucid.bytesize)
data.print ucid
continuation.write(Bytes[0x1a])
continuation.write(write_var_int(cursor.size))
continuation.print(cursor)
continuation.rewind
data.write_byte 0x1a
VarInt.to_io(data, cursor.bytesize)
data.print cursor
continuation = Base64.urlsafe_encode(continuation.to_slice)
data.rewind
buffer = IO::Memory.new
buffer.write Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02]
VarInt.to_io(buffer, data.size)
IO.copy data, buffer
continuation = Base64.urlsafe_encode(buffer)
continuation = URI.escape(continuation)
return continuation
@ -895,28 +896,25 @@ end
def extract_channel_community_cursor(continuation)
continuation = URI.unescape(continuation)
continuation = Base64.decode(continuation)
data = IO::Memory.new(Base64.decode(continuation))
# 0xe2 0xa9 0x85 0xb2 0x02
continuation += 5
data.pos += 5
total_size = read_var_int(continuation[0, 4])
continuation += write_var_int(total_size).size
continuation = Bytes.new(data.read_bytes(VarInt))
data.read continuation
data = IO::Memory.new(continuation)
# 0x12
continuation += 1
ucid_size = continuation[0]
continuation += 1
ucid = continuation[0, ucid_size]
continuation += ucid_size
data.read_byte # => 0x12
ucid = Bytes.new(data.read_bytes(VarInt))
data.read ucid
# 0x1a
continuation += 1
until continuation[0] == 'E'.ord
continuation += 1
data.read_byte # => 0x1a
until data.peek[0] == 'E'.ord
data.read_byte
end
return String.new(continuation)
return URI.unescape(data.gets_to_end)
end
def get_about_info(ucid, locale)

119
src/invidious/comments.cr
View File

@ -564,108 +564,105 @@ def content_to_comment_html(content)
end
def produce_comment_continuation(video_id, cursor = "", sort_by = "top")
continuation = IO::Memory.new
data = IO::Memory.new
continuation.write(Bytes[0x12, 0x26])
data.write Bytes[0x12, 0x26]
continuation.write(Bytes[0x12, video_id.size])
continuation.print(video_id)
data.write_byte 0x12
VarInt.to_io(data, video_id.bytesize)
data.print video_id
continuation.write(Bytes[0xc0, 0x01, 0x01])
continuation.write(Bytes[0xc8, 0x01, 0x01])
continuation.write(Bytes[0xe0, 0x01, 0x01])
data.write Bytes[0xc0, 0x01, 0x01]
data.write Bytes[0xc8, 0x01, 0x01]
data.write Bytes[0xe0, 0x01, 0x01]
continuation.write(Bytes[0xa2, 0x02, 0x0d])
continuation.write(Bytes[0x28, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01])
data.write Bytes[0xa2, 0x02, 0x0d]
data.write Bytes[0x28, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]
continuation.write(Bytes[0x40, 0x00])
continuation.write(Bytes[0x18, 0x06])
data.write Bytes[0x40, 0x00]
data.write Bytes[0x18, 0x06]
if cursor.empty?
continuation.write(Bytes[0x32])
continuation.write(write_var_int(video_id.size + 8))
data.write Bytes[0x32]
VarInt.to_io(data, cursor.bytesize + video_id.bytesize + 8)
continuation.write(Bytes[0x22, video_id.size + 4])
continuation.write(Bytes[0x22, video_id.size])
continuation.print(video_id)
data.write Bytes[0x22, video_id.bytesize + 4]
data.write Bytes[0x22, video_id.bytesize]
data.print video_id
case sort_by
when "top"
continuation.write(Bytes[0x30, 0x00])
data.write Bytes[0x30, 0x00]
when "new", "newest"
continuation.write(Bytes[0x30, 0x01])
data.write Bytes[0x30, 0x01]
end
continuation.write(Bytes[0x78, 0x02])
data.write(Bytes[0x78, 0x02])
else
continuation.write(Bytes[0x32])
continuation.write(write_var_int(cursor.size + video_id.size + 11))
data.write Bytes[0x32]
VarInt.to_io(data, cursor.bytesize + video_id.bytesize + 11)
continuation.write(Bytes[0x0a])
continuation.write(write_var_int(cursor.size))
continuation.print(cursor)
data.write_byte 0x0a
VarInt.to_io(data, cursor.bytesize)
data.print cursor
continuation.write(Bytes[0x22, video_id.size + 4])
continuation.write(Bytes[0x22, video_id.size])
continuation.print(video_id)
data.write Bytes[0x22, video_id.bytesize + 4]
data.write Bytes[0x22, video_id.bytesize]
data.print video_id
case sort_by
when "top"
continuation.write(Bytes[0x30, 0x00])
data.write Bytes[0x30, 0x00]
when "new", "newest"
continuation.write(Bytes[0x30, 0x01])
data.write Bytes[0x30, 0x01]
end
continuation.write(Bytes[0x28, 0x14])
data.write Bytes[0x28, 0x14]
end
continuation.rewind
continuation = continuation.gets_to_end
continuation = Base64.urlsafe_encode(continuation.to_slice)
continuation = Base64.urlsafe_encode(data)
continuation = URI.escape(continuation)
return continuation
end
def produce_comment_reply_continuation(video_id, ucid, comment_id)
continuation = IO::Memory.new
data = IO::Memory.new
continuation.write(Bytes[0x12, 0x26])
data.write Bytes[0x12, 0x26]
continuation.write(Bytes[0x12, video_id.size])
continuation.print(video_id)
data.write_byte 0x12
VarInt.to_io(data, video_id.size)
data.print video_id
continuation.write(Bytes[0xc0, 0x01, 0x01])
continuation.write(Bytes[0xc8, 0x01, 0x01])
continuation.write(Bytes[0xe0, 0x01, 0x01])
data.write Bytes[0xc0, 0x01, 0x01]
data.write Bytes[0xc8, 0x01, 0x01]
data.write Bytes[0xe0, 0x01, 0x01]
continuation.write(Bytes[0xa2, 0x02, 0x0d])
continuation.write(Bytes[0x28, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01])
data.write Bytes[0xa2, 0x02, 0x0d]
data.write Bytes[0x28, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]
continuation.write(Bytes[0x40, 0x00])
continuation.write(Bytes[0x18, 0x06])
data.write Bytes[0x40, 0x00]
data.write Bytes[0x18, 0x06]
continuation.write(Bytes[0x32, ucid.size + video_id.size + comment_id.size + 16])
continuation.write(Bytes[0x1a, ucid.size + video_id.size + comment_id.size + 14])
data.write(Bytes[0x32, ucid.size + video_id.size + comment_id.size + 16])
data.write(Bytes[0x1a, ucid.size + video_id.size + comment_id.size + 14])
continuation.write(Bytes[0x12, comment_id.size])
continuation.print(comment_id)
data.write_byte 0x12
VarInt.to_io(data, comment_id.size)
data.print comment_id
continuation.write(Bytes[0x22, 0x02, 0x08, 0x00]) # ??
data.write(Bytes[0x22, 0x02, 0x08, 0x00]) # ??
continuation.write(Bytes[ucid.size + video_id.size + 7])
continuation.write(Bytes[ucid.size])
continuation.print(ucid)
continuation.write(Bytes[0x32, video_id.size])
continuation.print(video_id)
continuation.write(Bytes[0x40, 0x01])
continuation.write(Bytes[0x48, 0x0a])
data.write(Bytes[ucid.size + video_id.size + 7])
data.write(Bytes[ucid.size])
data.print(ucid)
data.write(Bytes[0x32, video_id.size])
data.print(video_id)
data.write(Bytes[0x40, 0x01])
data.write(Bytes[0x48, 0x0a])
continuation.rewind
continuation = continuation.gets_to_end
continuation = Base64.urlsafe_encode(continuation.to_slice)
continuation = Base64.urlsafe_encode(data.to_slice)
continuation = URI.escape(continuation)
return continuation

50
src/invidious/helpers/utils.cr
View File

@ -266,50 +266,40 @@ def get_referer(env, fallback = "/", unroll = true)
return referer
end
def read_var_int(bytes)
num_read = 0
result = 0
struct VarInt
def self.from_io(io : IO, format = IO::ByteFormat::BigEndian) : Int32
result = 0_i32
num_read = 0
read = bytes[num_read]
if bytes.size == 1
result = bytes[0].to_i32
else
while ((read & 0b10000000) != 0)
read = bytes[num_read].to_u64
value = (read & 0b01111111)
result |= (value << (7 * num_read))
loop do
byte = io.read_byte
raise "Invalid VarInt" if !byte
value = byte & 0x7f
result |= value.to_i32 << (7 * num_read)
num_read += 1
if num_read > 5
raise "VarInt is too big"
end
break if byte & 0x80 == 0
raise "Invalid VarInt" if num_read > 5
end
result
end
return result
end
def self.to_io(io : IO, value : Int32)
io.write_byte 0x00 if value == 0x00
def write_var_int(value : Int)
bytes = [] of UInt8
value = value.to_u32
if value == 0
bytes = [0_u8]
else
while value != 0
temp = (value & 0b01111111).to_u8
value = value >> 7
byte = (value & 0x7f).to_u8
value >>= 7
if value != 0
temp |= 0b10000000
byte |= 0x80
end
bytes << temp
io.write_byte byte
end
end
return Slice.new(bytes.to_unsafe, bytes.size)
end
def sha256(text)

53
src/invidious/playlists.cr
View File

@ -157,37 +157,44 @@ def produce_playlist_url(id, index)
end
ucid = "VL" + id
meta = IO::Memory.new
meta.write(Bytes[0x08])
meta.write(write_var_int(index))
data = IO::Memory.new
data.write_byte 0x08
VarInt.to_io(data, index)
meta.rewind
meta = Base64.urlsafe_encode(meta.to_slice, false)
meta = "PT:#{meta}"
data.rewind
data = Base64.urlsafe_encode(data, false)
data = "PT:#{data}"
continuation = IO::Memory.new
continuation.write(Bytes[0x7a, meta.size])
continuation.print(meta)
continuation.write_byte 0x7a
VarInt.to_io(continuation, data.bytesize)
continuation.print data
continuation.rewind
meta = Base64.urlsafe_encode(continuation.to_slice)
meta = URI.escape(meta)
data = Base64.urlsafe_encode(continuation)
cursor = URI.escape(data)
continuation = IO::Memory.new
continuation.write(Bytes[0x12, ucid.size])
continuation.print(ucid)
continuation.write(Bytes[0x1a, meta.size])
continuation.print(meta)
data = IO::Memory.new
wrapper = IO::Memory.new
wrapper.write(Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02, continuation.size])
wrapper.print(continuation)
wrapper.rewind
data.write_byte 0x12
VarInt.to_io(data, ucid.bytesize)
data.print ucid
wrapper = Base64.urlsafe_encode(wrapper.to_slice)
wrapper = URI.escape(wrapper)
data.write_byte 0x1a
VarInt.to_io(data, cursor.bytesize)
data.print cursor
url = "/browse_ajax?continuation=#{wrapper}&gl=US&hl=en"
data.rewind
buffer = IO::Memory.new
buffer.write Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02]
VarInt.to_io(buffer, data.bytesize)
IO.copy data, buffer
continuation = Base64.urlsafe_encode(buffer)
continuation = URI.escape(continuation)
url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
return url
end

64
src/invidious/search.cr
View File

@ -374,45 +374,51 @@ end
def produce_channel_search_url(ucid, query, page)
page = "#{page}"
meta = IO::Memory.new
meta.write(Bytes[0x12, 0x06])
meta.print("search")
data = IO::Memory.new
data.write_byte 0x12
data.write_byte 0x06
data.print "search"
meta.write(Bytes[0x30, 0x02])
meta.write(Bytes[0x38, 0x01])
meta.write(Bytes[0x60, 0x01])
meta.write(Bytes[0x6a, 0x00])
meta.write(Bytes[0xb8, 0x01, 0x00])
data.write Bytes[0x30, 0x02]
data.write Bytes[0x38, 0x01]
data.write Bytes[0x60, 0x01]
data.write Bytes[0x6a, 0x00]
data.write Bytes[0xb8, 0x01, 0x00]
meta.write(Bytes[0x7a, page.size])
meta.print(page)
data.write_byte 0x7a
VarInt.to_io(data, page.bytesize)
data.print page
meta.rewind
meta = Base64.urlsafe_encode(meta.to_slice)
meta = URI.escape(meta)
data.rewind
data = Base64.urlsafe_encode(data)
continuation = URI.escape(data)
continuation = IO::Memory.new
continuation.write(Bytes[0x12, ucid.size])
continuation.print(ucid)
data = IO::Memory.new
continuation.write(Bytes[0x1a, meta.size])
continuation.print(meta)
data.write_byte 0x12
VarInt.to_io(data, ucid.bytesize)
data.print ucid
continuation.write(Bytes[0x5a, query.size])
continuation.print(query)
data.write_byte 0x1a
VarInt.to_io(data, continuation.bytesize)
data.print continuation
continuation.rewind
continuation = continuation.gets_to_end
data.write_byte 0x5a
VarInt.to_io(data, query.bytesize)
data.print query
wrapper = IO::Memory.new
wrapper.write(Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02, continuation.size])
wrapper.print(continuation)
wrapper.rewind
data.rewind
wrapper = Base64.urlsafe_encode(wrapper.to_slice)
wrapper = URI.escape(wrapper)
buffer = IO::Memory.new
buffer.write Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02]
VarInt.to_io(buffer, data.bytesize)
url = "/browse_ajax?continuation=#{wrapper}&gl=US&hl=en"
IO.copy data, buffer
continuation = Base64.urlsafe_encode(buffer)
continuation = URI.escape(continuation)
url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
return url
end