From 99bf5197812b470f17e66aec2ca6eccc55ca6e15 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 1 Dec 2022 20:09:31 +0100
Subject: [PATCH 01/17] shards: Bump protodec to v0.1.5

---
 shard.lock | 2 +-
 shard.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/shard.lock b/shard.lock
index cdce1160..235e4c25 100644
--- a/shard.lock
+++ b/shard.lock
@@ -34,7 +34,7 @@ shards:
 
   protodec:
     git: https://github.com/iv-org/protodec.git
-    version: 0.1.4
+    version: 0.1.5
 
   radix:
     git: https://github.com/luislavena/radix.git
diff --git a/shard.yml b/shard.yml
index 9c9b0d37..7ee0bb2a 100644
--- a/shard.yml
+++ b/shard.yml
@@ -24,7 +24,7 @@ dependencies:
     version: ~> 0.6.1
   protodec:
     github: iv-org/protodec
-    version: ~> 0.1.4
+    version: ~> 0.1.5
   lsquic:
     github: iv-org/lsquic.cr
     version: ~> 2.18.1-2

From fbcce57ce29b05c234c0c31b5f179d861e143260 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 11 Nov 2022 01:31:32 +0100
Subject: [PATCH 02/17] channel: use extractor utils to parse tabs (+ code
 cleaning)

---
 src/invidious/channels/about.cr | 54 ++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index 4c442959..bb9bd8c7 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -100,34 +100,40 @@ def get_about_info(ucid, locale) : AboutChannel
   total_views = 0_i64
   joined = Time.unix(0)
 
-  tabs = [] of String
+  tab_names = [] of String
 
-  tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a?
-  if !tabs_json.nil?
-    # Retrieve information from the tabs array. The index we are looking for varies between channels.
-    tabs_json.each do |node|
-      # Try to find the about section which is located in only one of the tabs.
-      channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]?
-        .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]?
-          .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]?
+  if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
+    # Get the name of the tabs available on this channel
+    tab_names = tabs_json.as_a
+      .compact_map(&.dig?("tabRenderer", "title").try &.as_s.downcase)
 
-      if !channel_about_meta.nil?
-        total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
+    # Get the currently active tab ("About")
+    about_tab = extract_selected_tab(tabs_json)
 
-        # The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
-        joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s }
-          .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
+    # Try to find the about metadata section
+    channel_about_meta = about_tab.dig?(
+      "content",
+      "sectionListRenderer", "contents", 0,
+      "itemSectionRenderer", "contents", 0,
+      "channelAboutFullMetadataRenderer"
+    )
 
-        # Normal Auto-generated channels
-        # https://support.google.com/youtube/answer/2579942
-        # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
-        if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) &&
-           (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube"
-          auto_generated = true
-        end
-      end
+    if !channel_about_meta.nil?
+      total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
+
+      # The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
+      joined = extract_text(channel_about_meta["joinedDateText"]?)
+        .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
+
+      # Normal Auto-generated channels
+      # https://support.google.com/youtube/answer/2579942
+      # For auto-generated channels, channel_about_meta only has
+      # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
+      auto_generated = (
+        (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
+           extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube"
+      )
     end
-    tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase)
   end
 
   sub_count = initdata
@@ -148,7 +154,7 @@ def get_about_info(ucid, locale) : AboutChannel
     joined: joined,
     is_family_friendly: is_family_friendly,
     allowed_regions: allowed_regions,
-    tabs: tabs,
+    tabs: tab_names,
     verified: author_verified || false,
   )
 end

From 9588fcb5d1dd90e8591ed53a342727a0df6923c4 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sat, 3 Dec 2022 22:39:24 +0100
Subject: [PATCH 03/17] frontend: remove paging on channel videos

---
 src/invidious/routes/channels.cr |  5 +----
 src/invidious/views/channel.ecr  | 18 +++---------------
 2 files changed, 4 insertions(+), 19 deletions(-)

diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index c6e02cbd..f26f29f5 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -12,9 +12,6 @@ module Invidious::Routes::Channels
     end
     locale, user, subscriptions, continuation, ucid, channel = data
 
-    page = env.params.query["page"]?.try &.to_i?
-    page ||= 1
-
     sort_by = env.params.query["sort_by"]?.try &.downcase
 
     if channel.auto_generated
@@ -35,7 +32,7 @@ module Invidious::Routes::Channels
       sort_options = {"newest", "oldest", "popular"}
       sort_by ||= "newest"
 
-      count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+      count, items = get_60_videos(channel.ucid, channel.author, 1, channel.auto_generated, sort_by)
     end
 
     templated "channel"
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index dea86abe..878587d4 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -90,7 +90,7 @@
                     <% if sort_by == sort %>
                         <b><%= translate(locale, sort) %></b>
                     <% else %>
-                        <a href="/channel/<%= ucid %>?page=<%= page %>&sort_by=<%= sort %>">
+                        <a href="/channel/<%= ucid %>?sort_by=<%= sort %>">
                             <%= translate(locale, sort) %>
                         </a>
                     <% end %>
@@ -111,19 +111,7 @@
 </div>
 
 <div class="pure-g h-box">
-    <div class="pure-u-1 pure-u-lg-1-5">
-        <% if page > 1 %>
-            <a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
-                <%= translate(locale, "Previous page") %>
-            </a>
-        <% end %>
-    </div>
+    <div class="pure-u-1 pure-u-lg-1-5"></div>
     <div class="pure-u-1 pure-u-lg-3-5"></div>
-    <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
-        <% if count == 60 %>
-            <a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
-                <%= translate(locale, "Next page") %>
-            </a>
-        <% end %>
-    </div>
+    <div class="pure-u-1 pure-u-lg-1-5"></div>
 </div>

From bdc51cd20fd2df99c2fe5ddc281aada86000a783 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 10 Nov 2022 23:32:51 +0100
Subject: [PATCH 04/17] extractors: separate 'extract' and 'parse' logic

---
 src/invidious/channels/playlists.cr    |  2 +-
 src/invidious/search/processors.cr     |  2 +-
 src/invidious/yt_backend/extractors.cr | 54 +++++++++++++++-----------
 3 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index d5628f6a..e6c0a1d5 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -8,7 +8,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
 
     items = [] of SearchItem
     continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
-      extract_item(item, author, ucid).try { |t| items << t }
+      parse_item(item, author, ucid).try { |t| items << t }
     }
 
     continuation = continuation_items.as_a.last["continuationItemRenderer"]?
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index d1409c06..683a4a7e 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -37,7 +37,7 @@ module Invidious::Search
 
       items = [] of SearchItem
       continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
-        extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
+        parse_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
       end
 
       return items
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index edc722cf..a4b20d04 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -20,6 +20,8 @@ private ITEM_PARSERS = {
   Parsers::ReelItemRendererParser,
 }
 
+private alias InitialData = Hash(String, JSON::Any)
+
 record AuthorFallback, name : String, id : String
 
 # Namespace for logic relating to parsing InnerTube data into various datastructs.
@@ -348,7 +350,7 @@ private module Parsers
       raw_contents = content_container["items"]?.try &.as_a
       if !raw_contents.nil?
         raw_contents.each do |item|
-          result = extract_item(item)
+          result = parse_item(item)
           if !result.nil?
             contents << result
           end
@@ -510,7 +512,7 @@ private module Extractors
   # }]
   #
   module YouTubeTabs
-    def self.process(initial_data : Hash(String, JSON::Any))
+    def self.process(initial_data : InitialData)
       if target = initial_data["twoColumnBrowseResultsRenderer"]?
         self.extract(target)
       end
@@ -575,7 +577,7 @@ private module Extractors
   # }
   #
   module SearchResults
-    def self.process(initial_data : Hash(String, JSON::Any))
+    def self.process(initial_data : InitialData)
       if target = initial_data["twoColumnSearchResultsRenderer"]?
         self.extract(target)
       end
@@ -608,8 +610,8 @@ private module Extractors
   # The way they are structured is too varied to be accurately written down here.
   # However, they all eventually lead to an array of parsable items after traversing
   # through the JSON structure.
-  module Continuation
-    def self.process(initial_data : Hash(String, JSON::Any))
+  module ContinuationContent
+    def self.process(initial_data : InitialData)
       if target = initial_data["continuationContents"]?
         self.extract(target)
       elsif target = initial_data["appendContinuationItemsAction"]?
@@ -691,8 +693,7 @@ end
 
 # Parses an item from Youtube's JSON response into a more usable structure.
 # The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.
-def extract_item(item : JSON::Any, author_fallback : String? = "",
-                 author_id_fallback : String? = "")
+def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "")
   # We "allow" nil values but secretly use empty strings instead. This is to save us the
   # hassle of modifying every author_fallback and author_id_fallback arg usage
   # which is more often than not nil.
@@ -702,24 +703,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "",
   # Each parser automatically validates the data given to see if the data is
   # applicable to itself. If not nil is returned and the next parser is attempted.
   ITEM_PARSERS.each do |parser|
-    LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
+    LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
 
     if result = parser.process(item, author_fallback)
-      LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}")
-
+      LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}")
       return result
     else
-      LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
+      LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
     end
   end
 end
 
 # Parses multiple items from YouTube's initial JSON response into a more usable structure.
 # The end result is an array of SearchItem.
-def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
-                  author_id_fallback : String? = nil) : Array(SearchItem)
-  items = [] of SearchItem
-
+#
+# This function yields the container so that items can be parsed separately.
+#
+def extract_items(initial_data : InitialData, &block)
   if unpackaged_data = initial_data["contents"]?.try &.as_h
   elsif unpackaged_data = initial_data["response"]?.try &.as_h
   elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h
@@ -727,24 +727,32 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
     unpackaged_data = initial_data
   end
 
-  # This is identical to the parser cycling of extract_item().
+  # This is identical to the parser cycling of parse_item().
   ITEM_CONTAINER_EXTRACTOR.each do |extractor|
     LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)")
 
     if container = extractor.process(unpackaged_data)
       LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"")
       # Extract items in container
-      container.each do |item|
-        if parsed_result = extract_item(item, author_fallback, author_id_fallback)
-          items << parsed_result
-        end
-      end
-
-      break
+      container.each { |item| yield item }
     else
       LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...")
     end
   end
+end
+
+# Wrapper using the block function above
+def extract_items(
+  initial_data : InitialData,
+  author_fallback : String? = nil,
+  author_id_fallback : String? = nil
+) : Array(SearchItem)
+  items = [] of SearchItem
+
+  extract_items(initial_data) do |item|
+    parsed = parse_item(item, author_fallback, author_id_fallback)
+    items << parsed if !parsed.nil?
+  end
 
   return items
 end

From ce7db8d2cb87111af15de2de9faf12aae38283bb Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sat, 5 Nov 2022 18:56:35 +0100
Subject: [PATCH 05/17] extractors: Add continuation token parser

---
 spec/invidious/hashtag_spec.cr               |  4 +-
 src/invidious/channels/playlists.cr          | 16 +-----
 src/invidious/hashtag.cr                     |  3 +-
 src/invidious/helpers/serialized_yt_data.cr  |  7 +++
 src/invidious/search/processors.cr           | 14 ++---
 src/invidious/yt_backend/extractors.cr       | 54 +++++++++++++++-----
 src/invidious/yt_backend/extractors_utils.cr | 27 ++--------
 7 files changed, 63 insertions(+), 62 deletions(-)

diff --git a/spec/invidious/hashtag_spec.cr b/spec/invidious/hashtag_spec.cr
index 77676878..266ec57b 100644
--- a/spec/invidious/hashtag_spec.cr
+++ b/spec/invidious/hashtag_spec.cr
@@ -4,7 +4,7 @@ Spectator.describe Invidious::Hashtag do
   it "parses richItemRenderer containers (test 1)" do
     # Enable mock
     test_content = load_mock("hashtag/martingarrix_page1")
-    videos = extract_items(test_content)
+    videos, _ = extract_items(test_content)
 
     expect(typeof(videos)).to eq(Array(SearchItem))
     expect(videos.size).to eq(60)
@@ -57,7 +57,7 @@ Spectator.describe Invidious::Hashtag do
   it "parses richItemRenderer containers (test 2)" do
     # Enable mock
     test_content = load_mock("hashtag/martingarrix_page2")
-    videos = extract_items(test_content)
+    videos, _ = extract_items(test_content)
 
     expect(typeof(videos)).to eq(Array(SearchItem))
     expect(videos.size).to eq(60)
diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index e6c0a1d5..0d46499a 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -1,18 +1,7 @@
 def fetch_channel_playlists(ucid, author, continuation, sort_by)
   if continuation
     response_json = YoutubeAPI.browse(continuation)
-    continuation_items = response_json["onResponseReceivedActions"]?
-      .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
-
-    return [] of SearchItem, nil if !continuation_items
-
-    items = [] of SearchItem
-    continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
-      parse_item(item, author, ucid).try { |t| items << t }
-    }
-
-    continuation = continuation_items.as_a.last["continuationItemRenderer"]?
-      .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
+    items, continuation = extract_items(response_json, author, ucid)
   else
     url = "/channel/#{ucid}/playlists?flow=list&view=1"
 
@@ -30,8 +19,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
     initial_data = extract_initial_data(response.body)
     return [] of SearchItem, nil if !initial_data
 
-    items = extract_items(initial_data, author, ucid)
-    continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]?
+    items, continuation = extract_items(initial_data, author, ucid)
   end
 
   return items, continuation
diff --git a/src/invidious/hashtag.cr b/src/invidious/hashtag.cr
index afe31a36..bc329205 100644
--- a/src/invidious/hashtag.cr
+++ b/src/invidious/hashtag.cr
@@ -8,7 +8,8 @@ module Invidious::Hashtag
     client_config = YoutubeAPI::ClientConfig.new(region: region)
     response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
 
-    return extract_items(response)
+    items, _ = extract_items(response)
+    return items
   end
 
   def generate_continuation(hashtag : String, cursor : Int)
diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr
index c52e2a0d..635f0984 100644
--- a/src/invidious/helpers/serialized_yt_data.cr
+++ b/src/invidious/helpers/serialized_yt_data.cr
@@ -265,4 +265,11 @@ class Category
   end
 end
 
+struct Continuation
+  getter token
+
+  def initialize(@token : String)
+  end
+end
+
 alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index 683a4a7e..7e909590 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -9,7 +9,8 @@ module Invidious::Search
       client_config = YoutubeAPI::ClientConfig.new(region: query.region)
       initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
 
-      return extract_items(initial_data)
+      items, _ = extract_items(initial_data)
+      return items
     end
 
     # Search a youtube channel
@@ -30,16 +31,7 @@ module Invidious::Search
       continuation = produce_channel_search_continuation(ucid, query.text, query.page)
       response_json = YoutubeAPI.browse(continuation)
 
-      continuation_items = response_json["onResponseReceivedActions"]?
-        .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
-
-      return [] of SearchItem if !continuation_items
-
-      items = [] of SearchItem
-      continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
-        parse_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
-      end
-
+      items, _ = extract_items(response_json, "", ucid)
       return items
     end
 
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index a4b20d04..baf52118 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -7,7 +7,7 @@ require "../helpers/serialized_yt_data"
 private ITEM_CONTAINER_EXTRACTOR = {
   Extractors::YouTubeTabs,
   Extractors::SearchResults,
-  Extractors::Continuation,
+  Extractors::ContinuationContent,
 }
 
 private ITEM_PARSERS = {
@@ -18,6 +18,7 @@ private ITEM_PARSERS = {
   Parsers::CategoryRendererParser,
   Parsers::RichItemRendererParser,
   Parsers::ReelItemRendererParser,
+  Parsers::ContinuationItemRendererParser,
 }
 
 private alias InitialData = Hash(String, JSON::Any)
@@ -347,14 +348,9 @@ private module Parsers
         content_container = item_contents["contents"]
       end
 
-      raw_contents = content_container["items"]?.try &.as_a
-      if !raw_contents.nil?
-        raw_contents.each do |item|
-          result = parse_item(item)
-          if !result.nil?
-            contents << result
-          end
-        end
+      content_container["items"]?.try &.as_a.each do |item|
+        result = parse_item(item, author_fallback.name, author_fallback.id)
+        contents << result if result.is_a?(SearchItem)
       end
 
       Category.new({
@@ -477,6 +473,35 @@ private module Parsers
       return {{@type.name}}
     end
   end
+
+  # Parses an InnerTube continuationItemRenderer into a Continuation.
+  # Returns nil when the given object isn't a continuationItemRenderer.
+  #
+  # continuationItemRenderer contains various metadata ued to load more
+  # content (i.e when the user scrolls down). The interesting bit is the
+  # protobuf object known as the "continutation token". Previously, those
+  # were generated from sratch, but recent (as of 11/2022) Youtube changes
+  # are forcing us to extract them from replies.
+  #
+  module ContinuationItemRendererParser
+    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+      if item_contents = item["continuationItemRenderer"]?
+        return self.parse(item_contents)
+      end
+    end
+
+    private def self.parse(item_contents)
+      token = item_contents
+        .dig?("continuationEndpoint", "continuationCommand", "token")
+        .try &.as_s
+
+      return Continuation.new(token) if token
+    end
+
+    def self.parser_name
+      return {{@type.name}}
+    end
+  end
 end
 
 # The following are the extractors for extracting an array of items from
@@ -746,13 +771,18 @@ def extract_items(
   initial_data : InitialData,
   author_fallback : String? = nil,
   author_id_fallback : String? = nil
-) : Array(SearchItem)
+) : {Array(SearchItem), String?}
   items = [] of SearchItem
+  continuation = nil
 
   extract_items(initial_data) do |item|
     parsed = parse_item(item, author_fallback, author_id_fallback)
-    items << parsed if !parsed.nil?
+
+    case parsed
+    when .is_a?(Continuation) then continuation = parsed.token
+    when .is_a?(SearchItem)   then items << parsed
+    end
   end
 
-  return items
+  return items, continuation
 end
diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr
index f8245160..0cb3c079 100644
--- a/src/invidious/yt_backend/extractors_utils.cr
+++ b/src/invidious/yt_backend/extractors_utils.cr
@@ -68,10 +68,10 @@ rescue ex
   return false
 end
 
-def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
-  extracted = extract_items(initial_data, author_fallback, author_id_fallback)
+def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo)
+  extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback)
 
-  target = [] of SearchItem
+  target = [] of (SearchItem | Continuation)
   extracted.each do |i|
     if i.is_a?(Category)
       i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
@@ -79,28 +79,11 @@ def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : Str
       target << i
     end
   end
-  return target.select(SearchVideo).map(&.as(SearchVideo))
+
+  return target.select(SearchVideo)
 end
 
 def extract_selected_tab(tabs)
   # Extract the selected tab from the array of tabs Youtube returns
   return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"]
 end
-
-def fetch_continuation_token(items : Array(JSON::Any))
-  # Fetches the continuation token from an array of items
-  return items.last["continuationItemRenderer"]?
-    .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
-end
-
-def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
-  # Fetches the continuation token from initial data
-  if initial_data["onResponseReceivedActions"]?
-    continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
-  else
-    tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
-    continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
-  end
-
-  return fetch_continuation_token(continuation_items.as_a)
-end

From 8e8ca4fcc5cfcb7bebc3f29440d6abc1de770513 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sat, 12 Nov 2022 00:04:27 +0100
Subject: [PATCH 06/17] Prepare to create a 'Channel' module

---
 src/invidious.cr                             | 9 ++++++++-
 src/invidious/jobs/notification_job.cr       | 4 ++--
 src/invidious/jobs/refresh_channels_job.cr   | 2 +-
 src/invidious/jobs/refresh_feeds_job.cr      | 2 +-
 src/invidious/jobs/subscribe_to_feeds_job.cr | 2 +-
 5 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/invidious.cr b/src/invidious.cr
index 2874cc71..5064f0b8 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -48,6 +48,13 @@ require "./invidious/search/*"
 require "./invidious/routes/**"
 require "./invidious/jobs/**"
 
+# Declare the base namespace for invidious
+module Invidious
+end
+
+# Simple alias to make code easier to read
+alias IV = Invidious
+
 CONFIG   = Config.load
 HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
 
@@ -172,7 +179,7 @@ if CONFIG.popular_enabled
   Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
 end
 
-CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32)
+CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
 Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
 
 Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr
index 2f525e08..b445107b 100644
--- a/src/invidious/jobs/notification_job.cr
+++ b/src/invidious/jobs/notification_job.cr
@@ -1,12 +1,12 @@
 class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
-  private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
+  private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
   private getter pg_url : URI
 
   def initialize(@connection_channel, @pg_url)
   end
 
   def begin
-    connections = [] of Channel(PQ::Notification)
+    connections = [] of ::Channel(PQ::Notification)
 
     PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
 
diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr
index 92681408..80812a63 100644
--- a/src/invidious/jobs/refresh_channels_job.cr
+++ b/src/invidious/jobs/refresh_channels_job.cr
@@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
     max_fibers = CONFIG.channel_threads
     lim_fibers = max_fibers
     active_fibers = 0
-    active_channel = Channel(Bool).new
+    active_channel = ::Channel(Bool).new
     backoff = 2.minutes
 
     loop do
diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr
index 4b52c959..4f8130df 100644
--- a/src/invidious/jobs/refresh_feeds_job.cr
+++ b/src/invidious/jobs/refresh_feeds_job.cr
@@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
   def begin
     max_fibers = CONFIG.feed_threads
     active_fibers = 0
-    active_channel = Channel(Bool).new
+    active_channel = ::Channel(Bool).new
 
     loop do
       db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr
index a431a48a..8584fb9c 100644
--- a/src/invidious/jobs/subscribe_to_feeds_job.cr
+++ b/src/invidious/jobs/subscribe_to_feeds_job.cr
@@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
     end
 
     active_fibers = 0
-    active_channel = Channel(Bool).new
+    active_channel = ::Channel(Bool).new
 
     loop do
       db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|

From c5ee2bfc0f5e485f91e53dedc879312c3e729be8 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 11 Nov 2022 00:44:24 +0100
Subject: [PATCH 07/17] channel: use YT API to fetch playlist items

---
 src/invidious/channels/playlists.cr | 40 +++++++++++++++--------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index 0d46499a..8fdac3a7 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -1,28 +1,30 @@
 def fetch_channel_playlists(ucid, author, continuation, sort_by)
   if continuation
-    response_json = YoutubeAPI.browse(continuation)
-    items, continuation = extract_items(response_json, author, ucid)
+    initial_data = YoutubeAPI.browse(continuation)
   else
-    url = "/channel/#{ucid}/playlists?flow=list&view=1"
+    params =
+      case sort_by
+      when "last", "last_added"
+        # Equivalent to "&sort=lad"
+        # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1}
+        "EglwbGF5bGlzdHMYBCABMAE%3D"
+      when "oldest", "oldest_created"
+        # formerly "&sort=da"
+        # Not available anymore :c or maybe ??
+        # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1}
+        "EglwbGF5bGlzdHMYAiABMAE%3D"
+        # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
+        # "EglwbGF5bGlzdHMYASABMAE%3D"
+      when "newest", "newest_created"
+        # Formerly "&sort=dd"
+        # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1}
+        "EglwbGF5bGlzdHMYAyABMAE%3D"
+      end
 
-    case sort_by
-    when "last", "last_added"
-      #
-    when "oldest", "oldest_created"
-      url += "&sort=da"
-    when "newest", "newest_created"
-      url += "&sort=dd"
-    else nil # Ignore
-    end
-
-    response = YT_POOL.client &.get(url)
-    initial_data = extract_initial_data(response.body)
-    return [] of SearchItem, nil if !initial_data
-
-    items, continuation = extract_items(initial_data, author, ucid)
+    initial_data = YoutubeAPI.browse(ucid, params: params || "")
   end
 
-  return items, continuation
+  return extract_items(initial_data, ucid, author)
 end
 
 # ## NOTE: DEPRECATED

From 2903e896ecf2404bf932438a33432125a6ad1fca Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 11 Nov 2022 20:26:34 +0100
Subject: [PATCH 08/17] channel: use YT API + extractors to fetch videos

---
 src/invidious/channels/channels.cr      | 65 ++++++++---------
 src/invidious/channels/playlists.cr     |  2 +-
 src/invidious/channels/videos.cr        | 94 ++++++++++++++++++-------
 src/invidious/routes/api/v1/channels.cr | 66 +++++++----------
 src/invidious/routes/channels.cr        |  4 +-
 5 files changed, 127 insertions(+), 104 deletions(-)

diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr
index e3d3d9ee..27369f12 100644
--- a/src/invidious/channels/channels.cr
+++ b/src/invidious/channels/channels.cr
@@ -180,11 +180,16 @@ def fetch_channel(ucid, pull_all_videos : Bool)
 
   LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
 
-  page = 1
+  channel = InvidiousChannel.new({
+    id:         ucid,
+    author:     author,
+    updated:    Time.utc,
+    deleted:    false,
+    subscribed: nil,
+  })
 
   LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
-  initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
-  videos = extract_videos(initial_data, author, ucid)
+  videos, continuation = IV::Channel::Tabs.get_videos(channel)
 
   LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
   rss.xpath_nodes("//feed/entry").each do |entry|
@@ -197,7 +202,9 @@ def fetch_channel(ucid, pull_all_videos : Bool)
     views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64?
     views ||= 0_i64
 
-    channel_video = videos.select { |video| video.id == video_id }[0]?
+    channel_video = videos
+      .select(SearchVideo)
+      .select(&.id.== video_id)[0]?
 
     length_seconds = channel_video.try &.length_seconds
     length_seconds ||= 0
@@ -235,30 +242,25 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   end
 
   if pull_all_videos
-    page += 1
-
-    ids = [] of String
-
     loop do
-      initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
-      videos = extract_videos(initial_data, author, ucid)
+      # Keep fetching videos using the continuation token retrieved earlier
+      videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
 
-      count = videos.size
-      videos = videos.map { |video| ChannelVideo.new({
-        id:                 video.id,
-        title:              video.title,
-        published:          video.published,
-        updated:            Time.utc,
-        ucid:               video.ucid,
-        author:             video.author,
-        length_seconds:     video.length_seconds,
-        live_now:           video.live_now,
-        premiere_timestamp: video.premiere_timestamp,
-        views:              video.views,
-      }) }
-
-      videos.each do |video|
-        ids << video.id
+      count = 0
+      videos.select(SearchVideo).each do |video|
+        count += 1
+        video = ChannelVideo.new({
+          id:                 video.id,
+          title:              video.title,
+          published:          video.published,
+          updated:            Time.utc,
+          ucid:               video.ucid,
+          author:             video.author,
+          length_seconds:     video.length_seconds,
+          live_now:           video.live_now,
+          premiere_timestamp: video.premiere_timestamp,
+          views:              video.views,
+        })
 
         # We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
         # so since they don't provide a published date here we can safely ignore them.
@@ -269,17 +271,10 @@ def fetch_channel(ucid, pull_all_videos : Bool)
       end
 
       break if count < 25
-      page += 1
+      sleep 500.milliseconds
     end
   end
 
-  channel = InvidiousChannel.new({
-    id:         ucid,
-    author:     author,
-    updated:    Time.utc,
-    deleted:    false,
-    subscribed: nil,
-  })
-
+  channel.updated = Time.utc
   return channel
 end
diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index 8fdac3a7..772eecb9 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -24,7 +24,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
     initial_data = YoutubeAPI.browse(ucid, params: params || "")
   end
 
-  return extract_items(initial_data, ucid, author)
+  return extract_items(initial_data, author, ucid)
 end
 
 # ## NOTE: DEPRECATED
diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index b495e597..23ad4e02 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -16,6 +16,14 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
     .try { |i| Base64.urlsafe_encode(i) }
     .try { |i| URI.encode_www_form(i) }
 
+  sort_by_numerical =
+    case sort_by
+    when "newest"  then 1_i64
+    when "popular" then 2_i64
+    when "oldest"  then 3_i64 # Broken as of 10/2022 :c
+    else                1_i64 # Fallback to "newest"
+    end
+
   object_inner_1 = {
     "110:embedded" => {
       "3:embedded" => {
@@ -24,7 +32,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
             "1:string" => object_inner_2_encoded,
             "2:string" => "00000000-0000-0000-0000-000000000000",
           },
-          "3:varint" => 1_i64,
+          "3:varint" => sort_by_numerical,
         },
       },
     },
@@ -52,34 +60,66 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
   return continuation
 end
 
-def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
-  continuation = produce_channel_videos_continuation(ucid, page,
-    auto_generated: auto_generated, sort_by: sort_by, v2: true)
-
-  return YoutubeAPI.browse(continuation)
-end
-
-def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
-  videos = [] of SearchVideo
-
-  # 2.times do |i|
-  # initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
-  initial_data = get_channel_videos_response(ucid, 1, auto_generated: auto_generated, sort_by: sort_by)
-  videos = extract_videos(initial_data, author, ucid)
-  # end
-
-  return videos.size, videos
-end
-
-def get_latest_videos(ucid)
-  initial_data = get_channel_videos_response(ucid)
-  author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
-
-  return extract_videos(initial_data, author, ucid)
-end
-
 # Used in bypass_captcha_job.cr
 def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
   continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2)
   return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
 end
+
+module Invidious::Channel::Tabs
+  extend self
+
+  # -------------------
+  #  Regular videos
+  # -------------------
+
+  def make_initial_video_ctoken(ucid, sort_by) : String
+    return produce_channel_videos_continuation(ucid, sort_by: sort_by)
+  end
+
+  # Wrapper for AboutChannel, as we still need to call get_videos with
+  # an author name and ucid directly (e.g in RSS feeds).
+  # TODO: figure out how to get rid of that
+  def get_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
+    return get_videos(
+      channel.author, channel.ucid,
+      continuation: continuation, sort_by: sort_by
+    )
+  end
+
+  # Wrapper for InvidiousChannel, as we still need to call get_videos with
+  # an author name and ucid directly (e.g in RSS feeds).
+  # TODO: figure out how to get rid of that
+  def get_videos(channel : InvidiousChannel, *, continuation : String? = nil, sort_by = "newest")
+    return get_videos(
+      channel.author, channel.id,
+      continuation: continuation, sort_by: sort_by
+    )
+  end
+
+  def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
+    continuation ||= make_initial_video_ctoken(ucid, sort_by)
+    initial_data = YoutubeAPI.browse(continuation: continuation)
+
+    return extract_items(initial_data, author, ucid)
+  end
+
+  def get_60_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
+    if continuation.nil?
+      # Fetch the first "page" of video
+      items, next_continuation = get_videos(channel, sort_by: sort_by)
+    else
+      # Fetch a "page" of videos using the given continuation token
+      items, next_continuation = get_videos(channel, continuation: continuation)
+    end
+
+    # If there is more to load, then load a second "page"
+    # and replace the previous continuation token
+    if !next_continuation.nil?
+      items_2, next_continuation = get_videos(channel, continuation: next_continuation)
+      items.concat items_2
+    end
+
+    return items, next_continuation
+  end
+end
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 6b81c546..72d9ae5f 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -5,8 +5,6 @@ module Invidious::Routes::API::V1::Channels
     env.response.content_type = "application/json"
 
     ucid = env.params.url["ucid"]
-    sort_by = env.params.query["sort_by"]?.try &.downcase
-    sort_by ||= "newest"
 
     begin
       channel = get_about_info(ucid, locale)
@@ -19,16 +17,13 @@ module Invidious::Routes::API::V1::Channels
       return error_json(500, ex)
     end
 
-    page = 1
-    if channel.auto_generated
-      videos = [] of SearchVideo
-      count = 0
-    else
-      begin
-        count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
-      rescue ex
-        return error_json(500, ex)
-      end
+    # Retrieve "sort by" setting from URL parameters
+    sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+
+    begin
+      videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
+    rescue ex
+      return error_json(500, ex)
     end
 
     JSON.build do |json|
@@ -134,25 +129,11 @@ module Invidious::Routes::API::V1::Channels
   end
 
   def self.latest(env)
-    locale = env.get("preferences").as(Preferences).locale
+    # Remove parameters that could affect this endpoint's behavior
+    env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
+    env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
 
-    env.response.content_type = "application/json"
-
-    ucid = env.params.url["ucid"]
-
-    begin
-      videos = get_latest_videos(ucid)
-    rescue ex
-      return error_json(500, ex)
-    end
-
-    JSON.build do |json|
-      json.array do
-        videos.each do |video|
-          video.to_json(locale, json)
-        end
-      end
-    end
+    return self.videos(env)
   end
 
   def self.videos(env)
@@ -161,11 +142,6 @@ module Invidious::Routes::API::V1::Channels
     env.response.content_type = "application/json"
 
     ucid = env.params.url["ucid"]
-    page = env.params.query["page"]?.try &.to_i?
-    page ||= 1
-    sort_by = env.params.query["sort"]?.try &.downcase
-    sort_by ||= env.params.query["sort_by"]?.try &.downcase
-    sort_by ||= "newest"
 
     begin
       channel = get_about_info(ucid, locale)
@@ -178,17 +154,27 @@ module Invidious::Routes::API::V1::Channels
       return error_json(500, ex)
     end
 
+    # Retrieve some URL parameters
+    sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+    continuation = env.params.query["continuation"]?
+
     begin
-      count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+      videos, next_continuation = Channel::Tabs.get_60_videos(
+        channel, continuation: continuation, sort_by: sort_by
+      )
     rescue ex
       return error_json(500, ex)
     end
 
-    JSON.build do |json|
-      json.array do
-        videos.each do |video|
-          video.to_json(locale, json)
+    return JSON.build do |json|
+      json.object do
+        json.field "videos" do
+          json.array do
+            videos.each &.to_json(locale, json)
+          end
         end
+
+        json.field "continuation", next_continuation if next_continuation
       end
     end
   end
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index f26f29f5..2773deb7 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -32,7 +32,9 @@ module Invidious::Routes::Channels
       sort_options = {"newest", "oldest", "popular"}
       sort_by ||= "newest"
 
-      count, items = get_60_videos(channel.ucid, channel.author, 1, channel.auto_generated, sort_by)
+      items, continuation = Channel::Tabs.get_60_videos(
+        channel, continuation: continuation, sort_by: sort_by
+      )
     end
 
     templated "channel"

From 52ef89f02d0ab29fd0f218abc4051328b3d96809 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sat, 12 Nov 2022 00:09:03 +0100
Subject: [PATCH 09/17] channel: Add support for shorts and livestreams
 (backend only)

---
 src/invidious/channels/videos.cr | 50 ++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index 23ad4e02..bea406c1 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -122,4 +122,54 @@ module Invidious::Channel::Tabs
 
     return items, next_continuation
   end
+
+  # -------------------
+  #  Shorts
+  # -------------------
+
+  def get_shorts(channel : AboutChannel, continuation : String? = nil)
+    if continuation.nil?
+      # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
+      # TODO: try to extract the continuation tokens that allows other sorting options
+      initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
+    else
+      initial_data = YoutubeAPI.browse(continuation: continuation)
+    end
+
+    return extract_items(initial_data, channel.author, channel.ucid)
+  end
+
+  # -------------------
+  #  Livestreams
+  # -------------------
+
+  def get_livestreams(channel : AboutChannel, continuation : String? = nil)
+    if continuation.nil?
+      # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams"
+      initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D")
+    else
+      initial_data = YoutubeAPI.browse(continuation: continuation)
+    end
+
+    return extract_items(initial_data, channel.author, channel.ucid)
+  end
+
+  def get_60_livestreams(channel : AboutChannel, continuation : String? = nil)
+    if continuation.nil?
+      # Fetch the first "page" of streams
+      items, next_continuation = get_livestreams(channel)
+    else
+      # Fetch a "page" of streams using the given continuation token
+      items, next_continuation = get_livestreams(channel, continuation: continuation)
+    end
+
+    # If there is more to load, then load a second "page"
+    # and replace the previous continuation token
+    if !next_continuation.nil?
+      items_2, next_continuation = get_livestreams(channel, continuation: next_continuation)
+      items.concat items_2
+    end
+
+    return items, next_continuation
+  end
 end

From 5d6abd5301b14c24475bf7ad477a43c60ff78993 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 1 Dec 2022 23:01:31 +0100
Subject: [PATCH 10/17] extractors: Fix ReelItemRendererParser

---
 src/invidious/yt_backend/extractors.cr | 32 ++++++++++++++++----------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index baf52118..bca0dcbd 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -382,7 +382,9 @@ private module Parsers
     end
 
     private def self.parse(item_contents, author_fallback)
-      return VideoRendererParser.process(item_contents, author_fallback)
+      child = VideoRendererParser.process(item_contents, author_fallback)
+      child ||= ReelItemRendererParser.process(item_contents, author_fallback)
+      return child
     end
 
     def self.parser_name
@@ -406,12 +408,18 @@ private module Parsers
     private def self.parse(item_contents, author_fallback)
       video_id = item_contents["videoId"].as_s
 
-      video_details_container = item_contents.dig(
-        "navigationEndpoint", "reelWatchEndpoint",
-        "overlay", "reelPlayerOverlayRenderer",
-        "reelPlayerHeaderSupportedRenderers",
-        "reelPlayerHeaderRenderer"
-      )
+      begin
+        video_details_container = item_contents.dig(
+          "navigationEndpoint", "reelWatchEndpoint",
+          "overlay", "reelPlayerOverlayRenderer",
+          "reelPlayerHeaderSupportedRenderers",
+          "reelPlayerHeaderRenderer"
+        )
+      rescue ex : KeyError
+        # Extract key name from original message
+        key = /"([^"]+)"/.match(ex.message || "").try &.[1]?
+        raise BrokenTubeException.new(key || "reelPlayerOverlayRenderer")
+      end
 
       # Author infos
 
@@ -434,9 +442,9 @@ private module Parsers
 
       # View count
 
-      view_count_text = video_details_container.dig?("viewCountText", "simpleText")
-      view_count_text ||= video_details_container
-        .dig?("viewCountText", "accessibility", "accessibilityData", "label")
+      # View count used to be in the reelWatchEndpoint, but that changed?
+      view_count_text = item_contents.dig?("viewCountText", "simpleText")
+      view_count_text ||= video_details_container.dig?("viewCountText", "simpleText")
 
       view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
 
@@ -448,8 +456,8 @@ private module Parsers
 
       regex_match = /- (?<min>\d+ minutes? )?(?<sec>\d+ seconds?)+ -/.match(a11y_data)
 
-      minutes = regex_match.try &.["min"].to_i(strict: false) || 0
-      seconds = regex_match.try &.["sec"].to_i(strict: false) || 0
+      minutes = regex_match.try &.["min"]?.try &.to_i(strict: false) || 0
+      seconds = regex_match.try &.["sec"]?.try &.to_i(strict: false) || 0
 
       duration = (minutes*60 + seconds)
 

From 6c9754e66316d903ed4f89d2cd59cd82940509f5 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Wed, 30 Nov 2022 00:29:48 +0100
Subject: [PATCH 11/17] frontend: Add support for shorts and livestreams

---
 locales/en-US.json                     |  9 +++--
 src/invidious/channels/about.cr        | 10 ++++-
 src/invidious/frontend/channel_page.cr | 43 ++++++++++++++++++++
 src/invidious/routes/channels.cr       | 54 ++++++++++++++++++++++++--
 src/invidious/routing.cr               |  4 +-
 src/invidious/views/channel.ecr        | 30 +++++---------
 src/invidious/views/community.ecr      | 15 +------
 src/invidious/views/playlists.ecr      | 14 +------
 8 files changed, 124 insertions(+), 55 deletions(-)
 create mode 100644 src/invidious/frontend/channel_page.cr

diff --git a/locales/en-US.json b/locales/en-US.json
index 5554b928..44b40c24 100644
--- a/locales/en-US.json
+++ b/locales/en-US.json
@@ -404,9 +404,7 @@
     "`x` marked it with a ❤": "`x` marked it with a ❤",
     "Audio mode": "Audio mode",
     "Video mode": "Video mode",
-    "Videos": "Videos",
     "Playlists": "Playlists",
-    "Community": "Community",
     "search_filters_title": "Filters",
     "search_filters_date_label": "Upload date",
     "search_filters_date_option_none": "Any date",
@@ -472,5 +470,10 @@
     "crash_page_read_the_faq": "read the <a href=\"`x`\">Frequently Asked Questions (FAQ)</a>",
     "crash_page_search_issue": "searched for <a href=\"`x`\">existing issues on GitHub</a>",
     "crash_page_report_issue": "If none of the above helped, please <a href=\"`x`\">open a new issue on GitHub</a> (preferably in English) and include the following text in your message (do NOT translate that text):",
-    "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>"
+    "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>",
+    "channel_tab_videos_label": "Videos",
+    "channel_tab_shorts_label": "Shorts",
+    "channel_tab_streams_label": "Livestreams",
+    "channel_tab_playlists_label": "Playlists",
+    "channel_tab_community_label": "Community"
 }
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index bb9bd8c7..09c3427a 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -104,8 +104,14 @@ def get_about_info(ucid, locale) : AboutChannel
 
   if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
     # Get the name of the tabs available on this channel
-    tab_names = tabs_json.as_a
-      .compact_map(&.dig?("tabRenderer", "title").try &.as_s.downcase)
+    tab_names = tabs_json.as_a.compact_map do |entry|
+      name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
+
+      # This is a small fix to not add extra code on the HTML side
+      # I.e, the URL for the "live" tab is .../streams, so use "streams"
+      # everywhere for the sake of simplicity
+      (name == "live") ? "streams" : name
+    end
 
     # Get the currently active tab ("About")
     about_tab = extract_selected_tab(tabs_json)
diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr
new file mode 100644
index 00000000..7ac0e071
--- /dev/null
+++ b/src/invidious/frontend/channel_page.cr
@@ -0,0 +1,43 @@
+module Invidious::Frontend::ChannelPage
+  extend self
+
+  enum TabsAvailable
+    Videos
+    Shorts
+    Streams
+    Playlists
+    Community
+  end
+
+  def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
+    return String.build(1500) do |str|
+      base_url = "/channel/#{channel.ucid}"
+
+      TabsAvailable.each do |tab|
+        # Ignore playlists, as it is not supported for auto-generated channels yet
+        next if (tab.playlists? && channel.auto_generated)
+
+        tab_name = tab.to_s.downcase
+
+        if channel.tabs.includes? tab_name
+          str << %(<div class="pure-u-1 pure-md-1-3">\n)
+
+          if tab == selected_tab
+            str << "\t<b>"
+            str << translate(locale, "channel_tab_#{tab_name}_label")
+            str << "</b>\n"
+          else
+            # Video tab doesn't have the last path component
+            url = tab.videos? ? base_url : "#{base_url}/#{tab_name}"
+
+            str << %(\t<a href=") << url << %(">)
+            str << translate(locale, "channel_tab_#{tab_name}_label")
+            str << "</a>\n"
+          end
+
+          str << "</div>"
+        end
+      end
+    end
+  end
+end
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index 2773deb7..78b38341 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -18,7 +18,7 @@ module Invidious::Routes::Channels
       sort_options = {"last", "oldest", "newest"}
       sort_by ||= "last"
 
-      items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+      items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
       items.uniq! do |item|
         if item.responds_to?(:title)
           item.title
@@ -32,11 +32,59 @@ module Invidious::Routes::Channels
       sort_options = {"newest", "oldest", "popular"}
       sort_by ||= "newest"
 
-      items, continuation = Channel::Tabs.get_60_videos(
+      # Fetch items and continuation token
+      items, next_continuation = Channel::Tabs.get_videos(
         channel, continuation: continuation, sort_by: sort_by
       )
     end
 
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
+    templated "channel"
+  end
+
+  def self.shorts(env)
+    data = self.fetch_basic_information(env)
+    return data if !data.is_a?(Tuple)
+
+    locale, user, subscriptions, continuation, ucid, channel = data
+
+    if !channel.tabs.includes? "shorts"
+      return env.redirect "/channel/#{channel.ucid}"
+    end
+
+    # TODO: support sort option for shorts
+    sort_by = ""
+    sort_options = [] of String
+
+    # Fetch items and continuation token
+    items, next_continuation = Channel::Tabs.get_shorts(
+      channel, continuation: continuation
+    )
+
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
+    templated "channel"
+  end
+
+  def self.streams(env)
+    data = self.fetch_basic_information(env)
+    return data if !data.is_a?(Tuple)
+
+    locale, user, subscriptions, continuation, ucid, channel = data
+
+    if !channel.tabs.includes? "streams"
+      return env.redirect "/channel/#{channel.ucid}"
+    end
+
+    # TODO: support sort option for livestreams
+    sort_by = ""
+    sort_options = [] of String
+
+    # Fetch items and continuation token
+    items, next_continuation = Channel::Tabs.get_60_livestreams(
+      channel, continuation: continuation
+    )
+
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
     templated "channel"
   end
 
@@ -124,7 +172,7 @@ module Invidious::Routes::Channels
     end
 
     selected_tab = env.request.path.split("/")[-1]
-    if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
+    if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab
       url = "/channel/#{ucid}/#{selected_tab}"
     else
       url = "/channel/#{ucid}"
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index f409f13c..08739c3d 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -115,6 +115,8 @@ module Invidious::Routing
     get "/channel/:ucid", Routes::Channels, :home
     get "/channel/:ucid/home", Routes::Channels, :home
     get "/channel/:ucid/videos", Routes::Channels, :videos
+    get "/channel/:ucid/shorts", Routes::Channels, :shorts
+    get "/channel/:ucid/streams", Routes::Channels, :streams
     get "/channel/:ucid/playlists", Routes::Channels, :playlists
     get "/channel/:ucid/community", Routes::Channels, :community
     get "/channel/:ucid/about", Routes::Channels, :about
@@ -122,7 +124,7 @@ module Invidious::Routing
     get "/user/:user/live", Routes::Channels, :live
     get "/c/:user/live", Routes::Channels, :live
 
-    ["", "/videos", "/playlists", "/community", "/about"].each do |path|
+    {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path|
       # /c/LinusTechTips
       get "/c/:user#{path}", Routes::Channels, :brand_redirect
       # /user/linustechtips | Not always the same as /c/
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 878587d4..f6cc3340 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -64,23 +64,8 @@
                 <a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a>
             <% end %>
         </div>
-        <% if !channel.auto_generated %>
-            <div class="pure-u-1 pure-md-1-3">
-                <b><%= translate(locale, "Videos") %></b>
-            </div>
-        <% end %>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if channel.auto_generated %>
-                <b><%= translate(locale, "Playlists") %></b>
-            <% else %>
-                <a href="/channel/<%= ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
-            <% end %>
-        </div>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if channel.tabs.includes? "community" %>
-                <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
-            <% end %>
-        </div>
+
+        <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
     </div>
     <div class="pure-u-1-3"></div>
     <div class="pure-u-1-3">
@@ -111,7 +96,12 @@
 </div>
 
 <div class="pure-g h-box">
-    <div class="pure-u-1 pure-u-lg-1-5"></div>
-    <div class="pure-u-1 pure-u-lg-3-5"></div>
-    <div class="pure-u-1 pure-u-lg-1-5"></div>
+    <div class="pure-u-1 pure-u-md-4-5"></div>
+    <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
+        <% if next_continuation %>
+            <a href="/channel/<%= ucid %>/<%= selected_tab %>?continuation=<%= next_continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+                <%= translate(locale, "Next page") %>
+            </a>
+        <% end %>
+    </div>
 </div>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index 3bc29e55..e467a679 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -50,19 +50,8 @@
                 <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
             <% end %>
         </div>
-        <% if !channel.auto_generated %>
-            <div class="pure-u-1 pure-md-1-3">
-                <a href="/channel/<%= channel.ucid %>"><%= translate(locale, "Videos") %></a>
-            </div>
-        <% end %>
-        <div class="pure-u-1 pure-md-1-3">
-            <a href="/channel/<%= channel.ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
-        </div>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if channel.tabs.includes? "community" %>
-                <b><%= translate(locale, "Community") %></b>
-            <% end %>
-        </div>
+
+        <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Community) %>
     </div>
     <div class="pure-u-2-3"></div>
 </div>
diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr
index c8718e7b..56d25ef5 100644
--- a/src/invidious/views/playlists.ecr
+++ b/src/invidious/views/playlists.ecr
@@ -54,19 +54,7 @@
             <% end %>
         </div>
 
-        <div class="pure-u-1 pure-md-1-3">
-            <a href="/channel/<%= ucid %>"><%= translate(locale, "Videos") %></a>
-        </div>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if !channel.auto_generated %>
-                <b><%= translate(locale, "Playlists") %></b>
-            <% end %>
-        </div>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if channel.tabs.includes? "community" %>
-                <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
-            <% end %>
-        </div>
+        <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Playlists) %>
     </div>
     <div class="pure-u-1-3"></div>
     <div class="pure-u-1-3">

From 40c666cab22693cf9d31895978ae4b4356e6579b Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 4 Dec 2022 19:24:51 +0100
Subject: [PATCH 12/17] api: Add support for shorts and livestreams

---
 src/invidious/routes/api/v1/channels.cr | 118 ++++++++++++++++++------
 src/invidious/routing.cr                |   3 +
 2 files changed, 92 insertions(+), 29 deletions(-)

diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 72d9ae5f..4e92b54e 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -1,11 +1,7 @@
 module Invidious::Routes::API::V1::Channels
-  def self.home(env)
-    locale = env.get("preferences").as(Preferences).locale
-
-    env.response.content_type = "application/json"
-
-    ucid = env.params.url["ucid"]
-
+  # Macro to avoid duplicating some code below
+  # This sets the `channel` variable, or handles Exceptions.
+  private macro get_channel
     begin
       channel = get_about_info(ucid, locale)
     rescue ex : ChannelRedirect
@@ -16,6 +12,17 @@ module Invidious::Routes::API::V1::Channels
     rescue ex
       return error_json(500, ex)
     end
+  end
+
+  def self.home(env)
+    locale = env.get("preferences").as(Preferences).locale
+    ucid = env.params.url["ucid"]
+
+    env.response.content_type = "application/json"
+
+    # Use the private macro defined above.
+    channel = nil # Make the compiler happy
+    get_channel()
 
     # Retrieve "sort by" setting from URL parameters
     sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
@@ -138,21 +145,13 @@ module Invidious::Routes::API::V1::Channels
 
   def self.videos(env)
     locale = env.get("preferences").as(Preferences).locale
+    ucid = env.params.url["ucid"]
 
     env.response.content_type = "application/json"
 
-    ucid = env.params.url["ucid"]
-
-    begin
-      channel = get_about_info(ucid, locale)
-    rescue ex : ChannelRedirect
-      env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
-      return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
-    rescue ex : NotFoundException
-      return error_json(404, ex)
-    rescue ex
-      return error_json(500, ex)
-    end
+    # Use the private macro defined above.
+    channel = nil # Make the compiler happy
+    get_channel()
 
     # Retrieve some URL parameters
     sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
@@ -179,6 +178,74 @@ module Invidious::Routes::API::V1::Channels
     end
   end
 
+  def self.shorts(env)
+    locale = env.get("preferences").as(Preferences).locale
+    ucid = env.params.url["ucid"]
+
+    env.response.content_type = "application/json"
+
+    # Use the private macro defined above.
+    channel = nil # Make the compiler happy
+    get_channel()
+
+    # Retrieve continuation from URL parameters
+    continuation = env.params.query["continuation"]?
+
+    begin
+      videos, next_continuation = Channel::Tabs.get_shorts(
+        channel, continuation: continuation
+      )
+    rescue ex
+      return error_json(500, ex)
+    end
+
+    return JSON.build do |json|
+      json.object do
+        json.field "videos" do
+          json.array do
+            videos.each &.to_json(locale, json)
+          end
+        end
+
+        json.field "continuation", next_continuation if next_continuation
+      end
+    end
+  end
+
+  def self.streams(env)
+    locale = env.get("preferences").as(Preferences).locale
+    ucid = env.params.url["ucid"]
+
+    env.response.content_type = "application/json"
+
+    # Use the private macro defined above.
+    channel = nil # Make the compiler happy
+    get_channel()
+
+    # Retrieve continuation from URL parameters
+    continuation = env.params.query["continuation"]?
+
+    begin
+      videos, next_continuation = Channel::Tabs.get_60_livestreams(
+        channel, continuation: continuation
+      )
+    rescue ex
+      return error_json(500, ex)
+    end
+
+    return JSON.build do |json|
+      json.object do
+        json.field "videos" do
+          json.array do
+            videos.each &.to_json(locale, json)
+          end
+        end
+
+        json.field "continuation", next_continuation if next_continuation
+      end
+    end
+  end
+
   def self.playlists(env)
     locale = env.get("preferences").as(Preferences).locale
 
@@ -190,16 +257,9 @@ module Invidious::Routes::API::V1::Channels
               env.params.query["sort_by"]?.try &.downcase ||
               "last"
 
-    begin
-      channel = get_about_info(ucid, locale)
-    rescue ex : ChannelRedirect
-      env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
-      return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
-    rescue ex : NotFoundException
-      return error_json(404, ex)
-    rescue ex
-      return error_json(500, ex)
-    end
+    # Use the macro defined above
+    channel = nil # Make the compiler happy
+    get_channel()
 
     items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
 
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 08739c3d..0e6fba21 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -222,6 +222,9 @@ module Invidious::Routing
 
       # Channels
       get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
+      get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
+      get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
+
       {% for route in {"videos", "latest", "playlists", "community", "search"} %}
         get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
         get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}

From b6a4de66a5414f8ae790033fc3fc9e9fda70a860 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 4 Dec 2022 23:19:25 +0100
Subject: [PATCH 13/17] frontend: Unify the various channel pages

---
 src/invidious/routes/channels.cr              | 33 ++++---
 src/invidious/views/channel.ecr               | 95 +++++-------------
 src/invidious/views/community.ecr             | 65 +++----------
 .../views/components/channel_info.ecr         | 60 ++++++++++++
 src/invidious/views/playlists.ecr             | 96 -------------------
 5 files changed, 116 insertions(+), 233 deletions(-)
 create mode 100644 src/invidious/views/components/channel_info.ecr
 delete mode 100644 src/invidious/views/playlists.ecr

diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index 78b38341..77d309fb 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -7,18 +7,19 @@ module Invidious::Routes::Channels
 
   def self.videos(env)
     data = self.fetch_basic_information(env)
-    if !data.is_a?(Tuple)
-      return data
-    end
+    return data if !data.is_a?(Tuple)
+
     locale, user, subscriptions, continuation, ucid, channel = data
 
     sort_by = env.params.query["sort_by"]?.try &.downcase
 
     if channel.auto_generated
       sort_options = {"last", "oldest", "newest"}
-      sort_by ||= "last"
 
-      items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+      items, next_continuation = fetch_channel_playlists(
+        channel.ucid, channel.author, continuation, (sort_by || "last")
+      )
+
       items.uniq! do |item|
         if item.responds_to?(:title)
           item.title
@@ -30,11 +31,10 @@ module Invidious::Routes::Channels
       items.each(&.author = "")
     else
       sort_options = {"newest", "oldest", "popular"}
-      sort_by ||= "newest"
 
       # Fetch items and continuation token
       items, next_continuation = Channel::Tabs.get_videos(
-        channel, continuation: continuation, sort_by: sort_by
+        channel, continuation: continuation, sort_by: (sort_by || "newest")
       )
     end
 
@@ -90,24 +90,26 @@ module Invidious::Routes::Channels
 
   def self.playlists(env)
     data = self.fetch_basic_information(env)
-    if !data.is_a?(Tuple)
-      return data
-    end
+    return data if !data.is_a?(Tuple)
+
     locale, user, subscriptions, continuation, ucid, channel = data
 
     sort_options = {"last", "oldest", "newest"}
     sort_by = env.params.query["sort_by"]?.try &.downcase
-    sort_by ||= "last"
 
     if channel.auto_generated
       return env.redirect "/channel/#{channel.ucid}"
     end
 
-    items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+    items, next_continuation = fetch_channel_playlists(
+      channel.ucid, channel.author, continuation, (sort_by || "last")
+    )
+
     items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
     items.each(&.author = "")
 
-    templated "playlists"
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
+    templated "channel"
   end
 
   def self.community(env)
@@ -121,12 +123,15 @@ module Invidious::Routes::Channels
     thin_mode = thin_mode == "true"
 
     continuation = env.params.query["continuation"]?
-    # sort_by = env.params.query["sort_by"]?.try &.downcase
 
     if !channel.tabs.includes? "community"
       return env.redirect "/channel/#{channel.ucid}"
     end
 
+    # TODO: support sort options for community posts
+    sort_by = ""
+    sort_options = [] of String
+
     begin
       items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
     rescue ex : InfoException
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index f6cc3340..039f8752 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -1,8 +1,23 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
-<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %>
+<%-
+  ucid = channel.ucid
+  author = HTML.escape(channel.author)
+  channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+  relative_url =
+    case selected_tab
+    when .shorts?    then "/channel/#{ucid}/shorts"
+    when .streams?   then "/channel/#{ucid}/streams"
+    when .playlists? then "/channel/#{ucid}/playlists"
+    else
+      "/channel/#{ucid}"
+    end
+
+  youtube_url = "https://www.youtube.com#{relative_url}"
+  redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+-%>
 
 <% content_for "header" do %>
+<%- if selected_tab.videos? -%>
 <meta name="description" content="<%= channel.description %>">
 <meta property="og:site_name" content="Invidious">
 <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>">
@@ -14,76 +29,14 @@
 <meta name="twitter:title" content="<%= author %>">
 <meta name="twitter:description" content="<%= channel.description %>">
 <meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>">
-<link rel="alternate" href="https://www.youtube.com/channel/<%= ucid %>">
-<title><%= author %> - Invidious</title>
 <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" />
+<%- end -%>
+
+<link rel="alternate" href="<%= youtube_url %>">
+<title><%= author %> - Invidious</title>
 <% end %>
 
-<% if channel.banner %>
-    <div class="h-box">
-        <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
-    </div>
-
-    <div class="h-box">
-        <hr>
-    </div>
-<% end %>
-
-<div class="pure-g h-box">
-    <div class="pure-u-2-3">
-        <div class="channel-profile">
-            <img src="/ggpht<%= channel_profile_pic %>">
-            <span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
-        </div>
-    </div>
-    <div class="pure-u-1-3">
-        <h3 style="text-align:right">
-            <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
-        </h3>
-    </div>
-</div>
-
-<div class="h-box">
-    <div id="descriptionWrapper">
-        <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
-    </div>
-</div>
-
-<div class="h-box">
-    <% sub_count_text = number_to_short_text(channel.sub_count) %>
-    <%= rendered "components/subscribe_widget" %>
-</div>
-
-<div class="pure-g h-box">
-    <div class="pure-u-1-3">
-        <a href="https://www.youtube.com/channel/<%= ucid %>"><%= translate(locale, "View channel on YouTube") %></a>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
-                <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% else %>
-                <a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% end %>
-        </div>
-
-        <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
-    </div>
-    <div class="pure-u-1-3"></div>
-    <div class="pure-u-1-3">
-        <div class="pure-g" style="text-align:right">
-            <% sort_options.each do |sort| %>
-                <div class="pure-u-1 pure-md-1-3">
-                    <% if sort_by == sort %>
-                        <b><%= translate(locale, sort) %></b>
-                    <% else %>
-                        <a href="/channel/<%= ucid %>?sort_by=<%= sort %>">
-                            <%= translate(locale, sort) %>
-                        </a>
-                    <% end %>
-                </div>
-            <% end %>
-        </div>
-    </div>
-</div>
+<%= rendered "components/channel_info" %>
 
 <div class="h-box">
     <hr>
@@ -99,7 +52,7 @@
     <div class="pure-u-1 pure-u-md-4-5"></div>
     <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
         <% if next_continuation %>
-            <a href="/channel/<%= ucid %>/<%= selected_tab %>?continuation=<%= next_continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+            <a href="<%= relative_url %>?continuation=<%= next_continuation %><% if sort_options.any? sort_by %>&sort_by=<%= sort_by %><% end %>">
                 <%= translate(locale, "Next page") %>
             </a>
         <% end %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index e467a679..9e11d562 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -1,60 +1,21 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
+<%-
+  ucid = channel.ucid
+  author = HTML.escape(channel.author)
+  channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+  relative_url = "/channel/#{ucid}/community"
+  youtube_url = "https://www.youtube.com#{relative_url}"
+  redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+  selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community
+-%>
 
 <% content_for "header" do %>
+<link rel="alternate" href="<%= youtube_url %>">
 <title><%= author %> - Invidious</title>
 <% end %>
 
-<% if channel.banner %>
-    <div class="h-box">
-        <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
-    </div>
-
-    <div class="h-box">
-        <hr>
-    </div>
-<% end %>
-
-<div class="pure-g h-box">
-    <div class="pure-u-2-3">
-        <div class="channel-profile">
-            <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
-            <span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
-        </div>
-    </div>
-    <div class="pure-u-1-3" style="text-align:right">
-        <h3 style="text-align:right">
-            <a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a>
-        </h3>
-    </div>
-</div>
-
-<div class="h-box">
-    <div id="descriptionWrapper">
-        <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p>
-    </div>
-</div>
-
-<div class="h-box">
-    <% sub_count_text = number_to_short_text(channel.sub_count) %>
-    <%= rendered "components/subscribe_widget" %>
-</div>
-
-<div class="pure-g h-box">
-    <div class="pure-u-1-3">
-        <a href="https://www.youtube.com/channel/<%= channel.ucid %>/community"><%= translate(locale, "View channel on YouTube") %></a>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
-                <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% else %>
-                <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% end %>
-        </div>
-
-        <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Community) %>
-    </div>
-    <div class="pure-u-2-3"></div>
-</div>
+<%= rendered "components/channel_info" %>
 
 <div class="h-box">
     <hr>
diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr
new file mode 100644
index 00000000..f216359f
--- /dev/null
+++ b/src/invidious/views/components/channel_info.ecr
@@ -0,0 +1,60 @@
+<% if channel.banner %>
+    <div class="h-box">
+        <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
+    </div>
+
+    <div class="h-box">
+        <hr>
+    </div>
+<% end %>
+
+<div class="pure-g h-box">
+    <div class="pure-u-2-3">
+        <div class="channel-profile">
+            <img src="/ggpht<%= channel_profile_pic %>">
+            <span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
+        </div>
+    </div>
+    <div class="pure-u-1-3">
+        <h3 style="text-align:right">
+            <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
+        </h3>
+    </div>
+</div>
+
+<div class="h-box">
+    <div id="descriptionWrapper">
+        <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
+    </div>
+</div>
+
+<div class="h-box">
+    <% sub_count_text = number_to_short_text(channel.sub_count) %>
+    <%= rendered "components/subscribe_widget" %>
+</div>
+
+<div class="pure-g h-box">
+    <div class="pure-u-1-2">
+        <div class="pure-u-1 pure-md-1-3">
+            <a href="<%= youtube_url %>"><%= translate(locale, "View channel on YouTube") %></a>
+        </div>
+        <div class="pure-u-1 pure-md-1-3">
+            <a href="<%= redirect_url %>"><%= translate(locale, "Switch Invidious Instance") %></a>
+        </div>
+
+        <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
+    </div>
+    <div class="pure-u-1-2">
+        <div class="pure-g" style="text-align:end">
+            <% sort_options.each do |sort| %>
+                <div class="pure-u-1 pure-md-1-3">
+                    <% if sort_by == sort %>
+                        <b><%= translate(locale, sort) %></b>
+                    <% else %>
+                        <a href="<%= relative_url %>?sort_by=<%= sort %>"><%= translate(locale, sort) %></a>
+                    <% end %>
+                </div>
+            <% end %>
+        </div>
+    </div>
+</div>
diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr
deleted file mode 100644
index 56d25ef5..00000000
--- a/src/invidious/views/playlists.ecr
+++ /dev/null
@@ -1,96 +0,0 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
-
-<% content_for "header" do %>
-<title><%= author %> - Invidious</title>
-<% end %>
-
-<% if channel.banner %>
-    <div class="h-box">
-        <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
-    </div>
-
-    <div class="h-box">
-        <hr>
-    </div>
-<% end %>
-
-<div class="pure-g h-box">
-    <div class="pure-u-2-3">
-        <div class="channel-profile">
-            <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
-            <span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
-        </div>
-    </div>
-    <div class="pure-u-1-3" style="text-align:right">
-        <h3 style="text-align:right">
-            <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
-        </h3>
-    </div>
-</div>
-
-<div class="h-box">
-    <div id="descriptionWrapper">
-        <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p>
-    </div>
-</div>
-
-<div class="h-box">
-    <% sub_count_text = number_to_short_text(channel.sub_count) %>
-    <%= rendered "components/subscribe_widget" %>
-</div>
-
-<div class="pure-g h-box">
-    <div class="pure-g pure-u-1-3">
-        <div class="pure-u-1 pure-md-1-3">
-            <a href="https://www.youtube.com/channel/<%= ucid %>/playlists"><%= translate(locale, "View channel on YouTube") %></a>
-        </div>
-
-        <div class="pure-u-1 pure-md-1-3">
-            <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
-                <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% else %>
-                <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% end %>
-        </div>
-
-        <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Playlists) %>
-    </div>
-    <div class="pure-u-1-3"></div>
-    <div class="pure-u-1-3">
-        <div class="pure-g" style="text-align:right">
-            <% {"last", "oldest", "newest"}.each do |sort| %>
-                <div class="pure-u-1 pure-md-1-3">
-                    <% if sort_by == sort %>
-                        <b><%= translate(locale, sort) %></b>
-                    <% else %>
-                        <a href="/channel/<%= ucid %>/playlists?sort_by=<%= sort %>">
-                            <%= translate(locale, sort) %>
-                        </a>
-                    <% end %>
-                </div>
-            <% end %>
-        </div>
-    </div>
-</div>
-
-<div class="h-box">
-    <hr>
-</div>
-
-<div class="pure-g">
-<% items.each do |item| %>
-    <%= rendered "components/item" %>
-<% end %>
-</div>
-
-<div class="pure-g h-box">
-    <div class="pure-u-1 pure-u-md-4-5"></div>
-    <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
-        <% if continuation %>
-            <a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
-                <%= translate(locale, "Next page") %>
-            </a>
-        <% end %>
-    </div>
-</div>

From 4e3a9306260b737e2d13c6a763899b946a6ecfbb Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Mon, 5 Dec 2022 00:50:04 +0100
Subject: [PATCH 14/17] frontend: Add support for the "featured channels" page

---
 locales/en-US.json                      |  3 +-
 src/invidious/channels/about.cr         | 50 +++++--------------------
 src/invidious/frontend/channel_page.cr  |  1 +
 src/invidious/routes/api/v1/channels.cr | 24 ++----------
 src/invidious/routes/channels.cr        | 20 ++++++++++
 src/invidious/routing.cr                |  1 +
 src/invidious/views/channel.ecr         |  1 +
 7 files changed, 37 insertions(+), 63 deletions(-)

diff --git a/locales/en-US.json b/locales/en-US.json
index 44b40c24..12955665 100644
--- a/locales/en-US.json
+++ b/locales/en-US.json
@@ -475,5 +475,6 @@
     "channel_tab_shorts_label": "Shorts",
     "channel_tab_streams_label": "Livestreams",
     "channel_tab_playlists_label": "Playlists",
-    "channel_tab_community_label": "Community"
+    "channel_tab_community_label": "Community",
+    "channel_tab_channels_label": "Channels"
 }
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index 09c3427a..0054f8f2 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -16,12 +16,6 @@ record AboutChannel,
   tabs : Array(String),
   verified : Bool
 
-record AboutRelatedChannel,
-  ucid : String,
-  author : String,
-  author_url : String,
-  author_thumbnail : String
-
 def get_about_info(ucid, locale) : AboutChannel
   begin
     # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
@@ -165,41 +159,15 @@ def get_about_info(ucid, locale) : AboutChannel
   )
 end
 
-def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel)
-  # params is {"2:string":"channels"} encoded
-  channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
-
-  tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any
-  tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels"))
-
-  return [] of AboutRelatedChannel if tab.nil?
-
-  items = tab.dig?(
-    "tabRenderer", "content",
-    "sectionListRenderer", "contents", 0,
-    "itemSectionRenderer", "contents", 0,
-    "gridRenderer", "items"
-  ).try &.as_a?
-
-  related = [] of AboutRelatedChannel
-  return related if (items.nil? || items.empty?)
-
-  items.each do |item|
-    renderer = item["gridChannelRenderer"]?
-    next if !renderer
-
-    related_id = renderer.dig("channelId").as_s
-    related_title = renderer.dig("title", "simpleText").as_s
-    related_author_url = renderer.dig("navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s
-    related_author_thumbnail = HelperExtractors.get_thumbnails(renderer)
-
-    related << AboutRelatedChannel.new(
-      ucid: related_id,
-      author: related_title,
-      author_url: related_author_url,
-      author_thumbnail: related_author_thumbnail,
-    )
+def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
+  if continuation.nil?
+    # params is {"2:string":"channels"} encoded
+    initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
+  else
+    initial_data = YoutubeAPI.browse(continuation)
   end
 
-  return related
+  items, continuation = extract_items(initial_data)
+
+  return items.select(SearchChannel), continuation
 end
diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr
index 7ac0e071..53745dd5 100644
--- a/src/invidious/frontend/channel_page.cr
+++ b/src/invidious/frontend/channel_page.cr
@@ -7,6 +7,7 @@ module Invidious::Frontend::ChannelPage
     Streams
     Playlists
     Community
+    Channels
   end
 
   def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 4e92b54e..28ccdab9 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -102,31 +102,13 @@ module Invidious::Routes::API::V1::Channels
           json.array do
             # Fetch related channels
             begin
-              related_channels = fetch_related_channels(channel)
+              related_channels, _ = fetch_related_channels(channel)
             rescue ex
-              related_channels = [] of AboutRelatedChannel
+              related_channels = [] of SearchChannel
             end
 
             related_channels.each do |related_channel|
-              json.object do
-                json.field "author", related_channel.author
-                json.field "authorId", related_channel.ucid
-                json.field "authorUrl", related_channel.author_url
-
-                json.field "authorThumbnails" do
-                  json.array do
-                    qualities = {32, 48, 76, 100, 176, 512}
-
-                    qualities.each do |quality|
-                      json.object do
-                        json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
-                        json.field "width", quality
-                        json.field "height", quality
-                      end
-                    end
-                  end
-                end
-              end
+              related_channel.to_json(locale, json)
             end
           end
         end # relatedChannels
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index 77d309fb..d3969d29 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -147,6 +147,26 @@ module Invidious::Routes::Channels
     templated "community"
   end
 
+  def self.channels(env)
+    data = self.fetch_basic_information(env)
+    return data if !data.is_a?(Tuple)
+
+    locale, user, subscriptions, continuation, ucid, channel = data
+
+    if channel.auto_generated
+      return env.redirect "/channel/#{channel.ucid}"
+    end
+
+    items, next_continuation = fetch_related_channels(channel, continuation)
+
+    # Featured/related channels can't be sorted
+    sort_options = [] of String
+    sort_by = nil
+
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
+    templated "channel"
+  end
+
   def self.about(env)
     data = self.fetch_basic_information(env)
     if !data.is_a?(Tuple)
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 0e6fba21..84dbed5b 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -119,6 +119,7 @@ module Invidious::Routing
     get "/channel/:ucid/streams", Routes::Channels, :streams
     get "/channel/:ucid/playlists", Routes::Channels, :playlists
     get "/channel/:ucid/community", Routes::Channels, :community
+    get "/channel/:ucid/channels", Routes::Channels, :channels
     get "/channel/:ucid/about", Routes::Channels, :about
     get "/channel/:ucid/live", Routes::Channels, :live
     get "/user/:user/live", Routes::Channels, :live
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 039f8752..a29315ef 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -8,6 +8,7 @@
     when .shorts?    then "/channel/#{ucid}/shorts"
     when .streams?   then "/channel/#{ucid}/streams"
     when .playlists? then "/channel/#{ucid}/playlists"
+    when .channels?  then "/channel/#{ucid}/channels"
     else
       "/channel/#{ucid}"
     end

From 69b8e0919fd0a410d35f5f5fccc4753f79faf940 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 22 Dec 2022 17:26:30 +0100
Subject: [PATCH 15/17] api: Add support for the "featured channels" endpoint

---
 src/invidious/routes/api/v1/channels.cr | 31 +++++++++++++++++++++++++
 src/invidious/routing.cr                |  1 +
 2 files changed, 32 insertions(+)

diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 28ccdab9..ca2b2734 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -283,6 +283,37 @@ module Invidious::Routes::API::V1::Channels
     end
   end
 
+  def self.channels(env)
+    locale = env.get("preferences").as(Preferences).locale
+    ucid = env.params.url["ucid"]
+
+    env.response.content_type = "application/json"
+
+    # Use the macro defined above
+    channel = nil # Make the compiler happy
+    get_channel()
+
+    continuation = env.params.query["continuation"]?
+
+    begin
+      items, next_continuation = fetch_related_channels(channel, continuation)
+    rescue ex
+      return error_json(500, ex)
+    end
+
+    JSON.build do |json|
+      json.object do
+        json.field "relatedChannels" do
+          json.array do
+            items.each &.to_json(locale, json)
+          end
+        end
+
+        json.field "continuation", next_continuation if next_continuation
+      end
+    end
+  end
+
   def self.search(env)
     locale = env.get("preferences").as(Preferences).locale
     region = env.params.query["region"]?
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 84dbed5b..54bd82a4 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -225,6 +225,7 @@ module Invidious::Routing
       get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
       get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
       get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
+      get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
 
       {% for route in {"videos", "latest", "playlists", "community", "search"} %}
         get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}

From f9eb839c7ae2c29e641495c4a2affd384445bf97 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 22 Dec 2022 13:05:13 +0100
Subject: [PATCH 16/17] channel: remove dead playlists code

---
 spec/invidious/helpers_spec.cr      |  6 ----
 src/invidious/channels/playlists.cr | 55 -----------------------------
 2 files changed, 61 deletions(-)

diff --git a/spec/invidious/helpers_spec.cr b/spec/invidious/helpers_spec.cr
index ab361770..f81cd29a 100644
--- a/spec/invidious/helpers_spec.cr
+++ b/spec/invidious/helpers_spec.cr
@@ -23,12 +23,6 @@ Spectator.describe "Helper" do
     end
   end
 
-  describe "#produce_channel_playlists_url" do
-    it "correctly produces a /browse_ajax URL with the given UCID and cursor" do
-      expect(produce_channel_playlists_url("UCCj956IF62FbT7Gouszaj9w", "AIOkY9EQpi_gyn1_QrFuZ1reN81_MMmI1YmlBblw8j7JHItEFG5h7qcJTNd4W9x5Quk_CVZ028gW")).to eq("/browse_ajax?continuation=4qmFsgLNARIYVUNDajk1NklGNjJGYlQ3R291c3phajl3GrABRWdsd2JHRjViR2x6ZEhNd0FqZ0JZQUZxQUxnQkFIcG1VVlZzVUdFeGF6VlNWa1ozWVZZNWJtVlhOSGhZTVVaNVVtNVdZVTFZU214VWFtZDRXREF4VG1KVmEzaFhWekZ6VVcxS2MyUjZhSEZPTUhCSlUxaFNSbEpyWXpGaFJHUjRXVEJ3VlZSdFVUQldlbXcwVGxaR01XRXhPVVJXYkc5M1RXcG9ibFozSUFFWUF3PT0%3D&gl=US&hl=en")
-    end
-  end
-
   describe "#produce_comment_continuation" do
     it "correctly produces a continuation token for comments" do
       expect(produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA")).to eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyjAMK9gJBRFNKX2kycXZKZUZ0TDBodG1TNV9LNUN0ajNlR0ZWQk1XTDlXZDQybzNrbVVMNl9tQXpkTHA4NS1saVFaTDBtWXJfMTZCaGFnZ1VxWDY1MlN2OUpxVjZWWGluU2hTUC1aVDZyTDROb2xQQmFQWFZ0SnNPNV9yQV9xRTNHdWJBdUxGdzl1eklJWFUyLUhucFhiZGdQTFdURmF2ZlgyMDZocVdtbXBId1VPcm14UVZfT1g2dFlrTTN1eDNyUEFLQ0RyVDhlV0w3TVUzYkxpTmNuYmdrVzhvMGg4S1lMTF84QlBhOExjSGJUdjhwQW9Oa2plcmxYMXg3SzRwcXhhWFBveXo4OXFObG5oNnJSeDZBWGdBenpvSEgxZG1jeVE4Q0lCZU9IZy1tNGk4WnhkWDRkUDg4WFdySUZnLWpKR2hwR1A4SlVNRGdaZ2F2eFZ4MjI1aFVFWVpNeXJMR2xlcjVlbTRGZ2JHNjJZV0M1MW1vTERMZVlFQSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D")
diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index 772eecb9..8dc824b2 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -26,58 +26,3 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
 
   return extract_items(initial_data, author, ucid)
 end
-
-# ## NOTE: DEPRECATED
-# Reason -> Unstable
-# The Protobuf object must be provided with an id of the last playlist from the current "page"
-# in order to fetch the next one accurately
-# (if the id isn't included, entries shift around erratically between pages,
-# leading to repetitions and skip overs)
-#
-# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user,
-# it's better to stick to continuation tokens provided by the first request and onward
-def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false)
-  object = {
-    "80226972:embedded" => {
-      "2:string" => ucid,
-      "3:base64" => {
-        "2:string"  => "playlists",
-        "6:varint"  => 2_i64,
-        "7:varint"  => 1_i64,
-        "12:varint" => 1_i64,
-        "13:string" => "",
-        "23:varint" => 0_i64,
-      },
-    },
-  }
-
-  if cursor
-    cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated
-    object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor
-  end
-
-  if auto_generated
-    object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64
-  else
-    object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64
-    case sort
-    when "oldest", "oldest_created"
-      object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64
-    when "newest", "newest_created"
-      object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64
-    when "last", "last_added"
-      object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64
-    else nil # Ignore
-    end
-  end
-
-  object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"])))
-  object["80226972:embedded"].delete("3:base64")
-
-  continuation = object.try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
-end

From a37522a03dc12f61386fc0529a9136ad296b1228 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 8 Jan 2023 13:50:52 +0100
Subject: [PATCH 17/17] Implement workaround for broken shorts objects

---
 src/invidious/channels/videos.cr       | 30 ++++++++++++++++++++++----
 src/invidious/exceptions.cr            |  5 +++++
 src/invidious/yt_backend/extractors.cr | 26 ++++++++++++----------
 3 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index bea406c1..befec03d 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -127,16 +127,38 @@ module Invidious::Channel::Tabs
   #  Shorts
   # -------------------
 
-  def get_shorts(channel : AboutChannel, continuation : String? = nil)
+  private def fetch_shorts_data(ucid : String, continuation : String? = nil)
     if continuation.nil?
       # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
       # TODO: try to extract the continuation tokens that allows other sorting options
-      initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
+      return YoutubeAPI.browse(ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
     else
-      initial_data = YoutubeAPI.browse(continuation: continuation)
+      return YoutubeAPI.browse(continuation: continuation)
     end
+  end
 
-    return extract_items(initial_data, channel.author, channel.ucid)
+  def get_shorts(channel : AboutChannel, continuation : String? = nil)
+    initial_data = self.fetch_shorts_data(channel.ucid, continuation)
+
+    begin
+      # Try to parse the initial data fetched above
+      return extract_items(initial_data, channel.author, channel.ucid)
+    rescue ex : RetryOnceException
+      # Sometimes, for a completely unknown reason, the "reelItemRenderer"
+      # object is missing some critical information (it happens once in about
+      # 20 subsequent requests). Refreshing the page is required to properly
+      # show the "shorts" tab.
+      #
+      # In order to make the experience smoother for the user, we simulate
+      # said page refresh by fetching again the JSON. If that still doesn't
+      # work, we raise a BrokenTubeException, as something is really broken.
+      begin
+        initial_data = self.fetch_shorts_data(channel.ucid, continuation)
+        return extract_items(initial_data, channel.author, channel.ucid)
+      rescue ex : RetryOnceException
+        raise BrokenTubeException.new "reelPlayerHeaderSupportedRenderers"
+      end
+    end
   end
 
   # -------------------
diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr
index 425c08da..690db907 100644
--- a/src/invidious/exceptions.cr
+++ b/src/invidious/exceptions.cr
@@ -33,3 +33,8 @@ end
 
 class VideoNotAvailableException < Exception
 end
+
+# Exception used to indicate that the JSON response from YT is missing
+# some important informations, and that the query should be sent again.
+class RetryOnceException < Exception
+end
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index bca0dcbd..65d107b2 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -408,19 +408,23 @@ private module Parsers
     private def self.parse(item_contents, author_fallback)
       video_id = item_contents["videoId"].as_s
 
-      begin
-        video_details_container = item_contents.dig(
-          "navigationEndpoint", "reelWatchEndpoint",
-          "overlay", "reelPlayerOverlayRenderer",
-          "reelPlayerHeaderSupportedRenderers",
-          "reelPlayerHeaderRenderer"
-        )
-      rescue ex : KeyError
-        # Extract key name from original message
-        key = /"([^"]+)"/.match(ex.message || "").try &.[1]?
-        raise BrokenTubeException.new(key || "reelPlayerOverlayRenderer")
+      reel_player_overlay = item_contents.dig(
+        "navigationEndpoint", "reelWatchEndpoint",
+        "overlay", "reelPlayerOverlayRenderer"
+      )
+
+      # Sometimes, the "reelPlayerOverlayRenderer" object is missing the
+      # important part of the response. We use this exception to tell
+      # the calling function to fetch the content again.
+      if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers")
+        raise RetryOnceException.new
       end
 
+      video_details_container = reel_player_overlay.dig(
+        "reelPlayerHeaderSupportedRenderers",
+        "reelPlayerHeaderRenderer"
+      )
+
       # Author infos
 
       author = video_details_container