Merge pull request 'Externalized items to new glade files and classes' (#10) from develop into master

Reviewed-on: #10

.gitignore

@@ -1,3 +1,7 @@
+docs/
+.idea/
+*.zip
+
 # ---> Python
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -137,4 +141,3 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
-

README.md

@@ -1,5 +1,3 @@
-# SolarFM
-
 # SolarFM
 SolarFM is a Gtk+ Python file manager.
 
@@ -13,8 +11,8 @@ sudo apt-get install python3.8 wget python3-setproctitle python3-gi ffmpegthumbn
 
 # TODO
 <ul>
-<li>Add simpleish plugin system to run bash/python scripts.</li>
+<li>Add simpleish preview plugin for various file types.</li>
-<li>Add DnD context awareness for over folder drop.</li>
+<li>Add simpleish bulk-renamer.</li>
 </ul>
 
 # Images

plugins/README.md

@@ -0,0 +1,42 @@
+### Note
+Copy the example and rename it to your desired name. Plugins define a ui target slot with the 'ui_target' requests data but don't have to if not directly interacted with.
+Plugins must have a run method defined; though, you do not need to necessarily do anything within it. The run method implies that the passed in event system or other data is ready for the plugin to use.
+
+
+### Manifest Example (All are required even if empty.)
+```
+class Manifest:
+    name: str     = "Example Plugin"
+    author: str   = "John Doe"
+    version: str  = "0.0.1"
+    support: str  = ""
+    requests: {}  = {
+        'ui_target': "plugin_control_list",
+        'pass_fm_events': "true"
+    }
+```
+
+
+### Requests
+```
+requests: {}  = {
+    'ui_target': "plugin_control_list",
+    'ui_target_id': "<some other Gtk Glade ID>",          # Only needed if using "other" in "ui_target". See below for predefined "ui_target" options...
+    'pass_fm_events': "true",                             # If empty or not present will be ignored.
+    "pass_ui_objects": [""],                              # Request reference to a UI component. Will be passed back as array to plugin.
+    'bind_keys': [f"{name}||send_message:<Control>f"],
+                  f"{name}||do_save:<Control>s"]          # Bind keys with method and key pare using list. Must pass "name" like shown with delimiter to its right.
+
+}
+```
+
+UI Targets:
+<ul>
+<li>main_Window</li>
+<li>main_menu_bar</li>
+<li>path_menu_bar</li>
+<li>plugin_control_list</li>
+<li>window_(1-4)</li>
+<li>context_menu</li>
+<li>other</li>
+</ul>

plugins/archiver/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/archiver/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/archiver/archiver.glade

@@ -0,0 +1,164 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated with glade 3.38.2 -->
+<interface>
+  <requires lib="gtk+" version="3.16"/>
+  <object class="GtkTextBuffer" id="arc_command_buffer">
+    <property name="text" translatable="yes">$(which 7za || echo 7zr) a %o %N</property>
+  </object>
+  <object class="GtkFileChooserDialog" id="archiver_dialogue">
+    <property name="can-focus">False</property>
+    <property name="modal">True</property>
+    <property name="window-position">center</property>
+    <property name="type-hint">dialog</property>
+    <property name="gravity">center</property>
+    <property name="do-overwrite-confirmation">True</property>
+    <property name="select-multiple">True</property>
+    <child internal-child="vbox">
+      <object class="GtkBox">
+        <property name="can-focus">False</property>
+        <property name="orientation">vertical</property>
+        <property name="spacing">2</property>
+        <child internal-child="action_area">
+          <object class="GtkButtonBox">
+            <property name="can-focus">False</property>
+            <property name="layout-style">end</property>
+            <child>
+              <object class="GtkButton" id="button21">
+                <property name="label">gtk-cancel</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="receives-default">True</property>
+                <property name="use-stock">True</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkButton" id="button22">
+                <property name="label">gtk-ok</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="receives-default">True</property>
+                <property name="use-stock">True</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">False</property>
+            <property name="position">0</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkBox">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <property name="orientation">vertical</property>
+            <child>
+              <object class="GtkBox">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="homogeneous">True</property>
+                <child>
+                  <object class="GtkLabel">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="label" translatable="yes">Compress Commands:</property>
+                    <property name="xalign">0.20000000298023224</property>
+                    <attributes>
+                      <attribute name="gravity" value="west"/>
+                    </attributes>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="fill">True</property>
+                    <property name="position">0</property>
+                  </packing>
+                </child>
+                <child>
+                  <placeholder/>
+                </child>
+                <child>
+                  <object class="GtkLabel">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="label" translatable="yes">Archive Format:</property>
+                    <property name="xalign">1</property>
+                    <attributes>
+                      <attribute name="gravity" value="east"/>
+                    </attributes>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="fill">True</property>
+                    <property name="position">2</property>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkComboBoxText">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="active">0</property>
+                    <property name="active-id">0</property>
+                    <items>
+                      <item id="0" translatable="yes">7Zip (*.7z)</item>
+                      <item id="1" translatable="yes">Zip (*.zip *.ZIP)</item>
+                      <item id="2" translatable="yes">RAR (*.rar *.RAR)</item>
+                      <item id="3" translatable="yes">Tar (*.tar)</item>
+                      <item id="4" translatable="yes">Tar bzip2 (*.tar.bz2)</item>
+                      <item id="5" translatable="yes">Tar Gzip (*.tar.gz *.tgz)</item>
+                      <item id="6" translatable="yes">Tar xz (*.tar.xz *.txz)</item>
+                      <item id="7" translatable="yes">Gzip (*.gz)</item>
+                      <item id="8" translatable="yes">XZ (*.xz)</item>
+                    </items>
+                    <signal name="changed" handler="set_arc_buffer_text" swapped="no"/>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="fill">True</property>
+                    <property name="position">3</property>
+                  </packing>
+                </child>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkTextView" id="arc_command">
+                <property name="height-request">72</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="buffer">arc_command_buffer</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">True</property>
+            <property name="position">2</property>
+          </packing>
+        </child>
+      </object>
+    </child>
+    <action-widgets>
+      <action-widget response="-6">button21</action-widget>
+      <action-widget response="-5">button22</action-widget>
+    </action-widgets>
+  </object>
+</interface>

plugins/archiver/manifest.json

@@ -0,0 +1,12 @@
+{
+    "manifest": {
+        "name": "Archiver",
+        "author": "ITDominator",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "context_menu_plugins",
+            "pass_fm_events": "true"
+        }
+    }
+}

plugins/archiver/plugin.py

@@ -0,0 +1,129 @@
+# Python imports
+import os
+import threading
+import subprocess
+import inspect
+import shlex
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+
+# Application imports
+from plugins.plugin_base import PluginBase
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+    return wrapper
+
+
+
+
+class Plugin(PluginBase):
+    def __init__(self):
+        super().__init__()
+        self.path               = os.path.dirname(os.path.realpath(__file__))
+        self._GLADE_FILE        = f"{self.path}/archiver.glade"
+        self.name = "Archiver"  # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                #       where self.name should not be needed for message comms
+        self._archiver_dialogue  = None
+        self._arc_command_buffer = None
+
+        # In compress commands:
+        #    %n: First selected filename/dir to archive
+        #    %N: All selected filenames/dirs to archive, or (with %O) a single filename
+        #    %o: Resulting single archive file
+        #    %O: Resulting archive per source file/directory (use changes %N meaning)
+        #
+        #  In extract commands:
+        #    %x: Archive file to extract
+        #    %g: Unique extraction target filename with optional subfolder
+        #    %G: Unique extraction target filename, never with subfolder
+        #
+        #  In list commands:
+        #      %x: Archive to list
+        #
+        #  Plus standard bash variables are accepted.
+        self.arc_commands            = [ '$(which 7za || echo 7zr) a %o %N',
+                                                                'zip -r %o %N',
+                                                                'rar a -r %o %N',
+                                                                'tar -cvf %o %N',
+                                                                'tar -cvjf %o %N',
+                                                                'tar -cvzf %o %N',
+                                                                'tar -cvJf %o %N',
+                                                                'gzip -c %N > %O',
+                                                                'xz -cz %N > %O'
+                                        ]
+
+
+    def generate_reference_ui_element(self):
+        self._builder           = Gtk.Builder()
+        self._builder.add_from_file(self._GLADE_FILE)
+
+        classes  = [self]
+        handlers = {}
+        for c in classes:
+            methods = None
+            try:
+                methods = inspect.getmembers(c, predicate=inspect.ismethod)
+                handlers.update(methods)
+            except Exception as e:
+                print(repr(e))
+
+        self._builder.connect_signals(handlers)
+
+        self._archiver_dialogue  = self._builder.get_object("archiver_dialogue")
+        self._arc_command_buffer = self._builder.get_object("arc_command_buffer")
+
+        item = Gtk.ImageMenuItem(self.name)
+        item.set_image( Gtk.Image(stock=Gtk.STOCK_FLOPPY) )
+        item.connect("activate", self.show_archiver_dialogue)
+        item.set_always_show_image(True)
+        return item
+
+
+    def run(self):
+        ...
+
+    def show_archiver_dialogue(self, widget=None, eve=None):
+        self._event_system.emit("get_current_state")
+        state = self._fm_state
+
+        self._archiver_dialogue.set_action(Gtk.FileChooserAction.SAVE)
+        self._archiver_dialogue.set_current_folder(state.tab.get_current_directory())
+        self._archiver_dialogue.set_current_name("arc.7z")
+
+        response = self._archiver_dialogue.run()
+        if response == Gtk.ResponseType.OK:
+            save_target = self._archiver_dialogue.get_filename()
+            self.archive_files(save_target, state)
+        if (response == Gtk.ResponseType.CANCEL) or (response == Gtk.ResponseType.DELETE_EVENT):
+            pass
+
+        self._archiver_dialogue.hide()
+
+    def archive_files(self, save_target, state):
+        paths       = [shlex.quote(p) for p in state.selected_files]
+
+        sItr, eItr  = self._arc_command_buffer.get_bounds()
+        pre_command = self._arc_command_buffer.get_text(sItr, eItr, False)
+        pre_command = pre_command.replace("%o", shlex.quote(save_target))
+        pre_command = pre_command.replace("%N", ' '.join(paths))
+        command     = f"{state.tab.terminal_app} -e {shlex.quote(pre_command)}"
+        current_dir = state.tab.get_current_directory()
+
+        state.tab.execute(shlex.split(command), start_dir=shlex.quote(current_dir))
+
+    def set_arc_buffer_text(self, widget=None, eve=None):
+        sid = widget.get_active_id()
+        self._arc_command_buffer.set_text(self.arc_commands[int(sid)])

plugins/disk_usage/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/disk_usage/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/disk_usage/du_usage.glade

@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated with glade 3.40.0 -->
+<interface>
+  <requires lib="gtk+" version="3.24"/>
+  <object class="GtkListStore" id="du_store">
+    <columns>
+      <!-- column-name Size -->
+      <column type="gchararray"/>
+      <!-- column-name Dir -->
+      <column type="gchararray"/>
+    </columns>
+  </object>
+  <object class="GtkDialog" id="du_dialog">
+    <property name="width-request">420</property>
+    <property name="height-request">450</property>
+    <property name="can-focus">False</property>
+    <property name="modal">True</property>
+    <property name="window-position">center</property>
+    <property name="destroy-with-parent">True</property>
+    <property name="type-hint">dialog</property>
+    <property name="skip-taskbar-hint">True</property>
+    <property name="skip-pager-hint">True</property>
+    <property name="decorated">False</property>
+    <property name="deletable">False</property>
+    <property name="gravity">center</property>
+    <child internal-child="vbox">
+      <object class="GtkBox">
+        <property name="can-focus">False</property>
+        <property name="orientation">vertical</property>
+        <property name="spacing">2</property>
+        <child internal-child="action_area">
+          <object class="GtkButtonBox">
+            <property name="can-focus">False</property>
+            <property name="layout-style">end</property>
+            <child>
+              <object class="GtkButton">
+                <property name="label">gtk-close</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="receives-default">True</property>
+                <property name="use-stock">True</property>
+                <signal name="released" handler="_hide_du_menu" swapped="no"/>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">2</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">False</property>
+            <property name="position">0</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkBox">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <property name="orientation">vertical</property>
+            <child>
+              <object class="GtkLabel" id="current_dir_lbl">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="margin-start">5</property>
+                <property name="margin-end">5</property>
+                <property name="margin-top">5</property>
+                <property name="margin-bottom">5</property>
+                <property name="label" translatable="yes">Current Directory:</property>
+                <property name="justify">center</property>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkScrolledWindow">
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="shadow-type">in</property>
+                <child>
+                  <object class="GtkTreeView">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="model">du_store</property>
+                    <property name="headers-clickable">False</property>
+                    <child internal-child="selection">
+                      <object class="GtkTreeSelection"/>
+                    </child>
+                    <child>
+                      <object class="GtkTreeViewColumn">
+                        <property name="title" translatable="yes">Disk Usage</property>
+                        <child>
+                          <object class="GtkCellRendererText"/>
+                          <attributes>
+                            <attribute name="text">0</attribute>
+                          </attributes>
+                        </child>
+                        <child>
+                          <object class="GtkCellRendererText"/>
+                          <attributes>
+                            <attribute name="text">1</attribute>
+                          </attributes>
+                        </child>
+                      </object>
+                    </child>
+                  </object>
+                </child>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">True</property>
+            <property name="fill">True</property>
+            <property name="position">1</property>
+          </packing>
+        </child>
+      </object>
+    </child>
+  </object>
+</interface>

plugins/disk_usage/manifest.json

@@ -0,0 +1,12 @@
+{
+    "manifest": {
+        "name": "Disk Usage",
+        "author": "ITDominator",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "context_menu_plugins",
+            "pass_fm_events": "true"
+        }
+    }
+}

plugins/disk_usage/plugin.py

@@ -0,0 +1,95 @@
+# Python imports
+import os
+import subprocess
+import time
+import inspect
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+
+# Application imports
+from plugins.plugin_base import PluginBase
+
+
+
+
+class Plugin(PluginBase):
+    def __init__(self):
+        super().__init__()
+
+        self.name        = "Disk Usage"  # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                         #       where self.name should not be needed for message comms
+
+        self.path        = os.path.dirname(os.path.realpath(__file__))
+        self._GLADE_FILE = f"{self.path}/du_usage.glade"
+        self._du_dialog  = None
+        self._du_store   = None
+
+
+    def run(self):
+        self._builder    = Gtk.Builder()
+        self._builder.add_from_file(self._GLADE_FILE)
+
+        classes  = [self]
+        handlers = {}
+        for c in classes:
+            methods = None
+            try:
+                methods = inspect.getmembers(c, predicate=inspect.ismethod)
+                handlers.update(methods)
+            except Exception as e:
+                print(repr(e))
+
+        self._builder.connect_signals(handlers)
+
+        self._du_dialog = self._builder.get_object("du_dialog")
+        self._du_store  = self._builder.get_object("du_store")
+        self._current_dir_lbl  = self._builder.get_object("current_dir_lbl")
+
+        self._event_system.subscribe("show_du_menu", self._show_du_menu)
+
+    def generate_reference_ui_element(self):
+        item = Gtk.ImageMenuItem(self.name)
+        item.set_image( Gtk.Image(stock=Gtk.STOCK_HARDDISK) )
+        item.connect("activate", self._show_du_menu)
+        item.set_always_show_image(True)
+        return item
+
+    def _get_state(self, widget=None, eve=None):
+        self._event_system.emit("get_current_state")
+
+    def _set_current_dir_lbl(self, widget=None, eve=None):
+        self._current_dir_lbl.set_label(f"Current Directory:\n{self._fm_state.tab.get_current_directory()}")
+
+    def _show_du_menu(self, widget=None, eve=None):
+        self._fm_state = None
+        self._get_state()
+        self._set_current_dir_lbl()
+        self.load_du_data()
+        self._du_dialog.run()
+
+    def load_du_data(self):
+        self._du_store.clear()
+
+        path     = self._fm_state.tab.get_current_directory()
+        # NOTE: -h = human readable, -d = depth asigned to 1
+        command  = ["du", "-h", "-d", "1", path]
+        proc     = subprocess.Popen(command, stdout=subprocess.PIPE, encoding="utf-8")
+        raw_data = proc.communicate()[0]
+        # NOTE: Will return data AFTER completion (if any)
+        data     = raw_data.strip()
+        parts    = data.split("\n")
+
+        # NOTE: Last entry is curret dir. Move to top of list and pop off...
+        size, file = parts[-1].split("\t")
+        self._du_store.append([size, file.split("/")[-1]])
+        parts.pop()
+
+        for part in parts:
+            size, file = part.split("\t")
+            self._du_store.append([size, file.split("/")[-1]])
+
+    def _hide_du_menu(self, widget=None, eve=None):
+        self._du_dialog.hide()

plugins/favorites/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/favorites/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/favorites/favorites.glade

@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated with glade 3.40.0 -->
+<interface>
+  <requires lib="gtk+" version="3.24"/>
+  <object class="GtkListStore" id="favorites_store">
+    <columns>
+      <!-- column-name Favorites -->
+      <column type="gchararray"/>
+      <!-- column-name Path -->
+      <column type="gchararray"/>
+    </columns>
+  </object>
+  <object class="GtkDialog" id="favorites_dialog">
+    <property name="width-request">420</property>
+    <property name="height-request">450</property>
+    <property name="can-focus">False</property>
+    <property name="modal">True</property>
+    <property name="window-position">center</property>
+    <property name="destroy-with-parent">True</property>
+    <property name="type-hint">dialog</property>
+    <property name="skip-taskbar-hint">True</property>
+    <property name="skip-pager-hint">True</property>
+    <property name="decorated">False</property>
+    <property name="deletable">False</property>
+    <property name="gravity">center</property>
+    <child internal-child="vbox">
+      <object class="GtkBox">
+        <property name="can-focus">False</property>
+        <property name="orientation">vertical</property>
+        <property name="spacing">2</property>
+        <child internal-child="action_area">
+          <object class="GtkButtonBox">
+            <property name="can-focus">False</property>
+            <property name="layout-style">end</property>
+            <child>
+              <object class="GtkButton">
+                <property name="label">gtk-delete</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="receives-default">True</property>
+                <property name="use-stock">True</property>
+                <property name="always-show-image">True</property>
+                <signal name="released" handler="_remove_from_favorite" swapped="no"/>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkButton">
+                <property name="label">gtk-add</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="receives-default">True</property>
+                <property name="use-stock">True</property>
+                <property name="always-show-image">True</property>
+                <signal name="released" handler="_add_to_favorite" swapped="no"/>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkButton">
+                <property name="label">gtk-close</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="receives-default">True</property>
+                <property name="use-stock">True</property>
+                <signal name="released" handler="_hide_favorites_menu" swapped="no"/>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">2</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">False</property>
+            <property name="position">0</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkBox">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <property name="orientation">vertical</property>
+            <child>
+              <object class="GtkLabel" id="current_dir_lbl">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="margin-start">5</property>
+                <property name="margin-end">5</property>
+                <property name="margin-top">5</property>
+                <property name="margin-bottom">5</property>
+                <property name="label" translatable="yes">Current Directory:</property>
+                <property name="justify">center</property>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkScrolledWindow">
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="shadow-type">in</property>
+                <child>
+                  <object class="GtkTreeView">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="model">favorites_store</property>
+                    <property name="headers-clickable">False</property>
+                    <signal name="button-release-event" handler="_set_selected_path" swapped="no"/>
+                    <child internal-child="selection">
+                      <object class="GtkTreeSelection">
+                        <signal name="changed" handler="_set_selected" swapped="no"/>
+                      </object>
+                    </child>
+                    <child>
+                      <object class="GtkTreeViewColumn">
+                        <property name="title" translatable="yes">Favorites</property>
+                        <child>
+                          <object class="GtkCellRendererText"/>
+                          <attributes>
+                            <attribute name="text">0</attribute>
+                          </attributes>
+                        </child>
+                      </object>
+                    </child>
+                  </object>
+                </child>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">True</property>
+            <property name="fill">True</property>
+            <property name="position">1</property>
+          </packing>
+        </child>
+      </object>
+    </child>
+  </object>
+</interface>

plugins/favorites/manifest.json

@@ -0,0 +1,14 @@
+{
+    "manifest": {
+        "name": "Favorites",
+        "author": "ITDominator",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "main_menu_bttn_box_bar",
+            "pass_fm_events": "true",
+            "pass_ui_objects": ["path_entry"],
+            "bind_keys": ["Favorites||show_favorites_menu:<Control>f"]
+        }
+    }
+}

plugins/favorites/plugin.py

@@ -0,0 +1,117 @@
+# Python imports
+import os
+import inspect
+import json
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+
+# Application imports
+from plugins.plugin_base import PluginBase
+
+
+
+
+class Plugin(PluginBase):
+    def __init__(self):
+        super().__init__()
+
+        self.name               = "Favorites"  # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                                #       where self.name should not be needed for message comms
+        self.path               = os.path.dirname(os.path.realpath(__file__))
+        self._GLADE_FILE        = f"{self.path}/favorites.glade"
+        self._FAVORITES_FILE    = f"{self.path}/favorites.json"
+
+        self._favorites_dialog  = None
+        self._favorites_store   = None
+        self._favorites         = None
+        self._selected          = None
+
+
+    def run(self):
+        self._builder          = Gtk.Builder()
+        self._builder.add_from_file(self._GLADE_FILE)
+
+        classes  = [self]
+        handlers = {}
+        for c in classes:
+            methods = None
+            try:
+                methods = inspect.getmembers(c, predicate=inspect.ismethod)
+                handlers.update(methods)
+            except Exception as e:
+                print(repr(e))
+
+        self._builder.connect_signals(handlers)
+
+        self._favorites_dialog = self._builder.get_object("favorites_dialog")
+        self._favorites_store  = self._builder.get_object("favorites_store")
+        self._current_dir_lbl  = self._builder.get_object("current_dir_lbl")
+
+        if os.path.exists(self._FAVORITES_FILE):
+            with open(self._FAVORITES_FILE) as f:
+                self._favorites = json.load(f)
+                for favorite in self._favorites:
+                    display, path = favorite
+                    self._favorites_store.append([display, path])
+        else:
+            with open(self._FAVORITES_FILE, 'a') as f:
+                f.write('[]')
+
+        self._event_system.subscribe("show_favorites_menu", self._show_favorites_menu)
+
+
+    def generate_reference_ui_element(self):
+        button = Gtk.Button(label=self.name)
+        button.connect("button-release-event", self._show_favorites_menu)
+        return button
+
+    def _get_state(self, widget=None, eve=None):
+        self._event_system.emit("get_current_state")
+
+    def _set_current_dir_lbl(self, widget=None, eve=None):
+        self._current_dir_lbl.set_label(f"Current Directory:\n{self._fm_state.tab.get_current_directory()}")
+
+    def _add_to_favorite(self, state):
+        path    = self._fm_state.tab.get_current_directory()
+        parts   = path.split("/")
+        display = '/'.join(parts[-3:]) if len(parts) > 3 else path
+
+        self._favorites_store.append([display, path])
+        self._favorites.append([display, path])
+        self._save_favorites()
+
+    def _remove_from_favorite(self, state):
+        path = self._favorites_store.get_value(self._selected, 1)
+        self._favorites_store.remove(self._selected)
+
+        for i, f in enumerate(self._favorites):
+            if f[1] == path:
+                self._favorites.remove( self._favorites[i] )
+
+        self._save_favorites()
+
+    def _save_favorites(self):
+        with open(self._FAVORITES_FILE, 'w') as outfile:
+            json.dump(self._favorites, outfile, separators=(',', ':'), indent=4)
+
+    def _set_selected_path(self, widget=None, eve=None):
+        path = self._favorites_store.get_value(self._selected, 1)
+        self._ui_objects[0].set_text(path)
+        self._set_current_dir_lbl()
+
+    def _show_favorites_menu(self, widget=None, eve=None):
+        self._fm_state = None
+        self._get_state()
+        self._set_current_dir_lbl()
+        self._favorites_dialog.run()
+
+    def _hide_favorites_menu(self, widget=None, eve=None):
+        self._favorites_dialog.hide()
+
+    def _set_selected(self, user_data):
+        selected = user_data.get_selected()[1]
+        if selected and not self._selected == selected:
+            self._selected = selected

plugins/file_properties/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/file_properties/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/file_properties/file_properties.glade

@@ -0,0 +1,685 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated with glade 3.38.2 -->
+<interface>
+  <requires lib="gtk+" version="3.16"/>
+  <object class="GtkDialog" id="file_properties_dialog">
+    <property name="can-focus">False</property>
+    <property name="border-width">6</property>
+    <property name="title" translatable="yes">File Properties</property>
+    <property name="modal">True</property>
+    <property name="window-position">center-on-parent</property>
+    <property name="default-width">420</property>
+    <property name="destroy-with-parent">True</property>
+    <property name="type-hint">dialog</property>
+    <property name="skip-taskbar-hint">True</property>
+    <property name="skip-pager-hint">True</property>
+    <property name="gravity">center</property>
+    <signal name="response" handler="on_filePropertiesDlg_response" swapped="no"/>
+    <child internal-child="vbox">
+      <object class="GtkBox" id="dialog_vbox">
+        <property name="visible">True</property>
+        <property name="can-focus">False</property>
+        <property name="spacing">12</property>
+        <child internal-child="action_area">
+          <object class="GtkButtonBox" id="dialog_action_area">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <property name="layout-style">end</property>
+            <child>
+              <object class="GtkButton" id="cancel_button">
+                <property name="label">gtk-cancel</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="can-default">True</property>
+                <property name="receives-default">False</property>
+                <property name="use-stock">True</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkButton" id="ok_button">
+                <property name="label">gtk-ok</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="can-default">True</property>
+                <property name="receives-default">False</property>
+                <property name="use-stock">True</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">False</property>
+            <property name="pack-type">end</property>
+            <property name="position">0</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkNotebook" id="notebook">
+            <property name="visible">True</property>
+            <property name="can-focus">True</property>
+            <property name="border-width">6</property>
+            <child>
+              <object class="GtkAlignment" id="alignment2">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="top-padding">6</property>
+                <property name="bottom-padding">6</property>
+                <property name="left-padding">12</property>
+                <child>
+                  <object class="GtkTable" id="general_table">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="border-width">4</property>
+                    <property name="n-rows">7</property>
+                    <property name="n-columns">2</property>
+                    <property name="column-spacing">12</property>
+                    <property name="row-spacing">6</property>
+                    <child>
+                      <object class="GtkLabel" id="label_filename">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">&lt;b&gt;File _Name:&lt;/b&gt;</property>
+                        <property name="use-markup">True</property>
+                        <property name="use-underline">True</property>
+                        <property name="mnemonic-widget">file_name</property>
+                        <property name="xalign">0</property>
+                      </object>
+                      <packing>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkEntry" id="file_name">
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                      </object>
+                      <packing>
+                        <property name="left-attach">1</property>
+                        <property name="right-attach">2</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label20">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">&lt;b&gt;_Location:&lt;/b&gt;</property>
+                        <property name="use-markup">True</property>
+                        <property name="use-underline">True</property>
+                        <property name="mnemonic-widget">file_location</property>
+                        <property name="xalign">0</property>
+                      </object>
+                      <packing>
+                        <property name="top-attach">1</property>
+                        <property name="bottom-attach">2</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkEntry" id="file_location">
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                      </object>
+                      <packing>
+                        <property name="left-attach">1</property>
+                        <property name="right-attach">2</property>
+                        <property name="top-attach">1</property>
+                        <property name="bottom-attach">2</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label_target">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Link _Target:&lt;/b&gt;</property>
+                        <property name="use-markup">True</property>
+                        <property name="use-underline">True</property>
+                        <property name="mnemonic-widget">file_target</property>
+                        <property name="xalign">0</property>
+                      </object>
+                      <packing>
+                        <property name="top-attach">2</property>
+                        <property name="bottom-attach">3</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkEntry" id="file_target">
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                      </object>
+                      <packing>
+                        <property name="left-attach">1</property>
+                        <property name="right-attach">2</property>
+                        <property name="top-attach">2</property>
+                        <property name="bottom-attach">3</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label4">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Type:&lt;/b&gt;</property>
+                        <property name="use-markup">True</property>
+                        <property name="use-underline">True</property>
+                        <property name="xalign">0</property>
+                        <property name="yalign">0</property>
+                      </object>
+                      <packing>
+                        <property name="top-attach">3</property>
+                        <property name="bottom-attach">4</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options">GTK_FILL</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="mime_type">
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                        <property name="selectable">True</property>
+                        <property name="ellipsize">end</property>
+                        <property name="xalign">0</property>
+                        <property name="yalign">0</property>
+                      </object>
+                      <packing>
+                        <property name="left-attach">1</property>
+                        <property name="right-attach">2</property>
+                        <property name="top-attach">3</property>
+                        <property name="bottom-attach">4</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label5">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">&lt;b&gt;Size:&lt;/b&gt;</property>
+                        <property name="use-markup">True</property>
+                        <property name="use-underline">True</property>
+                        <property name="xalign">0</property>
+                      </object>
+                      <packing>
+                        <property name="top-attach">4</property>
+                        <property name="bottom-attach">5</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="file_size">
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                        <property name="selectable">True</property>
+                        <property name="xalign">0</property>
+                      </object>
+                      <packing>
+                        <property name="left-attach">1</property>
+                        <property name="right-attach">2</property>
+                        <property name="top-attach">4</property>
+                        <property name="bottom-attach">5</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label7">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">&lt;b&gt;_Modified:&lt;/b&gt;</property>
+                        <property name="use-markup">True</property>
+                        <property name="use-underline">True</property>
+                        <property name="xalign">0</property>
+                      </object>
+                      <packing>
+                        <property name="top-attach">5</property>
+                        <property name="bottom-attach">6</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label13">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="label" translatable="yes">&lt;b&gt;_Accessed:&lt;/b&gt;</property>
+                        <property name="use-markup">True</property>
+                        <property name="use-underline">True</property>
+                        <property name="xalign">0</property>
+                      </object>
+                      <packing>
+                        <property name="top-attach">6</property>
+                        <property name="bottom-attach">7</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkEntry" id="mtime">
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                      </object>
+                      <packing>
+                        <property name="left-attach">1</property>
+                        <property name="right-attach">2</property>
+                        <property name="top-attach">5</property>
+                        <property name="bottom-attach">6</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkEntry" id="atime">
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                      </object>
+                      <packing>
+                        <property name="left-attach">1</property>
+                        <property name="right-attach">2</property>
+                        <property name="top-attach">6</property>
+                        <property name="bottom-attach">7</property>
+                        <property name="x-options">GTK_FILL</property>
+                        <property name="y-options"/>
+                      </packing>
+                    </child>
+                  </object>
+                </child>
+              </object>
+            </child>
+            <child type="tab">
+              <object class="GtkLabel" id="label1">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="label" translatable="yes">_Info</property>
+                <property name="use-underline">True</property>
+              </object>
+              <packing>
+                <property name="tab-fill">False</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkAlignment" id="alignment1">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="top-padding">6</property>
+                <property name="bottom-padding">6</property>
+                <property name="left-padding">12</property>
+                <child>
+                  <object class="GtkVBox" id="vbox1">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="spacing">6</property>
+                    <child>
+                      <object class="GtkTable" id="table3">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="border-width">2</property>
+                        <property name="n-rows">2</property>
+                        <property name="n-columns">2</property>
+                        <property name="column-spacing">12</property>
+                        <property name="row-spacing">6</property>
+                        <child>
+                          <object class="GtkLabel" id="owner_label">
+                            <property name="visible">True</property>
+                            <property name="can-focus">False</property>
+                            <property name="label" translatable="yes">&lt;b&gt;_Owner:&lt;/b&gt;</property>
+                            <property name="use-markup">True</property>
+                            <property name="use-underline">True</property>
+                            <property name="mnemonic-widget">file_owner</property>
+                            <property name="xalign">0</property>
+                          </object>
+                          <packing>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkLabel" id="group_label">
+                            <property name="visible">True</property>
+                            <property name="can-focus">False</property>
+                            <property name="label" translatable="yes">&lt;b&gt;_Group:&lt;/b&gt;</property>
+                            <property name="use-markup">True</property>
+                            <property name="use-underline">True</property>
+                            <property name="mnemonic-widget">file_group</property>
+                            <property name="xalign">0</property>
+                          </object>
+                          <packing>
+                            <property name="top-attach">1</property>
+                            <property name="bottom-attach">2</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkEntry" id="file_owner">
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">1</property>
+                            <property name="right-attach">2</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkEntry" id="file_group">
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">1</property>
+                            <property name="right-attach">2</property>
+                            <property name="top-attach">1</property>
+                            <property name="bottom-attach">2</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                      </object>
+                      <packing>
+                        <property name="expand">False</property>
+                        <property name="fill">False</property>
+                        <property name="position">0</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkHSeparator" id="hseparator1">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                      </object>
+                      <packing>
+                        <property name="expand">False</property>
+                        <property name="fill">False</property>
+                        <property name="position">1</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkTable" id="table2">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <property name="border-width">4</property>
+                        <property name="n-rows">3</property>
+                        <property name="n-columns">5</property>
+                        <property name="column-spacing">12</property>
+                        <property name="row-spacing">6</property>
+                        <child>
+                          <object class="GtkLabel" id="label17">
+                            <property name="visible">True</property>
+                            <property name="can-focus">False</property>
+                            <property name="label" translatable="yes">&lt;b&gt;Owner:&lt;/b&gt;</property>
+                            <property name="use-markup">True</property>
+                            <property name="use-underline">True</property>
+                            <property name="xalign">0</property>
+                          </object>
+                          <packing>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkLabel" id="label18">
+                            <property name="visible">True</property>
+                            <property name="can-focus">False</property>
+                            <property name="label" translatable="yes">&lt;b&gt;Group:&lt;/b&gt;</property>
+                            <property name="use-markup">True</property>
+                            <property name="use-underline">True</property>
+                            <property name="xalign">0</property>
+                          </object>
+                          <packing>
+                            <property name="top-attach">1</property>
+                            <property name="bottom-attach">2</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkLabel" id="label19">
+                            <property name="visible">True</property>
+                            <property name="can-focus">False</property>
+                            <property name="label" translatable="yes">&lt;b&gt;Other:&lt;/b&gt;</property>
+                            <property name="use-markup">True</property>
+                            <property name="use-underline">True</property>
+                            <property name="xalign">0</property>
+                          </object>
+                          <packing>
+                            <property name="top-attach">2</property>
+                            <property name="bottom-attach">3</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkCheckButton" id="owner_r">
+                            <property name="label" translatable="yes">Read</property>
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                            <property name="receives-default">False</property>
+                            <property name="border-width">2</property>
+                            <property name="use-underline">True</property>
+                            <property name="draw-indicator">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">1</property>
+                            <property name="right-attach">2</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkCheckButton" id="group_r">
+                            <property name="label" translatable="yes">Read</property>
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                            <property name="receives-default">False</property>
+                            <property name="border-width">2</property>
+                            <property name="use-underline">True</property>
+                            <property name="draw-indicator">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">1</property>
+                            <property name="right-attach">2</property>
+                            <property name="top-attach">1</property>
+                            <property name="bottom-attach">2</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkCheckButton" id="others_r">
+                            <property name="label" translatable="yes">Read</property>
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                            <property name="receives-default">False</property>
+                            <property name="border-width">2</property>
+                            <property name="use-underline">True</property>
+                            <property name="draw-indicator">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">1</property>
+                            <property name="right-attach">2</property>
+                            <property name="top-attach">2</property>
+                            <property name="bottom-attach">3</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkCheckButton" id="owner_w">
+                            <property name="label" translatable="yes">Write</property>
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                            <property name="receives-default">False</property>
+                            <property name="border-width">2</property>
+                            <property name="use-underline">True</property>
+                            <property name="draw-indicator">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">2</property>
+                            <property name="right-attach">3</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkCheckButton" id="group_w">
+                            <property name="label" translatable="yes">Write</property>
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                            <property name="receives-default">False</property>
+                            <property name="border-width">2</property>
+                            <property name="use-underline">True</property>
+                            <property name="draw-indicator">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">2</property>
+                            <property name="right-attach">3</property>
+                            <property name="top-attach">1</property>
+                            <property name="bottom-attach">2</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkCheckButton" id="others_w">
+                            <property name="label" translatable="yes">Write</property>
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                            <property name="receives-default">False</property>
+                            <property name="border-width">2</property>
+                            <property name="use-underline">True</property>
+                            <property name="draw-indicator">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">2</property>
+                            <property name="right-attach">3</property>
+                            <property name="top-attach">2</property>
+                            <property name="bottom-attach">3</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkCheckButton" id="owner_x">
+                            <property name="label" translatable="yes">Execute</property>
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                            <property name="receives-default">False</property>
+                            <property name="border-width">2</property>
+                            <property name="use-underline">True</property>
+                            <property name="draw-indicator">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">3</property>
+                            <property name="right-attach">4</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkCheckButton" id="group_x">
+                            <property name="label" translatable="yes">Execute</property>
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                            <property name="receives-default">False</property>
+                            <property name="border-width">2</property>
+                            <property name="use-underline">True</property>
+                            <property name="draw-indicator">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">3</property>
+                            <property name="right-attach">4</property>
+                            <property name="top-attach">1</property>
+                            <property name="bottom-attach">2</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkCheckButton" id="others_x">
+                            <property name="label" translatable="yes">Execute</property>
+                            <property name="visible">True</property>
+                            <property name="can-focus">True</property>
+                            <property name="receives-default">False</property>
+                            <property name="border-width">2</property>
+                            <property name="use-underline">True</property>
+                            <property name="draw-indicator">True</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">3</property>
+                            <property name="right-attach">4</property>
+                            <property name="top-attach">2</property>
+                            <property name="bottom-attach">3</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options"/>
+                          </packing>
+                        </child>
+                        <child>
+                          <object class="GtkVSeparator" id="vseparator1">
+                            <property name="visible">True</property>
+                            <property name="can-focus">False</property>
+                          </object>
+                          <packing>
+                            <property name="left-attach">4</property>
+                            <property name="right-attach">5</property>
+                            <property name="bottom-attach">3</property>
+                            <property name="x-options">GTK_FILL</property>
+                            <property name="y-options">GTK_FILL</property>
+                          </packing>
+                        </child>
+                      </object>
+                      <packing>
+                        <property name="expand">False</property>
+                        <property name="fill">True</property>
+                        <property name="position">2</property>
+                      </packing>
+                    </child>
+                  </object>
+                </child>
+              </object>
+              <packing>
+                <property name="position">1</property>
+              </packing>
+            </child>
+            <child type="tab">
+              <object class="GtkLabel" id="label2">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="label" translatable="yes">_Permissions</property>
+                <property name="use-underline">True</property>
+              </object>
+              <packing>
+                <property name="position">1</property>
+                <property name="tab-fill">False</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">True</property>
+            <property name="position">2</property>
+          </packing>
+        </child>
+      </object>
+    </child>
+    <action-widgets>
+      <action-widget response="-6">cancel_button</action-widget>
+      <action-widget response="-5">ok_button</action-widget>
+    </action-widgets>
+  </object>
+</interface>

plugins/file_properties/manifest.json

@@ -0,0 +1,12 @@
+{
+    "manifest": {
+        "name": "Properties",
+        "author": "ITDominator",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "context_menu",
+            "pass_fm_events": "true"
+        }
+    }
+}

plugins/file_properties/plugin.py

@@ -0,0 +1,249 @@
+# Python imports
+import os
+import threading
+import subprocess
+import time
+import pwd
+import grp
+from datetime import datetime
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+from gi.repository import GLib
+from gi.repository import Gio
+
+# Application imports
+from plugins.plugin_base import PluginBase
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+    return wrapper
+
+
+
+
+class Properties:
+    file_uri: str      = None
+    file_name: str     = None
+    file_location: str = None
+    file_target: str   = None
+    mime_type: str     = None
+    file_size: str     = None
+    mtime: int         = None
+    atime: int         = None
+    file_owner: str    = None
+    file_group: str    = None
+    chmod_stat: str    = None
+
+
+class Plugin(PluginBase):
+    def __init__(self):
+        super().__init__()
+
+        self.path               = os.path.dirname(os.path.realpath(__file__))
+        self._GLADE_FILE        = f"{self.path}/file_properties.glade"
+        self.name               = "Properties"  # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                                #       where self.name should not be needed for message comms
+
+        self._properties_dialog = None
+        self._file_name         = None
+        self._file_location     = None
+        self._file_target       = None
+        self._mime_type         = None
+        self._file_size         = None
+        self._mtime             = None
+        self._atime             = None
+        self._file_owner        = None
+        self._file_group        = None
+
+        self._chmod_map: {} = {
+                                "7": "rwx",
+                                "6": "rw",
+                                "5": "rx",
+                                "4": "r",
+                                "3": "wx",
+                                "2": "w",
+                                "1": "x",
+                                "0": ""
+        }
+
+        self._chmod_map_counter: {} = {
+                                 "rwx": "7",
+                                 "rw":  "6",
+                                 "rx":  "5",
+                                 "r":   "4",
+                                 "wx":  "3",
+                                 "w":   "2",
+                                 "x":   "1",
+                                 "":    "0"
+        }
+
+
+    def run(self):
+        self._builder           = Gtk.Builder()
+        self._builder.add_from_file(self._GLADE_FILE)
+
+        self._properties_dialog = self._builder.get_object("file_properties_dialog")
+        self._file_name     = self._builder.get_object("file_name")
+        self._file_location = self._builder.get_object("file_location")
+        self._file_target   = self._builder.get_object("file_target")
+        self._mime_type     = self._builder.get_object("mime_type")
+        self._file_size     = self._builder.get_object("file_size")
+        self._mtime         = self._builder.get_object("mtime")
+        self._atime         = self._builder.get_object("atime")
+        self._file_owner    = self._builder.get_object("file_owner")
+        self._file_group    = self._builder.get_object("file_group")
+
+    def generate_reference_ui_element(self):
+        item = Gtk.ImageMenuItem(self.name)
+        item.set_image( Gtk.Image(stock=Gtk.STOCK_PROPERTIES) )
+        item.connect("activate", self._show_properties_page)
+        item.set_always_show_image(True)
+        return item
+
+
+    @threaded
+    def _show_properties_page(self, widget=None, eve=None):
+        event_system.emit("get_current_state")
+
+        state               = self._fm_state
+        self._event_message = None
+
+        GLib.idle_add(self._process_changes, (state))
+
+    def _process_changes(self, state):
+        if len(state.selected_files) == 1:
+            uri  = state.selected_files[0]
+            path = state.tab.get_current_directory()
+
+
+            properties = self._set_ui_data(uri, path)
+            response   = self._properties_dialog.run()
+            if response in [Gtk.ResponseType.CANCEL, Gtk.ResponseType.DELETE_EVENT]:
+                self._properties_dialog.hide()
+
+            self._update_file(properties)
+            self._properties_dialog.hide()
+
+
+    def _update_file(self, properties):
+        chmod_stat = self._get_check_boxes()
+
+        if chmod_stat is not properties.chmod_stat:
+            try:
+                print("\nNew chmod flags...")
+                print(f"Old:  {''.join(properties.chmod_stat)}")
+                print(f"New:  {chmod_stat}")
+
+                command = ["chmod", f"{chmod_stat}", properties.file_uri]
+                with subprocess.Popen(command, stdout=subprocess.PIPE) as proc:
+                    result = proc.stdout.read().decode("UTF-8").strip()
+                    print(result)
+            except Exception as e:
+                print(f"Couldn't chmod\nFile:  {properties.file_uri}")
+                print( repr(e) )
+
+
+        owner = self._file_owner.get_text()
+        group = self._file_group.get_text()
+        if owner is not properties.file_owner or group is not properties.file_group:
+            try:
+                print("\nNew owner/group flags...")
+                print(f"Old:\n\tOwner: {properties.file_owner}\n\tGroup: {properties.file_group}")
+                print(f"New:\n\tOwner: {owner}\n\tGroup: {group}")
+
+                uid = pwd.getpwnam(owner).pw_uid
+                gid = grp.getgrnam(group).gr_gid
+                os.chown(properties.file_uri, uid, gid)
+            except Exception as e:
+                print(f"Couldn't chmod\nFile:  {properties.file_uri}")
+                print( repr(e) )
+
+
+    def _set_ui_data(self, uri, path):
+        properties = Properties()
+        file_info  = Gio.File.new_for_path(uri).query_info(attributes="standard::*,owner::*,time::access,time::changed",
+                                                            flags=Gio.FileQueryInfoFlags.NONE,
+                                                            cancellable=None)
+
+        is_symlink               = file_info.get_attribute_as_string("standard::is-symlink")
+        properties.file_uri      = uri
+        properties.file_target   = file_info.get_attribute_as_string("standard::symlink-target") if is_symlink else ""
+        properties.file_name     = file_info.get_display_name()
+        properties.file_location = path
+        properties.mime_type     = file_info.get_content_type()
+        properties.file_size     = self._sizeof_fmt(file_info.get_size())
+        properties.mtime         = datetime.fromtimestamp( int(file_info.get_attribute_as_string("time::changed")) ).strftime("%A, %B %d, %Y %I:%M:%S")
+        properties.atime         = datetime.fromtimestamp( int(file_info.get_attribute_as_string("time::access")) ).strftime("%A, %B %d, %Y %I:%M:%S")
+        properties.file_owner    = file_info.get_attribute_as_string("owner::user")
+        properties.file_group    = file_info.get_attribute_as_string("owner::group")
+
+        # NOTE: Read = 4,  Write = 2,  Exec = 1
+        command = ["stat", "-c", "%a", uri]
+        with subprocess.Popen(command, stdout=subprocess.PIPE) as proc:
+            properties.chmod_stat = list(proc.stdout.read().decode("UTF-8").strip())
+            owner  = self._chmod_map[f"{properties.chmod_stat[0]}"]
+            group  = self._chmod_map[f"{properties.chmod_stat[1]}"]
+            others = self._chmod_map[f"{properties.chmod_stat[2]}"]
+
+            self._reset_check_boxes()
+            self._set_check_boxes([["owner", owner], ["group", group], ["others", others]])
+
+        self._file_name.set_text(properties.file_name)
+        self._file_location.set_text(properties.file_location)
+        self._file_target.set_text(properties.file_target)
+        self._mime_type.set_label(properties.mime_type)
+        self._file_size.set_label(properties.file_size)
+        self._mtime.set_text(properties.mtime)
+        self._atime.set_text(properties.atime)
+        self._file_owner.set_text(properties.file_owner)
+        self._file_group.set_text(properties.file_group)
+
+        return properties
+
+
+
+
+    def _get_check_boxes(self):
+        perms = [[], [], []]
+
+        for i, target in enumerate(["owner", "group", "others"]):
+            for type in ["r", "w", "x"]:
+                is_active = self._builder.get_object(f"{target}_{type}").get_active()
+                if is_active:
+                    perms[i].append(type)
+
+        digits = []
+        for perm in perms:
+            digits.append(self._chmod_map_counter[ ''.join(perm) ])
+
+        return ''.join(digits)
+
+    def _set_check_boxes(self, targets):
+        for name, target in targets:
+            for type in list(target):
+                obj = f"{name}_{type}"
+                self._builder.get_object(obj).set_active(True)
+
+    def _reset_check_boxes(self):
+        for target in ["owner", "group", "others"]:
+            for type in ["r", "w", "x"]:
+                self._builder.get_object(f"{target}_{type}").set_active(False)
+
+    def _sizeof_fmt(self, num, suffix="B"):
+        for unit in ["", "K", "M", "G", "T", "Pi", "Ei", "Zi"]:
+            if abs(num) < 1024.0:
+                return f"{num:3.1f} {unit}{suffix}"
+            num /= 1024.0
+        return f"{num:.1f} Yi{suffix}"

plugins/movie_tv_info/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/movie_tv_info/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/movie_tv_info/manifest.json

@@ -0,0 +1,12 @@
+{
+    "manifest": {
+        "name": "Movie/TV Info",
+        "author": "ITDominator",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "context_menu_plugins",
+            "pass_fm_events": "true"
+        }
+    }
+}

plugins/movie_tv_info/movie_tv_info.glade

@@ -0,0 +1,220 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated with glade 3.38.2 -->
+<interface>
+  <requires lib="gtk+" version="3.16"/>
+  <object class="GtkImage" id="image1">
+    <property name="visible">True</property>
+    <property name="can-focus">False</property>
+    <property name="stock">gtk-media-play</property>
+  </object>
+  <object class="GtkAdjustment" id="scrub_step_adjuster">
+    <property name="lower">1</property>
+    <property name="upper">100</property>
+    <property name="value">65</property>
+    <property name="step-increment">1</property>
+    <property name="page-increment">10</property>
+  </object>
+  <object class="GtkTextBuffer" id="textbuffer"/>
+  <object class="GtkDialog" id="info_dialog">
+    <property name="can-focus">False</property>
+    <property name="border-width">6</property>
+    <property name="title" translatable="yes">Movie / TV Info</property>
+    <property name="modal">True</property>
+    <property name="window-position">center-on-parent</property>
+    <property name="default-width">420</property>
+    <property name="destroy-with-parent">True</property>
+    <property name="type-hint">dialog</property>
+    <property name="skip-taskbar-hint">True</property>
+    <property name="skip-pager-hint">True</property>
+    <property name="gravity">center</property>
+    <child internal-child="vbox">
+      <object class="GtkBox" id="dialog_vbox">
+        <property name="visible">True</property>
+        <property name="can-focus">False</property>
+        <property name="spacing">12</property>
+        <child internal-child="action_area">
+          <object class="GtkButtonBox" id="dialog_action_area">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <property name="layout-style">end</property>
+            <child>
+              <object class="GtkButton" id="cancel_button">
+                <property name="label">gtk-close</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="can-default">True</property>
+                <property name="receives-default">False</property>
+                <property name="use-stock">True</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">False</property>
+            <property name="pack-type">end</property>
+            <property name="position">0</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkBox">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <property name="orientation">vertical</property>
+            <child>
+              <object class="GtkImage" id="thumbnail_preview_img">
+                <property name="height-request">320</property>
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="margin-start">5</property>
+                <property name="margin-end">5</property>
+                <property name="margin-top">5</property>
+                <property name="stock">gtk-missing-image</property>
+                <property name="icon_size">6</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkBox">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="orientation">vertical</property>
+                <child>
+                  <object class="GtkTextView" id="overview_textview">
+                    <property name="height-request">120</property>
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="margin-start">5</property>
+                    <property name="margin-end">5</property>
+                    <property name="margin-top">10</property>
+                    <property name="margin-bottom">10</property>
+                    <property name="wrap-mode">word</property>
+                    <property name="cursor-visible">False</property>
+                    <property name="buffer">textbuffer</property>
+                    <property name="overwrite">True</property>
+                    <property name="monospace">True</property>
+                  </object>
+                  <packing>
+                    <property name="expand">True</property>
+                    <property name="fill">True</property>
+                    <property name="position">0</property>
+                  </packing>
+                </child>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkTable" id="general_table">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="border-width">4</property>
+                <property name="n-rows">3</property>
+                <property name="n-columns">2</property>
+                <property name="column-spacing">12</property>
+                <property name="row-spacing">6</property>
+                <child>
+                  <object class="GtkLinkButton" id="trailer_link">
+                    <property name="label" translatable="yes">Trailer</property>
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="receives-default">True</property>
+                    <property name="image">image1</property>
+                  </object>
+                  <packing>
+                    <property name="right-attach">2</property>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkLabel">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="label" translatable="yes">&lt;b&gt;File _Name:&lt;/b&gt;</property>
+                    <property name="use-markup">True</property>
+                    <property name="use-underline">True</property>
+                    <property name="xalign">0</property>
+                  </object>
+                  <packing>
+                    <property name="top-attach">1</property>
+                    <property name="bottom-attach">2</property>
+                    <property name="x-options">GTK_FILL</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkLabel">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="label" translatable="yes">&lt;b&gt;_Location:&lt;/b&gt;</property>
+                    <property name="use-markup">True</property>
+                    <property name="use-underline">True</property>
+                    <property name="xalign">0</property>
+                  </object>
+                  <packing>
+                    <property name="top-attach">2</property>
+                    <property name="bottom-attach">3</property>
+                    <property name="x-options">GTK_FILL</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkEntry" id="file_name">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="editable">False</property>
+                  </object>
+                  <packing>
+                    <property name="left-attach">1</property>
+                    <property name="right-attach">2</property>
+                    <property name="top-attach">1</property>
+                    <property name="bottom-attach">2</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkEntry" id="file_location">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="editable">False</property>
+                  </object>
+                  <packing>
+                    <property name="left-attach">1</property>
+                    <property name="right-attach">2</property>
+                    <property name="top-attach">2</property>
+                    <property name="bottom-attach">3</property>
+                    <property name="x-options">GTK_FILL</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">2</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">True</property>
+            <property name="fill">True</property>
+            <property name="position">1</property>
+          </packing>
+        </child>
+      </object>
+    </child>
+    <action-widgets>
+      <action-widget response="-7">cancel_button</action-widget>
+    </action-widgets>
+  </object>
+</interface>

plugins/movie_tv_info/plugin.py

@@ -0,0 +1,187 @@
+# Python imports
+import os
+import threading
+import subprocess
+import inspect
+import requests
+import shutil
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+gi.require_version('GdkPixbuf', '2.0')
+from gi.repository import Gtk
+from gi.repository import GLib
+from gi.repository import GdkPixbuf
+
+# Application imports
+from plugins.plugin_base import PluginBase
+from .tmdbscraper import scraper
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+    return wrapper
+
+
+
+
+class Plugin(PluginBase):
+    def __init__(self):
+        super().__init__()
+
+        self.path                   = os.path.dirname(os.path.realpath(__file__))
+        self.name                   = "Movie/TV Info"   # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                                        #       where self.name should not be needed for message comms
+        self._GLADE_FILE            = f"{self.path}/movie_tv_info.glade"
+
+        self._dialog                = None
+        self._thumbnail_preview_img = None
+        self._tmdb                  = scraper.get_tmdb_scraper()
+        self._overview              = None
+        self._file_name             = None
+        self._file_location         = None
+        self._trailer_link          = None
+
+
+    def run(self):
+        self._builder           = Gtk.Builder()
+        self._builder.add_from_file(self._GLADE_FILE)
+
+        classes  = [self]
+        handlers = {}
+        for c in classes:
+            methods = None
+            try:
+                methods = inspect.getmembers(c, predicate=inspect.ismethod)
+                handlers.update(methods)
+            except Exception as e:
+                print(repr(e))
+
+        self._builder.connect_signals(handlers)
+
+        self._thumbnailer_dialog    = self._builder.get_object("info_dialog")
+        self._overview              = self._builder.get_object("textbuffer")
+        self._file_name             = self._builder.get_object("file_name")
+        self._file_location         = self._builder.get_object("file_location")
+        self._thumbnail_preview_img = self._builder.get_object("thumbnail_preview_img")
+        self._file_hash             = self._builder.get_object("file_hash")
+        self._trailer_link          = self._builder.get_object("trailer_link")
+
+    def generate_reference_ui_element(self):
+        item = Gtk.ImageMenuItem(self.name)
+        item.set_image( Gtk.Image(stock=Gtk.STOCK_FIND) )
+        item.connect("activate", self._show_info_page)
+        item.set_always_show_image(True)
+        return item
+
+    @threaded
+    def _show_info_page(self, widget=None, eve=None):
+        self._event_system.emit("get_current_state")
+
+        state               = self._fm_state
+        self._event_message = None
+
+        GLib.idle_add(self._process_changes, (state))
+
+    def _process_changes(self, state):
+        self._fm_state = None
+
+        if len(state.selected_files) == 1:
+            self._fm_state = state
+            self._set_ui_data()
+            response   = self._thumbnailer_dialog.run()
+            if response in [Gtk.ResponseType.CLOSE, Gtk.ResponseType.DELETE_EVENT]:
+                self._thumbnailer_dialog.hide()
+
+    def _set_ui_data(self):
+        title, path, trailer, video_data = self.get_video_data()
+        keys = video_data.keys() if video_data else None
+
+        overview_text  = video_data["overview"] if video_data else f"...NO {self.name.upper()} DATA..."
+
+        self.set_text_data(title, path, overview_text)
+        self.set_thumbnail(video_data) if video_data else ...
+        self.set_trailer_link(trailer)
+
+        print(video_data["videos"]) if not keys in ("", None) and "videos" in keys else ...
+
+    def get_video_data(self):
+        uri            = self._fm_state.selected_files[0]
+        path           = self._fm_state.tab.get_current_directory()
+        parts          = uri.split("/")
+        _title         = parts[ len(parts) - 1 ]
+        trailer        = None
+
+        try:
+            title          = _title.split("(")[0].strip()
+            startIndex     = _title.index('(') + 1
+            endIndex       = _title.index(')')
+            date           = title[startIndex:endIndex]
+        except Exception as e:
+            print(repr(e))
+            title          = _title
+            date           = None
+
+        try:
+
+            video_data    = self._tmdb.search(title, date)[0]
+            video_id      = video_data["id"]
+            try:
+                results = self._tmdb.tmdbapi.get_movie(str(video_id), append_to_response="videos")["videos"]["results"]
+                for result in results:
+                    if "YouTube" in result["site"]:
+                        trailer = result["key"]
+
+                if not trailer:
+                    raise Exception("No key found. Defering to none...")
+            except Exception as e:
+                print("No trailer found...")
+
+        except Exception as e:
+            print(repr(e))
+            video_data    = None
+
+        return title, path, trailer, video_data
+
+
+    def set_text_data(self, title, path, overview_text):
+        self._file_name.set_text(title)
+        self._file_location.set_text(path)
+        self._overview.set_text(overview_text)
+
+    @threaded
+    def set_thumbnail(self, video_data):
+        background_url = video_data["backdrop_path"]
+        # background_url = video_data["poster_path"]
+        background_pth = "/tmp/sfm_mvtv_info.jpg"
+
+        try:
+            os.remove(background_pth)
+        except Exception as e:
+            ...
+
+        r = requests.get(background_url, stream = True)
+
+        if r.status_code == 200:
+            r.raw.decode_content = True
+            with open(background_pth,'wb') as f:
+                shutil.copyfileobj(r.raw, f)
+
+            print('Cover Background Image sucessfully retreived...')
+            preview_pixbuf = GdkPixbuf.Pixbuf.new_from_file(background_pth)
+            self._thumbnail_preview_img.set_from_pixbuf(preview_pixbuf)
+        else:
+            print('Cover Background Image Couldn\'t be retreived...')
+
+    def set_trailer_link(self, trailer):
+        if trailer:
+            self._trailer_link.set_uri(f"https://www.youtube.com/watch?v={trailer}")

plugins/movie_tv_info/tmdbscraper/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/movie_tv_info/tmdbscraper/scraper.py

@@ -0,0 +1,22 @@
+import json
+import sys
+
+from .lib.tmdbscraper.tmdb import TMDBMovieScraper
+from .lib.tmdbscraper.fanarttv import get_details as get_fanarttv_artwork
+from .lib.tmdbscraper.imdbratings import get_details as get_imdb_details
+from .lib.tmdbscraper.traktratings import get_trakt_ratinginfo
+from .scraper_datahelper import combine_scraped_details_info_and_ratings, \
+    combine_scraped_details_available_artwork, find_uniqueids_in_text, get_params
+from .scraper_config import configure_scraped_details, PathSpecificSettings, \
+    configure_tmdb_artwork, is_fanarttv_configured
+
+
+
+
+
+
+def get_tmdb_scraper():
+    language       = 'en-US'
+    certcountry    = 'us'
+    ADDON_SETTINGS = None
+    return TMDBMovieScraper(ADDON_SETTINGS, language, certcountry)

plugins/movie_tv_info/tmdbscraper/scraper_config.py

@@ -0,0 +1,111 @@
+def configure_scraped_details(details, settings):
+    details = _configure_rating_prefix(details, settings)
+    details = _configure_keeporiginaltitle(details, settings)
+    details = _configure_trailer(details, settings)
+    details = _configure_multiple_studios(details, settings)
+    details = _configure_default_rating(details, settings)
+    details = _configure_tags(details, settings)
+    return details
+
+def configure_tmdb_artwork(details, settings):
+    if 'available_art' not in details:
+        return details
+
+    art = details['available_art']
+    fanart_enabled = settings.getSettingBool('fanart')
+    if not fanart_enabled:
+        if 'fanart' in art:
+            del art['fanart']
+        if 'set.fanart' in art:
+            del art['set.fanart']
+    if not settings.getSettingBool('landscape'):
+        if 'landscape' in art:
+            if fanart_enabled:
+                art['fanart'] = art.get('fanart', []) + art['landscape']
+            del art['landscape']
+        if 'set.landscape' in art:
+            if fanart_enabled:
+                art['set.fanart'] = art.get('set.fanart', []) + art['set.landscape']
+            del art['set.landscape']
+
+    return details
+
+def is_fanarttv_configured(settings):
+    return settings.getSettingBool('enable_fanarttv_artwork')
+
+def _configure_rating_prefix(details, settings):
+    if details['info'].get('mpaa'):
+        details['info']['mpaa'] = settings.getSettingString('certprefix') + details['info']['mpaa']
+    return details
+
+def _configure_keeporiginaltitle(details, settings):
+    if settings.getSettingBool('keeporiginaltitle'):
+        details['info']['title'] = details['info']['originaltitle']
+    return details
+
+def _configure_trailer(details, settings):
+    if details['info'].get('trailer') and not settings.getSettingBool('trailer'):
+        del details['info']['trailer']
+    return details
+
+def _configure_multiple_studios(details, settings):
+    if not settings.getSettingBool('multiple_studios'):
+        details['info']['studio'] = details['info']['studio'][:1]
+    return details
+
+def _configure_default_rating(details, settings):
+    imdb_default = bool(details['ratings'].get('imdb')) and settings.getSettingString('RatingS') == 'IMDb'
+    trakt_default = bool(details['ratings'].get('trakt')) and settings.getSettingString('RatingS') == 'Trakt'
+    default_rating = 'themoviedb'
+    if imdb_default:
+        default_rating = 'imdb'
+    elif trakt_default:
+        default_rating = 'trakt'
+    if default_rating not in details['ratings']:
+        default_rating = list(details['ratings'].keys())[0] if details['ratings'] else None
+    for rating_type in details['ratings'].keys():
+        details['ratings'][rating_type]['default'] = rating_type == default_rating
+    return details
+
+def _configure_tags(details, settings):
+    if not settings.getSettingBool('add_tags'):
+        del details['info']['tag']
+    return details
+
+# pylint: disable=invalid-name
+try:
+    basestring
+except NameError: # py2 / py3
+    basestring = str
+
+#pylint: disable=redefined-builtin
+class PathSpecificSettings(object):
+    # read-only shim for typed `xbmcaddon.Addon().getSetting*` methods
+    def __init__(self, settings_dict, log_fn):
+        self.data = settings_dict
+        self.log = log_fn
+
+    def getSettingBool(self, id):
+        return self._inner_get_setting(id, bool, False)
+
+    def getSettingInt(self, id):
+        return self._inner_get_setting(id, int, 0)
+
+    def getSettingNumber(self, id):
+        return self._inner_get_setting(id, float, 0.0)
+
+    def getSettingString(self, id):
+        return self._inner_get_setting(id, basestring, '')
+
+    def _inner_get_setting(self, setting_id, setting_type, default):
+        value = self.data.get(setting_id)
+        if isinstance(value, setting_type):
+            return value
+        self._log_bad_value(value, setting_id)
+        return default
+
+    def _log_bad_value(self, value, setting_id):
+        if value is None:
+            self.log("requested setting ({0}) was not found.".format(setting_id))
+        else:
+            self.log('failed to load value "{0}" for setting {1}'.format(value, setting_id))

plugins/movie_tv_info/tmdbscraper/scraper_datahelper.py

@@ -0,0 +1,54 @@
+import re
+try:
+    from urlparse import parse_qsl
+except ImportError: # py2 / py3
+    from urllib.parse import parse_qsl
+
+# get addon params from the plugin path querystring
+def get_params(argv):
+    result = {'handle': int(argv[0])}
+    if len(argv) < 2 or not argv[1]:
+        return result
+
+    result.update(parse_qsl(argv[1].lstrip('?')))
+    return result
+
+def combine_scraped_details_info_and_ratings(original_details, additional_details):
+    def update_or_set(details, key, value):
+        if key in details:
+            details[key].update(value)
+        else:
+            details[key] = value
+
+    if additional_details:
+        if additional_details.get('info'):
+            update_or_set(original_details, 'info', additional_details['info'])
+        if additional_details.get('ratings'):
+            update_or_set(original_details, 'ratings', additional_details['ratings'])
+    return original_details
+
+def combine_scraped_details_available_artwork(original_details, additional_details):
+    if additional_details and additional_details.get('available_art'):
+        available_art = additional_details['available_art']
+        if not original_details.get('available_art'):
+            original_details['available_art'] = available_art
+        else:
+            for arttype, artlist in available_art.items():
+                original_details['available_art'][arttype] = \
+                    artlist + original_details['available_art'].get(arttype, [])
+
+    return original_details
+
+def find_uniqueids_in_text(input_text):
+    result = {}
+    res = re.search(r'(themoviedb.org/movie/)([0-9]+)', input_text)
+    if (res):
+        result['tmdb'] = res.group(2)
+    res = re.search(r'imdb....?/title/tt([0-9]+)', input_text)
+    if (res):
+        result['imdb'] = 'tt' + res.group(1)
+    else:
+        res = re.search(r'imdb....?/Title\?t{0,2}([0-9]+)', input_text)
+        if (res):
+            result['imdb'] = 'tt' + res.group(1)
+    return result

plugins/movie_tv_info/tmdbscraper/scraper_xbmc.py

@@ -0,0 +1,175 @@
+import json
+import sys
+import xbmc
+import xbmcaddon
+import xbmcgui
+import xbmcplugin
+
+from lib.tmdbscraper.tmdb import TMDBMovieScraper
+from lib.tmdbscraper.fanarttv import get_details as get_fanarttv_artwork
+from lib.tmdbscraper.imdbratings import get_details as get_imdb_details
+from lib.tmdbscraper.traktratings import get_trakt_ratinginfo
+from scraper_datahelper import combine_scraped_details_info_and_ratings, \
+    combine_scraped_details_available_artwork, find_uniqueids_in_text, get_params
+from scraper_config import configure_scraped_details, PathSpecificSettings, \
+    configure_tmdb_artwork, is_fanarttv_configured
+
+ADDON_SETTINGS = xbmcaddon.Addon()
+ID = ADDON_SETTINGS.getAddonInfo('id')
+
+def log(msg, level=xbmc.LOGDEBUG):
+    xbmc.log(msg='[{addon}]: {msg}'.format(addon=ID, msg=msg), level=level)
+
+def get_tmdb_scraper(settings):
+    language = settings.getSettingString('language')
+    certcountry = settings.getSettingString('tmdbcertcountry')
+    return TMDBMovieScraper(ADDON_SETTINGS, language, certcountry)
+
+def search_for_movie(title, year, handle, settings):
+    log("Find movie with title '{title}' from year '{year}'".format(title=title, year=year), xbmc.LOGINFO)
+    title = _strip_trailing_article(title)
+    search_results = get_tmdb_scraper(settings).search(title, year)
+    if not search_results:
+        return
+    if 'error' in search_results:
+        header = "The Movie Database Python error searching with web service TMDB"
+        xbmcgui.Dialog().notification(header, search_results['error'], xbmcgui.NOTIFICATION_WARNING)
+        log(header + ': ' + search_results['error'], xbmc.LOGWARNING)
+        return
+
+    for movie in search_results:
+        listitem = _searchresult_to_listitem(movie)
+        uniqueids = {'tmdb': str(movie['id'])}
+        xbmcplugin.addDirectoryItem(handle=handle, url=build_lookup_string(uniqueids),
+            listitem=listitem, isFolder=True)
+
+_articles = [prefix + article for prefix in (', ', ' ') for article in ("the", "a", "an")]
+def _strip_trailing_article(title):
+    title = title.lower()
+    for article in _articles:
+        if title.endswith(article):
+            return title[:-len(article)]
+    return title
+
+def _searchresult_to_listitem(movie):
+    movie_info = {'title': movie['title']}
+    movie_label = movie['title']
+
+    movie_year = movie['release_date'].split('-')[0] if movie.get('release_date') else None
+    if movie_year:
+        movie_label += ' ({})'.format(movie_year)
+        movie_info['year'] = movie_year
+
+    listitem = xbmcgui.ListItem(movie_label, offscreen=True)
+
+    listitem.setInfo('video', movie_info)
+    if movie['poster_path']:
+        listitem.setArt({'thumb': movie['poster_path']})
+
+    return listitem
+
+# Low limit because a big list of artwork can cause trouble in some cases
+# (a column can be too large for the MySQL integration),
+# and how useful is a big list anyway? Not exactly rhetorical, this is an experiment.
+IMAGE_LIMIT = 10
+
+def add_artworks(listitem, artworks):
+    for arttype, artlist in artworks.items():
+        if arttype == 'fanart':
+            continue
+        for image in artlist[:IMAGE_LIMIT]:
+            listitem.addAvailableArtwork(image['url'], arttype)
+
+    fanart_to_set = [{'image': image['url'], 'preview': image['preview']}
+        for image in artworks['fanart'][:IMAGE_LIMIT]]
+    listitem.setAvailableFanart(fanart_to_set)
+
+def get_details(input_uniqueids, handle, settings):
+    if not input_uniqueids:
+        return False
+    details = get_tmdb_scraper(settings).get_details(input_uniqueids)
+    if not details:
+        return False
+    if 'error' in details:
+        header = "The Movie Database Python error with web service TMDB"
+        xbmcgui.Dialog().notification(header, details['error'], xbmcgui.NOTIFICATION_WARNING)
+        log(header + ': ' + details['error'], xbmc.LOGWARNING)
+        return False
+
+    details = configure_tmdb_artwork(details, settings)
+
+    if settings.getSettingString('RatingS') == 'IMDb' or settings.getSettingBool('imdbanyway'):
+        imdbinfo = get_imdb_details(details['uniqueids'])
+        if 'error' in imdbinfo:
+            header = "The Movie Database Python error with website IMDB"
+            log(header + ': ' + imdbinfo['error'], xbmc.LOGWARNING)
+        else:
+            details = combine_scraped_details_info_and_ratings(details, imdbinfo)
+
+    if settings.getSettingString('RatingS') == 'Trakt' or settings.getSettingBool('traktanyway'):
+        traktinfo = get_trakt_ratinginfo(details['uniqueids'])
+        details = combine_scraped_details_info_and_ratings(details, traktinfo)
+
+    if is_fanarttv_configured(settings):
+        fanarttv_info = get_fanarttv_artwork(details['uniqueids'],
+            settings.getSettingString('fanarttv_clientkey'),
+            settings.getSettingString('fanarttv_language'),
+            details['_info']['set_tmdbid'])
+        details = combine_scraped_details_available_artwork(details, fanarttv_info)
+
+    details = configure_scraped_details(details, settings)
+
+    listitem = xbmcgui.ListItem(details['info']['title'], offscreen=True)
+    listitem.setInfo('video', details['info'])
+    listitem.setCast(details['cast'])
+    listitem.setUniqueIDs(details['uniqueids'], 'tmdb')
+    add_artworks(listitem, details['available_art'])
+
+    for rating_type, value in details['ratings'].items():
+        if 'votes' in value:
+            listitem.setRating(rating_type, value['rating'], value['votes'], value['default'])
+        else:
+            listitem.setRating(rating_type, value['rating'], defaultt=value['default'])
+
+    xbmcplugin.setResolvedUrl(handle=handle, succeeded=True, listitem=listitem)
+    return True
+
+def find_uniqueids_in_nfo(nfo, handle):
+    uniqueids = find_uniqueids_in_text(nfo)
+    if uniqueids:
+        listitem = xbmcgui.ListItem(offscreen=True)
+        xbmcplugin.addDirectoryItem(
+            handle=handle, url=build_lookup_string(uniqueids), listitem=listitem, isFolder=True)
+
+def build_lookup_string(uniqueids):
+    return json.dumps(uniqueids)
+
+def parse_lookup_string(uniqueids):
+    try:
+        return json.loads(uniqueids)
+    except ValueError:
+        log("Can't parse this lookup string, is it from another add-on?\n" + uniqueids, xbmc.LOGWARNING)
+        return None
+
+def run():
+    params = get_params(sys.argv[1:])
+    enddir = True
+    if 'action' in params:
+        settings = ADDON_SETTINGS if not params.get('pathSettings') else \
+            PathSpecificSettings(json.loads(params['pathSettings']), lambda msg: log(msg, xbmc.LOGWARNING))
+        action = params["action"]
+        if action == 'find' and 'title' in params:
+            search_for_movie(params["title"], params.get("year"), params['handle'], settings)
+        elif action == 'getdetails' and 'url' in params:
+            enddir = not get_details(parse_lookup_string(params["url"]), params['handle'], settings)
+        elif action == 'NfoUrl' and 'nfo' in params:
+            find_uniqueids_in_nfo(params["nfo"], params['handle'])
+        else:
+            log("unhandled action: " + action, xbmc.LOGWARNING)
+    else:
+        log("No action in 'params' to act on", xbmc.LOGWARNING)
+    if enddir:
+        xbmcplugin.endOfDirectory(params['handle'])
+
+if __name__ == '__main__':
+    run()

plugins/searcher/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/searcher/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/searcher/manifest.json

@@ -0,0 +1,13 @@
+{
+    "manifest": {
+        "name": "Search",
+        "author": "ITDominator",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "context_menu",
+            "pass_fm_events": "true",
+            "bind_keys": ["Search||show_search_page:<Control>s"]
+        }
+    }
+}

plugins/searcher/mixins/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Mixins Module
+"""

plugins/searcher/mixins/file_search_mixin.py

@@ -0,0 +1,82 @@
+# Python imports
+import threading
+import subprocess
+import signal
+import json
+import shlex
+from datetime import datetime
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+from gi.repository import GLib
+
+# Application imports
+from ..widgets.file_preview_widget import FilePreviewWidget
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+
+    return wrapper
+
+
+class FileSearchMixin:
+    def _run_find_file_query(self, widget=None, eve=None):
+        self._handle_find_file_query(query=widget)
+
+    # TODO: Merge this logic with nearly the exact same thing in grep_search_mixin
+    @daemon_threaded
+    def _handle_find_file_query(self, widget=None, eve=None, query=None):
+        # NOTE: Freeze IPC consumption
+        self.pause_fifo_update  = True
+        self.search_query       = ""
+        dt                      = datetime.now()
+        self.fsearch_time_stamp = datetime.timestamp(dt) # NOTE: Get timestamp
+
+        # NOTE: Kill the former process
+        if self._list_proc:
+            if self._list_proc.poll() == None:
+                self._list_proc.terminate()
+                while self._list_proc.poll() == None:
+                    ...
+
+            self._list_proc = None
+
+        # NOTE: Clear children from ui and make sure ui thread redraws
+        GLib.idle_add(self.reset_file_list_box)
+
+        # NOTE: If query create new process and do all new loop.
+        if query:
+            self.pause_fifo_update = False
+            GLib.idle_add(self._exec_find_file_query, query)
+
+    def _exec_find_file_query(self, widget=None, eve=None):
+        query = widget.get_text()
+
+        if not query in ("", None):
+            self.search_query = query
+            target_dir = shlex.quote( self._fm_state.tab.get_current_directory() )
+            command = ["python", f"{self.path}/utils/search.py", "-t", "file_search", "-d", f"{target_dir}", "-q", f"{query}"]
+            self._list_proc = subprocess.Popen(command, cwd=self.path, stdin=None, stdout=None, stderr=None)
+
+    def _load_file_ui(self, data):
+        Gtk.main_iteration()
+
+        if not data in ("", None):
+            jdata  = json.loads( data )
+            target = jdata[0]
+            file   = jdata[1]
+
+            widget = FilePreviewWidget(target, file)
+            self._file_list.add(widget)

plugins/searcher/mixins/grep_search_mixin.py

@@ -0,0 +1,86 @@
+# Python imports
+import ctypes
+import threading
+import subprocess
+import signal
+import json
+import shlex
+from datetime import datetime
+libgcc_s = ctypes.CDLL('libgcc_s.so.1')
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+from gi.repository import GLib
+
+# Application imports
+from ..widgets.grep_preview_widget import GrepPreviewWidget
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+
+    return wrapper
+
+
+class GrepSearchMixin:
+    def _run_grep_query(self, widget=None, eve=None):
+        self._handle_grep_query(query=widget)
+
+    # TODO: Merge this logic with nearly the exact same thing in file_search_mixin
+    @daemon_threaded
+    def _handle_grep_query(self, widget=None, eve=None, query=None):
+        # NOTE: Freeze IPC consumption
+        self.pause_fifo_update = True
+        self.grep_query        = ""
+        dt                     = datetime.now()
+        self.grep_time_stamp   = datetime.timestamp(dt) # NOTE: Get timestamp
+
+        # NOTE: Kill the former process
+        if self._grep_proc:
+            if self._grep_proc.poll() == None:
+                self._grep_proc.terminate()
+                while self._grep_proc.poll() == None:
+                    ...
+
+            self._grep_proc = None
+
+        # NOTE: Clear children from ui and make sure ui thread redraws
+        GLib.idle_add(self.reset_grep_box)
+
+        # NOTE: If query create new process and do all new loop.
+        if query:
+            self.pause_fifo_update = False
+            GLib.idle_add(self._exec_grep_query, query)
+
+    def _exec_grep_query(self, widget=None, eve=None):
+        query = widget.get_text()
+        if not query.strip() in ("", None):
+            self.grep_query = query
+
+            target_dir = shlex.quote( self._fm_state.tab.get_current_directory() )
+            command = ["python", f"{self.path}/utils/search.py", "-t", "grep_search", "-d", f"{target_dir}", "-q", f"{query}"]
+            self._grep_proc = subprocess.Popen(command, cwd=self.path, stdin=None, stdout=None, stderr=None)
+
+    def _load_grep_ui(self, data):
+        Gtk.main_iteration()
+
+        if not data in ("", None):
+            jdata = json.loads( data )
+            jkeys = jdata.keys()
+            for key in jkeys:
+                sub_keys    = jdata[key].keys()
+                grep_result = jdata[key]
+
+                widget = GrepPreviewWidget(key, sub_keys, grep_result, self.grep_query)
+                self._grep_list.add(widget)

plugins/searcher/plugin.py

@@ -0,0 +1,139 @@
+# Python imports
+import os
+import threading
+import inspect
+import time
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+from gi.repository import GLib
+
+# Application imports
+from plugins.plugin_base import PluginBase
+from .mixins.file_search_mixin import FileSearchMixin
+from .mixins.grep_search_mixin import GrepSearchMixin
+from .utils.ipc_server import IPCServer
+
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+    return wrapper
+
+
+
+
+class Plugin(IPCServer, FileSearchMixin, GrepSearchMixin, PluginBase):
+    def __init__(self):
+        super().__init__()
+
+        self.path               = os.path.dirname(os.path.realpath(__file__))
+        self.name               = "Search"  # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                            #       where self.name should not be needed for message comms
+        self._GLADE_FILE        = f"{self.path}/search_dialog.glade"
+
+        self.update_list_ui_buffer = ()
+        self._search_dialog     = None
+        self._active_path       = None
+        self.file_list_parent   = None
+        self.grep_list_parent   = None
+        self._file_list         = None
+        self._grep_list         = None
+        self._grep_proc         = None
+        self._list_proc         = None
+        self.pause_fifo_update  = False
+        self.grep_time_stamp    = None
+        self.fsearch_time_stamp = None
+        self.grep_query         = ""
+        self.search_query       = ""
+
+
+    def run(self):
+        self._builder          = Gtk.Builder()
+        self._builder.add_from_file(self._GLADE_FILE)
+
+        classes  = [self]
+        handlers = {}
+        for c in classes:
+            methods = None
+            try:
+                methods = inspect.getmembers(c, predicate=inspect.ismethod)
+                handlers.update(methods)
+            except Exception as e:
+                print(repr(e))
+
+        self._builder.connect_signals(handlers)
+
+        self._search_dialog = self._builder.get_object("search_dialog")
+        self.fsearch        = self._builder.get_object("fsearch")
+
+        self.grep_list_parent = self._builder.get_object("grep_list_parent")
+        self.file_list_parent = self._builder.get_object("file_list_parent")
+
+        self._event_system.subscribe("update-file-ui", self._load_file_ui)
+        self._event_system.subscribe("update-grep-ui", self._load_grep_ui)
+        self._event_system.subscribe("show_search_page", self._show_page)
+
+
+        self.create_ipc_listener()
+
+    def generate_reference_ui_element(self):
+        item = Gtk.ImageMenuItem(self.name)
+        item.set_image( Gtk.Image(stock=Gtk.STOCK_FIND) )
+        item.connect("activate", self._show_page)
+        item.set_always_show_image(True)
+        return item
+
+
+    def _show_page(self, widget=None, eve=None):
+        self._event_system.emit("get_current_state")
+
+        state               = self._fm_state
+        self._event_message = None
+
+        self._active_path   = state.tab.get_current_directory()
+        response            = self._search_dialog.run()
+        self._search_dialog.hide()
+
+    # TODO: Merge the below methods into some unified logic
+    def reset_grep_box(self) -> None:
+        try:
+            child = self.grep_list_parent.get_children()[0]
+            self._grep_list = None
+            self.grep_list_parent.remove(child)
+        except Exception:
+            ...
+
+        self._grep_list = Gtk.Box()
+        self._grep_list.set_orientation(Gtk.Orientation.VERTICAL)
+        self.grep_list_parent.add(self._grep_list)
+        self.grep_list_parent.show_all()
+
+        time.sleep(0.05)
+        Gtk.main_iteration()
+
+    def reset_file_list_box(self) -> None:
+        try:
+            child = self.file_list_parent.get_children()[0]
+            self._file_list = None
+            self.file_list_parent.remove(child)
+        except Exception:
+            ...
+
+        self._file_list = Gtk.Box()
+        self._file_list.set_orientation(Gtk.Orientation.VERTICAL)
+        self.file_list_parent.add(self._file_list)
+        self.file_list_parent.show_all()
+
+        time.sleep(0.05)
+        Gtk.main_iteration()

plugins/searcher/search_dialog.glade

@@ -0,0 +1,277 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated with glade 3.38.2 -->
+<interface>
+  <requires lib="gtk+" version="3.16"/>
+  <object class="GtkDialog" id="search_dialog">
+    <property name="can-focus">False</property>
+    <property name="border-width">6</property>
+    <property name="title" translatable="yes">Search...</property>
+    <property name="modal">True</property>
+    <property name="window-position">center-on-parent</property>
+    <property name="default-width">720</property>
+    <property name="default-height">620</property>
+    <property name="destroy-with-parent">True</property>
+    <property name="type-hint">dialog</property>
+    <property name="skip-taskbar-hint">True</property>
+    <property name="skip-pager-hint">True</property>
+    <property name="gravity">center</property>
+    <child internal-child="vbox">
+      <object class="GtkBox" id="dialog_vbox">
+        <property name="visible">True</property>
+        <property name="can-focus">False</property>
+        <property name="spacing">12</property>
+        <child internal-child="action_area">
+          <object class="GtkButtonBox" id="dialog_action_area">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <property name="layout-style">end</property>
+            <child>
+              <object class="GtkButton" id="cancel_button">
+                <property name="label">gtk-cancel</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="can-default">True</property>
+                <property name="receives-default">False</property>
+                <property name="use-stock">True</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkButton" id="ok_button">
+                <property name="label">gtk-ok</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="can-default">True</property>
+                <property name="receives-default">False</property>
+                <property name="use-stock">True</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">False</property>
+            <property name="pack-type">end</property>
+            <property name="position">0</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkNotebook">
+            <property name="visible">True</property>
+            <property name="can-focus">True</property>
+            <property name="show-border">False</property>
+            <child>
+              <object class="GtkBox">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="orientation">vertical</property>
+                <child>
+                  <object class="GtkBox">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <child>
+                      <object class="GtkSearchEntry" id="fsearch">
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                        <property name="tooltip-text" translatable="yes">Query...</property>
+                        <property name="primary-icon-name">edit-find-symbolic</property>
+                        <property name="primary-icon-activatable">False</property>
+                        <property name="primary-icon-sensitive">False</property>
+                        <property name="placeholder-text" translatable="yes">Search for file...</property>
+                        <signal name="search-changed" handler="_run_find_file_query" swapped="no"/>
+                      </object>
+                      <packing>
+                        <property name="expand">True</property>
+                        <property name="fill">True</property>
+                        <property name="position">0</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkButton">
+                        <property name="label">gtk-stop</property>
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                        <property name="receives-default">True</property>
+                        <property name="use-stock">True</property>
+                        <signal name="released" handler="_handle_find_file_query" swapped="no"/>
+                      </object>
+                      <packing>
+                        <property name="expand">False</property>
+                        <property name="fill">True</property>
+                        <property name="position">1</property>
+                      </packing>
+                    </child>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="fill">True</property>
+                    <property name="position">0</property>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkScrolledWindow">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="hexpand">True</property>
+                    <property name="vexpand">True</property>
+                    <property name="shadow-type">in</property>
+                    <child>
+                      <object class="GtkViewport">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <child>
+                          <object class="GtkBox" id="file_list_parent">
+                            <property name="visible">True</property>
+                            <property name="can-focus">False</property>
+                            <property name="orientation">vertical</property>
+                            <property name="spacing">5</property>
+                            <property name="baseline-position">top</property>
+                            <child>
+                              <placeholder/>
+                            </child>
+                          </object>
+                        </child>
+                      </object>
+                    </child>
+                  </object>
+                  <packing>
+                    <property name="expand">True</property>
+                    <property name="fill">True</property>
+                    <property name="position">1</property>
+                  </packing>
+                </child>
+              </object>
+            </child>
+            <child type="tab">
+              <object class="GtkLabel">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="label" translatable="yes">File Search</property>
+              </object>
+              <packing>
+                <property name="tab-fill">False</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkBox">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="orientation">vertical</property>
+                <child>
+                  <object class="GtkBox">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <child>
+                      <object class="GtkSearchEntry">
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                        <property name="tooltip-text" translatable="yes">Query...</property>
+                        <property name="primary-icon-name">edit-find-symbolic</property>
+                        <property name="primary-icon-activatable">False</property>
+                        <property name="primary-icon-sensitive">False</property>
+                        <property name="placeholder-text" translatable="yes">Query string in file...</property>
+                        <signal name="search-changed" handler="_run_grep_query" swapped="no"/>
+                      </object>
+                      <packing>
+                        <property name="expand">True</property>
+                        <property name="fill">True</property>
+                        <property name="position">0</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkButton">
+                        <property name="label">gtk-stop</property>
+                        <property name="visible">True</property>
+                        <property name="can-focus">True</property>
+                        <property name="receives-default">True</property>
+                        <property name="use-stock">True</property>
+                        <signal name="released" handler="_handle_grep_query" swapped="no"/>
+                      </object>
+                      <packing>
+                        <property name="expand">False</property>
+                        <property name="fill">True</property>
+                        <property name="position">1</property>
+                      </packing>
+                    </child>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="fill">True</property>
+                    <property name="position">0</property>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkScrolledWindow">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="hexpand">True</property>
+                    <property name="vexpand">True</property>
+                    <property name="shadow-type">in</property>
+                    <child>
+                      <object class="GtkViewport">
+                        <property name="visible">True</property>
+                        <property name="can-focus">False</property>
+                        <child>
+                          <object class="GtkBox" id="grep_list_parent">
+                            <property name="visible">True</property>
+                            <property name="can-focus">False</property>
+                            <property name="orientation">vertical</property>
+                            <child>
+                              <placeholder/>
+                            </child>
+                          </object>
+                        </child>
+                      </object>
+                    </child>
+                  </object>
+                  <packing>
+                    <property name="expand">True</property>
+                    <property name="fill">True</property>
+                    <property name="position">1</property>
+                  </packing>
+                </child>
+              </object>
+              <packing>
+                <property name="position">1</property>
+              </packing>
+            </child>
+            <child type="tab">
+              <object class="GtkLabel">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="label" translatable="yes">Grep Search</property>
+              </object>
+              <packing>
+                <property name="position">1</property>
+                <property name="tab-fill">False</property>
+              </packing>
+            </child>
+            <child>
+              <placeholder/>
+            </child>
+            <child type="tab">
+              <placeholder/>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">True</property>
+            <property name="fill">True</property>
+            <property name="position">1</property>
+          </packing>
+        </child>
+      </object>
+    </child>
+    <action-widgets>
+      <action-widget response="-6">cancel_button</action-widget>
+      <action-widget response="-5">ok_button</action-widget>
+    </action-widgets>
+  </object>
+</interface>

plugins/searcher/utils/ipc_server.py

@@ -0,0 +1,98 @@
+# Python imports
+import os, threading, pickle
+from multiprocessing.connection import Listener, Client
+
+# Lib imports
+from gi.repository import GLib
+
+# Application imports
+
+
+
+
+class IPCServer:
+    """ Create a listener so that other SolarFM instances send requests back to existing instance. """
+    def __init__(self, ipc_address: str = '127.0.0.1', conn_type: str = "socket"):
+        self.is_ipc_alive     = False
+        self._ipc_port        = 4848
+        self._ipc_address     = ipc_address
+        self._conn_type       = conn_type
+        self._ipc_authkey     = b'' + bytes(f'solarfm-search_grep-ipc', 'utf-8')
+        self._ipc_timeout     = 15.0
+
+        if conn_type == "socket":
+            self._ipc_address = f'/tmp/solarfm-search_grep-ipc.sock'
+        elif conn_type == "full_network":
+            self._ipc_address = '0.0.0.0'
+        elif conn_type == "full_network_unsecured":
+            self._ipc_authkey = None
+            self._ipc_address = '0.0.0.0'
+        elif conn_type == "local_network_unsecured":
+            self._ipc_authkey = None
+
+
+    @daemon_threaded
+    def create_ipc_listener(self) -> None:
+        if self._conn_type == "socket":
+            if os.path.exists(self._ipc_address):
+                os.unlink(self._ipc_address)
+
+            listener = Listener(address=self._ipc_address, family="AF_UNIX", authkey=self._ipc_authkey)
+        elif "unsecured" not in self._conn_type:
+            listener = Listener((self._ipc_address, self._ipc_port), authkey=self._ipc_authkey)
+        else:
+            listener = Listener((self._ipc_address, self._ipc_port))
+
+
+        self.is_ipc_alive = True
+        while True:
+            conn = listener.accept()
+
+            if not self.pause_fifo_update:
+                self.handle_message(conn)
+            else:
+                conn.close()
+
+        listener.close()
+
+    def handle_message(self, conn) -> None:
+        while True:
+            msg  = conn.recv()
+
+            if "SEARCH|" in msg:
+                ts, file = msg.split("SEARCH|")[1].strip().split("|", 1)
+                try:
+                    timestamp = float(ts)
+                    if timestamp > self.fsearch_time_stamp and file:
+                        GLib.idle_add(self._load_file_ui, file, priority=GLib.PRIORITY_LOW)
+                except Exception as e:
+                    ...
+
+            if "GREP|" in msg:
+                ts, data = msg.split("GREP|")[1].strip().split("|", 1)
+                try:
+                    timestamp = float(ts)
+                    if timestamp > self.grep_time_stamp and data:
+                        GLib.idle_add(self._load_grep_ui, data, priority=GLib.PRIORITY_LOW)
+                except Exception as e:
+                    ...
+
+
+            conn.close()
+            break
+
+    def send_ipc_message(self, message: str = "Empty Data...") -> None:
+        try:
+            if self._conn_type == "socket":
+                conn = Client(address=self._ipc_address, family="AF_UNIX", authkey=self._ipc_authkey)
+            elif "unsecured" not in self._conn_type:
+                conn = Client((self._ipc_address, self._ipc_port), authkey=self._ipc_authkey)
+            else:
+                conn = Client((self._ipc_address, self._ipc_port))
+
+            conn.send(message)
+            conn.close()
+        except ConnectionRefusedError as e:
+            print("Connection refused...")
+        except Exception as e:
+            print(repr(e))

plugins/searcher/utils/search.py

@@ -0,0 +1,111 @@
+#!/usr/bin/python3
+
+
+# Python imports
+import os
+import traceback
+import argparse
+import subprocess
+import json
+import base64
+import time
+from datetime import datetime
+from setproctitle import setproctitle
+from multiprocessing.connection import Client
+
+# Lib imports
+
+# Application imports
+
+
+
+
+_ipc_address = f'/tmp/solarfm-search_grep-ipc.sock'
+_ipc_authkey = b'' + bytes(f'solarfm-search_grep-ipc', 'utf-8')
+
+filter = (".cpp", ".css", ".c", ".go", ".html", ".htm", ".java", ".js", ".json", ".lua", ".md", ".py", ".rs", ".toml", ".xml", ".pom") + \
+            (".txt", ".text", ".sh", ".cfg", ".conf", ".log")
+
+# NOTE: Create timestamp of when this launched. Is used in IPC to see if
+# we are stale and that new call didn't fully kill this or older processes.
+dt = datetime.now()
+ts = datetime.timestamp(dt)
+
+
+def send_ipc_message(message) -> None:
+    conn = Client(address=_ipc_address, family="AF_UNIX", authkey=_ipc_authkey)
+    conn.send(message)
+    conn.close()
+
+    # NOTE: Kinda important as this prevents overloading the UI thread
+    time.sleep(0.05)
+
+
+def file_search(path, query):
+    try:
+        for _path, _dir, _files in os.walk(path, topdown = True):
+             for file in _files:
+                 if query in file.lower():
+                     target = os.path.join(_path, file)
+                     data = f"SEARCH|{ts}|{json.dumps([target, file])}"
+                     send_ipc_message(data)
+    except Exception as e:
+        print("Couldn't traverse to path. Might be permissions related...")
+        traceback.print_exc()
+
+
+def grep_search(target=None, query=None):
+    if not query or not target:
+        return
+
+    # NOTE: -n = provide line numbers, -R = Search recursive in given target
+    #       -i = insensitive, -F = don't do regex parsing. (Treat as raw string)
+    command    = ["grep", "-n", "-R", "-i", "-F", query, target]
+    proc       = subprocess.Popen(command, stdout=subprocess.PIPE, encoding="utf-8")
+    raw_data   = proc.communicate()[0].strip()
+    proc_data  = raw_data.split("\n")   # NOTE: Will return data AFTER completion (if any)
+    collection = {}
+
+    for line in proc_data:
+        file, line_no, data = line.split(":", 2)
+        b64_file = base64.urlsafe_b64encode(file.encode('utf-8')).decode('utf-8')
+        b64_data = base64.urlsafe_b64encode(data.encode('utf-8')).decode('utf-8')
+
+        if b64_file in collection.keys():
+            collection[f"{b64_file}"][f"{line_no}"] = b64_data
+        else:
+            collection[f"{b64_file}"] = {}
+            collection[f"{b64_file}"] = { f"{line_no}": b64_data}
+
+    try:
+        data = f"GREP|{ts}|{json.dumps(collection, separators=(',', ':'), indent=4)}"
+        send_ipc_message(data)
+    except Exception as e:
+        ...
+
+    collection = {}
+
+
+def search(args):
+    if args.type == "file_search":
+        file_search(args.dir, args.query.lower())
+
+    if args.type == "grep_search":
+        grep_search(args.dir, args.query.encode("utf-8"))
+
+
+if __name__ == "__main__":
+    try:
+        setproctitle('SolarFM: File Search - Grepy')
+
+        parser = argparse.ArgumentParser()
+        # Add long and short arguments
+        parser.add_argument("--type", "-t", default=None, help="Type of search to do.")
+        parser.add_argument("--dir", "-d", default=None, help="Directory root for search type.")
+        parser.add_argument("--query", "-q", default=None, help="Query search is working against.")
+
+        # Read arguments (If any...)
+        args = parser.parse_args()
+        search(args)
+    except Exception as e:
+        traceback.print_exc()

plugins/searcher/widgets/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Widgets Module
+"""

plugins/searcher/widgets/file_preview_widget.py

@@ -0,0 +1,16 @@
+# Python imports
+
+# Gtk imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+
+# Application imports
+
+
+class FilePreviewWidget(Gtk.LinkButton):
+    def __init__(self, path, file):
+        super(FilePreviewWidget, self).__init__()
+        self.set_label(file)
+        self.set_uri(f"file://{path}")
+        self.show_all()

plugins/searcher/widgets/grep_preview_widget.py

@@ -0,0 +1,60 @@
+# Python imports
+import base64
+import re
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+
+# Application imports
+
+
+class GrepPreviewWidget(Gtk.Box):
+    def __init__(self, _path, sub_keys, _data, _grep_query):
+        super(GrepPreviewWidget, self).__init__()
+
+        self.set_orientation(Gtk.Orientation.VERTICAL)
+        line_color      = "#e0cc64"
+        highlight_color = "#FBF719"
+        grep_query      = _grep_query
+
+        path  = self.decode_str(_path)
+        lbl   = '/'.join( path.split("/")[-3:] )
+        title = Gtk.LinkButton.new_with_label(uri=f"file://{path}", label=lbl)
+
+        text_view = Gtk.TextView()
+        buffer    = text_view.get_buffer()
+        text_view.set_editable(False)
+        text_view.set_monospace(True)
+        text_view.set_wrap_mode(Gtk.WrapMode.NONE)
+
+        for i, key in enumerate(sub_keys):
+            line_num = self.make_utf8_line_num(line_color, key)
+            itr      = buffer.get_end_iter()
+            buffer.insert_markup(itr, line_num, len(line_num))
+
+            decoded  = f"\t{self.decode_str(_data[key])}"
+            self.make_utf8_line_highlight(buffer, itr, i, highlight_color, decoded, grep_query)
+
+        self.add(title)
+        self.add(text_view)
+        self.show_all()
+
+    def decode_str(self, target):
+        return base64.urlsafe_b64decode(target.encode('utf-8')).decode('utf-8')
+
+    def make_utf8_line_num(self, color, target):
+        return bytes(f"\n<span foreground='{color}'>{target}</span>", "utf-8").decode("utf-8")
+
+    def make_utf8_line_highlight(self, buffer, itr, i, color, target, query):
+        parts = re.split(r"(" + query + ")(?i)", target.replace("\n", ""))
+        for part in parts:
+            itr  = buffer.get_end_iter()
+
+            if not query.lower() == part.lower() and not query.lower() in part.lower():
+                buffer.insert(itr, part, length=len(part))
+            else:
+                new_s = f"<span foreground='#000000' background='{color}'>{part}</span>"
+                _part = bytes(new_s, "utf-8").decode("utf-8")
+                buffer.insert_markup(itr, _part, len(_part))

plugins/template/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/template/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/template/manifest.json

@@ -0,0 +1,13 @@
+{
+    "manifest": {
+        "name": "Example Plugin",
+        "author": "John Doe",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "plugin_control_list",
+            "pass_fm_events": "true",
+            "bind_keys": ["Example Plugin||send_message:<Control>f"]
+        }
+    }
+}

plugins/template/plugin.py

@@ -0,0 +1,49 @@
+# Python imports
+import os
+import threading
+import subprocess
+import ime
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+
+# Application imports
+from plugins.plugin_base import PluginBase
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+    return wrapper
+
+
+
+
+class Plugin(PluginBase):
+    def __init__(self):
+        super().__init__()
+
+        self.name               = "Example Plugin"  # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                                    #       where self.name should not be needed for message comms
+
+
+    def generate_reference_ui_element(self):
+        button = Gtk.Button(label=self.name)
+        button.connect("button-release-event", self.send_message)
+        return button
+
+    def run(self):
+        ...
+
+    def send_message(self, widget=None, eve=None):
+        message = "Hello, World!"
+        event_system.emit("display_message", ("warning", message, None))

plugins/trasher/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/trasher/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/trasher/manifest.json

@@ -0,0 +1,16 @@
+{
+    "manifest": {
+        "name": "Trasher",
+        "author": "ITDominator",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "context_menu",
+            "pass_fm_events": "true",
+            "bind_keys": [
+                            "Trasher||delete_files:Delete",
+                            "Trasher||trash_files:<Control>d"
+                        ]
+        }
+    }
+}

plugins/trasher/plugin.py

@@ -0,0 +1,138 @@
+# Python imports
+import os
+import threading
+import subprocess
+import inspect
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+from gi.repository import GLib
+from gi.repository import Gio
+
+# Application imports
+from plugins.plugin_base import PluginBase
+from .xdgtrash import XDGTrash
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+    return wrapper
+
+
+
+
+class Plugin(PluginBase):
+    def __init__(self):
+        super().__init__()
+        self.path               = os.path.dirname(os.path.realpath(__file__))
+        self._GLADE_FILE        = f"{self.path}/trasher.glade"
+        self.name               = "Trasher"  # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                             #       where self.name should not be needed for message comms
+        self.trashman           = XDGTrash()
+        self.trash_files_path   = f"{GLib.get_user_data_dir()}/Trash/files"
+        self.trash_info_path    = f"{GLib.get_user_data_dir()}/Trash/info"
+
+        self.trashman.regenerate()
+
+
+    def run(self):
+        self._event_system.subscribe("show_trash_buttons", self._show_trash_buttons)
+        self._event_system.subscribe("hide_trash_buttons", self._hide_trash_buttons)
+        self._event_system.subscribe("delete_files", self.delete_files)
+        self._event_system.subscribe("trash_files", self.trash_files)
+
+    def generate_reference_ui_element(self):
+        trash_a = Gtk.MenuItem("Trash Actions")
+        trash_menu = Gtk.Menu()
+
+        self.restore = Gtk.MenuItem("Restore From Trash")
+        self.restore.connect("activate", self.restore_trash_files)
+
+        self.empty = Gtk.MenuItem("Empty Trash")
+        self.empty.connect("activate", self.empty_trash)
+
+        trash = Gtk.ImageMenuItem("Trash")
+        trash.set_image( Gtk.Image.new_from_icon_name("user-trash", 16) )
+        trash.connect("activate", self.trash_files)
+        trash.set_always_show_image(True)
+
+        go_to = Gtk.ImageMenuItem("Go To Trash")
+        go_to.set_image( Gtk.Image.new_from_icon_name("user-trash", 16) )
+        go_to.connect("activate", self.go_to_trash)
+        go_to.set_always_show_image(True)
+
+        delete = Gtk.ImageMenuItem("Delete")
+        delete.set_image( Gtk.Image(stock=Gtk.STOCK_DELETE) )
+        delete.connect("activate", self.delete_files)
+        delete.set_always_show_image(True)
+
+        trash_a.set_submenu(trash_menu)
+        trash_a.show_all()
+        self._appen_menu_items(trash_menu, [self.restore, self.empty, trash, go_to, delete])
+
+        return trash_a
+
+
+    def _appen_menu_items(self, menu, items):
+        for item in items:
+            menu.append(item)
+
+
+    def _show_trash_buttons(self):
+        self.restore.show()
+        self.empty.show()
+
+    def _hide_trash_buttons(self):
+        self.restore.hide()
+        self.empty.hide()
+
+    def delete_files(self, widget = None, eve = None):
+        self._event_system.emit("get_current_state")
+        state    = self._fm_state
+        uris     = state.selected_files
+        response = None
+
+        state.warning_alert.format_secondary_text(f"Do you really want to delete the {len(uris)} file(s)?")
+        for uri in uris:
+            file = Gio.File.new_for_path(uri)
+
+            if not response:
+                response = state.warning_alert.run()
+                state.warning_alert.hide()
+            if response == Gtk.ResponseType.YES:
+                type = file.query_file_type(flags=Gio.FileQueryInfoFlags.NONE)
+
+                if type == Gio.FileType.DIRECTORY:
+                    state.tab.delete_file( file.get_path() )
+                else:
+                    file.delete(cancellable=None)
+            else:
+                break
+
+    def trash_files(self, widget = None, eve = None, verbocity = False):
+        self._event_system.emit("get_current_state")
+        state = self._fm_state
+        for uri in state.selected_files:
+            self.trashman.trash(uri, verbocity)
+
+    def restore_trash_files(self, widget = None, eve = None, verbocity = False):
+        self._event_system.emit("get_current_state")
+        state = self._fm_state
+        for uri in state.selected_files:
+            self.trashman.restore(filename=uri.split("/")[-1], verbose = verbocity)
+
+    def empty_trash(self, widget = None, eve = None, verbocity = False):
+        self.trashman.empty(verbose = verbocity)
+
+    def go_to_trash(self, widget = None, eve = None, verbocity = False):
+        self._event_system.emit("go_to_path", self.trash_files_path)

plugins/trasher/trash.py

@@ -21,7 +21,7 @@ class Trash(object):
             if os.path.isfile(item):
                 size = size + os.path.getsize(item)
             elif os.path.isdir(item):
-                size = size + size_dir(item)
+                size = size + self.size_dir(item)
 
         return size
 

plugins/vod_thumbnailer/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/vod_thumbnailer/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/vod_thumbnailer/manifest.json

@@ -0,0 +1,12 @@
+{
+    "manifest": {
+        "name": "VOD Thumbnailer",
+        "author": "ITDominator",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "context_menu_plugins",
+            "pass_fm_events": "true"
+        }
+    }
+}

plugins/vod_thumbnailer/plugin.py

@@ -0,0 +1,146 @@
+# Python imports
+import os
+import threading
+import subprocess
+import time
+import inspect
+import hashlib
+from datetime import datetime
+
+# Gtk imports
+import gi
+gi.require_version('Gtk', '3.0')
+gi.require_version('GdkPixbuf', '2.0')
+from gi.repository import Gtk
+from gi.repository import GLib
+from gi.repository import Gio
+from gi.repository import GdkPixbuf
+
+# Application imports
+from plugins.plugin_base import PluginBase
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+    return wrapper
+
+
+
+
+class Plugin(PluginBase):
+    def __init__(self):
+        super().__init__()
+
+        self.path                   = os.path.dirname(os.path.realpath(__file__))
+        self._GLADE_FILE            = f"{self.path}/re_thumbnailer.glade"
+        self.name                   = "VOD Thumbnailer"  # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                                         #       where self.name should not be needed for message comms
+
+        self._thumbnailer_dialog    = None
+        self._thumbnail_preview_img = None
+        self._scrub_step            = None
+        self._file_name             = None
+        self._file_location         = None
+        self._file_hash             = None
+
+
+    def run(self):
+        self._builder           = Gtk.Builder()
+        self._builder.add_from_file(self._GLADE_FILE)
+
+        classes  = [self]
+        handlers = {}
+        for c in classes:
+            methods = None
+            try:
+                methods = inspect.getmembers(c, predicate=inspect.ismethod)
+                handlers.update(methods)
+            except Exception as e:
+                print(repr(e))
+
+        self._builder.connect_signals(handlers)
+
+        self._thumbnailer_dialog    = self._builder.get_object("thumbnailer_dialog")
+        self._scrub_step            = self._builder.get_object("scrub_step")
+        self._file_name             = self._builder.get_object("file_name")
+        self._file_location         = self._builder.get_object("file_location")
+        self._thumbnail_preview_img = self._builder.get_object("thumbnail_preview_img")
+        self._file_hash             = self._builder.get_object("file_hash")
+
+    def generate_reference_ui_element(self):
+        pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(f"{self.path}/../../icons/video.png", 16, 16, True)
+        icon   = Gtk.Image.new_from_pixbuf(pixbuf)
+        item   = Gtk.ImageMenuItem(self.name)
+
+        item.set_image( icon )
+        item.connect("activate", self._show_thumbnailer_page)
+        item.set_always_show_image(True)
+        return item
+
+
+    @threaded
+    def _show_thumbnailer_page(self, widget=None, eve=None):
+        self._event_system.emit("get_current_state")
+
+        state               = self._fm_state
+        self._event_message = None
+
+        GLib.idle_add(self._process_changes, (state))
+
+    def _process_changes(self, state):
+        self._fm_state = None
+
+        if len(state.selected_files) == 1:
+            if state.selected_files[0].lower().endswith(state.tab.fvideos):
+                self._fm_state = state
+                self._set_ui_data()
+                response   = self._thumbnailer_dialog.run()
+                if response in [Gtk.ResponseType.CLOSE, Gtk.ResponseType.DELETE_EVENT]:
+                    self._thumbnailer_dialog.hide()
+
+
+    def _regenerate_thumbnail(self, widget=None, eve=None):
+        scrub_percent = int(self._scrub_step.get_value())
+        file          = self._file_name.get_text()
+        dir           = self._file_location.get_text()
+        file_hash     = self._file_hash.get_text()
+        hash_img_pth  = f"{self._fm_state.tab.ABS_THUMBS_PTH}/{file_hash}.jpg"
+
+        try:
+            os.remove(hash_img_pth) if os.path.isfile(hash_img_pth) else ...
+
+            self._fm_state.tab.create_thumbnail(dir, file, f"{scrub_percent}%")
+            preview_pixbuf = GdkPixbuf.Pixbuf.new_from_file(hash_img_pth)
+            self._thumbnail_preview_img.set_from_pixbuf(preview_pixbuf)
+
+            img_pixbuf = self._fm_state.tab.create_scaled_image(hash_img_pth)
+            tree_pth   = self._fm_state.icon_grid.get_selected_items()[0]
+            itr        = self._fm_state.store.get_iter(tree_pth)
+            pixbuff    = self._fm_state.store.get(itr, 0)[0]
+            self._fm_state.store.set(itr, 0, img_pixbuf)
+        except Exception as e:
+            print(repr(e))
+            print("Couldn't regenerate thumbnail!")
+
+
+    def _set_ui_data(self):
+        uri            = self._fm_state.selected_files[0]
+        path           = self._fm_state.tab.get_current_directory()
+        parts          = uri.split("/")
+
+        file_hash      = hashlib.sha256(str.encode(uri)).hexdigest()
+        hash_img_pth   = f"{self._fm_state.tab.ABS_THUMBS_PTH}/{file_hash}.jpg"
+        preview_pixbuf = GdkPixbuf.Pixbuf.new_from_file(hash_img_pth)
+
+        self._thumbnail_preview_img.set_from_pixbuf(preview_pixbuf)
+        self._file_name.set_text(parts[ len(parts) - 1 ])
+        self._file_location.set_text(path)
+        self._file_hash.set_text(file_hash)

plugins/vod_thumbnailer/re_thumbnailer.glade

@@ -0,0 +1,239 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated with glade 3.38.2 -->
+<interface>
+  <requires lib="gtk+" version="3.16"/>
+  <object class="GtkAdjustment" id="scrub_step_adjuster">
+    <property name="lower">1</property>
+    <property name="upper">100</property>
+    <property name="value">65</property>
+    <property name="step-increment">1</property>
+    <property name="page-increment">10</property>
+  </object>
+  <object class="GtkDialog" id="thumbnailer_dialog">
+    <property name="can-focus">False</property>
+    <property name="border-width">6</property>
+    <property name="title" translatable="yes">VOD Thumbnailer</property>
+    <property name="modal">True</property>
+    <property name="window-position">center-on-parent</property>
+    <property name="default-width">420</property>
+    <property name="destroy-with-parent">True</property>
+    <property name="type-hint">dialog</property>
+    <property name="skip-taskbar-hint">True</property>
+    <property name="skip-pager-hint">True</property>
+    <property name="gravity">center</property>
+    <child internal-child="vbox">
+      <object class="GtkBox" id="dialog_vbox">
+        <property name="visible">True</property>
+        <property name="can-focus">False</property>
+        <property name="spacing">12</property>
+        <child internal-child="action_area">
+          <object class="GtkButtonBox" id="dialog_action_area">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <property name="layout-style">end</property>
+            <child>
+              <object class="GtkButton" id="cancel_button">
+                <property name="label">gtk-close</property>
+                <property name="visible">True</property>
+                <property name="can-focus">True</property>
+                <property name="can-default">True</property>
+                <property name="receives-default">False</property>
+                <property name="use-stock">True</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">False</property>
+            <property name="fill">False</property>
+            <property name="pack-type">end</property>
+            <property name="position">0</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkBox">
+            <property name="visible">True</property>
+            <property name="can-focus">False</property>
+            <property name="orientation">vertical</property>
+            <child>
+              <object class="GtkImage" id="thumbnail_preview_img">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="stock">gtk-missing-image</property>
+                <property name="icon_size">6</property>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkBox">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="orientation">vertical</property>
+                <child>
+                  <object class="GtkLabel">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="margin-start">5</property>
+                    <property name="label" translatable="yes">New Thumbnail Scrub Step: </property>
+                  </object>
+                  <packing>
+                    <property name="expand">False</property>
+                    <property name="fill">True</property>
+                    <property name="position">0</property>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkScale" id="scrub_step">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="margin-start">5</property>
+                    <property name="margin-end">5</property>
+                    <property name="adjustment">scrub_step_adjuster</property>
+                    <property name="show-fill-level">True</property>
+                    <property name="round-digits">1</property>
+                    <property name="digits">0</property>
+                    <property name="value-pos">right</property>
+                    <signal name="value-changed" handler="_regenerate_thumbnail" swapped="no"/>
+                  </object>
+                  <packing>
+                    <property name="expand">True</property>
+                    <property name="fill">True</property>
+                    <property name="position">1</property>
+                  </packing>
+                </child>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkTable" id="general_table">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="border-width">4</property>
+                <property name="n-rows">3</property>
+                <property name="n-columns">2</property>
+                <property name="column-spacing">12</property>
+                <property name="row-spacing">6</property>
+                <child>
+                  <object class="GtkLabel">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="label" translatable="yes">&lt;b&gt;File _Name:&lt;/b&gt;</property>
+                    <property name="use-markup">True</property>
+                    <property name="use-underline">True</property>
+                    <property name="mnemonic-widget">file_name</property>
+                    <property name="xalign">0</property>
+                  </object>
+                  <packing>
+                    <property name="x-options">GTK_FILL</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkEntry" id="file_name">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="editable">False</property>
+                  </object>
+                  <packing>
+                    <property name="left-attach">1</property>
+                    <property name="right-attach">2</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkLabel">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="label" translatable="yes">&lt;b&gt;_Location:&lt;/b&gt;</property>
+                    <property name="use-markup">True</property>
+                    <property name="use-underline">True</property>
+                    <property name="mnemonic-widget">file_location</property>
+                    <property name="xalign">0</property>
+                  </object>
+                  <packing>
+                    <property name="top-attach">1</property>
+                    <property name="bottom-attach">2</property>
+                    <property name="x-options">GTK_FILL</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkEntry" id="file_location">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="editable">False</property>
+                  </object>
+                  <packing>
+                    <property name="left-attach">1</property>
+                    <property name="right-attach">2</property>
+                    <property name="top-attach">1</property>
+                    <property name="bottom-attach">2</property>
+                    <property name="x-options">GTK_FILL</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkLabel" id="hash">
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="label" translatable="yes">&lt;b&gt;_Thumbnail Hash:&lt;/b&gt;</property>
+                    <property name="use-markup">True</property>
+                    <property name="use-underline">True</property>
+                    <property name="mnemonic-widget">file_location</property>
+                    <property name="xalign">0</property>
+                  </object>
+                  <packing>
+                    <property name="top-attach">2</property>
+                    <property name="bottom-attach">3</property>
+                    <property name="x-options">GTK_FILL</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkEntry" id="file_hash">
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="editable">False</property>
+                  </object>
+                  <packing>
+                    <property name="left-attach">1</property>
+                    <property name="right-attach">2</property>
+                    <property name="top-attach">2</property>
+                    <property name="bottom-attach">3</property>
+                    <property name="x-options">GTK_FILL</property>
+                    <property name="y-options"/>
+                  </packing>
+                </child>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">2</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="expand">True</property>
+            <property name="fill">True</property>
+            <property name="position">1</property>
+          </packing>
+        </child>
+      </object>
+    </child>
+    <action-widgets>
+      <action-widget response="-7">cancel_button</action-widget>
+    </action-widgets>
+  </object>
+</interface>

plugins/youtube_download/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Module
+"""

plugins/youtube_download/__main__.py

@@ -0,0 +1,3 @@
+"""
+    Pligin Package
+"""

plugins/youtube_download/download.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# . CONFIG.sh
+
+# set -o xtrace       ## To debug scripts
+# set -o errexit      ## To exit on error
+# set -o errunset     ## To exit if a variable is referenced but not set
+
+
+function main() {
+    cd "$(dirname "")"
+    echo "Working Dir: " $(pwd)
+    LINK=`xclip -selection clipboard -o`
+
+    python "${HOME}/.config/solarfm/plugins/youtube_download/yt_dlp/__main__.py" \
+            --cookies-from-browser firefox --write-sub --embed-sub --sub-langs en \
+            -o "${1}/%(title)s.%(ext)s" "${LINK}"
+}
+main "$@";

plugins/youtube_download/manifest.json

@@ -0,0 +1,12 @@
+{
+    "manifest": {
+        "name": "Youtube Download",
+        "author": "ITDominator",
+        "version": "0.0.1",
+        "support": "",
+        "requests": {
+            "ui_target": "plugin_control_list",
+            "pass_fm_events": "true"
+        }
+    }
+}

plugins/youtube_download/plugin.py

@@ -0,0 +1,53 @@
+# Python imports
+import os, threading, subprocess, time
+
+# Lib imports
+import gi
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk
+
+# Application imports
+from plugins.plugin_base import PluginBase
+
+
+# NOTE: Threads WILL NOT die with parent's destruction.
+def threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
+    return wrapper
+
+# NOTE: Threads WILL die with parent's destruction.
+def daemon_threaded(fn):
+    def wrapper(*args, **kwargs):
+        threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
+    return wrapper
+
+
+
+
+class Plugin(PluginBase):
+    def __init__(self):
+        super().__init__()
+
+        self.path              = os.path.dirname(os.path.realpath(__file__))
+        self.name              = "Youtube Download"  # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
+                                                     #       where self.name should not be needed for message comms
+
+    def generate_reference_ui_element(self):
+        button = Gtk.Button(label=self.name)
+        button.connect("button-release-event", self._do_download)
+        return button
+
+    def run(self):
+        ...
+
+
+    def _do_download(self, widget=None, eve=None):
+        self._event_system.emit("get_current_state")
+
+        dir = self._fm_state.tab.get_current_directory()
+        self._download(dir)
+
+    @threaded
+    def _download(self, dir):
+        subprocess.Popen([f'{self.path}/download.sh', dir])

plugins/youtube_download/yt_dlp-2022.2.4.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

plugins/youtube_download/yt_dlp-2022.2.4.dist-info/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

plugins/youtube_download/yt_dlp-2022.2.4.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.37.1)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

plugins/youtube_download/yt_dlp-2022.2.4.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+yt-dlp = yt_dlp:main
+

plugins/youtube_download/yt_dlp-2022.2.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+yt_dlp

plugins/youtube_download/yt_dlp/__init__.py

@@ -0,0 +1,871 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+f'You are using an unsupported version of Python. Only Python versions 3.6 and above are supported by yt-dlp'  # noqa: F541
+
+__license__ = 'Public Domain'
+
+import codecs
+import io
+import itertools
+import os
+import random
+import re
+import sys
+
+from .options import (
+    parseOpts,
+)
+from .compat import (
+    compat_getpass,
+    compat_os_name,
+    compat_shlex_quote,
+    workaround_optparse_bug9161,
+)
+from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS
+from .utils import (
+    DateRange,
+    decodeOption,
+    DownloadCancelled,
+    DownloadError,
+    error_to_compat_str,
+    expand_path,
+    GeoUtils,
+    float_or_none,
+    int_or_none,
+    match_filter_func,
+    parse_duration,
+    preferredencoding,
+    read_batch_urls,
+    render_table,
+    SameFileError,
+    setproctitle,
+    std_headers,
+    write_string,
+)
+from .update import run_update
+from .downloader import (
+    FileDownloader,
+)
+from .extractor import gen_extractors, list_extractors
+from .extractor.common import InfoExtractor
+from .extractor.adobepass import MSO_INFO
+from .postprocessor import (
+    FFmpegExtractAudioPP,
+    FFmpegSubtitlesConvertorPP,
+    FFmpegThumbnailsConvertorPP,
+    FFmpegVideoConvertorPP,
+    FFmpegVideoRemuxerPP,
+    MetadataFromFieldPP,
+    MetadataParserPP,
+)
+from .YoutubeDL import YoutubeDL
+
+
+def _real_main(argv=None):
+    # Compatibility fixes for Windows
+    if sys.platform == 'win32':
+        # https://github.com/ytdl-org/youtube-dl/issues/820
+        codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
+
+    workaround_optparse_bug9161()
+
+    setproctitle('yt-dlp')
+
+    parser, opts, args = parseOpts(argv)
+    warnings, deprecation_warnings = [], []
+
+    # Set user agent
+    if opts.user_agent is not None:
+        std_headers['User-Agent'] = opts.user_agent
+
+    # Set referer
+    if opts.referer is not None:
+        std_headers['Referer'] = opts.referer
+
+    # Custom HTTP headers
+    std_headers.update(opts.headers)
+
+    # Dump user agent
+    if opts.dump_user_agent:
+        write_string(std_headers['User-Agent'] + '\n', out=sys.stdout)
+        sys.exit(0)
+
+    # Batch file verification
+    batch_urls = []
+    if opts.batchfile is not None:
+        try:
+            if opts.batchfile == '-':
+                write_string('Reading URLs from stdin - EOF (%s) to end:\n' % (
+                    'Ctrl+Z' if compat_os_name == 'nt' else 'Ctrl+D'))
+                batchfd = sys.stdin
+            else:
+                batchfd = io.open(
+                    expand_path(opts.batchfile),
+                    'r', encoding='utf-8', errors='ignore')
+            batch_urls = read_batch_urls(batchfd)
+            if opts.verbose:
+                write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
+        except IOError:
+            sys.exit('ERROR: batch file %s could not be read' % opts.batchfile)
+    all_urls = batch_urls + [url.strip() for url in args]  # batch_urls are already striped in read_batch_urls
+    _enc = preferredencoding()
+    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
+
+    if opts.list_extractors:
+        for ie in list_extractors(opts.age_limit):
+            write_string(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie.working() else '') + '\n', out=sys.stdout)
+            matchedUrls = [url for url in all_urls if ie.suitable(url)]
+            for mu in matchedUrls:
+                write_string('  ' + mu + '\n', out=sys.stdout)
+        sys.exit(0)
+    if opts.list_extractor_descriptions:
+        for ie in list_extractors(opts.age_limit):
+            if not ie.working():
+                continue
+            desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
+            if desc is False:
+                continue
+            if getattr(ie, 'SEARCH_KEY', None) is not None:
+                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
+                _COUNTS = ('', '5', '10', 'all')
+                desc += f'; "{ie.SEARCH_KEY}:" prefix (Example: "{ie.SEARCH_KEY}{random.choice(_COUNTS)}:{random.choice(_SEARCHES)}")'
+            write_string(desc + '\n', out=sys.stdout)
+        sys.exit(0)
+    if opts.ap_list_mso:
+        table = [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()]
+        write_string('Supported TV Providers:\n' + render_table(['mso', 'mso name'], table) + '\n', out=sys.stdout)
+        sys.exit(0)
+
+    # Conflicting, missing and erroneous options
+    if opts.format == 'best':
+        warnings.append('.\n         '.join((
+            '"-f best" selects the best pre-merged format which is often not the best option',
+            'To let yt-dlp download and merge the best available formats, simply do not pass any format selection',
+            'If you know what you are doing and want only the best pre-merged format, use "-f b" instead to suppress this warning')))
+    if opts.exec_cmd.get('before_dl') and opts.exec_before_dl_cmd:
+        parser.error('using "--exec-before-download" conflicts with "--exec before_dl:"')
+    if opts.usenetrc and (opts.username is not None or opts.password is not None):
+        parser.error('using .netrc conflicts with giving username/password')
+    if opts.password is not None and opts.username is None:
+        parser.error('account username missing\n')
+    if opts.ap_password is not None and opts.ap_username is None:
+        parser.error('TV Provider account username missing\n')
+    if opts.autonumber_size is not None:
+        if opts.autonumber_size <= 0:
+            parser.error('auto number size must be positive')
+    if opts.autonumber_start is not None:
+        if opts.autonumber_start < 0:
+            parser.error('auto number start must be positive or 0')
+    if opts.username is not None and opts.password is None:
+        opts.password = compat_getpass('Type account password and press [Return]: ')
+    if opts.ap_username is not None and opts.ap_password is None:
+        opts.ap_password = compat_getpass('Type TV provider account password and press [Return]: ')
+    if opts.ratelimit is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
+        if numeric_limit is None:
+            parser.error('invalid rate limit specified')
+        opts.ratelimit = numeric_limit
+    if opts.throttledratelimit is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.throttledratelimit)
+        if numeric_limit is None:
+            parser.error('invalid rate limit specified')
+        opts.throttledratelimit = numeric_limit
+    if opts.min_filesize is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
+        if numeric_limit is None:
+            parser.error('invalid min_filesize specified')
+        opts.min_filesize = numeric_limit
+    if opts.max_filesize is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
+        if numeric_limit is None:
+            parser.error('invalid max_filesize specified')
+        opts.max_filesize = numeric_limit
+    if opts.sleep_interval is not None:
+        if opts.sleep_interval < 0:
+            parser.error('sleep interval must be positive or 0')
+    if opts.max_sleep_interval is not None:
+        if opts.max_sleep_interval < 0:
+            parser.error('max sleep interval must be positive or 0')
+        if opts.sleep_interval is None:
+            parser.error('min sleep interval must be specified, use --min-sleep-interval')
+        if opts.max_sleep_interval < opts.sleep_interval:
+            parser.error('max sleep interval must be greater than or equal to min sleep interval')
+    else:
+        opts.max_sleep_interval = opts.sleep_interval
+    if opts.sleep_interval_subtitles is not None:
+        if opts.sleep_interval_subtitles < 0:
+            parser.error('subtitles sleep interval must be positive or 0')
+    if opts.sleep_interval_requests is not None:
+        if opts.sleep_interval_requests < 0:
+            parser.error('requests sleep interval must be positive or 0')
+    if opts.ap_mso and opts.ap_mso not in MSO_INFO:
+        parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')
+    if opts.overwrites:  # --yes-overwrites implies --no-continue
+        opts.continue_dl = False
+    if opts.concurrent_fragment_downloads <= 0:
+        parser.error('Concurrent fragments must be positive')
+    if opts.wait_for_video is not None:
+        min_wait, max_wait, *_ = map(parse_duration, opts.wait_for_video.split('-', 1) + [None])
+        if min_wait is None or (max_wait is None and '-' in opts.wait_for_video):
+            parser.error('Invalid time range to wait')
+        elif max_wait is not None and max_wait < min_wait:
+            parser.error('Minimum time range to wait must not be longer than the maximum')
+        opts.wait_for_video = (min_wait, max_wait)
+
+    def parse_retries(retries, name=''):
+        if retries in ('inf', 'infinite'):
+            parsed_retries = float('inf')
+        else:
+            try:
+                parsed_retries = int(retries)
+            except (TypeError, ValueError):
+                parser.error('invalid %sretry count specified' % name)
+        return parsed_retries
+    if opts.retries is not None:
+        opts.retries = parse_retries(opts.retries)
+    if opts.file_access_retries is not None:
+        opts.file_access_retries = parse_retries(opts.file_access_retries, 'file access ')
+    if opts.fragment_retries is not None:
+        opts.fragment_retries = parse_retries(opts.fragment_retries, 'fragment ')
+    if opts.extractor_retries is not None:
+        opts.extractor_retries = parse_retries(opts.extractor_retries, 'extractor ')
+    if opts.buffersize is not None:
+        numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
+        if numeric_buffersize is None:
+            parser.error('invalid buffer size specified')
+        opts.buffersize = numeric_buffersize
+    if opts.http_chunk_size is not None:
+        numeric_chunksize = FileDownloader.parse_bytes(opts.http_chunk_size)
+        if not numeric_chunksize:
+            parser.error('invalid http chunk size specified')
+        opts.http_chunk_size = numeric_chunksize
+    if opts.playliststart <= 0:
+        raise parser.error('Playlist start must be positive')
+    if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
+        raise parser.error('Playlist end must be greater than playlist start')
+    if opts.extractaudio:
+        opts.audioformat = opts.audioformat.lower()
+        if opts.audioformat not in ['best'] + list(FFmpegExtractAudioPP.SUPPORTED_EXTS):
+            parser.error('invalid audio format specified')
+    if opts.audioquality:
+        opts.audioquality = opts.audioquality.strip('k').strip('K')
+        audioquality = int_or_none(float_or_none(opts.audioquality))  # int_or_none prevents inf, nan
+        if audioquality is None or audioquality < 0:
+            parser.error('invalid audio quality specified')
+    if opts.recodevideo is not None:
+        opts.recodevideo = opts.recodevideo.replace(' ', '')
+        if not re.match(FFmpegVideoConvertorPP.FORMAT_RE, opts.recodevideo):
+            parser.error('invalid video remux format specified')
+    if opts.remuxvideo is not None:
+        opts.remuxvideo = opts.remuxvideo.replace(' ', '')
+        if not re.match(FFmpegVideoRemuxerPP.FORMAT_RE, opts.remuxvideo):
+            parser.error('invalid video remux format specified')
+    if opts.convertsubtitles is not None:
+        if opts.convertsubtitles not in FFmpegSubtitlesConvertorPP.SUPPORTED_EXTS:
+            parser.error('invalid subtitle format specified')
+    if opts.convertthumbnails is not None:
+        if opts.convertthumbnails not in FFmpegThumbnailsConvertorPP.SUPPORTED_EXTS:
+            parser.error('invalid thumbnail format specified')
+    if opts.cookiesfrombrowser is not None:
+        mobj = re.match(r'(?P<name>[^+:]+)(\s*\+\s*(?P<keyring>[^:]+))?(\s*:(?P<profile>.+))?', opts.cookiesfrombrowser)
+        if mobj is None:
+            parser.error(f'invalid cookies from browser arguments: {opts.cookiesfrombrowser}')
+        browser_name, keyring, profile = mobj.group('name', 'keyring', 'profile')
+        browser_name = browser_name.lower()
+        if browser_name not in SUPPORTED_BROWSERS:
+            parser.error(f'unsupported browser specified for cookies: "{browser_name}". '
+                         f'Supported browsers are: {", ".join(sorted(SUPPORTED_BROWSERS))}')
+        if keyring is not None:
+            keyring = keyring.upper()
+            if keyring not in SUPPORTED_KEYRINGS:
+                parser.error(f'unsupported keyring specified for cookies: "{keyring}". '
+                             f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
+        opts.cookiesfrombrowser = (browser_name, profile, keyring)
+    geo_bypass_code = opts.geo_bypass_ip_block or opts.geo_bypass_country
+    if geo_bypass_code is not None:
+        try:
+            GeoUtils.random_ipv4(geo_bypass_code)
+        except Exception:
+            parser.error('unsupported geo-bypass country or ip-block')
+
+    if opts.date is not None:
+        date = DateRange.day(opts.date)
+    else:
+        date = DateRange(opts.dateafter, opts.datebefore)
+
+    compat_opts = opts.compat_opts
+
+    def report_conflict(arg1, arg2):
+        warnings.append(f'{arg2} is ignored since {arg1} was given')
+
+    def _unused_compat_opt(name):
+        if name not in compat_opts:
+            return False
+        compat_opts.discard(name)
+        compat_opts.update(['*%s' % name])
+        return True
+
+    def set_default_compat(compat_name, opt_name, default=True, remove_compat=True):
+        attr = getattr(opts, opt_name)
+        if compat_name in compat_opts:
+            if attr is None:
+                setattr(opts, opt_name, not default)
+                return True
+            else:
+                if remove_compat:
+                    _unused_compat_opt(compat_name)
+                return False
+        elif attr is None:
+            setattr(opts, opt_name, default)
+        return None
+
+    set_default_compat('abort-on-error', 'ignoreerrors', 'only_download')
+    set_default_compat('no-playlist-metafiles', 'allow_playlist_files')
+    set_default_compat('no-clean-infojson', 'clean_infojson')
+    if 'no-attach-info-json' in compat_opts:
+        if opts.embed_infojson:
+            _unused_compat_opt('no-attach-info-json')
+        else:
+            opts.embed_infojson = False
+    if 'format-sort' in compat_opts:
+        opts.format_sort.extend(InfoExtractor.FormatSort.ytdl_default)
+    _video_multistreams_set = set_default_compat('multistreams', 'allow_multiple_video_streams', False, remove_compat=False)
+    _audio_multistreams_set = set_default_compat('multistreams', 'allow_multiple_audio_streams', False, remove_compat=False)
+    if _video_multistreams_set is False and _audio_multistreams_set is False:
+        _unused_compat_opt('multistreams')
+    outtmpl_default = opts.outtmpl.get('default')
+    if outtmpl_default == '':
+        outtmpl_default, opts.skip_download = None, True
+        del opts.outtmpl['default']
+    if opts.useid:
+        if outtmpl_default is None:
+            outtmpl_default = opts.outtmpl['default'] = '%(id)s.%(ext)s'
+        else:
+            report_conflict('--output', '--id')
+    if 'filename' in compat_opts:
+        if outtmpl_default is None:
+            outtmpl_default = opts.outtmpl['default'] = '%(title)s-%(id)s.%(ext)s'
+        else:
+            _unused_compat_opt('filename')
+
+    def validate_outtmpl(tmpl, msg):
+        err = YoutubeDL.validate_outtmpl(tmpl)
+        if err:
+            parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err)))
+
+    for k, tmpl in opts.outtmpl.items():
+        validate_outtmpl(tmpl, f'{k} output template')
+    for type_, tmpl_list in opts.forceprint.items():
+        for tmpl in tmpl_list:
+            validate_outtmpl(tmpl, f'{type_} print template')
+    for type_, tmpl_list in opts.print_to_file.items():
+        for tmpl, file in tmpl_list:
+            validate_outtmpl(tmpl, f'{type_} print-to-file template')
+            validate_outtmpl(file, f'{type_} print-to-file filename')
+    validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title')
+    for k, tmpl in opts.progress_template.items():
+        k = f'{k[:-6]} console title' if '-title' in k else f'{k} progress'
+        validate_outtmpl(tmpl, f'{k} template')
+
+    if opts.extractaudio and not opts.keepvideo and opts.format is None:
+        opts.format = 'bestaudio/best'
+
+    if outtmpl_default is not None and not os.path.splitext(outtmpl_default)[1] and opts.extractaudio:
+        parser.error('Cannot download a video and extract audio into the same'
+                     ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
+                     ' template'.format(outtmpl_default))
+
+    for f in opts.format_sort:
+        if re.match(InfoExtractor.FormatSort.regex, f) is None:
+            parser.error('invalid format sort string "%s" specified' % f)
+
+    def metadataparser_actions(f):
+        if isinstance(f, str):
+            cmd = '--parse-metadata %s' % compat_shlex_quote(f)
+            try:
+                actions = [MetadataFromFieldPP.to_action(f)]
+            except Exception as err:
+                parser.error(f'{cmd} is invalid; {err}')
+        else:
+            cmd = '--replace-in-metadata %s' % ' '.join(map(compat_shlex_quote, f))
+            actions = ((MetadataParserPP.Actions.REPLACE, x, *f[1:]) for x in f[0].split(','))
+
+        for action in actions:
+            try:
+                MetadataParserPP.validate_action(*action)
+            except Exception as err:
+                parser.error(f'{cmd} is invalid; {err}')
+            yield action
+
+    if opts.parse_metadata is None:
+        opts.parse_metadata = []
+    if opts.metafromtitle is not None:
+        opts.parse_metadata.append('title:%s' % opts.metafromtitle)
+    opts.parse_metadata = list(itertools.chain(*map(metadataparser_actions, opts.parse_metadata)))
+
+    any_getting = (any(opts.forceprint.values()) or opts.dumpjson or opts.dump_single_json
+                   or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail
+                   or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration)
+
+    any_printing = opts.print_json
+    download_archive_fn = expand_path(opts.download_archive) if opts.download_archive is not None else opts.download_archive
+
+    # If JSON is not printed anywhere, but comments are requested, save it to file
+    printing_json = opts.dumpjson or opts.print_json or opts.dump_single_json
+    if opts.getcomments and not printing_json:
+        opts.writeinfojson = True
+
+    if opts.no_sponsorblock:
+        opts.sponsorblock_mark = set()
+        opts.sponsorblock_remove = set()
+    sponsorblock_query = opts.sponsorblock_mark | opts.sponsorblock_remove
+
+    opts.remove_chapters = opts.remove_chapters or []
+
+    if (opts.remove_chapters or sponsorblock_query) and opts.sponskrub is not False:
+        if opts.sponskrub:
+            if opts.remove_chapters:
+                report_conflict('--remove-chapters', '--sponskrub')
+            if opts.sponsorblock_mark:
+                report_conflict('--sponsorblock-mark', '--sponskrub')
+            if opts.sponsorblock_remove:
+                report_conflict('--sponsorblock-remove', '--sponskrub')
+        opts.sponskrub = False
+    if opts.sponskrub_cut and opts.split_chapters and opts.sponskrub is not False:
+        report_conflict('--split-chapter', '--sponskrub-cut')
+        opts.sponskrub_cut = False
+
+    if opts.remuxvideo and opts.recodevideo:
+        report_conflict('--recode-video', '--remux-video')
+        opts.remuxvideo = False
+
+    if opts.allow_unplayable_formats:
+        def report_unplayable_conflict(opt_name, arg, default=False, allowed=None):
+            val = getattr(opts, opt_name)
+            if (not allowed and val) or (allowed and not allowed(val)):
+                report_conflict('--allow-unplayable-formats', arg)
+                setattr(opts, opt_name, default)
+
+        report_unplayable_conflict('extractaudio', '--extract-audio')
+        report_unplayable_conflict('remuxvideo', '--remux-video')
+        report_unplayable_conflict('recodevideo', '--recode-video')
+        report_unplayable_conflict('addmetadata', '--embed-metadata')
+        report_unplayable_conflict('addchapters', '--embed-chapters')
+        report_unplayable_conflict('embed_infojson', '--embed-info-json')
+        opts.embed_infojson = False
+        report_unplayable_conflict('embedsubtitles', '--embed-subs')
+        report_unplayable_conflict('embedthumbnail', '--embed-thumbnail')
+        report_unplayable_conflict('xattrs', '--xattrs')
+        report_unplayable_conflict('fixup', '--fixup', default='never', allowed=lambda x: x in (None, 'never', 'ignore'))
+        opts.fixup = 'never'
+        report_unplayable_conflict('remove_chapters', '--remove-chapters', default=[])
+        report_unplayable_conflict('sponsorblock_remove', '--sponsorblock-remove', default=set())
+        report_unplayable_conflict('sponskrub', '--sponskrub', default=set())
+        opts.sponskrub = False
+
+    if (opts.addmetadata or opts.sponsorblock_mark) and opts.addchapters is None:
+        opts.addchapters = True
+
+    # PostProcessors
+    postprocessors = list(opts.add_postprocessors)
+    if sponsorblock_query:
+        postprocessors.append({
+            'key': 'SponsorBlock',
+            'categories': sponsorblock_query,
+            'api': opts.sponsorblock_api,
+            # Run this immediately after extraction is complete
+            'when': 'pre_process'
+        })
+    if opts.parse_metadata:
+        postprocessors.append({
+            'key': 'MetadataParser',
+            'actions': opts.parse_metadata,
+            # Run this immediately after extraction is complete
+            'when': 'pre_process'
+        })
+    if opts.convertsubtitles:
+        postprocessors.append({
+            'key': 'FFmpegSubtitlesConvertor',
+            'format': opts.convertsubtitles,
+            # Run this before the actual video download
+            'when': 'before_dl'
+        })
+    if opts.convertthumbnails:
+        postprocessors.append({
+            'key': 'FFmpegThumbnailsConvertor',
+            'format': opts.convertthumbnails,
+            # Run this before the actual video download
+            'when': 'before_dl'
+        })
+    if opts.extractaudio:
+        postprocessors.append({
+            'key': 'FFmpegExtractAudio',
+            'preferredcodec': opts.audioformat,
+            'preferredquality': opts.audioquality,
+            'nopostoverwrites': opts.nopostoverwrites,
+        })
+    if opts.remuxvideo:
+        postprocessors.append({
+            'key': 'FFmpegVideoRemuxer',
+            'preferedformat': opts.remuxvideo,
+        })
+    if opts.recodevideo:
+        postprocessors.append({
+            'key': 'FFmpegVideoConvertor',
+            'preferedformat': opts.recodevideo,
+        })
+    # If ModifyChapters is going to remove chapters, subtitles must already be in the container.
+    if opts.embedsubtitles:
+        already_have_subtitle = opts.writesubtitles and 'no-keep-subs' not in compat_opts
+        postprocessors.append({
+            'key': 'FFmpegEmbedSubtitle',
+            # already_have_subtitle = True prevents the file from being deleted after embedding
+            'already_have_subtitle': already_have_subtitle
+        })
+        if not opts.writeautomaticsub and 'no-keep-subs' not in compat_opts:
+            opts.writesubtitles = True
+    # --all-sub automatically sets --write-sub if --write-auto-sub is not given
+    # this was the old behaviour if only --all-sub was given.
+    if opts.allsubtitles and not opts.writeautomaticsub:
+        opts.writesubtitles = True
+    # ModifyChapters must run before FFmpegMetadataPP
+    remove_chapters_patterns, remove_ranges = [], []
+    for regex in opts.remove_chapters:
+        if regex.startswith('*'):
+            dur = list(map(parse_duration, regex[1:].split('-')))
+            if len(dur) == 2 and all(t is not None for t in dur):
+                remove_ranges.append(tuple(dur))
+                continue
+            parser.error(f'invalid --remove-chapters time range {regex!r}. Must be of the form *start-end')
+        try:
+            remove_chapters_patterns.append(re.compile(regex))
+        except re.error as err:
+            parser.error(f'invalid --remove-chapters regex {regex!r} - {err}')
+    if opts.remove_chapters or sponsorblock_query:
+        postprocessors.append({
+            'key': 'ModifyChapters',
+            'remove_chapters_patterns': remove_chapters_patterns,
+            'remove_sponsor_segments': opts.sponsorblock_remove,
+            'remove_ranges': remove_ranges,
+            'sponsorblock_chapter_title': opts.sponsorblock_chapter_title,
+            'force_keyframes': opts.force_keyframes_at_cuts
+        })
+    # FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and
+    # FFmpegExtractAudioPP as containers before conversion may not support
+    # metadata (3gp, webm, etc.)
+    # By default ffmpeg preserves metadata applicable for both
+    # source and target containers. From this point the container won't change,
+    # so metadata can be added here.
+    if opts.addmetadata or opts.addchapters or opts.embed_infojson:
+        if opts.embed_infojson is None:
+            opts.embed_infojson = 'if_exists'
+        postprocessors.append({
+            'key': 'FFmpegMetadata',
+            'add_chapters': opts.addchapters,
+            'add_metadata': opts.addmetadata,
+            'add_infojson': opts.embed_infojson,
+        })
+    # Deprecated
+    # This should be above EmbedThumbnail since sponskrub removes the thumbnail attachment
+    # but must be below EmbedSubtitle and FFmpegMetadata
+    # See https://github.com/yt-dlp/yt-dlp/issues/204 , https://github.com/faissaloo/SponSkrub/issues/29
+    # If opts.sponskrub is None, sponskrub is used, but it silently fails if the executable can't be found
+    if opts.sponskrub is not False:
+        postprocessors.append({
+            'key': 'SponSkrub',
+            'path': opts.sponskrub_path,
+            'args': opts.sponskrub_args,
+            'cut': opts.sponskrub_cut,
+            'force': opts.sponskrub_force,
+            'ignoreerror': opts.sponskrub is None,
+            '_from_cli': True,
+        })
+    if opts.embedthumbnail:
+        postprocessors.append({
+            'key': 'EmbedThumbnail',
+            # already_have_thumbnail = True prevents the file from being deleted after embedding
+            'already_have_thumbnail': opts.writethumbnail
+        })
+        if not opts.writethumbnail:
+            opts.writethumbnail = True
+            opts.outtmpl['pl_thumbnail'] = ''
+    if opts.split_chapters:
+        postprocessors.append({
+            'key': 'FFmpegSplitChapters',
+            'force_keyframes': opts.force_keyframes_at_cuts,
+        })
+    # XAttrMetadataPP should be run after post-processors that may change file contents
+    if opts.xattrs:
+        postprocessors.append({'key': 'XAttrMetadata'})
+    if opts.concat_playlist != 'never':
+        postprocessors.append({
+            'key': 'FFmpegConcat',
+            'only_multi_video': opts.concat_playlist != 'always',
+            'when': 'playlist',
+        })
+    # Exec must be the last PP of each category
+    if opts.exec_before_dl_cmd:
+        opts.exec_cmd.setdefault('before_dl', opts.exec_before_dl_cmd)
+    for when, exec_cmd in opts.exec_cmd.items():
+        postprocessors.append({
+            'key': 'Exec',
+            'exec_cmd': exec_cmd,
+            # Run this only after the files have been moved to their final locations
+            'when': when,
+        })
+
+    def report_args_compat(arg, name):
+        warnings.append('%s given without specifying name. The arguments will be given to all %s' % (arg, name))
+
+    if 'default' in opts.external_downloader_args:
+        report_args_compat('--downloader-args', 'external downloaders')
+
+    if 'default-compat' in opts.postprocessor_args and 'default' not in opts.postprocessor_args:
+        report_args_compat('--post-processor-args', 'post-processors')
+        opts.postprocessor_args.setdefault('sponskrub', [])
+        opts.postprocessor_args['default'] = opts.postprocessor_args['default-compat']
+
+    def report_deprecation(val, old, new=None):
+        if not val:
+            return
+        deprecation_warnings.append(
+            f'{old} is deprecated and may be removed in a future version. Use {new} instead' if new
+            else f'{old} is deprecated and may not work as expected')
+
+    report_deprecation(opts.sponskrub, '--sponskrub', '--sponsorblock-mark or --sponsorblock-remove')
+    report_deprecation(not opts.prefer_ffmpeg, '--prefer-avconv', 'ffmpeg')
+    report_deprecation(opts.include_ads, '--include-ads')
+    # report_deprecation(opts.call_home, '--call-home')  # We may re-implement this in future
+    # report_deprecation(opts.writeannotations, '--write-annotations')  # It's just that no website has it
+
+    final_ext = (
+        opts.recodevideo if opts.recodevideo in FFmpegVideoConvertorPP.SUPPORTED_EXTS
+        else opts.remuxvideo if opts.remuxvideo in FFmpegVideoRemuxerPP.SUPPORTED_EXTS
+        else opts.audioformat if (opts.extractaudio and opts.audioformat != 'best')
+        else None)
+
+    match_filter = (
+        None if opts.match_filter is None
+        else match_filter_func(opts.match_filter))
+
+    ydl_opts = {
+        'usenetrc': opts.usenetrc,
+        'netrc_location': opts.netrc_location,
+        'username': opts.username,
+        'password': opts.password,
+        'twofactor': opts.twofactor,
+        'videopassword': opts.videopassword,
+        'ap_mso': opts.ap_mso,
+        'ap_username': opts.ap_username,
+        'ap_password': opts.ap_password,
+        'quiet': (opts.quiet or any_getting or any_printing),
+        'no_warnings': opts.no_warnings,
+        'forceurl': opts.geturl,
+        'forcetitle': opts.gettitle,
+        'forceid': opts.getid,
+        'forcethumbnail': opts.getthumbnail,
+        'forcedescription': opts.getdescription,
+        'forceduration': opts.getduration,
+        'forcefilename': opts.getfilename,
+        'forceformat': opts.getformat,
+        'forceprint': opts.forceprint,
+        'print_to_file': opts.print_to_file,
+        'forcejson': opts.dumpjson or opts.print_json,
+        'dump_single_json': opts.dump_single_json,
+        'force_write_download_archive': opts.force_write_download_archive,
+        'simulate': (any_getting or None) if opts.simulate is None else opts.simulate,
+        'skip_download': opts.skip_download,
+        'format': opts.format,
+        'allow_unplayable_formats': opts.allow_unplayable_formats,
+        'ignore_no_formats_error': opts.ignore_no_formats_error,
+        'format_sort': opts.format_sort,
+        'format_sort_force': opts.format_sort_force,
+        'allow_multiple_video_streams': opts.allow_multiple_video_streams,
+        'allow_multiple_audio_streams': opts.allow_multiple_audio_streams,
+        'check_formats': opts.check_formats,
+        'listformats': opts.listformats,
+        'listformats_table': opts.listformats_table,
+        'outtmpl': opts.outtmpl,
+        'outtmpl_na_placeholder': opts.outtmpl_na_placeholder,
+        'paths': opts.paths,
+        'autonumber_size': opts.autonumber_size,
+        'autonumber_start': opts.autonumber_start,
+        'restrictfilenames': opts.restrictfilenames,
+        'windowsfilenames': opts.windowsfilenames,
+        'ignoreerrors': opts.ignoreerrors,
+        'force_generic_extractor': opts.force_generic_extractor,
+        'ratelimit': opts.ratelimit,
+        'throttledratelimit': opts.throttledratelimit,
+        'overwrites': opts.overwrites,
+        'retries': opts.retries,
+        'file_access_retries': opts.file_access_retries,
+        'fragment_retries': opts.fragment_retries,
+        'extractor_retries': opts.extractor_retries,
+        'skip_unavailable_fragments': opts.skip_unavailable_fragments,
+        'keep_fragments': opts.keep_fragments,
+        'concurrent_fragment_downloads': opts.concurrent_fragment_downloads,
+        'buffersize': opts.buffersize,
+        'noresizebuffer': opts.noresizebuffer,
+        'http_chunk_size': opts.http_chunk_size,
+        'continuedl': opts.continue_dl,
+        'noprogress': opts.quiet if opts.noprogress is None else opts.noprogress,
+        'progress_with_newline': opts.progress_with_newline,
+        'progress_template': opts.progress_template,
+        'playliststart': opts.playliststart,
+        'playlistend': opts.playlistend,
+        'playlistreverse': opts.playlist_reverse,
+        'playlistrandom': opts.playlist_random,
+        'noplaylist': opts.noplaylist,
+        'logtostderr': outtmpl_default == '-',
+        'consoletitle': opts.consoletitle,
+        'nopart': opts.nopart,
+        'updatetime': opts.updatetime,
+        'writedescription': opts.writedescription,
+        'writeannotations': opts.writeannotations,
+        'writeinfojson': opts.writeinfojson,
+        'allow_playlist_files': opts.allow_playlist_files,
+        'clean_infojson': opts.clean_infojson,
+        'getcomments': opts.getcomments,
+        'writethumbnail': opts.writethumbnail is True,
+        'write_all_thumbnails': opts.writethumbnail == 'all',
+        'writelink': opts.writelink,
+        'writeurllink': opts.writeurllink,
+        'writewebloclink': opts.writewebloclink,
+        'writedesktoplink': opts.writedesktoplink,
+        'writesubtitles': opts.writesubtitles,
+        'writeautomaticsub': opts.writeautomaticsub,
+        'allsubtitles': opts.allsubtitles,
+        'listsubtitles': opts.listsubtitles,
+        'subtitlesformat': opts.subtitlesformat,
+        'subtitleslangs': opts.subtitleslangs,
+        'matchtitle': decodeOption(opts.matchtitle),
+        'rejecttitle': decodeOption(opts.rejecttitle),
+        'max_downloads': opts.max_downloads,
+        'prefer_free_formats': opts.prefer_free_formats,
+        'trim_file_name': opts.trim_file_name,
+        'verbose': opts.verbose,
+        'dump_intermediate_pages': opts.dump_intermediate_pages,
+        'write_pages': opts.write_pages,
+        'test': opts.test,
+        'keepvideo': opts.keepvideo,
+        'min_filesize': opts.min_filesize,
+        'max_filesize': opts.max_filesize,
+        'min_views': opts.min_views,
+        'max_views': opts.max_views,
+        'daterange': date,
+        'cachedir': opts.cachedir,
+        'youtube_print_sig_code': opts.youtube_print_sig_code,
+        'age_limit': opts.age_limit,
+        'download_archive': download_archive_fn,
+        'break_on_existing': opts.break_on_existing,
+        'break_on_reject': opts.break_on_reject,
+        'break_per_url': opts.break_per_url,
+        'skip_playlist_after_errors': opts.skip_playlist_after_errors,
+        'cookiefile': opts.cookiefile,
+        'cookiesfrombrowser': opts.cookiesfrombrowser,
+        'legacyserverconnect': opts.legacy_server_connect,
+        'nocheckcertificate': opts.no_check_certificate,
+        'prefer_insecure': opts.prefer_insecure,
+        'proxy': opts.proxy,
+        'socket_timeout': opts.socket_timeout,
+        'bidi_workaround': opts.bidi_workaround,
+        'debug_printtraffic': opts.debug_printtraffic,
+        'prefer_ffmpeg': opts.prefer_ffmpeg,
+        'include_ads': opts.include_ads,
+        'default_search': opts.default_search,
+        'dynamic_mpd': opts.dynamic_mpd,
+        'extractor_args': opts.extractor_args,
+        'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
+        'youtube_include_hls_manifest': opts.youtube_include_hls_manifest,
+        'encoding': opts.encoding,
+        'extract_flat': opts.extract_flat,
+        'live_from_start': opts.live_from_start,
+        'wait_for_video': opts.wait_for_video,
+        'mark_watched': opts.mark_watched,
+        'merge_output_format': opts.merge_output_format,
+        'final_ext': final_ext,
+        'postprocessors': postprocessors,
+        'fixup': opts.fixup,
+        'source_address': opts.source_address,
+        'call_home': opts.call_home,
+        'sleep_interval_requests': opts.sleep_interval_requests,
+        'sleep_interval': opts.sleep_interval,
+        'max_sleep_interval': opts.max_sleep_interval,
+        'sleep_interval_subtitles': opts.sleep_interval_subtitles,
+        'external_downloader': opts.external_downloader,
+        'list_thumbnails': opts.list_thumbnails,
+        'playlist_items': opts.playlist_items,
+        'xattr_set_filesize': opts.xattr_set_filesize,
+        'match_filter': match_filter,
+        'no_color': opts.no_color,
+        'ffmpeg_location': opts.ffmpeg_location,
+        'hls_prefer_native': opts.hls_prefer_native,
+        'hls_use_mpegts': opts.hls_use_mpegts,
+        'hls_split_discontinuity': opts.hls_split_discontinuity,
+        'external_downloader_args': opts.external_downloader_args,
+        'postprocessor_args': opts.postprocessor_args,
+        'cn_verification_proxy': opts.cn_verification_proxy,
+        'geo_verification_proxy': opts.geo_verification_proxy,
+        'geo_bypass': opts.geo_bypass,
+        'geo_bypass_country': opts.geo_bypass_country,
+        'geo_bypass_ip_block': opts.geo_bypass_ip_block,
+        '_warnings': warnings,
+        '_deprecation_warnings': deprecation_warnings,
+        'compat_opts': compat_opts,
+    }
+
+    with YoutubeDL(ydl_opts) as ydl:
+        actual_use = all_urls or opts.load_info_filename
+
+        # Remove cache dir
+        if opts.rm_cachedir:
+            ydl.cache.remove()
+
+        # Update version
+        if opts.update_self:
+            # If updater returns True, exit. Required for windows
+            if run_update(ydl):
+                if actual_use:
+                    sys.exit('ERROR: The program must exit for the update to complete')
+                sys.exit()
+
+        # Maybe do nothing
+        if not actual_use:
+            if opts.update_self or opts.rm_cachedir:
+                sys.exit()
+
+            ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
+            parser.error(
+                'You must provide at least one URL.\n'
+                'Type yt-dlp --help to see a list of all options.')
+
+        try:
+            if opts.load_info_filename is not None:
+                retcode = ydl.download_with_info_file(expand_path(opts.load_info_filename))
+            else:
+                retcode = ydl.download(all_urls)
+        except DownloadCancelled:
+            ydl.to_screen('Aborting remaining downloads')
+            retcode = 101
+
+    sys.exit(retcode)
+
+
+def main(argv=None):
+    try:
+        _real_main(argv)
+    except DownloadError:
+        sys.exit(1)
+    except SameFileError as e:
+        sys.exit(f'ERROR: {e}')
+    except KeyboardInterrupt:
+        sys.exit('\nERROR: Interrupted by user')
+    except BrokenPipeError as e:
+        # https://docs.python.org/3/library/signal.html#note-on-sigpipe
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, sys.stdout.fileno())
+        sys.exit(f'\nERROR: {e}')
+
+
+__all__ = ['main', 'YoutubeDL', 'gen_extractors', 'list_extractors']

plugins/youtube_download/yt_dlp/__main__.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+from __future__ import unicode_literals
+
+# Execute with
+# $ python yt_dlp/__main__.py (2.6+)
+# $ python -m yt_dlp          (2.7+)
+
+import sys
+
+if __package__ is None and not hasattr(sys, 'frozen'):
+    # direct call of __main__.py
+    import os.path
+    path = os.path.realpath(os.path.abspath(__file__))
+    sys.path.insert(0, os.path.dirname(os.path.dirname(path)))
+
+import yt_dlp
+
+if __name__ == '__main__':
+    yt_dlp.main()

plugins/youtube_download/yt_dlp/aes.py

@@ -0,0 +1,522 @@
+from __future__ import unicode_literals
+
+from math import ceil
+
+from .compat import (
+    compat_b64decode,
+    compat_ord,
+    compat_pycrypto_AES,
+)
+from .utils import (
+    bytes_to_intlist,
+    intlist_to_bytes,
+)
+
+
+if compat_pycrypto_AES:
+    def aes_cbc_decrypt_bytes(data, key, iv):
+        """ Decrypt bytes with AES-CBC using pycryptodome """
+        return compat_pycrypto_AES.new(key, compat_pycrypto_AES.MODE_CBC, iv).decrypt(data)
+
+    def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce):
+        """ Decrypt bytes with AES-GCM using pycryptodome """
+        return compat_pycrypto_AES.new(key, compat_pycrypto_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag)
+
+else:
+    def aes_cbc_decrypt_bytes(data, key, iv):
+        """ Decrypt bytes with AES-CBC using native implementation since pycryptodome is unavailable """
+        return intlist_to_bytes(aes_cbc_decrypt(*map(bytes_to_intlist, (data, key, iv))))
+
+    def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce):
+        """ Decrypt bytes with AES-GCM using native implementation since pycryptodome is unavailable """
+        return intlist_to_bytes(aes_gcm_decrypt_and_verify(*map(bytes_to_intlist, (data, key, tag, nonce))))
+
+
+def unpad_pkcs7(data):
+    return data[:-compat_ord(data[-1])]
+
+
+BLOCK_SIZE_BYTES = 16
+
+
+def aes_ecb_encrypt(data, key, iv=None):
+    """
+    Encrypt with aes in ECB mode
+
+    @param {int[]} data        cleartext
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          Unused for this mode
+    @returns {int[]}           encrypted data
+    """
+    expanded_key = key_expansion(key)
+    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+    encrypted_data = []
+    for i in range(block_count):
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        encrypted_data += aes_encrypt(block, expanded_key)
+    encrypted_data = encrypted_data[:len(data)]
+
+    return encrypted_data
+
+
+def aes_ecb_decrypt(data, key, iv=None):
+    """
+    Decrypt with aes in ECB mode
+
+    @param {int[]} data        cleartext
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          Unused for this mode
+    @returns {int[]}           decrypted data
+    """
+    expanded_key = key_expansion(key)
+    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+    encrypted_data = []
+    for i in range(block_count):
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        encrypted_data += aes_decrypt(block, expanded_key)
+    encrypted_data = encrypted_data[:len(data)]
+
+    return encrypted_data
+
+
+def aes_ctr_decrypt(data, key, iv):
+    """
+    Decrypt with aes in counter mode
+
+    @param {int[]} data        cipher
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          16-Byte initialization vector
+    @returns {int[]}           decrypted data
+    """
+    return aes_ctr_encrypt(data, key, iv)
+
+
+def aes_ctr_encrypt(data, key, iv):
+    """
+    Encrypt with aes in counter mode
+
+    @param {int[]} data        cleartext
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          16-Byte initialization vector
+    @returns {int[]}           encrypted data
+    """
+    expanded_key = key_expansion(key)
+    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+    counter = iter_vector(iv)
+
+    encrypted_data = []
+    for i in range(block_count):
+        counter_block = next(counter)
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        block += [0] * (BLOCK_SIZE_BYTES - len(block))
+
+        cipher_counter_block = aes_encrypt(counter_block, expanded_key)
+        encrypted_data += xor(block, cipher_counter_block)
+    encrypted_data = encrypted_data[:len(data)]
+
+    return encrypted_data
+
+
+def aes_cbc_decrypt(data, key, iv):
+    """
+    Decrypt with aes in CBC mode
+
+    @param {int[]} data        cipher
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          16-Byte IV
+    @returns {int[]}           decrypted data
+    """
+    expanded_key = key_expansion(key)
+    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+    decrypted_data = []
+    previous_cipher_block = iv
+    for i in range(block_count):
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        block += [0] * (BLOCK_SIZE_BYTES - len(block))
+
+        decrypted_block = aes_decrypt(block, expanded_key)
+        decrypted_data += xor(decrypted_block, previous_cipher_block)
+        previous_cipher_block = block
+    decrypted_data = decrypted_data[:len(data)]
+
+    return decrypted_data
+
+
+def aes_cbc_encrypt(data, key, iv):
+    """
+    Encrypt with aes in CBC mode. Using PKCS#7 padding
+
+    @param {int[]} data        cleartext
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          16-Byte IV
+    @returns {int[]}           encrypted data
+    """
+    expanded_key = key_expansion(key)
+    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+    encrypted_data = []
+    previous_cipher_block = iv
+    for i in range(block_count):
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        remaining_length = BLOCK_SIZE_BYTES - len(block)
+        block += [remaining_length] * remaining_length
+        mixed_block = xor(block, previous_cipher_block)
+
+        encrypted_block = aes_encrypt(mixed_block, expanded_key)
+        encrypted_data += encrypted_block
+
+        previous_cipher_block = encrypted_block
+
+    return encrypted_data
+
+
+def aes_gcm_decrypt_and_verify(data, key, tag, nonce):
+    """
+    Decrypt with aes in GBM mode and checks authenticity using tag
+
+    @param {int[]} data        cipher
+    @param {int[]} key         16-Byte cipher key
+    @param {int[]} tag         authentication tag
+    @param {int[]} nonce       IV (recommended 12-Byte)
+    @returns {int[]}           decrypted data
+    """
+
+    # XXX: check aes, gcm param
+
+    hash_subkey = aes_encrypt([0] * BLOCK_SIZE_BYTES, key_expansion(key))
+
+    if len(nonce) == 12:
+        j0 = nonce + [0, 0, 0, 1]
+    else:
+        fill = (BLOCK_SIZE_BYTES - (len(nonce) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES + 8
+        ghash_in = nonce + [0] * fill + bytes_to_intlist((8 * len(nonce)).to_bytes(8, 'big'))
+        j0 = ghash(hash_subkey, ghash_in)
+
+    # TODO: add nonce support to aes_ctr_decrypt
+
+    # nonce_ctr = j0[:12]
+    iv_ctr = inc(j0)
+
+    decrypted_data = aes_ctr_decrypt(data, key, iv_ctr + [0] * (BLOCK_SIZE_BYTES - len(iv_ctr)))
+    pad_len = len(data) // 16 * 16
+    s_tag = ghash(
+        hash_subkey,
+        data
+        + [0] * (BLOCK_SIZE_BYTES - len(data) + pad_len)        # pad
+        + bytes_to_intlist((0 * 8).to_bytes(8, 'big')           # length of associated data
+                           + ((len(data) * 8).to_bytes(8, 'big')))  # length of data
+    )
+
+    if tag != aes_ctr_encrypt(s_tag, key, j0):
+        raise ValueError("Mismatching authentication tag")
+
+    return decrypted_data
+
+
+def aes_encrypt(data, expanded_key):
+    """
+    Encrypt one block with aes
+
+    @param {int[]} data          16-Byte state
+    @param {int[]} expanded_key  176/208/240-Byte expanded key
+    @returns {int[]}             16-Byte cipher
+    """
+    rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
+
+    data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
+    for i in range(1, rounds + 1):
+        data = sub_bytes(data)
+        data = shift_rows(data)
+        if i != rounds:
+            data = list(iter_mix_columns(data, MIX_COLUMN_MATRIX))
+        data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
+
+    return data
+
+
+def aes_decrypt(data, expanded_key):
+    """
+    Decrypt one block with aes
+
+    @param {int[]} data          16-Byte cipher
+    @param {int[]} expanded_key  176/208/240-Byte expanded key
+    @returns {int[]}             16-Byte state
+    """
+    rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
+
+    for i in range(rounds, 0, -1):
+        data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
+        if i != rounds:
+            data = list(iter_mix_columns(data, MIX_COLUMN_MATRIX_INV))
+        data = shift_rows_inv(data)
+        data = sub_bytes_inv(data)
+    data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
+
+    return data
+
+
+def aes_decrypt_text(data, password, key_size_bytes):
+    """
+    Decrypt text
+    - The first 8 Bytes of decoded 'data' are the 8 high Bytes of the counter
+    - The cipher key is retrieved by encrypting the first 16 Byte of 'password'
+      with the first 'key_size_bytes' Bytes from 'password' (if necessary filled with 0's)
+    - Mode of operation is 'counter'
+
+    @param {str} data                    Base64 encoded string
+    @param {str,unicode} password        Password (will be encoded with utf-8)
+    @param {int} key_size_bytes          Possible values: 16 for 128-Bit, 24 for 192-Bit or 32 for 256-Bit
+    @returns {str}                       Decrypted data
+    """
+    NONCE_LENGTH_BYTES = 8
+
+    data = bytes_to_intlist(compat_b64decode(data))
+    password = bytes_to_intlist(password.encode('utf-8'))
+
+    key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))
+    key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES)
+
+    nonce = data[:NONCE_LENGTH_BYTES]
+    cipher = data[NONCE_LENGTH_BYTES:]
+
+    decrypted_data = aes_ctr_decrypt(cipher, key, nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES))
+    plaintext = intlist_to_bytes(decrypted_data)
+
+    return plaintext
+
+
+RCON = (0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36)
+SBOX = (0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
+        0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0,
+        0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15,
+        0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75,
+        0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84,
+        0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF,
+        0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8,
+        0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2,
+        0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73,
+        0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB,
+        0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79,
+        0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08,
+        0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A,
+        0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E,
+        0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF,
+        0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16)
+SBOX_INV = (0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
+            0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
+            0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
+            0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
+            0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
+            0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
+            0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
+            0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
+            0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
+            0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
+            0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
+            0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
+            0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
+            0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
+            0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
+            0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d)
+MIX_COLUMN_MATRIX = ((0x2, 0x3, 0x1, 0x1),
+                     (0x1, 0x2, 0x3, 0x1),
+                     (0x1, 0x1, 0x2, 0x3),
+                     (0x3, 0x1, 0x1, 0x2))
+MIX_COLUMN_MATRIX_INV = ((0xE, 0xB, 0xD, 0x9),
+                         (0x9, 0xE, 0xB, 0xD),
+                         (0xD, 0x9, 0xE, 0xB),
+                         (0xB, 0xD, 0x9, 0xE))
+RIJNDAEL_EXP_TABLE = (0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35,
+                      0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA,
+                      0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31,
+                      0x53, 0xF5, 0x04, 0x0C, 0x14, 0x3C, 0x44, 0xCC, 0x4F, 0xD1, 0x68, 0xB8, 0xD3, 0x6E, 0xB2, 0xCD,
+                      0x4C, 0xD4, 0x67, 0xA9, 0xE0, 0x3B, 0x4D, 0xD7, 0x62, 0xA6, 0xF1, 0x08, 0x18, 0x28, 0x78, 0x88,
+                      0x83, 0x9E, 0xB9, 0xD0, 0x6B, 0xBD, 0xDC, 0x7F, 0x81, 0x98, 0xB3, 0xCE, 0x49, 0xDB, 0x76, 0x9A,
+                      0xB5, 0xC4, 0x57, 0xF9, 0x10, 0x30, 0x50, 0xF0, 0x0B, 0x1D, 0x27, 0x69, 0xBB, 0xD6, 0x61, 0xA3,
+                      0xFE, 0x19, 0x2B, 0x7D, 0x87, 0x92, 0xAD, 0xEC, 0x2F, 0x71, 0x93, 0xAE, 0xE9, 0x20, 0x60, 0xA0,
+                      0xFB, 0x16, 0x3A, 0x4E, 0xD2, 0x6D, 0xB7, 0xC2, 0x5D, 0xE7, 0x32, 0x56, 0xFA, 0x15, 0x3F, 0x41,
+                      0xC3, 0x5E, 0xE2, 0x3D, 0x47, 0xC9, 0x40, 0xC0, 0x5B, 0xED, 0x2C, 0x74, 0x9C, 0xBF, 0xDA, 0x75,
+                      0x9F, 0xBA, 0xD5, 0x64, 0xAC, 0xEF, 0x2A, 0x7E, 0x82, 0x9D, 0xBC, 0xDF, 0x7A, 0x8E, 0x89, 0x80,
+                      0x9B, 0xB6, 0xC1, 0x58, 0xE8, 0x23, 0x65, 0xAF, 0xEA, 0x25, 0x6F, 0xB1, 0xC8, 0x43, 0xC5, 0x54,
+                      0xFC, 0x1F, 0x21, 0x63, 0xA5, 0xF4, 0x07, 0x09, 0x1B, 0x2D, 0x77, 0x99, 0xB0, 0xCB, 0x46, 0xCA,
+                      0x45, 0xCF, 0x4A, 0xDE, 0x79, 0x8B, 0x86, 0x91, 0xA8, 0xE3, 0x3E, 0x42, 0xC6, 0x51, 0xF3, 0x0E,
+                      0x12, 0x36, 0x5A, 0xEE, 0x29, 0x7B, 0x8D, 0x8C, 0x8F, 0x8A, 0x85, 0x94, 0xA7, 0xF2, 0x0D, 0x17,
+                      0x39, 0x4B, 0xDD, 0x7C, 0x84, 0x97, 0xA2, 0xFD, 0x1C, 0x24, 0x6C, 0xB4, 0xC7, 0x52, 0xF6, 0x01)
+RIJNDAEL_LOG_TABLE = (0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7, 0x1b, 0x68, 0x33, 0xee, 0xdf, 0x03,
+                      0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef, 0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1,
+                      0x7d, 0xc2, 0x1d, 0xb5, 0xf9, 0xb9, 0x27, 0x6a, 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78,
+                      0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, 0x12, 0xf0, 0x82, 0x45, 0x35, 0x93, 0xda, 0x8e,
+                      0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94, 0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38,
+                      0x66, 0xdd, 0xfd, 0x30, 0xbf, 0x06, 0x8b, 0x62, 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10,
+                      0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, 0x3a, 0x6b, 0x28, 0x54, 0xfa, 0x85, 0x3d, 0xba,
+                      0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca, 0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57,
+                      0xaf, 0x58, 0xa8, 0x50, 0xf4, 0xea, 0xd6, 0x74, 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8,
+                      0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, 0x59, 0xcb, 0x5f, 0xb0, 0x9c, 0xa9, 0x51, 0xa0,
+                      0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec, 0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7,
+                      0xcc, 0xbb, 0x3e, 0x5a, 0xfb, 0x60, 0xb1, 0x86, 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d,
+                      0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, 0xbc, 0x95, 0xcf, 0xcd, 0x37, 0x3f, 0x5b, 0xd1,
+                      0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47, 0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab,
+                      0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5,
+                      0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07)
+
+
+def key_expansion(data):
+    """
+    Generate key schedule
+
+    @param {int[]} data  16/24/32-Byte cipher key
+    @returns {int[]}     176/208/240-Byte expanded key
+    """
+    data = data[:]  # copy
+    rcon_iteration = 1
+    key_size_bytes = len(data)
+    expanded_key_size_bytes = (key_size_bytes // 4 + 7) * BLOCK_SIZE_BYTES
+
+    while len(data) < expanded_key_size_bytes:
+        temp = data[-4:]
+        temp = key_schedule_core(temp, rcon_iteration)
+        rcon_iteration += 1
+        data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+
+        for _ in range(3):
+            temp = data[-4:]
+            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+
+        if key_size_bytes == 32:
+            temp = data[-4:]
+            temp = sub_bytes(temp)
+            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+
+        for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0):
+            temp = data[-4:]
+            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+    data = data[:expanded_key_size_bytes]
+
+    return data
+
+
+def iter_vector(iv):
+    while True:
+        yield iv
+        iv = inc(iv)
+
+
+def sub_bytes(data):
+    return [SBOX[x] for x in data]
+
+
+def sub_bytes_inv(data):
+    return [SBOX_INV[x] for x in data]
+
+
+def rotate(data):
+    return data[1:] + [data[0]]
+
+
+def key_schedule_core(data, rcon_iteration):
+    data = rotate(data)
+    data = sub_bytes(data)
+    data[0] = data[0] ^ RCON[rcon_iteration]
+
+    return data
+
+
+def xor(data1, data2):
+    return [x ^ y for x, y in zip(data1, data2)]
+
+
+def iter_mix_columns(data, matrix):
+    for i in (0, 4, 8, 12):
+        for row in matrix:
+            mixed = 0
+            for j in range(4):
+                # xor is (+) and (-)
+                mixed ^= (0 if data[i:i + 4][j] == 0 or row[j] == 0 else
+                          RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[data[i + j]] + RIJNDAEL_LOG_TABLE[row[j]]) % 0xFF])
+            yield mixed
+
+
+def shift_rows(data):
+    return [data[((column + row) & 0b11) * 4 + row] for column in range(4) for row in range(4)]
+
+
+def shift_rows_inv(data):
+    return [data[((column - row) & 0b11) * 4 + row] for column in range(4) for row in range(4)]
+
+
+def shift_block(data):
+    data_shifted = []
+
+    bit = 0
+    for n in data:
+        if bit:
+            n |= 0x100
+        bit = n & 1
+        n >>= 1
+        data_shifted.append(n)
+
+    return data_shifted
+
+
+def inc(data):
+    data = data[:]  # copy
+    for i in range(len(data) - 1, -1, -1):
+        if data[i] == 255:
+            data[i] = 0
+        else:
+            data[i] = data[i] + 1
+            break
+    return data
+
+
+def block_product(block_x, block_y):
+    # NIST SP 800-38D, Algorithm 1
+
+    if len(block_x) != BLOCK_SIZE_BYTES or len(block_y) != BLOCK_SIZE_BYTES:
+        raise ValueError("Length of blocks need to be %d bytes" % BLOCK_SIZE_BYTES)
+
+    block_r = [0xE1] + [0] * (BLOCK_SIZE_BYTES - 1)
+    block_v = block_y[:]
+    block_z = [0] * BLOCK_SIZE_BYTES
+
+    for i in block_x:
+        for bit in range(7, -1, -1):
+            if i & (1 << bit):
+                block_z = xor(block_z, block_v)
+
+            do_xor = block_v[-1] & 1
+            block_v = shift_block(block_v)
+            if do_xor:
+                block_v = xor(block_v, block_r)
+
+    return block_z
+
+
+def ghash(subkey, data):
+    # NIST SP 800-38D, Algorithm 2
+
+    if len(data) % BLOCK_SIZE_BYTES:
+        raise ValueError("Length of data should be %d bytes" % BLOCK_SIZE_BYTES)
+
+    last_y = [0] * BLOCK_SIZE_BYTES
+    for i in range(0, len(data), BLOCK_SIZE_BYTES):
+        block = data[i : i + BLOCK_SIZE_BYTES]  # noqa: E203
+        last_y = block_product(xor(last_y, block), subkey)
+
+    return last_y
+
+
+__all__ = [
+    'aes_ctr_decrypt',
+    'aes_cbc_decrypt',
+    'aes_cbc_decrypt_bytes',
+    'aes_decrypt_text',
+    'aes_encrypt',
+    'aes_gcm_decrypt_and_verify',
+    'aes_gcm_decrypt_and_verify_bytes',
+    'key_expansion',
+    'unpad_pkcs7',
+]

plugins/youtube_download/yt_dlp/cache.py

@@ -0,0 +1,98 @@
+from __future__ import unicode_literals
+
+import errno
+import io
+import json
+import os
+import re
+import shutil
+import traceback
+
+from .compat import compat_getenv
+from .utils import (
+    expand_path,
+    write_json_file,
+)
+
+
+class Cache(object):
+    def __init__(self, ydl):
+        self._ydl = ydl
+
+    def _get_root_dir(self):
+        res = self._ydl.params.get('cachedir')
+        if res is None:
+            cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
+            res = os.path.join(cache_root, 'yt-dlp')
+        return expand_path(res)
+
+    def _get_cache_fn(self, section, key, dtype):
+        assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
+            'invalid section %r' % section
+        assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key
+        return os.path.join(
+            self._get_root_dir(), section, '%s.%s' % (key, dtype))
+
+    @property
+    def enabled(self):
+        return self._ydl.params.get('cachedir') is not False
+
+    def store(self, section, key, data, dtype='json'):
+        assert dtype in ('json',)
+
+        if not self.enabled:
+            return
+
+        fn = self._get_cache_fn(section, key, dtype)
+        try:
+            try:
+                os.makedirs(os.path.dirname(fn))
+            except OSError as ose:
+                if ose.errno != errno.EEXIST:
+                    raise
+            self._ydl.write_debug(f'Saving {section}.{key} to cache')
+            write_json_file(data, fn)
+        except Exception:
+            tb = traceback.format_exc()
+            self._ydl.report_warning(
+                'Writing cache to %r failed: %s' % (fn, tb))
+
+    def load(self, section, key, dtype='json', default=None):
+        assert dtype in ('json',)
+
+        if not self.enabled:
+            return default
+
+        cache_fn = self._get_cache_fn(section, key, dtype)
+        try:
+            try:
+                with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
+                    self._ydl.write_debug(f'Loading {section}.{key} from cache')
+                    return json.load(cachef)
+            except ValueError:
+                try:
+                    file_size = os.path.getsize(cache_fn)
+                except (OSError, IOError) as oe:
+                    file_size = str(oe)
+                self._ydl.report_warning(
+                    'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
+        except IOError:
+            pass  # No cache available
+
+        return default
+
+    def remove(self):
+        if not self.enabled:
+            self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
+            return
+
+        cachedir = self._get_root_dir()
+        if not any((term in cachedir) for term in ('cache', 'tmp')):
+            raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir)
+
+        self._ydl.to_screen(
+            'Removing cache dir %s .' % cachedir, skip_eol=True)
+        if os.path.exists(cachedir):
+            self._ydl.to_screen('.', skip_eol=True)
+            shutil.rmtree(cachedir)
+        self._ydl.to_screen('.')

plugins/youtube_download/yt_dlp/compat.py

@@ -0,0 +1,311 @@
+# coding: utf-8
+
+import asyncio
+import base64
+import collections
+import ctypes
+import getpass
+import html
+import html.parser
+import http
+import http.client
+import http.cookiejar
+import http.cookies
+import http.server
+import itertools
+import optparse
+import os
+import re
+import shlex
+import shutil
+import socket
+import struct
+import subprocess
+import sys
+import tokenize
+import urllib
+import xml.etree.ElementTree as etree
+from subprocess import DEVNULL
+
+
+# HTMLParseError has been deprecated in Python 3.3 and removed in
+# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
+# and uniform cross-version exception handling
+class compat_HTMLParseError(Exception):
+    pass
+
+
+# compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE
+# will not work since ctypes.WINFUNCTYPE does not exist in UNIX machines
+def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
+    return ctypes.WINFUNCTYPE(*args, **kwargs)
+
+
+class _TreeBuilder(etree.TreeBuilder):
+    def doctype(self, name, pubid, system):
+        pass
+
+
+def compat_etree_fromstring(text):
+    return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
+
+
+compat_os_name = os._name if os.name == 'java' else os.name
+
+
+if compat_os_name == 'nt':
+    def compat_shlex_quote(s):
+        return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
+else:
+    from shlex import quote as compat_shlex_quote
+
+
+def compat_ord(c):
+    if type(c) is int:
+        return c
+    else:
+        return ord(c)
+
+
+def compat_setenv(key, value, env=os.environ):
+    env[key] = value
+
+
+if compat_os_name == 'nt' and sys.version_info < (3, 8):
+    # os.path.realpath on Windows does not follow symbolic links
+    # prior to Python 3.8 (see https://bugs.python.org/issue9949)
+    def compat_realpath(path):
+        while os.path.islink(path):
+            path = os.path.abspath(os.readlink(path))
+        return path
+else:
+    compat_realpath = os.path.realpath
+
+
+def compat_print(s):
+    assert isinstance(s, compat_str)
+    print(s)
+
+
+# Fix https://github.com/ytdl-org/youtube-dl/issues/4223
+# See http://bugs.python.org/issue9161 for what is broken
+def workaround_optparse_bug9161():
+    op = optparse.OptionParser()
+    og = optparse.OptionGroup(op, 'foo')
+    try:
+        og.add_option('-t')
+    except TypeError:
+        real_add_option = optparse.OptionGroup.add_option
+
+        def _compat_add_option(self, *args, **kwargs):
+            enc = lambda v: (
+                v.encode('ascii', 'replace') if isinstance(v, compat_str)
+                else v)
+            bargs = [enc(a) for a in args]
+            bkwargs = dict(
+                (k, enc(v)) for k, v in kwargs.items())
+            return real_add_option(self, *bargs, **bkwargs)
+        optparse.OptionGroup.add_option = _compat_add_option
+
+
+try:
+    compat_Pattern = re.Pattern
+except AttributeError:
+    compat_Pattern = type(re.compile(''))
+
+
+try:
+    compat_Match = re.Match
+except AttributeError:
+    compat_Match = type(re.compile('').match(''))
+
+
+try:
+    compat_asyncio_run = asyncio.run  # >= 3.7
+except AttributeError:
+    def compat_asyncio_run(coro):
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+        loop.run_until_complete(coro)
+
+    asyncio.run = compat_asyncio_run
+
+
+# Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl
+# See https://github.com/yt-dlp/yt-dlp/issues/792
+# https://docs.python.org/3/library/os.path.html#os.path.expanduser
+if compat_os_name in ('nt', 'ce') and 'HOME' in os.environ:
+    _userhome = os.environ['HOME']
+
+    def compat_expanduser(path):
+        if not path.startswith('~'):
+            return path
+        i = path.replace('\\', '/', 1).find('/')  # ~user
+        if i < 0:
+            i = len(path)
+        userhome = os.path.join(os.path.dirname(_userhome), path[1:i]) if i > 1 else _userhome
+        return userhome + path[i:]
+else:
+    compat_expanduser = os.path.expanduser
+
+
+try:
+    from Cryptodome.Cipher import AES as compat_pycrypto_AES
+except ImportError:
+    try:
+        from Crypto.Cipher import AES as compat_pycrypto_AES
+    except ImportError:
+        compat_pycrypto_AES = None
+
+
+WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
+
+
+def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.python.org/issue30075
+    if compat_os_name != 'nt':
+        return
+    global WINDOWS_VT_MODE
+    startupinfo = subprocess.STARTUPINFO()
+    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+    try:
+        subprocess.Popen('', shell=True, startupinfo=startupinfo)
+        WINDOWS_VT_MODE = True
+    except Exception:
+        pass
+
+
+#  Deprecated
+
+compat_basestring = str
+compat_chr = chr
+compat_filter = filter
+compat_input = input
+compat_integer_types = (int, )
+compat_kwargs = lambda kwargs: kwargs
+compat_map = map
+compat_numeric_types = (int, float, complex)
+compat_str = str
+compat_xpath = lambda xpath: xpath
+compat_zip = zip
+
+compat_collections_abc = collections.abc
+compat_HTMLParser = html.parser.HTMLParser
+compat_HTTPError = urllib.error.HTTPError
+compat_Struct = struct.Struct
+compat_b64decode = base64.b64decode
+compat_cookiejar = http.cookiejar
+compat_cookiejar_Cookie = compat_cookiejar.Cookie
+compat_cookies = http.cookies
+compat_cookies_SimpleCookie = compat_cookies.SimpleCookie
+compat_etree_Element = etree.Element
+compat_etree_register_namespace = etree.register_namespace
+compat_get_terminal_size = shutil.get_terminal_size
+compat_getenv = os.getenv
+compat_getpass = getpass.getpass
+compat_html_entities = html.entities
+compat_html_entities_html5 = compat_html_entities.html5
+compat_http_client = http.client
+compat_http_server = http.server
+compat_itertools_count = itertools.count
+compat_parse_qs = urllib.parse.parse_qs
+compat_shlex_split = shlex.split
+compat_socket_create_connection = socket.create_connection
+compat_struct_pack = struct.pack
+compat_struct_unpack = struct.unpack
+compat_subprocess_get_DEVNULL = lambda: DEVNULL
+compat_tokenize_tokenize = tokenize.tokenize
+compat_urllib_error = urllib.error
+compat_urllib_parse = urllib.parse
+compat_urllib_parse_quote = urllib.parse.quote
+compat_urllib_parse_quote_plus = urllib.parse.quote_plus
+compat_urllib_parse_unquote = urllib.parse.unquote
+compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus
+compat_urllib_parse_unquote_to_bytes = urllib.parse.unquote_to_bytes
+compat_urllib_parse_urlencode = urllib.parse.urlencode
+compat_urllib_parse_urlparse = urllib.parse.urlparse
+compat_urllib_parse_urlunparse = urllib.parse.urlunparse
+compat_urllib_request = urllib.request
+compat_urllib_request_DataHandler = urllib.request.DataHandler
+compat_urllib_response = urllib.response
+compat_urlparse = urllib.parse
+compat_urlretrieve = urllib.request.urlretrieve
+compat_xml_parse_error = etree.ParseError
+
+
+# Set public objects
+
+__all__ = [
+    'WINDOWS_VT_MODE',
+    'compat_HTMLParseError',
+    'compat_HTMLParser',
+    'compat_HTTPError',
+    'compat_Match',
+    'compat_Pattern',
+    'compat_Struct',
+    'compat_asyncio_run',
+    'compat_b64decode',
+    'compat_basestring',
+    'compat_chr',
+    'compat_collections_abc',
+    'compat_cookiejar',
+    'compat_cookiejar_Cookie',
+    'compat_cookies',
+    'compat_cookies_SimpleCookie',
+    'compat_ctypes_WINFUNCTYPE',
+    'compat_etree_Element',
+    'compat_etree_fromstring',
+    'compat_etree_register_namespace',
+    'compat_expanduser',
+    'compat_filter',
+    'compat_get_terminal_size',
+    'compat_getenv',
+    'compat_getpass',
+    'compat_html_entities',
+    'compat_html_entities_html5',
+    'compat_http_client',
+    'compat_http_server',
+    'compat_input',
+    'compat_integer_types',
+    'compat_itertools_count',
+    'compat_kwargs',
+    'compat_map',
+    'compat_numeric_types',
+    'compat_ord',
+    'compat_os_name',
+    'compat_parse_qs',
+    'compat_print',
+    'compat_pycrypto_AES',
+    'compat_realpath',
+    'compat_setenv',
+    'compat_shlex_quote',
+    'compat_shlex_split',
+    'compat_socket_create_connection',
+    'compat_str',
+    'compat_struct_pack',
+    'compat_struct_unpack',
+    'compat_subprocess_get_DEVNULL',
+    'compat_tokenize_tokenize',
+    'compat_urllib_error',
+    'compat_urllib_parse',
+    'compat_urllib_parse_quote',
+    'compat_urllib_parse_quote_plus',
+    'compat_urllib_parse_unquote',
+    'compat_urllib_parse_unquote_plus',
+    'compat_urllib_parse_unquote_to_bytes',
+    'compat_urllib_parse_urlencode',
+    'compat_urllib_parse_urlparse',
+    'compat_urllib_parse_urlunparse',
+    'compat_urllib_request',
+    'compat_urllib_request_DataHandler',
+    'compat_urllib_response',
+    'compat_urlparse',
+    'compat_urlretrieve',
+    'compat_xml_parse_error',
+    'compat_xpath',
+    'compat_zip',
+    'windows_enable_vt_mode',
+    'workaround_optparse_bug9161',
+]

plugins/youtube_download/yt_dlp/cookies.py

@@ -0,0 +1,955 @@
+import contextlib
+import ctypes
+import json
+import os
+import shutil
+import struct
+import subprocess
+import sys
+import tempfile
+from datetime import datetime, timedelta, timezone
+from enum import Enum, auto
+from hashlib import pbkdf2_hmac
+
+from .aes import (
+    aes_cbc_decrypt_bytes,
+    aes_gcm_decrypt_and_verify_bytes,
+    unpad_pkcs7,
+)
+from .compat import (
+    compat_b64decode,
+    compat_cookiejar_Cookie,
+)
+from .utils import (
+    expand_path,
+    Popen,
+    YoutubeDLCookieJar,
+)
+
+try:
+    import sqlite3
+    SQLITE_AVAILABLE = True
+except ImportError:
+    # although sqlite3 is part of the standard library, it is possible to compile python without
+    # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544
+    SQLITE_AVAILABLE = False
+
+
+try:
+    import secretstorage
+    SECRETSTORAGE_AVAILABLE = True
+except ImportError:
+    SECRETSTORAGE_AVAILABLE = False
+    SECRETSTORAGE_UNAVAILABLE_REASON = (
+        'as the `secretstorage` module is not installed. '
+        'Please install by running `python3 -m pip install secretstorage`.')
+except Exception as _err:
+    SECRETSTORAGE_AVAILABLE = False
+    SECRETSTORAGE_UNAVAILABLE_REASON = f'as the `secretstorage` module could not be initialized. {_err}'
+
+
+CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
+SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
+
+
+class YDLLogger:
+    def __init__(self, ydl=None):
+        self._ydl = ydl
+
+    def debug(self, message):
+        if self._ydl:
+            self._ydl.write_debug(message)
+
+    def info(self, message):
+        if self._ydl:
+            self._ydl.to_screen(f'[Cookies] {message}')
+
+    def warning(self, message, only_once=False):
+        if self._ydl:
+            self._ydl.report_warning(message, only_once)
+
+    def error(self, message):
+        if self._ydl:
+            self._ydl.report_error(message)
+
+
+def load_cookies(cookie_file, browser_specification, ydl):
+    cookie_jars = []
+    if browser_specification is not None:
+        browser_name, profile, keyring = _parse_browser_specification(*browser_specification)
+        cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring))
+
+    if cookie_file is not None:
+        cookie_file = expand_path(cookie_file)
+        jar = YoutubeDLCookieJar(cookie_file)
+        if os.access(cookie_file, os.R_OK):
+            jar.load(ignore_discard=True, ignore_expires=True)
+        cookie_jars.append(jar)
+
+    return _merge_cookie_jars(cookie_jars)
+
+
+def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None):
+    if browser_name == 'firefox':
+        return _extract_firefox_cookies(profile, logger)
+    elif browser_name == 'safari':
+        return _extract_safari_cookies(profile, logger)
+    elif browser_name in CHROMIUM_BASED_BROWSERS:
+        return _extract_chrome_cookies(browser_name, profile, keyring, logger)
+    else:
+        raise ValueError('unknown browser: {}'.format(browser_name))
+
+
+def _extract_firefox_cookies(profile, logger):
+    logger.info('Extracting cookies from firefox')
+    if not SQLITE_AVAILABLE:
+        logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
+                       'Please use a python interpreter compiled with sqlite3 support')
+        return YoutubeDLCookieJar()
+
+    if profile is None:
+        search_root = _firefox_browser_dir()
+    elif _is_path(profile):
+        search_root = profile
+    else:
+        search_root = os.path.join(_firefox_browser_dir(), profile)
+
+    cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite')
+    if cookie_database_path is None:
+        raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root))
+    logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
+
+    with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
+        cursor = None
+        try:
+            cursor = _open_database_copy(cookie_database_path, tmpdir)
+            cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
+            jar = YoutubeDLCookieJar()
+            for host, name, value, path, expiry, is_secure in cursor.fetchall():
+                cookie = compat_cookiejar_Cookie(
+                    version=0, name=name, value=value, port=None, port_specified=False,
+                    domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
+                    path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
+                    comment=None, comment_url=None, rest={})
+                jar.set_cookie(cookie)
+            logger.info('Extracted {} cookies from firefox'.format(len(jar)))
+            return jar
+        finally:
+            if cursor is not None:
+                cursor.connection.close()
+
+
+def _firefox_browser_dir():
+    if sys.platform in ('linux', 'linux2'):
+        return os.path.expanduser('~/.mozilla/firefox')
+    elif sys.platform == 'win32':
+        return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox\Profiles')
+    elif sys.platform == 'darwin':
+        return os.path.expanduser('~/Library/Application Support/Firefox')
+    else:
+        raise ValueError('unsupported platform: {}'.format(sys.platform))
+
+
+def _get_chromium_based_browser_settings(browser_name):
+    # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md
+    if sys.platform in ('linux', 'linux2'):
+        config = _config_home()
+        browser_dir = {
+            'brave': os.path.join(config, 'BraveSoftware/Brave-Browser'),
+            'chrome': os.path.join(config, 'google-chrome'),
+            'chromium': os.path.join(config, 'chromium'),
+            'edge': os.path.join(config, 'microsoft-edge'),
+            'opera': os.path.join(config, 'opera'),
+            'vivaldi': os.path.join(config, 'vivaldi'),
+        }[browser_name]
+
+    elif sys.platform == 'win32':
+        appdata_local = os.path.expandvars('%LOCALAPPDATA%')
+        appdata_roaming = os.path.expandvars('%APPDATA%')
+        browser_dir = {
+            'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser\User Data'),
+            'chrome': os.path.join(appdata_local, r'Google\Chrome\User Data'),
+            'chromium': os.path.join(appdata_local, r'Chromium\User Data'),
+            'edge': os.path.join(appdata_local, r'Microsoft\Edge\User Data'),
+            'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'),
+            'vivaldi': os.path.join(appdata_local, r'Vivaldi\User Data'),
+        }[browser_name]
+
+    elif sys.platform == 'darwin':
+        appdata = os.path.expanduser('~/Library/Application Support')
+        browser_dir = {
+            'brave': os.path.join(appdata, 'BraveSoftware/Brave-Browser'),
+            'chrome': os.path.join(appdata, 'Google/Chrome'),
+            'chromium': os.path.join(appdata, 'Chromium'),
+            'edge': os.path.join(appdata, 'Microsoft Edge'),
+            'opera': os.path.join(appdata, 'com.operasoftware.Opera'),
+            'vivaldi': os.path.join(appdata, 'Vivaldi'),
+        }[browser_name]
+
+    else:
+        raise ValueError('unsupported platform: {}'.format(sys.platform))
+
+    # Linux keyring names can be determined by snooping on dbus while opening the browser in KDE:
+    # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
+    keyring_name = {
+        'brave': 'Brave',
+        'chrome': 'Chrome',
+        'chromium': 'Chromium',
+        'edge': 'Microsoft Edge' if sys.platform == 'darwin' else 'Chromium',
+        'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium',
+        'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome',
+    }[browser_name]
+
+    browsers_without_profiles = {'opera'}
+
+    return {
+        'browser_dir': browser_dir,
+        'keyring_name': keyring_name,
+        'supports_profiles': browser_name not in browsers_without_profiles
+    }
+
+
+def _extract_chrome_cookies(browser_name, profile, keyring, logger):
+    logger.info('Extracting cookies from {}'.format(browser_name))
+
+    if not SQLITE_AVAILABLE:
+        logger.warning(('Cannot extract cookies from {} without sqlite3 support. '
+                        'Please use a python interpreter compiled with sqlite3 support').format(browser_name))
+        return YoutubeDLCookieJar()
+
+    config = _get_chromium_based_browser_settings(browser_name)
+
+    if profile is None:
+        search_root = config['browser_dir']
+    elif _is_path(profile):
+        search_root = profile
+        config['browser_dir'] = os.path.dirname(profile) if config['supports_profiles'] else profile
+    else:
+        if config['supports_profiles']:
+            search_root = os.path.join(config['browser_dir'], profile)
+        else:
+            logger.error('{} does not support profiles'.format(browser_name))
+            search_root = config['browser_dir']
+
+    cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies')
+    if cookie_database_path is None:
+        raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root))
+    logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
+
+    decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger, keyring=keyring)
+
+    with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
+        cursor = None
+        try:
+            cursor = _open_database_copy(cookie_database_path, tmpdir)
+            cursor.connection.text_factory = bytes
+            column_names = _get_column_names(cursor, 'cookies')
+            secure_column = 'is_secure' if 'is_secure' in column_names else 'secure'
+            cursor.execute('SELECT host_key, name, value, encrypted_value, path, '
+                           'expires_utc, {} FROM cookies'.format(secure_column))
+            jar = YoutubeDLCookieJar()
+            failed_cookies = 0
+            unencrypted_cookies = 0
+            for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall():
+                host_key = host_key.decode('utf-8')
+                name = name.decode('utf-8')
+                value = value.decode('utf-8')
+                path = path.decode('utf-8')
+
+                if not value and encrypted_value:
+                    value = decryptor.decrypt(encrypted_value)
+                    if value is None:
+                        failed_cookies += 1
+                        continue
+                else:
+                    unencrypted_cookies += 1
+
+                cookie = compat_cookiejar_Cookie(
+                    version=0, name=name, value=value, port=None, port_specified=False,
+                    domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
+                    path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
+                    comment=None, comment_url=None, rest={})
+                jar.set_cookie(cookie)
+            if failed_cookies > 0:
+                failed_message = ' ({} could not be decrypted)'.format(failed_cookies)
+            else:
+                failed_message = ''
+            logger.info('Extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message))
+            counts = decryptor.cookie_counts.copy()
+            counts['unencrypted'] = unencrypted_cookies
+            logger.debug('cookie version breakdown: {}'.format(counts))
+            return jar
+        finally:
+            if cursor is not None:
+                cursor.connection.close()
+
+
+class ChromeCookieDecryptor:
+    """
+    Overview:
+
+        Linux:
+        - cookies are either v10 or v11
+            - v10: AES-CBC encrypted with a fixed key
+            - v11: AES-CBC encrypted with an OS protected key (keyring)
+            - v11 keys can be stored in various places depending on the activate desktop environment [2]
+
+        Mac:
+        - cookies are either v10 or not v10
+            - v10: AES-CBC encrypted with an OS protected key (keyring) and more key derivation iterations than linux
+            - not v10: 'old data' stored as plaintext
+
+        Windows:
+        - cookies are either v10 or not v10
+            - v10: AES-GCM encrypted with a key which is encrypted with DPAPI
+            - not v10: encrypted with DPAPI
+
+    Sources:
+    - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/
+    - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc
+        - KeyStorageLinux::CreateService
+    """
+
+    def decrypt(self, encrypted_value):
+        raise NotImplementedError
+
+    @property
+    def cookie_counts(self):
+        raise NotImplementedError
+
+
+def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None):
+    if sys.platform in ('linux', 'linux2'):
+        return LinuxChromeCookieDecryptor(browser_keyring_name, logger, keyring=keyring)
+    elif sys.platform == 'darwin':
+        return MacChromeCookieDecryptor(browser_keyring_name, logger)
+    elif sys.platform == 'win32':
+        return WindowsChromeCookieDecryptor(browser_root, logger)
+    else:
+        raise NotImplementedError('Chrome cookie decryption is not supported '
+                                  'on this platform: {}'.format(sys.platform))
+
+
+class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
+    def __init__(self, browser_keyring_name, logger, *, keyring=None):
+        self._logger = logger
+        self._v10_key = self.derive_key(b'peanuts')
+        password = _get_linux_keyring_password(browser_keyring_name, keyring, logger)
+        self._v11_key = None if password is None else self.derive_key(password)
+        self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0}
+
+    @staticmethod
+    def derive_key(password):
+        # values from
+        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
+        return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
+
+    @property
+    def cookie_counts(self):
+        return self._cookie_counts
+
+    def decrypt(self, encrypted_value):
+        version = encrypted_value[:3]
+        ciphertext = encrypted_value[3:]
+
+        if version == b'v10':
+            self._cookie_counts['v10'] += 1
+            return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
+
+        elif version == b'v11':
+            self._cookie_counts['v11'] += 1
+            if self._v11_key is None:
+                self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True)
+                return None
+            return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
+
+        else:
+            self._cookie_counts['other'] += 1
+            return None
+
+
+class MacChromeCookieDecryptor(ChromeCookieDecryptor):
+    def __init__(self, browser_keyring_name, logger):
+        self._logger = logger
+        password = _get_mac_keyring_password(browser_keyring_name, logger)
+        self._v10_key = None if password is None else self.derive_key(password)
+        self._cookie_counts = {'v10': 0, 'other': 0}
+
+    @staticmethod
+    def derive_key(password):
+        # values from
+        # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
+        return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
+
+    @property
+    def cookie_counts(self):
+        return self._cookie_counts
+
+    def decrypt(self, encrypted_value):
+        version = encrypted_value[:3]
+        ciphertext = encrypted_value[3:]
+
+        if version == b'v10':
+            self._cookie_counts['v10'] += 1
+            if self._v10_key is None:
+                self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
+                return None
+
+            return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
+
+        else:
+            self._cookie_counts['other'] += 1
+            # other prefixes are considered 'old data' which were stored as plaintext
+            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
+            return encrypted_value
+
+
+class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
+    def __init__(self, browser_root, logger):
+        self._logger = logger
+        self._v10_key = _get_windows_v10_key(browser_root, logger)
+        self._cookie_counts = {'v10': 0, 'other': 0}
+
+    @property
+    def cookie_counts(self):
+        return self._cookie_counts
+
+    def decrypt(self, encrypted_value):
+        version = encrypted_value[:3]
+        ciphertext = encrypted_value[3:]
+
+        if version == b'v10':
+            self._cookie_counts['v10'] += 1
+            if self._v10_key is None:
+                self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
+                return None
+
+            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
+            #   kNonceLength
+            nonce_length = 96 // 8
+            # boringssl
+            #   EVP_AEAD_AES_GCM_TAG_LEN
+            authentication_tag_length = 16
+
+            raw_ciphertext = ciphertext
+            nonce = raw_ciphertext[:nonce_length]
+            ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length]
+            authentication_tag = raw_ciphertext[-authentication_tag_length:]
+
+            return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger)
+
+        else:
+            self._cookie_counts['other'] += 1
+            # any other prefix means the data is DPAPI encrypted
+            # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
+            return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8')
+
+
+def _extract_safari_cookies(profile, logger):
+    if profile is not None:
+        logger.error('safari does not support profiles')
+    if sys.platform != 'darwin':
+        raise ValueError('unsupported platform: {}'.format(sys.platform))
+
+    cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
+
+    if not os.path.isfile(cookies_path):
+        raise FileNotFoundError('could not find safari cookies database')
+
+    with open(cookies_path, 'rb') as f:
+        cookies_data = f.read()
+
+    jar = parse_safari_cookies(cookies_data, logger=logger)
+    logger.info('Extracted {} cookies from safari'.format(len(jar)))
+    return jar
+
+
+class ParserError(Exception):
+    pass
+
+
+class DataParser:
+    def __init__(self, data, logger):
+        self._data = data
+        self.cursor = 0
+        self._logger = logger
+
+    def read_bytes(self, num_bytes):
+        if num_bytes < 0:
+            raise ParserError('invalid read of {} bytes'.format(num_bytes))
+        end = self.cursor + num_bytes
+        if end > len(self._data):
+            raise ParserError('reached end of input')
+        data = self._data[self.cursor:end]
+        self.cursor = end
+        return data
+
+    def expect_bytes(self, expected_value, message):
+        value = self.read_bytes(len(expected_value))
+        if value != expected_value:
+            raise ParserError('unexpected value: {} != {} ({})'.format(value, expected_value, message))
+
+    def read_uint(self, big_endian=False):
+        data_format = '>I' if big_endian else '<I'
+        return struct.unpack(data_format, self.read_bytes(4))[0]
+
+    def read_double(self, big_endian=False):
+        data_format = '>d' if big_endian else '<d'
+        return struct.unpack(data_format, self.read_bytes(8))[0]
+
+    def read_cstring(self):
+        buffer = []
+        while True:
+            c = self.read_bytes(1)
+            if c == b'\x00':
+                return b''.join(buffer).decode('utf-8')
+            else:
+                buffer.append(c)
+
+    def skip(self, num_bytes, description='unknown'):
+        if num_bytes > 0:
+            self._logger.debug('skipping {} bytes ({}): {}'.format(
+                num_bytes, description, self.read_bytes(num_bytes)))
+        elif num_bytes < 0:
+            raise ParserError('invalid skip of {} bytes'.format(num_bytes))
+
+    def skip_to(self, offset, description='unknown'):
+        self.skip(offset - self.cursor, description)
+
+    def skip_to_end(self, description='unknown'):
+        self.skip_to(len(self._data), description)
+
+
+def _mac_absolute_time_to_posix(timestamp):
+    return int((datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=timestamp)).timestamp())
+
+
+def _parse_safari_cookies_header(data, logger):
+    p = DataParser(data, logger)
+    p.expect_bytes(b'cook', 'database signature')
+    number_of_pages = p.read_uint(big_endian=True)
+    page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)]
+    return page_sizes, p.cursor
+
+
+def _parse_safari_cookies_page(data, jar, logger):
+    p = DataParser(data, logger)
+    p.expect_bytes(b'\x00\x00\x01\x00', 'page signature')
+    number_of_cookies = p.read_uint()
+    record_offsets = [p.read_uint() for _ in range(number_of_cookies)]
+    if number_of_cookies == 0:
+        logger.debug('a cookies page of size {} has no cookies'.format(len(data)))
+        return
+
+    p.skip_to(record_offsets[0], 'unknown page header field')
+
+    for record_offset in record_offsets:
+        p.skip_to(record_offset, 'space between records')
+        record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger)
+        p.read_bytes(record_length)
+    p.skip_to_end('space in between pages')
+
+
+def _parse_safari_cookies_record(data, jar, logger):
+    p = DataParser(data, logger)
+    record_size = p.read_uint()
+    p.skip(4, 'unknown record field 1')
+    flags = p.read_uint()
+    is_secure = bool(flags & 0x0001)
+    p.skip(4, 'unknown record field 2')
+    domain_offset = p.read_uint()
+    name_offset = p.read_uint()
+    path_offset = p.read_uint()
+    value_offset = p.read_uint()
+    p.skip(8, 'unknown record field 3')
+    expiration_date = _mac_absolute_time_to_posix(p.read_double())
+    _creation_date = _mac_absolute_time_to_posix(p.read_double())  # noqa: F841
+
+    try:
+        p.skip_to(domain_offset)
+        domain = p.read_cstring()
+
+        p.skip_to(name_offset)
+        name = p.read_cstring()
+
+        p.skip_to(path_offset)
+        path = p.read_cstring()
+
+        p.skip_to(value_offset)
+        value = p.read_cstring()
+    except UnicodeDecodeError:
+        logger.warning('failed to parse Safari cookie because UTF-8 decoding failed', only_once=True)
+        return record_size
+
+    p.skip_to(record_size, 'space at the end of the record')
+
+    cookie = compat_cookiejar_Cookie(
+        version=0, name=name, value=value, port=None, port_specified=False,
+        domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith('.'),
+        path=path, path_specified=bool(path), secure=is_secure, expires=expiration_date, discard=False,
+        comment=None, comment_url=None, rest={})
+    jar.set_cookie(cookie)
+    return record_size
+
+
+def parse_safari_cookies(data, jar=None, logger=YDLLogger()):
+    """
+    References:
+        - https://github.com/libyal/dtformats/blob/main/documentation/Safari%20Cookies.asciidoc
+            - this data appears to be out of date but the important parts of the database structure is the same
+            - there are a few bytes here and there which are skipped during parsing
+    """
+    if jar is None:
+        jar = YoutubeDLCookieJar()
+    page_sizes, body_start = _parse_safari_cookies_header(data, logger)
+    p = DataParser(data[body_start:], logger)
+    for page_size in page_sizes:
+        _parse_safari_cookies_page(p.read_bytes(page_size), jar, logger)
+    p.skip_to_end('footer')
+    return jar
+
+
+class _LinuxDesktopEnvironment(Enum):
+    """
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.h
+    DesktopEnvironment
+    """
+    OTHER = auto()
+    CINNAMON = auto()
+    GNOME = auto()
+    KDE = auto()
+    PANTHEON = auto()
+    UNITY = auto()
+    XFCE = auto()
+
+
+class _LinuxKeyring(Enum):
+    """
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.h
+    SelectedLinuxBackend
+    """
+    KWALLET = auto()
+    GNOMEKEYRING = auto()
+    BASICTEXT = auto()
+
+
+SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
+
+
+def _get_linux_desktop_environment(env):
+    """
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.cc
+    GetDesktopEnvironment
+    """
+    xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None)
+    desktop_session = env.get('DESKTOP_SESSION', None)
+    if xdg_current_desktop is not None:
+        xdg_current_desktop = xdg_current_desktop.split(':')[0].strip()
+
+        if xdg_current_desktop == 'Unity':
+            if desktop_session is not None and 'gnome-fallback' in desktop_session:
+                return _LinuxDesktopEnvironment.GNOME
+            else:
+                return _LinuxDesktopEnvironment.UNITY
+        elif xdg_current_desktop == 'GNOME':
+            return _LinuxDesktopEnvironment.GNOME
+        elif xdg_current_desktop == 'X-Cinnamon':
+            return _LinuxDesktopEnvironment.CINNAMON
+        elif xdg_current_desktop == 'KDE':
+            return _LinuxDesktopEnvironment.KDE
+        elif xdg_current_desktop == 'Pantheon':
+            return _LinuxDesktopEnvironment.PANTHEON
+        elif xdg_current_desktop == 'XFCE':
+            return _LinuxDesktopEnvironment.XFCE
+    elif desktop_session is not None:
+        if desktop_session in ('mate', 'gnome'):
+            return _LinuxDesktopEnvironment.GNOME
+        elif 'kde' in desktop_session:
+            return _LinuxDesktopEnvironment.KDE
+        elif 'xfce' in desktop_session:
+            return _LinuxDesktopEnvironment.XFCE
+    else:
+        if 'GNOME_DESKTOP_SESSION_ID' in env:
+            return _LinuxDesktopEnvironment.GNOME
+        elif 'KDE_FULL_SESSION' in env:
+            return _LinuxDesktopEnvironment.KDE
+    return _LinuxDesktopEnvironment.OTHER
+
+
+def _choose_linux_keyring(logger):
+    """
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.cc
+    SelectBackend
+    """
+    desktop_environment = _get_linux_desktop_environment(os.environ)
+    logger.debug('detected desktop environment: {}'.format(desktop_environment.name))
+    if desktop_environment == _LinuxDesktopEnvironment.KDE:
+        linux_keyring = _LinuxKeyring.KWALLET
+    elif desktop_environment == _LinuxDesktopEnvironment.OTHER:
+        linux_keyring = _LinuxKeyring.BASICTEXT
+    else:
+        linux_keyring = _LinuxKeyring.GNOMEKEYRING
+    return linux_keyring
+
+
+def _get_kwallet_network_wallet(logger):
+    """ The name of the wallet used to store network passwords.
+
+    https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/kwallet_dbus.cc
+    KWalletDBus::NetworkWallet
+    which does a dbus call to the following function:
+    https://api.kde.org/frameworks/kwallet/html/classKWallet_1_1Wallet.html
+    Wallet::NetworkWallet
+    """
+    default_wallet = 'kdewallet'
+    try:
+        proc = Popen([
+            'dbus-send', '--session', '--print-reply=literal',
+            '--dest=org.kde.kwalletd5',
+            '/modules/kwalletd5',
+            'org.kde.KWallet.networkWallet'
+        ], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+
+        stdout, stderr = proc.communicate_or_kill()
+        if proc.returncode != 0:
+            logger.warning('failed to read NetworkWallet')
+            return default_wallet
+        else:
+            network_wallet = stdout.decode('utf-8').strip()
+            logger.debug('NetworkWallet = "{}"'.format(network_wallet))
+            return network_wallet
+    except BaseException as e:
+        logger.warning('exception while obtaining NetworkWallet: {}'.format(e))
+        return default_wallet
+
+
+def _get_kwallet_password(browser_keyring_name, logger):
+    logger.debug('using kwallet-query to obtain password from kwallet')
+
+    if shutil.which('kwallet-query') is None:
+        logger.error('kwallet-query command not found. KWallet and kwallet-query '
+                     'must be installed to read from KWallet. kwallet-query should be'
+                     'included in the kwallet package for your distribution')
+        return b''
+
+    network_wallet = _get_kwallet_network_wallet(logger)
+
+    try:
+        proc = Popen([
+            'kwallet-query',
+            '--read-password', '{} Safe Storage'.format(browser_keyring_name),
+            '--folder', '{} Keys'.format(browser_keyring_name),
+            network_wallet
+        ], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+
+        stdout, stderr = proc.communicate_or_kill()
+        if proc.returncode != 0:
+            logger.error('kwallet-query failed with return code {}. Please consult '
+                         'the kwallet-query man page for details'.format(proc.returncode))
+            return b''
+        else:
+            if stdout.lower().startswith(b'failed to read'):
+                logger.debug('failed to read password from kwallet. Using empty string instead')
+                # this sometimes occurs in KDE because chrome does not check hasEntry and instead
+                # just tries to read the value (which kwallet returns "") whereas kwallet-query
+                # checks hasEntry. To verify this:
+                # dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
+                # while starting chrome.
+                # this may be a bug as the intended behaviour is to generate a random password and store
+                # it, but that doesn't matter here.
+                return b''
+            else:
+                logger.debug('password found')
+                if stdout[-1:] == b'\n':
+                    stdout = stdout[:-1]
+                return stdout
+    except BaseException as e:
+        logger.warning(f'exception running kwallet-query: {type(e).__name__}({e})')
+        return b''
+
+
+def _get_gnome_keyring_password(browser_keyring_name, logger):
+    if not SECRETSTORAGE_AVAILABLE:
+        logger.error('secretstorage not available {}'.format(SECRETSTORAGE_UNAVAILABLE_REASON))
+        return b''
+    # the Gnome keyring does not seem to organise keys in the same way as KWallet,
+    # using `dbus-monitor` during startup, it can be observed that chromium lists all keys
+    # and presumably searches for its key in the list. It appears that we must do the same.
+    # https://github.com/jaraco/keyring/issues/556
+    with contextlib.closing(secretstorage.dbus_init()) as con:
+        col = secretstorage.get_default_collection(con)
+        for item in col.get_all_items():
+            if item.get_label() == '{} Safe Storage'.format(browser_keyring_name):
+                return item.get_secret()
+        else:
+            logger.error('failed to read from keyring')
+            return b''
+
+
+def _get_linux_keyring_password(browser_keyring_name, keyring, logger):
+    # note: chrome/chromium can be run with the following flags to determine which keyring backend
+    # it has chosen to use
+    # chromium --enable-logging=stderr --v=1 2>&1 | grep key_storage_
+    # Chromium supports a flag: --password-store=<basic|gnome|kwallet> so the automatic detection
+    # will not be sufficient in all cases.
+
+    keyring = _LinuxKeyring[keyring] if keyring else _choose_linux_keyring(logger)
+    logger.debug(f'Chosen keyring: {keyring.name}')
+
+    if keyring == _LinuxKeyring.KWALLET:
+        return _get_kwallet_password(browser_keyring_name, logger)
+    elif keyring == _LinuxKeyring.GNOMEKEYRING:
+        return _get_gnome_keyring_password(browser_keyring_name, logger)
+    elif keyring == _LinuxKeyring.BASICTEXT:
+        # when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
+        return None
+    assert False, f'Unknown keyring {keyring}'
+
+
+def _get_mac_keyring_password(browser_keyring_name, logger):
+    logger.debug('using find-generic-password to obtain password from OSX keychain')
+    try:
+        proc = Popen(
+            ['security', 'find-generic-password',
+             '-w',  # write password to stdout
+             '-a', browser_keyring_name,  # match 'account'
+             '-s', '{} Safe Storage'.format(browser_keyring_name)],  # match 'service'
+            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+
+        stdout, stderr = proc.communicate_or_kill()
+        if stdout[-1:] == b'\n':
+            stdout = stdout[:-1]
+        return stdout
+    except BaseException as e:
+        logger.warning(f'exception running find-generic-password: {type(e).__name__}({e})')
+        return None
+
+
+def _get_windows_v10_key(browser_root, logger):
+    path = _find_most_recently_used_file(browser_root, 'Local State')
+    if path is None:
+        logger.error('could not find local state file')
+        return None
+    with open(path, 'r', encoding='utf8') as f:
+        data = json.load(f)
+    try:
+        base64_key = data['os_crypt']['encrypted_key']
+    except KeyError:
+        logger.error('no encrypted key in Local State')
+        return None
+    encrypted_key = compat_b64decode(base64_key)
+    prefix = b'DPAPI'
+    if not encrypted_key.startswith(prefix):
+        logger.error('invalid key')
+        return None
+    return _decrypt_windows_dpapi(encrypted_key[len(prefix):], logger)
+
+
+def pbkdf2_sha1(password, salt, iterations, key_length):
+    return pbkdf2_hmac('sha1', password, salt, iterations, key_length)
+
+
+def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
+    plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector))
+    try:
+        return plaintext.decode('utf-8')
+    except UnicodeDecodeError:
+        logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
+        return None
+
+
+def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):
+    try:
+        plaintext = aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce)
+    except ValueError:
+        logger.warning('failed to decrypt cookie (AES-GCM) because the MAC check failed. Possibly the key is wrong?', only_once=True)
+        return None
+
+    try:
+        return plaintext.decode('utf-8')
+    except UnicodeDecodeError:
+        logger.warning('failed to decrypt cookie (AES-GCM) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
+        return None
+
+
+def _decrypt_windows_dpapi(ciphertext, logger):
+    """
+    References:
+        - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata
+    """
+    from ctypes.wintypes import DWORD
+
+    class DATA_BLOB(ctypes.Structure):
+        _fields_ = [('cbData', DWORD),
+                    ('pbData', ctypes.POINTER(ctypes.c_char))]
+
+    buffer = ctypes.create_string_buffer(ciphertext)
+    blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer)
+    blob_out = DATA_BLOB()
+    ret = ctypes.windll.crypt32.CryptUnprotectData(
+        ctypes.byref(blob_in),  # pDataIn
+        None,  # ppszDataDescr: human readable description of pDataIn
+        None,  # pOptionalEntropy: salt?
+        None,  # pvReserved: must be NULL
+        None,  # pPromptStruct: information about prompts to display
+        0,  # dwFlags
+        ctypes.byref(blob_out)  # pDataOut
+    )
+    if not ret:
+        logger.warning('failed to decrypt with DPAPI', only_once=True)
+        return None
+
+    result = ctypes.string_at(blob_out.pbData, blob_out.cbData)
+    ctypes.windll.kernel32.LocalFree(blob_out.pbData)
+    return result
+
+
+def _config_home():
+    return os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
+
+
+def _open_database_copy(database_path, tmpdir):
+    # cannot open sqlite databases if they are already in use (e.g. by the browser)
+    database_copy_path = os.path.join(tmpdir, 'temporary.sqlite')
+    shutil.copy(database_path, database_copy_path)
+    conn = sqlite3.connect(database_copy_path)
+    return conn.cursor()
+
+
+def _get_column_names(cursor, table_name):
+    table_info = cursor.execute('PRAGMA table_info({})'.format(table_name)).fetchall()
+    return [row[1].decode('utf-8') for row in table_info]
+
+
+def _find_most_recently_used_file(root, filename):
+    # if there are multiple browser profiles, take the most recently used one
+    paths = []
+    for root, dirs, files in os.walk(root):
+        for file in files:
+            if file == filename:
+                paths.append(os.path.join(root, file))
+    return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
+
+
+def _merge_cookie_jars(jars):
+    output_jar = YoutubeDLCookieJar()
+    for jar in jars:
+        for cookie in jar:
+            output_jar.set_cookie(cookie)
+        if jar.filename is not None:
+            output_jar.filename = jar.filename
+    return output_jar
+
+
+def _is_path(value):
+    return os.path.sep in value
+
+
+def _parse_browser_specification(browser_name, profile=None, keyring=None):
+    if browser_name not in SUPPORTED_BROWSERS:
+        raise ValueError(f'unsupported browser: "{browser_name}"')
+    if keyring not in (None, *SUPPORTED_KEYRINGS):
+        raise ValueError(f'unsupported keyring: "{keyring}"')
+    if profile is not None and _is_path(profile):
+        profile = os.path.expanduser(profile)
+    return browser_name, profile, keyring

plugins/youtube_download/yt_dlp/downloader/__init__.py

@@ -0,0 +1,135 @@
+from __future__ import unicode_literals
+
+from ..compat import compat_str
+from ..utils import (
+    determine_protocol,
+    NO_DEFAULT
+)
+
+
+def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=None, to_stdout=False):
+    info_dict['protocol'] = determine_protocol(info_dict)
+    info_copy = info_dict.copy()
+    info_copy['to_stdout'] = to_stdout
+
+    protocols = (protocol or info_copy['protocol']).split('+')
+    downloaders = [_get_suitable_downloader(info_copy, proto, params, default) for proto in protocols]
+
+    if set(downloaders) == {FFmpegFD} and FFmpegFD.can_merge_formats(info_copy, params):
+        return FFmpegFD
+    elif (set(downloaders) == {DashSegmentsFD}
+          and not (to_stdout and len(protocols) > 1)
+          and set(protocols) == {'http_dash_segments_generator'}):
+        return DashSegmentsFD
+    elif len(downloaders) == 1:
+        return downloaders[0]
+    return None
+
+
+# Some of these require get_suitable_downloader
+from .common import FileDownloader
+from .dash import DashSegmentsFD
+from .f4m import F4mFD
+from .hls import HlsFD
+from .http import HttpFD
+from .rtmp import RtmpFD
+from .rtsp import RtspFD
+from .ism import IsmFD
+from .mhtml import MhtmlFD
+from .niconico import NiconicoDmcFD
+from .websocket import WebSocketFragmentFD
+from .youtube_live_chat import YoutubeLiveChatFD
+from .external import (
+    get_external_downloader,
+    FFmpegFD,
+)
+
+PROTOCOL_MAP = {
+    'rtmp': RtmpFD,
+    'rtmpe': RtmpFD,
+    'rtmp_ffmpeg': FFmpegFD,
+    'm3u8_native': HlsFD,
+    'm3u8': FFmpegFD,
+    'mms': RtspFD,
+    'rtsp': RtspFD,
+    'f4m': F4mFD,
+    'http_dash_segments': DashSegmentsFD,
+    'http_dash_segments_generator': DashSegmentsFD,
+    'ism': IsmFD,
+    'mhtml': MhtmlFD,
+    'niconico_dmc': NiconicoDmcFD,
+    'websocket_frag': WebSocketFragmentFD,
+    'youtube_live_chat': YoutubeLiveChatFD,
+    'youtube_live_chat_replay': YoutubeLiveChatFD,
+}
+
+
+def shorten_protocol_name(proto, simplify=False):
+    short_protocol_names = {
+        'm3u8_native': 'm3u8_n',
+        'rtmp_ffmpeg': 'rtmp_f',
+        'http_dash_segments': 'dash',
+        'http_dash_segments_generator': 'dash_g',
+        'niconico_dmc': 'dmc',
+        'websocket_frag': 'WSfrag',
+    }
+    if simplify:
+        short_protocol_names.update({
+            'https': 'http',
+            'ftps': 'ftp',
+            'm3u8_native': 'm3u8',
+            'http_dash_segments_generator': 'dash',
+            'rtmp_ffmpeg': 'rtmp',
+            'm3u8_frag_urls': 'm3u8',
+            'dash_frag_urls': 'dash',
+        })
+    return short_protocol_names.get(proto, proto)
+
+
+def _get_suitable_downloader(info_dict, protocol, params, default):
+    """Get the downloader class that can handle the info dict."""
+    if default is NO_DEFAULT:
+        default = HttpFD
+
+    # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
+    #     return FFmpegFD
+
+    info_dict['protocol'] = protocol
+    downloaders = params.get('external_downloader')
+    external_downloader = (
+        downloaders if isinstance(downloaders, compat_str) or downloaders is None
+        else downloaders.get(shorten_protocol_name(protocol, True), downloaders.get('default')))
+
+    if external_downloader is None:
+        if info_dict['to_stdout'] and FFmpegFD.can_merge_formats(info_dict, params):
+            return FFmpegFD
+    elif external_downloader.lower() != 'native':
+        ed = get_external_downloader(external_downloader)
+        if ed.can_download(info_dict, external_downloader):
+            return ed
+
+    if protocol == 'http_dash_segments':
+        if info_dict.get('is_live') and (external_downloader or '').lower() != 'native':
+            return FFmpegFD
+
+    if protocol in ('m3u8', 'm3u8_native'):
+        if info_dict.get('is_live'):
+            return FFmpegFD
+        elif (external_downloader or '').lower() == 'native':
+            return HlsFD
+        elif get_suitable_downloader(
+                info_dict, params, None, protocol='m3u8_frag_urls', to_stdout=info_dict['to_stdout']):
+            return HlsFD
+        elif params.get('hls_prefer_native') is True:
+            return HlsFD
+        elif params.get('hls_prefer_native') is False:
+            return FFmpegFD
+
+    return PROTOCOL_MAP.get(protocol, default)
+
+
+__all__ = [
+    'FileDownloader',
+    'get_suitable_downloader',
+    'shorten_protocol_name',
+]

plugins/youtube_download/yt_dlp/downloader/common.py

@@ -0,0 +1,472 @@
+from __future__ import division, unicode_literals
+
+import os
+import re
+import time
+import random
+import errno
+
+from ..utils import (
+    decodeArgument,
+    encodeFilename,
+    error_to_compat_str,
+    format_bytes,
+    sanitize_open,
+    shell_quote,
+    timeconvert,
+    timetuple_from_msec,
+)
+from ..minicurses import (
+    MultilineLogger,
+    MultilinePrinter,
+    QuietMultilinePrinter,
+    BreaklineStatusPrinter
+)
+
+
+class FileDownloader(object):
+    """File Downloader class.
+
+    File downloader objects are the ones responsible of downloading the
+    actual video file and writing it to disk.
+
+    File downloaders accept a lot of parameters. In order not to saturate
+    the object constructor with arguments, it receives a dictionary of
+    options instead.
+
+    Available options:
+
+    verbose:            Print additional info to stdout.
+    quiet:              Do not print messages to stdout.
+    ratelimit:          Download speed limit, in bytes/sec.
+    throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
+    retries:            Number of times to retry for HTTP error 5xx
+    file_access_retries:   Number of times to retry on file access error
+    buffersize:         Size of download buffer in bytes.
+    noresizebuffer:     Do not automatically resize the download buffer.
+    continuedl:         Try to continue downloads if possible.
+    noprogress:         Do not print the progress bar.
+    nopart:             Do not use temporary .part files.
+    updatetime:         Use the Last-modified header to set output file timestamps.
+    test:               Download only first bytes to test the downloader.
+    min_filesize:       Skip files smaller than this size
+    max_filesize:       Skip files larger than this size
+    xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
+    external_downloader_args:  A dictionary of downloader keys (in lower case)
+                        and a list of additional command-line arguments for the
+                        executable. Use 'default' as the name for arguments to be
+                        passed to all downloaders. For compatibility with youtube-dl,
+                        a single list of args can also be used
+    hls_use_mpegts:     Use the mpegts container for HLS videos.
+    http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
+                        useful for bypassing bandwidth throttling imposed by
+                        a webserver (experimental)
+    progress_template:  See YoutubeDL.py
+
+    Subclasses of this one must re-define the real_download method.
+    """
+
+    _TEST_FILE_SIZE = 10241
+    params = None
+
+    def __init__(self, ydl, params):
+        """Create a FileDownloader object with the given options."""
+        self.ydl = ydl
+        self._progress_hooks = []
+        self.params = params
+        self._prepare_multiline_status()
+        self.add_progress_hook(self.report_progress)
+
+    @staticmethod
+    def format_seconds(seconds):
+        time = timetuple_from_msec(seconds * 1000)
+        if time.hours > 99:
+            return '--:--:--'
+        if not time.hours:
+            return '%02d:%02d' % time[1:-1]
+        return '%02d:%02d:%02d' % time[:-1]
+
+    @staticmethod
+    def calc_percent(byte_counter, data_len):
+        if data_len is None:
+            return None
+        return float(byte_counter) / float(data_len) * 100.0
+
+    @staticmethod
+    def format_percent(percent):
+        if percent is None:
+            return '---.-%'
+        elif percent == 100:
+            return '100%'
+        return '%6s' % ('%3.1f%%' % percent)
+
+    @staticmethod
+    def calc_eta(start, now, total, current):
+        if total is None:
+            return None
+        if now is None:
+            now = time.time()
+        dif = now - start
+        if current == 0 or dif < 0.001:  # One millisecond
+            return None
+        rate = float(current) / dif
+        return int((float(total) - float(current)) / rate)
+
+    @staticmethod
+    def format_eta(eta):
+        if eta is None:
+            return '--:--'
+        return FileDownloader.format_seconds(eta)
+
+    @staticmethod
+    def calc_speed(start, now, bytes):
+        dif = now - start
+        if bytes == 0 or dif < 0.001:  # One millisecond
+            return None
+        return float(bytes) / dif
+
+    @staticmethod
+    def format_speed(speed):
+        if speed is None:
+            return '%10s' % '---b/s'
+        return '%10s' % ('%s/s' % format_bytes(speed))
+
+    @staticmethod
+    def format_retries(retries):
+        return 'inf' if retries == float('inf') else '%.0f' % retries
+
+    @staticmethod
+    def best_block_size(elapsed_time, bytes):
+        new_min = max(bytes / 2.0, 1.0)
+        new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
+        if elapsed_time < 0.001:
+            return int(new_max)
+        rate = bytes / elapsed_time
+        if rate > new_max:
+            return int(new_max)
+        if rate < new_min:
+            return int(new_min)
+        return int(rate)
+
+    @staticmethod
+    def parse_bytes(bytestr):
+        """Parse a string indicating a byte quantity into an integer."""
+        matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
+        if matchobj is None:
+            return None
+        number = float(matchobj.group(1))
+        multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
+        return int(round(number * multiplier))
+
+    def to_screen(self, *args, **kargs):
+        self.ydl.to_stdout(*args, quiet=self.params.get('quiet'), **kargs)
+
+    def to_stderr(self, message):
+        self.ydl.to_stderr(message)
+
+    def to_console_title(self, message):
+        self.ydl.to_console_title(message)
+
+    def trouble(self, *args, **kargs):
+        self.ydl.trouble(*args, **kargs)
+
+    def report_warning(self, *args, **kargs):
+        self.ydl.report_warning(*args, **kargs)
+
+    def report_error(self, *args, **kargs):
+        self.ydl.report_error(*args, **kargs)
+
+    def write_debug(self, *args, **kargs):
+        self.ydl.write_debug(*args, **kargs)
+
+    def slow_down(self, start_time, now, byte_counter):
+        """Sleep if the download speed is over the rate limit."""
+        rate_limit = self.params.get('ratelimit')
+        if rate_limit is None or byte_counter == 0:
+            return
+        if now is None:
+            now = time.time()
+        elapsed = now - start_time
+        if elapsed <= 0.0:
+            return
+        speed = float(byte_counter) / elapsed
+        if speed > rate_limit:
+            sleep_time = float(byte_counter) / rate_limit - elapsed
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+
+    def temp_name(self, filename):
+        """Returns a temporary filename for the given filename."""
+        if self.params.get('nopart', False) or filename == '-' or \
+                (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
+            return filename
+        return filename + '.part'
+
+    def undo_temp_name(self, filename):
+        if filename.endswith('.part'):
+            return filename[:-len('.part')]
+        return filename
+
+    def ytdl_filename(self, filename):
+        return filename + '.ytdl'
+
+    def sanitize_open(self, filename, open_mode):
+        file_access_retries = self.params.get('file_access_retries', 10)
+        retry = 0
+        while True:
+            try:
+                return sanitize_open(filename, open_mode)
+            except (IOError, OSError) as err:
+                retry = retry + 1
+                if retry > file_access_retries or err.errno not in (errno.EACCES,):
+                    raise
+                self.to_screen(
+                    '[download] Got file access error. Retrying (attempt %d of %s) ...'
+                    % (retry, self.format_retries(file_access_retries)))
+                time.sleep(0.01)
+
+    def try_rename(self, old_filename, new_filename):
+        if old_filename == new_filename:
+            return
+        try:
+            os.replace(old_filename, new_filename)
+        except (IOError, OSError) as err:
+            self.report_error(f'unable to rename file: {err}')
+
+    def try_utime(self, filename, last_modified_hdr):
+        """Try to set the last-modified time of the given file."""
+        if last_modified_hdr is None:
+            return
+        if not os.path.isfile(encodeFilename(filename)):
+            return
+        timestr = last_modified_hdr
+        if timestr is None:
+            return
+        filetime = timeconvert(timestr)
+        if filetime is None:
+            return filetime
+        # Ignore obviously invalid dates
+        if filetime == 0:
+            return
+        try:
+            os.utime(filename, (time.time(), filetime))
+        except Exception:
+            pass
+        return filetime
+
+    def report_destination(self, filename):
+        """Report destination filename."""
+        self.to_screen('[download] Destination: ' + filename)
+
+    def _prepare_multiline_status(self, lines=1):
+        if self.params.get('noprogress'):
+            self._multiline = QuietMultilinePrinter()
+        elif self.ydl.params.get('logger'):
+            self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
+        elif self.params.get('progress_with_newline'):
+            self._multiline = BreaklineStatusPrinter(self.ydl._screen_file, lines)
+        else:
+            self._multiline = MultilinePrinter(self.ydl._screen_file, lines, not self.params.get('quiet'))
+        self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self.params.get('no_color')
+
+    def _finish_multiline_status(self):
+        self._multiline.end()
+
+    _progress_styles = {
+        'downloaded_bytes': 'light blue',
+        'percent': 'light blue',
+        'eta': 'yellow',
+        'speed': 'green',
+        'elapsed': 'bold white',
+        'total_bytes': '',
+        'total_bytes_estimate': '',
+    }
+
+    def _report_progress_status(self, s, default_template):
+        for name, style in self._progress_styles.items():
+            name = f'_{name}_str'
+            if name not in s:
+                continue
+            s[name] = self._format_progress(s[name], style)
+        s['_default_template'] = default_template % s
+
+        progress_dict = s.copy()
+        progress_dict.pop('info_dict')
+        progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
+
+        progress_template = self.params.get('progress_template', {})
+        self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
+            progress_template.get('download') or '[download] %(progress._default_template)s',
+            progress_dict), s.get('progress_idx') or 0)
+        self.to_console_title(self.ydl.evaluate_outtmpl(
+            progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
+            progress_dict))
+
+    def _format_progress(self, *args, **kwargs):
+        return self.ydl._format_text(
+            self._multiline.stream, self._multiline.allow_colors, *args, **kwargs)
+
+    def report_progress(self, s):
+        if s['status'] == 'finished':
+            if self.params.get('noprogress'):
+                self.to_screen('[download] Download completed')
+            msg_template = '100%%'
+            if s.get('total_bytes') is not None:
+                s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+                msg_template += ' of %(_total_bytes_str)s'
+            if s.get('elapsed') is not None:
+                s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                msg_template += ' in %(_elapsed_str)s'
+            s['_percent_str'] = self.format_percent(100)
+            self._report_progress_status(s, msg_template)
+            return
+
+        if s['status'] != 'downloading':
+            return
+
+        if s.get('eta') is not None:
+            s['_eta_str'] = self.format_eta(s['eta'])
+        else:
+            s['_eta_str'] = 'Unknown'
+
+        if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
+            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
+        elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
+            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
+        else:
+            if s.get('downloaded_bytes') == 0:
+                s['_percent_str'] = self.format_percent(0)
+            else:
+                s['_percent_str'] = 'Unknown %'
+
+        if s.get('speed') is not None:
+            s['_speed_str'] = self.format_speed(s['speed'])
+        else:
+            s['_speed_str'] = 'Unknown speed'
+
+        if s.get('total_bytes') is not None:
+            s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+            msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
+        elif s.get('total_bytes_estimate') is not None:
+            s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
+            msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
+        else:
+            if s.get('downloaded_bytes') is not None:
+                s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
+                if s.get('elapsed'):
+                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
+                else:
+                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
+            else:
+                msg_template = '%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s'
+        if s.get('fragment_index') and s.get('fragment_count'):
+            msg_template += ' (frag %(fragment_index)s/%(fragment_count)s)'
+        elif s.get('fragment_index'):
+            msg_template += ' (frag %(fragment_index)s)'
+        self._report_progress_status(s, msg_template)
+
+    def report_resuming_byte(self, resume_len):
+        """Report attempt to resume at given byte."""
+        self.to_screen('[download] Resuming download at byte %s' % resume_len)
+
+    def report_retry(self, err, count, retries):
+        """Report retry in case of HTTP error 5xx"""
+        self.to_screen(
+            '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
+            % (error_to_compat_str(err), count, self.format_retries(retries)))
+
+    def report_file_already_downloaded(self, *args, **kwargs):
+        """Report file has already been fully downloaded."""
+        return self.ydl.report_file_already_downloaded(*args, **kwargs)
+
+    def report_unable_to_resume(self):
+        """Report it was impossible to resume download."""
+        self.to_screen('[download] Unable to resume')
+
+    @staticmethod
+    def supports_manifest(manifest):
+        """ Whether the downloader can download the fragments from the manifest.
+        Redefine in subclasses if needed. """
+        pass
+
+    def download(self, filename, info_dict, subtitle=False):
+        """Download to a filename using the info from info_dict
+        Return True on success and False otherwise
+        """
+
+        nooverwrites_and_exists = (
+            not self.params.get('overwrites', True)
+            and os.path.exists(encodeFilename(filename))
+        )
+
+        if not hasattr(filename, 'write'):
+            continuedl_and_exists = (
+                self.params.get('continuedl', True)
+                and os.path.isfile(encodeFilename(filename))
+                and not self.params.get('nopart', False)
+            )
+
+            # Check file already present
+            if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
+                self.report_file_already_downloaded(filename)
+                self._hook_progress({
+                    'filename': filename,
+                    'status': 'finished',
+                    'total_bytes': os.path.getsize(encodeFilename(filename)),
+                }, info_dict)
+                self._finish_multiline_status()
+                return True, False
+
+        if subtitle is False:
+            min_sleep_interval = self.params.get('sleep_interval')
+            if min_sleep_interval:
+                max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
+                sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
+                self.to_screen(
+                    '[download] Sleeping %s seconds ...' % (
+                        int(sleep_interval) if sleep_interval.is_integer()
+                        else '%.2f' % sleep_interval))
+                time.sleep(sleep_interval)
+        else:
+            sleep_interval_sub = 0
+            if type(self.params.get('sleep_interval_subtitles')) is int:
+                sleep_interval_sub = self.params.get('sleep_interval_subtitles')
+            if sleep_interval_sub > 0:
+                self.to_screen(
+                    '[download] Sleeping %s seconds ...' % (
+                        sleep_interval_sub))
+                time.sleep(sleep_interval_sub)
+        ret = self.real_download(filename, info_dict)
+        self._finish_multiline_status()
+        return ret, True
+
+    def real_download(self, filename, info_dict):
+        """Real download process. Redefine in subclasses."""
+        raise NotImplementedError('This method must be implemented by subclasses')
+
+    def _hook_progress(self, status, info_dict):
+        if not self._progress_hooks:
+            return
+        status['info_dict'] = info_dict
+        # youtube-dl passes the same status object to all the hooks.
+        # Some third party scripts seems to be relying on this.
+        # So keep this behavior if possible
+        for ph in self._progress_hooks:
+            ph(status)
+
+    def add_progress_hook(self, ph):
+        # See YoutubeDl.py (search for progress_hooks) for a description of
+        # this interface
+        self._progress_hooks.append(ph)
+
+    def _debug_cmd(self, args, exe=None):
+        if not self.params.get('verbose', False):
+            return
+
+        str_args = [decodeArgument(a) for a in args]
+
+        if exe is None:
+            exe = os.path.basename(str_args[0])
+
+        self.write_debug('%s command line: %s' % (exe, shell_quote(str_args)))

plugins/youtube_download/yt_dlp/downloader/dash.py

@@ -0,0 +1,80 @@
+from __future__ import unicode_literals
+import time
+
+from ..downloader import get_suitable_downloader
+from .fragment import FragmentFD
+
+from ..utils import urljoin
+
+
+class DashSegmentsFD(FragmentFD):
+    """
+    Download segments in a DASH manifest. External downloaders can take over
+    the fragment downloads by supporting the 'dash_frag_urls' protocol
+    """
+
+    FD_NAME = 'dashsegments'
+
+    def real_download(self, filename, info_dict):
+        if info_dict.get('is_live') and set(info_dict['protocol'].split('+')) != {'http_dash_segments_generator'}:
+            self.report_error('Live DASH videos are not supported')
+
+        real_start = time.time()
+        real_downloader = get_suitable_downloader(
+            info_dict, self.params, None, protocol='dash_frag_urls', to_stdout=(filename == '-'))
+
+        requested_formats = [{**info_dict, **fmt} for fmt in info_dict.get('requested_formats', [])]
+        args = []
+        for fmt in requested_formats or [info_dict]:
+            try:
+                fragment_count = 1 if self.params.get('test') else len(fmt['fragments'])
+            except TypeError:
+                fragment_count = None
+            ctx = {
+                'filename': fmt.get('filepath') or filename,
+                'live': 'is_from_start' if fmt.get('is_from_start') else fmt.get('is_live'),
+                'total_frags': fragment_count,
+            }
+
+            if real_downloader:
+                self._prepare_external_frag_download(ctx)
+            else:
+                self._prepare_and_start_frag_download(ctx, fmt)
+            ctx['start'] = real_start
+
+            fragments_to_download = self._get_fragments(fmt, ctx)
+
+            if real_downloader:
+                self.to_screen(
+                    '[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
+                info_dict['fragments'] = list(fragments_to_download)
+                fd = real_downloader(self.ydl, self.params)
+                return fd.real_download(filename, info_dict)
+
+            args.append([ctx, fragments_to_download, fmt])
+
+        return self.download_and_append_fragments_multiple(*args)
+
+    def _resolve_fragments(self, fragments, ctx):
+        fragments = fragments(ctx) if callable(fragments) else fragments
+        return [next(iter(fragments))] if self.params.get('test') else fragments
+
+    def _get_fragments(self, fmt, ctx):
+        fragment_base_url = fmt.get('fragment_base_url')
+        fragments = self._resolve_fragments(fmt['fragments'], ctx)
+
+        frag_index = 0
+        for i, fragment in enumerate(fragments):
+            frag_index += 1
+            if frag_index <= ctx['fragment_index']:
+                continue
+            fragment_url = fragment.get('url')
+            if not fragment_url:
+                assert fragment_base_url
+                fragment_url = urljoin(fragment_base_url, fragment['path'])
+
+            yield {
+                'frag_index': frag_index,
+                'index': i,
+                'url': fragment_url,
+            }

plugins/youtube_download/yt_dlp/downloader/external.py

@@ -0,0 +1,526 @@
+from __future__ import unicode_literals
+
+import os.path
+import re
+import subprocess
+import sys
+import time
+
+from .fragment import FragmentFD
+from ..compat import (
+    compat_setenv,
+    compat_str,
+)
+from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
+from ..utils import (
+    cli_option,
+    cli_valueless_option,
+    cli_bool_option,
+    _configuration_args,
+    determine_ext,
+    encodeFilename,
+    encodeArgument,
+    handle_youtubedl_headers,
+    check_executable,
+    Popen,
+    remove_end,
+)
+
+
+class ExternalFD(FragmentFD):
+    SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps')
+    can_download_to_stdout = False
+
+    def real_download(self, filename, info_dict):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        try:
+            started = time.time()
+            retval = self._call_downloader(tmpfilename, info_dict)
+        except KeyboardInterrupt:
+            if not info_dict.get('is_live'):
+                raise
+            # Live stream downloading cancellation should be considered as
+            # correct and expected termination thus all postprocessing
+            # should take place
+            retval = 0
+            self.to_screen('[%s] Interrupted by user' % self.get_basename())
+
+        if retval == 0:
+            status = {
+                'filename': filename,
+                'status': 'finished',
+                'elapsed': time.time() - started,
+            }
+            if filename != '-':
+                fsize = os.path.getsize(encodeFilename(tmpfilename))
+                self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
+                self.try_rename(tmpfilename, filename)
+                status.update({
+                    'downloaded_bytes': fsize,
+                    'total_bytes': fsize,
+                })
+            self._hook_progress(status, info_dict)
+            return True
+        else:
+            self.to_stderr('\n')
+            self.report_error('%s exited with code %d' % (
+                self.get_basename(), retval))
+            return False
+
+    @classmethod
+    def get_basename(cls):
+        return cls.__name__[:-2].lower()
+
+    @property
+    def exe(self):
+        return self.get_basename()
+
+    @classmethod
+    def available(cls, path=None):
+        path = check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT])
+        if path:
+            cls.exe = path
+            return path
+        return False
+
+    @classmethod
+    def supports(cls, info_dict):
+        return (
+            (cls.can_download_to_stdout or not info_dict.get('to_stdout'))
+            and info_dict['protocol'] in cls.SUPPORTED_PROTOCOLS)
+
+    @classmethod
+    def can_download(cls, info_dict, path=None):
+        return cls.available(path) and cls.supports(info_dict)
+
+    def _option(self, command_option, param):
+        return cli_option(self.params, command_option, param)
+
+    def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
+        return cli_bool_option(self.params, command_option, param, true_value, false_value, separator)
+
+    def _valueless_option(self, command_option, param, expected_value=True):
+        return cli_valueless_option(self.params, command_option, param, expected_value)
+
+    def _configuration_args(self, keys=None, *args, **kwargs):
+        return _configuration_args(
+            self.get_basename(), self.params.get('external_downloader_args'), self.get_basename(),
+            keys, *args, **kwargs)
+
+    def _call_downloader(self, tmpfilename, info_dict):
+        """ Either overwrite this or implement _make_cmd """
+        cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
+
+        self._debug_cmd(cmd)
+
+        if 'fragments' not in info_dict:
+            p = Popen(cmd, stderr=subprocess.PIPE)
+            _, stderr = p.communicate_or_kill()
+            if p.returncode != 0:
+                self.to_stderr(stderr.decode('utf-8', 'replace'))
+            return p.returncode
+
+        fragment_retries = self.params.get('fragment_retries', 0)
+        skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
+
+        count = 0
+        while count <= fragment_retries:
+            p = Popen(cmd, stderr=subprocess.PIPE)
+            _, stderr = p.communicate_or_kill()
+            if p.returncode == 0:
+                break
+            # TODO: Decide whether to retry based on error code
+            # https://aria2.github.io/manual/en/html/aria2c.html#exit-status
+            self.to_stderr(stderr.decode('utf-8', 'replace'))
+            count += 1
+            if count <= fragment_retries:
+                self.to_screen(
+                    '[%s] Got error. Retrying fragments (attempt %d of %s)...'
+                    % (self.get_basename(), count, self.format_retries(fragment_retries)))
+        if count > fragment_retries:
+            if not skip_unavailable_fragments:
+                self.report_error('Giving up after %s fragment retries' % fragment_retries)
+                return -1
+
+        decrypt_fragment = self.decrypter(info_dict)
+        dest, _ = self.sanitize_open(tmpfilename, 'wb')
+        for frag_index, fragment in enumerate(info_dict['fragments']):
+            fragment_filename = '%s-Frag%d' % (tmpfilename, frag_index)
+            try:
+                src, _ = self.sanitize_open(fragment_filename, 'rb')
+            except IOError as err:
+                if skip_unavailable_fragments and frag_index > 1:
+                    self.report_skip_fragment(frag_index, err)
+                    continue
+                self.report_error(f'Unable to open fragment {frag_index}; {err}')
+                return -1
+            dest.write(decrypt_fragment(fragment, src.read()))
+            src.close()
+            if not self.params.get('keep_fragments', False):
+                os.remove(encodeFilename(fragment_filename))
+        dest.close()
+        os.remove(encodeFilename('%s.frag.urls' % tmpfilename))
+        return 0
+
+
+class CurlFD(ExternalFD):
+    AVAILABLE_OPT = '-V'
+
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '--location', '-o', tmpfilename]
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['--header', '%s: %s' % (key, val)]
+
+        cmd += self._bool_option('--continue-at', 'continuedl', '-', '0')
+        cmd += self._valueless_option('--silent', 'noprogress')
+        cmd += self._valueless_option('--verbose', 'verbose')
+        cmd += self._option('--limit-rate', 'ratelimit')
+        retry = self._option('--retry', 'retries')
+        if len(retry) == 2:
+            if retry[1] in ('inf', 'infinite'):
+                retry[1] = '2147483647'
+            cmd += retry
+        cmd += self._option('--max-filesize', 'max_filesize')
+        cmd += self._option('--interface', 'source_address')
+        cmd += self._option('--proxy', 'proxy')
+        cmd += self._valueless_option('--insecure', 'nocheckcertificate')
+        cmd += self._configuration_args()
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+    def _call_downloader(self, tmpfilename, info_dict):
+        cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
+
+        self._debug_cmd(cmd)
+
+        # curl writes the progress to stderr so don't capture it.
+        p = Popen(cmd)
+        p.communicate_or_kill()
+        return p.returncode
+
+
+class AxelFD(ExternalFD):
+    AVAILABLE_OPT = '-V'
+
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-o', tmpfilename]
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['-H', '%s: %s' % (key, val)]
+        cmd += self._configuration_args()
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
+class WgetFD(ExternalFD):
+    AVAILABLE_OPT = '--version'
+
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._option('--limit-rate', 'ratelimit')
+        retry = self._option('--tries', 'retries')
+        if len(retry) == 2:
+            if retry[1] in ('inf', 'infinite'):
+                retry[1] = '0'
+            cmd += retry
+        cmd += self._option('--bind-address', 'source_address')
+        cmd += self._option('--proxy', 'proxy')
+        cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
+        cmd += self._configuration_args()
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
+class Aria2cFD(ExternalFD):
+    AVAILABLE_OPT = '-v'
+    SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'dash_frag_urls', 'm3u8_frag_urls')
+
+    @staticmethod
+    def supports_manifest(manifest):
+        UNSUPPORTED_FEATURES = [
+            r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [1]
+            # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
+        ]
+        check_results = (not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES)
+        return all(check_results)
+
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-c',
+               '--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
+               '--file-allocation=none', '-x16', '-j16', '-s16']
+        if 'fragments' in info_dict:
+            cmd += ['--allow-overwrite=true', '--allow-piece-length-change=true']
+        else:
+            cmd += ['--min-split-size', '1M']
+
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._option('--max-overall-download-limit', 'ratelimit')
+        cmd += self._option('--interface', 'source_address')
+        cmd += self._option('--all-proxy', 'proxy')
+        cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
+        cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=')
+        cmd += self._bool_option('--show-console-readout', 'noprogress', 'false', 'true', '=')
+        cmd += self._configuration_args()
+
+        # aria2c strips out spaces from the beginning/end of filenames and paths.
+        # We work around this issue by adding a "./" to the beginning of the
+        # filename and relative path, and adding a "/" at the end of the path.
+        # See: https://github.com/yt-dlp/yt-dlp/issues/276
+        # https://github.com/ytdl-org/youtube-dl/issues/20312
+        # https://github.com/aria2/aria2/issues/1373
+        dn = os.path.dirname(tmpfilename)
+        if dn:
+            if not os.path.isabs(dn):
+                dn = '.%s%s' % (os.path.sep, dn)
+            cmd += ['--dir', dn + os.path.sep]
+        if 'fragments' not in info_dict:
+            cmd += ['--out', '.%s%s' % (os.path.sep, os.path.basename(tmpfilename))]
+        cmd += ['--auto-file-renaming=false']
+
+        if 'fragments' in info_dict:
+            cmd += ['--file-allocation=none', '--uri-selector=inorder']
+            url_list_file = '%s.frag.urls' % tmpfilename
+            url_list = []
+            for frag_index, fragment in enumerate(info_dict['fragments']):
+                fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index)
+                url_list.append('%s\n\tout=%s' % (fragment['url'], fragment_filename))
+            stream, _ = self.sanitize_open(url_list_file, 'wb')
+            stream.write('\n'.join(url_list).encode('utf-8'))
+            stream.close()
+            cmd += ['-i', url_list_file]
+        else:
+            cmd += ['--', info_dict['url']]
+        return cmd
+
+
+class HttpieFD(ExternalFD):
+    AVAILABLE_OPT = '--version'
+
+    @classmethod
+    def available(cls, path=None):
+        return super().available(path or 'http')
+
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
+
+        if info_dict.get('http_headers') is not None:
+            for key, val in info_dict['http_headers'].items():
+                cmd += ['%s:%s' % (key, val)]
+        return cmd
+
+
+class FFmpegFD(ExternalFD):
+    SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'm3u8_native', 'rtsp', 'rtmp', 'rtmp_ffmpeg', 'mms', 'http_dash_segments')
+    can_download_to_stdout = True
+
+    @classmethod
+    def available(cls, path=None):
+        # TODO: Fix path for ffmpeg
+        # Fixme: This may be wrong when --ffmpeg-location is used
+        return FFmpegPostProcessor().available
+
+    @classmethod
+    def supports(cls, info_dict):
+        return all(proto in cls.SUPPORTED_PROTOCOLS for proto in info_dict['protocol'].split('+'))
+
+    def on_process_started(self, proc, stdin):
+        """ Override this in subclasses  """
+        pass
+
+    @classmethod
+    def can_merge_formats(cls, info_dict, params):
+        return (
+            info_dict.get('requested_formats')
+            and info_dict.get('protocol')
+            and not params.get('allow_unplayable_formats')
+            and 'no-direct-merge' not in params.get('compat_opts', [])
+            and cls.can_download(info_dict))
+
+    def _call_downloader(self, tmpfilename, info_dict):
+        urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']]
+        ffpp = FFmpegPostProcessor(downloader=self)
+        if not ffpp.available:
+            self.report_error('m3u8 download detected but ffmpeg could not be found. Please install')
+            return False
+        ffpp.check_version()
+
+        args = [ffpp.executable, '-y']
+
+        for log_level in ('quiet', 'verbose'):
+            if self.params.get(log_level, False):
+                args += ['-loglevel', log_level]
+                break
+        if not self.params.get('verbose'):
+            args += ['-hide_banner']
+
+        args += info_dict.get('_ffmpeg_args', [])
+
+        # This option exists only for compatibility. Extractors should use `_ffmpeg_args` instead
+        seekable = info_dict.get('_seekable')
+        if seekable is not None:
+            # setting -seekable prevents ffmpeg from guessing if the server
+            # supports seeking(by adding the header `Range: bytes=0-`), which
+            # can cause problems in some cases
+            # https://github.com/ytdl-org/youtube-dl/issues/11800#issuecomment-275037127
+            # http://trac.ffmpeg.org/ticket/6125#comment:10
+            args += ['-seekable', '1' if seekable else '0']
+
+        # start_time = info_dict.get('start_time') or 0
+        # if start_time:
+        #     args += ['-ss', compat_str(start_time)]
+        # end_time = info_dict.get('end_time')
+        # if end_time:
+        #     args += ['-t', compat_str(end_time - start_time)]
+
+        if info_dict.get('http_headers') is not None and re.match(r'^https?://', urls[0]):
+            # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
+            # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
+            headers = handle_youtubedl_headers(info_dict['http_headers'])
+            args += [
+                '-headers',
+                ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
+
+        env = None
+        proxy = self.params.get('proxy')
+        if proxy:
+            if not re.match(r'^[\da-zA-Z]+://', proxy):
+                proxy = 'http://%s' % proxy
+
+            if proxy.startswith('socks'):
+                self.report_warning(
+                    '%s does not support SOCKS proxies. Downloading is likely to fail. '
+                    'Consider adding --hls-prefer-native to your command.' % self.get_basename())
+
+            # Since December 2015 ffmpeg supports -http_proxy option (see
+            # http://git.videolan.org/?p=ffmpeg.git;a=commit;h=b4eb1f29ebddd60c41a2eb39f5af701e38e0d3fd)
+            # We could switch to the following code if we are able to detect version properly
+            # args += ['-http_proxy', proxy]
+            env = os.environ.copy()
+            compat_setenv('HTTP_PROXY', proxy, env=env)
+            compat_setenv('http_proxy', proxy, env=env)
+
+        protocol = info_dict.get('protocol')
+
+        if protocol == 'rtmp':
+            player_url = info_dict.get('player_url')
+            page_url = info_dict.get('page_url')
+            app = info_dict.get('app')
+            play_path = info_dict.get('play_path')
+            tc_url = info_dict.get('tc_url')
+            flash_version = info_dict.get('flash_version')
+            live = info_dict.get('rtmp_live', False)
+            conn = info_dict.get('rtmp_conn')
+            if player_url is not None:
+                args += ['-rtmp_swfverify', player_url]
+            if page_url is not None:
+                args += ['-rtmp_pageurl', page_url]
+            if app is not None:
+                args += ['-rtmp_app', app]
+            if play_path is not None:
+                args += ['-rtmp_playpath', play_path]
+            if tc_url is not None:
+                args += ['-rtmp_tcurl', tc_url]
+            if flash_version is not None:
+                args += ['-rtmp_flashver', flash_version]
+            if live:
+                args += ['-rtmp_live', 'live']
+            if isinstance(conn, list):
+                for entry in conn:
+                    args += ['-rtmp_conn', entry]
+            elif isinstance(conn, compat_str):
+                args += ['-rtmp_conn', conn]
+
+        for i, url in enumerate(urls):
+            args += self._configuration_args((f'_i{i + 1}', '_i')) + ['-i', url]
+
+        args += ['-c', 'copy']
+        if info_dict.get('requested_formats') or protocol == 'http_dash_segments':
+            for (i, fmt) in enumerate(info_dict.get('requested_formats') or [info_dict]):
+                stream_number = fmt.get('manifest_stream_number', 0)
+                args.extend(['-map', f'{i}:{stream_number}'])
+
+        if self.params.get('test', False):
+            args += ['-fs', compat_str(self._TEST_FILE_SIZE)]
+
+        ext = info_dict['ext']
+        if protocol in ('m3u8', 'm3u8_native'):
+            use_mpegts = (tmpfilename == '-') or self.params.get('hls_use_mpegts')
+            if use_mpegts is None:
+                use_mpegts = info_dict.get('is_live')
+            if use_mpegts:
+                args += ['-f', 'mpegts']
+            else:
+                args += ['-f', 'mp4']
+                if (ffpp.basename == 'ffmpeg' and ffpp._features.get('needs_adtstoasc')) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
+                    args += ['-bsf:a', 'aac_adtstoasc']
+        elif protocol == 'rtmp':
+            args += ['-f', 'flv']
+        elif ext == 'mp4' and tmpfilename == '-':
+            args += ['-f', 'mpegts']
+        elif ext == 'unknown_video':
+            ext = determine_ext(remove_end(tmpfilename, '.part'))
+            if ext == 'unknown_video':
+                self.report_warning(
+                    'The video format is unknown and cannot be downloaded by ffmpeg. '
+                    'Explicitly set the extension in the filename to attempt download in that format')
+            else:
+                self.report_warning(f'The video format is unknown. Trying to download as {ext} according to the filename')
+                args += ['-f', EXT_TO_OUT_FORMATS.get(ext, ext)]
+        else:
+            args += ['-f', EXT_TO_OUT_FORMATS.get(ext, ext)]
+
+        args += self._configuration_args(('_o1', '_o', ''))
+
+        args = [encodeArgument(opt) for opt in args]
+        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
+        self._debug_cmd(args)
+
+        proc = Popen(args, stdin=subprocess.PIPE, env=env)
+        if url in ('-', 'pipe:'):
+            self.on_process_started(proc, proc.stdin)
+        try:
+            retval = proc.wait()
+        except BaseException as e:
+            # subprocces.run would send the SIGKILL signal to ffmpeg and the
+            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
+            # produces a file that is playable (this is mostly useful for live
+            # streams). Note that Windows is not affected and produces playable
+            # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
+            if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32' and url not in ('-', 'pipe:'):
+                proc.communicate_or_kill(b'q')
+            else:
+                proc.kill()
+                proc.wait()
+            raise
+        return retval
+
+
+class AVconvFD(FFmpegFD):
+    pass
+
+
+_BY_NAME = dict(
+    (klass.get_basename(), klass)
+    for name, klass in globals().items()
+    if name.endswith('FD') and name not in ('ExternalFD', 'FragmentFD')
+)
+
+
+def list_external_downloaders():
+    return sorted(_BY_NAME.keys())
+
+
+def get_external_downloader(external_downloader):
+    """ Given the name of the executable, see whether we support the given
+        downloader . """
+    # Drop .exe extension on Windows
+    bn = os.path.splitext(os.path.basename(external_downloader))[0]
+    return _BY_NAME.get(bn)

plugins/youtube_download/yt_dlp/downloader/f4m.py

@@ -0,0 +1,439 @@
+from __future__ import division, unicode_literals
+
+import io
+import itertools
+import time
+
+from .fragment import FragmentFD
+from ..compat import (
+    compat_b64decode,
+    compat_etree_fromstring,
+    compat_urlparse,
+    compat_urllib_error,
+    compat_urllib_parse_urlparse,
+    compat_struct_pack,
+    compat_struct_unpack,
+)
+from ..utils import (
+    fix_xml_ampersands,
+    xpath_text,
+)
+
+
+class DataTruncatedError(Exception):
+    pass
+
+
+class FlvReader(io.BytesIO):
+    """
+    Reader for Flv files
+    The file format is documented in https://www.adobe.com/devnet/f4v.html
+    """
+
+    def read_bytes(self, n):
+        data = self.read(n)
+        if len(data) < n:
+            raise DataTruncatedError(
+                'FlvReader error: need %d bytes while only %d bytes got' % (
+                    n, len(data)))
+        return data
+
+    # Utility functions for reading numbers and strings
+    def read_unsigned_long_long(self):
+        return compat_struct_unpack('!Q', self.read_bytes(8))[0]
+
+    def read_unsigned_int(self):
+        return compat_struct_unpack('!I', self.read_bytes(4))[0]
+
+    def read_unsigned_char(self):
+        return compat_struct_unpack('!B', self.read_bytes(1))[0]
+
+    def read_string(self):
+        res = b''
+        while True:
+            char = self.read_bytes(1)
+            if char == b'\x00':
+                break
+            res += char
+        return res
+
+    def read_box_info(self):
+        """
+        Read a box and return the info as a tuple: (box_size, box_type, box_data)
+        """
+        real_size = size = self.read_unsigned_int()
+        box_type = self.read_bytes(4)
+        header_end = 8
+        if size == 1:
+            real_size = self.read_unsigned_long_long()
+            header_end = 16
+        return real_size, box_type, self.read_bytes(real_size - header_end)
+
+    def read_asrt(self):
+        # version
+        self.read_unsigned_char()
+        # flags
+        self.read_bytes(3)
+        quality_entry_count = self.read_unsigned_char()
+        # QualityEntryCount
+        for i in range(quality_entry_count):
+            self.read_string()
+
+        segment_run_count = self.read_unsigned_int()
+        segments = []
+        for i in range(segment_run_count):
+            first_segment = self.read_unsigned_int()
+            fragments_per_segment = self.read_unsigned_int()
+            segments.append((first_segment, fragments_per_segment))
+
+        return {
+            'segment_run': segments,
+        }
+
+    def read_afrt(self):
+        # version
+        self.read_unsigned_char()
+        # flags
+        self.read_bytes(3)
+        # time scale
+        self.read_unsigned_int()
+
+        quality_entry_count = self.read_unsigned_char()
+        # QualitySegmentUrlModifiers
+        for i in range(quality_entry_count):
+            self.read_string()
+
+        fragments_count = self.read_unsigned_int()
+        fragments = []
+        for i in range(fragments_count):
+            first = self.read_unsigned_int()
+            first_ts = self.read_unsigned_long_long()
+            duration = self.read_unsigned_int()
+            if duration == 0:
+                discontinuity_indicator = self.read_unsigned_char()
+            else:
+                discontinuity_indicator = None
+            fragments.append({
+                'first': first,
+                'ts': first_ts,
+                'duration': duration,
+                'discontinuity_indicator': discontinuity_indicator,
+            })
+
+        return {
+            'fragments': fragments,
+        }
+
+    def read_abst(self):
+        # version
+        self.read_unsigned_char()
+        # flags
+        self.read_bytes(3)
+
+        self.read_unsigned_int()  # BootstrapinfoVersion
+        # Profile,Live,Update,Reserved
+        flags = self.read_unsigned_char()
+        live = flags & 0x20 != 0
+        # time scale
+        self.read_unsigned_int()
+        # CurrentMediaTime
+        self.read_unsigned_long_long()
+        # SmpteTimeCodeOffset
+        self.read_unsigned_long_long()
+
+        self.read_string()  # MovieIdentifier
+        server_count = self.read_unsigned_char()
+        # ServerEntryTable
+        for i in range(server_count):
+            self.read_string()
+        quality_count = self.read_unsigned_char()
+        # QualityEntryTable
+        for i in range(quality_count):
+            self.read_string()
+        # DrmData
+        self.read_string()
+        # MetaData
+        self.read_string()
+
+        segments_count = self.read_unsigned_char()
+        segments = []
+        for i in range(segments_count):
+            box_size, box_type, box_data = self.read_box_info()
+            assert box_type == b'asrt'
+            segment = FlvReader(box_data).read_asrt()
+            segments.append(segment)
+        fragments_run_count = self.read_unsigned_char()
+        fragments = []
+        for i in range(fragments_run_count):
+            box_size, box_type, box_data = self.read_box_info()
+            assert box_type == b'afrt'
+            fragments.append(FlvReader(box_data).read_afrt())
+
+        return {
+            'segments': segments,
+            'fragments': fragments,
+            'live': live,
+        }
+
+    def read_bootstrap_info(self):
+        total_size, box_type, box_data = self.read_box_info()
+        assert box_type == b'abst'
+        return FlvReader(box_data).read_abst()
+
+
+def read_bootstrap_info(bootstrap_bytes):
+    return FlvReader(bootstrap_bytes).read_bootstrap_info()
+
+
+def build_fragments_list(boot_info):
+    """ Return a list of (segment, fragment) for each fragment in the video """
+    res = []
+    segment_run_table = boot_info['segments'][0]
+    fragment_run_entry_table = boot_info['fragments'][0]['fragments']
+    first_frag_number = fragment_run_entry_table[0]['first']
+    fragments_counter = itertools.count(first_frag_number)
+    for segment, fragments_count in segment_run_table['segment_run']:
+        # In some live HDS streams (for example Rai), `fragments_count` is
+        # abnormal and causing out-of-memory errors. It's OK to change the
+        # number of fragments for live streams as they are updated periodically
+        if fragments_count == 4294967295 and boot_info['live']:
+            fragments_count = 2
+        for _ in range(fragments_count):
+            res.append((segment, next(fragments_counter)))
+
+    if boot_info['live']:
+        res = res[-2:]
+
+    return res
+
+
+def write_unsigned_int(stream, val):
+    stream.write(compat_struct_pack('!I', val))
+
+
+def write_unsigned_int_24(stream, val):
+    stream.write(compat_struct_pack('!I', val)[1:])
+
+
+def write_flv_header(stream):
+    """Writes the FLV header to stream"""
+    # FLV header
+    stream.write(b'FLV\x01')
+    stream.write(b'\x05')
+    stream.write(b'\x00\x00\x00\x09')
+    stream.write(b'\x00\x00\x00\x00')
+
+
+def write_metadata_tag(stream, metadata):
+    """Writes optional metadata tag to stream"""
+    SCRIPT_TAG = b'\x12'
+    FLV_TAG_HEADER_LEN = 11
+
+    if metadata:
+        stream.write(SCRIPT_TAG)
+        write_unsigned_int_24(stream, len(metadata))
+        stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
+        stream.write(metadata)
+        write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata))
+
+
+def remove_encrypted_media(media):
+    return list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib
+                                 and 'drmAdditionalHeaderSetId' not in e.attrib,
+                       media))
+
+
+def _add_ns(prop, ver=1):
+    return '{http://ns.adobe.com/f4m/%d.0}%s' % (ver, prop)
+
+
+def get_base_url(manifest):
+    base_url = xpath_text(
+        manifest, [_add_ns('baseURL'), _add_ns('baseURL', 2)],
+        'base URL', default=None)
+    if base_url:
+        base_url = base_url.strip()
+    return base_url
+
+
+class F4mFD(FragmentFD):
+    """
+    A downloader for f4m manifests or AdobeHDS.
+    """
+
+    FD_NAME = 'f4m'
+
+    def _get_unencrypted_media(self, doc):
+        media = doc.findall(_add_ns('media'))
+        if not media:
+            self.report_error('No media found')
+        if not self.params.get('allow_unplayable_formats'):
+            for e in (doc.findall(_add_ns('drmAdditionalHeader'))
+                      + doc.findall(_add_ns('drmAdditionalHeaderSet'))):
+                # If id attribute is missing it's valid for all media nodes
+                # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
+                if 'id' not in e.attrib:
+                    self.report_error('Missing ID in f4m DRM')
+            media = remove_encrypted_media(media)
+        if not media:
+            self.report_error('Unsupported DRM')
+        return media
+
+    def _get_bootstrap_from_url(self, bootstrap_url):
+        bootstrap = self.ydl.urlopen(bootstrap_url).read()
+        return read_bootstrap_info(bootstrap)
+
+    def _update_live_fragments(self, bootstrap_url, latest_fragment):
+        fragments_list = []
+        retries = 30
+        while (not fragments_list) and (retries > 0):
+            boot_info = self._get_bootstrap_from_url(bootstrap_url)
+            fragments_list = build_fragments_list(boot_info)
+            fragments_list = [f for f in fragments_list if f[1] > latest_fragment]
+            if not fragments_list:
+                # Retry after a while
+                time.sleep(5.0)
+                retries -= 1
+
+        if not fragments_list:
+            self.report_error('Failed to update fragments')
+
+        return fragments_list
+
+    def _parse_bootstrap_node(self, node, base_url):
+        # Sometimes non empty inline bootstrap info can be specified along
+        # with bootstrap url attribute (e.g. dummy inline bootstrap info
+        # contains whitespace characters in [1]). We will prefer bootstrap
+        # url over inline bootstrap info when present.
+        # 1. http://live-1-1.rutube.ru/stream/1024/HDS/SD/C2NKsS85HQNckgn5HdEmOQ/1454167650/S-s604419906/move/four/dirs/upper/1024-576p.f4m
+        bootstrap_url = node.get('url')
+        if bootstrap_url:
+            bootstrap_url = compat_urlparse.urljoin(
+                base_url, bootstrap_url)
+            boot_info = self._get_bootstrap_from_url(bootstrap_url)
+        else:
+            bootstrap_url = None
+            bootstrap = compat_b64decode(node.text)
+            boot_info = read_bootstrap_info(bootstrap)
+        return boot_info, bootstrap_url
+
+    def real_download(self, filename, info_dict):
+        man_url = info_dict['url']
+        requested_bitrate = info_dict.get('tbr')
+        self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
+
+        urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
+        man_url = urlh.geturl()
+        # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
+        # (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244
+        # and https://github.com/ytdl-org/youtube-dl/issues/7823)
+        manifest = fix_xml_ampersands(urlh.read().decode('utf-8', 'ignore')).strip()
+
+        doc = compat_etree_fromstring(manifest)
+        formats = [(int(f.attrib.get('bitrate', -1)), f)
+                   for f in self._get_unencrypted_media(doc)]
+        if requested_bitrate is None or len(formats) == 1:
+            # get the best format
+            formats = sorted(formats, key=lambda f: f[0])
+            rate, media = formats[-1]
+        else:
+            rate, media = list(filter(
+                lambda f: int(f[0]) == requested_bitrate, formats))[0]
+
+        # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec.
+        man_base_url = get_base_url(doc) or man_url
+
+        base_url = compat_urlparse.urljoin(man_base_url, media.attrib['url'])
+        bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
+        boot_info, bootstrap_url = self._parse_bootstrap_node(
+            bootstrap_node, man_base_url)
+        live = boot_info['live']
+        metadata_node = media.find(_add_ns('metadata'))
+        if metadata_node is not None:
+            metadata = compat_b64decode(metadata_node.text)
+        else:
+            metadata = None
+
+        fragments_list = build_fragments_list(boot_info)
+        test = self.params.get('test', False)
+        if test:
+            # We only download the first fragment
+            fragments_list = fragments_list[:1]
+        total_frags = len(fragments_list)
+        # For some akamai manifests we'll need to add a query to the fragment url
+        akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
+
+        ctx = {
+            'filename': filename,
+            'total_frags': total_frags,
+            'live': bool(live),
+        }
+
+        self._prepare_frag_download(ctx)
+
+        dest_stream = ctx['dest_stream']
+
+        if ctx['complete_frags_downloaded_bytes'] == 0:
+            write_flv_header(dest_stream)
+            if not live:
+                write_metadata_tag(dest_stream, metadata)
+
+        base_url_parsed = compat_urllib_parse_urlparse(base_url)
+
+        self._start_frag_download(ctx, info_dict)
+
+        frag_index = 0
+        while fragments_list:
+            seg_i, frag_i = fragments_list.pop(0)
+            frag_index += 1
+            if frag_index <= ctx['fragment_index']:
+                continue
+            name = 'Seg%d-Frag%d' % (seg_i, frag_i)
+            query = []
+            if base_url_parsed.query:
+                query.append(base_url_parsed.query)
+            if akamai_pv:
+                query.append(akamai_pv.strip(';'))
+            if info_dict.get('extra_param_to_segment_url'):
+                query.append(info_dict['extra_param_to_segment_url'])
+            url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
+            try:
+                success, down_data = self._download_fragment(ctx, url_parsed.geturl(), info_dict)
+                if not success:
+                    return False
+                reader = FlvReader(down_data)
+                while True:
+                    try:
+                        _, box_type, box_data = reader.read_box_info()
+                    except DataTruncatedError:
+                        if test:
+                            # In tests, segments may be truncated, and thus
+                            # FlvReader may not be able to parse the whole
+                            # chunk. If so, write the segment as is
+                            # See https://github.com/ytdl-org/youtube-dl/issues/9214
+                            dest_stream.write(down_data)
+                            break
+                        raise
+                    if box_type == b'mdat':
+                        self._append_fragment(ctx, box_data)
+                        break
+            except (compat_urllib_error.HTTPError, ) as err:
+                if live and (err.code == 404 or err.code == 410):
+                    # We didn't keep up with the live window. Continue
+                    # with the next available fragment.
+                    msg = 'Fragment %d unavailable' % frag_i
+                    self.report_warning(msg)
+                    fragments_list = []
+                else:
+                    raise
+
+            if not fragments_list and not test and live and bootstrap_url:
+                fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
+                total_frags += len(fragments_list)
+                if fragments_list and (fragments_list[0][1] > frag_i + 1):
+                    msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
+                    self.report_warning(msg)
+
+        self._finish_frag_download(ctx, info_dict)
+
+        return True

plugins/youtube_download/yt_dlp/downloader/fragment.py

@@ -0,0 +1,523 @@
+from __future__ import division, unicode_literals
+
+import http.client
+import json
+import math
+import os
+import time
+
+try:
+    import concurrent.futures
+    can_threaded_download = True
+except ImportError:
+    can_threaded_download = False
+
+from .common import FileDownloader
+from .http import HttpFD
+from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
+from ..compat import (
+    compat_os_name,
+    compat_urllib_error,
+    compat_struct_pack,
+)
+from ..utils import (
+    DownloadError,
+    error_to_compat_str,
+    encodeFilename,
+    sanitized_Request,
+)
+
+
+class HttpQuietDownloader(HttpFD):
+    def to_screen(self, *args, **kargs):
+        pass
+
+    def report_retry(self, err, count, retries):
+        super().to_screen(
+            f'[download] Got server HTTP error: {err}. Retrying (attempt {count} of {self.format_retries(retries)}) ...')
+
+
+class FragmentFD(FileDownloader):
+    """
+    A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
+
+    Available options:
+
+    fragment_retries:   Number of times to retry a fragment for HTTP error (DASH
+                        and hlsnative only)
+    skip_unavailable_fragments:
+                        Skip unavailable fragments (DASH and hlsnative only)
+    keep_fragments:     Keep downloaded fragments on disk after downloading is
+                        finished
+    concurrent_fragment_downloads:  The number of threads to use for native hls and dash downloads
+    _no_ytdl_file:      Don't use .ytdl file
+
+    For each incomplete fragment download yt-dlp keeps on disk a special
+    bookkeeping file with download state and metadata (in future such files will
+    be used for any incomplete download handled by yt-dlp). This file is
+    used to properly handle resuming, check download file consistency and detect
+    potential errors. The file has a .ytdl extension and represents a standard
+    JSON file of the following format:
+
+    extractor:
+        Dictionary of extractor related data. TBD.
+
+    downloader:
+        Dictionary of downloader related data. May contain following data:
+            current_fragment:
+                Dictionary with current (being downloaded) fragment data:
+                index:  0-based index of current fragment among all fragments
+            fragment_count:
+                Total count of fragments
+
+    This feature is experimental and file format may change in future.
+    """
+
+    def report_retry_fragment(self, err, frag_index, count, retries):
+        self.to_screen(
+            '\r[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s) ...'
+            % (error_to_compat_str(err), frag_index, count, self.format_retries(retries)))
+
+    def report_skip_fragment(self, frag_index, err=None):
+        err = f' {err};' if err else ''
+        self.to_screen(f'[download]{err} Skipping fragment {frag_index:d} ...')
+
+    def _prepare_url(self, info_dict, url):
+        headers = info_dict.get('http_headers')
+        return sanitized_Request(url, None, headers) if headers else url
+
+    def _prepare_and_start_frag_download(self, ctx, info_dict):
+        self._prepare_frag_download(ctx)
+        self._start_frag_download(ctx, info_dict)
+
+    def __do_ytdl_file(self, ctx):
+        return ctx['live'] is not True and ctx['tmpfilename'] != '-' and not self.params.get('_no_ytdl_file')
+
+    def _read_ytdl_file(self, ctx):
+        assert 'ytdl_corrupt' not in ctx
+        stream, _ = self.sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
+        try:
+            ytdl_data = json.loads(stream.read())
+            ctx['fragment_index'] = ytdl_data['downloader']['current_fragment']['index']
+            if 'extra_state' in ytdl_data['downloader']:
+                ctx['extra_state'] = ytdl_data['downloader']['extra_state']
+        except Exception:
+            ctx['ytdl_corrupt'] = True
+        finally:
+            stream.close()
+
+    def _write_ytdl_file(self, ctx):
+        frag_index_stream, _ = self.sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
+        try:
+            downloader = {
+                'current_fragment': {
+                    'index': ctx['fragment_index'],
+                },
+            }
+            if 'extra_state' in ctx:
+                downloader['extra_state'] = ctx['extra_state']
+            if ctx.get('fragment_count') is not None:
+                downloader['fragment_count'] = ctx['fragment_count']
+            frag_index_stream.write(json.dumps({'downloader': downloader}))
+        finally:
+            frag_index_stream.close()
+
+    def _download_fragment(self, ctx, frag_url, info_dict, headers=None, request_data=None):
+        fragment_filename = '%s-Frag%d' % (ctx['tmpfilename'], ctx['fragment_index'])
+        fragment_info_dict = {
+            'url': frag_url,
+            'http_headers': headers or info_dict.get('http_headers'),
+            'request_data': request_data,
+            'ctx_id': ctx.get('ctx_id'),
+        }
+        success = ctx['dl'].download(fragment_filename, fragment_info_dict)
+        if not success:
+            return False, None
+        if fragment_info_dict.get('filetime'):
+            ctx['fragment_filetime'] = fragment_info_dict.get('filetime')
+        ctx['fragment_filename_sanitized'] = fragment_filename
+        return True, self._read_fragment(ctx)
+
+    def _read_fragment(self, ctx):
+        down, frag_sanitized = self.sanitize_open(ctx['fragment_filename_sanitized'], 'rb')
+        ctx['fragment_filename_sanitized'] = frag_sanitized
+        frag_content = down.read()
+        down.close()
+        return frag_content
+
+    def _append_fragment(self, ctx, frag_content):
+        try:
+            ctx['dest_stream'].write(frag_content)
+            ctx['dest_stream'].flush()
+        finally:
+            if self.__do_ytdl_file(ctx):
+                self._write_ytdl_file(ctx)
+            if not self.params.get('keep_fragments', False):
+                os.remove(encodeFilename(ctx['fragment_filename_sanitized']))
+            del ctx['fragment_filename_sanitized']
+
+    def _prepare_frag_download(self, ctx):
+        if 'live' not in ctx:
+            ctx['live'] = False
+        if not ctx['live']:
+            total_frags_str = '%d' % ctx['total_frags']
+            ad_frags = ctx.get('ad_frags', 0)
+            if ad_frags:
+                total_frags_str += ' (not including %d ad)' % ad_frags
+        else:
+            total_frags_str = 'unknown (live)'
+        self.to_screen(
+            '[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
+        self.report_destination(ctx['filename'])
+        dl = HttpQuietDownloader(
+            self.ydl,
+            {
+                'continuedl': True,
+                'quiet': self.params.get('quiet'),
+                'noprogress': True,
+                'ratelimit': self.params.get('ratelimit'),
+                'retries': self.params.get('retries', 0),
+                'nopart': self.params.get('nopart', False),
+                'test': self.params.get('test', False),
+            }
+        )
+        tmpfilename = self.temp_name(ctx['filename'])
+        open_mode = 'wb'
+        resume_len = 0
+
+        # Establish possible resume length
+        if os.path.isfile(encodeFilename(tmpfilename)):
+            open_mode = 'ab'
+            resume_len = os.path.getsize(encodeFilename(tmpfilename))
+
+        # Should be initialized before ytdl file check
+        ctx.update({
+            'tmpfilename': tmpfilename,
+            'fragment_index': 0,
+        })
+
+        if self.__do_ytdl_file(ctx):
+            if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+                self._read_ytdl_file(ctx)
+                is_corrupt = ctx.get('ytdl_corrupt') is True
+                is_inconsistent = ctx['fragment_index'] > 0 and resume_len == 0
+                if is_corrupt or is_inconsistent:
+                    message = (
+                        '.ytdl file is corrupt' if is_corrupt else
+                        'Inconsistent state of incomplete fragment download')
+                    self.report_warning(
+                        '%s. Restarting from the beginning ...' % message)
+                    ctx['fragment_index'] = resume_len = 0
+                    if 'ytdl_corrupt' in ctx:
+                        del ctx['ytdl_corrupt']
+                    self._write_ytdl_file(ctx)
+            else:
+                self._write_ytdl_file(ctx)
+                assert ctx['fragment_index'] == 0
+
+        dest_stream, tmpfilename = self.sanitize_open(tmpfilename, open_mode)
+
+        ctx.update({
+            'dl': dl,
+            'dest_stream': dest_stream,
+            'tmpfilename': tmpfilename,
+            # Total complete fragments downloaded so far in bytes
+            'complete_frags_downloaded_bytes': resume_len,
+        })
+
+    def _start_frag_download(self, ctx, info_dict):
+        resume_len = ctx['complete_frags_downloaded_bytes']
+        total_frags = ctx['total_frags']
+        ctx_id = ctx.get('ctx_id')
+        # This dict stores the download progress, it's updated by the progress
+        # hook
+        state = {
+            'status': 'downloading',
+            'downloaded_bytes': resume_len,
+            'fragment_index': ctx['fragment_index'],
+            'fragment_count': total_frags,
+            'filename': ctx['filename'],
+            'tmpfilename': ctx['tmpfilename'],
+        }
+
+        start = time.time()
+        ctx.update({
+            'started': start,
+            'fragment_started': start,
+            # Amount of fragment's bytes downloaded by the time of the previous
+            # frag progress hook invocation
+            'prev_frag_downloaded_bytes': 0,
+        })
+
+        def frag_progress_hook(s):
+            if s['status'] not in ('downloading', 'finished'):
+                return
+
+            if ctx_id is not None and s.get('ctx_id') != ctx_id:
+                return
+
+            state['max_progress'] = ctx.get('max_progress')
+            state['progress_idx'] = ctx.get('progress_idx')
+
+            time_now = time.time()
+            state['elapsed'] = time_now - start
+            frag_total_bytes = s.get('total_bytes') or 0
+            s['fragment_info_dict'] = s.pop('info_dict', {})
+            if not ctx['live']:
+                estimated_size = (
+                    (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes)
+                    / (state['fragment_index'] + 1) * total_frags)
+                state['total_bytes_estimate'] = estimated_size
+
+            if s['status'] == 'finished':
+                state['fragment_index'] += 1
+                ctx['fragment_index'] = state['fragment_index']
+                state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
+                ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
+                ctx['speed'] = state['speed'] = self.calc_speed(
+                    ctx['fragment_started'], time_now, frag_total_bytes)
+                ctx['fragment_started'] = time.time()
+                ctx['prev_frag_downloaded_bytes'] = 0
+            else:
+                frag_downloaded_bytes = s['downloaded_bytes']
+                state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
+                if not ctx['live']:
+                    state['eta'] = self.calc_eta(
+                        start, time_now, estimated_size - resume_len,
+                        state['downloaded_bytes'] - resume_len)
+                ctx['speed'] = state['speed'] = self.calc_speed(
+                    ctx['fragment_started'], time_now, frag_downloaded_bytes)
+                ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
+            self._hook_progress(state, info_dict)
+
+        ctx['dl'].add_progress_hook(frag_progress_hook)
+
+        return start
+
+    def _finish_frag_download(self, ctx, info_dict):
+        ctx['dest_stream'].close()
+        if self.__do_ytdl_file(ctx):
+            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+            if os.path.isfile(ytdl_filename):
+                os.remove(ytdl_filename)
+        elapsed = time.time() - ctx['started']
+
+        if ctx['tmpfilename'] == '-':
+            downloaded_bytes = ctx['complete_frags_downloaded_bytes']
+        else:
+            self.try_rename(ctx['tmpfilename'], ctx['filename'])
+            if self.params.get('updatetime', True):
+                filetime = ctx.get('fragment_filetime')
+                if filetime:
+                    try:
+                        os.utime(ctx['filename'], (time.time(), filetime))
+                    except Exception:
+                        pass
+            downloaded_bytes = os.path.getsize(encodeFilename(ctx['filename']))
+
+        self._hook_progress({
+            'downloaded_bytes': downloaded_bytes,
+            'total_bytes': downloaded_bytes,
+            'filename': ctx['filename'],
+            'status': 'finished',
+            'elapsed': elapsed,
+            'ctx_id': ctx.get('ctx_id'),
+            'max_progress': ctx.get('max_progress'),
+            'progress_idx': ctx.get('progress_idx'),
+        }, info_dict)
+
+    def _prepare_external_frag_download(self, ctx):
+        if 'live' not in ctx:
+            ctx['live'] = False
+        if not ctx['live']:
+            total_frags_str = '%d' % ctx['total_frags']
+            ad_frags = ctx.get('ad_frags', 0)
+            if ad_frags:
+                total_frags_str += ' (not including %d ad)' % ad_frags
+        else:
+            total_frags_str = 'unknown (live)'
+        self.to_screen(
+            '[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
+
+        tmpfilename = self.temp_name(ctx['filename'])
+
+        # Should be initialized before ytdl file check
+        ctx.update({
+            'tmpfilename': tmpfilename,
+            'fragment_index': 0,
+        })
+
+    def decrypter(self, info_dict):
+        _key_cache = {}
+
+        def _get_key(url):
+            if url not in _key_cache:
+                _key_cache[url] = self.ydl.urlopen(self._prepare_url(info_dict, url)).read()
+            return _key_cache[url]
+
+        def decrypt_fragment(fragment, frag_content):
+            decrypt_info = fragment.get('decrypt_info')
+            if not decrypt_info or decrypt_info['METHOD'] != 'AES-128':
+                return frag_content
+            iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', fragment['media_sequence'])
+            decrypt_info['KEY'] = decrypt_info.get('KEY') or _get_key(info_dict.get('_decryption_key_url') or decrypt_info['URI'])
+            # Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block
+            # size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded,
+            # not what it decrypts to.
+            if self.params.get('test', False):
+                return frag_content
+            return unpad_pkcs7(aes_cbc_decrypt_bytes(frag_content, decrypt_info['KEY'], iv))
+
+        return decrypt_fragment
+
+    def download_and_append_fragments_multiple(self, *args, pack_func=None, finish_func=None):
+        '''
+        @params (ctx1, fragments1, info_dict1), (ctx2, fragments2, info_dict2), ...
+                all args must be either tuple or list
+        '''
+        interrupt_trigger = [True]
+        max_progress = len(args)
+        if max_progress == 1:
+            return self.download_and_append_fragments(*args[0], pack_func=pack_func, finish_func=finish_func)
+        max_workers = self.params.get('concurrent_fragment_downloads', 1)
+        if max_progress > 1:
+            self._prepare_multiline_status(max_progress)
+
+        def thread_func(idx, ctx, fragments, info_dict, tpe):
+            ctx['max_progress'] = max_progress
+            ctx['progress_idx'] = idx
+            return self.download_and_append_fragments(
+                ctx, fragments, info_dict, pack_func=pack_func, finish_func=finish_func,
+                tpe=tpe, interrupt_trigger=interrupt_trigger)
+
+        class FTPE(concurrent.futures.ThreadPoolExecutor):
+            # has to stop this or it's going to wait on the worker thread itself
+            def __exit__(self, exc_type, exc_val, exc_tb):
+                pass
+
+        spins = []
+        if compat_os_name == 'nt':
+            self.report_warning('Ctrl+C does not work on Windows when used with parallel threads. '
+                                'This is a known issue and patches are welcome')
+        for idx, (ctx, fragments, info_dict) in enumerate(args):
+            tpe = FTPE(math.ceil(max_workers / max_progress))
+            job = tpe.submit(thread_func, idx, ctx, fragments, info_dict, tpe)
+            spins.append((tpe, job))
+
+        result = True
+        for tpe, job in spins:
+            try:
+                result = result and job.result()
+            except KeyboardInterrupt:
+                interrupt_trigger[0] = False
+            finally:
+                tpe.shutdown(wait=True)
+        if not interrupt_trigger[0]:
+            raise KeyboardInterrupt()
+        return result
+
+    def download_and_append_fragments(
+            self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None,
+            tpe=None, interrupt_trigger=None):
+        if not interrupt_trigger:
+            interrupt_trigger = (True, )
+
+        fragment_retries = self.params.get('fragment_retries', 0)
+        is_fatal = (
+            ((lambda _: False) if info_dict.get('is_live') else (lambda idx: idx == 0))
+            if self.params.get('skip_unavailable_fragments', True) else (lambda _: True))
+
+        if not pack_func:
+            pack_func = lambda frag_content, _: frag_content
+
+        def download_fragment(fragment, ctx):
+            frag_index = ctx['fragment_index'] = fragment['frag_index']
+            ctx['last_error'] = None
+            if not interrupt_trigger[0]:
+                return False, frag_index
+            headers = info_dict.get('http_headers', {}).copy()
+            byte_range = fragment.get('byte_range')
+            if byte_range:
+                headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
+
+            # Never skip the first fragment
+            fatal = is_fatal(fragment.get('index') or (frag_index - 1))
+            count, frag_content = 0, None
+            while count <= fragment_retries:
+                try:
+                    success, frag_content = self._download_fragment(ctx, fragment['url'], info_dict, headers)
+                    if not success:
+                        return False, frag_index
+                    break
+                except (compat_urllib_error.HTTPError, http.client.IncompleteRead) as err:
+                    # Unavailable (possibly temporary) fragments may be served.
+                    # First we try to retry then either skip or abort.
+                    # See https://github.com/ytdl-org/youtube-dl/issues/10165,
+                    # https://github.com/ytdl-org/youtube-dl/issues/10448).
+                    count += 1
+                    ctx['last_error'] = err
+                    if count <= fragment_retries:
+                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
+                except DownloadError:
+                    # Don't retry fragment if error occurred during HTTP downloading
+                    # itself since it has own retry settings
+                    if not fatal:
+                        break
+                    raise
+
+            if count > fragment_retries:
+                if not fatal:
+                    return False, frag_index
+                ctx['dest_stream'].close()
+                self.report_error('Giving up after %s fragment retries' % fragment_retries)
+                return False, frag_index
+            return frag_content, frag_index
+
+        def append_fragment(frag_content, frag_index, ctx):
+            if not frag_content:
+                if not is_fatal(frag_index - 1):
+                    self.report_skip_fragment(frag_index, 'fragment not found')
+                    return True
+                else:
+                    ctx['dest_stream'].close()
+                    self.report_error(
+                        'fragment %s not found, unable to continue' % frag_index)
+                    return False
+            self._append_fragment(ctx, pack_func(frag_content, frag_index))
+            return True
+
+        decrypt_fragment = self.decrypter(info_dict)
+
+        max_workers = math.ceil(
+            self.params.get('concurrent_fragment_downloads', 1) / ctx.get('max_progress', 1))
+        if can_threaded_download and max_workers > 1:
+
+            def _download_fragment(fragment):
+                ctx_copy = ctx.copy()
+                frag_content, frag_index = download_fragment(fragment, ctx_copy)
+                return fragment, frag_content, frag_index, ctx_copy.get('fragment_filename_sanitized')
+
+            self.report_warning('The download speed shown is only of one thread. This is a known issue and patches are welcome')
+            with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool:
+                for fragment, frag_content, frag_index, frag_filename in pool.map(_download_fragment, fragments):
+                    if not interrupt_trigger[0]:
+                        break
+                    ctx['fragment_filename_sanitized'] = frag_filename
+                    ctx['fragment_index'] = frag_index
+                    result = append_fragment(decrypt_fragment(fragment, frag_content), frag_index, ctx)
+                    if not result:
+                        return False
+        else:
+            for fragment in fragments:
+                if not interrupt_trigger[0]:
+                    break
+                frag_content, frag_index = download_fragment(fragment, ctx)
+                result = append_fragment(decrypt_fragment(fragment, frag_content), frag_index, ctx)
+                if not result:
+                    return False
+
+        if finish_func is not None:
+            ctx['dest_stream'].write(finish_func())
+            ctx['dest_stream'].flush()
+        self._finish_frag_download(ctx, info_dict)
+        return True

plugins/youtube_download/yt_dlp/downloader/hls.py

@@ -0,0 +1,358 @@
+from __future__ import unicode_literals
+
+import re
+import io
+import binascii
+
+from ..downloader import get_suitable_downloader
+from .fragment import FragmentFD
+from .external import FFmpegFD
+
+from ..compat import (
+    compat_pycrypto_AES,
+    compat_urlparse,
+)
+from ..utils import (
+    parse_m3u8_attributes,
+    update_url_query,
+    bug_reports_message,
+)
+from .. import webvtt
+
+
+class HlsFD(FragmentFD):
+    """
+    Download segments in a m3u8 manifest. External downloaders can take over
+    the fragment downloads by supporting the 'm3u8_frag_urls' protocol and
+    re-defining 'supports_manifest' function
+    """
+
+    FD_NAME = 'hlsnative'
+
+    @staticmethod
+    def can_download(manifest, info_dict, allow_unplayable_formats=False):
+        UNSUPPORTED_FEATURES = [
+            # r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
+
+            # Live streams heuristic does not always work (e.g. geo restricted to Germany
+            # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
+            # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
+
+            # This heuristic also is not correct since segments may not be appended as well.
+            # Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite
+            # no segments will definitely be appended to the end of the playlist.
+            # r'#EXT-X-PLAYLIST-TYPE:EVENT',  # media segments may be appended to the end of
+            #                                 # event media playlists [4]
+            # r'#EXT-X-MAP:',  # media initialization [5]
+            # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
+            # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
+            # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
+            # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
+            # 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5
+        ]
+        if not allow_unplayable_formats:
+            UNSUPPORTED_FEATURES += [
+                r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
+            ]
+
+        def check_results():
+            yield not info_dict.get('is_live')
+            for feature in UNSUPPORTED_FEATURES:
+                yield not re.search(feature, manifest)
+        return all(check_results())
+
+    def real_download(self, filename, info_dict):
+        man_url = info_dict['url']
+        self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
+
+        urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
+        man_url = urlh.geturl()
+        s = urlh.read().decode('utf-8', 'ignore')
+
+        can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
+        if can_download and not compat_pycrypto_AES and '#EXT-X-KEY:METHOD=AES-128' in s:
+            if FFmpegFD.available():
+                can_download, message = False, 'The stream has AES-128 encryption and pycryptodomex is not available'
+            else:
+                message = ('The stream has AES-128 encryption and neither ffmpeg nor pycryptodomex are available; '
+                           'Decryption will be performed natively, but will be extremely slow')
+        if not can_download:
+            has_drm = re.search('|'.join([
+                r'#EXT-X-FAXS-CM:',  # Adobe Flash Access
+                r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://',  # Apple FairPlay
+            ]), s)
+            if has_drm and not self.params.get('allow_unplayable_formats'):
+                self.report_error(
+                    'This video is DRM protected; Try selecting another format with --format or '
+                    'add --check-formats to automatically fallback to the next best format')
+                return False
+            message = message or 'Unsupported features have been detected'
+            fd = FFmpegFD(self.ydl, self.params)
+            self.report_warning(f'{message}; extraction will be delegated to {fd.get_basename()}')
+            return fd.real_download(filename, info_dict)
+        elif message:
+            self.report_warning(message)
+
+        is_webvtt = info_dict['ext'] == 'vtt'
+        if is_webvtt:
+            real_downloader = None  # Packing the fragments is not currently supported for external downloader
+        else:
+            real_downloader = get_suitable_downloader(
+                info_dict, self.params, None, protocol='m3u8_frag_urls', to_stdout=(filename == '-'))
+        if real_downloader and not real_downloader.supports_manifest(s):
+            real_downloader = None
+        if real_downloader:
+            self.to_screen(
+                '[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
+
+        def is_ad_fragment_start(s):
+            return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
+                    or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',ad'))
+
+        def is_ad_fragment_end(s):
+            return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s
+                    or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',segment'))
+
+        fragments = []
+
+        media_frags = 0
+        ad_frags = 0
+        ad_frag_next = False
+        for line in s.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            if line.startswith('#'):
+                if is_ad_fragment_start(line):
+                    ad_frag_next = True
+                elif is_ad_fragment_end(line):
+                    ad_frag_next = False
+                continue
+            if ad_frag_next:
+                ad_frags += 1
+                continue
+            media_frags += 1
+
+        ctx = {
+            'filename': filename,
+            'total_frags': media_frags,
+            'ad_frags': ad_frags,
+        }
+
+        if real_downloader:
+            self._prepare_external_frag_download(ctx)
+        else:
+            self._prepare_and_start_frag_download(ctx, info_dict)
+
+        extra_state = ctx.setdefault('extra_state', {})
+
+        format_index = info_dict.get('format_index')
+        extra_query = None
+        extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
+        if extra_param_to_segment_url:
+            extra_query = compat_urlparse.parse_qs(extra_param_to_segment_url)
+        i = 0
+        media_sequence = 0
+        decrypt_info = {'METHOD': 'NONE'}
+        byte_range = {}
+        discontinuity_count = 0
+        frag_index = 0
+        ad_frag_next = False
+        for line in s.splitlines():
+            line = line.strip()
+            if line:
+                if not line.startswith('#'):
+                    if format_index and discontinuity_count != format_index:
+                        continue
+                    if ad_frag_next:
+                        continue
+                    frag_index += 1
+                    if frag_index <= ctx['fragment_index']:
+                        continue
+                    frag_url = (
+                        line
+                        if re.match(r'^https?://', line)
+                        else compat_urlparse.urljoin(man_url, line))
+                    if extra_query:
+                        frag_url = update_url_query(frag_url, extra_query)
+
+                    fragments.append({
+                        'frag_index': frag_index,
+                        'url': frag_url,
+                        'decrypt_info': decrypt_info,
+                        'byte_range': byte_range,
+                        'media_sequence': media_sequence,
+                    })
+                    media_sequence += 1
+
+                elif line.startswith('#EXT-X-MAP'):
+                    if format_index and discontinuity_count != format_index:
+                        continue
+                    if frag_index > 0:
+                        self.report_error(
+                            'Initialization fragment found after media fragments, unable to download')
+                        return False
+                    frag_index += 1
+                    map_info = parse_m3u8_attributes(line[11:])
+                    frag_url = (
+                        map_info.get('URI')
+                        if re.match(r'^https?://', map_info.get('URI'))
+                        else compat_urlparse.urljoin(man_url, map_info.get('URI')))
+                    if extra_query:
+                        frag_url = update_url_query(frag_url, extra_query)
+
+                    fragments.append({
+                        'frag_index': frag_index,
+                        'url': frag_url,
+                        'decrypt_info': decrypt_info,
+                        'byte_range': byte_range,
+                        'media_sequence': media_sequence
+                    })
+                    media_sequence += 1
+
+                    if map_info.get('BYTERANGE'):
+                        splitted_byte_range = map_info.get('BYTERANGE').split('@')
+                        sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end']
+                        byte_range = {
+                            'start': sub_range_start,
+                            'end': sub_range_start + int(splitted_byte_range[0]),
+                        }
+
+                elif line.startswith('#EXT-X-KEY'):
+                    decrypt_url = decrypt_info.get('URI')
+                    decrypt_info = parse_m3u8_attributes(line[11:])
+                    if decrypt_info['METHOD'] == 'AES-128':
+                        if 'IV' in decrypt_info:
+                            decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:].zfill(32))
+                        if not re.match(r'^https?://', decrypt_info['URI']):
+                            decrypt_info['URI'] = compat_urlparse.urljoin(
+                                man_url, decrypt_info['URI'])
+                        if extra_query:
+                            decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
+                        if decrypt_url != decrypt_info['URI']:
+                            decrypt_info['KEY'] = None
+
+                elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
+                    media_sequence = int(line[22:])
+                elif line.startswith('#EXT-X-BYTERANGE'):
+                    splitted_byte_range = line[17:].split('@')
+                    sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end']
+                    byte_range = {
+                        'start': sub_range_start,
+                        'end': sub_range_start + int(splitted_byte_range[0]),
+                    }
+                elif is_ad_fragment_start(line):
+                    ad_frag_next = True
+                elif is_ad_fragment_end(line):
+                    ad_frag_next = False
+                elif line.startswith('#EXT-X-DISCONTINUITY'):
+                    discontinuity_count += 1
+                i += 1
+
+        # We only download the first fragment during the test
+        if self.params.get('test', False):
+            fragments = [fragments[0] if fragments else None]
+
+        if real_downloader:
+            info_dict['fragments'] = fragments
+            fd = real_downloader(self.ydl, self.params)
+            # TODO: Make progress updates work without hooking twice
+            # for ph in self._progress_hooks:
+            #     fd.add_progress_hook(ph)
+            return fd.real_download(filename, info_dict)
+
+        if is_webvtt:
+            def pack_fragment(frag_content, frag_index):
+                output = io.StringIO()
+                adjust = 0
+                overflow = False
+                mpegts_last = None
+                for block in webvtt.parse_fragment(frag_content):
+                    if isinstance(block, webvtt.CueBlock):
+                        extra_state['webvtt_mpegts_last'] = mpegts_last
+                        if overflow:
+                            extra_state['webvtt_mpegts_adjust'] += 1
+                            overflow = False
+                        block.start += adjust
+                        block.end += adjust
+
+                        dedup_window = extra_state.setdefault('webvtt_dedup_window', [])
+
+                        ready = []
+
+                        i = 0
+                        is_new = True
+                        while i < len(dedup_window):
+                            wcue = dedup_window[i]
+                            wblock = webvtt.CueBlock.from_json(wcue)
+                            i += 1
+                            if wblock.hinges(block):
+                                wcue['end'] = block.end
+                                is_new = False
+                                continue
+                            if wblock == block:
+                                is_new = False
+                                continue
+                            if wblock.end > block.start:
+                                continue
+                            ready.append(wblock)
+                            i -= 1
+                            del dedup_window[i]
+
+                        if is_new:
+                            dedup_window.append(block.as_json)
+                        for block in ready:
+                            block.write_into(output)
+
+                        # we only emit cues once they fall out of the duplicate window
+                        continue
+                    elif isinstance(block, webvtt.Magic):
+                        # take care of MPEG PES timestamp overflow
+                        if block.mpegts is None:
+                            block.mpegts = 0
+                        extra_state.setdefault('webvtt_mpegts_adjust', 0)
+                        block.mpegts += extra_state['webvtt_mpegts_adjust'] << 33
+                        if block.mpegts < extra_state.get('webvtt_mpegts_last', 0):
+                            overflow = True
+                            block.mpegts += 1 << 33
+                        mpegts_last = block.mpegts
+
+                        if frag_index == 1:
+                            extra_state['webvtt_mpegts'] = block.mpegts or 0
+                            extra_state['webvtt_local'] = block.local or 0
+                            # XXX: block.local = block.mpegts = None ?
+                        else:
+                            if block.mpegts is not None and block.local is not None:
+                                adjust = (
+                                    (block.mpegts - extra_state.get('webvtt_mpegts', 0))
+                                    - (block.local - extra_state.get('webvtt_local', 0))
+                                )
+                            continue
+                    elif isinstance(block, webvtt.HeaderBlock):
+                        if frag_index != 1:
+                            # XXX: this should probably be silent as well
+                            # or verify that all segments contain the same data
+                            self.report_warning(bug_reports_message(
+                                'Discarding a %s block found in the middle of the stream; '
+                                'if the subtitles display incorrectly,'
+                                % (type(block).__name__)))
+                            continue
+                    block.write_into(output)
+
+                return output.getvalue().encode('utf-8')
+
+            def fin_fragments():
+                dedup_window = extra_state.get('webvtt_dedup_window')
+                if not dedup_window:
+                    return b''
+
+                output = io.StringIO()
+                for cue in dedup_window:
+                    webvtt.CueBlock.from_json(cue).write_into(output)
+
+                return output.getvalue().encode('utf-8')
+
+            self.download_and_append_fragments(
+                ctx, fragments, info_dict, pack_func=pack_fragment, finish_func=fin_fragments)
+        else:
+            return self.download_and_append_fragments(ctx, fragments, info_dict)

plugins/youtube_download/yt_dlp/downloader/http.py

@@ -0,0 +1,386 @@
+from __future__ import unicode_literals
+
+import errno
+import os
+import socket
+import time
+import random
+import re
+
+from .common import FileDownloader
+from ..compat import (
+    compat_str,
+    compat_urllib_error,
+)
+from ..utils import (
+    ContentTooShortError,
+    encodeFilename,
+    int_or_none,
+    sanitized_Request,
+    ThrottledDownload,
+    write_xattr,
+    XAttrMetadataError,
+    XAttrUnavailableError,
+)
+
+
+class HttpFD(FileDownloader):
+    def real_download(self, filename, info_dict):
+        url = info_dict['url']
+        request_data = info_dict.get('request_data', None)
+
+        class DownloadContext(dict):
+            __getattr__ = dict.get
+            __setattr__ = dict.__setitem__
+            __delattr__ = dict.__delitem__
+
+        ctx = DownloadContext()
+        ctx.filename = filename
+        ctx.tmpfilename = self.temp_name(filename)
+        ctx.stream = None
+
+        # Do not include the Accept-Encoding header
+        headers = {'Youtubedl-no-compression': 'True'}
+        add_headers = info_dict.get('http_headers')
+        if add_headers:
+            headers.update(add_headers)
+
+        is_test = self.params.get('test', False)
+        chunk_size = self._TEST_FILE_SIZE if is_test else (
+            self.params.get('http_chunk_size')
+            or info_dict.get('downloader_options', {}).get('http_chunk_size')
+            or 0)
+
+        ctx.open_mode = 'wb'
+        ctx.resume_len = 0
+        ctx.data_len = None
+        ctx.block_size = self.params.get('buffersize', 1024)
+        ctx.start_time = time.time()
+        ctx.chunk_size = None
+        throttle_start = None
+
+        if self.params.get('continuedl', True):
+            # Establish possible resume length
+            if os.path.isfile(encodeFilename(ctx.tmpfilename)):
+                ctx.resume_len = os.path.getsize(
+                    encodeFilename(ctx.tmpfilename))
+
+        ctx.is_resume = ctx.resume_len > 0
+
+        count = 0
+        retries = self.params.get('retries', 0)
+
+        class SucceedDownload(Exception):
+            pass
+
+        class RetryDownload(Exception):
+            def __init__(self, source_error):
+                self.source_error = source_error
+
+        class NextFragment(Exception):
+            pass
+
+        def set_range(req, start, end):
+            range_header = 'bytes=%d-' % start
+            if end:
+                range_header += compat_str(end)
+            req.add_header('Range', range_header)
+
+        def establish_connection():
+            ctx.chunk_size = (random.randint(int(chunk_size * 0.95), chunk_size)
+                              if not is_test and chunk_size else chunk_size)
+            if ctx.resume_len > 0:
+                range_start = ctx.resume_len
+                if ctx.is_resume:
+                    self.report_resuming_byte(ctx.resume_len)
+                ctx.open_mode = 'ab'
+            elif ctx.chunk_size > 0:
+                range_start = 0
+            else:
+                range_start = None
+            ctx.is_resume = False
+            range_end = range_start + ctx.chunk_size - 1 if ctx.chunk_size else None
+            if range_end and ctx.data_len is not None and range_end >= ctx.data_len:
+                range_end = ctx.data_len - 1
+            has_range = range_start is not None
+            ctx.has_range = has_range
+            request = sanitized_Request(url, request_data, headers)
+            if has_range:
+                set_range(request, range_start, range_end)
+            # Establish connection
+            try:
+                try:
+                    ctx.data = self.ydl.urlopen(request)
+                except (compat_urllib_error.URLError, ) as err:
+                    # reason may not be available, e.g. for urllib2.HTTPError on python 2.6
+                    reason = getattr(err, 'reason', None)
+                    if isinstance(reason, socket.timeout):
+                        raise RetryDownload(err)
+                    raise err
+                # When trying to resume, Content-Range HTTP header of response has to be checked
+                # to match the value of requested Range HTTP header. This is due to a webservers
+                # that don't support resuming and serve a whole file with no Content-Range
+                # set in response despite of requested Range (see
+                # https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
+                if has_range:
+                    content_range = ctx.data.headers.get('Content-Range')
+                    if content_range:
+                        content_range_m = re.search(r'bytes (\d+)-(\d+)?(?:/(\d+))?', content_range)
+                        # Content-Range is present and matches requested Range, resume is possible
+                        if content_range_m:
+                            if range_start == int(content_range_m.group(1)):
+                                content_range_end = int_or_none(content_range_m.group(2))
+                                content_len = int_or_none(content_range_m.group(3))
+                                accept_content_len = (
+                                    # Non-chunked download
+                                    not ctx.chunk_size
+                                    # Chunked download and requested piece or
+                                    # its part is promised to be served
+                                    or content_range_end == range_end
+                                    or content_len < range_end)
+                                if accept_content_len:
+                                    ctx.data_len = content_len
+                                    return
+                    # Content-Range is either not present or invalid. Assuming remote webserver is
+                    # trying to send the whole file, resume is not possible, so wiping the local file
+                    # and performing entire redownload
+                    self.report_unable_to_resume()
+                    ctx.resume_len = 0
+                    ctx.open_mode = 'wb'
+                ctx.data_len = int_or_none(ctx.data.info().get('Content-length', None))
+                return
+            except (compat_urllib_error.HTTPError, ) as err:
+                if err.code == 416:
+                    # Unable to resume (requested range not satisfiable)
+                    try:
+                        # Open the connection again without the range header
+                        ctx.data = self.ydl.urlopen(
+                            sanitized_Request(url, request_data, headers))
+                        content_length = ctx.data.info()['Content-Length']
+                    except (compat_urllib_error.HTTPError, ) as err:
+                        if err.code < 500 or err.code >= 600:
+                            raise
+                    else:
+                        # Examine the reported length
+                        if (content_length is not None
+                                and (ctx.resume_len - 100 < int(content_length) < ctx.resume_len + 100)):
+                            # The file had already been fully downloaded.
+                            # Explanation to the above condition: in issue #175 it was revealed that
+                            # YouTube sometimes adds or removes a few bytes from the end of the file,
+                            # changing the file size slightly and causing problems for some users. So
+                            # I decided to implement a suggested change and consider the file
+                            # completely downloaded if the file size differs less than 100 bytes from
+                            # the one in the hard drive.
+                            self.report_file_already_downloaded(ctx.filename)
+                            self.try_rename(ctx.tmpfilename, ctx.filename)
+                            self._hook_progress({
+                                'filename': ctx.filename,
+                                'status': 'finished',
+                                'downloaded_bytes': ctx.resume_len,
+                                'total_bytes': ctx.resume_len,
+                            }, info_dict)
+                            raise SucceedDownload()
+                        else:
+                            # The length does not match, we start the download over
+                            self.report_unable_to_resume()
+                            ctx.resume_len = 0
+                            ctx.open_mode = 'wb'
+                            return
+                elif err.code < 500 or err.code >= 600:
+                    # Unexpected HTTP error
+                    raise
+                raise RetryDownload(err)
+            except socket.timeout as err:
+                raise RetryDownload(err)
+            except socket.error as err:
+                if err.errno in (errno.ECONNRESET, errno.ETIMEDOUT):
+                    # Connection reset is no problem, just retry
+                    raise RetryDownload(err)
+                raise
+
+        def download():
+            nonlocal throttle_start
+            data_len = ctx.data.info().get('Content-length', None)
+
+            # Range HTTP header may be ignored/unsupported by a webserver
+            # (e.g. extractor/scivee.py, extractor/bambuser.py).
+            # However, for a test we still would like to download just a piece of a file.
+            # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
+            # block size when downloading a file.
+            if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
+                data_len = self._TEST_FILE_SIZE
+
+            if data_len is not None:
+                data_len = int(data_len) + ctx.resume_len
+                min_data_len = self.params.get('min_filesize')
+                max_data_len = self.params.get('max_filesize')
+                if min_data_len is not None and data_len < min_data_len:
+                    self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
+                    return False
+                if max_data_len is not None and data_len > max_data_len:
+                    self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
+                    return False
+
+            byte_counter = 0 + ctx.resume_len
+            block_size = ctx.block_size
+            start = time.time()
+
+            # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
+            now = None  # needed for slow_down() in the first loop run
+            before = start  # start measuring
+
+            def retry(e):
+                to_stdout = ctx.tmpfilename == '-'
+                if ctx.stream is not None:
+                    if not to_stdout:
+                        ctx.stream.close()
+                    ctx.stream = None
+                ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
+                raise RetryDownload(e)
+
+            while True:
+                try:
+                    # Download and write
+                    data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
+                # socket.timeout is a subclass of socket.error but may not have
+                # errno set
+                except socket.timeout as e:
+                    retry(e)
+                except socket.error as e:
+                    # SSLError on python 2 (inherits socket.error) may have
+                    # no errno set but this error message
+                    if e.errno in (errno.ECONNRESET, errno.ETIMEDOUT) or getattr(e, 'message', None) == 'The read operation timed out':
+                        retry(e)
+                    raise
+
+                byte_counter += len(data_block)
+
+                # exit loop when download is finished
+                if len(data_block) == 0:
+                    break
+
+                # Open destination file just in time
+                if ctx.stream is None:
+                    try:
+                        ctx.stream, ctx.tmpfilename = self.sanitize_open(
+                            ctx.tmpfilename, ctx.open_mode)
+                        assert ctx.stream is not None
+                        ctx.filename = self.undo_temp_name(ctx.tmpfilename)
+                        self.report_destination(ctx.filename)
+                    except (OSError, IOError) as err:
+                        self.report_error('unable to open for writing: %s' % str(err))
+                        return False
+
+                    if self.params.get('xattr_set_filesize', False) and data_len is not None:
+                        try:
+                            write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode('utf-8'))
+                        except (XAttrUnavailableError, XAttrMetadataError) as err:
+                            self.report_error('unable to set filesize xattr: %s' % str(err))
+
+                try:
+                    ctx.stream.write(data_block)
+                except (IOError, OSError) as err:
+                    self.to_stderr('\n')
+                    self.report_error('unable to write data: %s' % str(err))
+                    return False
+
+                # Apply rate limit
+                self.slow_down(start, now, byte_counter - ctx.resume_len)
+
+                # end measuring of one loop run
+                now = time.time()
+                after = now
+
+                # Adjust block size
+                if not self.params.get('noresizebuffer', False):
+                    block_size = self.best_block_size(after - before, len(data_block))
+
+                before = after
+
+                # Progress message
+                speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
+                if ctx.data_len is None:
+                    eta = None
+                else:
+                    eta = self.calc_eta(start, time.time(), ctx.data_len - ctx.resume_len, byte_counter - ctx.resume_len)
+
+                self._hook_progress({
+                    'status': 'downloading',
+                    'downloaded_bytes': byte_counter,
+                    'total_bytes': ctx.data_len,
+                    'tmpfilename': ctx.tmpfilename,
+                    'filename': ctx.filename,
+                    'eta': eta,
+                    'speed': speed,
+                    'elapsed': now - ctx.start_time,
+                    'ctx_id': info_dict.get('ctx_id'),
+                }, info_dict)
+
+                if data_len is not None and byte_counter == data_len:
+                    break
+
+                if speed and speed < (self.params.get('throttledratelimit') or 0):
+                    # The speed must stay below the limit for 3 seconds
+                    # This prevents raising error when the speed temporarily goes down
+                    if throttle_start is None:
+                        throttle_start = now
+                    elif now - throttle_start > 3:
+                        if ctx.stream is not None and ctx.tmpfilename != '-':
+                            ctx.stream.close()
+                        raise ThrottledDownload()
+                elif speed:
+                    throttle_start = None
+
+            if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len:
+                ctx.resume_len = byte_counter
+                # ctx.block_size = block_size
+                raise NextFragment()
+
+            if ctx.stream is None:
+                self.to_stderr('\n')
+                self.report_error('Did not get any data blocks')
+                return False
+            if ctx.tmpfilename != '-':
+                ctx.stream.close()
+
+            if data_len is not None and byte_counter != data_len:
+                err = ContentTooShortError(byte_counter, int(data_len))
+                if count <= retries:
+                    retry(err)
+                raise err
+
+            self.try_rename(ctx.tmpfilename, ctx.filename)
+
+            # Update file modification time
+            if self.params.get('updatetime', True):
+                info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))
+
+            self._hook_progress({
+                'downloaded_bytes': byte_counter,
+                'total_bytes': byte_counter,
+                'filename': ctx.filename,
+                'status': 'finished',
+                'elapsed': time.time() - ctx.start_time,
+                'ctx_id': info_dict.get('ctx_id'),
+            }, info_dict)
+
+            return True
+
+        while count <= retries:
+            try:
+                establish_connection()
+                return download()
+            except RetryDownload as e:
+                count += 1
+                if count <= retries:
+                    self.report_retry(e.source_error, count, retries)
+                else:
+                    self.to_screen(f'[download] Got server HTTP error: {e.source_error}')
+                continue
+            except NextFragment:
+                continue
+            except SucceedDownload:
+                return True
+
+        self.report_error('giving up after %s retries' % retries)
+        return False

plugins/youtube_download/yt_dlp/downloader/ism.py

@@ -0,0 +1,289 @@
+from __future__ import unicode_literals
+
+import time
+import binascii
+import io
+
+from .fragment import FragmentFD
+from ..compat import (
+    compat_Struct,
+    compat_urllib_error,
+)
+
+
+u8 = compat_Struct('>B')
+u88 = compat_Struct('>Bx')
+u16 = compat_Struct('>H')
+u1616 = compat_Struct('>Hxx')
+u32 = compat_Struct('>I')
+u64 = compat_Struct('>Q')
+
+s88 = compat_Struct('>bx')
+s16 = compat_Struct('>h')
+s1616 = compat_Struct('>hxx')
+s32 = compat_Struct('>i')
+
+unity_matrix = (s32.pack(0x10000) + s32.pack(0) * 3) * 2 + s32.pack(0x40000000)
+
+TRACK_ENABLED = 0x1
+TRACK_IN_MOVIE = 0x2
+TRACK_IN_PREVIEW = 0x4
+
+SELF_CONTAINED = 0x1
+
+
+def box(box_type, payload):
+    return u32.pack(8 + len(payload)) + box_type + payload
+
+
+def full_box(box_type, version, flags, payload):
+    return box(box_type, u8.pack(version) + u32.pack(flags)[1:] + payload)
+
+
+def write_piff_header(stream, params):
+    track_id = params['track_id']
+    fourcc = params['fourcc']
+    duration = params['duration']
+    timescale = params.get('timescale', 10000000)
+    language = params.get('language', 'und')
+    height = params.get('height', 0)
+    width = params.get('width', 0)
+    stream_type = params['stream_type']
+    creation_time = modification_time = int(time.time())
+
+    ftyp_payload = b'isml'  # major brand
+    ftyp_payload += u32.pack(1)  # minor version
+    ftyp_payload += b'piff' + b'iso2'  # compatible brands
+    stream.write(box(b'ftyp', ftyp_payload))  # File Type Box
+
+    mvhd_payload = u64.pack(creation_time)
+    mvhd_payload += u64.pack(modification_time)
+    mvhd_payload += u32.pack(timescale)
+    mvhd_payload += u64.pack(duration)
+    mvhd_payload += s1616.pack(1)  # rate
+    mvhd_payload += s88.pack(1)  # volume
+    mvhd_payload += u16.pack(0)  # reserved
+    mvhd_payload += u32.pack(0) * 2  # reserved
+    mvhd_payload += unity_matrix
+    mvhd_payload += u32.pack(0) * 6  # pre defined
+    mvhd_payload += u32.pack(0xffffffff)  # next track id
+    moov_payload = full_box(b'mvhd', 1, 0, mvhd_payload)  # Movie Header Box
+
+    tkhd_payload = u64.pack(creation_time)
+    tkhd_payload += u64.pack(modification_time)
+    tkhd_payload += u32.pack(track_id)  # track id
+    tkhd_payload += u32.pack(0)  # reserved
+    tkhd_payload += u64.pack(duration)
+    tkhd_payload += u32.pack(0) * 2  # reserved
+    tkhd_payload += s16.pack(0)  # layer
+    tkhd_payload += s16.pack(0)  # alternate group
+    tkhd_payload += s88.pack(1 if stream_type == 'audio' else 0)  # volume
+    tkhd_payload += u16.pack(0)  # reserved
+    tkhd_payload += unity_matrix
+    tkhd_payload += u1616.pack(width)
+    tkhd_payload += u1616.pack(height)
+    trak_payload = full_box(b'tkhd', 1, TRACK_ENABLED | TRACK_IN_MOVIE | TRACK_IN_PREVIEW, tkhd_payload)  # Track Header Box
+
+    mdhd_payload = u64.pack(creation_time)
+    mdhd_payload += u64.pack(modification_time)
+    mdhd_payload += u32.pack(timescale)
+    mdhd_payload += u64.pack(duration)
+    mdhd_payload += u16.pack(((ord(language[0]) - 0x60) << 10) | ((ord(language[1]) - 0x60) << 5) | (ord(language[2]) - 0x60))
+    mdhd_payload += u16.pack(0)  # pre defined
+    mdia_payload = full_box(b'mdhd', 1, 0, mdhd_payload)  # Media Header Box
+
+    hdlr_payload = u32.pack(0)  # pre defined
+    if stream_type == 'audio':  # handler type
+        hdlr_payload += b'soun'
+        hdlr_payload += u32.pack(0) * 3  # reserved
+        hdlr_payload += b'SoundHandler\0'  # name
+    elif stream_type == 'video':
+        hdlr_payload += b'vide'
+        hdlr_payload += u32.pack(0) * 3  # reserved
+        hdlr_payload += b'VideoHandler\0'  # name
+    elif stream_type == 'text':
+        hdlr_payload += b'subt'
+        hdlr_payload += u32.pack(0) * 3  # reserved
+        hdlr_payload += b'SubtitleHandler\0'  # name
+    else:
+        assert False
+    mdia_payload += full_box(b'hdlr', 0, 0, hdlr_payload)  # Handler Reference Box
+
+    if stream_type == 'audio':
+        smhd_payload = s88.pack(0)  # balance
+        smhd_payload += u16.pack(0)  # reserved
+        media_header_box = full_box(b'smhd', 0, 0, smhd_payload)  # Sound Media Header
+    elif stream_type == 'video':
+        vmhd_payload = u16.pack(0)  # graphics mode
+        vmhd_payload += u16.pack(0) * 3  # opcolor
+        media_header_box = full_box(b'vmhd', 0, 1, vmhd_payload)  # Video Media Header
+    elif stream_type == 'text':
+        media_header_box = full_box(b'sthd', 0, 0, b'')  # Subtitle Media Header
+    else:
+        assert False
+    minf_payload = media_header_box
+
+    dref_payload = u32.pack(1)  # entry count
+    dref_payload += full_box(b'url ', 0, SELF_CONTAINED, b'')  # Data Entry URL Box
+    dinf_payload = full_box(b'dref', 0, 0, dref_payload)  # Data Reference Box
+    minf_payload += box(b'dinf', dinf_payload)  # Data Information Box
+
+    stsd_payload = u32.pack(1)  # entry count
+
+    sample_entry_payload = u8.pack(0) * 6  # reserved
+    sample_entry_payload += u16.pack(1)  # data reference index
+    if stream_type == 'audio':
+        sample_entry_payload += u32.pack(0) * 2  # reserved
+        sample_entry_payload += u16.pack(params.get('channels', 2))
+        sample_entry_payload += u16.pack(params.get('bits_per_sample', 16))
+        sample_entry_payload += u16.pack(0)  # pre defined
+        sample_entry_payload += u16.pack(0)  # reserved
+        sample_entry_payload += u1616.pack(params['sampling_rate'])
+
+        if fourcc == 'AACL':
+            sample_entry_box = box(b'mp4a', sample_entry_payload)
+    elif stream_type == 'video':
+        sample_entry_payload += u16.pack(0)  # pre defined
+        sample_entry_payload += u16.pack(0)  # reserved
+        sample_entry_payload += u32.pack(0) * 3  # pre defined
+        sample_entry_payload += u16.pack(width)
+        sample_entry_payload += u16.pack(height)
+        sample_entry_payload += u1616.pack(0x48)  # horiz resolution 72 dpi
+        sample_entry_payload += u1616.pack(0x48)  # vert resolution 72 dpi
+        sample_entry_payload += u32.pack(0)  # reserved
+        sample_entry_payload += u16.pack(1)  # frame count
+        sample_entry_payload += u8.pack(0) * 32  # compressor name
+        sample_entry_payload += u16.pack(0x18)  # depth
+        sample_entry_payload += s16.pack(-1)  # pre defined
+
+        codec_private_data = binascii.unhexlify(params['codec_private_data'].encode('utf-8'))
+        if fourcc in ('H264', 'AVC1'):
+            sps, pps = codec_private_data.split(u32.pack(1))[1:]
+            avcc_payload = u8.pack(1)  # configuration version
+            avcc_payload += sps[1:4]  # avc profile indication + profile compatibility + avc level indication
+            avcc_payload += u8.pack(0xfc | (params.get('nal_unit_length_field', 4) - 1))  # complete representation (1) + reserved (11111) + length size minus one
+            avcc_payload += u8.pack(1)  # reserved (0) + number of sps (0000001)
+            avcc_payload += u16.pack(len(sps))
+            avcc_payload += sps
+            avcc_payload += u8.pack(1)  # number of pps
+            avcc_payload += u16.pack(len(pps))
+            avcc_payload += pps
+            sample_entry_payload += box(b'avcC', avcc_payload)  # AVC Decoder Configuration Record
+            sample_entry_box = box(b'avc1', sample_entry_payload)  # AVC Simple Entry
+        else:
+            assert False
+    elif stream_type == 'text':
+        if fourcc == 'TTML':
+            sample_entry_payload += b'http://www.w3.org/ns/ttml\0'  # namespace
+            sample_entry_payload += b'\0'  # schema location
+            sample_entry_payload += b'\0'  # auxilary mime types(??)
+            sample_entry_box = box(b'stpp', sample_entry_payload)
+        else:
+            assert False
+    else:
+        assert False
+    stsd_payload += sample_entry_box
+
+    stbl_payload = full_box(b'stsd', 0, 0, stsd_payload)  # Sample Description Box
+
+    stts_payload = u32.pack(0)  # entry count
+    stbl_payload += full_box(b'stts', 0, 0, stts_payload)  # Decoding Time to Sample Box
+
+    stsc_payload = u32.pack(0)  # entry count
+    stbl_payload += full_box(b'stsc', 0, 0, stsc_payload)  # Sample To Chunk Box
+
+    stco_payload = u32.pack(0)  # entry count
+    stbl_payload += full_box(b'stco', 0, 0, stco_payload)  # Chunk Offset Box
+
+    minf_payload += box(b'stbl', stbl_payload)  # Sample Table Box
+
+    mdia_payload += box(b'minf', minf_payload)  # Media Information Box
+
+    trak_payload += box(b'mdia', mdia_payload)  # Media Box
+
+    moov_payload += box(b'trak', trak_payload)  # Track Box
+
+    mehd_payload = u64.pack(duration)
+    mvex_payload = full_box(b'mehd', 1, 0, mehd_payload)  # Movie Extends Header Box
+
+    trex_payload = u32.pack(track_id)  # track id
+    trex_payload += u32.pack(1)  # default sample description index
+    trex_payload += u32.pack(0)  # default sample duration
+    trex_payload += u32.pack(0)  # default sample size
+    trex_payload += u32.pack(0)  # default sample flags
+    mvex_payload += full_box(b'trex', 0, 0, trex_payload)  # Track Extends Box
+
+    moov_payload += box(b'mvex', mvex_payload)  # Movie Extends Box
+    stream.write(box(b'moov', moov_payload))  # Movie Box
+
+
+def extract_box_data(data, box_sequence):
+    data_reader = io.BytesIO(data)
+    while True:
+        box_size = u32.unpack(data_reader.read(4))[0]
+        box_type = data_reader.read(4)
+        if box_type == box_sequence[0]:
+            box_data = data_reader.read(box_size - 8)
+            if len(box_sequence) == 1:
+                return box_data
+            return extract_box_data(box_data, box_sequence[1:])
+        data_reader.seek(box_size - 8, 1)
+
+
+class IsmFD(FragmentFD):
+    """
+    Download segments in a ISM manifest
+    """
+
+    FD_NAME = 'ism'
+
+    def real_download(self, filename, info_dict):
+        segments = info_dict['fragments'][:1] if self.params.get(
+            'test', False) else info_dict['fragments']
+
+        ctx = {
+            'filename': filename,
+            'total_frags': len(segments),
+        }
+
+        self._prepare_and_start_frag_download(ctx, info_dict)
+
+        extra_state = ctx.setdefault('extra_state', {
+            'ism_track_written': False,
+        })
+
+        fragment_retries = self.params.get('fragment_retries', 0)
+        skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
+
+        frag_index = 0
+        for i, segment in enumerate(segments):
+            frag_index += 1
+            if frag_index <= ctx['fragment_index']:
+                continue
+            count = 0
+            while count <= fragment_retries:
+                try:
+                    success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
+                    if not success:
+                        return False
+                    if not extra_state['ism_track_written']:
+                        tfhd_data = extract_box_data(frag_content, [b'moof', b'traf', b'tfhd'])
+                        info_dict['_download_params']['track_id'] = u32.unpack(tfhd_data[4:8])[0]
+                        write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
+                        extra_state['ism_track_written'] = True
+                    self._append_fragment(ctx, frag_content)
+                    break
+                except compat_urllib_error.HTTPError as err:
+                    count += 1
+                    if count <= fragment_retries:
+                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
+            if count > fragment_retries:
+                if skip_unavailable_fragments:
+                    self.report_skip_fragment(frag_index)
+                    continue
+                self.report_error('giving up after %s fragment retries' % fragment_retries)
+                return False
+
+        self._finish_frag_download(ctx, info_dict)
+
+        return True

plugins/youtube_download/yt_dlp/downloader/mhtml.py

@@ -0,0 +1,202 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import io
+import quopri
+import re
+import uuid
+
+from .fragment import FragmentFD
+from ..utils import (
+    escapeHTML,
+    formatSeconds,
+    srt_subtitles_timecode,
+    urljoin,
+)
+from ..version import __version__ as YT_DLP_VERSION
+
+
+class MhtmlFD(FragmentFD):
+    FD_NAME = 'mhtml'
+
+    _STYLESHEET = """\
+html, body {
+    margin: 0;
+    padding: 0;
+    height: 100vh;
+}
+
+html {
+    overflow-y: scroll;
+    scroll-snap-type: y mandatory;
+}
+
+body {
+    scroll-snap-type: y mandatory;
+    display: flex;
+    flex-flow: column;
+}
+
+body > figure {
+    max-width: 100vw;
+    max-height: 100vh;
+    scroll-snap-align: center;
+}
+
+body > figure > figcaption {
+    text-align: center;
+    height: 2.5em;
+}
+
+body > figure > img {
+    display: block;
+    margin: auto;
+    max-width: 100%;
+    max-height: calc(100vh - 5em);
+}
+"""
+    _STYLESHEET = re.sub(r'\s+', ' ', _STYLESHEET)
+    _STYLESHEET = re.sub(r'\B \B|(?<=[\w\-]) (?=[^\w\-])|(?<=[^\w\-]) (?=[\w\-])', '', _STYLESHEET)
+
+    @staticmethod
+    def _escape_mime(s):
+        return '=?utf-8?Q?' + (b''.join(
+            bytes((b,)) if b >= 0x20 else b'=%02X' % b
+            for b in quopri.encodestring(s.encode('utf-8'), header=True)
+        )).decode('us-ascii') + '?='
+
+    def _gen_cid(self, i, fragment, frag_boundary):
+        return '%u.%s@yt-dlp.github.io.invalid' % (i, frag_boundary)
+
+    def _gen_stub(self, *, fragments, frag_boundary, title):
+        output = io.StringIO()
+
+        output.write((
+            '<!DOCTYPE html>'
+            '<html>'
+            '<head>'
+            ''  '<meta name="generator" content="yt-dlp {version}">'
+            ''  '<title>{title}</title>'
+            ''  '<style>{styles}</style>'
+            '<body>'
+        ).format(
+            version=escapeHTML(YT_DLP_VERSION),
+            styles=self._STYLESHEET,
+            title=escapeHTML(title)
+        ))
+
+        t0 = 0
+        for i, frag in enumerate(fragments):
+            output.write('<figure>')
+            try:
+                t1 = t0 + frag['duration']
+                output.write((
+                    '<figcaption>Slide #{num}: {t0} – {t1} (duration: {duration})</figcaption>'
+                ).format(
+                    num=i + 1,
+                    t0=srt_subtitles_timecode(t0),
+                    t1=srt_subtitles_timecode(t1),
+                    duration=formatSeconds(frag['duration'], msec=True)
+                ))
+            except (KeyError, ValueError, TypeError):
+                t1 = None
+                output.write((
+                    '<figcaption>Slide #{num}</figcaption>'
+                ).format(num=i + 1))
+            output.write('<img src="cid:{cid}">'.format(
+                cid=self._gen_cid(i, frag, frag_boundary)))
+            output.write('</figure>')
+            t0 = t1
+
+        return output.getvalue()
+
+    def real_download(self, filename, info_dict):
+        fragment_base_url = info_dict.get('fragment_base_url')
+        fragments = info_dict['fragments'][:1] if self.params.get(
+            'test', False) else info_dict['fragments']
+        title = info_dict.get('title', info_dict['format_id'])
+        origin = info_dict.get('webpage_url', info_dict['url'])
+
+        ctx = {
+            'filename': filename,
+            'total_frags': len(fragments),
+        }
+
+        self._prepare_and_start_frag_download(ctx, info_dict)
+
+        extra_state = ctx.setdefault('extra_state', {
+            'header_written': False,
+            'mime_boundary': str(uuid.uuid4()).replace('-', ''),
+        })
+
+        frag_boundary = extra_state['mime_boundary']
+
+        if not extra_state['header_written']:
+            stub = self._gen_stub(
+                fragments=fragments,
+                frag_boundary=frag_boundary,
+                title=title
+            )
+
+            ctx['dest_stream'].write((
+                'MIME-Version: 1.0\r\n'
+                'From: <nowhere@yt-dlp.github.io.invalid>\r\n'
+                'To: <nowhere@yt-dlp.github.io.invalid>\r\n'
+                'Subject: {title}\r\n'
+                'Content-type: multipart/related; '
+                ''  'boundary="{boundary}"; '
+                ''  'type="text/html"\r\n'
+                'X.yt-dlp.Origin: {origin}\r\n'
+                '\r\n'
+                '--{boundary}\r\n'
+                'Content-Type: text/html; charset=utf-8\r\n'
+                'Content-Length: {length}\r\n'
+                '\r\n'
+                '{stub}\r\n'
+            ).format(
+                origin=origin,
+                boundary=frag_boundary,
+                length=len(stub),
+                title=self._escape_mime(title),
+                stub=stub
+            ).encode('utf-8'))
+            extra_state['header_written'] = True
+
+        for i, fragment in enumerate(fragments):
+            if (i + 1) <= ctx['fragment_index']:
+                continue
+
+            fragment_url = urljoin(fragment_base_url, fragment['path'])
+            success, frag_content = self._download_fragment(ctx, fragment_url, info_dict)
+            if not success:
+                continue
+
+            mime_type = b'image/jpeg'
+            if frag_content.startswith(b'\x89PNG\r\n\x1a\n'):
+                mime_type = b'image/png'
+            if frag_content.startswith((b'GIF87a', b'GIF89a')):
+                mime_type = b'image/gif'
+            if frag_content.startswith(b'RIFF') and frag_content[8:12] == 'WEBP':
+                mime_type = b'image/webp'
+
+            frag_header = io.BytesIO()
+            frag_header.write(
+                b'--%b\r\n' % frag_boundary.encode('us-ascii'))
+            frag_header.write(
+                b'Content-ID: <%b>\r\n' % self._gen_cid(i, fragment, frag_boundary).encode('us-ascii'))
+            frag_header.write(
+                b'Content-type: %b\r\n' % mime_type)
+            frag_header.write(
+                b'Content-length: %u\r\n' % len(frag_content))
+            frag_header.write(
+                b'Content-location: %b\r\n' % fragment_url.encode('us-ascii'))
+            frag_header.write(
+                b'X.yt-dlp.Duration: %f\r\n' % fragment['duration'])
+            frag_header.write(b'\r\n')
+            self._append_fragment(
+                ctx, frag_header.getvalue() + frag_content + b'\r\n')
+
+        ctx['dest_stream'].write(
+            b'--%b--\r\n\r\n' % frag_boundary.encode('us-ascii'))
+        self._finish_frag_download(ctx, info_dict)
+        return True

plugins/youtube_download/yt_dlp/downloader/niconico.py

@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import threading
+
+from .common import FileDownloader
+from ..downloader import get_suitable_downloader
+from ..extractor.niconico import NiconicoIE
+from ..utils import sanitized_Request
+
+
+class NiconicoDmcFD(FileDownloader):
+    """ Downloading niconico douga from DMC with heartbeat """
+
+    FD_NAME = 'niconico_dmc'
+
+    def real_download(self, filename, info_dict):
+        self.to_screen('[%s] Downloading from DMC' % self.FD_NAME)
+
+        ie = NiconicoIE(self.ydl)
+        info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
+
+        fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params)
+
+        success = download_complete = False
+        timer = [None]
+        heartbeat_lock = threading.Lock()
+        heartbeat_url = heartbeat_info_dict['url']
+        heartbeat_data = heartbeat_info_dict['data'].encode()
+        heartbeat_interval = heartbeat_info_dict.get('interval', 30)
+
+        request = sanitized_Request(heartbeat_url, heartbeat_data)
+
+        def heartbeat():
+            try:
+                self.ydl.urlopen(request).read()
+            except Exception:
+                self.to_screen('[%s] Heartbeat failed' % self.FD_NAME)
+
+            with heartbeat_lock:
+                if not download_complete:
+                    timer[0] = threading.Timer(heartbeat_interval, heartbeat)
+                    timer[0].start()
+
+        heartbeat_info_dict['ping']()
+        self.to_screen('[%s] Heartbeat with %d second interval ...' % (self.FD_NAME, heartbeat_interval))
+        try:
+            heartbeat()
+            if type(fd).__name__ == 'HlsFD':
+                info_dict.update(ie._extract_m3u8_formats(info_dict['url'], info_dict['id'])[0])
+            success = fd.real_download(filename, info_dict)
+        finally:
+            if heartbeat_lock:
+                with heartbeat_lock:
+                    timer[0].cancel()
+                    download_complete = True
+            return success

plugins/youtube_download/yt_dlp/downloader/rtmp.py

@@ -0,0 +1,217 @@
+from __future__ import unicode_literals
+
+import os
+import re
+import subprocess
+import time
+
+from .common import FileDownloader
+from ..compat import compat_str
+from ..utils import (
+    check_executable,
+    encodeFilename,
+    encodeArgument,
+    get_exe_version,
+    Popen,
+)
+
+
+def rtmpdump_version():
+    return get_exe_version(
+        'rtmpdump', ['--help'], r'(?i)RTMPDump\s*v?([0-9a-zA-Z._-]+)')
+
+
+class RtmpFD(FileDownloader):
+    def real_download(self, filename, info_dict):
+        def run_rtmpdump(args):
+            start = time.time()
+            resume_percent = None
+            resume_downloaded_data_len = None
+            proc = Popen(args, stderr=subprocess.PIPE)
+            cursor_in_new_line = True
+            proc_stderr_closed = False
+            try:
+                while not proc_stderr_closed:
+                    # read line from stderr
+                    line = ''
+                    while True:
+                        char = proc.stderr.read(1)
+                        if not char:
+                            proc_stderr_closed = True
+                            break
+                        if char in [b'\r', b'\n']:
+                            break
+                        line += char.decode('ascii', 'replace')
+                    if not line:
+                        # proc_stderr_closed is True
+                        continue
+                    mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line)
+                    if mobj:
+                        downloaded_data_len = int(float(mobj.group(1)) * 1024)
+                        percent = float(mobj.group(2))
+                        if not resume_percent:
+                            resume_percent = percent
+                            resume_downloaded_data_len = downloaded_data_len
+                        time_now = time.time()
+                        eta = self.calc_eta(start, time_now, 100 - resume_percent, percent - resume_percent)
+                        speed = self.calc_speed(start, time_now, downloaded_data_len - resume_downloaded_data_len)
+                        data_len = None
+                        if percent > 0:
+                            data_len = int(downloaded_data_len * 100 / percent)
+                        self._hook_progress({
+                            'status': 'downloading',
+                            'downloaded_bytes': downloaded_data_len,
+                            'total_bytes_estimate': data_len,
+                            'tmpfilename': tmpfilename,
+                            'filename': filename,
+                            'eta': eta,
+                            'elapsed': time_now - start,
+                            'speed': speed,
+                        }, info_dict)
+                        cursor_in_new_line = False
+                    else:
+                        # no percent for live streams
+                        mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line)
+                        if mobj:
+                            downloaded_data_len = int(float(mobj.group(1)) * 1024)
+                            time_now = time.time()
+                            speed = self.calc_speed(start, time_now, downloaded_data_len)
+                            self._hook_progress({
+                                'downloaded_bytes': downloaded_data_len,
+                                'tmpfilename': tmpfilename,
+                                'filename': filename,
+                                'status': 'downloading',
+                                'elapsed': time_now - start,
+                                'speed': speed,
+                            }, info_dict)
+                            cursor_in_new_line = False
+                        elif self.params.get('verbose', False):
+                            if not cursor_in_new_line:
+                                self.to_screen('')
+                            cursor_in_new_line = True
+                            self.to_screen('[rtmpdump] ' + line)
+                if not cursor_in_new_line:
+                    self.to_screen('')
+                return proc.wait()
+            except BaseException:  # Including KeyboardInterrupt
+                proc.kill()
+                proc.wait()
+                raise
+
+        url = info_dict['url']
+        player_url = info_dict.get('player_url')
+        page_url = info_dict.get('page_url')
+        app = info_dict.get('app')
+        play_path = info_dict.get('play_path')
+        tc_url = info_dict.get('tc_url')
+        flash_version = info_dict.get('flash_version')
+        live = info_dict.get('rtmp_live', False)
+        conn = info_dict.get('rtmp_conn')
+        protocol = info_dict.get('rtmp_protocol')
+        real_time = info_dict.get('rtmp_real_time', False)
+        no_resume = info_dict.get('no_resume', False)
+        continue_dl = self.params.get('continuedl', True)
+
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+        test = self.params.get('test', False)
+
+        # Check for rtmpdump first
+        if not check_executable('rtmpdump', ['-h']):
+            self.report_error('RTMP download detected but "rtmpdump" could not be run. Please install')
+            return False
+
+        # Download using rtmpdump. rtmpdump returns exit code 2 when
+        # the connection was interrupted and resuming appears to be
+        # possible. This is part of rtmpdump's normal usage, AFAIK.
+        basic_args = [
+            'rtmpdump', '--verbose', '-r', url,
+            '-o', tmpfilename]
+        if player_url is not None:
+            basic_args += ['--swfVfy', player_url]
+        if page_url is not None:
+            basic_args += ['--pageUrl', page_url]
+        if app is not None:
+            basic_args += ['--app', app]
+        if play_path is not None:
+            basic_args += ['--playpath', play_path]
+        if tc_url is not None:
+            basic_args += ['--tcUrl', tc_url]
+        if test:
+            basic_args += ['--stop', '1']
+        if flash_version is not None:
+            basic_args += ['--flashVer', flash_version]
+        if live:
+            basic_args += ['--live']
+        if isinstance(conn, list):
+            for entry in conn:
+                basic_args += ['--conn', entry]
+        elif isinstance(conn, compat_str):
+            basic_args += ['--conn', conn]
+        if protocol is not None:
+            basic_args += ['--protocol', protocol]
+        if real_time:
+            basic_args += ['--realtime']
+
+        args = basic_args
+        if not no_resume and continue_dl and not live:
+            args += ['--resume']
+        if not live and continue_dl:
+            args += ['--skip', '1']
+
+        args = [encodeArgument(a) for a in args]
+
+        self._debug_cmd(args, exe='rtmpdump')
+
+        RD_SUCCESS = 0
+        RD_FAILED = 1
+        RD_INCOMPLETE = 2
+        RD_NO_CONNECT = 3
+
+        started = time.time()
+
+        try:
+            retval = run_rtmpdump(args)
+        except KeyboardInterrupt:
+            if not info_dict.get('is_live'):
+                raise
+            retval = RD_SUCCESS
+            self.to_screen('\n[rtmpdump] Interrupted by user')
+
+        if retval == RD_NO_CONNECT:
+            self.report_error('[rtmpdump] Could not connect to RTMP server.')
+            return False
+
+        while retval in (RD_INCOMPLETE, RD_FAILED) and not test and not live:
+            prevsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen('[rtmpdump] Downloaded %s bytes' % prevsize)
+            time.sleep(5.0)  # This seems to be needed
+            args = basic_args + ['--resume']
+            if retval == RD_FAILED:
+                args += ['--skip', '1']
+            args = [encodeArgument(a) for a in args]
+            retval = run_rtmpdump(args)
+            cursize = os.path.getsize(encodeFilename(tmpfilename))
+            if prevsize == cursize and retval == RD_FAILED:
+                break
+            # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
+            if prevsize == cursize and retval == RD_INCOMPLETE and cursize > 1024:
+                self.to_screen('[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
+                retval = RD_SUCCESS
+                break
+        if retval == RD_SUCCESS or (test and retval == RD_INCOMPLETE):
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen('[rtmpdump] Downloaded %s bytes' % fsize)
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+                'elapsed': time.time() - started,
+            }, info_dict)
+            return True
+        else:
+            self.to_stderr('\n')
+            self.report_error('rtmpdump exited with code %d' % retval)
+            return False

plugins/youtube_download/yt_dlp/downloader/rtsp.py

@@ -0,0 +1,47 @@
+from __future__ import unicode_literals
+
+import os
+import subprocess
+
+from .common import FileDownloader
+from ..utils import (
+    check_executable,
+    encodeFilename,
+)
+
+
+class RtspFD(FileDownloader):
+    def real_download(self, filename, info_dict):
+        url = info_dict['url']
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        if check_executable('mplayer', ['-h']):
+            args = [
+                'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
+                '-dumpstream', '-dumpfile', tmpfilename, url]
+        elif check_executable('mpv', ['-h']):
+            args = [
+                'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
+        else:
+            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install one')
+            return False
+
+        self._debug_cmd(args)
+
+        retval = subprocess.call(args)
+        if retval == 0:
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            }, info_dict)
+            return True
+        else:
+            self.to_stderr('\n')
+            self.report_error('%s exited with code %d' % (args[0], retval))
+            return False

plugins/youtube_download/yt_dlp/downloader/websocket.py

@@ -0,0 +1,62 @@
+import os
+import signal
+import asyncio
+import threading
+
+try:
+    import websockets
+except (ImportError, SyntaxError):
+    # websockets 3.10 on python 3.6 causes SyntaxError
+    # See https://github.com/yt-dlp/yt-dlp/issues/2633
+    has_websockets = False
+else:
+    has_websockets = True
+
+from .common import FileDownloader
+from .external import FFmpegFD
+
+
+class FFmpegSinkFD(FileDownloader):
+    """ A sink to ffmpeg for downloading fragments in any form """
+
+    def real_download(self, filename, info_dict):
+        info_copy = info_dict.copy()
+        info_copy['url'] = '-'
+
+        async def call_conn(proc, stdin):
+            try:
+                await self.real_connection(stdin, info_dict)
+            except (BrokenPipeError, OSError):
+                pass
+            finally:
+                try:
+                    stdin.flush()
+                    stdin.close()
+                except OSError:
+                    pass
+                os.kill(os.getpid(), signal.SIGINT)
+
+        class FFmpegStdinFD(FFmpegFD):
+            @classmethod
+            def get_basename(cls):
+                return FFmpegFD.get_basename()
+
+            def on_process_started(self, proc, stdin):
+                thread = threading.Thread(target=asyncio.run, daemon=True, args=(call_conn(proc, stdin), ))
+                thread.start()
+
+        return FFmpegStdinFD(self.ydl, self.params or {}).download(filename, info_copy)
+
+    async def real_connection(self, sink, info_dict):
+        """ Override this in subclasses """
+        raise NotImplementedError('This method must be implemented by subclasses')
+
+
+class WebSocketFragmentFD(FFmpegSinkFD):
+    async def real_connection(self, sink, info_dict):
+        async with websockets.connect(info_dict['url'], extra_headers=info_dict.get('http_headers', {})) as ws:
+            while True:
+                recv = await ws.recv()
+                if isinstance(recv, str):
+                    recv = recv.encode('utf8')
+                sink.write(recv)

plugins/youtube_download/yt_dlp/downloader/youtube_live_chat.py

@@ -0,0 +1,236 @@
+from __future__ import division, unicode_literals
+
+import json
+import time
+
+from .fragment import FragmentFD
+from ..compat import compat_urllib_error
+from ..utils import (
+    try_get,
+    dict_get,
+    int_or_none,
+    RegexNotFoundError,
+)
+from ..extractor.youtube import YoutubeBaseInfoExtractor as YT_BaseIE
+
+
+class YoutubeLiveChatFD(FragmentFD):
+    """ Downloads YouTube live chats fragment by fragment """
+
+    FD_NAME = 'youtube_live_chat'
+
+    def real_download(self, filename, info_dict):
+        video_id = info_dict['video_id']
+        self.to_screen('[%s] Downloading live chat' % self.FD_NAME)
+
+        fragment_retries = self.params.get('fragment_retries', 0)
+        test = self.params.get('test', False)
+
+        ctx = {
+            'filename': filename,
+            'live': True,
+            'total_frags': None,
+        }
+
+        ie = YT_BaseIE(self.ydl)
+
+        start_time = int(time.time() * 1000)
+
+        def dl_fragment(url, data=None, headers=None):
+            http_headers = info_dict.get('http_headers', {})
+            if headers:
+                http_headers = http_headers.copy()
+                http_headers.update(headers)
+            return self._download_fragment(ctx, url, info_dict, http_headers, data)
+
+        def parse_actions_replay(live_chat_continuation):
+            offset = continuation_id = click_tracking_params = None
+            processed_fragment = bytearray()
+            for action in live_chat_continuation.get('actions', []):
+                if 'replayChatItemAction' in action:
+                    replay_chat_item_action = action['replayChatItemAction']
+                    offset = int(replay_chat_item_action['videoOffsetTimeMsec'])
+                processed_fragment.extend(
+                    json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n')
+            if offset is not None:
+                continuation = try_get(
+                    live_chat_continuation,
+                    lambda x: x['continuations'][0]['liveChatReplayContinuationData'], dict)
+                if continuation:
+                    continuation_id = continuation.get('continuation')
+                    click_tracking_params = continuation.get('clickTrackingParams')
+            self._append_fragment(ctx, processed_fragment)
+            return continuation_id, offset, click_tracking_params
+
+        def try_refresh_replay_beginning(live_chat_continuation):
+            # choose the second option that contains the unfiltered live chat replay
+            refresh_continuation = try_get(
+                live_chat_continuation,
+                lambda x: x['header']['liveChatHeaderRenderer']['viewSelector']['sortFilterSubMenuRenderer']['subMenuItems'][1]['continuation']['reloadContinuationData'], dict)
+            if refresh_continuation:
+                # no data yet but required to call _append_fragment
+                self._append_fragment(ctx, b'')
+                refresh_continuation_id = refresh_continuation.get('continuation')
+                offset = 0
+                click_tracking_params = refresh_continuation.get('trackingParams')
+                return refresh_continuation_id, offset, click_tracking_params
+            return parse_actions_replay(live_chat_continuation)
+
+        live_offset = 0
+
+        def parse_actions_live(live_chat_continuation):
+            nonlocal live_offset
+            continuation_id = click_tracking_params = None
+            processed_fragment = bytearray()
+            for action in live_chat_continuation.get('actions', []):
+                timestamp = self.parse_live_timestamp(action)
+                if timestamp is not None:
+                    live_offset = timestamp - start_time
+                # compatibility with replay format
+                pseudo_action = {
+                    'replayChatItemAction': {'actions': [action]},
+                    'videoOffsetTimeMsec': str(live_offset),
+                    'isLive': True,
+                }
+                processed_fragment.extend(
+                    json.dumps(pseudo_action, ensure_ascii=False).encode('utf-8') + b'\n')
+            continuation_data_getters = [
+                lambda x: x['continuations'][0]['invalidationContinuationData'],
+                lambda x: x['continuations'][0]['timedContinuationData'],
+            ]
+            continuation_data = try_get(live_chat_continuation, continuation_data_getters, dict)
+            if continuation_data:
+                continuation_id = continuation_data.get('continuation')
+                click_tracking_params = continuation_data.get('clickTrackingParams')
+                timeout_ms = int_or_none(continuation_data.get('timeoutMs'))
+                if timeout_ms is not None:
+                    time.sleep(timeout_ms / 1000)
+            self._append_fragment(ctx, processed_fragment)
+            return continuation_id, live_offset, click_tracking_params
+
+        def download_and_parse_fragment(url, frag_index, request_data=None, headers=None):
+            count = 0
+            while count <= fragment_retries:
+                try:
+                    success, raw_fragment = dl_fragment(url, request_data, headers)
+                    if not success:
+                        return False, None, None, None
+                    try:
+                        data = ie.extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace'))
+                    except RegexNotFoundError:
+                        data = None
+                    if not data:
+                        data = json.loads(raw_fragment)
+                    live_chat_continuation = try_get(
+                        data,
+                        lambda x: x['continuationContents']['liveChatContinuation'], dict) or {}
+                    if info_dict['protocol'] == 'youtube_live_chat_replay':
+                        if frag_index == 1:
+                            continuation_id, offset, click_tracking_params = try_refresh_replay_beginning(live_chat_continuation)
+                        else:
+                            continuation_id, offset, click_tracking_params = parse_actions_replay(live_chat_continuation)
+                    elif info_dict['protocol'] == 'youtube_live_chat':
+                        continuation_id, offset, click_tracking_params = parse_actions_live(live_chat_continuation)
+                    return True, continuation_id, offset, click_tracking_params
+                except compat_urllib_error.HTTPError as err:
+                    count += 1
+                    if count <= fragment_retries:
+                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
+            if count > fragment_retries:
+                self.report_error('giving up after %s fragment retries' % fragment_retries)
+                return False, None, None, None
+
+        self._prepare_and_start_frag_download(ctx, info_dict)
+
+        success, raw_fragment = dl_fragment(info_dict['url'])
+        if not success:
+            return False
+        try:
+            data = ie.extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace'))
+        except RegexNotFoundError:
+            return False
+        continuation_id = try_get(
+            data,
+            lambda x: x['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation'])
+        # no data yet but required to call _append_fragment
+        self._append_fragment(ctx, b'')
+
+        ytcfg = ie.extract_ytcfg(video_id, raw_fragment.decode('utf-8', 'replace'))
+
+        if not ytcfg:
+            return False
+        api_key = try_get(ytcfg, lambda x: x['INNERTUBE_API_KEY'])
+        innertube_context = try_get(ytcfg, lambda x: x['INNERTUBE_CONTEXT'])
+        if not api_key or not innertube_context:
+            return False
+        visitor_data = try_get(innertube_context, lambda x: x['client']['visitorData'], str)
+        if info_dict['protocol'] == 'youtube_live_chat_replay':
+            url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay?key=' + api_key
+            chat_page_url = 'https://www.youtube.com/live_chat_replay?continuation=' + continuation_id
+        elif info_dict['protocol'] == 'youtube_live_chat':
+            url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat?key=' + api_key
+            chat_page_url = 'https://www.youtube.com/live_chat?continuation=' + continuation_id
+
+        frag_index = offset = 0
+        click_tracking_params = None
+        while continuation_id is not None:
+            frag_index += 1
+            request_data = {
+                'context': innertube_context,
+                'continuation': continuation_id,
+            }
+            if frag_index > 1:
+                request_data['currentPlayerState'] = {'playerOffsetMs': str(max(offset - 5000, 0))}
+                if click_tracking_params:
+                    request_data['context']['clickTracking'] = {'clickTrackingParams': click_tracking_params}
+                headers = ie.generate_api_headers(ytcfg=ytcfg, visitor_data=visitor_data)
+                headers.update({'content-type': 'application/json'})
+                fragment_request_data = json.dumps(request_data, ensure_ascii=False).encode('utf-8') + b'\n'
+                success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
+                    url, frag_index, fragment_request_data, headers)
+            else:
+                success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
+                    chat_page_url, frag_index)
+            if not success:
+                return False
+            if test:
+                break
+
+        self._finish_frag_download(ctx, info_dict)
+        return True
+
+    @staticmethod
+    def parse_live_timestamp(action):
+        action_content = dict_get(
+            action,
+            ['addChatItemAction', 'addLiveChatTickerItemAction', 'addBannerToLiveChatCommand'])
+        if not isinstance(action_content, dict):
+            return None
+        item = dict_get(action_content, ['item', 'bannerRenderer'])
+        if not isinstance(item, dict):
+            return None
+        renderer = dict_get(item, [
+            # text
+            'liveChatTextMessageRenderer', 'liveChatPaidMessageRenderer',
+            'liveChatMembershipItemRenderer', 'liveChatPaidStickerRenderer',
+            # ticker
+            'liveChatTickerPaidMessageItemRenderer',
+            'liveChatTickerSponsorItemRenderer',
+            # banner
+            'liveChatBannerRenderer',
+        ])
+        if not isinstance(renderer, dict):
+            return None
+        parent_item_getters = [
+            lambda x: x['showItemEndpoint']['showLiveChatItemEndpoint']['renderer'],
+            lambda x: x['contents'],
+        ]
+        parent_item = try_get(renderer, parent_item_getters, dict)
+        if parent_item:
+            renderer = dict_get(parent_item, [
+                'liveChatTextMessageRenderer', 'liveChatPaidMessageRenderer',
+                'liveChatMembershipItemRenderer', 'liveChatPaidStickerRenderer',
+            ])
+            if not isinstance(renderer, dict):
+                return None
+        return int_or_none(renderer.get('timestampUsec'), 1000)

plugins/youtube_download/yt_dlp/extractor/__init__.py

@@ -0,0 +1,54 @@
+import os
+
+from ..utils import load_plugins
+
+_LAZY_LOADER = False
+if not os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'):
+    try:
+        from .lazy_extractors import *
+        from .lazy_extractors import _ALL_CLASSES
+        _LAZY_LOADER = True
+    except ImportError:
+        pass
+
+if not _LAZY_LOADER:
+    from .extractors import *
+    _ALL_CLASSES = [
+        klass
+        for name, klass in globals().items()
+        if name.endswith('IE') and name != 'GenericIE'
+    ]
+    _ALL_CLASSES.append(GenericIE)
+
+_PLUGIN_CLASSES = load_plugins('extractor', 'IE', globals())
+_ALL_CLASSES = list(_PLUGIN_CLASSES.values()) + _ALL_CLASSES
+
+
+def gen_extractor_classes():
+    """ Return a list of supported extractors.
+    The order does matter; the first extractor matched is the one handling the URL.
+    """
+    return _ALL_CLASSES
+
+
+def gen_extractors():
+    """ Return a list of an instance of every supported extractor.
+    The order does matter; the first extractor matched is the one handling the URL.
+    """
+    return [klass() for klass in gen_extractor_classes()]
+
+
+def list_extractors(age_limit):
+    """
+    Return a list of extractors that are suitable for the given age,
+    sorted by extractor ID.
+    """
+
+    return sorted(
+        filter(lambda ie: ie.is_suitable(age_limit), gen_extractors()),
+        key=lambda ie: ie.IE_NAME.lower())
+
+
+def get_info_extractor(ie_name):
+    """Returns the info extractor class with the given ie_name"""
+    return globals()[ie_name + 'IE']

plugins/youtube_download/yt_dlp/extractor/abc.py

@@ -0,0 +1,318 @@
+from __future__ import unicode_literals
+
+import hashlib
+import hmac
+import re
+import time
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    dict_get,
+    ExtractorError,
+    js_to_json,
+    int_or_none,
+    parse_iso8601,
+    str_or_none,
+    try_get,
+    unescapeHTML,
+    update_url_query,
+)
+
+
+class ABCIE(InfoExtractor):
+    IE_NAME = 'abc.net.au'
+    _VALID_URL = r'https?://(?:www\.)?abc\.net\.au/(?:news|btn)/(?:[^/]+/){1,4}(?P<id>\d{5,})'
+
+    _TESTS = [{
+        'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
+        'md5': 'cb3dd03b18455a661071ee1e28344d9f',
+        'info_dict': {
+            'id': '5868334',
+            'ext': 'mp4',
+            'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
+            'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
+        },
+        'skip': 'this video has expired',
+    }, {
+        'url': 'http://www.abc.net.au/news/2015-08-17/warren-entsch-introduces-same-sex-marriage-bill/6702326',
+        'md5': '4ebd61bdc82d9a8b722f64f1f4b4d121',
+        'info_dict': {
+            'id': 'NvqvPeNZsHU',
+            'ext': 'mp4',
+            'upload_date': '20150816',
+            'uploader': 'ABC News (Australia)',
+            'description': 'Government backbencher Warren Entsch introduces a cross-party sponsored bill to legalise same-sex marriage, saying the bill is designed to promote "an inclusive Australia, not a divided one.". Read more here: http://ab.co/1Mwc6ef',
+            'uploader_id': 'NewsOnABC',
+            'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
+        },
+        'add_ie': ['Youtube'],
+        'skip': 'Not accessible from Travis CI server',
+    }, {
+        'url': 'http://www.abc.net.au/news/2015-10-23/nab-lifts-interest-rates-following-westpac-and-cba/6880080',
+        'md5': 'b96eee7c9edf4fc5a358a0252881cc1f',
+        'info_dict': {
+            'id': '6880080',
+            'ext': 'mp3',
+            'title': 'NAB lifts interest rates, following Westpac and CBA',
+            'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
+        },
+    }, {
+        'url': 'http://www.abc.net.au/news/2015-10-19/6866214',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.abc.net.au/btn/classroom/wwi-centenary/10527914',
+        'info_dict': {
+            'id': '10527914',
+            'ext': 'mp4',
+            'title': 'WWI Centenary',
+            'description': 'md5:c2379ec0ca84072e86b446e536954546',
+        }
+    }, {
+        'url': 'https://www.abc.net.au/news/programs/the-world/2020-06-10/black-lives-matter-protests-spawn-support-for/12342074',
+        'info_dict': {
+            'id': '12342074',
+            'ext': 'mp4',
+            'title': 'Black Lives Matter protests spawn support for Papuans in Indonesia',
+            'description': 'md5:2961a17dc53abc558589ccd0fb8edd6f',
+        }
+    }, {
+        'url': 'https://www.abc.net.au/btn/newsbreak/btn-newsbreak-20200814/12560476',
+        'info_dict': {
+            'id': 'tDL8Ld4dK_8',
+            'ext': 'mp4',
+            'title': 'Fortnite Banned From Apple and Google App Stores',
+            'description': 'md5:a6df3f36ce8f816b74af4bd6462f5651',
+            'upload_date': '20200813',
+            'uploader': 'Behind the News',
+            'uploader_id': 'behindthenews',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        mobj = re.search(r'<a\s+href="(?P<url>[^"]+)"\s+data-duration="\d+"\s+title="Download audio directly">', webpage)
+        if mobj:
+            urls_info = mobj.groupdict()
+            youtube = False
+            video = False
+        else:
+            mobj = re.search(r'<a href="(?P<url>http://www\.youtube\.com/watch\?v=[^"]+)"><span><strong>External Link:</strong>',
+                             webpage)
+            if mobj is None:
+                mobj = re.search(r'<iframe width="100%" src="(?P<url>//www\.youtube-nocookie\.com/embed/[^?"]+)', webpage)
+            if mobj:
+                urls_info = mobj.groupdict()
+                youtube = True
+                video = True
+
+        if mobj is None:
+            mobj = re.search(r'(?P<type>)"sources": (?P<json_data>\[[^\]]+\]),', webpage)
+            if mobj is None:
+                mobj = re.search(
+                    r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
+                    webpage)
+                if mobj is None:
+                    expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?<span>(.+?)</span>', webpage, 'expired', None)
+                    if expired:
+                        raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True)
+                    raise ExtractorError('Unable to extract video urls')
+
+            urls_info = self._parse_json(
+                mobj.group('json_data'), video_id, transform_source=js_to_json)
+            youtube = mobj.group('type') == 'YouTube'
+            video = mobj.group('type') == 'Video' or urls_info[0]['contentType'] == 'video/mp4'
+
+        if not isinstance(urls_info, list):
+            urls_info = [urls_info]
+
+        if youtube:
+            return self.playlist_result([
+                self.url_result(url_info['url']) for url_info in urls_info])
+
+        formats = []
+        for url_info in urls_info:
+            height = int_or_none(url_info.get('height'))
+            bitrate = int_or_none(url_info.get('bitrate'))
+            width = int_or_none(url_info.get('width'))
+            format_id = None
+            mobj = re.search(r'_(?:(?P<height>\d+)|(?P<bitrate>\d+)k)\.mp4$', url_info['url'])
+            if mobj:
+                height_from_url = mobj.group('height')
+                if height_from_url:
+                    height = height or int_or_none(height_from_url)
+                    width = width or int_or_none(url_info.get('label'))
+                else:
+                    bitrate = bitrate or int_or_none(mobj.group('bitrate'))
+                    format_id = str_or_none(url_info.get('label'))
+            formats.append({
+                'url': url_info['url'],
+                'vcodec': url_info.get('codec') if video else 'none',
+                'width': width,
+                'height': height,
+                'tbr': bitrate,
+                'filesize': int_or_none(url_info.get('filesize')),
+                'format_id': format_id
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+        }
+
+
+class ABCIViewIE(InfoExtractor):
+    IE_NAME = 'abc.net.au:iview'
+    _VALID_URL = r'https?://iview\.abc\.net\.au/(?:[^/]+/)*video/(?P<id>[^/?#]+)'
+    _GEO_COUNTRIES = ['AU']
+
+    # ABC iview programs are normally available for 14 days only.
+    _TESTS = [{
+        'url': 'https://iview.abc.net.au/show/gruen/series/11/video/LE1927H001S00',
+        'md5': '67715ce3c78426b11ba167d875ac6abf',
+        'info_dict': {
+            'id': 'LE1927H001S00',
+            'ext': 'mp4',
+            'title': "Series 11 Ep 1",
+            'series': "Gruen",
+            'description': 'md5:52cc744ad35045baf6aded2ce7287f67',
+            'upload_date': '20190925',
+            'uploader_id': 'abc1',
+            'timestamp': 1569445289,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_params = self._download_json(
+            'https://iview.abc.net.au/api/programs/' + video_id, video_id)
+        title = unescapeHTML(video_params.get('title') or video_params['seriesTitle'])
+        stream = next(s for s in video_params['playlist'] if s.get('type') in ('program', 'livestream'))
+
+        house_number = video_params.get('episodeHouseNumber') or video_id
+        path = '/auth/hls/sign?ts={0}&hn={1}&d=android-tablet'.format(
+            int(time.time()), house_number)
+        sig = hmac.new(
+            b'android.content.res.Resources',
+            path.encode('utf-8'), hashlib.sha256).hexdigest()
+        token = self._download_webpage(
+            'http://iview.abc.net.au{0}&sig={1}'.format(path, sig), video_id)
+
+        def tokenize_url(url, token):
+            return update_url_query(url, {
+                'hdnea': token,
+            })
+
+        for sd in ('720', 'sd', 'sd-low'):
+            sd_url = try_get(
+                stream, lambda x: x['streams']['hls'][sd], compat_str)
+            if not sd_url:
+                continue
+            formats = self._extract_m3u8_formats(
+                tokenize_url(sd_url, token), video_id, 'mp4',
+                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+            if formats:
+                break
+        self._sort_formats(formats)
+
+        subtitles = {}
+        src_vtt = stream.get('captions', {}).get('src-vtt')
+        if src_vtt:
+            subtitles['en'] = [{
+                'url': src_vtt,
+                'ext': 'vtt',
+            }]
+
+        is_live = video_params.get('livestream') == '1'
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_params.get('description'),
+            'thumbnail': video_params.get('thumbnail'),
+            'duration': int_or_none(video_params.get('eventDuration')),
+            'timestamp': parse_iso8601(video_params.get('pubDate'), ' '),
+            'series': unescapeHTML(video_params.get('seriesTitle')),
+            'series_id': video_params.get('seriesHouseNumber') or video_id[:7],
+            'season_number': int_or_none(self._search_regex(
+                r'\bSeries\s+(\d+)\b', title, 'season number', default=None)),
+            'episode_number': int_or_none(self._search_regex(
+                r'\bEp\s+(\d+)\b', title, 'episode number', default=None)),
+            'episode_id': house_number,
+            'uploader_id': video_params.get('channel'),
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': is_live,
+        }
+
+
+class ABCIViewShowSeriesIE(InfoExtractor):
+    IE_NAME = 'abc.net.au:iview:showseries'
+    _VALID_URL = r'https?://iview\.abc\.net\.au/show/(?P<id>[^/]+)(?:/series/\d+)?$'
+    _GEO_COUNTRIES = ['AU']
+
+    _TESTS = [{
+        'url': 'https://iview.abc.net.au/show/upper-middle-bogan',
+        'info_dict': {
+            'id': '124870-1',
+            'title': 'Series 1',
+            'description': 'md5:93119346c24a7c322d446d8eece430ff',
+            'series': 'Upper Middle Bogan',
+            'season': 'Series 1',
+            'thumbnail': r're:^https?://cdn\.iview\.abc\.net\.au/thumbs/.*\.jpg$'
+        },
+        'playlist_count': 8,
+    }, {
+        'url': 'https://iview.abc.net.au/show/upper-middle-bogan',
+        'info_dict': {
+            'id': 'CO1108V001S00',
+            'ext': 'mp4',
+            'title': 'Series 1 Ep 1 I\'m A Swan',
+            'description': 'md5:7b676758c1de11a30b79b4d301e8da93',
+            'series': 'Upper Middle Bogan',
+            'uploader_id': 'abc1',
+            'upload_date': '20210630',
+            'timestamp': 1625036400,
+        },
+        'params': {
+            'noplaylist': True,
+            'skip_download': 'm3u8',
+        },
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        webpage = self._download_webpage(url, show_id)
+        webpage_data = self._search_regex(
+            r'window\.__INITIAL_STATE__\s*=\s*[\'"](.+?)[\'"]\s*;',
+            webpage, 'initial state')
+        video_data = self._parse_json(
+            unescapeHTML(webpage_data).encode('utf-8').decode('unicode_escape'), show_id)
+        video_data = video_data['route']['pageData']['_embedded']
+
+        highlight = try_get(video_data, lambda x: x['highlightVideo']['shareUrl'])
+        if not self._yes_playlist(show_id, bool(highlight), video_label='highlight video'):
+            return self.url_result(highlight, ie=ABCIViewIE.ie_key())
+
+        series = video_data['selectedSeries']
+        return {
+            '_type': 'playlist',
+            'entries': [self.url_result(episode['shareUrl'])
+                        for episode in series['_embedded']['videoEpisodes']],
+            'id': series.get('id'),
+            'title': dict_get(series, ('title', 'displaySubtitle')),
+            'description': series.get('description'),
+            'series': dict_get(series, ('showTitle', 'displayTitle')),
+            'season': dict_get(series, ('title', 'displaySubtitle')),
+            'thumbnail': series.get('thumbnail'),
+        }