Compare commits
123 Commits
convert-to
...
da70244a54
| Author | SHA1 | Date | |
|---|---|---|---|
| da70244a54 | |||
| a237757e5e | |||
| f5eac69c20 | |||
| 0e0802a375 | |||
| 59adbe58da | |||
| 0d01030be3 | |||
| 8ed011ab8c | |||
| ba0cf50ad5 | |||
| d91266886f | |||
| 8c595bdf0c | |||
| b84fd38523 | |||
| e4e5e08cb4 | |||
| c01e81af27 | |||
| f2090a7d46 | |||
| 9bd5697677 | |||
| dfce2f0125 | |||
| da9a8c024b | |||
| 95f790a7a4 | |||
| 8a3146fd03 | |||
| f51a860de5 | |||
| 4f9fe37613 | |||
| 9cde8345cf | |||
| ee123c4916 | |||
| eeef0a4330 | |||
| d0612a2b37 | |||
| 9697e8ca16 | |||
| efa42a301c | |||
| 75da08d081 | |||
| 49ed89201a | |||
| e96d9e682d | |||
| 206f67f2f0 | |||
| e929e9b742 | |||
| 0dece2cec9 | |||
| 982e586936 | |||
| 867c651a04 | |||
| f48d84a004 | |||
| dc9cae6d38 | |||
| 72f0236e58 | |||
| da63e6e44e | |||
| d3e42b3ae0 | |||
| bdd532060a | |||
| ded86b81ec | |||
| a7fbc6eadb | |||
| 41f39ba8cc | |||
| 51a565a79d | |||
| bebe0c7cba | |||
| 43fe513bb1 | |||
| 3d0a714106 | |||
| 74d53690e2 | |||
| aabcd46d75 | |||
| 061dbf19ad | |||
| 247f1a1165 | |||
| 09d8170953 | |||
| 1798213bfc | |||
| bff54995fd | |||
| b058dc3667 | |||
| 3f5664da5b | |||
| 7abbee9182 | |||
| 6bd4d97db2 | |||
| 92d8069f3a | |||
| b1bf8785c6 | |||
| bcc04dda3c | |||
| 8f64066049 | |||
| 111c535876 | |||
| 793621745a | |||
| ee086f67f4 | |||
| e4656c771a | |||
| 9442453d43 | |||
| 0539fa41f0 | |||
| 2278cdc0c3 | |||
| 22c9fa301b | |||
| 1fc1609b0a | |||
| 09a85abb79 | |||
| 51ac26048c | |||
| c3f637b5fd | |||
| 45ca8abbdd | |||
| 4aeaffdd44 | |||
| 8eccdfce7c | |||
| 8e242f5475 | |||
| 32f061ff76 | |||
| 8f1c1848fd | |||
| 7e5d603eb9 | |||
| 674dac5918 | |||
| bddcc8e3e6 | |||
| 56b8ee6117 | |||
| 312a782a87 | |||
| 7b4bbd7c2b | |||
| f79aa4e852 | |||
| 918eec1053 | |||
| 07714c9cd4 | |||
| 2622600e92 | |||
| 6eed25efd6 | |||
| eafc8613e6 | |||
| f77becc21c | |||
| 67c13d264a | |||
| 52aa14dcb4 | |||
| 7534bf141e | |||
| ca855712b1 | |||
| 95c6f79627 | |||
| a863dbc586 | |||
| 7737e3ad6d | |||
| 628740fd31 | |||
| ed2a27ed9a | |||
| 3bedd83793 | |||
| ad70e8c819 | |||
| ecfb586f53 | |||
| b0991cb776 | |||
| 3d719ad6f6 | |||
| 3c914e64dd | |||
| 63c41d5e2a | |||
| 2a0fe9eb15 | |||
| a380c01573 | |||
| a1c27792ee | |||
| 353ee2a966 | |||
| 8a0057f78e | |||
| f2314500b7 | |||
| 2c258d470b | |||
| 59d67874ad | |||
| bee66ee001 | |||
| 5bf6d04fdd | |||
| 216cc9d34c | |||
| 5a9fa8253b | |||
| 9b578859e0 |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -1,3 +1,7 @@
|
||||
docs/
|
||||
.idea/
|
||||
*.zip
|
||||
|
||||
# ---> Python
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
@@ -137,4 +141,3 @@ dmypy.json
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
# SolarFM
|
||||
|
||||
# SolarFM
|
||||
SolarFM is a Gtk+ Python file manager.
|
||||
|
||||
@@ -13,8 +11,8 @@ sudo apt-get install python3.8 wget python3-setproctitle python3-gi ffmpegthumbn
|
||||
|
||||
# TODO
|
||||
<ul>
|
||||
<li>Add simpleish plugin system to run bash/python scripts.</li>
|
||||
<li>Add DnD context awareness for over folder drop.</li>
|
||||
<li>Add simpleish preview plugin for various file types.</li>
|
||||
<li>Add simpleish bulk-renamer.</li>
|
||||
</ul>
|
||||
|
||||
# Images
|
||||
|
||||
BIN
bin/solarfm-0-0-1-x64.deb
Normal file
BIN
bin/solarfm-0-0-1-x64.deb
Normal file
Binary file not shown.
42
plugins/README.md
Normal file
42
plugins/README.md
Normal file
@@ -0,0 +1,42 @@
|
||||
### Note
|
||||
Copy the example and rename it to your desired name. Plugins define a ui target slot with the 'ui_target' requests data but don't have to if not directly interacted with.
|
||||
Plugins must have a run method defined; though, you do not need to necessarily do anything within it. The run method implies that the passed in event system or other data is ready for the plugin to use.
|
||||
|
||||
|
||||
### Manifest Example (All are required even if empty.)
|
||||
```
|
||||
class Manifest:
|
||||
name: str = "Example Plugin"
|
||||
author: str = "John Doe"
|
||||
version: str = "0.0.1"
|
||||
support: str = ""
|
||||
requests: {} = {
|
||||
'ui_target': "plugin_control_list",
|
||||
'pass_fm_events': "true"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
### Requests
|
||||
```
|
||||
requests: {} = {
|
||||
'ui_target': "plugin_control_list",
|
||||
'ui_target_id': "<some other Gtk Glade ID>", # Only needed if using "other" in "ui_target". See below for predefined "ui_target" options...
|
||||
'pass_fm_events': "true", # If empty or not present will be ignored.
|
||||
"pass_ui_objects": [""], # Request reference to a UI component. Will be passed back as array to plugin.
|
||||
'bind_keys': [f"{name}||send_message:<Control>f"],
|
||||
f"{name}||do_save:<Control>s"] # Bind keys with method and key pare using list. Must pass "name" like shown with delimiter to its right.
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
UI Targets:
|
||||
<ul>
|
||||
<li>main_Window</li>
|
||||
<li>main_menu_bar</li>
|
||||
<li>path_menu_bar</li>
|
||||
<li>plugin_control_list</li>
|
||||
<li>window_(1-4)</li>
|
||||
<li>context_menu</li>
|
||||
<li>other</li>
|
||||
</ul>
|
||||
3
plugins/archiver/__init__.py
Normal file
3
plugins/archiver/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/archiver/__main__.py
Normal file
3
plugins/archiver/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
164
plugins/archiver/archiver.glade
Normal file
164
plugins/archiver/archiver.glade
Normal file
@@ -0,0 +1,164 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated with glade 3.38.2 -->
|
||||
<interface>
|
||||
<requires lib="gtk+" version="3.16"/>
|
||||
<object class="GtkTextBuffer" id="arc_command_buffer">
|
||||
<property name="text" translatable="yes">$(which 7za || echo 7zr) a %o %N</property>
|
||||
</object>
|
||||
<object class="GtkFileChooserDialog" id="archiver_dialogue">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="modal">True</property>
|
||||
<property name="window-position">center</property>
|
||||
<property name="type-hint">dialog</property>
|
||||
<property name="gravity">center</property>
|
||||
<property name="do-overwrite-confirmation">True</property>
|
||||
<property name="select-multiple">True</property>
|
||||
<child internal-child="vbox">
|
||||
<object class="GtkBox">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<property name="spacing">2</property>
|
||||
<child internal-child="action_area">
|
||||
<object class="GtkButtonBox">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="layout-style">end</property>
|
||||
<child>
|
||||
<object class="GtkButton" id="button21">
|
||||
<property name="label">gtk-cancel</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">True</property>
|
||||
<property name="use-stock">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkButton" id="button22">
|
||||
<property name="label">gtk-ok</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">True</property>
|
||||
<property name="use-stock">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">False</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="homogeneous">True</property>
|
||||
<child>
|
||||
<object class="GtkLabel">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes">Compress Commands:</property>
|
||||
<property name="xalign">0.20000000298023224</property>
|
||||
<attributes>
|
||||
<attribute name="gravity" value="west"/>
|
||||
</attributes>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<placeholder/>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes">Archive Format:</property>
|
||||
<property name="xalign">1</property>
|
||||
<attributes>
|
||||
<attribute name="gravity" value="east"/>
|
||||
</attributes>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">2</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkComboBoxText">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="active">0</property>
|
||||
<property name="active-id">0</property>
|
||||
<items>
|
||||
<item id="0" translatable="yes">7Zip (*.7z)</item>
|
||||
<item id="1" translatable="yes">Zip (*.zip *.ZIP)</item>
|
||||
<item id="2" translatable="yes">RAR (*.rar *.RAR)</item>
|
||||
<item id="3" translatable="yes">Tar (*.tar)</item>
|
||||
<item id="4" translatable="yes">Tar bzip2 (*.tar.bz2)</item>
|
||||
<item id="5" translatable="yes">Tar Gzip (*.tar.gz *.tgz)</item>
|
||||
<item id="6" translatable="yes">Tar xz (*.tar.xz *.txz)</item>
|
||||
<item id="7" translatable="yes">Gzip (*.gz)</item>
|
||||
<item id="8" translatable="yes">XZ (*.xz)</item>
|
||||
</items>
|
||||
<signal name="changed" handler="set_arc_buffer_text" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">3</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkTextView" id="arc_command">
|
||||
<property name="height-request">72</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="buffer">arc_command_buffer</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">2</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
<action-widgets>
|
||||
<action-widget response="-6">button21</action-widget>
|
||||
<action-widget response="-5">button22</action-widget>
|
||||
</action-widgets>
|
||||
</object>
|
||||
</interface>
|
||||
12
plugins/archiver/manifest.json
Normal file
12
plugins/archiver/manifest.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "Archiver",
|
||||
"author": "ITDominator",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "context_menu_plugins",
|
||||
"pass_fm_events": "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
129
plugins/archiver/plugin.py
Normal file
129
plugins/archiver/plugin.py
Normal file
@@ -0,0 +1,129 @@
|
||||
# Python imports
|
||||
import os
|
||||
import threading
|
||||
import subprocess
|
||||
import inspect
|
||||
import shlex
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
return wrapper
|
||||
|
||||
|
||||
|
||||
|
||||
class Plugin(PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.path = os.path.dirname(os.path.realpath(__file__))
|
||||
self._GLADE_FILE = f"{self.path}/archiver.glade"
|
||||
self.name = "Archiver" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
self._archiver_dialogue = None
|
||||
self._arc_command_buffer = None
|
||||
|
||||
# In compress commands:
|
||||
# %n: First selected filename/dir to archive
|
||||
# %N: All selected filenames/dirs to archive, or (with %O) a single filename
|
||||
# %o: Resulting single archive file
|
||||
# %O: Resulting archive per source file/directory (use changes %N meaning)
|
||||
#
|
||||
# In extract commands:
|
||||
# %x: Archive file to extract
|
||||
# %g: Unique extraction target filename with optional subfolder
|
||||
# %G: Unique extraction target filename, never with subfolder
|
||||
#
|
||||
# In list commands:
|
||||
# %x: Archive to list
|
||||
#
|
||||
# Plus standard bash variables are accepted.
|
||||
self.arc_commands = [ '$(which 7za || echo 7zr) a %o %N',
|
||||
'zip -r %o %N',
|
||||
'rar a -r %o %N',
|
||||
'tar -cvf %o %N',
|
||||
'tar -cvjf %o %N',
|
||||
'tar -cvzf %o %N',
|
||||
'tar -cvJf %o %N',
|
||||
'gzip -c %N > %O',
|
||||
'xz -cz %N > %O'
|
||||
]
|
||||
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
self._builder = Gtk.Builder()
|
||||
self._builder.add_from_file(self._GLADE_FILE)
|
||||
|
||||
classes = [self]
|
||||
handlers = {}
|
||||
for c in classes:
|
||||
methods = None
|
||||
try:
|
||||
methods = inspect.getmembers(c, predicate=inspect.ismethod)
|
||||
handlers.update(methods)
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
|
||||
self._builder.connect_signals(handlers)
|
||||
|
||||
self._archiver_dialogue = self._builder.get_object("archiver_dialogue")
|
||||
self._arc_command_buffer = self._builder.get_object("arc_command_buffer")
|
||||
|
||||
item = Gtk.ImageMenuItem(self.name)
|
||||
item.set_image( Gtk.Image(stock=Gtk.STOCK_FLOPPY) )
|
||||
item.connect("activate", self.show_archiver_dialogue)
|
||||
item.set_always_show_image(True)
|
||||
return item
|
||||
|
||||
|
||||
def run(self):
|
||||
...
|
||||
|
||||
def show_archiver_dialogue(self, widget=None, eve=None):
|
||||
self._event_system.emit("get_current_state")
|
||||
state = self._fm_state
|
||||
|
||||
self._archiver_dialogue.set_action(Gtk.FileChooserAction.SAVE)
|
||||
self._archiver_dialogue.set_current_folder(state.tab.get_current_directory())
|
||||
self._archiver_dialogue.set_current_name("arc.7z")
|
||||
|
||||
response = self._archiver_dialogue.run()
|
||||
if response == Gtk.ResponseType.OK:
|
||||
save_target = self._archiver_dialogue.get_filename()
|
||||
self.archive_files(save_target, state)
|
||||
if (response == Gtk.ResponseType.CANCEL) or (response == Gtk.ResponseType.DELETE_EVENT):
|
||||
pass
|
||||
|
||||
self._archiver_dialogue.hide()
|
||||
|
||||
def archive_files(self, save_target, state):
|
||||
paths = [shlex.quote(p) for p in state.selected_files]
|
||||
|
||||
sItr, eItr = self._arc_command_buffer.get_bounds()
|
||||
pre_command = self._arc_command_buffer.get_text(sItr, eItr, False)
|
||||
pre_command = pre_command.replace("%o", shlex.quote(save_target))
|
||||
pre_command = pre_command.replace("%N", ' '.join(paths))
|
||||
command = f"{state.tab.terminal_app} -e {shlex.quote(pre_command)}"
|
||||
current_dir = state.tab.get_current_directory()
|
||||
|
||||
state.tab.execute(shlex.split(command), start_dir=shlex.quote(current_dir))
|
||||
|
||||
def set_arc_buffer_text(self, widget=None, eve=None):
|
||||
sid = widget.get_active_id()
|
||||
self._arc_command_buffer.set_text(self.arc_commands[int(sid)])
|
||||
3
plugins/disk_usage/__init__.py
Normal file
3
plugins/disk_usage/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/disk_usage/__main__.py
Normal file
3
plugins/disk_usage/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
129
plugins/disk_usage/du_usage.glade
Normal file
129
plugins/disk_usage/du_usage.glade
Normal file
@@ -0,0 +1,129 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated with glade 3.40.0 -->
|
||||
<interface>
|
||||
<requires lib="gtk+" version="3.24"/>
|
||||
<object class="GtkListStore" id="du_store">
|
||||
<columns>
|
||||
<!-- column-name Size -->
|
||||
<column type="gchararray"/>
|
||||
<!-- column-name Dir -->
|
||||
<column type="gchararray"/>
|
||||
</columns>
|
||||
</object>
|
||||
<object class="GtkDialog" id="du_dialog">
|
||||
<property name="width-request">420</property>
|
||||
<property name="height-request">450</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="modal">True</property>
|
||||
<property name="window-position">center</property>
|
||||
<property name="destroy-with-parent">True</property>
|
||||
<property name="type-hint">dialog</property>
|
||||
<property name="skip-taskbar-hint">True</property>
|
||||
<property name="skip-pager-hint">True</property>
|
||||
<property name="decorated">False</property>
|
||||
<property name="deletable">False</property>
|
||||
<property name="gravity">center</property>
|
||||
<child internal-child="vbox">
|
||||
<object class="GtkBox">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<property name="spacing">2</property>
|
||||
<child internal-child="action_area">
|
||||
<object class="GtkButtonBox">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="layout-style">end</property>
|
||||
<child>
|
||||
<object class="GtkButton">
|
||||
<property name="label">gtk-close</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">True</property>
|
||||
<property name="use-stock">True</property>
|
||||
<signal name="released" handler="_hide_du_menu" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">2</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">False</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<object class="GtkLabel" id="current_dir_lbl">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="margin-start">5</property>
|
||||
<property name="margin-end">5</property>
|
||||
<property name="margin-top">5</property>
|
||||
<property name="margin-bottom">5</property>
|
||||
<property name="label" translatable="yes">Current Directory:</property>
|
||||
<property name="justify">center</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkScrolledWindow">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="shadow-type">in</property>
|
||||
<child>
|
||||
<object class="GtkTreeView">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="model">du_store</property>
|
||||
<property name="headers-clickable">False</property>
|
||||
<child internal-child="selection">
|
||||
<object class="GtkTreeSelection"/>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkTreeViewColumn">
|
||||
<property name="title" translatable="yes">Disk Usage</property>
|
||||
<child>
|
||||
<object class="GtkCellRendererText"/>
|
||||
<attributes>
|
||||
<attribute name="text">0</attribute>
|
||||
</attributes>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCellRendererText"/>
|
||||
<attributes>
|
||||
<attribute name="text">1</attribute>
|
||||
</attributes>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
</interface>
|
||||
12
plugins/disk_usage/manifest.json
Normal file
12
plugins/disk_usage/manifest.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "Disk Usage",
|
||||
"author": "ITDominator",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "context_menu_plugins",
|
||||
"pass_fm_events": "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
95
plugins/disk_usage/plugin.py
Normal file
95
plugins/disk_usage/plugin.py
Normal file
@@ -0,0 +1,95 @@
|
||||
# Python imports
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import inspect
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
|
||||
|
||||
|
||||
|
||||
class Plugin(PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.name = "Disk Usage" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
|
||||
self.path = os.path.dirname(os.path.realpath(__file__))
|
||||
self._GLADE_FILE = f"{self.path}/du_usage.glade"
|
||||
self._du_dialog = None
|
||||
self._du_store = None
|
||||
|
||||
|
||||
def run(self):
|
||||
self._builder = Gtk.Builder()
|
||||
self._builder.add_from_file(self._GLADE_FILE)
|
||||
|
||||
classes = [self]
|
||||
handlers = {}
|
||||
for c in classes:
|
||||
methods = None
|
||||
try:
|
||||
methods = inspect.getmembers(c, predicate=inspect.ismethod)
|
||||
handlers.update(methods)
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
|
||||
self._builder.connect_signals(handlers)
|
||||
|
||||
self._du_dialog = self._builder.get_object("du_dialog")
|
||||
self._du_store = self._builder.get_object("du_store")
|
||||
self._current_dir_lbl = self._builder.get_object("current_dir_lbl")
|
||||
|
||||
self._event_system.subscribe("show_du_menu", self._show_du_menu)
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
item = Gtk.ImageMenuItem(self.name)
|
||||
item.set_image( Gtk.Image(stock=Gtk.STOCK_HARDDISK) )
|
||||
item.connect("activate", self._show_du_menu)
|
||||
item.set_always_show_image(True)
|
||||
return item
|
||||
|
||||
def _get_state(self, widget=None, eve=None):
|
||||
self._event_system.emit("get_current_state")
|
||||
|
||||
def _set_current_dir_lbl(self, widget=None, eve=None):
|
||||
self._current_dir_lbl.set_label(f"Current Directory:\n{self._fm_state.tab.get_current_directory()}")
|
||||
|
||||
def _show_du_menu(self, widget=None, eve=None):
|
||||
self._fm_state = None
|
||||
self._get_state()
|
||||
self._set_current_dir_lbl()
|
||||
self.load_du_data()
|
||||
self._du_dialog.run()
|
||||
|
||||
def load_du_data(self):
|
||||
self._du_store.clear()
|
||||
|
||||
path = self._fm_state.tab.get_current_directory()
|
||||
# NOTE: -h = human readable, -d = depth asigned to 1
|
||||
command = ["du", "-h", "-d", "1", path]
|
||||
proc = subprocess.Popen(command, stdout=subprocess.PIPE, encoding="utf-8")
|
||||
raw_data = proc.communicate()[0]
|
||||
# NOTE: Will return data AFTER completion (if any)
|
||||
data = raw_data.strip()
|
||||
parts = data.split("\n")
|
||||
|
||||
# NOTE: Last entry is curret dir. Move to top of list and pop off...
|
||||
size, file = parts[-1].split("\t")
|
||||
self._du_store.append([size, file.split("/")[-1]])
|
||||
parts.pop()
|
||||
|
||||
for part in parts:
|
||||
size, file = part.split("\t")
|
||||
self._du_store.append([size, file.split("/")[-1]])
|
||||
|
||||
def _hide_du_menu(self, widget=None, eve=None):
|
||||
self._du_dialog.hide()
|
||||
3
plugins/favorites/__init__.py
Normal file
3
plugins/favorites/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/favorites/__main__.py
Normal file
3
plugins/favorites/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
158
plugins/favorites/favorites.glade
Normal file
158
plugins/favorites/favorites.glade
Normal file
@@ -0,0 +1,158 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated with glade 3.40.0 -->
|
||||
<interface>
|
||||
<requires lib="gtk+" version="3.24"/>
|
||||
<object class="GtkListStore" id="favorites_store">
|
||||
<columns>
|
||||
<!-- column-name Favorites -->
|
||||
<column type="gchararray"/>
|
||||
<!-- column-name Path -->
|
||||
<column type="gchararray"/>
|
||||
</columns>
|
||||
</object>
|
||||
<object class="GtkDialog" id="favorites_dialog">
|
||||
<property name="width-request">420</property>
|
||||
<property name="height-request">450</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="modal">True</property>
|
||||
<property name="window-position">center</property>
|
||||
<property name="destroy-with-parent">True</property>
|
||||
<property name="type-hint">dialog</property>
|
||||
<property name="skip-taskbar-hint">True</property>
|
||||
<property name="skip-pager-hint">True</property>
|
||||
<property name="decorated">False</property>
|
||||
<property name="deletable">False</property>
|
||||
<property name="gravity">center</property>
|
||||
<child internal-child="vbox">
|
||||
<object class="GtkBox">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<property name="spacing">2</property>
|
||||
<child internal-child="action_area">
|
||||
<object class="GtkButtonBox">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="layout-style">end</property>
|
||||
<child>
|
||||
<object class="GtkButton">
|
||||
<property name="label">gtk-delete</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">True</property>
|
||||
<property name="use-stock">True</property>
|
||||
<property name="always-show-image">True</property>
|
||||
<signal name="released" handler="_remove_from_favorite" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkButton">
|
||||
<property name="label">gtk-add</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">True</property>
|
||||
<property name="use-stock">True</property>
|
||||
<property name="always-show-image">True</property>
|
||||
<signal name="released" handler="_add_to_favorite" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkButton">
|
||||
<property name="label">gtk-close</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">True</property>
|
||||
<property name="use-stock">True</property>
|
||||
<signal name="released" handler="_hide_favorites_menu" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">2</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">False</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<object class="GtkLabel" id="current_dir_lbl">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="margin-start">5</property>
|
||||
<property name="margin-end">5</property>
|
||||
<property name="margin-top">5</property>
|
||||
<property name="margin-bottom">5</property>
|
||||
<property name="label" translatable="yes">Current Directory:</property>
|
||||
<property name="justify">center</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkScrolledWindow">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="shadow-type">in</property>
|
||||
<child>
|
||||
<object class="GtkTreeView">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="model">favorites_store</property>
|
||||
<property name="headers-clickable">False</property>
|
||||
<signal name="button-release-event" handler="_set_selected_path" swapped="no"/>
|
||||
<child internal-child="selection">
|
||||
<object class="GtkTreeSelection">
|
||||
<signal name="changed" handler="_set_selected" swapped="no"/>
|
||||
</object>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkTreeViewColumn">
|
||||
<property name="title" translatable="yes">Favorites</property>
|
||||
<child>
|
||||
<object class="GtkCellRendererText"/>
|
||||
<attributes>
|
||||
<attribute name="text">0</attribute>
|
||||
</attributes>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
</interface>
|
||||
14
plugins/favorites/manifest.json
Normal file
14
plugins/favorites/manifest.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "Favorites",
|
||||
"author": "ITDominator",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "main_menu_bttn_box_bar",
|
||||
"pass_fm_events": "true",
|
||||
"pass_ui_objects": ["path_entry"],
|
||||
"bind_keys": ["Favorites||show_favorites_menu:<Control>f"]
|
||||
}
|
||||
}
|
||||
}
|
||||
117
plugins/favorites/plugin.py
Normal file
117
plugins/favorites/plugin.py
Normal file
@@ -0,0 +1,117 @@
|
||||
# Python imports
|
||||
import os
|
||||
import inspect
|
||||
import json
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
|
||||
|
||||
|
||||
|
||||
class Plugin(PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.name = "Favorites" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
self.path = os.path.dirname(os.path.realpath(__file__))
|
||||
self._GLADE_FILE = f"{self.path}/favorites.glade"
|
||||
self._FAVORITES_FILE = f"{self.path}/favorites.json"
|
||||
|
||||
self._favorites_dialog = None
|
||||
self._favorites_store = None
|
||||
self._favorites = None
|
||||
self._selected = None
|
||||
|
||||
|
||||
def run(self):
|
||||
self._builder = Gtk.Builder()
|
||||
self._builder.add_from_file(self._GLADE_FILE)
|
||||
|
||||
classes = [self]
|
||||
handlers = {}
|
||||
for c in classes:
|
||||
methods = None
|
||||
try:
|
||||
methods = inspect.getmembers(c, predicate=inspect.ismethod)
|
||||
handlers.update(methods)
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
|
||||
self._builder.connect_signals(handlers)
|
||||
|
||||
self._favorites_dialog = self._builder.get_object("favorites_dialog")
|
||||
self._favorites_store = self._builder.get_object("favorites_store")
|
||||
self._current_dir_lbl = self._builder.get_object("current_dir_lbl")
|
||||
|
||||
if os.path.exists(self._FAVORITES_FILE):
|
||||
with open(self._FAVORITES_FILE) as f:
|
||||
self._favorites = json.load(f)
|
||||
for favorite in self._favorites:
|
||||
display, path = favorite
|
||||
self._favorites_store.append([display, path])
|
||||
else:
|
||||
with open(self._FAVORITES_FILE, 'a') as f:
|
||||
f.write('[]')
|
||||
|
||||
self._event_system.subscribe("show_favorites_menu", self._show_favorites_menu)
|
||||
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
button = Gtk.Button(label=self.name)
|
||||
button.connect("button-release-event", self._show_favorites_menu)
|
||||
return button
|
||||
|
||||
def _get_state(self, widget=None, eve=None):
|
||||
self._event_system.emit("get_current_state")
|
||||
|
||||
def _set_current_dir_lbl(self, widget=None, eve=None):
|
||||
self._current_dir_lbl.set_label(f"Current Directory:\n{self._fm_state.tab.get_current_directory()}")
|
||||
|
||||
def _add_to_favorite(self, state):
|
||||
path = self._fm_state.tab.get_current_directory()
|
||||
parts = path.split("/")
|
||||
display = '/'.join(parts[-3:]) if len(parts) > 3 else path
|
||||
|
||||
self._favorites_store.append([display, path])
|
||||
self._favorites.append([display, path])
|
||||
self._save_favorites()
|
||||
|
||||
def _remove_from_favorite(self, state):
|
||||
path = self._favorites_store.get_value(self._selected, 1)
|
||||
self._favorites_store.remove(self._selected)
|
||||
|
||||
for i, f in enumerate(self._favorites):
|
||||
if f[1] == path:
|
||||
self._favorites.remove( self._favorites[i] )
|
||||
|
||||
self._save_favorites()
|
||||
|
||||
def _save_favorites(self):
|
||||
with open(self._FAVORITES_FILE, 'w') as outfile:
|
||||
json.dump(self._favorites, outfile, separators=(',', ':'), indent=4)
|
||||
|
||||
def _set_selected_path(self, widget=None, eve=None):
|
||||
path = self._favorites_store.get_value(self._selected, 1)
|
||||
self._ui_objects[0].set_text(path)
|
||||
self._set_current_dir_lbl()
|
||||
|
||||
def _show_favorites_menu(self, widget=None, eve=None):
|
||||
self._fm_state = None
|
||||
self._get_state()
|
||||
self._set_current_dir_lbl()
|
||||
self._favorites_dialog.run()
|
||||
|
||||
def _hide_favorites_menu(self, widget=None, eve=None):
|
||||
self._favorites_dialog.hide()
|
||||
|
||||
def _set_selected(self, user_data):
|
||||
selected = user_data.get_selected()[1]
|
||||
if selected and not self._selected == selected:
|
||||
self._selected = selected
|
||||
3
plugins/file_properties/__init__.py
Normal file
3
plugins/file_properties/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/file_properties/__main__.py
Normal file
3
plugins/file_properties/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
685
plugins/file_properties/file_properties.glade
Normal file
685
plugins/file_properties/file_properties.glade
Normal file
@@ -0,0 +1,685 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated with glade 3.38.2 -->
|
||||
<interface>
|
||||
<requires lib="gtk+" version="3.16"/>
|
||||
<object class="GtkDialog" id="file_properties_dialog">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="border-width">6</property>
|
||||
<property name="title" translatable="yes">File Properties</property>
|
||||
<property name="modal">True</property>
|
||||
<property name="window-position">center-on-parent</property>
|
||||
<property name="default-width">420</property>
|
||||
<property name="destroy-with-parent">True</property>
|
||||
<property name="type-hint">dialog</property>
|
||||
<property name="skip-taskbar-hint">True</property>
|
||||
<property name="skip-pager-hint">True</property>
|
||||
<property name="gravity">center</property>
|
||||
<signal name="response" handler="on_filePropertiesDlg_response" swapped="no"/>
|
||||
<child internal-child="vbox">
|
||||
<object class="GtkBox" id="dialog_vbox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="spacing">12</property>
|
||||
<child internal-child="action_area">
|
||||
<object class="GtkButtonBox" id="dialog_action_area">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="layout-style">end</property>
|
||||
<child>
|
||||
<object class="GtkButton" id="cancel_button">
|
||||
<property name="label">gtk-cancel</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="can-default">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="use-stock">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkButton" id="ok_button">
|
||||
<property name="label">gtk-ok</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="can-default">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="use-stock">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">False</property>
|
||||
<property name="pack-type">end</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkNotebook" id="notebook">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="border-width">6</property>
|
||||
<child>
|
||||
<object class="GtkAlignment" id="alignment2">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="top-padding">6</property>
|
||||
<property name="bottom-padding">6</property>
|
||||
<property name="left-padding">12</property>
|
||||
<child>
|
||||
<object class="GtkTable" id="general_table">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="border-width">4</property>
|
||||
<property name="n-rows">7</property>
|
||||
<property name="n-columns">2</property>
|
||||
<property name="column-spacing">12</property>
|
||||
<property name="row-spacing">6</property>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label_filename">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>File _Name:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="mnemonic-widget">file_name</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_name">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label20">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>_Location:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="mnemonic-widget">file_location</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_location">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label_target">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>Link _Target:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="mnemonic-widget">file_target</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_target">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label4">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>Type:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="xalign">0</property>
|
||||
<property name="yalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">3</property>
|
||||
<property name="bottom-attach">4</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options">GTK_FILL</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="mime_type">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="selectable">True</property>
|
||||
<property name="ellipsize">end</property>
|
||||
<property name="xalign">0</property>
|
||||
<property name="yalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">3</property>
|
||||
<property name="bottom-attach">4</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label5">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>Size:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">4</property>
|
||||
<property name="bottom-attach">5</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="file_size">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="selectable">True</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">4</property>
|
||||
<property name="bottom-attach">5</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label7">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>_Modified:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">5</property>
|
||||
<property name="bottom-attach">6</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label13">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>_Accessed:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">6</property>
|
||||
<property name="bottom-attach">7</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="mtime">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">5</property>
|
||||
<property name="bottom-attach">6</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="atime">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">6</property>
|
||||
<property name="bottom-attach">7</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
<child type="tab">
|
||||
<object class="GtkLabel" id="label1">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes">_Info</property>
|
||||
<property name="use-underline">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="tab-fill">False</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkAlignment" id="alignment1">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="top-padding">6</property>
|
||||
<property name="bottom-padding">6</property>
|
||||
<property name="left-padding">12</property>
|
||||
<child>
|
||||
<object class="GtkVBox" id="vbox1">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="spacing">6</property>
|
||||
<child>
|
||||
<object class="GtkTable" id="table3">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="n-rows">2</property>
|
||||
<property name="n-columns">2</property>
|
||||
<property name="column-spacing">12</property>
|
||||
<property name="row-spacing">6</property>
|
||||
<child>
|
||||
<object class="GtkLabel" id="owner_label">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>_Owner:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="mnemonic-widget">file_owner</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="group_label">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>_Group:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="mnemonic-widget">file_group</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_owner">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_group">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">False</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkHSeparator" id="hseparator1">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">False</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkTable" id="table2">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="border-width">4</property>
|
||||
<property name="n-rows">3</property>
|
||||
<property name="n-columns">5</property>
|
||||
<property name="column-spacing">12</property>
|
||||
<property name="row-spacing">6</property>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label17">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>Owner:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label18">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>Group:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="label19">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>Other:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCheckButton" id="owner_r">
|
||||
<property name="label" translatable="yes">Read</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="draw-indicator">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCheckButton" id="group_r">
|
||||
<property name="label" translatable="yes">Read</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="draw-indicator">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCheckButton" id="others_r">
|
||||
<property name="label" translatable="yes">Read</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="draw-indicator">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCheckButton" id="owner_w">
|
||||
<property name="label" translatable="yes">Write</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="draw-indicator">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">2</property>
|
||||
<property name="right-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCheckButton" id="group_w">
|
||||
<property name="label" translatable="yes">Write</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="draw-indicator">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">2</property>
|
||||
<property name="right-attach">3</property>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCheckButton" id="others_w">
|
||||
<property name="label" translatable="yes">Write</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="draw-indicator">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">2</property>
|
||||
<property name="right-attach">3</property>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCheckButton" id="owner_x">
|
||||
<property name="label" translatable="yes">Execute</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="draw-indicator">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">3</property>
|
||||
<property name="right-attach">4</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCheckButton" id="group_x">
|
||||
<property name="label" translatable="yes">Execute</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="draw-indicator">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">3</property>
|
||||
<property name="right-attach">4</property>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkCheckButton" id="others_x">
|
||||
<property name="label" translatable="yes">Execute</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="border-width">2</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="draw-indicator">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">3</property>
|
||||
<property name="right-attach">4</property>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkVSeparator" id="vseparator1">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">4</property>
|
||||
<property name="right-attach">5</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options">GTK_FILL</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">2</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child type="tab">
|
||||
<object class="GtkLabel" id="label2">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes">_Permissions</property>
|
||||
<property name="use-underline">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="position">1</property>
|
||||
<property name="tab-fill">False</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">2</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
<action-widgets>
|
||||
<action-widget response="-6">cancel_button</action-widget>
|
||||
<action-widget response="-5">ok_button</action-widget>
|
||||
</action-widgets>
|
||||
</object>
|
||||
</interface>
|
||||
12
plugins/file_properties/manifest.json
Normal file
12
plugins/file_properties/manifest.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "Properties",
|
||||
"author": "ITDominator",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "context_menu",
|
||||
"pass_fm_events": "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
249
plugins/file_properties/plugin.py
Normal file
249
plugins/file_properties/plugin.py
Normal file
@@ -0,0 +1,249 @@
|
||||
# Python imports
|
||||
import os
|
||||
import threading
|
||||
import subprocess
|
||||
import time
|
||||
import pwd
|
||||
import grp
|
||||
from datetime import datetime
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
from gi.repository import GLib
|
||||
from gi.repository import Gio
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
return wrapper
|
||||
|
||||
|
||||
|
||||
|
||||
class Properties:
|
||||
file_uri: str = None
|
||||
file_name: str = None
|
||||
file_location: str = None
|
||||
file_target: str = None
|
||||
mime_type: str = None
|
||||
file_size: str = None
|
||||
mtime: int = None
|
||||
atime: int = None
|
||||
file_owner: str = None
|
||||
file_group: str = None
|
||||
chmod_stat: str = None
|
||||
|
||||
|
||||
class Plugin(PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.path = os.path.dirname(os.path.realpath(__file__))
|
||||
self._GLADE_FILE = f"{self.path}/file_properties.glade"
|
||||
self.name = "Properties" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
|
||||
self._properties_dialog = None
|
||||
self._file_name = None
|
||||
self._file_location = None
|
||||
self._file_target = None
|
||||
self._mime_type = None
|
||||
self._file_size = None
|
||||
self._mtime = None
|
||||
self._atime = None
|
||||
self._file_owner = None
|
||||
self._file_group = None
|
||||
|
||||
self._chmod_map: {} = {
|
||||
"7": "rwx",
|
||||
"6": "rw",
|
||||
"5": "rx",
|
||||
"4": "r",
|
||||
"3": "wx",
|
||||
"2": "w",
|
||||
"1": "x",
|
||||
"0": ""
|
||||
}
|
||||
|
||||
self._chmod_map_counter: {} = {
|
||||
"rwx": "7",
|
||||
"rw": "6",
|
||||
"rx": "5",
|
||||
"r": "4",
|
||||
"wx": "3",
|
||||
"w": "2",
|
||||
"x": "1",
|
||||
"": "0"
|
||||
}
|
||||
|
||||
|
||||
def run(self):
|
||||
self._builder = Gtk.Builder()
|
||||
self._builder.add_from_file(self._GLADE_FILE)
|
||||
|
||||
self._properties_dialog = self._builder.get_object("file_properties_dialog")
|
||||
self._file_name = self._builder.get_object("file_name")
|
||||
self._file_location = self._builder.get_object("file_location")
|
||||
self._file_target = self._builder.get_object("file_target")
|
||||
self._mime_type = self._builder.get_object("mime_type")
|
||||
self._file_size = self._builder.get_object("file_size")
|
||||
self._mtime = self._builder.get_object("mtime")
|
||||
self._atime = self._builder.get_object("atime")
|
||||
self._file_owner = self._builder.get_object("file_owner")
|
||||
self._file_group = self._builder.get_object("file_group")
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
item = Gtk.ImageMenuItem(self.name)
|
||||
item.set_image( Gtk.Image(stock=Gtk.STOCK_PROPERTIES) )
|
||||
item.connect("activate", self._show_properties_page)
|
||||
item.set_always_show_image(True)
|
||||
return item
|
||||
|
||||
|
||||
@threaded
|
||||
def _show_properties_page(self, widget=None, eve=None):
|
||||
event_system.emit("get_current_state")
|
||||
|
||||
state = self._fm_state
|
||||
self._event_message = None
|
||||
|
||||
GLib.idle_add(self._process_changes, (state))
|
||||
|
||||
def _process_changes(self, state):
|
||||
if len(state.selected_files) == 1:
|
||||
uri = state.selected_files[0]
|
||||
path = state.tab.get_current_directory()
|
||||
|
||||
|
||||
properties = self._set_ui_data(uri, path)
|
||||
response = self._properties_dialog.run()
|
||||
if response in [Gtk.ResponseType.CANCEL, Gtk.ResponseType.DELETE_EVENT]:
|
||||
self._properties_dialog.hide()
|
||||
|
||||
self._update_file(properties)
|
||||
self._properties_dialog.hide()
|
||||
|
||||
|
||||
def _update_file(self, properties):
|
||||
chmod_stat = self._get_check_boxes()
|
||||
|
||||
if chmod_stat is not properties.chmod_stat:
|
||||
try:
|
||||
print("\nNew chmod flags...")
|
||||
print(f"Old: {''.join(properties.chmod_stat)}")
|
||||
print(f"New: {chmod_stat}")
|
||||
|
||||
command = ["chmod", f"{chmod_stat}", properties.file_uri]
|
||||
with subprocess.Popen(command, stdout=subprocess.PIPE) as proc:
|
||||
result = proc.stdout.read().decode("UTF-8").strip()
|
||||
print(result)
|
||||
except Exception as e:
|
||||
print(f"Couldn't chmod\nFile: {properties.file_uri}")
|
||||
print( repr(e) )
|
||||
|
||||
|
||||
owner = self._file_owner.get_text()
|
||||
group = self._file_group.get_text()
|
||||
if owner is not properties.file_owner or group is not properties.file_group:
|
||||
try:
|
||||
print("\nNew owner/group flags...")
|
||||
print(f"Old:\n\tOwner: {properties.file_owner}\n\tGroup: {properties.file_group}")
|
||||
print(f"New:\n\tOwner: {owner}\n\tGroup: {group}")
|
||||
|
||||
uid = pwd.getpwnam(owner).pw_uid
|
||||
gid = grp.getgrnam(group).gr_gid
|
||||
os.chown(properties.file_uri, uid, gid)
|
||||
except Exception as e:
|
||||
print(f"Couldn't chmod\nFile: {properties.file_uri}")
|
||||
print( repr(e) )
|
||||
|
||||
|
||||
def _set_ui_data(self, uri, path):
|
||||
properties = Properties()
|
||||
file_info = Gio.File.new_for_path(uri).query_info(attributes="standard::*,owner::*,time::access,time::changed",
|
||||
flags=Gio.FileQueryInfoFlags.NONE,
|
||||
cancellable=None)
|
||||
|
||||
is_symlink = file_info.get_attribute_as_string("standard::is-symlink")
|
||||
properties.file_uri = uri
|
||||
properties.file_target = file_info.get_attribute_as_string("standard::symlink-target") if is_symlink else ""
|
||||
properties.file_name = file_info.get_display_name()
|
||||
properties.file_location = path
|
||||
properties.mime_type = file_info.get_content_type()
|
||||
properties.file_size = self._sizeof_fmt(file_info.get_size())
|
||||
properties.mtime = datetime.fromtimestamp( int(file_info.get_attribute_as_string("time::changed")) ).strftime("%A, %B %d, %Y %I:%M:%S")
|
||||
properties.atime = datetime.fromtimestamp( int(file_info.get_attribute_as_string("time::access")) ).strftime("%A, %B %d, %Y %I:%M:%S")
|
||||
properties.file_owner = file_info.get_attribute_as_string("owner::user")
|
||||
properties.file_group = file_info.get_attribute_as_string("owner::group")
|
||||
|
||||
# NOTE: Read = 4, Write = 2, Exec = 1
|
||||
command = ["stat", "-c", "%a", uri]
|
||||
with subprocess.Popen(command, stdout=subprocess.PIPE) as proc:
|
||||
properties.chmod_stat = list(proc.stdout.read().decode("UTF-8").strip())
|
||||
owner = self._chmod_map[f"{properties.chmod_stat[0]}"]
|
||||
group = self._chmod_map[f"{properties.chmod_stat[1]}"]
|
||||
others = self._chmod_map[f"{properties.chmod_stat[2]}"]
|
||||
|
||||
self._reset_check_boxes()
|
||||
self._set_check_boxes([["owner", owner], ["group", group], ["others", others]])
|
||||
|
||||
self._file_name.set_text(properties.file_name)
|
||||
self._file_location.set_text(properties.file_location)
|
||||
self._file_target.set_text(properties.file_target)
|
||||
self._mime_type.set_label(properties.mime_type)
|
||||
self._file_size.set_label(properties.file_size)
|
||||
self._mtime.set_text(properties.mtime)
|
||||
self._atime.set_text(properties.atime)
|
||||
self._file_owner.set_text(properties.file_owner)
|
||||
self._file_group.set_text(properties.file_group)
|
||||
|
||||
return properties
|
||||
|
||||
|
||||
|
||||
|
||||
def _get_check_boxes(self):
|
||||
perms = [[], [], []]
|
||||
|
||||
for i, target in enumerate(["owner", "group", "others"]):
|
||||
for type in ["r", "w", "x"]:
|
||||
is_active = self._builder.get_object(f"{target}_{type}").get_active()
|
||||
if is_active:
|
||||
perms[i].append(type)
|
||||
|
||||
digits = []
|
||||
for perm in perms:
|
||||
digits.append(self._chmod_map_counter[ ''.join(perm) ])
|
||||
|
||||
return ''.join(digits)
|
||||
|
||||
def _set_check_boxes(self, targets):
|
||||
for name, target in targets:
|
||||
for type in list(target):
|
||||
obj = f"{name}_{type}"
|
||||
self._builder.get_object(obj).set_active(True)
|
||||
|
||||
def _reset_check_boxes(self):
|
||||
for target in ["owner", "group", "others"]:
|
||||
for type in ["r", "w", "x"]:
|
||||
self._builder.get_object(f"{target}_{type}").set_active(False)
|
||||
|
||||
def _sizeof_fmt(self, num, suffix="B"):
|
||||
for unit in ["", "K", "M", "G", "T", "Pi", "Ei", "Zi"]:
|
||||
if abs(num) < 1024.0:
|
||||
return f"{num:3.1f} {unit}{suffix}"
|
||||
num /= 1024.0
|
||||
return f"{num:.1f} Yi{suffix}"
|
||||
3
plugins/movie_tv_info/__init__.py
Normal file
3
plugins/movie_tv_info/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/movie_tv_info/__main__.py
Normal file
3
plugins/movie_tv_info/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
12
plugins/movie_tv_info/manifest.json
Normal file
12
plugins/movie_tv_info/manifest.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "Movie/TV Info",
|
||||
"author": "ITDominator",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "context_menu_plugins",
|
||||
"pass_fm_events": "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
220
plugins/movie_tv_info/movie_tv_info.glade
Normal file
220
plugins/movie_tv_info/movie_tv_info.glade
Normal file
@@ -0,0 +1,220 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated with glade 3.38.2 -->
|
||||
<interface>
|
||||
<requires lib="gtk+" version="3.16"/>
|
||||
<object class="GtkImage" id="image1">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="stock">gtk-media-play</property>
|
||||
</object>
|
||||
<object class="GtkAdjustment" id="scrub_step_adjuster">
|
||||
<property name="lower">1</property>
|
||||
<property name="upper">100</property>
|
||||
<property name="value">65</property>
|
||||
<property name="step-increment">1</property>
|
||||
<property name="page-increment">10</property>
|
||||
</object>
|
||||
<object class="GtkTextBuffer" id="textbuffer"/>
|
||||
<object class="GtkDialog" id="info_dialog">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="border-width">6</property>
|
||||
<property name="title" translatable="yes">Movie / TV Info</property>
|
||||
<property name="modal">True</property>
|
||||
<property name="window-position">center-on-parent</property>
|
||||
<property name="default-width">420</property>
|
||||
<property name="destroy-with-parent">True</property>
|
||||
<property name="type-hint">dialog</property>
|
||||
<property name="skip-taskbar-hint">True</property>
|
||||
<property name="skip-pager-hint">True</property>
|
||||
<property name="gravity">center</property>
|
||||
<child internal-child="vbox">
|
||||
<object class="GtkBox" id="dialog_vbox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="spacing">12</property>
|
||||
<child internal-child="action_area">
|
||||
<object class="GtkButtonBox" id="dialog_action_area">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="layout-style">end</property>
|
||||
<child>
|
||||
<object class="GtkButton" id="cancel_button">
|
||||
<property name="label">gtk-close</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="can-default">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="use-stock">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">False</property>
|
||||
<property name="pack-type">end</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<object class="GtkImage" id="thumbnail_preview_img">
|
||||
<property name="height-request">320</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="margin-start">5</property>
|
||||
<property name="margin-end">5</property>
|
||||
<property name="margin-top">5</property>
|
||||
<property name="stock">gtk-missing-image</property>
|
||||
<property name="icon_size">6</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<object class="GtkTextView" id="overview_textview">
|
||||
<property name="height-request">120</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="margin-start">5</property>
|
||||
<property name="margin-end">5</property>
|
||||
<property name="margin-top">10</property>
|
||||
<property name="margin-bottom">10</property>
|
||||
<property name="wrap-mode">word</property>
|
||||
<property name="cursor-visible">False</property>
|
||||
<property name="buffer">textbuffer</property>
|
||||
<property name="overwrite">True</property>
|
||||
<property name="monospace">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkTable" id="general_table">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="border-width">4</property>
|
||||
<property name="n-rows">3</property>
|
||||
<property name="n-columns">2</property>
|
||||
<property name="column-spacing">12</property>
|
||||
<property name="row-spacing">6</property>
|
||||
<child>
|
||||
<object class="GtkLinkButton" id="trailer_link">
|
||||
<property name="label" translatable="yes">Trailer</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">True</property>
|
||||
<property name="image">image1</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="right-attach">2</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>File _Name:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>_Location:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_name">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="editable">False</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_location">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="editable">False</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">2</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
<action-widgets>
|
||||
<action-widget response="-7">cancel_button</action-widget>
|
||||
</action-widgets>
|
||||
</object>
|
||||
</interface>
|
||||
187
plugins/movie_tv_info/plugin.py
Normal file
187
plugins/movie_tv_info/plugin.py
Normal file
@@ -0,0 +1,187 @@
|
||||
# Python imports
|
||||
import os
|
||||
import threading
|
||||
import subprocess
|
||||
import inspect
|
||||
import requests
|
||||
import shutil
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
gi.require_version('GdkPixbuf', '2.0')
|
||||
from gi.repository import Gtk
|
||||
from gi.repository import GLib
|
||||
from gi.repository import GdkPixbuf
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
from .tmdbscraper import scraper
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
return wrapper
|
||||
|
||||
|
||||
|
||||
|
||||
class Plugin(PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.path = os.path.dirname(os.path.realpath(__file__))
|
||||
self.name = "Movie/TV Info" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
self._GLADE_FILE = f"{self.path}/movie_tv_info.glade"
|
||||
|
||||
self._dialog = None
|
||||
self._thumbnail_preview_img = None
|
||||
self._tmdb = scraper.get_tmdb_scraper()
|
||||
self._overview = None
|
||||
self._file_name = None
|
||||
self._file_location = None
|
||||
self._trailer_link = None
|
||||
|
||||
|
||||
def run(self):
|
||||
self._builder = Gtk.Builder()
|
||||
self._builder.add_from_file(self._GLADE_FILE)
|
||||
|
||||
classes = [self]
|
||||
handlers = {}
|
||||
for c in classes:
|
||||
methods = None
|
||||
try:
|
||||
methods = inspect.getmembers(c, predicate=inspect.ismethod)
|
||||
handlers.update(methods)
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
|
||||
self._builder.connect_signals(handlers)
|
||||
|
||||
self._thumbnailer_dialog = self._builder.get_object("info_dialog")
|
||||
self._overview = self._builder.get_object("textbuffer")
|
||||
self._file_name = self._builder.get_object("file_name")
|
||||
self._file_location = self._builder.get_object("file_location")
|
||||
self._thumbnail_preview_img = self._builder.get_object("thumbnail_preview_img")
|
||||
self._file_hash = self._builder.get_object("file_hash")
|
||||
self._trailer_link = self._builder.get_object("trailer_link")
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
item = Gtk.ImageMenuItem(self.name)
|
||||
item.set_image( Gtk.Image(stock=Gtk.STOCK_FIND) )
|
||||
item.connect("activate", self._show_info_page)
|
||||
item.set_always_show_image(True)
|
||||
return item
|
||||
|
||||
@threaded
|
||||
def _show_info_page(self, widget=None, eve=None):
|
||||
self._event_system.emit("get_current_state")
|
||||
|
||||
state = self._fm_state
|
||||
self._event_message = None
|
||||
|
||||
GLib.idle_add(self._process_changes, (state))
|
||||
|
||||
def _process_changes(self, state):
|
||||
self._fm_state = None
|
||||
|
||||
if len(state.selected_files) == 1:
|
||||
self._fm_state = state
|
||||
self._set_ui_data()
|
||||
response = self._thumbnailer_dialog.run()
|
||||
if response in [Gtk.ResponseType.CLOSE, Gtk.ResponseType.DELETE_EVENT]:
|
||||
self._thumbnailer_dialog.hide()
|
||||
|
||||
def _set_ui_data(self):
|
||||
title, path, trailer, video_data = self.get_video_data()
|
||||
keys = video_data.keys() if video_data else None
|
||||
|
||||
overview_text = video_data["overview"] if video_data else f"...NO {self.name.upper()} DATA..."
|
||||
|
||||
self.set_text_data(title, path, overview_text)
|
||||
self.set_thumbnail(video_data) if video_data else ...
|
||||
self.set_trailer_link(trailer)
|
||||
|
||||
print(video_data["videos"]) if not keys in ("", None) and "videos" in keys else ...
|
||||
|
||||
def get_video_data(self):
|
||||
uri = self._fm_state.selected_files[0]
|
||||
path = self._fm_state.tab.get_current_directory()
|
||||
parts = uri.split("/")
|
||||
_title = parts[ len(parts) - 1 ]
|
||||
trailer = None
|
||||
|
||||
try:
|
||||
title = _title.split("(")[0].strip()
|
||||
startIndex = _title.index('(') + 1
|
||||
endIndex = _title.index(')')
|
||||
date = title[startIndex:endIndex]
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
title = _title
|
||||
date = None
|
||||
|
||||
try:
|
||||
|
||||
video_data = self._tmdb.search(title, date)[0]
|
||||
video_id = video_data["id"]
|
||||
try:
|
||||
results = self._tmdb.tmdbapi.get_movie(str(video_id), append_to_response="videos")["videos"]["results"]
|
||||
for result in results:
|
||||
if "YouTube" in result["site"]:
|
||||
trailer = result["key"]
|
||||
|
||||
if not trailer:
|
||||
raise Exception("No key found. Defering to none...")
|
||||
except Exception as e:
|
||||
print("No trailer found...")
|
||||
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
video_data = None
|
||||
|
||||
return title, path, trailer, video_data
|
||||
|
||||
|
||||
def set_text_data(self, title, path, overview_text):
|
||||
self._file_name.set_text(title)
|
||||
self._file_location.set_text(path)
|
||||
self._overview.set_text(overview_text)
|
||||
|
||||
@threaded
|
||||
def set_thumbnail(self, video_data):
|
||||
background_url = video_data["backdrop_path"]
|
||||
# background_url = video_data["poster_path"]
|
||||
background_pth = "/tmp/sfm_mvtv_info.jpg"
|
||||
|
||||
try:
|
||||
os.remove(background_pth)
|
||||
except Exception as e:
|
||||
...
|
||||
|
||||
r = requests.get(background_url, stream = True)
|
||||
|
||||
if r.status_code == 200:
|
||||
r.raw.decode_content = True
|
||||
with open(background_pth,'wb') as f:
|
||||
shutil.copyfileobj(r.raw, f)
|
||||
|
||||
print('Cover Background Image sucessfully retreived...')
|
||||
preview_pixbuf = GdkPixbuf.Pixbuf.new_from_file(background_pth)
|
||||
self._thumbnail_preview_img.set_from_pixbuf(preview_pixbuf)
|
||||
else:
|
||||
print('Cover Background Image Couldn\'t be retreived...')
|
||||
|
||||
def set_trailer_link(self, trailer):
|
||||
if trailer:
|
||||
self._trailer_link.set_uri(f"https://www.youtube.com/watch?v={trailer}")
|
||||
3
plugins/movie_tv_info/tmdbscraper/__init__.py
Normal file
3
plugins/movie_tv_info/tmdbscraper/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
22
plugins/movie_tv_info/tmdbscraper/scraper.py
Normal file
22
plugins/movie_tv_info/tmdbscraper/scraper.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
from .lib.tmdbscraper.tmdb import TMDBMovieScraper
|
||||
from .lib.tmdbscraper.fanarttv import get_details as get_fanarttv_artwork
|
||||
from .lib.tmdbscraper.imdbratings import get_details as get_imdb_details
|
||||
from .lib.tmdbscraper.traktratings import get_trakt_ratinginfo
|
||||
from .scraper_datahelper import combine_scraped_details_info_and_ratings, \
|
||||
combine_scraped_details_available_artwork, find_uniqueids_in_text, get_params
|
||||
from .scraper_config import configure_scraped_details, PathSpecificSettings, \
|
||||
configure_tmdb_artwork, is_fanarttv_configured
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def get_tmdb_scraper():
|
||||
language = 'en-US'
|
||||
certcountry = 'us'
|
||||
ADDON_SETTINGS = None
|
||||
return TMDBMovieScraper(ADDON_SETTINGS, language, certcountry)
|
||||
111
plugins/movie_tv_info/tmdbscraper/scraper_config.py
Normal file
111
plugins/movie_tv_info/tmdbscraper/scraper_config.py
Normal file
@@ -0,0 +1,111 @@
|
||||
def configure_scraped_details(details, settings):
|
||||
details = _configure_rating_prefix(details, settings)
|
||||
details = _configure_keeporiginaltitle(details, settings)
|
||||
details = _configure_trailer(details, settings)
|
||||
details = _configure_multiple_studios(details, settings)
|
||||
details = _configure_default_rating(details, settings)
|
||||
details = _configure_tags(details, settings)
|
||||
return details
|
||||
|
||||
def configure_tmdb_artwork(details, settings):
|
||||
if 'available_art' not in details:
|
||||
return details
|
||||
|
||||
art = details['available_art']
|
||||
fanart_enabled = settings.getSettingBool('fanart')
|
||||
if not fanart_enabled:
|
||||
if 'fanart' in art:
|
||||
del art['fanart']
|
||||
if 'set.fanart' in art:
|
||||
del art['set.fanart']
|
||||
if not settings.getSettingBool('landscape'):
|
||||
if 'landscape' in art:
|
||||
if fanart_enabled:
|
||||
art['fanart'] = art.get('fanart', []) + art['landscape']
|
||||
del art['landscape']
|
||||
if 'set.landscape' in art:
|
||||
if fanart_enabled:
|
||||
art['set.fanart'] = art.get('set.fanart', []) + art['set.landscape']
|
||||
del art['set.landscape']
|
||||
|
||||
return details
|
||||
|
||||
def is_fanarttv_configured(settings):
|
||||
return settings.getSettingBool('enable_fanarttv_artwork')
|
||||
|
||||
def _configure_rating_prefix(details, settings):
|
||||
if details['info'].get('mpaa'):
|
||||
details['info']['mpaa'] = settings.getSettingString('certprefix') + details['info']['mpaa']
|
||||
return details
|
||||
|
||||
def _configure_keeporiginaltitle(details, settings):
|
||||
if settings.getSettingBool('keeporiginaltitle'):
|
||||
details['info']['title'] = details['info']['originaltitle']
|
||||
return details
|
||||
|
||||
def _configure_trailer(details, settings):
|
||||
if details['info'].get('trailer') and not settings.getSettingBool('trailer'):
|
||||
del details['info']['trailer']
|
||||
return details
|
||||
|
||||
def _configure_multiple_studios(details, settings):
|
||||
if not settings.getSettingBool('multiple_studios'):
|
||||
details['info']['studio'] = details['info']['studio'][:1]
|
||||
return details
|
||||
|
||||
def _configure_default_rating(details, settings):
|
||||
imdb_default = bool(details['ratings'].get('imdb')) and settings.getSettingString('RatingS') == 'IMDb'
|
||||
trakt_default = bool(details['ratings'].get('trakt')) and settings.getSettingString('RatingS') == 'Trakt'
|
||||
default_rating = 'themoviedb'
|
||||
if imdb_default:
|
||||
default_rating = 'imdb'
|
||||
elif trakt_default:
|
||||
default_rating = 'trakt'
|
||||
if default_rating not in details['ratings']:
|
||||
default_rating = list(details['ratings'].keys())[0] if details['ratings'] else None
|
||||
for rating_type in details['ratings'].keys():
|
||||
details['ratings'][rating_type]['default'] = rating_type == default_rating
|
||||
return details
|
||||
|
||||
def _configure_tags(details, settings):
|
||||
if not settings.getSettingBool('add_tags'):
|
||||
del details['info']['tag']
|
||||
return details
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
try:
|
||||
basestring
|
||||
except NameError: # py2 / py3
|
||||
basestring = str
|
||||
|
||||
#pylint: disable=redefined-builtin
|
||||
class PathSpecificSettings(object):
|
||||
# read-only shim for typed `xbmcaddon.Addon().getSetting*` methods
|
||||
def __init__(self, settings_dict, log_fn):
|
||||
self.data = settings_dict
|
||||
self.log = log_fn
|
||||
|
||||
def getSettingBool(self, id):
|
||||
return self._inner_get_setting(id, bool, False)
|
||||
|
||||
def getSettingInt(self, id):
|
||||
return self._inner_get_setting(id, int, 0)
|
||||
|
||||
def getSettingNumber(self, id):
|
||||
return self._inner_get_setting(id, float, 0.0)
|
||||
|
||||
def getSettingString(self, id):
|
||||
return self._inner_get_setting(id, basestring, '')
|
||||
|
||||
def _inner_get_setting(self, setting_id, setting_type, default):
|
||||
value = self.data.get(setting_id)
|
||||
if isinstance(value, setting_type):
|
||||
return value
|
||||
self._log_bad_value(value, setting_id)
|
||||
return default
|
||||
|
||||
def _log_bad_value(self, value, setting_id):
|
||||
if value is None:
|
||||
self.log("requested setting ({0}) was not found.".format(setting_id))
|
||||
else:
|
||||
self.log('failed to load value "{0}" for setting {1}'.format(value, setting_id))
|
||||
54
plugins/movie_tv_info/tmdbscraper/scraper_datahelper.py
Normal file
54
plugins/movie_tv_info/tmdbscraper/scraper_datahelper.py
Normal file
@@ -0,0 +1,54 @@
|
||||
import re
|
||||
try:
|
||||
from urlparse import parse_qsl
|
||||
except ImportError: # py2 / py3
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
# get addon params from the plugin path querystring
|
||||
def get_params(argv):
|
||||
result = {'handle': int(argv[0])}
|
||||
if len(argv) < 2 or not argv[1]:
|
||||
return result
|
||||
|
||||
result.update(parse_qsl(argv[1].lstrip('?')))
|
||||
return result
|
||||
|
||||
def combine_scraped_details_info_and_ratings(original_details, additional_details):
|
||||
def update_or_set(details, key, value):
|
||||
if key in details:
|
||||
details[key].update(value)
|
||||
else:
|
||||
details[key] = value
|
||||
|
||||
if additional_details:
|
||||
if additional_details.get('info'):
|
||||
update_or_set(original_details, 'info', additional_details['info'])
|
||||
if additional_details.get('ratings'):
|
||||
update_or_set(original_details, 'ratings', additional_details['ratings'])
|
||||
return original_details
|
||||
|
||||
def combine_scraped_details_available_artwork(original_details, additional_details):
|
||||
if additional_details and additional_details.get('available_art'):
|
||||
available_art = additional_details['available_art']
|
||||
if not original_details.get('available_art'):
|
||||
original_details['available_art'] = available_art
|
||||
else:
|
||||
for arttype, artlist in available_art.items():
|
||||
original_details['available_art'][arttype] = \
|
||||
artlist + original_details['available_art'].get(arttype, [])
|
||||
|
||||
return original_details
|
||||
|
||||
def find_uniqueids_in_text(input_text):
|
||||
result = {}
|
||||
res = re.search(r'(themoviedb.org/movie/)([0-9]+)', input_text)
|
||||
if (res):
|
||||
result['tmdb'] = res.group(2)
|
||||
res = re.search(r'imdb....?/title/tt([0-9]+)', input_text)
|
||||
if (res):
|
||||
result['imdb'] = 'tt' + res.group(1)
|
||||
else:
|
||||
res = re.search(r'imdb....?/Title\?t{0,2}([0-9]+)', input_text)
|
||||
if (res):
|
||||
result['imdb'] = 'tt' + res.group(1)
|
||||
return result
|
||||
175
plugins/movie_tv_info/tmdbscraper/scraper_xbmc.py
Normal file
175
plugins/movie_tv_info/tmdbscraper/scraper_xbmc.py
Normal file
@@ -0,0 +1,175 @@
|
||||
import json
|
||||
import sys
|
||||
import xbmc
|
||||
import xbmcaddon
|
||||
import xbmcgui
|
||||
import xbmcplugin
|
||||
|
||||
from lib.tmdbscraper.tmdb import TMDBMovieScraper
|
||||
from lib.tmdbscraper.fanarttv import get_details as get_fanarttv_artwork
|
||||
from lib.tmdbscraper.imdbratings import get_details as get_imdb_details
|
||||
from lib.tmdbscraper.traktratings import get_trakt_ratinginfo
|
||||
from scraper_datahelper import combine_scraped_details_info_and_ratings, \
|
||||
combine_scraped_details_available_artwork, find_uniqueids_in_text, get_params
|
||||
from scraper_config import configure_scraped_details, PathSpecificSettings, \
|
||||
configure_tmdb_artwork, is_fanarttv_configured
|
||||
|
||||
ADDON_SETTINGS = xbmcaddon.Addon()
|
||||
ID = ADDON_SETTINGS.getAddonInfo('id')
|
||||
|
||||
def log(msg, level=xbmc.LOGDEBUG):
|
||||
xbmc.log(msg='[{addon}]: {msg}'.format(addon=ID, msg=msg), level=level)
|
||||
|
||||
def get_tmdb_scraper(settings):
|
||||
language = settings.getSettingString('language')
|
||||
certcountry = settings.getSettingString('tmdbcertcountry')
|
||||
return TMDBMovieScraper(ADDON_SETTINGS, language, certcountry)
|
||||
|
||||
def search_for_movie(title, year, handle, settings):
|
||||
log("Find movie with title '{title}' from year '{year}'".format(title=title, year=year), xbmc.LOGINFO)
|
||||
title = _strip_trailing_article(title)
|
||||
search_results = get_tmdb_scraper(settings).search(title, year)
|
||||
if not search_results:
|
||||
return
|
||||
if 'error' in search_results:
|
||||
header = "The Movie Database Python error searching with web service TMDB"
|
||||
xbmcgui.Dialog().notification(header, search_results['error'], xbmcgui.NOTIFICATION_WARNING)
|
||||
log(header + ': ' + search_results['error'], xbmc.LOGWARNING)
|
||||
return
|
||||
|
||||
for movie in search_results:
|
||||
listitem = _searchresult_to_listitem(movie)
|
||||
uniqueids = {'tmdb': str(movie['id'])}
|
||||
xbmcplugin.addDirectoryItem(handle=handle, url=build_lookup_string(uniqueids),
|
||||
listitem=listitem, isFolder=True)
|
||||
|
||||
_articles = [prefix + article for prefix in (', ', ' ') for article in ("the", "a", "an")]
|
||||
def _strip_trailing_article(title):
|
||||
title = title.lower()
|
||||
for article in _articles:
|
||||
if title.endswith(article):
|
||||
return title[:-len(article)]
|
||||
return title
|
||||
|
||||
def _searchresult_to_listitem(movie):
|
||||
movie_info = {'title': movie['title']}
|
||||
movie_label = movie['title']
|
||||
|
||||
movie_year = movie['release_date'].split('-')[0] if movie.get('release_date') else None
|
||||
if movie_year:
|
||||
movie_label += ' ({})'.format(movie_year)
|
||||
movie_info['year'] = movie_year
|
||||
|
||||
listitem = xbmcgui.ListItem(movie_label, offscreen=True)
|
||||
|
||||
listitem.setInfo('video', movie_info)
|
||||
if movie['poster_path']:
|
||||
listitem.setArt({'thumb': movie['poster_path']})
|
||||
|
||||
return listitem
|
||||
|
||||
# Low limit because a big list of artwork can cause trouble in some cases
|
||||
# (a column can be too large for the MySQL integration),
|
||||
# and how useful is a big list anyway? Not exactly rhetorical, this is an experiment.
|
||||
IMAGE_LIMIT = 10
|
||||
|
||||
def add_artworks(listitem, artworks):
|
||||
for arttype, artlist in artworks.items():
|
||||
if arttype == 'fanart':
|
||||
continue
|
||||
for image in artlist[:IMAGE_LIMIT]:
|
||||
listitem.addAvailableArtwork(image['url'], arttype)
|
||||
|
||||
fanart_to_set = [{'image': image['url'], 'preview': image['preview']}
|
||||
for image in artworks['fanart'][:IMAGE_LIMIT]]
|
||||
listitem.setAvailableFanart(fanart_to_set)
|
||||
|
||||
def get_details(input_uniqueids, handle, settings):
|
||||
if not input_uniqueids:
|
||||
return False
|
||||
details = get_tmdb_scraper(settings).get_details(input_uniqueids)
|
||||
if not details:
|
||||
return False
|
||||
if 'error' in details:
|
||||
header = "The Movie Database Python error with web service TMDB"
|
||||
xbmcgui.Dialog().notification(header, details['error'], xbmcgui.NOTIFICATION_WARNING)
|
||||
log(header + ': ' + details['error'], xbmc.LOGWARNING)
|
||||
return False
|
||||
|
||||
details = configure_tmdb_artwork(details, settings)
|
||||
|
||||
if settings.getSettingString('RatingS') == 'IMDb' or settings.getSettingBool('imdbanyway'):
|
||||
imdbinfo = get_imdb_details(details['uniqueids'])
|
||||
if 'error' in imdbinfo:
|
||||
header = "The Movie Database Python error with website IMDB"
|
||||
log(header + ': ' + imdbinfo['error'], xbmc.LOGWARNING)
|
||||
else:
|
||||
details = combine_scraped_details_info_and_ratings(details, imdbinfo)
|
||||
|
||||
if settings.getSettingString('RatingS') == 'Trakt' or settings.getSettingBool('traktanyway'):
|
||||
traktinfo = get_trakt_ratinginfo(details['uniqueids'])
|
||||
details = combine_scraped_details_info_and_ratings(details, traktinfo)
|
||||
|
||||
if is_fanarttv_configured(settings):
|
||||
fanarttv_info = get_fanarttv_artwork(details['uniqueids'],
|
||||
settings.getSettingString('fanarttv_clientkey'),
|
||||
settings.getSettingString('fanarttv_language'),
|
||||
details['_info']['set_tmdbid'])
|
||||
details = combine_scraped_details_available_artwork(details, fanarttv_info)
|
||||
|
||||
details = configure_scraped_details(details, settings)
|
||||
|
||||
listitem = xbmcgui.ListItem(details['info']['title'], offscreen=True)
|
||||
listitem.setInfo('video', details['info'])
|
||||
listitem.setCast(details['cast'])
|
||||
listitem.setUniqueIDs(details['uniqueids'], 'tmdb')
|
||||
add_artworks(listitem, details['available_art'])
|
||||
|
||||
for rating_type, value in details['ratings'].items():
|
||||
if 'votes' in value:
|
||||
listitem.setRating(rating_type, value['rating'], value['votes'], value['default'])
|
||||
else:
|
||||
listitem.setRating(rating_type, value['rating'], defaultt=value['default'])
|
||||
|
||||
xbmcplugin.setResolvedUrl(handle=handle, succeeded=True, listitem=listitem)
|
||||
return True
|
||||
|
||||
def find_uniqueids_in_nfo(nfo, handle):
|
||||
uniqueids = find_uniqueids_in_text(nfo)
|
||||
if uniqueids:
|
||||
listitem = xbmcgui.ListItem(offscreen=True)
|
||||
xbmcplugin.addDirectoryItem(
|
||||
handle=handle, url=build_lookup_string(uniqueids), listitem=listitem, isFolder=True)
|
||||
|
||||
def build_lookup_string(uniqueids):
|
||||
return json.dumps(uniqueids)
|
||||
|
||||
def parse_lookup_string(uniqueids):
|
||||
try:
|
||||
return json.loads(uniqueids)
|
||||
except ValueError:
|
||||
log("Can't parse this lookup string, is it from another add-on?\n" + uniqueids, xbmc.LOGWARNING)
|
||||
return None
|
||||
|
||||
def run():
|
||||
params = get_params(sys.argv[1:])
|
||||
enddir = True
|
||||
if 'action' in params:
|
||||
settings = ADDON_SETTINGS if not params.get('pathSettings') else \
|
||||
PathSpecificSettings(json.loads(params['pathSettings']), lambda msg: log(msg, xbmc.LOGWARNING))
|
||||
action = params["action"]
|
||||
if action == 'find' and 'title' in params:
|
||||
search_for_movie(params["title"], params.get("year"), params['handle'], settings)
|
||||
elif action == 'getdetails' and 'url' in params:
|
||||
enddir = not get_details(parse_lookup_string(params["url"]), params['handle'], settings)
|
||||
elif action == 'NfoUrl' and 'nfo' in params:
|
||||
find_uniqueids_in_nfo(params["nfo"], params['handle'])
|
||||
else:
|
||||
log("unhandled action: " + action, xbmc.LOGWARNING)
|
||||
else:
|
||||
log("No action in 'params' to act on", xbmc.LOGWARNING)
|
||||
if enddir:
|
||||
xbmcplugin.endOfDirectory(params['handle'])
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
3
plugins/searcher/__init__.py
Normal file
3
plugins/searcher/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/searcher/__main__.py
Normal file
3
plugins/searcher/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
13
plugins/searcher/manifest.json
Normal file
13
plugins/searcher/manifest.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "Search",
|
||||
"author": "ITDominator",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "context_menu",
|
||||
"pass_fm_events": "true",
|
||||
"bind_keys": ["Search||show_search_page:<Control>s"]
|
||||
}
|
||||
}
|
||||
}
|
||||
3
plugins/searcher/mixins/__init__.py
Normal file
3
plugins/searcher/mixins/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Mixins Module
|
||||
"""
|
||||
82
plugins/searcher/mixins/file_search_mixin.py
Normal file
82
plugins/searcher/mixins/file_search_mixin.py
Normal file
@@ -0,0 +1,82 @@
|
||||
# Python imports
|
||||
import threading
|
||||
import subprocess
|
||||
import signal
|
||||
import json
|
||||
import shlex
|
||||
from datetime import datetime
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
from gi.repository import GLib
|
||||
|
||||
# Application imports
|
||||
from ..widgets.file_preview_widget import FilePreviewWidget
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class FileSearchMixin:
|
||||
def _run_find_file_query(self, widget=None, eve=None):
|
||||
self._handle_find_file_query(query=widget)
|
||||
|
||||
# TODO: Merge this logic with nearly the exact same thing in grep_search_mixin
|
||||
@daemon_threaded
|
||||
def _handle_find_file_query(self, widget=None, eve=None, query=None):
|
||||
# NOTE: Freeze IPC consumption
|
||||
self.pause_fifo_update = True
|
||||
self.search_query = ""
|
||||
dt = datetime.now()
|
||||
self.fsearch_time_stamp = datetime.timestamp(dt) # NOTE: Get timestamp
|
||||
|
||||
# NOTE: Kill the former process
|
||||
if self._list_proc:
|
||||
if self._list_proc.poll() == None:
|
||||
self._list_proc.terminate()
|
||||
while self._list_proc.poll() == None:
|
||||
...
|
||||
|
||||
self._list_proc = None
|
||||
|
||||
# NOTE: Clear children from ui and make sure ui thread redraws
|
||||
GLib.idle_add(self.reset_file_list_box)
|
||||
|
||||
# NOTE: If query create new process and do all new loop.
|
||||
if query:
|
||||
self.pause_fifo_update = False
|
||||
GLib.idle_add(self._exec_find_file_query, query)
|
||||
|
||||
def _exec_find_file_query(self, widget=None, eve=None):
|
||||
query = widget.get_text()
|
||||
|
||||
if not query in ("", None):
|
||||
self.search_query = query
|
||||
target_dir = shlex.quote( self._fm_state.tab.get_current_directory() )
|
||||
command = ["python", f"{self.path}/utils/search.py", "-t", "file_search", "-d", f"{target_dir}", "-q", f"{query}"]
|
||||
self._list_proc = subprocess.Popen(command, cwd=self.path, stdin=None, stdout=None, stderr=None)
|
||||
|
||||
def _load_file_ui(self, data):
|
||||
Gtk.main_iteration()
|
||||
|
||||
if not data in ("", None):
|
||||
jdata = json.loads( data )
|
||||
target = jdata[0]
|
||||
file = jdata[1]
|
||||
|
||||
widget = FilePreviewWidget(target, file)
|
||||
self._file_list.add(widget)
|
||||
86
plugins/searcher/mixins/grep_search_mixin.py
Normal file
86
plugins/searcher/mixins/grep_search_mixin.py
Normal file
@@ -0,0 +1,86 @@
|
||||
# Python imports
|
||||
import ctypes
|
||||
import threading
|
||||
import subprocess
|
||||
import signal
|
||||
import json
|
||||
import shlex
|
||||
from datetime import datetime
|
||||
libgcc_s = ctypes.CDLL('libgcc_s.so.1')
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
from gi.repository import GLib
|
||||
|
||||
# Application imports
|
||||
from ..widgets.grep_preview_widget import GrepPreviewWidget
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class GrepSearchMixin:
|
||||
def _run_grep_query(self, widget=None, eve=None):
|
||||
self._handle_grep_query(query=widget)
|
||||
|
||||
# TODO: Merge this logic with nearly the exact same thing in file_search_mixin
|
||||
@daemon_threaded
|
||||
def _handle_grep_query(self, widget=None, eve=None, query=None):
|
||||
# NOTE: Freeze IPC consumption
|
||||
self.pause_fifo_update = True
|
||||
self.grep_query = ""
|
||||
dt = datetime.now()
|
||||
self.grep_time_stamp = datetime.timestamp(dt) # NOTE: Get timestamp
|
||||
|
||||
# NOTE: Kill the former process
|
||||
if self._grep_proc:
|
||||
if self._grep_proc.poll() == None:
|
||||
self._grep_proc.terminate()
|
||||
while self._grep_proc.poll() == None:
|
||||
...
|
||||
|
||||
self._grep_proc = None
|
||||
|
||||
# NOTE: Clear children from ui and make sure ui thread redraws
|
||||
GLib.idle_add(self.reset_grep_box)
|
||||
|
||||
# NOTE: If query create new process and do all new loop.
|
||||
if query:
|
||||
self.pause_fifo_update = False
|
||||
GLib.idle_add(self._exec_grep_query, query)
|
||||
|
||||
def _exec_grep_query(self, widget=None, eve=None):
|
||||
query = widget.get_text()
|
||||
if not query.strip() in ("", None):
|
||||
self.grep_query = query
|
||||
|
||||
target_dir = shlex.quote( self._fm_state.tab.get_current_directory() )
|
||||
command = ["python", f"{self.path}/utils/search.py", "-t", "grep_search", "-d", f"{target_dir}", "-q", f"{query}"]
|
||||
self._grep_proc = subprocess.Popen(command, cwd=self.path, stdin=None, stdout=None, stderr=None)
|
||||
|
||||
def _load_grep_ui(self, data):
|
||||
Gtk.main_iteration()
|
||||
|
||||
if not data in ("", None):
|
||||
jdata = json.loads( data )
|
||||
jkeys = jdata.keys()
|
||||
for key in jkeys:
|
||||
sub_keys = jdata[key].keys()
|
||||
grep_result = jdata[key]
|
||||
|
||||
widget = GrepPreviewWidget(key, sub_keys, grep_result, self.grep_query)
|
||||
self._grep_list.add(widget)
|
||||
139
plugins/searcher/plugin.py
Normal file
139
plugins/searcher/plugin.py
Normal file
@@ -0,0 +1,139 @@
|
||||
# Python imports
|
||||
import os
|
||||
import threading
|
||||
import inspect
|
||||
import time
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
from gi.repository import GLib
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
from .mixins.file_search_mixin import FileSearchMixin
|
||||
from .mixins.grep_search_mixin import GrepSearchMixin
|
||||
from .utils.ipc_server import IPCServer
|
||||
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
return wrapper
|
||||
|
||||
|
||||
|
||||
|
||||
class Plugin(IPCServer, FileSearchMixin, GrepSearchMixin, PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.path = os.path.dirname(os.path.realpath(__file__))
|
||||
self.name = "Search" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
self._GLADE_FILE = f"{self.path}/search_dialog.glade"
|
||||
|
||||
self.update_list_ui_buffer = ()
|
||||
self._search_dialog = None
|
||||
self._active_path = None
|
||||
self.file_list_parent = None
|
||||
self.grep_list_parent = None
|
||||
self._file_list = None
|
||||
self._grep_list = None
|
||||
self._grep_proc = None
|
||||
self._list_proc = None
|
||||
self.pause_fifo_update = False
|
||||
self.grep_time_stamp = None
|
||||
self.fsearch_time_stamp = None
|
||||
self.grep_query = ""
|
||||
self.search_query = ""
|
||||
|
||||
|
||||
def run(self):
|
||||
self._builder = Gtk.Builder()
|
||||
self._builder.add_from_file(self._GLADE_FILE)
|
||||
|
||||
classes = [self]
|
||||
handlers = {}
|
||||
for c in classes:
|
||||
methods = None
|
||||
try:
|
||||
methods = inspect.getmembers(c, predicate=inspect.ismethod)
|
||||
handlers.update(methods)
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
|
||||
self._builder.connect_signals(handlers)
|
||||
|
||||
self._search_dialog = self._builder.get_object("search_dialog")
|
||||
self.fsearch = self._builder.get_object("fsearch")
|
||||
|
||||
self.grep_list_parent = self._builder.get_object("grep_list_parent")
|
||||
self.file_list_parent = self._builder.get_object("file_list_parent")
|
||||
|
||||
self._event_system.subscribe("update-file-ui", self._load_file_ui)
|
||||
self._event_system.subscribe("update-grep-ui", self._load_grep_ui)
|
||||
self._event_system.subscribe("show_search_page", self._show_page)
|
||||
|
||||
|
||||
self.create_ipc_listener()
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
item = Gtk.ImageMenuItem(self.name)
|
||||
item.set_image( Gtk.Image(stock=Gtk.STOCK_FIND) )
|
||||
item.connect("activate", self._show_page)
|
||||
item.set_always_show_image(True)
|
||||
return item
|
||||
|
||||
|
||||
def _show_page(self, widget=None, eve=None):
|
||||
self._event_system.emit("get_current_state")
|
||||
|
||||
state = self._fm_state
|
||||
self._event_message = None
|
||||
|
||||
self._active_path = state.tab.get_current_directory()
|
||||
response = self._search_dialog.run()
|
||||
self._search_dialog.hide()
|
||||
|
||||
# TODO: Merge the below methods into some unified logic
|
||||
def reset_grep_box(self) -> None:
|
||||
try:
|
||||
child = self.grep_list_parent.get_children()[0]
|
||||
self._grep_list = None
|
||||
self.grep_list_parent.remove(child)
|
||||
except Exception:
|
||||
...
|
||||
|
||||
self._grep_list = Gtk.Box()
|
||||
self._grep_list.set_orientation(Gtk.Orientation.VERTICAL)
|
||||
self.grep_list_parent.add(self._grep_list)
|
||||
self.grep_list_parent.show_all()
|
||||
|
||||
time.sleep(0.05)
|
||||
Gtk.main_iteration()
|
||||
|
||||
def reset_file_list_box(self) -> None:
|
||||
try:
|
||||
child = self.file_list_parent.get_children()[0]
|
||||
self._file_list = None
|
||||
self.file_list_parent.remove(child)
|
||||
except Exception:
|
||||
...
|
||||
|
||||
self._file_list = Gtk.Box()
|
||||
self._file_list.set_orientation(Gtk.Orientation.VERTICAL)
|
||||
self.file_list_parent.add(self._file_list)
|
||||
self.file_list_parent.show_all()
|
||||
|
||||
time.sleep(0.05)
|
||||
Gtk.main_iteration()
|
||||
277
plugins/searcher/search_dialog.glade
Normal file
277
plugins/searcher/search_dialog.glade
Normal file
@@ -0,0 +1,277 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated with glade 3.38.2 -->
|
||||
<interface>
|
||||
<requires lib="gtk+" version="3.16"/>
|
||||
<object class="GtkDialog" id="search_dialog">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="border-width">6</property>
|
||||
<property name="title" translatable="yes">Search...</property>
|
||||
<property name="modal">True</property>
|
||||
<property name="window-position">center-on-parent</property>
|
||||
<property name="default-width">720</property>
|
||||
<property name="default-height">620</property>
|
||||
<property name="destroy-with-parent">True</property>
|
||||
<property name="type-hint">dialog</property>
|
||||
<property name="skip-taskbar-hint">True</property>
|
||||
<property name="skip-pager-hint">True</property>
|
||||
<property name="gravity">center</property>
|
||||
<child internal-child="vbox">
|
||||
<object class="GtkBox" id="dialog_vbox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="spacing">12</property>
|
||||
<child internal-child="action_area">
|
||||
<object class="GtkButtonBox" id="dialog_action_area">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="layout-style">end</property>
|
||||
<child>
|
||||
<object class="GtkButton" id="cancel_button">
|
||||
<property name="label">gtk-cancel</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="can-default">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="use-stock">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkButton" id="ok_button">
|
||||
<property name="label">gtk-ok</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="can-default">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="use-stock">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">False</property>
|
||||
<property name="pack-type">end</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkNotebook">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="show-border">False</property>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<child>
|
||||
<object class="GtkSearchEntry" id="fsearch">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="tooltip-text" translatable="yes">Query...</property>
|
||||
<property name="primary-icon-name">edit-find-symbolic</property>
|
||||
<property name="primary-icon-activatable">False</property>
|
||||
<property name="primary-icon-sensitive">False</property>
|
||||
<property name="placeholder-text" translatable="yes">Search for file...</property>
|
||||
<signal name="search-changed" handler="_run_find_file_query" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkButton">
|
||||
<property name="label">gtk-stop</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">True</property>
|
||||
<property name="use-stock">True</property>
|
||||
<signal name="released" handler="_handle_find_file_query" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkScrolledWindow">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="hexpand">True</property>
|
||||
<property name="vexpand">True</property>
|
||||
<property name="shadow-type">in</property>
|
||||
<child>
|
||||
<object class="GtkViewport">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<child>
|
||||
<object class="GtkBox" id="file_list_parent">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<property name="spacing">5</property>
|
||||
<property name="baseline-position">top</property>
|
||||
<child>
|
||||
<placeholder/>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
<child type="tab">
|
||||
<object class="GtkLabel">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes">File Search</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="tab-fill">False</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<child>
|
||||
<object class="GtkSearchEntry">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="tooltip-text" translatable="yes">Query...</property>
|
||||
<property name="primary-icon-name">edit-find-symbolic</property>
|
||||
<property name="primary-icon-activatable">False</property>
|
||||
<property name="primary-icon-sensitive">False</property>
|
||||
<property name="placeholder-text" translatable="yes">Query string in file...</property>
|
||||
<signal name="search-changed" handler="_run_grep_query" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkButton">
|
||||
<property name="label">gtk-stop</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="receives-default">True</property>
|
||||
<property name="use-stock">True</property>
|
||||
<signal name="released" handler="_handle_grep_query" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkScrolledWindow">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="hexpand">True</property>
|
||||
<property name="vexpand">True</property>
|
||||
<property name="shadow-type">in</property>
|
||||
<child>
|
||||
<object class="GtkViewport">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<child>
|
||||
<object class="GtkBox" id="grep_list_parent">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<placeholder/>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child type="tab">
|
||||
<object class="GtkLabel">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes">Grep Search</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="position">1</property>
|
||||
<property name="tab-fill">False</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<placeholder/>
|
||||
</child>
|
||||
<child type="tab">
|
||||
<placeholder/>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
<action-widgets>
|
||||
<action-widget response="-6">cancel_button</action-widget>
|
||||
<action-widget response="-5">ok_button</action-widget>
|
||||
</action-widgets>
|
||||
</object>
|
||||
</interface>
|
||||
98
plugins/searcher/utils/ipc_server.py
Normal file
98
plugins/searcher/utils/ipc_server.py
Normal file
@@ -0,0 +1,98 @@
|
||||
# Python imports
|
||||
import os, threading, pickle
|
||||
from multiprocessing.connection import Listener, Client
|
||||
|
||||
# Lib imports
|
||||
from gi.repository import GLib
|
||||
|
||||
# Application imports
|
||||
|
||||
|
||||
|
||||
|
||||
class IPCServer:
|
||||
""" Create a listener so that other SolarFM instances send requests back to existing instance. """
|
||||
def __init__(self, ipc_address: str = '127.0.0.1', conn_type: str = "socket"):
|
||||
self.is_ipc_alive = False
|
||||
self._ipc_port = 4848
|
||||
self._ipc_address = ipc_address
|
||||
self._conn_type = conn_type
|
||||
self._ipc_authkey = b'' + bytes(f'solarfm-search_grep-ipc', 'utf-8')
|
||||
self._ipc_timeout = 15.0
|
||||
|
||||
if conn_type == "socket":
|
||||
self._ipc_address = f'/tmp/solarfm-search_grep-ipc.sock'
|
||||
elif conn_type == "full_network":
|
||||
self._ipc_address = '0.0.0.0'
|
||||
elif conn_type == "full_network_unsecured":
|
||||
self._ipc_authkey = None
|
||||
self._ipc_address = '0.0.0.0'
|
||||
elif conn_type == "local_network_unsecured":
|
||||
self._ipc_authkey = None
|
||||
|
||||
|
||||
@daemon_threaded
|
||||
def create_ipc_listener(self) -> None:
|
||||
if self._conn_type == "socket":
|
||||
if os.path.exists(self._ipc_address):
|
||||
os.unlink(self._ipc_address)
|
||||
|
||||
listener = Listener(address=self._ipc_address, family="AF_UNIX", authkey=self._ipc_authkey)
|
||||
elif "unsecured" not in self._conn_type:
|
||||
listener = Listener((self._ipc_address, self._ipc_port), authkey=self._ipc_authkey)
|
||||
else:
|
||||
listener = Listener((self._ipc_address, self._ipc_port))
|
||||
|
||||
|
||||
self.is_ipc_alive = True
|
||||
while True:
|
||||
conn = listener.accept()
|
||||
|
||||
if not self.pause_fifo_update:
|
||||
self.handle_message(conn)
|
||||
else:
|
||||
conn.close()
|
||||
|
||||
listener.close()
|
||||
|
||||
def handle_message(self, conn) -> None:
|
||||
while True:
|
||||
msg = conn.recv()
|
||||
|
||||
if "SEARCH|" in msg:
|
||||
ts, file = msg.split("SEARCH|")[1].strip().split("|", 1)
|
||||
try:
|
||||
timestamp = float(ts)
|
||||
if timestamp > self.fsearch_time_stamp and file:
|
||||
GLib.idle_add(self._load_file_ui, file, priority=GLib.PRIORITY_LOW)
|
||||
except Exception as e:
|
||||
...
|
||||
|
||||
if "GREP|" in msg:
|
||||
ts, data = msg.split("GREP|")[1].strip().split("|", 1)
|
||||
try:
|
||||
timestamp = float(ts)
|
||||
if timestamp > self.grep_time_stamp and data:
|
||||
GLib.idle_add(self._load_grep_ui, data, priority=GLib.PRIORITY_LOW)
|
||||
except Exception as e:
|
||||
...
|
||||
|
||||
|
||||
conn.close()
|
||||
break
|
||||
|
||||
def send_ipc_message(self, message: str = "Empty Data...") -> None:
|
||||
try:
|
||||
if self._conn_type == "socket":
|
||||
conn = Client(address=self._ipc_address, family="AF_UNIX", authkey=self._ipc_authkey)
|
||||
elif "unsecured" not in self._conn_type:
|
||||
conn = Client((self._ipc_address, self._ipc_port), authkey=self._ipc_authkey)
|
||||
else:
|
||||
conn = Client((self._ipc_address, self._ipc_port))
|
||||
|
||||
conn.send(message)
|
||||
conn.close()
|
||||
except ConnectionRefusedError as e:
|
||||
print("Connection refused...")
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
111
plugins/searcher/utils/search.py
Executable file
111
plugins/searcher/utils/search.py
Executable file
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
|
||||
# Python imports
|
||||
import os
|
||||
import traceback
|
||||
import argparse
|
||||
import subprocess
|
||||
import json
|
||||
import base64
|
||||
import time
|
||||
from datetime import datetime
|
||||
from setproctitle import setproctitle
|
||||
from multiprocessing.connection import Client
|
||||
|
||||
# Lib imports
|
||||
|
||||
# Application imports
|
||||
|
||||
|
||||
|
||||
|
||||
_ipc_address = f'/tmp/solarfm-search_grep-ipc.sock'
|
||||
_ipc_authkey = b'' + bytes(f'solarfm-search_grep-ipc', 'utf-8')
|
||||
|
||||
filter = (".cpp", ".css", ".c", ".go", ".html", ".htm", ".java", ".js", ".json", ".lua", ".md", ".py", ".rs", ".toml", ".xml", ".pom") + \
|
||||
(".txt", ".text", ".sh", ".cfg", ".conf", ".log")
|
||||
|
||||
# NOTE: Create timestamp of when this launched. Is used in IPC to see if
|
||||
# we are stale and that new call didn't fully kill this or older processes.
|
||||
dt = datetime.now()
|
||||
ts = datetime.timestamp(dt)
|
||||
|
||||
|
||||
def send_ipc_message(message) -> None:
|
||||
conn = Client(address=_ipc_address, family="AF_UNIX", authkey=_ipc_authkey)
|
||||
conn.send(message)
|
||||
conn.close()
|
||||
|
||||
# NOTE: Kinda important as this prevents overloading the UI thread
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def file_search(path, query):
|
||||
try:
|
||||
for _path, _dir, _files in os.walk(path, topdown = True):
|
||||
for file in _files:
|
||||
if query in file.lower():
|
||||
target = os.path.join(_path, file)
|
||||
data = f"SEARCH|{ts}|{json.dumps([target, file])}"
|
||||
send_ipc_message(data)
|
||||
except Exception as e:
|
||||
print("Couldn't traverse to path. Might be permissions related...")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def grep_search(target=None, query=None):
|
||||
if not query or not target:
|
||||
return
|
||||
|
||||
# NOTE: -n = provide line numbers, -R = Search recursive in given target
|
||||
# -i = insensitive, -F = don't do regex parsing. (Treat as raw string)
|
||||
command = ["grep", "-n", "-R", "-i", "-F", query, target]
|
||||
proc = subprocess.Popen(command, stdout=subprocess.PIPE, encoding="utf-8")
|
||||
raw_data = proc.communicate()[0].strip()
|
||||
proc_data = raw_data.split("\n") # NOTE: Will return data AFTER completion (if any)
|
||||
collection = {}
|
||||
|
||||
for line in proc_data:
|
||||
file, line_no, data = line.split(":", 2)
|
||||
b64_file = base64.urlsafe_b64encode(file.encode('utf-8')).decode('utf-8')
|
||||
b64_data = base64.urlsafe_b64encode(data.encode('utf-8')).decode('utf-8')
|
||||
|
||||
if b64_file in collection.keys():
|
||||
collection[f"{b64_file}"][f"{line_no}"] = b64_data
|
||||
else:
|
||||
collection[f"{b64_file}"] = {}
|
||||
collection[f"{b64_file}"] = { f"{line_no}": b64_data}
|
||||
|
||||
try:
|
||||
data = f"GREP|{ts}|{json.dumps(collection, separators=(',', ':'), indent=4)}"
|
||||
send_ipc_message(data)
|
||||
except Exception as e:
|
||||
...
|
||||
|
||||
collection = {}
|
||||
|
||||
|
||||
def search(args):
|
||||
if args.type == "file_search":
|
||||
file_search(args.dir, args.query.lower())
|
||||
|
||||
if args.type == "grep_search":
|
||||
grep_search(args.dir, args.query.encode("utf-8"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
setproctitle('SolarFM: File Search - Grepy')
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
# Add long and short arguments
|
||||
parser.add_argument("--type", "-t", default=None, help="Type of search to do.")
|
||||
parser.add_argument("--dir", "-d", default=None, help="Directory root for search type.")
|
||||
parser.add_argument("--query", "-q", default=None, help="Query search is working against.")
|
||||
|
||||
# Read arguments (If any...)
|
||||
args = parser.parse_args()
|
||||
search(args)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
3
plugins/searcher/widgets/__init__.py
Normal file
3
plugins/searcher/widgets/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Widgets Module
|
||||
"""
|
||||
16
plugins/searcher/widgets/file_preview_widget.py
Normal file
16
plugins/searcher/widgets/file_preview_widget.py
Normal file
@@ -0,0 +1,16 @@
|
||||
# Python imports
|
||||
|
||||
# Gtk imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
|
||||
# Application imports
|
||||
|
||||
|
||||
class FilePreviewWidget(Gtk.LinkButton):
|
||||
def __init__(self, path, file):
|
||||
super(FilePreviewWidget, self).__init__()
|
||||
self.set_label(file)
|
||||
self.set_uri(f"file://{path}")
|
||||
self.show_all()
|
||||
60
plugins/searcher/widgets/grep_preview_widget.py
Normal file
60
plugins/searcher/widgets/grep_preview_widget.py
Normal file
@@ -0,0 +1,60 @@
|
||||
# Python imports
|
||||
import base64
|
||||
import re
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
|
||||
# Application imports
|
||||
|
||||
|
||||
class GrepPreviewWidget(Gtk.Box):
|
||||
def __init__(self, _path, sub_keys, _data, _grep_query):
|
||||
super(GrepPreviewWidget, self).__init__()
|
||||
|
||||
self.set_orientation(Gtk.Orientation.VERTICAL)
|
||||
line_color = "#e0cc64"
|
||||
highlight_color = "#FBF719"
|
||||
grep_query = _grep_query
|
||||
|
||||
path = self.decode_str(_path)
|
||||
lbl = '/'.join( path.split("/")[-3:] )
|
||||
title = Gtk.LinkButton.new_with_label(uri=f"file://{path}", label=lbl)
|
||||
|
||||
text_view = Gtk.TextView()
|
||||
buffer = text_view.get_buffer()
|
||||
text_view.set_editable(False)
|
||||
text_view.set_monospace(True)
|
||||
text_view.set_wrap_mode(Gtk.WrapMode.NONE)
|
||||
|
||||
for i, key in enumerate(sub_keys):
|
||||
line_num = self.make_utf8_line_num(line_color, key)
|
||||
itr = buffer.get_end_iter()
|
||||
buffer.insert_markup(itr, line_num, len(line_num))
|
||||
|
||||
decoded = f"\t{self.decode_str(_data[key])}"
|
||||
self.make_utf8_line_highlight(buffer, itr, i, highlight_color, decoded, grep_query)
|
||||
|
||||
self.add(title)
|
||||
self.add(text_view)
|
||||
self.show_all()
|
||||
|
||||
def decode_str(self, target):
|
||||
return base64.urlsafe_b64decode(target.encode('utf-8')).decode('utf-8')
|
||||
|
||||
def make_utf8_line_num(self, color, target):
|
||||
return bytes(f"\n<span foreground='{color}'>{target}</span>", "utf-8").decode("utf-8")
|
||||
|
||||
def make_utf8_line_highlight(self, buffer, itr, i, color, target, query):
|
||||
parts = re.split(r"(" + query + ")(?i)", target.replace("\n", ""))
|
||||
for part in parts:
|
||||
itr = buffer.get_end_iter()
|
||||
|
||||
if not query.lower() == part.lower() and not query.lower() in part.lower():
|
||||
buffer.insert(itr, part, length=len(part))
|
||||
else:
|
||||
new_s = f"<span foreground='#000000' background='{color}'>{part}</span>"
|
||||
_part = bytes(new_s, "utf-8").decode("utf-8")
|
||||
buffer.insert_markup(itr, _part, len(_part))
|
||||
3
plugins/template/__init__.py
Normal file
3
plugins/template/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/template/__main__.py
Normal file
3
plugins/template/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
13
plugins/template/manifest.json
Normal file
13
plugins/template/manifest.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "Example Plugin",
|
||||
"author": "John Doe",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "plugin_control_list",
|
||||
"pass_fm_events": "true",
|
||||
"bind_keys": ["Example Plugin||send_message:<Control>f"]
|
||||
}
|
||||
}
|
||||
}
|
||||
49
plugins/template/plugin.py
Normal file
49
plugins/template/plugin.py
Normal file
@@ -0,0 +1,49 @@
|
||||
# Python imports
|
||||
import os
|
||||
import threading
|
||||
import subprocess
|
||||
import ime
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
return wrapper
|
||||
|
||||
|
||||
|
||||
|
||||
class Plugin(PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.name = "Example Plugin" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
button = Gtk.Button(label=self.name)
|
||||
button.connect("button-release-event", self.send_message)
|
||||
return button
|
||||
|
||||
def run(self):
|
||||
...
|
||||
|
||||
def send_message(self, widget=None, eve=None):
|
||||
message = "Hello, World!"
|
||||
event_system.emit("display_message", ("warning", message, None))
|
||||
3
plugins/trasher/__init__.py
Normal file
3
plugins/trasher/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/trasher/__main__.py
Normal file
3
plugins/trasher/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
16
plugins/trasher/manifest.json
Normal file
16
plugins/trasher/manifest.json
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "Trasher",
|
||||
"author": "ITDominator",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "context_menu",
|
||||
"pass_fm_events": "true",
|
||||
"bind_keys": [
|
||||
"Trasher||delete_files:Delete",
|
||||
"Trasher||trash_files:<Control>d"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
138
plugins/trasher/plugin.py
Normal file
138
plugins/trasher/plugin.py
Normal file
@@ -0,0 +1,138 @@
|
||||
# Python imports
|
||||
import os
|
||||
import threading
|
||||
import subprocess
|
||||
import inspect
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
from gi.repository import GLib
|
||||
from gi.repository import Gio
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
from .xdgtrash import XDGTrash
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
return wrapper
|
||||
|
||||
|
||||
|
||||
|
||||
class Plugin(PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.path = os.path.dirname(os.path.realpath(__file__))
|
||||
self._GLADE_FILE = f"{self.path}/trasher.glade"
|
||||
self.name = "Trasher" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
self.trashman = XDGTrash()
|
||||
self.trash_files_path = f"{GLib.get_user_data_dir()}/Trash/files"
|
||||
self.trash_info_path = f"{GLib.get_user_data_dir()}/Trash/info"
|
||||
|
||||
self.trashman.regenerate()
|
||||
|
||||
|
||||
def run(self):
|
||||
self._event_system.subscribe("show_trash_buttons", self._show_trash_buttons)
|
||||
self._event_system.subscribe("hide_trash_buttons", self._hide_trash_buttons)
|
||||
self._event_system.subscribe("delete_files", self.delete_files)
|
||||
self._event_system.subscribe("trash_files", self.trash_files)
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
trash_a = Gtk.MenuItem("Trash Actions")
|
||||
trash_menu = Gtk.Menu()
|
||||
|
||||
self.restore = Gtk.MenuItem("Restore From Trash")
|
||||
self.restore.connect("activate", self.restore_trash_files)
|
||||
|
||||
self.empty = Gtk.MenuItem("Empty Trash")
|
||||
self.empty.connect("activate", self.empty_trash)
|
||||
|
||||
trash = Gtk.ImageMenuItem("Trash")
|
||||
trash.set_image( Gtk.Image.new_from_icon_name("user-trash", 16) )
|
||||
trash.connect("activate", self.trash_files)
|
||||
trash.set_always_show_image(True)
|
||||
|
||||
go_to = Gtk.ImageMenuItem("Go To Trash")
|
||||
go_to.set_image( Gtk.Image.new_from_icon_name("user-trash", 16) )
|
||||
go_to.connect("activate", self.go_to_trash)
|
||||
go_to.set_always_show_image(True)
|
||||
|
||||
delete = Gtk.ImageMenuItem("Delete")
|
||||
delete.set_image( Gtk.Image(stock=Gtk.STOCK_DELETE) )
|
||||
delete.connect("activate", self.delete_files)
|
||||
delete.set_always_show_image(True)
|
||||
|
||||
trash_a.set_submenu(trash_menu)
|
||||
trash_a.show_all()
|
||||
self._appen_menu_items(trash_menu, [self.restore, self.empty, trash, go_to, delete])
|
||||
|
||||
return trash_a
|
||||
|
||||
|
||||
def _appen_menu_items(self, menu, items):
|
||||
for item in items:
|
||||
menu.append(item)
|
||||
|
||||
|
||||
def _show_trash_buttons(self):
|
||||
self.restore.show()
|
||||
self.empty.show()
|
||||
|
||||
def _hide_trash_buttons(self):
|
||||
self.restore.hide()
|
||||
self.empty.hide()
|
||||
|
||||
def delete_files(self, widget = None, eve = None):
|
||||
self._event_system.emit("get_current_state")
|
||||
state = self._fm_state
|
||||
uris = state.selected_files
|
||||
response = None
|
||||
|
||||
state.warning_alert.format_secondary_text(f"Do you really want to delete the {len(uris)} file(s)?")
|
||||
for uri in uris:
|
||||
file = Gio.File.new_for_path(uri)
|
||||
|
||||
if not response:
|
||||
response = state.warning_alert.run()
|
||||
state.warning_alert.hide()
|
||||
if response == Gtk.ResponseType.YES:
|
||||
type = file.query_file_type(flags=Gio.FileQueryInfoFlags.NONE)
|
||||
|
||||
if type == Gio.FileType.DIRECTORY:
|
||||
state.tab.delete_file( file.get_path() )
|
||||
else:
|
||||
file.delete(cancellable=None)
|
||||
else:
|
||||
break
|
||||
|
||||
def trash_files(self, widget = None, eve = None, verbocity = False):
|
||||
self._event_system.emit("get_current_state")
|
||||
state = self._fm_state
|
||||
for uri in state.selected_files:
|
||||
self.trashman.trash(uri, verbocity)
|
||||
|
||||
def restore_trash_files(self, widget = None, eve = None, verbocity = False):
|
||||
self._event_system.emit("get_current_state")
|
||||
state = self._fm_state
|
||||
for uri in state.selected_files:
|
||||
self.trashman.restore(filename=uri.split("/")[-1], verbose = verbocity)
|
||||
|
||||
def empty_trash(self, widget = None, eve = None, verbocity = False):
|
||||
self.trashman.empty(verbose = verbocity)
|
||||
|
||||
def go_to_trash(self, widget = None, eve = None, verbocity = False):
|
||||
self._event_system.emit("go_to_path", self.trash_files_path)
|
||||
@@ -21,7 +21,7 @@ class Trash(object):
|
||||
if os.path.isfile(item):
|
||||
size = size + os.path.getsize(item)
|
||||
elif os.path.isdir(item):
|
||||
size = size + size_dir(item)
|
||||
size = size + self.size_dir(item)
|
||||
|
||||
return size
|
||||
|
||||
3
plugins/vod_thumbnailer/__init__.py
Normal file
3
plugins/vod_thumbnailer/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/vod_thumbnailer/__main__.py
Normal file
3
plugins/vod_thumbnailer/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
12
plugins/vod_thumbnailer/manifest.json
Normal file
12
plugins/vod_thumbnailer/manifest.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "VOD Thumbnailer",
|
||||
"author": "ITDominator",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "context_menu_plugins",
|
||||
"pass_fm_events": "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
146
plugins/vod_thumbnailer/plugin.py
Normal file
146
plugins/vod_thumbnailer/plugin.py
Normal file
@@ -0,0 +1,146 @@
|
||||
# Python imports
|
||||
import os
|
||||
import threading
|
||||
import subprocess
|
||||
import time
|
||||
import inspect
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
|
||||
# Gtk imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
gi.require_version('GdkPixbuf', '2.0')
|
||||
from gi.repository import Gtk
|
||||
from gi.repository import GLib
|
||||
from gi.repository import Gio
|
||||
from gi.repository import GdkPixbuf
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
return wrapper
|
||||
|
||||
|
||||
|
||||
|
||||
class Plugin(PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.path = os.path.dirname(os.path.realpath(__file__))
|
||||
self._GLADE_FILE = f"{self.path}/re_thumbnailer.glade"
|
||||
self.name = "VOD Thumbnailer" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
|
||||
self._thumbnailer_dialog = None
|
||||
self._thumbnail_preview_img = None
|
||||
self._scrub_step = None
|
||||
self._file_name = None
|
||||
self._file_location = None
|
||||
self._file_hash = None
|
||||
|
||||
|
||||
def run(self):
|
||||
self._builder = Gtk.Builder()
|
||||
self._builder.add_from_file(self._GLADE_FILE)
|
||||
|
||||
classes = [self]
|
||||
handlers = {}
|
||||
for c in classes:
|
||||
methods = None
|
||||
try:
|
||||
methods = inspect.getmembers(c, predicate=inspect.ismethod)
|
||||
handlers.update(methods)
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
|
||||
self._builder.connect_signals(handlers)
|
||||
|
||||
self._thumbnailer_dialog = self._builder.get_object("thumbnailer_dialog")
|
||||
self._scrub_step = self._builder.get_object("scrub_step")
|
||||
self._file_name = self._builder.get_object("file_name")
|
||||
self._file_location = self._builder.get_object("file_location")
|
||||
self._thumbnail_preview_img = self._builder.get_object("thumbnail_preview_img")
|
||||
self._file_hash = self._builder.get_object("file_hash")
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(f"{self.path}/../../icons/video.png", 16, 16, True)
|
||||
icon = Gtk.Image.new_from_pixbuf(pixbuf)
|
||||
item = Gtk.ImageMenuItem(self.name)
|
||||
|
||||
item.set_image( icon )
|
||||
item.connect("activate", self._show_thumbnailer_page)
|
||||
item.set_always_show_image(True)
|
||||
return item
|
||||
|
||||
|
||||
@threaded
|
||||
def _show_thumbnailer_page(self, widget=None, eve=None):
|
||||
self._event_system.emit("get_current_state")
|
||||
|
||||
state = self._fm_state
|
||||
self._event_message = None
|
||||
|
||||
GLib.idle_add(self._process_changes, (state))
|
||||
|
||||
def _process_changes(self, state):
|
||||
self._fm_state = None
|
||||
|
||||
if len(state.selected_files) == 1:
|
||||
if state.selected_files[0].lower().endswith(state.tab.fvideos):
|
||||
self._fm_state = state
|
||||
self._set_ui_data()
|
||||
response = self._thumbnailer_dialog.run()
|
||||
if response in [Gtk.ResponseType.CLOSE, Gtk.ResponseType.DELETE_EVENT]:
|
||||
self._thumbnailer_dialog.hide()
|
||||
|
||||
|
||||
def _regenerate_thumbnail(self, widget=None, eve=None):
|
||||
scrub_percent = int(self._scrub_step.get_value())
|
||||
file = self._file_name.get_text()
|
||||
dir = self._file_location.get_text()
|
||||
file_hash = self._file_hash.get_text()
|
||||
hash_img_pth = f"{self._fm_state.tab.ABS_THUMBS_PTH}/{file_hash}.jpg"
|
||||
|
||||
try:
|
||||
os.remove(hash_img_pth) if os.path.isfile(hash_img_pth) else ...
|
||||
|
||||
self._fm_state.tab.create_thumbnail(dir, file, f"{scrub_percent}%")
|
||||
preview_pixbuf = GdkPixbuf.Pixbuf.new_from_file(hash_img_pth)
|
||||
self._thumbnail_preview_img.set_from_pixbuf(preview_pixbuf)
|
||||
|
||||
img_pixbuf = self._fm_state.tab.create_scaled_image(hash_img_pth)
|
||||
tree_pth = self._fm_state.icon_grid.get_selected_items()[0]
|
||||
itr = self._fm_state.store.get_iter(tree_pth)
|
||||
pixbuff = self._fm_state.store.get(itr, 0)[0]
|
||||
self._fm_state.store.set(itr, 0, img_pixbuf)
|
||||
except Exception as e:
|
||||
print(repr(e))
|
||||
print("Couldn't regenerate thumbnail!")
|
||||
|
||||
|
||||
def _set_ui_data(self):
|
||||
uri = self._fm_state.selected_files[0]
|
||||
path = self._fm_state.tab.get_current_directory()
|
||||
parts = uri.split("/")
|
||||
|
||||
file_hash = hashlib.sha256(str.encode(uri)).hexdigest()
|
||||
hash_img_pth = f"{self._fm_state.tab.ABS_THUMBS_PTH}/{file_hash}.jpg"
|
||||
preview_pixbuf = GdkPixbuf.Pixbuf.new_from_file(hash_img_pth)
|
||||
|
||||
self._thumbnail_preview_img.set_from_pixbuf(preview_pixbuf)
|
||||
self._file_name.set_text(parts[ len(parts) - 1 ])
|
||||
self._file_location.set_text(path)
|
||||
self._file_hash.set_text(file_hash)
|
||||
239
plugins/vod_thumbnailer/re_thumbnailer.glade
Normal file
239
plugins/vod_thumbnailer/re_thumbnailer.glade
Normal file
@@ -0,0 +1,239 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated with glade 3.38.2 -->
|
||||
<interface>
|
||||
<requires lib="gtk+" version="3.16"/>
|
||||
<object class="GtkAdjustment" id="scrub_step_adjuster">
|
||||
<property name="lower">1</property>
|
||||
<property name="upper">100</property>
|
||||
<property name="value">65</property>
|
||||
<property name="step-increment">1</property>
|
||||
<property name="page-increment">10</property>
|
||||
</object>
|
||||
<object class="GtkDialog" id="thumbnailer_dialog">
|
||||
<property name="can-focus">False</property>
|
||||
<property name="border-width">6</property>
|
||||
<property name="title" translatable="yes">VOD Thumbnailer</property>
|
||||
<property name="modal">True</property>
|
||||
<property name="window-position">center-on-parent</property>
|
||||
<property name="default-width">420</property>
|
||||
<property name="destroy-with-parent">True</property>
|
||||
<property name="type-hint">dialog</property>
|
||||
<property name="skip-taskbar-hint">True</property>
|
||||
<property name="skip-pager-hint">True</property>
|
||||
<property name="gravity">center</property>
|
||||
<child internal-child="vbox">
|
||||
<object class="GtkBox" id="dialog_vbox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="spacing">12</property>
|
||||
<child internal-child="action_area">
|
||||
<object class="GtkButtonBox" id="dialog_action_area">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="layout-style">end</property>
|
||||
<child>
|
||||
<object class="GtkButton" id="cancel_button">
|
||||
<property name="label">gtk-close</property>
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="can-default">True</property>
|
||||
<property name="receives-default">False</property>
|
||||
<property name="use-stock">True</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">False</property>
|
||||
<property name="pack-type">end</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<object class="GtkImage" id="thumbnail_preview_img">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="stock">gtk-missing-image</property>
|
||||
<property name="icon_size">6</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkBox">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="orientation">vertical</property>
|
||||
<child>
|
||||
<object class="GtkLabel">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="margin-start">5</property>
|
||||
<property name="label" translatable="yes">New Thumbnail Scrub Step: </property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">0</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkScale" id="scrub_step">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="margin-start">5</property>
|
||||
<property name="margin-end">5</property>
|
||||
<property name="adjustment">scrub_step_adjuster</property>
|
||||
<property name="show-fill-level">True</property>
|
||||
<property name="round-digits">1</property>
|
||||
<property name="digits">0</property>
|
||||
<property name="value-pos">right</property>
|
||||
<signal name="value-changed" handler="_regenerate_thumbnail" swapped="no"/>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkTable" id="general_table">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="border-width">4</property>
|
||||
<property name="n-rows">3</property>
|
||||
<property name="n-columns">2</property>
|
||||
<property name="column-spacing">12</property>
|
||||
<property name="row-spacing">6</property>
|
||||
<child>
|
||||
<object class="GtkLabel">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>File _Name:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="mnemonic-widget">file_name</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_name">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="editable">False</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>_Location:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="mnemonic-widget">file_location</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_location">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="editable">False</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">1</property>
|
||||
<property name="bottom-attach">2</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkLabel" id="hash">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">False</property>
|
||||
<property name="label" translatable="yes"><b>_Thumbnail Hash:</b></property>
|
||||
<property name="use-markup">True</property>
|
||||
<property name="use-underline">True</property>
|
||||
<property name="mnemonic-widget">file_location</property>
|
||||
<property name="xalign">0</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
<child>
|
||||
<object class="GtkEntry" id="file_hash">
|
||||
<property name="visible">True</property>
|
||||
<property name="can-focus">True</property>
|
||||
<property name="editable">False</property>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="left-attach">1</property>
|
||||
<property name="right-attach">2</property>
|
||||
<property name="top-attach">2</property>
|
||||
<property name="bottom-attach">3</property>
|
||||
<property name="x-options">GTK_FILL</property>
|
||||
<property name="y-options"/>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">False</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">2</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
<packing>
|
||||
<property name="expand">True</property>
|
||||
<property name="fill">True</property>
|
||||
<property name="position">1</property>
|
||||
</packing>
|
||||
</child>
|
||||
</object>
|
||||
</child>
|
||||
<action-widgets>
|
||||
<action-widget response="-7">cancel_button</action-widget>
|
||||
</action-widgets>
|
||||
</object>
|
||||
</interface>
|
||||
3
plugins/youtube_download/__init__.py
Normal file
3
plugins/youtube_download/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Module
|
||||
"""
|
||||
3
plugins/youtube_download/__main__.py
Normal file
3
plugins/youtube_download/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Pligin Package
|
||||
"""
|
||||
19
plugins/youtube_download/download.sh
Executable file
19
plugins/youtube_download/download.sh
Executable file
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
|
||||
# . CONFIG.sh
|
||||
|
||||
# set -o xtrace ## To debug scripts
|
||||
# set -o errexit ## To exit on error
|
||||
# set -o errunset ## To exit if a variable is referenced but not set
|
||||
|
||||
|
||||
function main() {
|
||||
cd "$(dirname "")"
|
||||
echo "Working Dir: " $(pwd)
|
||||
LINK=`xclip -selection clipboard -o`
|
||||
|
||||
python "${HOME}/.config/solarfm/plugins/youtube_download/yt_dlp/__main__.py" \
|
||||
--cookies-from-browser firefox --write-sub --embed-sub --sub-langs en \
|
||||
-o "${1}/%(title)s.%(ext)s" "${LINK}"
|
||||
}
|
||||
main "$@";
|
||||
12
plugins/youtube_download/manifest.json
Normal file
12
plugins/youtube_download/manifest.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"manifest": {
|
||||
"name": "Youtube Download",
|
||||
"author": "ITDominator",
|
||||
"version": "0.0.1",
|
||||
"support": "",
|
||||
"requests": {
|
||||
"ui_target": "plugin_control_list",
|
||||
"pass_fm_events": "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
53
plugins/youtube_download/plugin.py
Normal file
53
plugins/youtube_download/plugin.py
Normal file
@@ -0,0 +1,53 @@
|
||||
# Python imports
|
||||
import os, threading, subprocess, time
|
||||
|
||||
# Lib imports
|
||||
import gi
|
||||
gi.require_version('Gtk', '3.0')
|
||||
from gi.repository import Gtk
|
||||
|
||||
# Application imports
|
||||
from plugins.plugin_base import PluginBase
|
||||
|
||||
|
||||
# NOTE: Threads WILL NOT die with parent's destruction.
|
||||
def threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
|
||||
return wrapper
|
||||
|
||||
# NOTE: Threads WILL die with parent's destruction.
|
||||
def daemon_threaded(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
|
||||
return wrapper
|
||||
|
||||
|
||||
|
||||
|
||||
class Plugin(PluginBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.path = os.path.dirname(os.path.realpath(__file__))
|
||||
self.name = "Youtube Download" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
|
||||
# where self.name should not be needed for message comms
|
||||
|
||||
def generate_reference_ui_element(self):
|
||||
button = Gtk.Button(label=self.name)
|
||||
button.connect("button-release-event", self._do_download)
|
||||
return button
|
||||
|
||||
def run(self):
|
||||
...
|
||||
|
||||
|
||||
def _do_download(self, widget=None, eve=None):
|
||||
self._event_system.emit("get_current_state")
|
||||
|
||||
dir = self._fm_state.tab.get_current_directory()
|
||||
self._download(dir)
|
||||
|
||||
@threaded
|
||||
def _download(self, dir):
|
||||
subprocess.Popen([f'{self.path}/download.sh', dir])
|
||||
0
src/versions/solarfm-0.0.1/SolarFM/solarfm/trasher/__init__.py → plugins/youtube_download/yt_dlp-2022.2.4.dist-info/AUTHORS
Executable file → Normal file
0
src/versions/solarfm-0.0.1/SolarFM/solarfm/trasher/__init__.py → plugins/youtube_download/yt_dlp-2022.2.4.dist-info/AUTHORS
Executable file → Normal file
@@ -0,0 +1 @@
|
||||
pip
|
||||
24
plugins/youtube_download/yt_dlp-2022.2.4.dist-info/LICENSE
Normal file
24
plugins/youtube_download/yt_dlp-2022.2.4.dist-info/LICENSE
Normal file
@@ -0,0 +1,24 @@
|
||||
This is free and unencumbered software released into the public domain.
|
||||
|
||||
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||
distribute this software, either in source code form or as a compiled
|
||||
binary, for any purpose, commercial or non-commercial, and by any
|
||||
means.
|
||||
|
||||
In jurisdictions that recognize copyright laws, the author or authors
|
||||
of this software dedicate any and all copyright interest in the
|
||||
software to the public domain. We make this dedication for the benefit
|
||||
of the public at large and to the detriment of our heirs and
|
||||
successors. We intend this dedication to be an overt act of
|
||||
relinquishment in perpetuity of all present and future rights to this
|
||||
software under copyright law.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
For more information, please refer to <http://unlicense.org/>
|
||||
1992
plugins/youtube_download/yt_dlp-2022.2.4.dist-info/METADATA
Normal file
1992
plugins/youtube_download/yt_dlp-2022.2.4.dist-info/METADATA
Normal file
File diff suppressed because it is too large
Load Diff
1939
plugins/youtube_download/yt_dlp-2022.2.4.dist-info/RECORD
Normal file
1939
plugins/youtube_download/yt_dlp-2022.2.4.dist-info/RECORD
Normal file
File diff suppressed because it is too large
Load Diff
6
plugins/youtube_download/yt_dlp-2022.2.4.dist-info/WHEEL
Normal file
6
plugins/youtube_download/yt_dlp-2022.2.4.dist-info/WHEEL
Normal file
@@ -0,0 +1,6 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: bdist_wheel (0.37.1)
|
||||
Root-Is-Purelib: true
|
||||
Tag: py2-none-any
|
||||
Tag: py3-none-any
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
[console_scripts]
|
||||
yt-dlp = yt_dlp:main
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
yt_dlp
|
||||
3874
plugins/youtube_download/yt_dlp/YoutubeDL.py
Normal file
3874
plugins/youtube_download/yt_dlp/YoutubeDL.py
Normal file
File diff suppressed because it is too large
Load Diff
871
plugins/youtube_download/yt_dlp/__init__.py
Normal file
871
plugins/youtube_download/yt_dlp/__init__.py
Normal file
@@ -0,0 +1,871 @@
|
||||
#!/usr/bin/env python3
|
||||
# coding: utf-8
|
||||
|
||||
f'You are using an unsupported version of Python. Only Python versions 3.6 and above are supported by yt-dlp' # noqa: F541
|
||||
|
||||
__license__ = 'Public Domain'
|
||||
|
||||
import codecs
|
||||
import io
|
||||
import itertools
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
|
||||
from .options import (
|
||||
parseOpts,
|
||||
)
|
||||
from .compat import (
|
||||
compat_getpass,
|
||||
compat_os_name,
|
||||
compat_shlex_quote,
|
||||
workaround_optparse_bug9161,
|
||||
)
|
||||
from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS
|
||||
from .utils import (
|
||||
DateRange,
|
||||
decodeOption,
|
||||
DownloadCancelled,
|
||||
DownloadError,
|
||||
error_to_compat_str,
|
||||
expand_path,
|
||||
GeoUtils,
|
||||
float_or_none,
|
||||
int_or_none,
|
||||
match_filter_func,
|
||||
parse_duration,
|
||||
preferredencoding,
|
||||
read_batch_urls,
|
||||
render_table,
|
||||
SameFileError,
|
||||
setproctitle,
|
||||
std_headers,
|
||||
write_string,
|
||||
)
|
||||
from .update import run_update
|
||||
from .downloader import (
|
||||
FileDownloader,
|
||||
)
|
||||
from .extractor import gen_extractors, list_extractors
|
||||
from .extractor.common import InfoExtractor
|
||||
from .extractor.adobepass import MSO_INFO
|
||||
from .postprocessor import (
|
||||
FFmpegExtractAudioPP,
|
||||
FFmpegSubtitlesConvertorPP,
|
||||
FFmpegThumbnailsConvertorPP,
|
||||
FFmpegVideoConvertorPP,
|
||||
FFmpegVideoRemuxerPP,
|
||||
MetadataFromFieldPP,
|
||||
MetadataParserPP,
|
||||
)
|
||||
from .YoutubeDL import YoutubeDL
|
||||
|
||||
|
||||
def _real_main(argv=None):
|
||||
# Compatibility fixes for Windows
|
||||
if sys.platform == 'win32':
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/820
|
||||
codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
|
||||
|
||||
workaround_optparse_bug9161()
|
||||
|
||||
setproctitle('yt-dlp')
|
||||
|
||||
parser, opts, args = parseOpts(argv)
|
||||
warnings, deprecation_warnings = [], []
|
||||
|
||||
# Set user agent
|
||||
if opts.user_agent is not None:
|
||||
std_headers['User-Agent'] = opts.user_agent
|
||||
|
||||
# Set referer
|
||||
if opts.referer is not None:
|
||||
std_headers['Referer'] = opts.referer
|
||||
|
||||
# Custom HTTP headers
|
||||
std_headers.update(opts.headers)
|
||||
|
||||
# Dump user agent
|
||||
if opts.dump_user_agent:
|
||||
write_string(std_headers['User-Agent'] + '\n', out=sys.stdout)
|
||||
sys.exit(0)
|
||||
|
||||
# Batch file verification
|
||||
batch_urls = []
|
||||
if opts.batchfile is not None:
|
||||
try:
|
||||
if opts.batchfile == '-':
|
||||
write_string('Reading URLs from stdin - EOF (%s) to end:\n' % (
|
||||
'Ctrl+Z' if compat_os_name == 'nt' else 'Ctrl+D'))
|
||||
batchfd = sys.stdin
|
||||
else:
|
||||
batchfd = io.open(
|
||||
expand_path(opts.batchfile),
|
||||
'r', encoding='utf-8', errors='ignore')
|
||||
batch_urls = read_batch_urls(batchfd)
|
||||
if opts.verbose:
|
||||
write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
|
||||
except IOError:
|
||||
sys.exit('ERROR: batch file %s could not be read' % opts.batchfile)
|
||||
all_urls = batch_urls + [url.strip() for url in args] # batch_urls are already striped in read_batch_urls
|
||||
_enc = preferredencoding()
|
||||
all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
|
||||
|
||||
if opts.list_extractors:
|
||||
for ie in list_extractors(opts.age_limit):
|
||||
write_string(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie.working() else '') + '\n', out=sys.stdout)
|
||||
matchedUrls = [url for url in all_urls if ie.suitable(url)]
|
||||
for mu in matchedUrls:
|
||||
write_string(' ' + mu + '\n', out=sys.stdout)
|
||||
sys.exit(0)
|
||||
if opts.list_extractor_descriptions:
|
||||
for ie in list_extractors(opts.age_limit):
|
||||
if not ie.working():
|
||||
continue
|
||||
desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
|
||||
if desc is False:
|
||||
continue
|
||||
if getattr(ie, 'SEARCH_KEY', None) is not None:
|
||||
_SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
|
||||
_COUNTS = ('', '5', '10', 'all')
|
||||
desc += f'; "{ie.SEARCH_KEY}:" prefix (Example: "{ie.SEARCH_KEY}{random.choice(_COUNTS)}:{random.choice(_SEARCHES)}")'
|
||||
write_string(desc + '\n', out=sys.stdout)
|
||||
sys.exit(0)
|
||||
if opts.ap_list_mso:
|
||||
table = [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()]
|
||||
write_string('Supported TV Providers:\n' + render_table(['mso', 'mso name'], table) + '\n', out=sys.stdout)
|
||||
sys.exit(0)
|
||||
|
||||
# Conflicting, missing and erroneous options
|
||||
if opts.format == 'best':
|
||||
warnings.append('.\n '.join((
|
||||
'"-f best" selects the best pre-merged format which is often not the best option',
|
||||
'To let yt-dlp download and merge the best available formats, simply do not pass any format selection',
|
||||
'If you know what you are doing and want only the best pre-merged format, use "-f b" instead to suppress this warning')))
|
||||
if opts.exec_cmd.get('before_dl') and opts.exec_before_dl_cmd:
|
||||
parser.error('using "--exec-before-download" conflicts with "--exec before_dl:"')
|
||||
if opts.usenetrc and (opts.username is not None or opts.password is not None):
|
||||
parser.error('using .netrc conflicts with giving username/password')
|
||||
if opts.password is not None and opts.username is None:
|
||||
parser.error('account username missing\n')
|
||||
if opts.ap_password is not None and opts.ap_username is None:
|
||||
parser.error('TV Provider account username missing\n')
|
||||
if opts.autonumber_size is not None:
|
||||
if opts.autonumber_size <= 0:
|
||||
parser.error('auto number size must be positive')
|
||||
if opts.autonumber_start is not None:
|
||||
if opts.autonumber_start < 0:
|
||||
parser.error('auto number start must be positive or 0')
|
||||
if opts.username is not None and opts.password is None:
|
||||
opts.password = compat_getpass('Type account password and press [Return]: ')
|
||||
if opts.ap_username is not None and opts.ap_password is None:
|
||||
opts.ap_password = compat_getpass('Type TV provider account password and press [Return]: ')
|
||||
if opts.ratelimit is not None:
|
||||
numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
|
||||
if numeric_limit is None:
|
||||
parser.error('invalid rate limit specified')
|
||||
opts.ratelimit = numeric_limit
|
||||
if opts.throttledratelimit is not None:
|
||||
numeric_limit = FileDownloader.parse_bytes(opts.throttledratelimit)
|
||||
if numeric_limit is None:
|
||||
parser.error('invalid rate limit specified')
|
||||
opts.throttledratelimit = numeric_limit
|
||||
if opts.min_filesize is not None:
|
||||
numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
|
||||
if numeric_limit is None:
|
||||
parser.error('invalid min_filesize specified')
|
||||
opts.min_filesize = numeric_limit
|
||||
if opts.max_filesize is not None:
|
||||
numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
|
||||
if numeric_limit is None:
|
||||
parser.error('invalid max_filesize specified')
|
||||
opts.max_filesize = numeric_limit
|
||||
if opts.sleep_interval is not None:
|
||||
if opts.sleep_interval < 0:
|
||||
parser.error('sleep interval must be positive or 0')
|
||||
if opts.max_sleep_interval is not None:
|
||||
if opts.max_sleep_interval < 0:
|
||||
parser.error('max sleep interval must be positive or 0')
|
||||
if opts.sleep_interval is None:
|
||||
parser.error('min sleep interval must be specified, use --min-sleep-interval')
|
||||
if opts.max_sleep_interval < opts.sleep_interval:
|
||||
parser.error('max sleep interval must be greater than or equal to min sleep interval')
|
||||
else:
|
||||
opts.max_sleep_interval = opts.sleep_interval
|
||||
if opts.sleep_interval_subtitles is not None:
|
||||
if opts.sleep_interval_subtitles < 0:
|
||||
parser.error('subtitles sleep interval must be positive or 0')
|
||||
if opts.sleep_interval_requests is not None:
|
||||
if opts.sleep_interval_requests < 0:
|
||||
parser.error('requests sleep interval must be positive or 0')
|
||||
if opts.ap_mso and opts.ap_mso not in MSO_INFO:
|
||||
parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')
|
||||
if opts.overwrites: # --yes-overwrites implies --no-continue
|
||||
opts.continue_dl = False
|
||||
if opts.concurrent_fragment_downloads <= 0:
|
||||
parser.error('Concurrent fragments must be positive')
|
||||
if opts.wait_for_video is not None:
|
||||
min_wait, max_wait, *_ = map(parse_duration, opts.wait_for_video.split('-', 1) + [None])
|
||||
if min_wait is None or (max_wait is None and '-' in opts.wait_for_video):
|
||||
parser.error('Invalid time range to wait')
|
||||
elif max_wait is not None and max_wait < min_wait:
|
||||
parser.error('Minimum time range to wait must not be longer than the maximum')
|
||||
opts.wait_for_video = (min_wait, max_wait)
|
||||
|
||||
def parse_retries(retries, name=''):
|
||||
if retries in ('inf', 'infinite'):
|
||||
parsed_retries = float('inf')
|
||||
else:
|
||||
try:
|
||||
parsed_retries = int(retries)
|
||||
except (TypeError, ValueError):
|
||||
parser.error('invalid %sretry count specified' % name)
|
||||
return parsed_retries
|
||||
if opts.retries is not None:
|
||||
opts.retries = parse_retries(opts.retries)
|
||||
if opts.file_access_retries is not None:
|
||||
opts.file_access_retries = parse_retries(opts.file_access_retries, 'file access ')
|
||||
if opts.fragment_retries is not None:
|
||||
opts.fragment_retries = parse_retries(opts.fragment_retries, 'fragment ')
|
||||
if opts.extractor_retries is not None:
|
||||
opts.extractor_retries = parse_retries(opts.extractor_retries, 'extractor ')
|
||||
if opts.buffersize is not None:
|
||||
numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
|
||||
if numeric_buffersize is None:
|
||||
parser.error('invalid buffer size specified')
|
||||
opts.buffersize = numeric_buffersize
|
||||
if opts.http_chunk_size is not None:
|
||||
numeric_chunksize = FileDownloader.parse_bytes(opts.http_chunk_size)
|
||||
if not numeric_chunksize:
|
||||
parser.error('invalid http chunk size specified')
|
||||
opts.http_chunk_size = numeric_chunksize
|
||||
if opts.playliststart <= 0:
|
||||
raise parser.error('Playlist start must be positive')
|
||||
if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
|
||||
raise parser.error('Playlist end must be greater than playlist start')
|
||||
if opts.extractaudio:
|
||||
opts.audioformat = opts.audioformat.lower()
|
||||
if opts.audioformat not in ['best'] + list(FFmpegExtractAudioPP.SUPPORTED_EXTS):
|
||||
parser.error('invalid audio format specified')
|
||||
if opts.audioquality:
|
||||
opts.audioquality = opts.audioquality.strip('k').strip('K')
|
||||
audioquality = int_or_none(float_or_none(opts.audioquality)) # int_or_none prevents inf, nan
|
||||
if audioquality is None or audioquality < 0:
|
||||
parser.error('invalid audio quality specified')
|
||||
if opts.recodevideo is not None:
|
||||
opts.recodevideo = opts.recodevideo.replace(' ', '')
|
||||
if not re.match(FFmpegVideoConvertorPP.FORMAT_RE, opts.recodevideo):
|
||||
parser.error('invalid video remux format specified')
|
||||
if opts.remuxvideo is not None:
|
||||
opts.remuxvideo = opts.remuxvideo.replace(' ', '')
|
||||
if not re.match(FFmpegVideoRemuxerPP.FORMAT_RE, opts.remuxvideo):
|
||||
parser.error('invalid video remux format specified')
|
||||
if opts.convertsubtitles is not None:
|
||||
if opts.convertsubtitles not in FFmpegSubtitlesConvertorPP.SUPPORTED_EXTS:
|
||||
parser.error('invalid subtitle format specified')
|
||||
if opts.convertthumbnails is not None:
|
||||
if opts.convertthumbnails not in FFmpegThumbnailsConvertorPP.SUPPORTED_EXTS:
|
||||
parser.error('invalid thumbnail format specified')
|
||||
if opts.cookiesfrombrowser is not None:
|
||||
mobj = re.match(r'(?P<name>[^+:]+)(\s*\+\s*(?P<keyring>[^:]+))?(\s*:(?P<profile>.+))?', opts.cookiesfrombrowser)
|
||||
if mobj is None:
|
||||
parser.error(f'invalid cookies from browser arguments: {opts.cookiesfrombrowser}')
|
||||
browser_name, keyring, profile = mobj.group('name', 'keyring', 'profile')
|
||||
browser_name = browser_name.lower()
|
||||
if browser_name not in SUPPORTED_BROWSERS:
|
||||
parser.error(f'unsupported browser specified for cookies: "{browser_name}". '
|
||||
f'Supported browsers are: {", ".join(sorted(SUPPORTED_BROWSERS))}')
|
||||
if keyring is not None:
|
||||
keyring = keyring.upper()
|
||||
if keyring not in SUPPORTED_KEYRINGS:
|
||||
parser.error(f'unsupported keyring specified for cookies: "{keyring}". '
|
||||
f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
|
||||
opts.cookiesfrombrowser = (browser_name, profile, keyring)
|
||||
geo_bypass_code = opts.geo_bypass_ip_block or opts.geo_bypass_country
|
||||
if geo_bypass_code is not None:
|
||||
try:
|
||||
GeoUtils.random_ipv4(geo_bypass_code)
|
||||
except Exception:
|
||||
parser.error('unsupported geo-bypass country or ip-block')
|
||||
|
||||
if opts.date is not None:
|
||||
date = DateRange.day(opts.date)
|
||||
else:
|
||||
date = DateRange(opts.dateafter, opts.datebefore)
|
||||
|
||||
compat_opts = opts.compat_opts
|
||||
|
||||
def report_conflict(arg1, arg2):
|
||||
warnings.append(f'{arg2} is ignored since {arg1} was given')
|
||||
|
||||
def _unused_compat_opt(name):
|
||||
if name not in compat_opts:
|
||||
return False
|
||||
compat_opts.discard(name)
|
||||
compat_opts.update(['*%s' % name])
|
||||
return True
|
||||
|
||||
def set_default_compat(compat_name, opt_name, default=True, remove_compat=True):
|
||||
attr = getattr(opts, opt_name)
|
||||
if compat_name in compat_opts:
|
||||
if attr is None:
|
||||
setattr(opts, opt_name, not default)
|
||||
return True
|
||||
else:
|
||||
if remove_compat:
|
||||
_unused_compat_opt(compat_name)
|
||||
return False
|
||||
elif attr is None:
|
||||
setattr(opts, opt_name, default)
|
||||
return None
|
||||
|
||||
set_default_compat('abort-on-error', 'ignoreerrors', 'only_download')
|
||||
set_default_compat('no-playlist-metafiles', 'allow_playlist_files')
|
||||
set_default_compat('no-clean-infojson', 'clean_infojson')
|
||||
if 'no-attach-info-json' in compat_opts:
|
||||
if opts.embed_infojson:
|
||||
_unused_compat_opt('no-attach-info-json')
|
||||
else:
|
||||
opts.embed_infojson = False
|
||||
if 'format-sort' in compat_opts:
|
||||
opts.format_sort.extend(InfoExtractor.FormatSort.ytdl_default)
|
||||
_video_multistreams_set = set_default_compat('multistreams', 'allow_multiple_video_streams', False, remove_compat=False)
|
||||
_audio_multistreams_set = set_default_compat('multistreams', 'allow_multiple_audio_streams', False, remove_compat=False)
|
||||
if _video_multistreams_set is False and _audio_multistreams_set is False:
|
||||
_unused_compat_opt('multistreams')
|
||||
outtmpl_default = opts.outtmpl.get('default')
|
||||
if outtmpl_default == '':
|
||||
outtmpl_default, opts.skip_download = None, True
|
||||
del opts.outtmpl['default']
|
||||
if opts.useid:
|
||||
if outtmpl_default is None:
|
||||
outtmpl_default = opts.outtmpl['default'] = '%(id)s.%(ext)s'
|
||||
else:
|
||||
report_conflict('--output', '--id')
|
||||
if 'filename' in compat_opts:
|
||||
if outtmpl_default is None:
|
||||
outtmpl_default = opts.outtmpl['default'] = '%(title)s-%(id)s.%(ext)s'
|
||||
else:
|
||||
_unused_compat_opt('filename')
|
||||
|
||||
def validate_outtmpl(tmpl, msg):
|
||||
err = YoutubeDL.validate_outtmpl(tmpl)
|
||||
if err:
|
||||
parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err)))
|
||||
|
||||
for k, tmpl in opts.outtmpl.items():
|
||||
validate_outtmpl(tmpl, f'{k} output template')
|
||||
for type_, tmpl_list in opts.forceprint.items():
|
||||
for tmpl in tmpl_list:
|
||||
validate_outtmpl(tmpl, f'{type_} print template')
|
||||
for type_, tmpl_list in opts.print_to_file.items():
|
||||
for tmpl, file in tmpl_list:
|
||||
validate_outtmpl(tmpl, f'{type_} print-to-file template')
|
||||
validate_outtmpl(file, f'{type_} print-to-file filename')
|
||||
validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title')
|
||||
for k, tmpl in opts.progress_template.items():
|
||||
k = f'{k[:-6]} console title' if '-title' in k else f'{k} progress'
|
||||
validate_outtmpl(tmpl, f'{k} template')
|
||||
|
||||
if opts.extractaudio and not opts.keepvideo and opts.format is None:
|
||||
opts.format = 'bestaudio/best'
|
||||
|
||||
if outtmpl_default is not None and not os.path.splitext(outtmpl_default)[1] and opts.extractaudio:
|
||||
parser.error('Cannot download a video and extract audio into the same'
|
||||
' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
|
||||
' template'.format(outtmpl_default))
|
||||
|
||||
for f in opts.format_sort:
|
||||
if re.match(InfoExtractor.FormatSort.regex, f) is None:
|
||||
parser.error('invalid format sort string "%s" specified' % f)
|
||||
|
||||
def metadataparser_actions(f):
|
||||
if isinstance(f, str):
|
||||
cmd = '--parse-metadata %s' % compat_shlex_quote(f)
|
||||
try:
|
||||
actions = [MetadataFromFieldPP.to_action(f)]
|
||||
except Exception as err:
|
||||
parser.error(f'{cmd} is invalid; {err}')
|
||||
else:
|
||||
cmd = '--replace-in-metadata %s' % ' '.join(map(compat_shlex_quote, f))
|
||||
actions = ((MetadataParserPP.Actions.REPLACE, x, *f[1:]) for x in f[0].split(','))
|
||||
|
||||
for action in actions:
|
||||
try:
|
||||
MetadataParserPP.validate_action(*action)
|
||||
except Exception as err:
|
||||
parser.error(f'{cmd} is invalid; {err}')
|
||||
yield action
|
||||
|
||||
if opts.parse_metadata is None:
|
||||
opts.parse_metadata = []
|
||||
if opts.metafromtitle is not None:
|
||||
opts.parse_metadata.append('title:%s' % opts.metafromtitle)
|
||||
opts.parse_metadata = list(itertools.chain(*map(metadataparser_actions, opts.parse_metadata)))
|
||||
|
||||
any_getting = (any(opts.forceprint.values()) or opts.dumpjson or opts.dump_single_json
|
||||
or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail
|
||||
or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration)
|
||||
|
||||
any_printing = opts.print_json
|
||||
download_archive_fn = expand_path(opts.download_archive) if opts.download_archive is not None else opts.download_archive
|
||||
|
||||
# If JSON is not printed anywhere, but comments are requested, save it to file
|
||||
printing_json = opts.dumpjson or opts.print_json or opts.dump_single_json
|
||||
if opts.getcomments and not printing_json:
|
||||
opts.writeinfojson = True
|
||||
|
||||
if opts.no_sponsorblock:
|
||||
opts.sponsorblock_mark = set()
|
||||
opts.sponsorblock_remove = set()
|
||||
sponsorblock_query = opts.sponsorblock_mark | opts.sponsorblock_remove
|
||||
|
||||
opts.remove_chapters = opts.remove_chapters or []
|
||||
|
||||
if (opts.remove_chapters or sponsorblock_query) and opts.sponskrub is not False:
|
||||
if opts.sponskrub:
|
||||
if opts.remove_chapters:
|
||||
report_conflict('--remove-chapters', '--sponskrub')
|
||||
if opts.sponsorblock_mark:
|
||||
report_conflict('--sponsorblock-mark', '--sponskrub')
|
||||
if opts.sponsorblock_remove:
|
||||
report_conflict('--sponsorblock-remove', '--sponskrub')
|
||||
opts.sponskrub = False
|
||||
if opts.sponskrub_cut and opts.split_chapters and opts.sponskrub is not False:
|
||||
report_conflict('--split-chapter', '--sponskrub-cut')
|
||||
opts.sponskrub_cut = False
|
||||
|
||||
if opts.remuxvideo and opts.recodevideo:
|
||||
report_conflict('--recode-video', '--remux-video')
|
||||
opts.remuxvideo = False
|
||||
|
||||
if opts.allow_unplayable_formats:
|
||||
def report_unplayable_conflict(opt_name, arg, default=False, allowed=None):
|
||||
val = getattr(opts, opt_name)
|
||||
if (not allowed and val) or (allowed and not allowed(val)):
|
||||
report_conflict('--allow-unplayable-formats', arg)
|
||||
setattr(opts, opt_name, default)
|
||||
|
||||
report_unplayable_conflict('extractaudio', '--extract-audio')
|
||||
report_unplayable_conflict('remuxvideo', '--remux-video')
|
||||
report_unplayable_conflict('recodevideo', '--recode-video')
|
||||
report_unplayable_conflict('addmetadata', '--embed-metadata')
|
||||
report_unplayable_conflict('addchapters', '--embed-chapters')
|
||||
report_unplayable_conflict('embed_infojson', '--embed-info-json')
|
||||
opts.embed_infojson = False
|
||||
report_unplayable_conflict('embedsubtitles', '--embed-subs')
|
||||
report_unplayable_conflict('embedthumbnail', '--embed-thumbnail')
|
||||
report_unplayable_conflict('xattrs', '--xattrs')
|
||||
report_unplayable_conflict('fixup', '--fixup', default='never', allowed=lambda x: x in (None, 'never', 'ignore'))
|
||||
opts.fixup = 'never'
|
||||
report_unplayable_conflict('remove_chapters', '--remove-chapters', default=[])
|
||||
report_unplayable_conflict('sponsorblock_remove', '--sponsorblock-remove', default=set())
|
||||
report_unplayable_conflict('sponskrub', '--sponskrub', default=set())
|
||||
opts.sponskrub = False
|
||||
|
||||
if (opts.addmetadata or opts.sponsorblock_mark) and opts.addchapters is None:
|
||||
opts.addchapters = True
|
||||
|
||||
# PostProcessors
|
||||
postprocessors = list(opts.add_postprocessors)
|
||||
if sponsorblock_query:
|
||||
postprocessors.append({
|
||||
'key': 'SponsorBlock',
|
||||
'categories': sponsorblock_query,
|
||||
'api': opts.sponsorblock_api,
|
||||
# Run this immediately after extraction is complete
|
||||
'when': 'pre_process'
|
||||
})
|
||||
if opts.parse_metadata:
|
||||
postprocessors.append({
|
||||
'key': 'MetadataParser',
|
||||
'actions': opts.parse_metadata,
|
||||
# Run this immediately after extraction is complete
|
||||
'when': 'pre_process'
|
||||
})
|
||||
if opts.convertsubtitles:
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegSubtitlesConvertor',
|
||||
'format': opts.convertsubtitles,
|
||||
# Run this before the actual video download
|
||||
'when': 'before_dl'
|
||||
})
|
||||
if opts.convertthumbnails:
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegThumbnailsConvertor',
|
||||
'format': opts.convertthumbnails,
|
||||
# Run this before the actual video download
|
||||
'when': 'before_dl'
|
||||
})
|
||||
if opts.extractaudio:
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegExtractAudio',
|
||||
'preferredcodec': opts.audioformat,
|
||||
'preferredquality': opts.audioquality,
|
||||
'nopostoverwrites': opts.nopostoverwrites,
|
||||
})
|
||||
if opts.remuxvideo:
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegVideoRemuxer',
|
||||
'preferedformat': opts.remuxvideo,
|
||||
})
|
||||
if opts.recodevideo:
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegVideoConvertor',
|
||||
'preferedformat': opts.recodevideo,
|
||||
})
|
||||
# If ModifyChapters is going to remove chapters, subtitles must already be in the container.
|
||||
if opts.embedsubtitles:
|
||||
already_have_subtitle = opts.writesubtitles and 'no-keep-subs' not in compat_opts
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegEmbedSubtitle',
|
||||
# already_have_subtitle = True prevents the file from being deleted after embedding
|
||||
'already_have_subtitle': already_have_subtitle
|
||||
})
|
||||
if not opts.writeautomaticsub and 'no-keep-subs' not in compat_opts:
|
||||
opts.writesubtitles = True
|
||||
# --all-sub automatically sets --write-sub if --write-auto-sub is not given
|
||||
# this was the old behaviour if only --all-sub was given.
|
||||
if opts.allsubtitles and not opts.writeautomaticsub:
|
||||
opts.writesubtitles = True
|
||||
# ModifyChapters must run before FFmpegMetadataPP
|
||||
remove_chapters_patterns, remove_ranges = [], []
|
||||
for regex in opts.remove_chapters:
|
||||
if regex.startswith('*'):
|
||||
dur = list(map(parse_duration, regex[1:].split('-')))
|
||||
if len(dur) == 2 and all(t is not None for t in dur):
|
||||
remove_ranges.append(tuple(dur))
|
||||
continue
|
||||
parser.error(f'invalid --remove-chapters time range {regex!r}. Must be of the form *start-end')
|
||||
try:
|
||||
remove_chapters_patterns.append(re.compile(regex))
|
||||
except re.error as err:
|
||||
parser.error(f'invalid --remove-chapters regex {regex!r} - {err}')
|
||||
if opts.remove_chapters or sponsorblock_query:
|
||||
postprocessors.append({
|
||||
'key': 'ModifyChapters',
|
||||
'remove_chapters_patterns': remove_chapters_patterns,
|
||||
'remove_sponsor_segments': opts.sponsorblock_remove,
|
||||
'remove_ranges': remove_ranges,
|
||||
'sponsorblock_chapter_title': opts.sponsorblock_chapter_title,
|
||||
'force_keyframes': opts.force_keyframes_at_cuts
|
||||
})
|
||||
# FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and
|
||||
# FFmpegExtractAudioPP as containers before conversion may not support
|
||||
# metadata (3gp, webm, etc.)
|
||||
# By default ffmpeg preserves metadata applicable for both
|
||||
# source and target containers. From this point the container won't change,
|
||||
# so metadata can be added here.
|
||||
if opts.addmetadata or opts.addchapters or opts.embed_infojson:
|
||||
if opts.embed_infojson is None:
|
||||
opts.embed_infojson = 'if_exists'
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegMetadata',
|
||||
'add_chapters': opts.addchapters,
|
||||
'add_metadata': opts.addmetadata,
|
||||
'add_infojson': opts.embed_infojson,
|
||||
})
|
||||
# Deprecated
|
||||
# This should be above EmbedThumbnail since sponskrub removes the thumbnail attachment
|
||||
# but must be below EmbedSubtitle and FFmpegMetadata
|
||||
# See https://github.com/yt-dlp/yt-dlp/issues/204 , https://github.com/faissaloo/SponSkrub/issues/29
|
||||
# If opts.sponskrub is None, sponskrub is used, but it silently fails if the executable can't be found
|
||||
if opts.sponskrub is not False:
|
||||
postprocessors.append({
|
||||
'key': 'SponSkrub',
|
||||
'path': opts.sponskrub_path,
|
||||
'args': opts.sponskrub_args,
|
||||
'cut': opts.sponskrub_cut,
|
||||
'force': opts.sponskrub_force,
|
||||
'ignoreerror': opts.sponskrub is None,
|
||||
'_from_cli': True,
|
||||
})
|
||||
if opts.embedthumbnail:
|
||||
postprocessors.append({
|
||||
'key': 'EmbedThumbnail',
|
||||
# already_have_thumbnail = True prevents the file from being deleted after embedding
|
||||
'already_have_thumbnail': opts.writethumbnail
|
||||
})
|
||||
if not opts.writethumbnail:
|
||||
opts.writethumbnail = True
|
||||
opts.outtmpl['pl_thumbnail'] = ''
|
||||
if opts.split_chapters:
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegSplitChapters',
|
||||
'force_keyframes': opts.force_keyframes_at_cuts,
|
||||
})
|
||||
# XAttrMetadataPP should be run after post-processors that may change file contents
|
||||
if opts.xattrs:
|
||||
postprocessors.append({'key': 'XAttrMetadata'})
|
||||
if opts.concat_playlist != 'never':
|
||||
postprocessors.append({
|
||||
'key': 'FFmpegConcat',
|
||||
'only_multi_video': opts.concat_playlist != 'always',
|
||||
'when': 'playlist',
|
||||
})
|
||||
# Exec must be the last PP of each category
|
||||
if opts.exec_before_dl_cmd:
|
||||
opts.exec_cmd.setdefault('before_dl', opts.exec_before_dl_cmd)
|
||||
for when, exec_cmd in opts.exec_cmd.items():
|
||||
postprocessors.append({
|
||||
'key': 'Exec',
|
||||
'exec_cmd': exec_cmd,
|
||||
# Run this only after the files have been moved to their final locations
|
||||
'when': when,
|
||||
})
|
||||
|
||||
def report_args_compat(arg, name):
|
||||
warnings.append('%s given without specifying name. The arguments will be given to all %s' % (arg, name))
|
||||
|
||||
if 'default' in opts.external_downloader_args:
|
||||
report_args_compat('--downloader-args', 'external downloaders')
|
||||
|
||||
if 'default-compat' in opts.postprocessor_args and 'default' not in opts.postprocessor_args:
|
||||
report_args_compat('--post-processor-args', 'post-processors')
|
||||
opts.postprocessor_args.setdefault('sponskrub', [])
|
||||
opts.postprocessor_args['default'] = opts.postprocessor_args['default-compat']
|
||||
|
||||
def report_deprecation(val, old, new=None):
|
||||
if not val:
|
||||
return
|
||||
deprecation_warnings.append(
|
||||
f'{old} is deprecated and may be removed in a future version. Use {new} instead' if new
|
||||
else f'{old} is deprecated and may not work as expected')
|
||||
|
||||
report_deprecation(opts.sponskrub, '--sponskrub', '--sponsorblock-mark or --sponsorblock-remove')
|
||||
report_deprecation(not opts.prefer_ffmpeg, '--prefer-avconv', 'ffmpeg')
|
||||
report_deprecation(opts.include_ads, '--include-ads')
|
||||
# report_deprecation(opts.call_home, '--call-home') # We may re-implement this in future
|
||||
# report_deprecation(opts.writeannotations, '--write-annotations') # It's just that no website has it
|
||||
|
||||
final_ext = (
|
||||
opts.recodevideo if opts.recodevideo in FFmpegVideoConvertorPP.SUPPORTED_EXTS
|
||||
else opts.remuxvideo if opts.remuxvideo in FFmpegVideoRemuxerPP.SUPPORTED_EXTS
|
||||
else opts.audioformat if (opts.extractaudio and opts.audioformat != 'best')
|
||||
else None)
|
||||
|
||||
match_filter = (
|
||||
None if opts.match_filter is None
|
||||
else match_filter_func(opts.match_filter))
|
||||
|
||||
ydl_opts = {
|
||||
'usenetrc': opts.usenetrc,
|
||||
'netrc_location': opts.netrc_location,
|
||||
'username': opts.username,
|
||||
'password': opts.password,
|
||||
'twofactor': opts.twofactor,
|
||||
'videopassword': opts.videopassword,
|
||||
'ap_mso': opts.ap_mso,
|
||||
'ap_username': opts.ap_username,
|
||||
'ap_password': opts.ap_password,
|
||||
'quiet': (opts.quiet or any_getting or any_printing),
|
||||
'no_warnings': opts.no_warnings,
|
||||
'forceurl': opts.geturl,
|
||||
'forcetitle': opts.gettitle,
|
||||
'forceid': opts.getid,
|
||||
'forcethumbnail': opts.getthumbnail,
|
||||
'forcedescription': opts.getdescription,
|
||||
'forceduration': opts.getduration,
|
||||
'forcefilename': opts.getfilename,
|
||||
'forceformat': opts.getformat,
|
||||
'forceprint': opts.forceprint,
|
||||
'print_to_file': opts.print_to_file,
|
||||
'forcejson': opts.dumpjson or opts.print_json,
|
||||
'dump_single_json': opts.dump_single_json,
|
||||
'force_write_download_archive': opts.force_write_download_archive,
|
||||
'simulate': (any_getting or None) if opts.simulate is None else opts.simulate,
|
||||
'skip_download': opts.skip_download,
|
||||
'format': opts.format,
|
||||
'allow_unplayable_formats': opts.allow_unplayable_formats,
|
||||
'ignore_no_formats_error': opts.ignore_no_formats_error,
|
||||
'format_sort': opts.format_sort,
|
||||
'format_sort_force': opts.format_sort_force,
|
||||
'allow_multiple_video_streams': opts.allow_multiple_video_streams,
|
||||
'allow_multiple_audio_streams': opts.allow_multiple_audio_streams,
|
||||
'check_formats': opts.check_formats,
|
||||
'listformats': opts.listformats,
|
||||
'listformats_table': opts.listformats_table,
|
||||
'outtmpl': opts.outtmpl,
|
||||
'outtmpl_na_placeholder': opts.outtmpl_na_placeholder,
|
||||
'paths': opts.paths,
|
||||
'autonumber_size': opts.autonumber_size,
|
||||
'autonumber_start': opts.autonumber_start,
|
||||
'restrictfilenames': opts.restrictfilenames,
|
||||
'windowsfilenames': opts.windowsfilenames,
|
||||
'ignoreerrors': opts.ignoreerrors,
|
||||
'force_generic_extractor': opts.force_generic_extractor,
|
||||
'ratelimit': opts.ratelimit,
|
||||
'throttledratelimit': opts.throttledratelimit,
|
||||
'overwrites': opts.overwrites,
|
||||
'retries': opts.retries,
|
||||
'file_access_retries': opts.file_access_retries,
|
||||
'fragment_retries': opts.fragment_retries,
|
||||
'extractor_retries': opts.extractor_retries,
|
||||
'skip_unavailable_fragments': opts.skip_unavailable_fragments,
|
||||
'keep_fragments': opts.keep_fragments,
|
||||
'concurrent_fragment_downloads': opts.concurrent_fragment_downloads,
|
||||
'buffersize': opts.buffersize,
|
||||
'noresizebuffer': opts.noresizebuffer,
|
||||
'http_chunk_size': opts.http_chunk_size,
|
||||
'continuedl': opts.continue_dl,
|
||||
'noprogress': opts.quiet if opts.noprogress is None else opts.noprogress,
|
||||
'progress_with_newline': opts.progress_with_newline,
|
||||
'progress_template': opts.progress_template,
|
||||
'playliststart': opts.playliststart,
|
||||
'playlistend': opts.playlistend,
|
||||
'playlistreverse': opts.playlist_reverse,
|
||||
'playlistrandom': opts.playlist_random,
|
||||
'noplaylist': opts.noplaylist,
|
||||
'logtostderr': outtmpl_default == '-',
|
||||
'consoletitle': opts.consoletitle,
|
||||
'nopart': opts.nopart,
|
||||
'updatetime': opts.updatetime,
|
||||
'writedescription': opts.writedescription,
|
||||
'writeannotations': opts.writeannotations,
|
||||
'writeinfojson': opts.writeinfojson,
|
||||
'allow_playlist_files': opts.allow_playlist_files,
|
||||
'clean_infojson': opts.clean_infojson,
|
||||
'getcomments': opts.getcomments,
|
||||
'writethumbnail': opts.writethumbnail is True,
|
||||
'write_all_thumbnails': opts.writethumbnail == 'all',
|
||||
'writelink': opts.writelink,
|
||||
'writeurllink': opts.writeurllink,
|
||||
'writewebloclink': opts.writewebloclink,
|
||||
'writedesktoplink': opts.writedesktoplink,
|
||||
'writesubtitles': opts.writesubtitles,
|
||||
'writeautomaticsub': opts.writeautomaticsub,
|
||||
'allsubtitles': opts.allsubtitles,
|
||||
'listsubtitles': opts.listsubtitles,
|
||||
'subtitlesformat': opts.subtitlesformat,
|
||||
'subtitleslangs': opts.subtitleslangs,
|
||||
'matchtitle': decodeOption(opts.matchtitle),
|
||||
'rejecttitle': decodeOption(opts.rejecttitle),
|
||||
'max_downloads': opts.max_downloads,
|
||||
'prefer_free_formats': opts.prefer_free_formats,
|
||||
'trim_file_name': opts.trim_file_name,
|
||||
'verbose': opts.verbose,
|
||||
'dump_intermediate_pages': opts.dump_intermediate_pages,
|
||||
'write_pages': opts.write_pages,
|
||||
'test': opts.test,
|
||||
'keepvideo': opts.keepvideo,
|
||||
'min_filesize': opts.min_filesize,
|
||||
'max_filesize': opts.max_filesize,
|
||||
'min_views': opts.min_views,
|
||||
'max_views': opts.max_views,
|
||||
'daterange': date,
|
||||
'cachedir': opts.cachedir,
|
||||
'youtube_print_sig_code': opts.youtube_print_sig_code,
|
||||
'age_limit': opts.age_limit,
|
||||
'download_archive': download_archive_fn,
|
||||
'break_on_existing': opts.break_on_existing,
|
||||
'break_on_reject': opts.break_on_reject,
|
||||
'break_per_url': opts.break_per_url,
|
||||
'skip_playlist_after_errors': opts.skip_playlist_after_errors,
|
||||
'cookiefile': opts.cookiefile,
|
||||
'cookiesfrombrowser': opts.cookiesfrombrowser,
|
||||
'legacyserverconnect': opts.legacy_server_connect,
|
||||
'nocheckcertificate': opts.no_check_certificate,
|
||||
'prefer_insecure': opts.prefer_insecure,
|
||||
'proxy': opts.proxy,
|
||||
'socket_timeout': opts.socket_timeout,
|
||||
'bidi_workaround': opts.bidi_workaround,
|
||||
'debug_printtraffic': opts.debug_printtraffic,
|
||||
'prefer_ffmpeg': opts.prefer_ffmpeg,
|
||||
'include_ads': opts.include_ads,
|
||||
'default_search': opts.default_search,
|
||||
'dynamic_mpd': opts.dynamic_mpd,
|
||||
'extractor_args': opts.extractor_args,
|
||||
'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
|
||||
'youtube_include_hls_manifest': opts.youtube_include_hls_manifest,
|
||||
'encoding': opts.encoding,
|
||||
'extract_flat': opts.extract_flat,
|
||||
'live_from_start': opts.live_from_start,
|
||||
'wait_for_video': opts.wait_for_video,
|
||||
'mark_watched': opts.mark_watched,
|
||||
'merge_output_format': opts.merge_output_format,
|
||||
'final_ext': final_ext,
|
||||
'postprocessors': postprocessors,
|
||||
'fixup': opts.fixup,
|
||||
'source_address': opts.source_address,
|
||||
'call_home': opts.call_home,
|
||||
'sleep_interval_requests': opts.sleep_interval_requests,
|
||||
'sleep_interval': opts.sleep_interval,
|
||||
'max_sleep_interval': opts.max_sleep_interval,
|
||||
'sleep_interval_subtitles': opts.sleep_interval_subtitles,
|
||||
'external_downloader': opts.external_downloader,
|
||||
'list_thumbnails': opts.list_thumbnails,
|
||||
'playlist_items': opts.playlist_items,
|
||||
'xattr_set_filesize': opts.xattr_set_filesize,
|
||||
'match_filter': match_filter,
|
||||
'no_color': opts.no_color,
|
||||
'ffmpeg_location': opts.ffmpeg_location,
|
||||
'hls_prefer_native': opts.hls_prefer_native,
|
||||
'hls_use_mpegts': opts.hls_use_mpegts,
|
||||
'hls_split_discontinuity': opts.hls_split_discontinuity,
|
||||
'external_downloader_args': opts.external_downloader_args,
|
||||
'postprocessor_args': opts.postprocessor_args,
|
||||
'cn_verification_proxy': opts.cn_verification_proxy,
|
||||
'geo_verification_proxy': opts.geo_verification_proxy,
|
||||
'geo_bypass': opts.geo_bypass,
|
||||
'geo_bypass_country': opts.geo_bypass_country,
|
||||
'geo_bypass_ip_block': opts.geo_bypass_ip_block,
|
||||
'_warnings': warnings,
|
||||
'_deprecation_warnings': deprecation_warnings,
|
||||
'compat_opts': compat_opts,
|
||||
}
|
||||
|
||||
with YoutubeDL(ydl_opts) as ydl:
|
||||
actual_use = all_urls or opts.load_info_filename
|
||||
|
||||
# Remove cache dir
|
||||
if opts.rm_cachedir:
|
||||
ydl.cache.remove()
|
||||
|
||||
# Update version
|
||||
if opts.update_self:
|
||||
# If updater returns True, exit. Required for windows
|
||||
if run_update(ydl):
|
||||
if actual_use:
|
||||
sys.exit('ERROR: The program must exit for the update to complete')
|
||||
sys.exit()
|
||||
|
||||
# Maybe do nothing
|
||||
if not actual_use:
|
||||
if opts.update_self or opts.rm_cachedir:
|
||||
sys.exit()
|
||||
|
||||
ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
|
||||
parser.error(
|
||||
'You must provide at least one URL.\n'
|
||||
'Type yt-dlp --help to see a list of all options.')
|
||||
|
||||
try:
|
||||
if opts.load_info_filename is not None:
|
||||
retcode = ydl.download_with_info_file(expand_path(opts.load_info_filename))
|
||||
else:
|
||||
retcode = ydl.download(all_urls)
|
||||
except DownloadCancelled:
|
||||
ydl.to_screen('Aborting remaining downloads')
|
||||
retcode = 101
|
||||
|
||||
sys.exit(retcode)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
_real_main(argv)
|
||||
except DownloadError:
|
||||
sys.exit(1)
|
||||
except SameFileError as e:
|
||||
sys.exit(f'ERROR: {e}')
|
||||
except KeyboardInterrupt:
|
||||
sys.exit('\nERROR: Interrupted by user')
|
||||
except BrokenPipeError as e:
|
||||
# https://docs.python.org/3/library/signal.html#note-on-sigpipe
|
||||
devnull = os.open(os.devnull, os.O_WRONLY)
|
||||
os.dup2(devnull, sys.stdout.fileno())
|
||||
sys.exit(f'\nERROR: {e}')
|
||||
|
||||
|
||||
__all__ = ['main', 'YoutubeDL', 'gen_extractors', 'list_extractors']
|
||||
19
plugins/youtube_download/yt_dlp/__main__.py
Normal file
19
plugins/youtube_download/yt_dlp/__main__.py
Normal file
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import unicode_literals
|
||||
|
||||
# Execute with
|
||||
# $ python yt_dlp/__main__.py (2.6+)
|
||||
# $ python -m yt_dlp (2.7+)
|
||||
|
||||
import sys
|
||||
|
||||
if __package__ is None and not hasattr(sys, 'frozen'):
|
||||
# direct call of __main__.py
|
||||
import os.path
|
||||
path = os.path.realpath(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(path)))
|
||||
|
||||
import yt_dlp
|
||||
|
||||
if __name__ == '__main__':
|
||||
yt_dlp.main()
|
||||
522
plugins/youtube_download/yt_dlp/aes.py
Normal file
522
plugins/youtube_download/yt_dlp/aes.py
Normal file
@@ -0,0 +1,522 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from math import ceil
|
||||
|
||||
from .compat import (
|
||||
compat_b64decode,
|
||||
compat_ord,
|
||||
compat_pycrypto_AES,
|
||||
)
|
||||
from .utils import (
|
||||
bytes_to_intlist,
|
||||
intlist_to_bytes,
|
||||
)
|
||||
|
||||
|
||||
if compat_pycrypto_AES:
|
||||
def aes_cbc_decrypt_bytes(data, key, iv):
|
||||
""" Decrypt bytes with AES-CBC using pycryptodome """
|
||||
return compat_pycrypto_AES.new(key, compat_pycrypto_AES.MODE_CBC, iv).decrypt(data)
|
||||
|
||||
def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce):
|
||||
""" Decrypt bytes with AES-GCM using pycryptodome """
|
||||
return compat_pycrypto_AES.new(key, compat_pycrypto_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag)
|
||||
|
||||
else:
|
||||
def aes_cbc_decrypt_bytes(data, key, iv):
|
||||
""" Decrypt bytes with AES-CBC using native implementation since pycryptodome is unavailable """
|
||||
return intlist_to_bytes(aes_cbc_decrypt(*map(bytes_to_intlist, (data, key, iv))))
|
||||
|
||||
def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce):
|
||||
""" Decrypt bytes with AES-GCM using native implementation since pycryptodome is unavailable """
|
||||
return intlist_to_bytes(aes_gcm_decrypt_and_verify(*map(bytes_to_intlist, (data, key, tag, nonce))))
|
||||
|
||||
|
||||
def unpad_pkcs7(data):
|
||||
return data[:-compat_ord(data[-1])]
|
||||
|
||||
|
||||
BLOCK_SIZE_BYTES = 16
|
||||
|
||||
|
||||
def aes_ecb_encrypt(data, key, iv=None):
|
||||
"""
|
||||
Encrypt with aes in ECB mode
|
||||
|
||||
@param {int[]} data cleartext
|
||||
@param {int[]} key 16/24/32-Byte cipher key
|
||||
@param {int[]} iv Unused for this mode
|
||||
@returns {int[]} encrypted data
|
||||
"""
|
||||
expanded_key = key_expansion(key)
|
||||
block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
|
||||
|
||||
encrypted_data = []
|
||||
for i in range(block_count):
|
||||
block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
|
||||
encrypted_data += aes_encrypt(block, expanded_key)
|
||||
encrypted_data = encrypted_data[:len(data)]
|
||||
|
||||
return encrypted_data
|
||||
|
||||
|
||||
def aes_ecb_decrypt(data, key, iv=None):
|
||||
"""
|
||||
Decrypt with aes in ECB mode
|
||||
|
||||
@param {int[]} data cleartext
|
||||
@param {int[]} key 16/24/32-Byte cipher key
|
||||
@param {int[]} iv Unused for this mode
|
||||
@returns {int[]} decrypted data
|
||||
"""
|
||||
expanded_key = key_expansion(key)
|
||||
block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
|
||||
|
||||
encrypted_data = []
|
||||
for i in range(block_count):
|
||||
block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
|
||||
encrypted_data += aes_decrypt(block, expanded_key)
|
||||
encrypted_data = encrypted_data[:len(data)]
|
||||
|
||||
return encrypted_data
|
||||
|
||||
|
||||
def aes_ctr_decrypt(data, key, iv):
|
||||
"""
|
||||
Decrypt with aes in counter mode
|
||||
|
||||
@param {int[]} data cipher
|
||||
@param {int[]} key 16/24/32-Byte cipher key
|
||||
@param {int[]} iv 16-Byte initialization vector
|
||||
@returns {int[]} decrypted data
|
||||
"""
|
||||
return aes_ctr_encrypt(data, key, iv)
|
||||
|
||||
|
||||
def aes_ctr_encrypt(data, key, iv):
|
||||
"""
|
||||
Encrypt with aes in counter mode
|
||||
|
||||
@param {int[]} data cleartext
|
||||
@param {int[]} key 16/24/32-Byte cipher key
|
||||
@param {int[]} iv 16-Byte initialization vector
|
||||
@returns {int[]} encrypted data
|
||||
"""
|
||||
expanded_key = key_expansion(key)
|
||||
block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
|
||||
counter = iter_vector(iv)
|
||||
|
||||
encrypted_data = []
|
||||
for i in range(block_count):
|
||||
counter_block = next(counter)
|
||||
block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
|
||||
block += [0] * (BLOCK_SIZE_BYTES - len(block))
|
||||
|
||||
cipher_counter_block = aes_encrypt(counter_block, expanded_key)
|
||||
encrypted_data += xor(block, cipher_counter_block)
|
||||
encrypted_data = encrypted_data[:len(data)]
|
||||
|
||||
return encrypted_data
|
||||
|
||||
|
||||
def aes_cbc_decrypt(data, key, iv):
|
||||
"""
|
||||
Decrypt with aes in CBC mode
|
||||
|
||||
@param {int[]} data cipher
|
||||
@param {int[]} key 16/24/32-Byte cipher key
|
||||
@param {int[]} iv 16-Byte IV
|
||||
@returns {int[]} decrypted data
|
||||
"""
|
||||
expanded_key = key_expansion(key)
|
||||
block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
|
||||
|
||||
decrypted_data = []
|
||||
previous_cipher_block = iv
|
||||
for i in range(block_count):
|
||||
block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
|
||||
block += [0] * (BLOCK_SIZE_BYTES - len(block))
|
||||
|
||||
decrypted_block = aes_decrypt(block, expanded_key)
|
||||
decrypted_data += xor(decrypted_block, previous_cipher_block)
|
||||
previous_cipher_block = block
|
||||
decrypted_data = decrypted_data[:len(data)]
|
||||
|
||||
return decrypted_data
|
||||
|
||||
|
||||
def aes_cbc_encrypt(data, key, iv):
|
||||
"""
|
||||
Encrypt with aes in CBC mode. Using PKCS#7 padding
|
||||
|
||||
@param {int[]} data cleartext
|
||||
@param {int[]} key 16/24/32-Byte cipher key
|
||||
@param {int[]} iv 16-Byte IV
|
||||
@returns {int[]} encrypted data
|
||||
"""
|
||||
expanded_key = key_expansion(key)
|
||||
block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
|
||||
|
||||
encrypted_data = []
|
||||
previous_cipher_block = iv
|
||||
for i in range(block_count):
|
||||
block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
|
||||
remaining_length = BLOCK_SIZE_BYTES - len(block)
|
||||
block += [remaining_length] * remaining_length
|
||||
mixed_block = xor(block, previous_cipher_block)
|
||||
|
||||
encrypted_block = aes_encrypt(mixed_block, expanded_key)
|
||||
encrypted_data += encrypted_block
|
||||
|
||||
previous_cipher_block = encrypted_block
|
||||
|
||||
return encrypted_data
|
||||
|
||||
|
||||
def aes_gcm_decrypt_and_verify(data, key, tag, nonce):
|
||||
"""
|
||||
Decrypt with aes in GBM mode and checks authenticity using tag
|
||||
|
||||
@param {int[]} data cipher
|
||||
@param {int[]} key 16-Byte cipher key
|
||||
@param {int[]} tag authentication tag
|
||||
@param {int[]} nonce IV (recommended 12-Byte)
|
||||
@returns {int[]} decrypted data
|
||||
"""
|
||||
|
||||
# XXX: check aes, gcm param
|
||||
|
||||
hash_subkey = aes_encrypt([0] * BLOCK_SIZE_BYTES, key_expansion(key))
|
||||
|
||||
if len(nonce) == 12:
|
||||
j0 = nonce + [0, 0, 0, 1]
|
||||
else:
|
||||
fill = (BLOCK_SIZE_BYTES - (len(nonce) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES + 8
|
||||
ghash_in = nonce + [0] * fill + bytes_to_intlist((8 * len(nonce)).to_bytes(8, 'big'))
|
||||
j0 = ghash(hash_subkey, ghash_in)
|
||||
|
||||
# TODO: add nonce support to aes_ctr_decrypt
|
||||
|
||||
# nonce_ctr = j0[:12]
|
||||
iv_ctr = inc(j0)
|
||||
|
||||
decrypted_data = aes_ctr_decrypt(data, key, iv_ctr + [0] * (BLOCK_SIZE_BYTES - len(iv_ctr)))
|
||||
pad_len = len(data) // 16 * 16
|
||||
s_tag = ghash(
|
||||
hash_subkey,
|
||||
data
|
||||
+ [0] * (BLOCK_SIZE_BYTES - len(data) + pad_len) # pad
|
||||
+ bytes_to_intlist((0 * 8).to_bytes(8, 'big') # length of associated data
|
||||
+ ((len(data) * 8).to_bytes(8, 'big'))) # length of data
|
||||
)
|
||||
|
||||
if tag != aes_ctr_encrypt(s_tag, key, j0):
|
||||
raise ValueError("Mismatching authentication tag")
|
||||
|
||||
return decrypted_data
|
||||
|
||||
|
||||
def aes_encrypt(data, expanded_key):
|
||||
"""
|
||||
Encrypt one block with aes
|
||||
|
||||
@param {int[]} data 16-Byte state
|
||||
@param {int[]} expanded_key 176/208/240-Byte expanded key
|
||||
@returns {int[]} 16-Byte cipher
|
||||
"""
|
||||
rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
|
||||
|
||||
data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
|
||||
for i in range(1, rounds + 1):
|
||||
data = sub_bytes(data)
|
||||
data = shift_rows(data)
|
||||
if i != rounds:
|
||||
data = list(iter_mix_columns(data, MIX_COLUMN_MATRIX))
|
||||
data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def aes_decrypt(data, expanded_key):
|
||||
"""
|
||||
Decrypt one block with aes
|
||||
|
||||
@param {int[]} data 16-Byte cipher
|
||||
@param {int[]} expanded_key 176/208/240-Byte expanded key
|
||||
@returns {int[]} 16-Byte state
|
||||
"""
|
||||
rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
|
||||
|
||||
for i in range(rounds, 0, -1):
|
||||
data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
|
||||
if i != rounds:
|
||||
data = list(iter_mix_columns(data, MIX_COLUMN_MATRIX_INV))
|
||||
data = shift_rows_inv(data)
|
||||
data = sub_bytes_inv(data)
|
||||
data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def aes_decrypt_text(data, password, key_size_bytes):
|
||||
"""
|
||||
Decrypt text
|
||||
- The first 8 Bytes of decoded 'data' are the 8 high Bytes of the counter
|
||||
- The cipher key is retrieved by encrypting the first 16 Byte of 'password'
|
||||
with the first 'key_size_bytes' Bytes from 'password' (if necessary filled with 0's)
|
||||
- Mode of operation is 'counter'
|
||||
|
||||
@param {str} data Base64 encoded string
|
||||
@param {str,unicode} password Password (will be encoded with utf-8)
|
||||
@param {int} key_size_bytes Possible values: 16 for 128-Bit, 24 for 192-Bit or 32 for 256-Bit
|
||||
@returns {str} Decrypted data
|
||||
"""
|
||||
NONCE_LENGTH_BYTES = 8
|
||||
|
||||
data = bytes_to_intlist(compat_b64decode(data))
|
||||
password = bytes_to_intlist(password.encode('utf-8'))
|
||||
|
||||
key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))
|
||||
key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES)
|
||||
|
||||
nonce = data[:NONCE_LENGTH_BYTES]
|
||||
cipher = data[NONCE_LENGTH_BYTES:]
|
||||
|
||||
decrypted_data = aes_ctr_decrypt(cipher, key, nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES))
|
||||
plaintext = intlist_to_bytes(decrypted_data)
|
||||
|
||||
return plaintext
|
||||
|
||||
|
||||
RCON = (0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36)
|
||||
SBOX = (0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
|
||||
0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0,
|
||||
0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15,
|
||||
0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75,
|
||||
0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84,
|
||||
0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF,
|
||||
0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8,
|
||||
0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2,
|
||||
0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73,
|
||||
0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB,
|
||||
0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79,
|
||||
0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08,
|
||||
0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A,
|
||||
0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E,
|
||||
0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF,
|
||||
0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16)
|
||||
SBOX_INV = (0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
|
||||
0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
|
||||
0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
|
||||
0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
|
||||
0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
|
||||
0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
|
||||
0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
|
||||
0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
|
||||
0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
|
||||
0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
|
||||
0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
|
||||
0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
|
||||
0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
|
||||
0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
|
||||
0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
|
||||
0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d)
|
||||
MIX_COLUMN_MATRIX = ((0x2, 0x3, 0x1, 0x1),
|
||||
(0x1, 0x2, 0x3, 0x1),
|
||||
(0x1, 0x1, 0x2, 0x3),
|
||||
(0x3, 0x1, 0x1, 0x2))
|
||||
MIX_COLUMN_MATRIX_INV = ((0xE, 0xB, 0xD, 0x9),
|
||||
(0x9, 0xE, 0xB, 0xD),
|
||||
(0xD, 0x9, 0xE, 0xB),
|
||||
(0xB, 0xD, 0x9, 0xE))
|
||||
RIJNDAEL_EXP_TABLE = (0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35,
|
||||
0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA,
|
||||
0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31,
|
||||
0x53, 0xF5, 0x04, 0x0C, 0x14, 0x3C, 0x44, 0xCC, 0x4F, 0xD1, 0x68, 0xB8, 0xD3, 0x6E, 0xB2, 0xCD,
|
||||
0x4C, 0xD4, 0x67, 0xA9, 0xE0, 0x3B, 0x4D, 0xD7, 0x62, 0xA6, 0xF1, 0x08, 0x18, 0x28, 0x78, 0x88,
|
||||
0x83, 0x9E, 0xB9, 0xD0, 0x6B, 0xBD, 0xDC, 0x7F, 0x81, 0x98, 0xB3, 0xCE, 0x49, 0xDB, 0x76, 0x9A,
|
||||
0xB5, 0xC4, 0x57, 0xF9, 0x10, 0x30, 0x50, 0xF0, 0x0B, 0x1D, 0x27, 0x69, 0xBB, 0xD6, 0x61, 0xA3,
|
||||
0xFE, 0x19, 0x2B, 0x7D, 0x87, 0x92, 0xAD, 0xEC, 0x2F, 0x71, 0x93, 0xAE, 0xE9, 0x20, 0x60, 0xA0,
|
||||
0xFB, 0x16, 0x3A, 0x4E, 0xD2, 0x6D, 0xB7, 0xC2, 0x5D, 0xE7, 0x32, 0x56, 0xFA, 0x15, 0x3F, 0x41,
|
||||
0xC3, 0x5E, 0xE2, 0x3D, 0x47, 0xC9, 0x40, 0xC0, 0x5B, 0xED, 0x2C, 0x74, 0x9C, 0xBF, 0xDA, 0x75,
|
||||
0x9F, 0xBA, 0xD5, 0x64, 0xAC, 0xEF, 0x2A, 0x7E, 0x82, 0x9D, 0xBC, 0xDF, 0x7A, 0x8E, 0x89, 0x80,
|
||||
0x9B, 0xB6, 0xC1, 0x58, 0xE8, 0x23, 0x65, 0xAF, 0xEA, 0x25, 0x6F, 0xB1, 0xC8, 0x43, 0xC5, 0x54,
|
||||
0xFC, 0x1F, 0x21, 0x63, 0xA5, 0xF4, 0x07, 0x09, 0x1B, 0x2D, 0x77, 0x99, 0xB0, 0xCB, 0x46, 0xCA,
|
||||
0x45, 0xCF, 0x4A, 0xDE, 0x79, 0x8B, 0x86, 0x91, 0xA8, 0xE3, 0x3E, 0x42, 0xC6, 0x51, 0xF3, 0x0E,
|
||||
0x12, 0x36, 0x5A, 0xEE, 0x29, 0x7B, 0x8D, 0x8C, 0x8F, 0x8A, 0x85, 0x94, 0xA7, 0xF2, 0x0D, 0x17,
|
||||
0x39, 0x4B, 0xDD, 0x7C, 0x84, 0x97, 0xA2, 0xFD, 0x1C, 0x24, 0x6C, 0xB4, 0xC7, 0x52, 0xF6, 0x01)
|
||||
RIJNDAEL_LOG_TABLE = (0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7, 0x1b, 0x68, 0x33, 0xee, 0xdf, 0x03,
|
||||
0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef, 0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1,
|
||||
0x7d, 0xc2, 0x1d, 0xb5, 0xf9, 0xb9, 0x27, 0x6a, 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78,
|
||||
0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, 0x12, 0xf0, 0x82, 0x45, 0x35, 0x93, 0xda, 0x8e,
|
||||
0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94, 0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38,
|
||||
0x66, 0xdd, 0xfd, 0x30, 0xbf, 0x06, 0x8b, 0x62, 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10,
|
||||
0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, 0x3a, 0x6b, 0x28, 0x54, 0xfa, 0x85, 0x3d, 0xba,
|
||||
0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca, 0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57,
|
||||
0xaf, 0x58, 0xa8, 0x50, 0xf4, 0xea, 0xd6, 0x74, 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8,
|
||||
0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, 0x59, 0xcb, 0x5f, 0xb0, 0x9c, 0xa9, 0x51, 0xa0,
|
||||
0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec, 0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7,
|
||||
0xcc, 0xbb, 0x3e, 0x5a, 0xfb, 0x60, 0xb1, 0x86, 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d,
|
||||
0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, 0xbc, 0x95, 0xcf, 0xcd, 0x37, 0x3f, 0x5b, 0xd1,
|
||||
0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47, 0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab,
|
||||
0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5,
|
||||
0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07)
|
||||
|
||||
|
||||
def key_expansion(data):
|
||||
"""
|
||||
Generate key schedule
|
||||
|
||||
@param {int[]} data 16/24/32-Byte cipher key
|
||||
@returns {int[]} 176/208/240-Byte expanded key
|
||||
"""
|
||||
data = data[:] # copy
|
||||
rcon_iteration = 1
|
||||
key_size_bytes = len(data)
|
||||
expanded_key_size_bytes = (key_size_bytes // 4 + 7) * BLOCK_SIZE_BYTES
|
||||
|
||||
while len(data) < expanded_key_size_bytes:
|
||||
temp = data[-4:]
|
||||
temp = key_schedule_core(temp, rcon_iteration)
|
||||
rcon_iteration += 1
|
||||
data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
|
||||
|
||||
for _ in range(3):
|
||||
temp = data[-4:]
|
||||
data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
|
||||
|
||||
if key_size_bytes == 32:
|
||||
temp = data[-4:]
|
||||
temp = sub_bytes(temp)
|
||||
data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
|
||||
|
||||
for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0):
|
||||
temp = data[-4:]
|
||||
data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
|
||||
data = data[:expanded_key_size_bytes]
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def iter_vector(iv):
|
||||
while True:
|
||||
yield iv
|
||||
iv = inc(iv)
|
||||
|
||||
|
||||
def sub_bytes(data):
|
||||
return [SBOX[x] for x in data]
|
||||
|
||||
|
||||
def sub_bytes_inv(data):
|
||||
return [SBOX_INV[x] for x in data]
|
||||
|
||||
|
||||
def rotate(data):
|
||||
return data[1:] + [data[0]]
|
||||
|
||||
|
||||
def key_schedule_core(data, rcon_iteration):
|
||||
data = rotate(data)
|
||||
data = sub_bytes(data)
|
||||
data[0] = data[0] ^ RCON[rcon_iteration]
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def xor(data1, data2):
|
||||
return [x ^ y for x, y in zip(data1, data2)]
|
||||
|
||||
|
||||
def iter_mix_columns(data, matrix):
|
||||
for i in (0, 4, 8, 12):
|
||||
for row in matrix:
|
||||
mixed = 0
|
||||
for j in range(4):
|
||||
# xor is (+) and (-)
|
||||
mixed ^= (0 if data[i:i + 4][j] == 0 or row[j] == 0 else
|
||||
RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[data[i + j]] + RIJNDAEL_LOG_TABLE[row[j]]) % 0xFF])
|
||||
yield mixed
|
||||
|
||||
|
||||
def shift_rows(data):
|
||||
return [data[((column + row) & 0b11) * 4 + row] for column in range(4) for row in range(4)]
|
||||
|
||||
|
||||
def shift_rows_inv(data):
|
||||
return [data[((column - row) & 0b11) * 4 + row] for column in range(4) for row in range(4)]
|
||||
|
||||
|
||||
def shift_block(data):
|
||||
data_shifted = []
|
||||
|
||||
bit = 0
|
||||
for n in data:
|
||||
if bit:
|
||||
n |= 0x100
|
||||
bit = n & 1
|
||||
n >>= 1
|
||||
data_shifted.append(n)
|
||||
|
||||
return data_shifted
|
||||
|
||||
|
||||
def inc(data):
|
||||
data = data[:] # copy
|
||||
for i in range(len(data) - 1, -1, -1):
|
||||
if data[i] == 255:
|
||||
data[i] = 0
|
||||
else:
|
||||
data[i] = data[i] + 1
|
||||
break
|
||||
return data
|
||||
|
||||
|
||||
def block_product(block_x, block_y):
|
||||
# NIST SP 800-38D, Algorithm 1
|
||||
|
||||
if len(block_x) != BLOCK_SIZE_BYTES or len(block_y) != BLOCK_SIZE_BYTES:
|
||||
raise ValueError("Length of blocks need to be %d bytes" % BLOCK_SIZE_BYTES)
|
||||
|
||||
block_r = [0xE1] + [0] * (BLOCK_SIZE_BYTES - 1)
|
||||
block_v = block_y[:]
|
||||
block_z = [0] * BLOCK_SIZE_BYTES
|
||||
|
||||
for i in block_x:
|
||||
for bit in range(7, -1, -1):
|
||||
if i & (1 << bit):
|
||||
block_z = xor(block_z, block_v)
|
||||
|
||||
do_xor = block_v[-1] & 1
|
||||
block_v = shift_block(block_v)
|
||||
if do_xor:
|
||||
block_v = xor(block_v, block_r)
|
||||
|
||||
return block_z
|
||||
|
||||
|
||||
def ghash(subkey, data):
|
||||
# NIST SP 800-38D, Algorithm 2
|
||||
|
||||
if len(data) % BLOCK_SIZE_BYTES:
|
||||
raise ValueError("Length of data should be %d bytes" % BLOCK_SIZE_BYTES)
|
||||
|
||||
last_y = [0] * BLOCK_SIZE_BYTES
|
||||
for i in range(0, len(data), BLOCK_SIZE_BYTES):
|
||||
block = data[i : i + BLOCK_SIZE_BYTES] # noqa: E203
|
||||
last_y = block_product(xor(last_y, block), subkey)
|
||||
|
||||
return last_y
|
||||
|
||||
|
||||
__all__ = [
|
||||
'aes_ctr_decrypt',
|
||||
'aes_cbc_decrypt',
|
||||
'aes_cbc_decrypt_bytes',
|
||||
'aes_decrypt_text',
|
||||
'aes_encrypt',
|
||||
'aes_gcm_decrypt_and_verify',
|
||||
'aes_gcm_decrypt_and_verify_bytes',
|
||||
'key_expansion',
|
||||
'unpad_pkcs7',
|
||||
]
|
||||
98
plugins/youtube_download/yt_dlp/cache.py
Normal file
98
plugins/youtube_download/yt_dlp/cache.py
Normal file
@@ -0,0 +1,98 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import errno
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import traceback
|
||||
|
||||
from .compat import compat_getenv
|
||||
from .utils import (
|
||||
expand_path,
|
||||
write_json_file,
|
||||
)
|
||||
|
||||
|
||||
class Cache(object):
|
||||
def __init__(self, ydl):
|
||||
self._ydl = ydl
|
||||
|
||||
def _get_root_dir(self):
|
||||
res = self._ydl.params.get('cachedir')
|
||||
if res is None:
|
||||
cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
|
||||
res = os.path.join(cache_root, 'yt-dlp')
|
||||
return expand_path(res)
|
||||
|
||||
def _get_cache_fn(self, section, key, dtype):
|
||||
assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
|
||||
'invalid section %r' % section
|
||||
assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key
|
||||
return os.path.join(
|
||||
self._get_root_dir(), section, '%s.%s' % (key, dtype))
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self._ydl.params.get('cachedir') is not False
|
||||
|
||||
def store(self, section, key, data, dtype='json'):
|
||||
assert dtype in ('json',)
|
||||
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
fn = self._get_cache_fn(section, key, dtype)
|
||||
try:
|
||||
try:
|
||||
os.makedirs(os.path.dirname(fn))
|
||||
except OSError as ose:
|
||||
if ose.errno != errno.EEXIST:
|
||||
raise
|
||||
self._ydl.write_debug(f'Saving {section}.{key} to cache')
|
||||
write_json_file(data, fn)
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
self._ydl.report_warning(
|
||||
'Writing cache to %r failed: %s' % (fn, tb))
|
||||
|
||||
def load(self, section, key, dtype='json', default=None):
|
||||
assert dtype in ('json',)
|
||||
|
||||
if not self.enabled:
|
||||
return default
|
||||
|
||||
cache_fn = self._get_cache_fn(section, key, dtype)
|
||||
try:
|
||||
try:
|
||||
with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
|
||||
self._ydl.write_debug(f'Loading {section}.{key} from cache')
|
||||
return json.load(cachef)
|
||||
except ValueError:
|
||||
try:
|
||||
file_size = os.path.getsize(cache_fn)
|
||||
except (OSError, IOError) as oe:
|
||||
file_size = str(oe)
|
||||
self._ydl.report_warning(
|
||||
'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
|
||||
except IOError:
|
||||
pass # No cache available
|
||||
|
||||
return default
|
||||
|
||||
def remove(self):
|
||||
if not self.enabled:
|
||||
self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
|
||||
return
|
||||
|
||||
cachedir = self._get_root_dir()
|
||||
if not any((term in cachedir) for term in ('cache', 'tmp')):
|
||||
raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir)
|
||||
|
||||
self._ydl.to_screen(
|
||||
'Removing cache dir %s .' % cachedir, skip_eol=True)
|
||||
if os.path.exists(cachedir):
|
||||
self._ydl.to_screen('.', skip_eol=True)
|
||||
shutil.rmtree(cachedir)
|
||||
self._ydl.to_screen('.')
|
||||
311
plugins/youtube_download/yt_dlp/compat.py
Normal file
311
plugins/youtube_download/yt_dlp/compat.py
Normal file
@@ -0,0 +1,311 @@
|
||||
# coding: utf-8
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import collections
|
||||
import ctypes
|
||||
import getpass
|
||||
import html
|
||||
import html.parser
|
||||
import http
|
||||
import http.client
|
||||
import http.cookiejar
|
||||
import http.cookies
|
||||
import http.server
|
||||
import itertools
|
||||
import optparse
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tokenize
|
||||
import urllib
|
||||
import xml.etree.ElementTree as etree
|
||||
from subprocess import DEVNULL
|
||||
|
||||
|
||||
# HTMLParseError has been deprecated in Python 3.3 and removed in
|
||||
# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
|
||||
# and uniform cross-version exception handling
|
||||
class compat_HTMLParseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE
|
||||
# will not work since ctypes.WINFUNCTYPE does not exist in UNIX machines
|
||||
def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
|
||||
return ctypes.WINFUNCTYPE(*args, **kwargs)
|
||||
|
||||
|
||||
class _TreeBuilder(etree.TreeBuilder):
|
||||
def doctype(self, name, pubid, system):
|
||||
pass
|
||||
|
||||
|
||||
def compat_etree_fromstring(text):
|
||||
return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
|
||||
|
||||
|
||||
compat_os_name = os._name if os.name == 'java' else os.name
|
||||
|
||||
|
||||
if compat_os_name == 'nt':
|
||||
def compat_shlex_quote(s):
|
||||
return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
|
||||
else:
|
||||
from shlex import quote as compat_shlex_quote
|
||||
|
||||
|
||||
def compat_ord(c):
|
||||
if type(c) is int:
|
||||
return c
|
||||
else:
|
||||
return ord(c)
|
||||
|
||||
|
||||
def compat_setenv(key, value, env=os.environ):
|
||||
env[key] = value
|
||||
|
||||
|
||||
if compat_os_name == 'nt' and sys.version_info < (3, 8):
|
||||
# os.path.realpath on Windows does not follow symbolic links
|
||||
# prior to Python 3.8 (see https://bugs.python.org/issue9949)
|
||||
def compat_realpath(path):
|
||||
while os.path.islink(path):
|
||||
path = os.path.abspath(os.readlink(path))
|
||||
return path
|
||||
else:
|
||||
compat_realpath = os.path.realpath
|
||||
|
||||
|
||||
def compat_print(s):
|
||||
assert isinstance(s, compat_str)
|
||||
print(s)
|
||||
|
||||
|
||||
# Fix https://github.com/ytdl-org/youtube-dl/issues/4223
|
||||
# See http://bugs.python.org/issue9161 for what is broken
|
||||
def workaround_optparse_bug9161():
|
||||
op = optparse.OptionParser()
|
||||
og = optparse.OptionGroup(op, 'foo')
|
||||
try:
|
||||
og.add_option('-t')
|
||||
except TypeError:
|
||||
real_add_option = optparse.OptionGroup.add_option
|
||||
|
||||
def _compat_add_option(self, *args, **kwargs):
|
||||
enc = lambda v: (
|
||||
v.encode('ascii', 'replace') if isinstance(v, compat_str)
|
||||
else v)
|
||||
bargs = [enc(a) for a in args]
|
||||
bkwargs = dict(
|
||||
(k, enc(v)) for k, v in kwargs.items())
|
||||
return real_add_option(self, *bargs, **bkwargs)
|
||||
optparse.OptionGroup.add_option = _compat_add_option
|
||||
|
||||
|
||||
try:
|
||||
compat_Pattern = re.Pattern
|
||||
except AttributeError:
|
||||
compat_Pattern = type(re.compile(''))
|
||||
|
||||
|
||||
try:
|
||||
compat_Match = re.Match
|
||||
except AttributeError:
|
||||
compat_Match = type(re.compile('').match(''))
|
||||
|
||||
|
||||
try:
|
||||
compat_asyncio_run = asyncio.run # >= 3.7
|
||||
except AttributeError:
|
||||
def compat_asyncio_run(coro):
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_until_complete(coro)
|
||||
|
||||
asyncio.run = compat_asyncio_run
|
||||
|
||||
|
||||
# Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl
|
||||
# See https://github.com/yt-dlp/yt-dlp/issues/792
|
||||
# https://docs.python.org/3/library/os.path.html#os.path.expanduser
|
||||
if compat_os_name in ('nt', 'ce') and 'HOME' in os.environ:
|
||||
_userhome = os.environ['HOME']
|
||||
|
||||
def compat_expanduser(path):
|
||||
if not path.startswith('~'):
|
||||
return path
|
||||
i = path.replace('\\', '/', 1).find('/') # ~user
|
||||
if i < 0:
|
||||
i = len(path)
|
||||
userhome = os.path.join(os.path.dirname(_userhome), path[1:i]) if i > 1 else _userhome
|
||||
return userhome + path[i:]
|
||||
else:
|
||||
compat_expanduser = os.path.expanduser
|
||||
|
||||
|
||||
try:
|
||||
from Cryptodome.Cipher import AES as compat_pycrypto_AES
|
||||
except ImportError:
|
||||
try:
|
||||
from Crypto.Cipher import AES as compat_pycrypto_AES
|
||||
except ImportError:
|
||||
compat_pycrypto_AES = None
|
||||
|
||||
|
||||
WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
|
||||
|
||||
|
||||
def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.python.org/issue30075
|
||||
if compat_os_name != 'nt':
|
||||
return
|
||||
global WINDOWS_VT_MODE
|
||||
startupinfo = subprocess.STARTUPINFO()
|
||||
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
||||
try:
|
||||
subprocess.Popen('', shell=True, startupinfo=startupinfo)
|
||||
WINDOWS_VT_MODE = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# Deprecated
|
||||
|
||||
compat_basestring = str
|
||||
compat_chr = chr
|
||||
compat_filter = filter
|
||||
compat_input = input
|
||||
compat_integer_types = (int, )
|
||||
compat_kwargs = lambda kwargs: kwargs
|
||||
compat_map = map
|
||||
compat_numeric_types = (int, float, complex)
|
||||
compat_str = str
|
||||
compat_xpath = lambda xpath: xpath
|
||||
compat_zip = zip
|
||||
|
||||
compat_collections_abc = collections.abc
|
||||
compat_HTMLParser = html.parser.HTMLParser
|
||||
compat_HTTPError = urllib.error.HTTPError
|
||||
compat_Struct = struct.Struct
|
||||
compat_b64decode = base64.b64decode
|
||||
compat_cookiejar = http.cookiejar
|
||||
compat_cookiejar_Cookie = compat_cookiejar.Cookie
|
||||
compat_cookies = http.cookies
|
||||
compat_cookies_SimpleCookie = compat_cookies.SimpleCookie
|
||||
compat_etree_Element = etree.Element
|
||||
compat_etree_register_namespace = etree.register_namespace
|
||||
compat_get_terminal_size = shutil.get_terminal_size
|
||||
compat_getenv = os.getenv
|
||||
compat_getpass = getpass.getpass
|
||||
compat_html_entities = html.entities
|
||||
compat_html_entities_html5 = compat_html_entities.html5
|
||||
compat_http_client = http.client
|
||||
compat_http_server = http.server
|
||||
compat_itertools_count = itertools.count
|
||||
compat_parse_qs = urllib.parse.parse_qs
|
||||
compat_shlex_split = shlex.split
|
||||
compat_socket_create_connection = socket.create_connection
|
||||
compat_struct_pack = struct.pack
|
||||
compat_struct_unpack = struct.unpack
|
||||
compat_subprocess_get_DEVNULL = lambda: DEVNULL
|
||||
compat_tokenize_tokenize = tokenize.tokenize
|
||||
compat_urllib_error = urllib.error
|
||||
compat_urllib_parse = urllib.parse
|
||||
compat_urllib_parse_quote = urllib.parse.quote
|
||||
compat_urllib_parse_quote_plus = urllib.parse.quote_plus
|
||||
compat_urllib_parse_unquote = urllib.parse.unquote
|
||||
compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus
|
||||
compat_urllib_parse_unquote_to_bytes = urllib.parse.unquote_to_bytes
|
||||
compat_urllib_parse_urlencode = urllib.parse.urlencode
|
||||
compat_urllib_parse_urlparse = urllib.parse.urlparse
|
||||
compat_urllib_parse_urlunparse = urllib.parse.urlunparse
|
||||
compat_urllib_request = urllib.request
|
||||
compat_urllib_request_DataHandler = urllib.request.DataHandler
|
||||
compat_urllib_response = urllib.response
|
||||
compat_urlparse = urllib.parse
|
||||
compat_urlretrieve = urllib.request.urlretrieve
|
||||
compat_xml_parse_error = etree.ParseError
|
||||
|
||||
|
||||
# Set public objects
|
||||
|
||||
__all__ = [
|
||||
'WINDOWS_VT_MODE',
|
||||
'compat_HTMLParseError',
|
||||
'compat_HTMLParser',
|
||||
'compat_HTTPError',
|
||||
'compat_Match',
|
||||
'compat_Pattern',
|
||||
'compat_Struct',
|
||||
'compat_asyncio_run',
|
||||
'compat_b64decode',
|
||||
'compat_basestring',
|
||||
'compat_chr',
|
||||
'compat_collections_abc',
|
||||
'compat_cookiejar',
|
||||
'compat_cookiejar_Cookie',
|
||||
'compat_cookies',
|
||||
'compat_cookies_SimpleCookie',
|
||||
'compat_ctypes_WINFUNCTYPE',
|
||||
'compat_etree_Element',
|
||||
'compat_etree_fromstring',
|
||||
'compat_etree_register_namespace',
|
||||
'compat_expanduser',
|
||||
'compat_filter',
|
||||
'compat_get_terminal_size',
|
||||
'compat_getenv',
|
||||
'compat_getpass',
|
||||
'compat_html_entities',
|
||||
'compat_html_entities_html5',
|
||||
'compat_http_client',
|
||||
'compat_http_server',
|
||||
'compat_input',
|
||||
'compat_integer_types',
|
||||
'compat_itertools_count',
|
||||
'compat_kwargs',
|
||||
'compat_map',
|
||||
'compat_numeric_types',
|
||||
'compat_ord',
|
||||
'compat_os_name',
|
||||
'compat_parse_qs',
|
||||
'compat_print',
|
||||
'compat_pycrypto_AES',
|
||||
'compat_realpath',
|
||||
'compat_setenv',
|
||||
'compat_shlex_quote',
|
||||
'compat_shlex_split',
|
||||
'compat_socket_create_connection',
|
||||
'compat_str',
|
||||
'compat_struct_pack',
|
||||
'compat_struct_unpack',
|
||||
'compat_subprocess_get_DEVNULL',
|
||||
'compat_tokenize_tokenize',
|
||||
'compat_urllib_error',
|
||||
'compat_urllib_parse',
|
||||
'compat_urllib_parse_quote',
|
||||
'compat_urllib_parse_quote_plus',
|
||||
'compat_urllib_parse_unquote',
|
||||
'compat_urllib_parse_unquote_plus',
|
||||
'compat_urllib_parse_unquote_to_bytes',
|
||||
'compat_urllib_parse_urlencode',
|
||||
'compat_urllib_parse_urlparse',
|
||||
'compat_urllib_parse_urlunparse',
|
||||
'compat_urllib_request',
|
||||
'compat_urllib_request_DataHandler',
|
||||
'compat_urllib_response',
|
||||
'compat_urlparse',
|
||||
'compat_urlretrieve',
|
||||
'compat_xml_parse_error',
|
||||
'compat_xpath',
|
||||
'compat_zip',
|
||||
'windows_enable_vt_mode',
|
||||
'workaround_optparse_bug9161',
|
||||
]
|
||||
955
plugins/youtube_download/yt_dlp/cookies.py
Normal file
955
plugins/youtube_download/yt_dlp/cookies.py
Normal file
@@ -0,0 +1,955 @@
|
||||
import contextlib
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import Enum, auto
|
||||
from hashlib import pbkdf2_hmac
|
||||
|
||||
from .aes import (
|
||||
aes_cbc_decrypt_bytes,
|
||||
aes_gcm_decrypt_and_verify_bytes,
|
||||
unpad_pkcs7,
|
||||
)
|
||||
from .compat import (
|
||||
compat_b64decode,
|
||||
compat_cookiejar_Cookie,
|
||||
)
|
||||
from .utils import (
|
||||
expand_path,
|
||||
Popen,
|
||||
YoutubeDLCookieJar,
|
||||
)
|
||||
|
||||
try:
|
||||
import sqlite3
|
||||
SQLITE_AVAILABLE = True
|
||||
except ImportError:
|
||||
# although sqlite3 is part of the standard library, it is possible to compile python without
|
||||
# sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544
|
||||
SQLITE_AVAILABLE = False
|
||||
|
||||
|
||||
try:
|
||||
import secretstorage
|
||||
SECRETSTORAGE_AVAILABLE = True
|
||||
except ImportError:
|
||||
SECRETSTORAGE_AVAILABLE = False
|
||||
SECRETSTORAGE_UNAVAILABLE_REASON = (
|
||||
'as the `secretstorage` module is not installed. '
|
||||
'Please install by running `python3 -m pip install secretstorage`.')
|
||||
except Exception as _err:
|
||||
SECRETSTORAGE_AVAILABLE = False
|
||||
SECRETSTORAGE_UNAVAILABLE_REASON = f'as the `secretstorage` module could not be initialized. {_err}'
|
||||
|
||||
|
||||
CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'}
|
||||
SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'}
|
||||
|
||||
|
||||
class YDLLogger:
|
||||
def __init__(self, ydl=None):
|
||||
self._ydl = ydl
|
||||
|
||||
def debug(self, message):
|
||||
if self._ydl:
|
||||
self._ydl.write_debug(message)
|
||||
|
||||
def info(self, message):
|
||||
if self._ydl:
|
||||
self._ydl.to_screen(f'[Cookies] {message}')
|
||||
|
||||
def warning(self, message, only_once=False):
|
||||
if self._ydl:
|
||||
self._ydl.report_warning(message, only_once)
|
||||
|
||||
def error(self, message):
|
||||
if self._ydl:
|
||||
self._ydl.report_error(message)
|
||||
|
||||
|
||||
def load_cookies(cookie_file, browser_specification, ydl):
|
||||
cookie_jars = []
|
||||
if browser_specification is not None:
|
||||
browser_name, profile, keyring = _parse_browser_specification(*browser_specification)
|
||||
cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring))
|
||||
|
||||
if cookie_file is not None:
|
||||
cookie_file = expand_path(cookie_file)
|
||||
jar = YoutubeDLCookieJar(cookie_file)
|
||||
if os.access(cookie_file, os.R_OK):
|
||||
jar.load(ignore_discard=True, ignore_expires=True)
|
||||
cookie_jars.append(jar)
|
||||
|
||||
return _merge_cookie_jars(cookie_jars)
|
||||
|
||||
|
||||
def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None):
|
||||
if browser_name == 'firefox':
|
||||
return _extract_firefox_cookies(profile, logger)
|
||||
elif browser_name == 'safari':
|
||||
return _extract_safari_cookies(profile, logger)
|
||||
elif browser_name in CHROMIUM_BASED_BROWSERS:
|
||||
return _extract_chrome_cookies(browser_name, profile, keyring, logger)
|
||||
else:
|
||||
raise ValueError('unknown browser: {}'.format(browser_name))
|
||||
|
||||
|
||||
def _extract_firefox_cookies(profile, logger):
|
||||
logger.info('Extracting cookies from firefox')
|
||||
if not SQLITE_AVAILABLE:
|
||||
logger.warning('Cannot extract cookies from firefox without sqlite3 support. '
|
||||
'Please use a python interpreter compiled with sqlite3 support')
|
||||
return YoutubeDLCookieJar()
|
||||
|
||||
if profile is None:
|
||||
search_root = _firefox_browser_dir()
|
||||
elif _is_path(profile):
|
||||
search_root = profile
|
||||
else:
|
||||
search_root = os.path.join(_firefox_browser_dir(), profile)
|
||||
|
||||
cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite')
|
||||
if cookie_database_path is None:
|
||||
raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root))
|
||||
logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
|
||||
cursor = None
|
||||
try:
|
||||
cursor = _open_database_copy(cookie_database_path, tmpdir)
|
||||
cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
|
||||
jar = YoutubeDLCookieJar()
|
||||
for host, name, value, path, expiry, is_secure in cursor.fetchall():
|
||||
cookie = compat_cookiejar_Cookie(
|
||||
version=0, name=name, value=value, port=None, port_specified=False,
|
||||
domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
|
||||
path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
|
||||
comment=None, comment_url=None, rest={})
|
||||
jar.set_cookie(cookie)
|
||||
logger.info('Extracted {} cookies from firefox'.format(len(jar)))
|
||||
return jar
|
||||
finally:
|
||||
if cursor is not None:
|
||||
cursor.connection.close()
|
||||
|
||||
|
||||
def _firefox_browser_dir():
|
||||
if sys.platform in ('linux', 'linux2'):
|
||||
return os.path.expanduser('~/.mozilla/firefox')
|
||||
elif sys.platform == 'win32':
|
||||
return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox\Profiles')
|
||||
elif sys.platform == 'darwin':
|
||||
return os.path.expanduser('~/Library/Application Support/Firefox')
|
||||
else:
|
||||
raise ValueError('unsupported platform: {}'.format(sys.platform))
|
||||
|
||||
|
||||
def _get_chromium_based_browser_settings(browser_name):
|
||||
# https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md
|
||||
if sys.platform in ('linux', 'linux2'):
|
||||
config = _config_home()
|
||||
browser_dir = {
|
||||
'brave': os.path.join(config, 'BraveSoftware/Brave-Browser'),
|
||||
'chrome': os.path.join(config, 'google-chrome'),
|
||||
'chromium': os.path.join(config, 'chromium'),
|
||||
'edge': os.path.join(config, 'microsoft-edge'),
|
||||
'opera': os.path.join(config, 'opera'),
|
||||
'vivaldi': os.path.join(config, 'vivaldi'),
|
||||
}[browser_name]
|
||||
|
||||
elif sys.platform == 'win32':
|
||||
appdata_local = os.path.expandvars('%LOCALAPPDATA%')
|
||||
appdata_roaming = os.path.expandvars('%APPDATA%')
|
||||
browser_dir = {
|
||||
'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser\User Data'),
|
||||
'chrome': os.path.join(appdata_local, r'Google\Chrome\User Data'),
|
||||
'chromium': os.path.join(appdata_local, r'Chromium\User Data'),
|
||||
'edge': os.path.join(appdata_local, r'Microsoft\Edge\User Data'),
|
||||
'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'),
|
||||
'vivaldi': os.path.join(appdata_local, r'Vivaldi\User Data'),
|
||||
}[browser_name]
|
||||
|
||||
elif sys.platform == 'darwin':
|
||||
appdata = os.path.expanduser('~/Library/Application Support')
|
||||
browser_dir = {
|
||||
'brave': os.path.join(appdata, 'BraveSoftware/Brave-Browser'),
|
||||
'chrome': os.path.join(appdata, 'Google/Chrome'),
|
||||
'chromium': os.path.join(appdata, 'Chromium'),
|
||||
'edge': os.path.join(appdata, 'Microsoft Edge'),
|
||||
'opera': os.path.join(appdata, 'com.operasoftware.Opera'),
|
||||
'vivaldi': os.path.join(appdata, 'Vivaldi'),
|
||||
}[browser_name]
|
||||
|
||||
else:
|
||||
raise ValueError('unsupported platform: {}'.format(sys.platform))
|
||||
|
||||
# Linux keyring names can be determined by snooping on dbus while opening the browser in KDE:
|
||||
# dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
|
||||
keyring_name = {
|
||||
'brave': 'Brave',
|
||||
'chrome': 'Chrome',
|
||||
'chromium': 'Chromium',
|
||||
'edge': 'Microsoft Edge' if sys.platform == 'darwin' else 'Chromium',
|
||||
'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium',
|
||||
'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome',
|
||||
}[browser_name]
|
||||
|
||||
browsers_without_profiles = {'opera'}
|
||||
|
||||
return {
|
||||
'browser_dir': browser_dir,
|
||||
'keyring_name': keyring_name,
|
||||
'supports_profiles': browser_name not in browsers_without_profiles
|
||||
}
|
||||
|
||||
|
||||
def _extract_chrome_cookies(browser_name, profile, keyring, logger):
|
||||
logger.info('Extracting cookies from {}'.format(browser_name))
|
||||
|
||||
if not SQLITE_AVAILABLE:
|
||||
logger.warning(('Cannot extract cookies from {} without sqlite3 support. '
|
||||
'Please use a python interpreter compiled with sqlite3 support').format(browser_name))
|
||||
return YoutubeDLCookieJar()
|
||||
|
||||
config = _get_chromium_based_browser_settings(browser_name)
|
||||
|
||||
if profile is None:
|
||||
search_root = config['browser_dir']
|
||||
elif _is_path(profile):
|
||||
search_root = profile
|
||||
config['browser_dir'] = os.path.dirname(profile) if config['supports_profiles'] else profile
|
||||
else:
|
||||
if config['supports_profiles']:
|
||||
search_root = os.path.join(config['browser_dir'], profile)
|
||||
else:
|
||||
logger.error('{} does not support profiles'.format(browser_name))
|
||||
search_root = config['browser_dir']
|
||||
|
||||
cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies')
|
||||
if cookie_database_path is None:
|
||||
raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root))
|
||||
logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
|
||||
|
||||
decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger, keyring=keyring)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
|
||||
cursor = None
|
||||
try:
|
||||
cursor = _open_database_copy(cookie_database_path, tmpdir)
|
||||
cursor.connection.text_factory = bytes
|
||||
column_names = _get_column_names(cursor, 'cookies')
|
||||
secure_column = 'is_secure' if 'is_secure' in column_names else 'secure'
|
||||
cursor.execute('SELECT host_key, name, value, encrypted_value, path, '
|
||||
'expires_utc, {} FROM cookies'.format(secure_column))
|
||||
jar = YoutubeDLCookieJar()
|
||||
failed_cookies = 0
|
||||
unencrypted_cookies = 0
|
||||
for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall():
|
||||
host_key = host_key.decode('utf-8')
|
||||
name = name.decode('utf-8')
|
||||
value = value.decode('utf-8')
|
||||
path = path.decode('utf-8')
|
||||
|
||||
if not value and encrypted_value:
|
||||
value = decryptor.decrypt(encrypted_value)
|
||||
if value is None:
|
||||
failed_cookies += 1
|
||||
continue
|
||||
else:
|
||||
unencrypted_cookies += 1
|
||||
|
||||
cookie = compat_cookiejar_Cookie(
|
||||
version=0, name=name, value=value, port=None, port_specified=False,
|
||||
domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
|
||||
path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
|
||||
comment=None, comment_url=None, rest={})
|
||||
jar.set_cookie(cookie)
|
||||
if failed_cookies > 0:
|
||||
failed_message = ' ({} could not be decrypted)'.format(failed_cookies)
|
||||
else:
|
||||
failed_message = ''
|
||||
logger.info('Extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message))
|
||||
counts = decryptor.cookie_counts.copy()
|
||||
counts['unencrypted'] = unencrypted_cookies
|
||||
logger.debug('cookie version breakdown: {}'.format(counts))
|
||||
return jar
|
||||
finally:
|
||||
if cursor is not None:
|
||||
cursor.connection.close()
|
||||
|
||||
|
||||
class ChromeCookieDecryptor:
|
||||
"""
|
||||
Overview:
|
||||
|
||||
Linux:
|
||||
- cookies are either v10 or v11
|
||||
- v10: AES-CBC encrypted with a fixed key
|
||||
- v11: AES-CBC encrypted with an OS protected key (keyring)
|
||||
- v11 keys can be stored in various places depending on the activate desktop environment [2]
|
||||
|
||||
Mac:
|
||||
- cookies are either v10 or not v10
|
||||
- v10: AES-CBC encrypted with an OS protected key (keyring) and more key derivation iterations than linux
|
||||
- not v10: 'old data' stored as plaintext
|
||||
|
||||
Windows:
|
||||
- cookies are either v10 or not v10
|
||||
- v10: AES-GCM encrypted with a key which is encrypted with DPAPI
|
||||
- not v10: encrypted with DPAPI
|
||||
|
||||
Sources:
|
||||
- [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/
|
||||
- [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc
|
||||
- KeyStorageLinux::CreateService
|
||||
"""
|
||||
|
||||
def decrypt(self, encrypted_value):
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def cookie_counts(self):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None):
|
||||
if sys.platform in ('linux', 'linux2'):
|
||||
return LinuxChromeCookieDecryptor(browser_keyring_name, logger, keyring=keyring)
|
||||
elif sys.platform == 'darwin':
|
||||
return MacChromeCookieDecryptor(browser_keyring_name, logger)
|
||||
elif sys.platform == 'win32':
|
||||
return WindowsChromeCookieDecryptor(browser_root, logger)
|
||||
else:
|
||||
raise NotImplementedError('Chrome cookie decryption is not supported '
|
||||
'on this platform: {}'.format(sys.platform))
|
||||
|
||||
|
||||
class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
def __init__(self, browser_keyring_name, logger, *, keyring=None):
|
||||
self._logger = logger
|
||||
self._v10_key = self.derive_key(b'peanuts')
|
||||
password = _get_linux_keyring_password(browser_keyring_name, keyring, logger)
|
||||
self._v11_key = None if password is None else self.derive_key(password)
|
||||
self._cookie_counts = {'v10': 0, 'v11': 0, 'other': 0}
|
||||
|
||||
@staticmethod
|
||||
def derive_key(password):
|
||||
# values from
|
||||
# https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc
|
||||
return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16)
|
||||
|
||||
@property
|
||||
def cookie_counts(self):
|
||||
return self._cookie_counts
|
||||
|
||||
def decrypt(self, encrypted_value):
|
||||
version = encrypted_value[:3]
|
||||
ciphertext = encrypted_value[3:]
|
||||
|
||||
if version == b'v10':
|
||||
self._cookie_counts['v10'] += 1
|
||||
return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
|
||||
|
||||
elif version == b'v11':
|
||||
self._cookie_counts['v11'] += 1
|
||||
if self._v11_key is None:
|
||||
self._logger.warning('cannot decrypt v11 cookies: no key found', only_once=True)
|
||||
return None
|
||||
return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger)
|
||||
|
||||
else:
|
||||
self._cookie_counts['other'] += 1
|
||||
return None
|
||||
|
||||
|
||||
class MacChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
def __init__(self, browser_keyring_name, logger):
|
||||
self._logger = logger
|
||||
password = _get_mac_keyring_password(browser_keyring_name, logger)
|
||||
self._v10_key = None if password is None else self.derive_key(password)
|
||||
self._cookie_counts = {'v10': 0, 'other': 0}
|
||||
|
||||
@staticmethod
|
||||
def derive_key(password):
|
||||
# values from
|
||||
# https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
|
||||
return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16)
|
||||
|
||||
@property
|
||||
def cookie_counts(self):
|
||||
return self._cookie_counts
|
||||
|
||||
def decrypt(self, encrypted_value):
|
||||
version = encrypted_value[:3]
|
||||
ciphertext = encrypted_value[3:]
|
||||
|
||||
if version == b'v10':
|
||||
self._cookie_counts['v10'] += 1
|
||||
if self._v10_key is None:
|
||||
self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
|
||||
return None
|
||||
|
||||
return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger)
|
||||
|
||||
else:
|
||||
self._cookie_counts['other'] += 1
|
||||
# other prefixes are considered 'old data' which were stored as plaintext
|
||||
# https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm
|
||||
return encrypted_value
|
||||
|
||||
|
||||
class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||
def __init__(self, browser_root, logger):
|
||||
self._logger = logger
|
||||
self._v10_key = _get_windows_v10_key(browser_root, logger)
|
||||
self._cookie_counts = {'v10': 0, 'other': 0}
|
||||
|
||||
@property
|
||||
def cookie_counts(self):
|
||||
return self._cookie_counts
|
||||
|
||||
def decrypt(self, encrypted_value):
|
||||
version = encrypted_value[:3]
|
||||
ciphertext = encrypted_value[3:]
|
||||
|
||||
if version == b'v10':
|
||||
self._cookie_counts['v10'] += 1
|
||||
if self._v10_key is None:
|
||||
self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
|
||||
return None
|
||||
|
||||
# https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
|
||||
# kNonceLength
|
||||
nonce_length = 96 // 8
|
||||
# boringssl
|
||||
# EVP_AEAD_AES_GCM_TAG_LEN
|
||||
authentication_tag_length = 16
|
||||
|
||||
raw_ciphertext = ciphertext
|
||||
nonce = raw_ciphertext[:nonce_length]
|
||||
ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length]
|
||||
authentication_tag = raw_ciphertext[-authentication_tag_length:]
|
||||
|
||||
return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger)
|
||||
|
||||
else:
|
||||
self._cookie_counts['other'] += 1
|
||||
# any other prefix means the data is DPAPI encrypted
|
||||
# https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
|
||||
return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8')
|
||||
|
||||
|
||||
def _extract_safari_cookies(profile, logger):
|
||||
if profile is not None:
|
||||
logger.error('safari does not support profiles')
|
||||
if sys.platform != 'darwin':
|
||||
raise ValueError('unsupported platform: {}'.format(sys.platform))
|
||||
|
||||
cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies')
|
||||
|
||||
if not os.path.isfile(cookies_path):
|
||||
raise FileNotFoundError('could not find safari cookies database')
|
||||
|
||||
with open(cookies_path, 'rb') as f:
|
||||
cookies_data = f.read()
|
||||
|
||||
jar = parse_safari_cookies(cookies_data, logger=logger)
|
||||
logger.info('Extracted {} cookies from safari'.format(len(jar)))
|
||||
return jar
|
||||
|
||||
|
||||
class ParserError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DataParser:
|
||||
def __init__(self, data, logger):
|
||||
self._data = data
|
||||
self.cursor = 0
|
||||
self._logger = logger
|
||||
|
||||
def read_bytes(self, num_bytes):
|
||||
if num_bytes < 0:
|
||||
raise ParserError('invalid read of {} bytes'.format(num_bytes))
|
||||
end = self.cursor + num_bytes
|
||||
if end > len(self._data):
|
||||
raise ParserError('reached end of input')
|
||||
data = self._data[self.cursor:end]
|
||||
self.cursor = end
|
||||
return data
|
||||
|
||||
def expect_bytes(self, expected_value, message):
|
||||
value = self.read_bytes(len(expected_value))
|
||||
if value != expected_value:
|
||||
raise ParserError('unexpected value: {} != {} ({})'.format(value, expected_value, message))
|
||||
|
||||
def read_uint(self, big_endian=False):
|
||||
data_format = '>I' if big_endian else '<I'
|
||||
return struct.unpack(data_format, self.read_bytes(4))[0]
|
||||
|
||||
def read_double(self, big_endian=False):
|
||||
data_format = '>d' if big_endian else '<d'
|
||||
return struct.unpack(data_format, self.read_bytes(8))[0]
|
||||
|
||||
def read_cstring(self):
|
||||
buffer = []
|
||||
while True:
|
||||
c = self.read_bytes(1)
|
||||
if c == b'\x00':
|
||||
return b''.join(buffer).decode('utf-8')
|
||||
else:
|
||||
buffer.append(c)
|
||||
|
||||
def skip(self, num_bytes, description='unknown'):
|
||||
if num_bytes > 0:
|
||||
self._logger.debug('skipping {} bytes ({}): {}'.format(
|
||||
num_bytes, description, self.read_bytes(num_bytes)))
|
||||
elif num_bytes < 0:
|
||||
raise ParserError('invalid skip of {} bytes'.format(num_bytes))
|
||||
|
||||
def skip_to(self, offset, description='unknown'):
|
||||
self.skip(offset - self.cursor, description)
|
||||
|
||||
def skip_to_end(self, description='unknown'):
|
||||
self.skip_to(len(self._data), description)
|
||||
|
||||
|
||||
def _mac_absolute_time_to_posix(timestamp):
|
||||
return int((datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=timestamp)).timestamp())
|
||||
|
||||
|
||||
def _parse_safari_cookies_header(data, logger):
|
||||
p = DataParser(data, logger)
|
||||
p.expect_bytes(b'cook', 'database signature')
|
||||
number_of_pages = p.read_uint(big_endian=True)
|
||||
page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)]
|
||||
return page_sizes, p.cursor
|
||||
|
||||
|
||||
def _parse_safari_cookies_page(data, jar, logger):
|
||||
p = DataParser(data, logger)
|
||||
p.expect_bytes(b'\x00\x00\x01\x00', 'page signature')
|
||||
number_of_cookies = p.read_uint()
|
||||
record_offsets = [p.read_uint() for _ in range(number_of_cookies)]
|
||||
if number_of_cookies == 0:
|
||||
logger.debug('a cookies page of size {} has no cookies'.format(len(data)))
|
||||
return
|
||||
|
||||
p.skip_to(record_offsets[0], 'unknown page header field')
|
||||
|
||||
for record_offset in record_offsets:
|
||||
p.skip_to(record_offset, 'space between records')
|
||||
record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger)
|
||||
p.read_bytes(record_length)
|
||||
p.skip_to_end('space in between pages')
|
||||
|
||||
|
||||
def _parse_safari_cookies_record(data, jar, logger):
|
||||
p = DataParser(data, logger)
|
||||
record_size = p.read_uint()
|
||||
p.skip(4, 'unknown record field 1')
|
||||
flags = p.read_uint()
|
||||
is_secure = bool(flags & 0x0001)
|
||||
p.skip(4, 'unknown record field 2')
|
||||
domain_offset = p.read_uint()
|
||||
name_offset = p.read_uint()
|
||||
path_offset = p.read_uint()
|
||||
value_offset = p.read_uint()
|
||||
p.skip(8, 'unknown record field 3')
|
||||
expiration_date = _mac_absolute_time_to_posix(p.read_double())
|
||||
_creation_date = _mac_absolute_time_to_posix(p.read_double()) # noqa: F841
|
||||
|
||||
try:
|
||||
p.skip_to(domain_offset)
|
||||
domain = p.read_cstring()
|
||||
|
||||
p.skip_to(name_offset)
|
||||
name = p.read_cstring()
|
||||
|
||||
p.skip_to(path_offset)
|
||||
path = p.read_cstring()
|
||||
|
||||
p.skip_to(value_offset)
|
||||
value = p.read_cstring()
|
||||
except UnicodeDecodeError:
|
||||
logger.warning('failed to parse Safari cookie because UTF-8 decoding failed', only_once=True)
|
||||
return record_size
|
||||
|
||||
p.skip_to(record_size, 'space at the end of the record')
|
||||
|
||||
cookie = compat_cookiejar_Cookie(
|
||||
version=0, name=name, value=value, port=None, port_specified=False,
|
||||
domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith('.'),
|
||||
path=path, path_specified=bool(path), secure=is_secure, expires=expiration_date, discard=False,
|
||||
comment=None, comment_url=None, rest={})
|
||||
jar.set_cookie(cookie)
|
||||
return record_size
|
||||
|
||||
|
||||
def parse_safari_cookies(data, jar=None, logger=YDLLogger()):
|
||||
"""
|
||||
References:
|
||||
- https://github.com/libyal/dtformats/blob/main/documentation/Safari%20Cookies.asciidoc
|
||||
- this data appears to be out of date but the important parts of the database structure is the same
|
||||
- there are a few bytes here and there which are skipped during parsing
|
||||
"""
|
||||
if jar is None:
|
||||
jar = YoutubeDLCookieJar()
|
||||
page_sizes, body_start = _parse_safari_cookies_header(data, logger)
|
||||
p = DataParser(data[body_start:], logger)
|
||||
for page_size in page_sizes:
|
||||
_parse_safari_cookies_page(p.read_bytes(page_size), jar, logger)
|
||||
p.skip_to_end('footer')
|
||||
return jar
|
||||
|
||||
|
||||
class _LinuxDesktopEnvironment(Enum):
|
||||
"""
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.h
|
||||
DesktopEnvironment
|
||||
"""
|
||||
OTHER = auto()
|
||||
CINNAMON = auto()
|
||||
GNOME = auto()
|
||||
KDE = auto()
|
||||
PANTHEON = auto()
|
||||
UNITY = auto()
|
||||
XFCE = auto()
|
||||
|
||||
|
||||
class _LinuxKeyring(Enum):
|
||||
"""
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.h
|
||||
SelectedLinuxBackend
|
||||
"""
|
||||
KWALLET = auto()
|
||||
GNOMEKEYRING = auto()
|
||||
BASICTEXT = auto()
|
||||
|
||||
|
||||
SUPPORTED_KEYRINGS = _LinuxKeyring.__members__.keys()
|
||||
|
||||
|
||||
def _get_linux_desktop_environment(env):
|
||||
"""
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/base/nix/xdg_util.cc
|
||||
GetDesktopEnvironment
|
||||
"""
|
||||
xdg_current_desktop = env.get('XDG_CURRENT_DESKTOP', None)
|
||||
desktop_session = env.get('DESKTOP_SESSION', None)
|
||||
if xdg_current_desktop is not None:
|
||||
xdg_current_desktop = xdg_current_desktop.split(':')[0].strip()
|
||||
|
||||
if xdg_current_desktop == 'Unity':
|
||||
if desktop_session is not None and 'gnome-fallback' in desktop_session:
|
||||
return _LinuxDesktopEnvironment.GNOME
|
||||
else:
|
||||
return _LinuxDesktopEnvironment.UNITY
|
||||
elif xdg_current_desktop == 'GNOME':
|
||||
return _LinuxDesktopEnvironment.GNOME
|
||||
elif xdg_current_desktop == 'X-Cinnamon':
|
||||
return _LinuxDesktopEnvironment.CINNAMON
|
||||
elif xdg_current_desktop == 'KDE':
|
||||
return _LinuxDesktopEnvironment.KDE
|
||||
elif xdg_current_desktop == 'Pantheon':
|
||||
return _LinuxDesktopEnvironment.PANTHEON
|
||||
elif xdg_current_desktop == 'XFCE':
|
||||
return _LinuxDesktopEnvironment.XFCE
|
||||
elif desktop_session is not None:
|
||||
if desktop_session in ('mate', 'gnome'):
|
||||
return _LinuxDesktopEnvironment.GNOME
|
||||
elif 'kde' in desktop_session:
|
||||
return _LinuxDesktopEnvironment.KDE
|
||||
elif 'xfce' in desktop_session:
|
||||
return _LinuxDesktopEnvironment.XFCE
|
||||
else:
|
||||
if 'GNOME_DESKTOP_SESSION_ID' in env:
|
||||
return _LinuxDesktopEnvironment.GNOME
|
||||
elif 'KDE_FULL_SESSION' in env:
|
||||
return _LinuxDesktopEnvironment.KDE
|
||||
return _LinuxDesktopEnvironment.OTHER
|
||||
|
||||
|
||||
def _choose_linux_keyring(logger):
|
||||
"""
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_util_linux.cc
|
||||
SelectBackend
|
||||
"""
|
||||
desktop_environment = _get_linux_desktop_environment(os.environ)
|
||||
logger.debug('detected desktop environment: {}'.format(desktop_environment.name))
|
||||
if desktop_environment == _LinuxDesktopEnvironment.KDE:
|
||||
linux_keyring = _LinuxKeyring.KWALLET
|
||||
elif desktop_environment == _LinuxDesktopEnvironment.OTHER:
|
||||
linux_keyring = _LinuxKeyring.BASICTEXT
|
||||
else:
|
||||
linux_keyring = _LinuxKeyring.GNOMEKEYRING
|
||||
return linux_keyring
|
||||
|
||||
|
||||
def _get_kwallet_network_wallet(logger):
|
||||
""" The name of the wallet used to store network passwords.
|
||||
|
||||
https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/kwallet_dbus.cc
|
||||
KWalletDBus::NetworkWallet
|
||||
which does a dbus call to the following function:
|
||||
https://api.kde.org/frameworks/kwallet/html/classKWallet_1_1Wallet.html
|
||||
Wallet::NetworkWallet
|
||||
"""
|
||||
default_wallet = 'kdewallet'
|
||||
try:
|
||||
proc = Popen([
|
||||
'dbus-send', '--session', '--print-reply=literal',
|
||||
'--dest=org.kde.kwalletd5',
|
||||
'/modules/kwalletd5',
|
||||
'org.kde.KWallet.networkWallet'
|
||||
], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if proc.returncode != 0:
|
||||
logger.warning('failed to read NetworkWallet')
|
||||
return default_wallet
|
||||
else:
|
||||
network_wallet = stdout.decode('utf-8').strip()
|
||||
logger.debug('NetworkWallet = "{}"'.format(network_wallet))
|
||||
return network_wallet
|
||||
except BaseException as e:
|
||||
logger.warning('exception while obtaining NetworkWallet: {}'.format(e))
|
||||
return default_wallet
|
||||
|
||||
|
||||
def _get_kwallet_password(browser_keyring_name, logger):
|
||||
logger.debug('using kwallet-query to obtain password from kwallet')
|
||||
|
||||
if shutil.which('kwallet-query') is None:
|
||||
logger.error('kwallet-query command not found. KWallet and kwallet-query '
|
||||
'must be installed to read from KWallet. kwallet-query should be'
|
||||
'included in the kwallet package for your distribution')
|
||||
return b''
|
||||
|
||||
network_wallet = _get_kwallet_network_wallet(logger)
|
||||
|
||||
try:
|
||||
proc = Popen([
|
||||
'kwallet-query',
|
||||
'--read-password', '{} Safe Storage'.format(browser_keyring_name),
|
||||
'--folder', '{} Keys'.format(browser_keyring_name),
|
||||
network_wallet
|
||||
], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if proc.returncode != 0:
|
||||
logger.error('kwallet-query failed with return code {}. Please consult '
|
||||
'the kwallet-query man page for details'.format(proc.returncode))
|
||||
return b''
|
||||
else:
|
||||
if stdout.lower().startswith(b'failed to read'):
|
||||
logger.debug('failed to read password from kwallet. Using empty string instead')
|
||||
# this sometimes occurs in KDE because chrome does not check hasEntry and instead
|
||||
# just tries to read the value (which kwallet returns "") whereas kwallet-query
|
||||
# checks hasEntry. To verify this:
|
||||
# dbus-monitor "interface='org.kde.KWallet'" "type=method_return"
|
||||
# while starting chrome.
|
||||
# this may be a bug as the intended behaviour is to generate a random password and store
|
||||
# it, but that doesn't matter here.
|
||||
return b''
|
||||
else:
|
||||
logger.debug('password found')
|
||||
if stdout[-1:] == b'\n':
|
||||
stdout = stdout[:-1]
|
||||
return stdout
|
||||
except BaseException as e:
|
||||
logger.warning(f'exception running kwallet-query: {type(e).__name__}({e})')
|
||||
return b''
|
||||
|
||||
|
||||
def _get_gnome_keyring_password(browser_keyring_name, logger):
|
||||
if not SECRETSTORAGE_AVAILABLE:
|
||||
logger.error('secretstorage not available {}'.format(SECRETSTORAGE_UNAVAILABLE_REASON))
|
||||
return b''
|
||||
# the Gnome keyring does not seem to organise keys in the same way as KWallet,
|
||||
# using `dbus-monitor` during startup, it can be observed that chromium lists all keys
|
||||
# and presumably searches for its key in the list. It appears that we must do the same.
|
||||
# https://github.com/jaraco/keyring/issues/556
|
||||
with contextlib.closing(secretstorage.dbus_init()) as con:
|
||||
col = secretstorage.get_default_collection(con)
|
||||
for item in col.get_all_items():
|
||||
if item.get_label() == '{} Safe Storage'.format(browser_keyring_name):
|
||||
return item.get_secret()
|
||||
else:
|
||||
logger.error('failed to read from keyring')
|
||||
return b''
|
||||
|
||||
|
||||
def _get_linux_keyring_password(browser_keyring_name, keyring, logger):
|
||||
# note: chrome/chromium can be run with the following flags to determine which keyring backend
|
||||
# it has chosen to use
|
||||
# chromium --enable-logging=stderr --v=1 2>&1 | grep key_storage_
|
||||
# Chromium supports a flag: --password-store=<basic|gnome|kwallet> so the automatic detection
|
||||
# will not be sufficient in all cases.
|
||||
|
||||
keyring = _LinuxKeyring[keyring] if keyring else _choose_linux_keyring(logger)
|
||||
logger.debug(f'Chosen keyring: {keyring.name}')
|
||||
|
||||
if keyring == _LinuxKeyring.KWALLET:
|
||||
return _get_kwallet_password(browser_keyring_name, logger)
|
||||
elif keyring == _LinuxKeyring.GNOMEKEYRING:
|
||||
return _get_gnome_keyring_password(browser_keyring_name, logger)
|
||||
elif keyring == _LinuxKeyring.BASICTEXT:
|
||||
# when basic text is chosen, all cookies are stored as v10 (so no keyring password is required)
|
||||
return None
|
||||
assert False, f'Unknown keyring {keyring}'
|
||||
|
||||
|
||||
def _get_mac_keyring_password(browser_keyring_name, logger):
|
||||
logger.debug('using find-generic-password to obtain password from OSX keychain')
|
||||
try:
|
||||
proc = Popen(
|
||||
['security', 'find-generic-password',
|
||||
'-w', # write password to stdout
|
||||
'-a', browser_keyring_name, # match 'account'
|
||||
'-s', '{} Safe Storage'.format(browser_keyring_name)], # match 'service'
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if stdout[-1:] == b'\n':
|
||||
stdout = stdout[:-1]
|
||||
return stdout
|
||||
except BaseException as e:
|
||||
logger.warning(f'exception running find-generic-password: {type(e).__name__}({e})')
|
||||
return None
|
||||
|
||||
|
||||
def _get_windows_v10_key(browser_root, logger):
|
||||
path = _find_most_recently_used_file(browser_root, 'Local State')
|
||||
if path is None:
|
||||
logger.error('could not find local state file')
|
||||
return None
|
||||
with open(path, 'r', encoding='utf8') as f:
|
||||
data = json.load(f)
|
||||
try:
|
||||
base64_key = data['os_crypt']['encrypted_key']
|
||||
except KeyError:
|
||||
logger.error('no encrypted key in Local State')
|
||||
return None
|
||||
encrypted_key = compat_b64decode(base64_key)
|
||||
prefix = b'DPAPI'
|
||||
if not encrypted_key.startswith(prefix):
|
||||
logger.error('invalid key')
|
||||
return None
|
||||
return _decrypt_windows_dpapi(encrypted_key[len(prefix):], logger)
|
||||
|
||||
|
||||
def pbkdf2_sha1(password, salt, iterations, key_length):
|
||||
return pbkdf2_hmac('sha1', password, salt, iterations, key_length)
|
||||
|
||||
|
||||
def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
|
||||
plaintext = unpad_pkcs7(aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector))
|
||||
try:
|
||||
return plaintext.decode('utf-8')
|
||||
except UnicodeDecodeError:
|
||||
logger.warning('failed to decrypt cookie (AES-CBC) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
|
||||
return None
|
||||
|
||||
|
||||
def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):
|
||||
try:
|
||||
plaintext = aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce)
|
||||
except ValueError:
|
||||
logger.warning('failed to decrypt cookie (AES-GCM) because the MAC check failed. Possibly the key is wrong?', only_once=True)
|
||||
return None
|
||||
|
||||
try:
|
||||
return plaintext.decode('utf-8')
|
||||
except UnicodeDecodeError:
|
||||
logger.warning('failed to decrypt cookie (AES-GCM) because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
|
||||
return None
|
||||
|
||||
|
||||
def _decrypt_windows_dpapi(ciphertext, logger):
|
||||
"""
|
||||
References:
|
||||
- https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata
|
||||
"""
|
||||
from ctypes.wintypes import DWORD
|
||||
|
||||
class DATA_BLOB(ctypes.Structure):
|
||||
_fields_ = [('cbData', DWORD),
|
||||
('pbData', ctypes.POINTER(ctypes.c_char))]
|
||||
|
||||
buffer = ctypes.create_string_buffer(ciphertext)
|
||||
blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer)
|
||||
blob_out = DATA_BLOB()
|
||||
ret = ctypes.windll.crypt32.CryptUnprotectData(
|
||||
ctypes.byref(blob_in), # pDataIn
|
||||
None, # ppszDataDescr: human readable description of pDataIn
|
||||
None, # pOptionalEntropy: salt?
|
||||
None, # pvReserved: must be NULL
|
||||
None, # pPromptStruct: information about prompts to display
|
||||
0, # dwFlags
|
||||
ctypes.byref(blob_out) # pDataOut
|
||||
)
|
||||
if not ret:
|
||||
logger.warning('failed to decrypt with DPAPI', only_once=True)
|
||||
return None
|
||||
|
||||
result = ctypes.string_at(blob_out.pbData, blob_out.cbData)
|
||||
ctypes.windll.kernel32.LocalFree(blob_out.pbData)
|
||||
return result
|
||||
|
||||
|
||||
def _config_home():
|
||||
return os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
|
||||
|
||||
|
||||
def _open_database_copy(database_path, tmpdir):
|
||||
# cannot open sqlite databases if they are already in use (e.g. by the browser)
|
||||
database_copy_path = os.path.join(tmpdir, 'temporary.sqlite')
|
||||
shutil.copy(database_path, database_copy_path)
|
||||
conn = sqlite3.connect(database_copy_path)
|
||||
return conn.cursor()
|
||||
|
||||
|
||||
def _get_column_names(cursor, table_name):
|
||||
table_info = cursor.execute('PRAGMA table_info({})'.format(table_name)).fetchall()
|
||||
return [row[1].decode('utf-8') for row in table_info]
|
||||
|
||||
|
||||
def _find_most_recently_used_file(root, filename):
|
||||
# if there are multiple browser profiles, take the most recently used one
|
||||
paths = []
|
||||
for root, dirs, files in os.walk(root):
|
||||
for file in files:
|
||||
if file == filename:
|
||||
paths.append(os.path.join(root, file))
|
||||
return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
|
||||
|
||||
|
||||
def _merge_cookie_jars(jars):
|
||||
output_jar = YoutubeDLCookieJar()
|
||||
for jar in jars:
|
||||
for cookie in jar:
|
||||
output_jar.set_cookie(cookie)
|
||||
if jar.filename is not None:
|
||||
output_jar.filename = jar.filename
|
||||
return output_jar
|
||||
|
||||
|
||||
def _is_path(value):
|
||||
return os.path.sep in value
|
||||
|
||||
|
||||
def _parse_browser_specification(browser_name, profile=None, keyring=None):
|
||||
if browser_name not in SUPPORTED_BROWSERS:
|
||||
raise ValueError(f'unsupported browser: "{browser_name}"')
|
||||
if keyring not in (None, *SUPPORTED_KEYRINGS):
|
||||
raise ValueError(f'unsupported keyring: "{keyring}"')
|
||||
if profile is not None and _is_path(profile):
|
||||
profile = os.path.expanduser(profile)
|
||||
return browser_name, profile, keyring
|
||||
135
plugins/youtube_download/yt_dlp/downloader/__init__.py
Normal file
135
plugins/youtube_download/yt_dlp/downloader/__init__.py
Normal file
@@ -0,0 +1,135 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from ..compat import compat_str
|
||||
from ..utils import (
|
||||
determine_protocol,
|
||||
NO_DEFAULT
|
||||
)
|
||||
|
||||
|
||||
def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=None, to_stdout=False):
|
||||
info_dict['protocol'] = determine_protocol(info_dict)
|
||||
info_copy = info_dict.copy()
|
||||
info_copy['to_stdout'] = to_stdout
|
||||
|
||||
protocols = (protocol or info_copy['protocol']).split('+')
|
||||
downloaders = [_get_suitable_downloader(info_copy, proto, params, default) for proto in protocols]
|
||||
|
||||
if set(downloaders) == {FFmpegFD} and FFmpegFD.can_merge_formats(info_copy, params):
|
||||
return FFmpegFD
|
||||
elif (set(downloaders) == {DashSegmentsFD}
|
||||
and not (to_stdout and len(protocols) > 1)
|
||||
and set(protocols) == {'http_dash_segments_generator'}):
|
||||
return DashSegmentsFD
|
||||
elif len(downloaders) == 1:
|
||||
return downloaders[0]
|
||||
return None
|
||||
|
||||
|
||||
# Some of these require get_suitable_downloader
|
||||
from .common import FileDownloader
|
||||
from .dash import DashSegmentsFD
|
||||
from .f4m import F4mFD
|
||||
from .hls import HlsFD
|
||||
from .http import HttpFD
|
||||
from .rtmp import RtmpFD
|
||||
from .rtsp import RtspFD
|
||||
from .ism import IsmFD
|
||||
from .mhtml import MhtmlFD
|
||||
from .niconico import NiconicoDmcFD
|
||||
from .websocket import WebSocketFragmentFD
|
||||
from .youtube_live_chat import YoutubeLiveChatFD
|
||||
from .external import (
|
||||
get_external_downloader,
|
||||
FFmpegFD,
|
||||
)
|
||||
|
||||
PROTOCOL_MAP = {
|
||||
'rtmp': RtmpFD,
|
||||
'rtmpe': RtmpFD,
|
||||
'rtmp_ffmpeg': FFmpegFD,
|
||||
'm3u8_native': HlsFD,
|
||||
'm3u8': FFmpegFD,
|
||||
'mms': RtspFD,
|
||||
'rtsp': RtspFD,
|
||||
'f4m': F4mFD,
|
||||
'http_dash_segments': DashSegmentsFD,
|
||||
'http_dash_segments_generator': DashSegmentsFD,
|
||||
'ism': IsmFD,
|
||||
'mhtml': MhtmlFD,
|
||||
'niconico_dmc': NiconicoDmcFD,
|
||||
'websocket_frag': WebSocketFragmentFD,
|
||||
'youtube_live_chat': YoutubeLiveChatFD,
|
||||
'youtube_live_chat_replay': YoutubeLiveChatFD,
|
||||
}
|
||||
|
||||
|
||||
def shorten_protocol_name(proto, simplify=False):
|
||||
short_protocol_names = {
|
||||
'm3u8_native': 'm3u8_n',
|
||||
'rtmp_ffmpeg': 'rtmp_f',
|
||||
'http_dash_segments': 'dash',
|
||||
'http_dash_segments_generator': 'dash_g',
|
||||
'niconico_dmc': 'dmc',
|
||||
'websocket_frag': 'WSfrag',
|
||||
}
|
||||
if simplify:
|
||||
short_protocol_names.update({
|
||||
'https': 'http',
|
||||
'ftps': 'ftp',
|
||||
'm3u8_native': 'm3u8',
|
||||
'http_dash_segments_generator': 'dash',
|
||||
'rtmp_ffmpeg': 'rtmp',
|
||||
'm3u8_frag_urls': 'm3u8',
|
||||
'dash_frag_urls': 'dash',
|
||||
})
|
||||
return short_protocol_names.get(proto, proto)
|
||||
|
||||
|
||||
def _get_suitable_downloader(info_dict, protocol, params, default):
|
||||
"""Get the downloader class that can handle the info dict."""
|
||||
if default is NO_DEFAULT:
|
||||
default = HttpFD
|
||||
|
||||
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
|
||||
# return FFmpegFD
|
||||
|
||||
info_dict['protocol'] = protocol
|
||||
downloaders = params.get('external_downloader')
|
||||
external_downloader = (
|
||||
downloaders if isinstance(downloaders, compat_str) or downloaders is None
|
||||
else downloaders.get(shorten_protocol_name(protocol, True), downloaders.get('default')))
|
||||
|
||||
if external_downloader is None:
|
||||
if info_dict['to_stdout'] and FFmpegFD.can_merge_formats(info_dict, params):
|
||||
return FFmpegFD
|
||||
elif external_downloader.lower() != 'native':
|
||||
ed = get_external_downloader(external_downloader)
|
||||
if ed.can_download(info_dict, external_downloader):
|
||||
return ed
|
||||
|
||||
if protocol == 'http_dash_segments':
|
||||
if info_dict.get('is_live') and (external_downloader or '').lower() != 'native':
|
||||
return FFmpegFD
|
||||
|
||||
if protocol in ('m3u8', 'm3u8_native'):
|
||||
if info_dict.get('is_live'):
|
||||
return FFmpegFD
|
||||
elif (external_downloader or '').lower() == 'native':
|
||||
return HlsFD
|
||||
elif get_suitable_downloader(
|
||||
info_dict, params, None, protocol='m3u8_frag_urls', to_stdout=info_dict['to_stdout']):
|
||||
return HlsFD
|
||||
elif params.get('hls_prefer_native') is True:
|
||||
return HlsFD
|
||||
elif params.get('hls_prefer_native') is False:
|
||||
return FFmpegFD
|
||||
|
||||
return PROTOCOL_MAP.get(protocol, default)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'FileDownloader',
|
||||
'get_suitable_downloader',
|
||||
'shorten_protocol_name',
|
||||
]
|
||||
472
plugins/youtube_download/yt_dlp/downloader/common.py
Normal file
472
plugins/youtube_download/yt_dlp/downloader/common.py
Normal file
@@ -0,0 +1,472 @@
|
||||
from __future__ import division, unicode_literals
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import random
|
||||
import errno
|
||||
|
||||
from ..utils import (
|
||||
decodeArgument,
|
||||
encodeFilename,
|
||||
error_to_compat_str,
|
||||
format_bytes,
|
||||
sanitize_open,
|
||||
shell_quote,
|
||||
timeconvert,
|
||||
timetuple_from_msec,
|
||||
)
|
||||
from ..minicurses import (
|
||||
MultilineLogger,
|
||||
MultilinePrinter,
|
||||
QuietMultilinePrinter,
|
||||
BreaklineStatusPrinter
|
||||
)
|
||||
|
||||
|
||||
class FileDownloader(object):
|
||||
"""File Downloader class.
|
||||
|
||||
File downloader objects are the ones responsible of downloading the
|
||||
actual video file and writing it to disk.
|
||||
|
||||
File downloaders accept a lot of parameters. In order not to saturate
|
||||
the object constructor with arguments, it receives a dictionary of
|
||||
options instead.
|
||||
|
||||
Available options:
|
||||
|
||||
verbose: Print additional info to stdout.
|
||||
quiet: Do not print messages to stdout.
|
||||
ratelimit: Download speed limit, in bytes/sec.
|
||||
throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
|
||||
retries: Number of times to retry for HTTP error 5xx
|
||||
file_access_retries: Number of times to retry on file access error
|
||||
buffersize: Size of download buffer in bytes.
|
||||
noresizebuffer: Do not automatically resize the download buffer.
|
||||
continuedl: Try to continue downloads if possible.
|
||||
noprogress: Do not print the progress bar.
|
||||
nopart: Do not use temporary .part files.
|
||||
updatetime: Use the Last-modified header to set output file timestamps.
|
||||
test: Download only first bytes to test the downloader.
|
||||
min_filesize: Skip files smaller than this size
|
||||
max_filesize: Skip files larger than this size
|
||||
xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
|
||||
external_downloader_args: A dictionary of downloader keys (in lower case)
|
||||
and a list of additional command-line arguments for the
|
||||
executable. Use 'default' as the name for arguments to be
|
||||
passed to all downloaders. For compatibility with youtube-dl,
|
||||
a single list of args can also be used
|
||||
hls_use_mpegts: Use the mpegts container for HLS videos.
|
||||
http_chunk_size: Size of a chunk for chunk-based HTTP downloading. May be
|
||||
useful for bypassing bandwidth throttling imposed by
|
||||
a webserver (experimental)
|
||||
progress_template: See YoutubeDL.py
|
||||
|
||||
Subclasses of this one must re-define the real_download method.
|
||||
"""
|
||||
|
||||
_TEST_FILE_SIZE = 10241
|
||||
params = None
|
||||
|
||||
def __init__(self, ydl, params):
|
||||
"""Create a FileDownloader object with the given options."""
|
||||
self.ydl = ydl
|
||||
self._progress_hooks = []
|
||||
self.params = params
|
||||
self._prepare_multiline_status()
|
||||
self.add_progress_hook(self.report_progress)
|
||||
|
||||
@staticmethod
|
||||
def format_seconds(seconds):
|
||||
time = timetuple_from_msec(seconds * 1000)
|
||||
if time.hours > 99:
|
||||
return '--:--:--'
|
||||
if not time.hours:
|
||||
return '%02d:%02d' % time[1:-1]
|
||||
return '%02d:%02d:%02d' % time[:-1]
|
||||
|
||||
@staticmethod
|
||||
def calc_percent(byte_counter, data_len):
|
||||
if data_len is None:
|
||||
return None
|
||||
return float(byte_counter) / float(data_len) * 100.0
|
||||
|
||||
@staticmethod
|
||||
def format_percent(percent):
|
||||
if percent is None:
|
||||
return '---.-%'
|
||||
elif percent == 100:
|
||||
return '100%'
|
||||
return '%6s' % ('%3.1f%%' % percent)
|
||||
|
||||
@staticmethod
|
||||
def calc_eta(start, now, total, current):
|
||||
if total is None:
|
||||
return None
|
||||
if now is None:
|
||||
now = time.time()
|
||||
dif = now - start
|
||||
if current == 0 or dif < 0.001: # One millisecond
|
||||
return None
|
||||
rate = float(current) / dif
|
||||
return int((float(total) - float(current)) / rate)
|
||||
|
||||
@staticmethod
|
||||
def format_eta(eta):
|
||||
if eta is None:
|
||||
return '--:--'
|
||||
return FileDownloader.format_seconds(eta)
|
||||
|
||||
@staticmethod
|
||||
def calc_speed(start, now, bytes):
|
||||
dif = now - start
|
||||
if bytes == 0 or dif < 0.001: # One millisecond
|
||||
return None
|
||||
return float(bytes) / dif
|
||||
|
||||
@staticmethod
|
||||
def format_speed(speed):
|
||||
if speed is None:
|
||||
return '%10s' % '---b/s'
|
||||
return '%10s' % ('%s/s' % format_bytes(speed))
|
||||
|
||||
@staticmethod
|
||||
def format_retries(retries):
|
||||
return 'inf' if retries == float('inf') else '%.0f' % retries
|
||||
|
||||
@staticmethod
|
||||
def best_block_size(elapsed_time, bytes):
|
||||
new_min = max(bytes / 2.0, 1.0)
|
||||
new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
|
||||
if elapsed_time < 0.001:
|
||||
return int(new_max)
|
||||
rate = bytes / elapsed_time
|
||||
if rate > new_max:
|
||||
return int(new_max)
|
||||
if rate < new_min:
|
||||
return int(new_min)
|
||||
return int(rate)
|
||||
|
||||
@staticmethod
|
||||
def parse_bytes(bytestr):
|
||||
"""Parse a string indicating a byte quantity into an integer."""
|
||||
matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
|
||||
if matchobj is None:
|
||||
return None
|
||||
number = float(matchobj.group(1))
|
||||
multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
|
||||
return int(round(number * multiplier))
|
||||
|
||||
def to_screen(self, *args, **kargs):
|
||||
self.ydl.to_stdout(*args, quiet=self.params.get('quiet'), **kargs)
|
||||
|
||||
def to_stderr(self, message):
|
||||
self.ydl.to_stderr(message)
|
||||
|
||||
def to_console_title(self, message):
|
||||
self.ydl.to_console_title(message)
|
||||
|
||||
def trouble(self, *args, **kargs):
|
||||
self.ydl.trouble(*args, **kargs)
|
||||
|
||||
def report_warning(self, *args, **kargs):
|
||||
self.ydl.report_warning(*args, **kargs)
|
||||
|
||||
def report_error(self, *args, **kargs):
|
||||
self.ydl.report_error(*args, **kargs)
|
||||
|
||||
def write_debug(self, *args, **kargs):
|
||||
self.ydl.write_debug(*args, **kargs)
|
||||
|
||||
def slow_down(self, start_time, now, byte_counter):
|
||||
"""Sleep if the download speed is over the rate limit."""
|
||||
rate_limit = self.params.get('ratelimit')
|
||||
if rate_limit is None or byte_counter == 0:
|
||||
return
|
||||
if now is None:
|
||||
now = time.time()
|
||||
elapsed = now - start_time
|
||||
if elapsed <= 0.0:
|
||||
return
|
||||
speed = float(byte_counter) / elapsed
|
||||
if speed > rate_limit:
|
||||
sleep_time = float(byte_counter) / rate_limit - elapsed
|
||||
if sleep_time > 0:
|
||||
time.sleep(sleep_time)
|
||||
|
||||
def temp_name(self, filename):
|
||||
"""Returns a temporary filename for the given filename."""
|
||||
if self.params.get('nopart', False) or filename == '-' or \
|
||||
(os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
|
||||
return filename
|
||||
return filename + '.part'
|
||||
|
||||
def undo_temp_name(self, filename):
|
||||
if filename.endswith('.part'):
|
||||
return filename[:-len('.part')]
|
||||
return filename
|
||||
|
||||
def ytdl_filename(self, filename):
|
||||
return filename + '.ytdl'
|
||||
|
||||
def sanitize_open(self, filename, open_mode):
|
||||
file_access_retries = self.params.get('file_access_retries', 10)
|
||||
retry = 0
|
||||
while True:
|
||||
try:
|
||||
return sanitize_open(filename, open_mode)
|
||||
except (IOError, OSError) as err:
|
||||
retry = retry + 1
|
||||
if retry > file_access_retries or err.errno not in (errno.EACCES,):
|
||||
raise
|
||||
self.to_screen(
|
||||
'[download] Got file access error. Retrying (attempt %d of %s) ...'
|
||||
% (retry, self.format_retries(file_access_retries)))
|
||||
time.sleep(0.01)
|
||||
|
||||
def try_rename(self, old_filename, new_filename):
|
||||
if old_filename == new_filename:
|
||||
return
|
||||
try:
|
||||
os.replace(old_filename, new_filename)
|
||||
except (IOError, OSError) as err:
|
||||
self.report_error(f'unable to rename file: {err}')
|
||||
|
||||
def try_utime(self, filename, last_modified_hdr):
|
||||
"""Try to set the last-modified time of the given file."""
|
||||
if last_modified_hdr is None:
|
||||
return
|
||||
if not os.path.isfile(encodeFilename(filename)):
|
||||
return
|
||||
timestr = last_modified_hdr
|
||||
if timestr is None:
|
||||
return
|
||||
filetime = timeconvert(timestr)
|
||||
if filetime is None:
|
||||
return filetime
|
||||
# Ignore obviously invalid dates
|
||||
if filetime == 0:
|
||||
return
|
||||
try:
|
||||
os.utime(filename, (time.time(), filetime))
|
||||
except Exception:
|
||||
pass
|
||||
return filetime
|
||||
|
||||
def report_destination(self, filename):
|
||||
"""Report destination filename."""
|
||||
self.to_screen('[download] Destination: ' + filename)
|
||||
|
||||
def _prepare_multiline_status(self, lines=1):
|
||||
if self.params.get('noprogress'):
|
||||
self._multiline = QuietMultilinePrinter()
|
||||
elif self.ydl.params.get('logger'):
|
||||
self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
|
||||
elif self.params.get('progress_with_newline'):
|
||||
self._multiline = BreaklineStatusPrinter(self.ydl._screen_file, lines)
|
||||
else:
|
||||
self._multiline = MultilinePrinter(self.ydl._screen_file, lines, not self.params.get('quiet'))
|
||||
self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self.params.get('no_color')
|
||||
|
||||
def _finish_multiline_status(self):
|
||||
self._multiline.end()
|
||||
|
||||
_progress_styles = {
|
||||
'downloaded_bytes': 'light blue',
|
||||
'percent': 'light blue',
|
||||
'eta': 'yellow',
|
||||
'speed': 'green',
|
||||
'elapsed': 'bold white',
|
||||
'total_bytes': '',
|
||||
'total_bytes_estimate': '',
|
||||
}
|
||||
|
||||
def _report_progress_status(self, s, default_template):
|
||||
for name, style in self._progress_styles.items():
|
||||
name = f'_{name}_str'
|
||||
if name not in s:
|
||||
continue
|
||||
s[name] = self._format_progress(s[name], style)
|
||||
s['_default_template'] = default_template % s
|
||||
|
||||
progress_dict = s.copy()
|
||||
progress_dict.pop('info_dict')
|
||||
progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
|
||||
|
||||
progress_template = self.params.get('progress_template', {})
|
||||
self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
|
||||
progress_template.get('download') or '[download] %(progress._default_template)s',
|
||||
progress_dict), s.get('progress_idx') or 0)
|
||||
self.to_console_title(self.ydl.evaluate_outtmpl(
|
||||
progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
|
||||
progress_dict))
|
||||
|
||||
def _format_progress(self, *args, **kwargs):
|
||||
return self.ydl._format_text(
|
||||
self._multiline.stream, self._multiline.allow_colors, *args, **kwargs)
|
||||
|
||||
def report_progress(self, s):
|
||||
if s['status'] == 'finished':
|
||||
if self.params.get('noprogress'):
|
||||
self.to_screen('[download] Download completed')
|
||||
msg_template = '100%%'
|
||||
if s.get('total_bytes') is not None:
|
||||
s['_total_bytes_str'] = format_bytes(s['total_bytes'])
|
||||
msg_template += ' of %(_total_bytes_str)s'
|
||||
if s.get('elapsed') is not None:
|
||||
s['_elapsed_str'] = self.format_seconds(s['elapsed'])
|
||||
msg_template += ' in %(_elapsed_str)s'
|
||||
s['_percent_str'] = self.format_percent(100)
|
||||
self._report_progress_status(s, msg_template)
|
||||
return
|
||||
|
||||
if s['status'] != 'downloading':
|
||||
return
|
||||
|
||||
if s.get('eta') is not None:
|
||||
s['_eta_str'] = self.format_eta(s['eta'])
|
||||
else:
|
||||
s['_eta_str'] = 'Unknown'
|
||||
|
||||
if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
|
||||
s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
|
||||
elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
|
||||
s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
|
||||
else:
|
||||
if s.get('downloaded_bytes') == 0:
|
||||
s['_percent_str'] = self.format_percent(0)
|
||||
else:
|
||||
s['_percent_str'] = 'Unknown %'
|
||||
|
||||
if s.get('speed') is not None:
|
||||
s['_speed_str'] = self.format_speed(s['speed'])
|
||||
else:
|
||||
s['_speed_str'] = 'Unknown speed'
|
||||
|
||||
if s.get('total_bytes') is not None:
|
||||
s['_total_bytes_str'] = format_bytes(s['total_bytes'])
|
||||
msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
|
||||
elif s.get('total_bytes_estimate') is not None:
|
||||
s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
|
||||
msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
|
||||
else:
|
||||
if s.get('downloaded_bytes') is not None:
|
||||
s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
|
||||
if s.get('elapsed'):
|
||||
s['_elapsed_str'] = self.format_seconds(s['elapsed'])
|
||||
msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
|
||||
else:
|
||||
msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
|
||||
else:
|
||||
msg_template = '%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s'
|
||||
if s.get('fragment_index') and s.get('fragment_count'):
|
||||
msg_template += ' (frag %(fragment_index)s/%(fragment_count)s)'
|
||||
elif s.get('fragment_index'):
|
||||
msg_template += ' (frag %(fragment_index)s)'
|
||||
self._report_progress_status(s, msg_template)
|
||||
|
||||
def report_resuming_byte(self, resume_len):
|
||||
"""Report attempt to resume at given byte."""
|
||||
self.to_screen('[download] Resuming download at byte %s' % resume_len)
|
||||
|
||||
def report_retry(self, err, count, retries):
|
||||
"""Report retry in case of HTTP error 5xx"""
|
||||
self.to_screen(
|
||||
'[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
|
||||
% (error_to_compat_str(err), count, self.format_retries(retries)))
|
||||
|
||||
def report_file_already_downloaded(self, *args, **kwargs):
|
||||
"""Report file has already been fully downloaded."""
|
||||
return self.ydl.report_file_already_downloaded(*args, **kwargs)
|
||||
|
||||
def report_unable_to_resume(self):
|
||||
"""Report it was impossible to resume download."""
|
||||
self.to_screen('[download] Unable to resume')
|
||||
|
||||
@staticmethod
|
||||
def supports_manifest(manifest):
|
||||
""" Whether the downloader can download the fragments from the manifest.
|
||||
Redefine in subclasses if needed. """
|
||||
pass
|
||||
|
||||
def download(self, filename, info_dict, subtitle=False):
|
||||
"""Download to a filename using the info from info_dict
|
||||
Return True on success and False otherwise
|
||||
"""
|
||||
|
||||
nooverwrites_and_exists = (
|
||||
not self.params.get('overwrites', True)
|
||||
and os.path.exists(encodeFilename(filename))
|
||||
)
|
||||
|
||||
if not hasattr(filename, 'write'):
|
||||
continuedl_and_exists = (
|
||||
self.params.get('continuedl', True)
|
||||
and os.path.isfile(encodeFilename(filename))
|
||||
and not self.params.get('nopart', False)
|
||||
)
|
||||
|
||||
# Check file already present
|
||||
if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
|
||||
self.report_file_already_downloaded(filename)
|
||||
self._hook_progress({
|
||||
'filename': filename,
|
||||
'status': 'finished',
|
||||
'total_bytes': os.path.getsize(encodeFilename(filename)),
|
||||
}, info_dict)
|
||||
self._finish_multiline_status()
|
||||
return True, False
|
||||
|
||||
if subtitle is False:
|
||||
min_sleep_interval = self.params.get('sleep_interval')
|
||||
if min_sleep_interval:
|
||||
max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
|
||||
sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
|
||||
self.to_screen(
|
||||
'[download] Sleeping %s seconds ...' % (
|
||||
int(sleep_interval) if sleep_interval.is_integer()
|
||||
else '%.2f' % sleep_interval))
|
||||
time.sleep(sleep_interval)
|
||||
else:
|
||||
sleep_interval_sub = 0
|
||||
if type(self.params.get('sleep_interval_subtitles')) is int:
|
||||
sleep_interval_sub = self.params.get('sleep_interval_subtitles')
|
||||
if sleep_interval_sub > 0:
|
||||
self.to_screen(
|
||||
'[download] Sleeping %s seconds ...' % (
|
||||
sleep_interval_sub))
|
||||
time.sleep(sleep_interval_sub)
|
||||
ret = self.real_download(filename, info_dict)
|
||||
self._finish_multiline_status()
|
||||
return ret, True
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
"""Real download process. Redefine in subclasses."""
|
||||
raise NotImplementedError('This method must be implemented by subclasses')
|
||||
|
||||
def _hook_progress(self, status, info_dict):
|
||||
if not self._progress_hooks:
|
||||
return
|
||||
status['info_dict'] = info_dict
|
||||
# youtube-dl passes the same status object to all the hooks.
|
||||
# Some third party scripts seems to be relying on this.
|
||||
# So keep this behavior if possible
|
||||
for ph in self._progress_hooks:
|
||||
ph(status)
|
||||
|
||||
def add_progress_hook(self, ph):
|
||||
# See YoutubeDl.py (search for progress_hooks) for a description of
|
||||
# this interface
|
||||
self._progress_hooks.append(ph)
|
||||
|
||||
def _debug_cmd(self, args, exe=None):
|
||||
if not self.params.get('verbose', False):
|
||||
return
|
||||
|
||||
str_args = [decodeArgument(a) for a in args]
|
||||
|
||||
if exe is None:
|
||||
exe = os.path.basename(str_args[0])
|
||||
|
||||
self.write_debug('%s command line: %s' % (exe, shell_quote(str_args)))
|
||||
80
plugins/youtube_download/yt_dlp/downloader/dash.py
Normal file
80
plugins/youtube_download/yt_dlp/downloader/dash.py
Normal file
@@ -0,0 +1,80 @@
|
||||
from __future__ import unicode_literals
|
||||
import time
|
||||
|
||||
from ..downloader import get_suitable_downloader
|
||||
from .fragment import FragmentFD
|
||||
|
||||
from ..utils import urljoin
|
||||
|
||||
|
||||
class DashSegmentsFD(FragmentFD):
|
||||
"""
|
||||
Download segments in a DASH manifest. External downloaders can take over
|
||||
the fragment downloads by supporting the 'dash_frag_urls' protocol
|
||||
"""
|
||||
|
||||
FD_NAME = 'dashsegments'
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
if info_dict.get('is_live') and set(info_dict['protocol'].split('+')) != {'http_dash_segments_generator'}:
|
||||
self.report_error('Live DASH videos are not supported')
|
||||
|
||||
real_start = time.time()
|
||||
real_downloader = get_suitable_downloader(
|
||||
info_dict, self.params, None, protocol='dash_frag_urls', to_stdout=(filename == '-'))
|
||||
|
||||
requested_formats = [{**info_dict, **fmt} for fmt in info_dict.get('requested_formats', [])]
|
||||
args = []
|
||||
for fmt in requested_formats or [info_dict]:
|
||||
try:
|
||||
fragment_count = 1 if self.params.get('test') else len(fmt['fragments'])
|
||||
except TypeError:
|
||||
fragment_count = None
|
||||
ctx = {
|
||||
'filename': fmt.get('filepath') or filename,
|
||||
'live': 'is_from_start' if fmt.get('is_from_start') else fmt.get('is_live'),
|
||||
'total_frags': fragment_count,
|
||||
}
|
||||
|
||||
if real_downloader:
|
||||
self._prepare_external_frag_download(ctx)
|
||||
else:
|
||||
self._prepare_and_start_frag_download(ctx, fmt)
|
||||
ctx['start'] = real_start
|
||||
|
||||
fragments_to_download = self._get_fragments(fmt, ctx)
|
||||
|
||||
if real_downloader:
|
||||
self.to_screen(
|
||||
'[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
|
||||
info_dict['fragments'] = list(fragments_to_download)
|
||||
fd = real_downloader(self.ydl, self.params)
|
||||
return fd.real_download(filename, info_dict)
|
||||
|
||||
args.append([ctx, fragments_to_download, fmt])
|
||||
|
||||
return self.download_and_append_fragments_multiple(*args)
|
||||
|
||||
def _resolve_fragments(self, fragments, ctx):
|
||||
fragments = fragments(ctx) if callable(fragments) else fragments
|
||||
return [next(iter(fragments))] if self.params.get('test') else fragments
|
||||
|
||||
def _get_fragments(self, fmt, ctx):
|
||||
fragment_base_url = fmt.get('fragment_base_url')
|
||||
fragments = self._resolve_fragments(fmt['fragments'], ctx)
|
||||
|
||||
frag_index = 0
|
||||
for i, fragment in enumerate(fragments):
|
||||
frag_index += 1
|
||||
if frag_index <= ctx['fragment_index']:
|
||||
continue
|
||||
fragment_url = fragment.get('url')
|
||||
if not fragment_url:
|
||||
assert fragment_base_url
|
||||
fragment_url = urljoin(fragment_base_url, fragment['path'])
|
||||
|
||||
yield {
|
||||
'frag_index': frag_index,
|
||||
'index': i,
|
||||
'url': fragment_url,
|
||||
}
|
||||
526
plugins/youtube_download/yt_dlp/downloader/external.py
Normal file
526
plugins/youtube_download/yt_dlp/downloader/external.py
Normal file
@@ -0,0 +1,526 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import os.path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from .fragment import FragmentFD
|
||||
from ..compat import (
|
||||
compat_setenv,
|
||||
compat_str,
|
||||
)
|
||||
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
|
||||
from ..utils import (
|
||||
cli_option,
|
||||
cli_valueless_option,
|
||||
cli_bool_option,
|
||||
_configuration_args,
|
||||
determine_ext,
|
||||
encodeFilename,
|
||||
encodeArgument,
|
||||
handle_youtubedl_headers,
|
||||
check_executable,
|
||||
Popen,
|
||||
remove_end,
|
||||
)
|
||||
|
||||
|
||||
class ExternalFD(FragmentFD):
|
||||
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps')
|
||||
can_download_to_stdout = False
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
self.report_destination(filename)
|
||||
tmpfilename = self.temp_name(filename)
|
||||
|
||||
try:
|
||||
started = time.time()
|
||||
retval = self._call_downloader(tmpfilename, info_dict)
|
||||
except KeyboardInterrupt:
|
||||
if not info_dict.get('is_live'):
|
||||
raise
|
||||
# Live stream downloading cancellation should be considered as
|
||||
# correct and expected termination thus all postprocessing
|
||||
# should take place
|
||||
retval = 0
|
||||
self.to_screen('[%s] Interrupted by user' % self.get_basename())
|
||||
|
||||
if retval == 0:
|
||||
status = {
|
||||
'filename': filename,
|
||||
'status': 'finished',
|
||||
'elapsed': time.time() - started,
|
||||
}
|
||||
if filename != '-':
|
||||
fsize = os.path.getsize(encodeFilename(tmpfilename))
|
||||
self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
|
||||
self.try_rename(tmpfilename, filename)
|
||||
status.update({
|
||||
'downloaded_bytes': fsize,
|
||||
'total_bytes': fsize,
|
||||
})
|
||||
self._hook_progress(status, info_dict)
|
||||
return True
|
||||
else:
|
||||
self.to_stderr('\n')
|
||||
self.report_error('%s exited with code %d' % (
|
||||
self.get_basename(), retval))
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def get_basename(cls):
|
||||
return cls.__name__[:-2].lower()
|
||||
|
||||
@property
|
||||
def exe(self):
|
||||
return self.get_basename()
|
||||
|
||||
@classmethod
|
||||
def available(cls, path=None):
|
||||
path = check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT])
|
||||
if path:
|
||||
cls.exe = path
|
||||
return path
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports(cls, info_dict):
|
||||
return (
|
||||
(cls.can_download_to_stdout or not info_dict.get('to_stdout'))
|
||||
and info_dict['protocol'] in cls.SUPPORTED_PROTOCOLS)
|
||||
|
||||
@classmethod
|
||||
def can_download(cls, info_dict, path=None):
|
||||
return cls.available(path) and cls.supports(info_dict)
|
||||
|
||||
def _option(self, command_option, param):
|
||||
return cli_option(self.params, command_option, param)
|
||||
|
||||
def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
|
||||
return cli_bool_option(self.params, command_option, param, true_value, false_value, separator)
|
||||
|
||||
def _valueless_option(self, command_option, param, expected_value=True):
|
||||
return cli_valueless_option(self.params, command_option, param, expected_value)
|
||||
|
||||
def _configuration_args(self, keys=None, *args, **kwargs):
|
||||
return _configuration_args(
|
||||
self.get_basename(), self.params.get('external_downloader_args'), self.get_basename(),
|
||||
keys, *args, **kwargs)
|
||||
|
||||
def _call_downloader(self, tmpfilename, info_dict):
|
||||
""" Either overwrite this or implement _make_cmd """
|
||||
cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
|
||||
|
||||
self._debug_cmd(cmd)
|
||||
|
||||
if 'fragments' not in info_dict:
|
||||
p = Popen(cmd, stderr=subprocess.PIPE)
|
||||
_, stderr = p.communicate_or_kill()
|
||||
if p.returncode != 0:
|
||||
self.to_stderr(stderr.decode('utf-8', 'replace'))
|
||||
return p.returncode
|
||||
|
||||
fragment_retries = self.params.get('fragment_retries', 0)
|
||||
skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
|
||||
|
||||
count = 0
|
||||
while count <= fragment_retries:
|
||||
p = Popen(cmd, stderr=subprocess.PIPE)
|
||||
_, stderr = p.communicate_or_kill()
|
||||
if p.returncode == 0:
|
||||
break
|
||||
# TODO: Decide whether to retry based on error code
|
||||
# https://aria2.github.io/manual/en/html/aria2c.html#exit-status
|
||||
self.to_stderr(stderr.decode('utf-8', 'replace'))
|
||||
count += 1
|
||||
if count <= fragment_retries:
|
||||
self.to_screen(
|
||||
'[%s] Got error. Retrying fragments (attempt %d of %s)...'
|
||||
% (self.get_basename(), count, self.format_retries(fragment_retries)))
|
||||
if count > fragment_retries:
|
||||
if not skip_unavailable_fragments:
|
||||
self.report_error('Giving up after %s fragment retries' % fragment_retries)
|
||||
return -1
|
||||
|
||||
decrypt_fragment = self.decrypter(info_dict)
|
||||
dest, _ = self.sanitize_open(tmpfilename, 'wb')
|
||||
for frag_index, fragment in enumerate(info_dict['fragments']):
|
||||
fragment_filename = '%s-Frag%d' % (tmpfilename, frag_index)
|
||||
try:
|
||||
src, _ = self.sanitize_open(fragment_filename, 'rb')
|
||||
except IOError as err:
|
||||
if skip_unavailable_fragments and frag_index > 1:
|
||||
self.report_skip_fragment(frag_index, err)
|
||||
continue
|
||||
self.report_error(f'Unable to open fragment {frag_index}; {err}')
|
||||
return -1
|
||||
dest.write(decrypt_fragment(fragment, src.read()))
|
||||
src.close()
|
||||
if not self.params.get('keep_fragments', False):
|
||||
os.remove(encodeFilename(fragment_filename))
|
||||
dest.close()
|
||||
os.remove(encodeFilename('%s.frag.urls' % tmpfilename))
|
||||
return 0
|
||||
|
||||
|
||||
class CurlFD(ExternalFD):
|
||||
AVAILABLE_OPT = '-V'
|
||||
|
||||
def _make_cmd(self, tmpfilename, info_dict):
|
||||
cmd = [self.exe, '--location', '-o', tmpfilename]
|
||||
if info_dict.get('http_headers') is not None:
|
||||
for key, val in info_dict['http_headers'].items():
|
||||
cmd += ['--header', '%s: %s' % (key, val)]
|
||||
|
||||
cmd += self._bool_option('--continue-at', 'continuedl', '-', '0')
|
||||
cmd += self._valueless_option('--silent', 'noprogress')
|
||||
cmd += self._valueless_option('--verbose', 'verbose')
|
||||
cmd += self._option('--limit-rate', 'ratelimit')
|
||||
retry = self._option('--retry', 'retries')
|
||||
if len(retry) == 2:
|
||||
if retry[1] in ('inf', 'infinite'):
|
||||
retry[1] = '2147483647'
|
||||
cmd += retry
|
||||
cmd += self._option('--max-filesize', 'max_filesize')
|
||||
cmd += self._option('--interface', 'source_address')
|
||||
cmd += self._option('--proxy', 'proxy')
|
||||
cmd += self._valueless_option('--insecure', 'nocheckcertificate')
|
||||
cmd += self._configuration_args()
|
||||
cmd += ['--', info_dict['url']]
|
||||
return cmd
|
||||
|
||||
def _call_downloader(self, tmpfilename, info_dict):
|
||||
cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
|
||||
|
||||
self._debug_cmd(cmd)
|
||||
|
||||
# curl writes the progress to stderr so don't capture it.
|
||||
p = Popen(cmd)
|
||||
p.communicate_or_kill()
|
||||
return p.returncode
|
||||
|
||||
|
||||
class AxelFD(ExternalFD):
|
||||
AVAILABLE_OPT = '-V'
|
||||
|
||||
def _make_cmd(self, tmpfilename, info_dict):
|
||||
cmd = [self.exe, '-o', tmpfilename]
|
||||
if info_dict.get('http_headers') is not None:
|
||||
for key, val in info_dict['http_headers'].items():
|
||||
cmd += ['-H', '%s: %s' % (key, val)]
|
||||
cmd += self._configuration_args()
|
||||
cmd += ['--', info_dict['url']]
|
||||
return cmd
|
||||
|
||||
|
||||
class WgetFD(ExternalFD):
|
||||
AVAILABLE_OPT = '--version'
|
||||
|
||||
def _make_cmd(self, tmpfilename, info_dict):
|
||||
cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
|
||||
if info_dict.get('http_headers') is not None:
|
||||
for key, val in info_dict['http_headers'].items():
|
||||
cmd += ['--header', '%s: %s' % (key, val)]
|
||||
cmd += self._option('--limit-rate', 'ratelimit')
|
||||
retry = self._option('--tries', 'retries')
|
||||
if len(retry) == 2:
|
||||
if retry[1] in ('inf', 'infinite'):
|
||||
retry[1] = '0'
|
||||
cmd += retry
|
||||
cmd += self._option('--bind-address', 'source_address')
|
||||
cmd += self._option('--proxy', 'proxy')
|
||||
cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
|
||||
cmd += self._configuration_args()
|
||||
cmd += ['--', info_dict['url']]
|
||||
return cmd
|
||||
|
||||
|
||||
class Aria2cFD(ExternalFD):
|
||||
AVAILABLE_OPT = '-v'
|
||||
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'dash_frag_urls', 'm3u8_frag_urls')
|
||||
|
||||
@staticmethod
|
||||
def supports_manifest(manifest):
|
||||
UNSUPPORTED_FEATURES = [
|
||||
r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [1]
|
||||
# 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
|
||||
]
|
||||
check_results = (not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES)
|
||||
return all(check_results)
|
||||
|
||||
def _make_cmd(self, tmpfilename, info_dict):
|
||||
cmd = [self.exe, '-c',
|
||||
'--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
|
||||
'--file-allocation=none', '-x16', '-j16', '-s16']
|
||||
if 'fragments' in info_dict:
|
||||
cmd += ['--allow-overwrite=true', '--allow-piece-length-change=true']
|
||||
else:
|
||||
cmd += ['--min-split-size', '1M']
|
||||
|
||||
if info_dict.get('http_headers') is not None:
|
||||
for key, val in info_dict['http_headers'].items():
|
||||
cmd += ['--header', '%s: %s' % (key, val)]
|
||||
cmd += self._option('--max-overall-download-limit', 'ratelimit')
|
||||
cmd += self._option('--interface', 'source_address')
|
||||
cmd += self._option('--all-proxy', 'proxy')
|
||||
cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
|
||||
cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=')
|
||||
cmd += self._bool_option('--show-console-readout', 'noprogress', 'false', 'true', '=')
|
||||
cmd += self._configuration_args()
|
||||
|
||||
# aria2c strips out spaces from the beginning/end of filenames and paths.
|
||||
# We work around this issue by adding a "./" to the beginning of the
|
||||
# filename and relative path, and adding a "/" at the end of the path.
|
||||
# See: https://github.com/yt-dlp/yt-dlp/issues/276
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/20312
|
||||
# https://github.com/aria2/aria2/issues/1373
|
||||
dn = os.path.dirname(tmpfilename)
|
||||
if dn:
|
||||
if not os.path.isabs(dn):
|
||||
dn = '.%s%s' % (os.path.sep, dn)
|
||||
cmd += ['--dir', dn + os.path.sep]
|
||||
if 'fragments' not in info_dict:
|
||||
cmd += ['--out', '.%s%s' % (os.path.sep, os.path.basename(tmpfilename))]
|
||||
cmd += ['--auto-file-renaming=false']
|
||||
|
||||
if 'fragments' in info_dict:
|
||||
cmd += ['--file-allocation=none', '--uri-selector=inorder']
|
||||
url_list_file = '%s.frag.urls' % tmpfilename
|
||||
url_list = []
|
||||
for frag_index, fragment in enumerate(info_dict['fragments']):
|
||||
fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index)
|
||||
url_list.append('%s\n\tout=%s' % (fragment['url'], fragment_filename))
|
||||
stream, _ = self.sanitize_open(url_list_file, 'wb')
|
||||
stream.write('\n'.join(url_list).encode('utf-8'))
|
||||
stream.close()
|
||||
cmd += ['-i', url_list_file]
|
||||
else:
|
||||
cmd += ['--', info_dict['url']]
|
||||
return cmd
|
||||
|
||||
|
||||
class HttpieFD(ExternalFD):
|
||||
AVAILABLE_OPT = '--version'
|
||||
|
||||
@classmethod
|
||||
def available(cls, path=None):
|
||||
return super().available(path or 'http')
|
||||
|
||||
def _make_cmd(self, tmpfilename, info_dict):
|
||||
cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
|
||||
|
||||
if info_dict.get('http_headers') is not None:
|
||||
for key, val in info_dict['http_headers'].items():
|
||||
cmd += ['%s:%s' % (key, val)]
|
||||
return cmd
|
||||
|
||||
|
||||
class FFmpegFD(ExternalFD):
|
||||
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'm3u8_native', 'rtsp', 'rtmp', 'rtmp_ffmpeg', 'mms', 'http_dash_segments')
|
||||
can_download_to_stdout = True
|
||||
|
||||
@classmethod
|
||||
def available(cls, path=None):
|
||||
# TODO: Fix path for ffmpeg
|
||||
# Fixme: This may be wrong when --ffmpeg-location is used
|
||||
return FFmpegPostProcessor().available
|
||||
|
||||
@classmethod
|
||||
def supports(cls, info_dict):
|
||||
return all(proto in cls.SUPPORTED_PROTOCOLS for proto in info_dict['protocol'].split('+'))
|
||||
|
||||
def on_process_started(self, proc, stdin):
|
||||
""" Override this in subclasses """
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def can_merge_formats(cls, info_dict, params):
|
||||
return (
|
||||
info_dict.get('requested_formats')
|
||||
and info_dict.get('protocol')
|
||||
and not params.get('allow_unplayable_formats')
|
||||
and 'no-direct-merge' not in params.get('compat_opts', [])
|
||||
and cls.can_download(info_dict))
|
||||
|
||||
def _call_downloader(self, tmpfilename, info_dict):
|
||||
urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']]
|
||||
ffpp = FFmpegPostProcessor(downloader=self)
|
||||
if not ffpp.available:
|
||||
self.report_error('m3u8 download detected but ffmpeg could not be found. Please install')
|
||||
return False
|
||||
ffpp.check_version()
|
||||
|
||||
args = [ffpp.executable, '-y']
|
||||
|
||||
for log_level in ('quiet', 'verbose'):
|
||||
if self.params.get(log_level, False):
|
||||
args += ['-loglevel', log_level]
|
||||
break
|
||||
if not self.params.get('verbose'):
|
||||
args += ['-hide_banner']
|
||||
|
||||
args += info_dict.get('_ffmpeg_args', [])
|
||||
|
||||
# This option exists only for compatibility. Extractors should use `_ffmpeg_args` instead
|
||||
seekable = info_dict.get('_seekable')
|
||||
if seekable is not None:
|
||||
# setting -seekable prevents ffmpeg from guessing if the server
|
||||
# supports seeking(by adding the header `Range: bytes=0-`), which
|
||||
# can cause problems in some cases
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/11800#issuecomment-275037127
|
||||
# http://trac.ffmpeg.org/ticket/6125#comment:10
|
||||
args += ['-seekable', '1' if seekable else '0']
|
||||
|
||||
# start_time = info_dict.get('start_time') or 0
|
||||
# if start_time:
|
||||
# args += ['-ss', compat_str(start_time)]
|
||||
# end_time = info_dict.get('end_time')
|
||||
# if end_time:
|
||||
# args += ['-t', compat_str(end_time - start_time)]
|
||||
|
||||
if info_dict.get('http_headers') is not None and re.match(r'^https?://', urls[0]):
|
||||
# Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
|
||||
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
|
||||
headers = handle_youtubedl_headers(info_dict['http_headers'])
|
||||
args += [
|
||||
'-headers',
|
||||
''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
|
||||
|
||||
env = None
|
||||
proxy = self.params.get('proxy')
|
||||
if proxy:
|
||||
if not re.match(r'^[\da-zA-Z]+://', proxy):
|
||||
proxy = 'http://%s' % proxy
|
||||
|
||||
if proxy.startswith('socks'):
|
||||
self.report_warning(
|
||||
'%s does not support SOCKS proxies. Downloading is likely to fail. '
|
||||
'Consider adding --hls-prefer-native to your command.' % self.get_basename())
|
||||
|
||||
# Since December 2015 ffmpeg supports -http_proxy option (see
|
||||
# http://git.videolan.org/?p=ffmpeg.git;a=commit;h=b4eb1f29ebddd60c41a2eb39f5af701e38e0d3fd)
|
||||
# We could switch to the following code if we are able to detect version properly
|
||||
# args += ['-http_proxy', proxy]
|
||||
env = os.environ.copy()
|
||||
compat_setenv('HTTP_PROXY', proxy, env=env)
|
||||
compat_setenv('http_proxy', proxy, env=env)
|
||||
|
||||
protocol = info_dict.get('protocol')
|
||||
|
||||
if protocol == 'rtmp':
|
||||
player_url = info_dict.get('player_url')
|
||||
page_url = info_dict.get('page_url')
|
||||
app = info_dict.get('app')
|
||||
play_path = info_dict.get('play_path')
|
||||
tc_url = info_dict.get('tc_url')
|
||||
flash_version = info_dict.get('flash_version')
|
||||
live = info_dict.get('rtmp_live', False)
|
||||
conn = info_dict.get('rtmp_conn')
|
||||
if player_url is not None:
|
||||
args += ['-rtmp_swfverify', player_url]
|
||||
if page_url is not None:
|
||||
args += ['-rtmp_pageurl', page_url]
|
||||
if app is not None:
|
||||
args += ['-rtmp_app', app]
|
||||
if play_path is not None:
|
||||
args += ['-rtmp_playpath', play_path]
|
||||
if tc_url is not None:
|
||||
args += ['-rtmp_tcurl', tc_url]
|
||||
if flash_version is not None:
|
||||
args += ['-rtmp_flashver', flash_version]
|
||||
if live:
|
||||
args += ['-rtmp_live', 'live']
|
||||
if isinstance(conn, list):
|
||||
for entry in conn:
|
||||
args += ['-rtmp_conn', entry]
|
||||
elif isinstance(conn, compat_str):
|
||||
args += ['-rtmp_conn', conn]
|
||||
|
||||
for i, url in enumerate(urls):
|
||||
args += self._configuration_args((f'_i{i + 1}', '_i')) + ['-i', url]
|
||||
|
||||
args += ['-c', 'copy']
|
||||
if info_dict.get('requested_formats') or protocol == 'http_dash_segments':
|
||||
for (i, fmt) in enumerate(info_dict.get('requested_formats') or [info_dict]):
|
||||
stream_number = fmt.get('manifest_stream_number', 0)
|
||||
args.extend(['-map', f'{i}:{stream_number}'])
|
||||
|
||||
if self.params.get('test', False):
|
||||
args += ['-fs', compat_str(self._TEST_FILE_SIZE)]
|
||||
|
||||
ext = info_dict['ext']
|
||||
if protocol in ('m3u8', 'm3u8_native'):
|
||||
use_mpegts = (tmpfilename == '-') or self.params.get('hls_use_mpegts')
|
||||
if use_mpegts is None:
|
||||
use_mpegts = info_dict.get('is_live')
|
||||
if use_mpegts:
|
||||
args += ['-f', 'mpegts']
|
||||
else:
|
||||
args += ['-f', 'mp4']
|
||||
if (ffpp.basename == 'ffmpeg' and ffpp._features.get('needs_adtstoasc')) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
|
||||
args += ['-bsf:a', 'aac_adtstoasc']
|
||||
elif protocol == 'rtmp':
|
||||
args += ['-f', 'flv']
|
||||
elif ext == 'mp4' and tmpfilename == '-':
|
||||
args += ['-f', 'mpegts']
|
||||
elif ext == 'unknown_video':
|
||||
ext = determine_ext(remove_end(tmpfilename, '.part'))
|
||||
if ext == 'unknown_video':
|
||||
self.report_warning(
|
||||
'The video format is unknown and cannot be downloaded by ffmpeg. '
|
||||
'Explicitly set the extension in the filename to attempt download in that format')
|
||||
else:
|
||||
self.report_warning(f'The video format is unknown. Trying to download as {ext} according to the filename')
|
||||
args += ['-f', EXT_TO_OUT_FORMATS.get(ext, ext)]
|
||||
else:
|
||||
args += ['-f', EXT_TO_OUT_FORMATS.get(ext, ext)]
|
||||
|
||||
args += self._configuration_args(('_o1', '_o', ''))
|
||||
|
||||
args = [encodeArgument(opt) for opt in args]
|
||||
args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
|
||||
self._debug_cmd(args)
|
||||
|
||||
proc = Popen(args, stdin=subprocess.PIPE, env=env)
|
||||
if url in ('-', 'pipe:'):
|
||||
self.on_process_started(proc, proc.stdin)
|
||||
try:
|
||||
retval = proc.wait()
|
||||
except BaseException as e:
|
||||
# subprocces.run would send the SIGKILL signal to ffmpeg and the
|
||||
# mp4 file couldn't be played, but if we ask ffmpeg to quit it
|
||||
# produces a file that is playable (this is mostly useful for live
|
||||
# streams). Note that Windows is not affected and produces playable
|
||||
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
|
||||
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32' and url not in ('-', 'pipe:'):
|
||||
proc.communicate_or_kill(b'q')
|
||||
else:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
raise
|
||||
return retval
|
||||
|
||||
|
||||
class AVconvFD(FFmpegFD):
|
||||
pass
|
||||
|
||||
|
||||
_BY_NAME = dict(
|
||||
(klass.get_basename(), klass)
|
||||
for name, klass in globals().items()
|
||||
if name.endswith('FD') and name not in ('ExternalFD', 'FragmentFD')
|
||||
)
|
||||
|
||||
|
||||
def list_external_downloaders():
|
||||
return sorted(_BY_NAME.keys())
|
||||
|
||||
|
||||
def get_external_downloader(external_downloader):
|
||||
""" Given the name of the executable, see whether we support the given
|
||||
downloader . """
|
||||
# Drop .exe extension on Windows
|
||||
bn = os.path.splitext(os.path.basename(external_downloader))[0]
|
||||
return _BY_NAME.get(bn)
|
||||
439
plugins/youtube_download/yt_dlp/downloader/f4m.py
Normal file
439
plugins/youtube_download/yt_dlp/downloader/f4m.py
Normal file
@@ -0,0 +1,439 @@
|
||||
from __future__ import division, unicode_literals
|
||||
|
||||
import io
|
||||
import itertools
|
||||
import time
|
||||
|
||||
from .fragment import FragmentFD
|
||||
from ..compat import (
|
||||
compat_b64decode,
|
||||
compat_etree_fromstring,
|
||||
compat_urlparse,
|
||||
compat_urllib_error,
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_struct_pack,
|
||||
compat_struct_unpack,
|
||||
)
|
||||
from ..utils import (
|
||||
fix_xml_ampersands,
|
||||
xpath_text,
|
||||
)
|
||||
|
||||
|
||||
class DataTruncatedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class FlvReader(io.BytesIO):
|
||||
"""
|
||||
Reader for Flv files
|
||||
The file format is documented in https://www.adobe.com/devnet/f4v.html
|
||||
"""
|
||||
|
||||
def read_bytes(self, n):
|
||||
data = self.read(n)
|
||||
if len(data) < n:
|
||||
raise DataTruncatedError(
|
||||
'FlvReader error: need %d bytes while only %d bytes got' % (
|
||||
n, len(data)))
|
||||
return data
|
||||
|
||||
# Utility functions for reading numbers and strings
|
||||
def read_unsigned_long_long(self):
|
||||
return compat_struct_unpack('!Q', self.read_bytes(8))[0]
|
||||
|
||||
def read_unsigned_int(self):
|
||||
return compat_struct_unpack('!I', self.read_bytes(4))[0]
|
||||
|
||||
def read_unsigned_char(self):
|
||||
return compat_struct_unpack('!B', self.read_bytes(1))[0]
|
||||
|
||||
def read_string(self):
|
||||
res = b''
|
||||
while True:
|
||||
char = self.read_bytes(1)
|
||||
if char == b'\x00':
|
||||
break
|
||||
res += char
|
||||
return res
|
||||
|
||||
def read_box_info(self):
|
||||
"""
|
||||
Read a box and return the info as a tuple: (box_size, box_type, box_data)
|
||||
"""
|
||||
real_size = size = self.read_unsigned_int()
|
||||
box_type = self.read_bytes(4)
|
||||
header_end = 8
|
||||
if size == 1:
|
||||
real_size = self.read_unsigned_long_long()
|
||||
header_end = 16
|
||||
return real_size, box_type, self.read_bytes(real_size - header_end)
|
||||
|
||||
def read_asrt(self):
|
||||
# version
|
||||
self.read_unsigned_char()
|
||||
# flags
|
||||
self.read_bytes(3)
|
||||
quality_entry_count = self.read_unsigned_char()
|
||||
# QualityEntryCount
|
||||
for i in range(quality_entry_count):
|
||||
self.read_string()
|
||||
|
||||
segment_run_count = self.read_unsigned_int()
|
||||
segments = []
|
||||
for i in range(segment_run_count):
|
||||
first_segment = self.read_unsigned_int()
|
||||
fragments_per_segment = self.read_unsigned_int()
|
||||
segments.append((first_segment, fragments_per_segment))
|
||||
|
||||
return {
|
||||
'segment_run': segments,
|
||||
}
|
||||
|
||||
def read_afrt(self):
|
||||
# version
|
||||
self.read_unsigned_char()
|
||||
# flags
|
||||
self.read_bytes(3)
|
||||
# time scale
|
||||
self.read_unsigned_int()
|
||||
|
||||
quality_entry_count = self.read_unsigned_char()
|
||||
# QualitySegmentUrlModifiers
|
||||
for i in range(quality_entry_count):
|
||||
self.read_string()
|
||||
|
||||
fragments_count = self.read_unsigned_int()
|
||||
fragments = []
|
||||
for i in range(fragments_count):
|
||||
first = self.read_unsigned_int()
|
||||
first_ts = self.read_unsigned_long_long()
|
||||
duration = self.read_unsigned_int()
|
||||
if duration == 0:
|
||||
discontinuity_indicator = self.read_unsigned_char()
|
||||
else:
|
||||
discontinuity_indicator = None
|
||||
fragments.append({
|
||||
'first': first,
|
||||
'ts': first_ts,
|
||||
'duration': duration,
|
||||
'discontinuity_indicator': discontinuity_indicator,
|
||||
})
|
||||
|
||||
return {
|
||||
'fragments': fragments,
|
||||
}
|
||||
|
||||
def read_abst(self):
|
||||
# version
|
||||
self.read_unsigned_char()
|
||||
# flags
|
||||
self.read_bytes(3)
|
||||
|
||||
self.read_unsigned_int() # BootstrapinfoVersion
|
||||
# Profile,Live,Update,Reserved
|
||||
flags = self.read_unsigned_char()
|
||||
live = flags & 0x20 != 0
|
||||
# time scale
|
||||
self.read_unsigned_int()
|
||||
# CurrentMediaTime
|
||||
self.read_unsigned_long_long()
|
||||
# SmpteTimeCodeOffset
|
||||
self.read_unsigned_long_long()
|
||||
|
||||
self.read_string() # MovieIdentifier
|
||||
server_count = self.read_unsigned_char()
|
||||
# ServerEntryTable
|
||||
for i in range(server_count):
|
||||
self.read_string()
|
||||
quality_count = self.read_unsigned_char()
|
||||
# QualityEntryTable
|
||||
for i in range(quality_count):
|
||||
self.read_string()
|
||||
# DrmData
|
||||
self.read_string()
|
||||
# MetaData
|
||||
self.read_string()
|
||||
|
||||
segments_count = self.read_unsigned_char()
|
||||
segments = []
|
||||
for i in range(segments_count):
|
||||
box_size, box_type, box_data = self.read_box_info()
|
||||
assert box_type == b'asrt'
|
||||
segment = FlvReader(box_data).read_asrt()
|
||||
segments.append(segment)
|
||||
fragments_run_count = self.read_unsigned_char()
|
||||
fragments = []
|
||||
for i in range(fragments_run_count):
|
||||
box_size, box_type, box_data = self.read_box_info()
|
||||
assert box_type == b'afrt'
|
||||
fragments.append(FlvReader(box_data).read_afrt())
|
||||
|
||||
return {
|
||||
'segments': segments,
|
||||
'fragments': fragments,
|
||||
'live': live,
|
||||
}
|
||||
|
||||
def read_bootstrap_info(self):
|
||||
total_size, box_type, box_data = self.read_box_info()
|
||||
assert box_type == b'abst'
|
||||
return FlvReader(box_data).read_abst()
|
||||
|
||||
|
||||
def read_bootstrap_info(bootstrap_bytes):
|
||||
return FlvReader(bootstrap_bytes).read_bootstrap_info()
|
||||
|
||||
|
||||
def build_fragments_list(boot_info):
|
||||
""" Return a list of (segment, fragment) for each fragment in the video """
|
||||
res = []
|
||||
segment_run_table = boot_info['segments'][0]
|
||||
fragment_run_entry_table = boot_info['fragments'][0]['fragments']
|
||||
first_frag_number = fragment_run_entry_table[0]['first']
|
||||
fragments_counter = itertools.count(first_frag_number)
|
||||
for segment, fragments_count in segment_run_table['segment_run']:
|
||||
# In some live HDS streams (for example Rai), `fragments_count` is
|
||||
# abnormal and causing out-of-memory errors. It's OK to change the
|
||||
# number of fragments for live streams as they are updated periodically
|
||||
if fragments_count == 4294967295 and boot_info['live']:
|
||||
fragments_count = 2
|
||||
for _ in range(fragments_count):
|
||||
res.append((segment, next(fragments_counter)))
|
||||
|
||||
if boot_info['live']:
|
||||
res = res[-2:]
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def write_unsigned_int(stream, val):
|
||||
stream.write(compat_struct_pack('!I', val))
|
||||
|
||||
|
||||
def write_unsigned_int_24(stream, val):
|
||||
stream.write(compat_struct_pack('!I', val)[1:])
|
||||
|
||||
|
||||
def write_flv_header(stream):
|
||||
"""Writes the FLV header to stream"""
|
||||
# FLV header
|
||||
stream.write(b'FLV\x01')
|
||||
stream.write(b'\x05')
|
||||
stream.write(b'\x00\x00\x00\x09')
|
||||
stream.write(b'\x00\x00\x00\x00')
|
||||
|
||||
|
||||
def write_metadata_tag(stream, metadata):
|
||||
"""Writes optional metadata tag to stream"""
|
||||
SCRIPT_TAG = b'\x12'
|
||||
FLV_TAG_HEADER_LEN = 11
|
||||
|
||||
if metadata:
|
||||
stream.write(SCRIPT_TAG)
|
||||
write_unsigned_int_24(stream, len(metadata))
|
||||
stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
|
||||
stream.write(metadata)
|
||||
write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata))
|
||||
|
||||
|
||||
def remove_encrypted_media(media):
|
||||
return list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib
|
||||
and 'drmAdditionalHeaderSetId' not in e.attrib,
|
||||
media))
|
||||
|
||||
|
||||
def _add_ns(prop, ver=1):
|
||||
return '{http://ns.adobe.com/f4m/%d.0}%s' % (ver, prop)
|
||||
|
||||
|
||||
def get_base_url(manifest):
|
||||
base_url = xpath_text(
|
||||
manifest, [_add_ns('baseURL'), _add_ns('baseURL', 2)],
|
||||
'base URL', default=None)
|
||||
if base_url:
|
||||
base_url = base_url.strip()
|
||||
return base_url
|
||||
|
||||
|
||||
class F4mFD(FragmentFD):
|
||||
"""
|
||||
A downloader for f4m manifests or AdobeHDS.
|
||||
"""
|
||||
|
||||
FD_NAME = 'f4m'
|
||||
|
||||
def _get_unencrypted_media(self, doc):
|
||||
media = doc.findall(_add_ns('media'))
|
||||
if not media:
|
||||
self.report_error('No media found')
|
||||
if not self.params.get('allow_unplayable_formats'):
|
||||
for e in (doc.findall(_add_ns('drmAdditionalHeader'))
|
||||
+ doc.findall(_add_ns('drmAdditionalHeaderSet'))):
|
||||
# If id attribute is missing it's valid for all media nodes
|
||||
# without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
|
||||
if 'id' not in e.attrib:
|
||||
self.report_error('Missing ID in f4m DRM')
|
||||
media = remove_encrypted_media(media)
|
||||
if not media:
|
||||
self.report_error('Unsupported DRM')
|
||||
return media
|
||||
|
||||
def _get_bootstrap_from_url(self, bootstrap_url):
|
||||
bootstrap = self.ydl.urlopen(bootstrap_url).read()
|
||||
return read_bootstrap_info(bootstrap)
|
||||
|
||||
def _update_live_fragments(self, bootstrap_url, latest_fragment):
|
||||
fragments_list = []
|
||||
retries = 30
|
||||
while (not fragments_list) and (retries > 0):
|
||||
boot_info = self._get_bootstrap_from_url(bootstrap_url)
|
||||
fragments_list = build_fragments_list(boot_info)
|
||||
fragments_list = [f for f in fragments_list if f[1] > latest_fragment]
|
||||
if not fragments_list:
|
||||
# Retry after a while
|
||||
time.sleep(5.0)
|
||||
retries -= 1
|
||||
|
||||
if not fragments_list:
|
||||
self.report_error('Failed to update fragments')
|
||||
|
||||
return fragments_list
|
||||
|
||||
def _parse_bootstrap_node(self, node, base_url):
|
||||
# Sometimes non empty inline bootstrap info can be specified along
|
||||
# with bootstrap url attribute (e.g. dummy inline bootstrap info
|
||||
# contains whitespace characters in [1]). We will prefer bootstrap
|
||||
# url over inline bootstrap info when present.
|
||||
# 1. http://live-1-1.rutube.ru/stream/1024/HDS/SD/C2NKsS85HQNckgn5HdEmOQ/1454167650/S-s604419906/move/four/dirs/upper/1024-576p.f4m
|
||||
bootstrap_url = node.get('url')
|
||||
if bootstrap_url:
|
||||
bootstrap_url = compat_urlparse.urljoin(
|
||||
base_url, bootstrap_url)
|
||||
boot_info = self._get_bootstrap_from_url(bootstrap_url)
|
||||
else:
|
||||
bootstrap_url = None
|
||||
bootstrap = compat_b64decode(node.text)
|
||||
boot_info = read_bootstrap_info(bootstrap)
|
||||
return boot_info, bootstrap_url
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
man_url = info_dict['url']
|
||||
requested_bitrate = info_dict.get('tbr')
|
||||
self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
|
||||
|
||||
urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
|
||||
man_url = urlh.geturl()
|
||||
# Some manifests may be malformed, e.g. prosiebensat1 generated manifests
|
||||
# (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244
|
||||
# and https://github.com/ytdl-org/youtube-dl/issues/7823)
|
||||
manifest = fix_xml_ampersands(urlh.read().decode('utf-8', 'ignore')).strip()
|
||||
|
||||
doc = compat_etree_fromstring(manifest)
|
||||
formats = [(int(f.attrib.get('bitrate', -1)), f)
|
||||
for f in self._get_unencrypted_media(doc)]
|
||||
if requested_bitrate is None or len(formats) == 1:
|
||||
# get the best format
|
||||
formats = sorted(formats, key=lambda f: f[0])
|
||||
rate, media = formats[-1]
|
||||
else:
|
||||
rate, media = list(filter(
|
||||
lambda f: int(f[0]) == requested_bitrate, formats))[0]
|
||||
|
||||
# Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec.
|
||||
man_base_url = get_base_url(doc) or man_url
|
||||
|
||||
base_url = compat_urlparse.urljoin(man_base_url, media.attrib['url'])
|
||||
bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
|
||||
boot_info, bootstrap_url = self._parse_bootstrap_node(
|
||||
bootstrap_node, man_base_url)
|
||||
live = boot_info['live']
|
||||
metadata_node = media.find(_add_ns('metadata'))
|
||||
if metadata_node is not None:
|
||||
metadata = compat_b64decode(metadata_node.text)
|
||||
else:
|
||||
metadata = None
|
||||
|
||||
fragments_list = build_fragments_list(boot_info)
|
||||
test = self.params.get('test', False)
|
||||
if test:
|
||||
# We only download the first fragment
|
||||
fragments_list = fragments_list[:1]
|
||||
total_frags = len(fragments_list)
|
||||
# For some akamai manifests we'll need to add a query to the fragment url
|
||||
akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
|
||||
|
||||
ctx = {
|
||||
'filename': filename,
|
||||
'total_frags': total_frags,
|
||||
'live': bool(live),
|
||||
}
|
||||
|
||||
self._prepare_frag_download(ctx)
|
||||
|
||||
dest_stream = ctx['dest_stream']
|
||||
|
||||
if ctx['complete_frags_downloaded_bytes'] == 0:
|
||||
write_flv_header(dest_stream)
|
||||
if not live:
|
||||
write_metadata_tag(dest_stream, metadata)
|
||||
|
||||
base_url_parsed = compat_urllib_parse_urlparse(base_url)
|
||||
|
||||
self._start_frag_download(ctx, info_dict)
|
||||
|
||||
frag_index = 0
|
||||
while fragments_list:
|
||||
seg_i, frag_i = fragments_list.pop(0)
|
||||
frag_index += 1
|
||||
if frag_index <= ctx['fragment_index']:
|
||||
continue
|
||||
name = 'Seg%d-Frag%d' % (seg_i, frag_i)
|
||||
query = []
|
||||
if base_url_parsed.query:
|
||||
query.append(base_url_parsed.query)
|
||||
if akamai_pv:
|
||||
query.append(akamai_pv.strip(';'))
|
||||
if info_dict.get('extra_param_to_segment_url'):
|
||||
query.append(info_dict['extra_param_to_segment_url'])
|
||||
url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
|
||||
try:
|
||||
success, down_data = self._download_fragment(ctx, url_parsed.geturl(), info_dict)
|
||||
if not success:
|
||||
return False
|
||||
reader = FlvReader(down_data)
|
||||
while True:
|
||||
try:
|
||||
_, box_type, box_data = reader.read_box_info()
|
||||
except DataTruncatedError:
|
||||
if test:
|
||||
# In tests, segments may be truncated, and thus
|
||||
# FlvReader may not be able to parse the whole
|
||||
# chunk. If so, write the segment as is
|
||||
# See https://github.com/ytdl-org/youtube-dl/issues/9214
|
||||
dest_stream.write(down_data)
|
||||
break
|
||||
raise
|
||||
if box_type == b'mdat':
|
||||
self._append_fragment(ctx, box_data)
|
||||
break
|
||||
except (compat_urllib_error.HTTPError, ) as err:
|
||||
if live and (err.code == 404 or err.code == 410):
|
||||
# We didn't keep up with the live window. Continue
|
||||
# with the next available fragment.
|
||||
msg = 'Fragment %d unavailable' % frag_i
|
||||
self.report_warning(msg)
|
||||
fragments_list = []
|
||||
else:
|
||||
raise
|
||||
|
||||
if not fragments_list and not test and live and bootstrap_url:
|
||||
fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
|
||||
total_frags += len(fragments_list)
|
||||
if fragments_list and (fragments_list[0][1] > frag_i + 1):
|
||||
msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
|
||||
self.report_warning(msg)
|
||||
|
||||
self._finish_frag_download(ctx, info_dict)
|
||||
|
||||
return True
|
||||
523
plugins/youtube_download/yt_dlp/downloader/fragment.py
Normal file
523
plugins/youtube_download/yt_dlp/downloader/fragment.py
Normal file
@@ -0,0 +1,523 @@
|
||||
from __future__ import division, unicode_literals
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
|
||||
try:
|
||||
import concurrent.futures
|
||||
can_threaded_download = True
|
||||
except ImportError:
|
||||
can_threaded_download = False
|
||||
|
||||
from .common import FileDownloader
|
||||
from .http import HttpFD
|
||||
from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
|
||||
from ..compat import (
|
||||
compat_os_name,
|
||||
compat_urllib_error,
|
||||
compat_struct_pack,
|
||||
)
|
||||
from ..utils import (
|
||||
DownloadError,
|
||||
error_to_compat_str,
|
||||
encodeFilename,
|
||||
sanitized_Request,
|
||||
)
|
||||
|
||||
|
||||
class HttpQuietDownloader(HttpFD):
|
||||
def to_screen(self, *args, **kargs):
|
||||
pass
|
||||
|
||||
def report_retry(self, err, count, retries):
|
||||
super().to_screen(
|
||||
f'[download] Got server HTTP error: {err}. Retrying (attempt {count} of {self.format_retries(retries)}) ...')
|
||||
|
||||
|
||||
class FragmentFD(FileDownloader):
|
||||
"""
|
||||
A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
|
||||
|
||||
Available options:
|
||||
|
||||
fragment_retries: Number of times to retry a fragment for HTTP error (DASH
|
||||
and hlsnative only)
|
||||
skip_unavailable_fragments:
|
||||
Skip unavailable fragments (DASH and hlsnative only)
|
||||
keep_fragments: Keep downloaded fragments on disk after downloading is
|
||||
finished
|
||||
concurrent_fragment_downloads: The number of threads to use for native hls and dash downloads
|
||||
_no_ytdl_file: Don't use .ytdl file
|
||||
|
||||
For each incomplete fragment download yt-dlp keeps on disk a special
|
||||
bookkeeping file with download state and metadata (in future such files will
|
||||
be used for any incomplete download handled by yt-dlp). This file is
|
||||
used to properly handle resuming, check download file consistency and detect
|
||||
potential errors. The file has a .ytdl extension and represents a standard
|
||||
JSON file of the following format:
|
||||
|
||||
extractor:
|
||||
Dictionary of extractor related data. TBD.
|
||||
|
||||
downloader:
|
||||
Dictionary of downloader related data. May contain following data:
|
||||
current_fragment:
|
||||
Dictionary with current (being downloaded) fragment data:
|
||||
index: 0-based index of current fragment among all fragments
|
||||
fragment_count:
|
||||
Total count of fragments
|
||||
|
||||
This feature is experimental and file format may change in future.
|
||||
"""
|
||||
|
||||
def report_retry_fragment(self, err, frag_index, count, retries):
|
||||
self.to_screen(
|
||||
'\r[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s) ...'
|
||||
% (error_to_compat_str(err), frag_index, count, self.format_retries(retries)))
|
||||
|
||||
def report_skip_fragment(self, frag_index, err=None):
|
||||
err = f' {err};' if err else ''
|
||||
self.to_screen(f'[download]{err} Skipping fragment {frag_index:d} ...')
|
||||
|
||||
def _prepare_url(self, info_dict, url):
|
||||
headers = info_dict.get('http_headers')
|
||||
return sanitized_Request(url, None, headers) if headers else url
|
||||
|
||||
def _prepare_and_start_frag_download(self, ctx, info_dict):
|
||||
self._prepare_frag_download(ctx)
|
||||
self._start_frag_download(ctx, info_dict)
|
||||
|
||||
def __do_ytdl_file(self, ctx):
|
||||
return ctx['live'] is not True and ctx['tmpfilename'] != '-' and not self.params.get('_no_ytdl_file')
|
||||
|
||||
def _read_ytdl_file(self, ctx):
|
||||
assert 'ytdl_corrupt' not in ctx
|
||||
stream, _ = self.sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
|
||||
try:
|
||||
ytdl_data = json.loads(stream.read())
|
||||
ctx['fragment_index'] = ytdl_data['downloader']['current_fragment']['index']
|
||||
if 'extra_state' in ytdl_data['downloader']:
|
||||
ctx['extra_state'] = ytdl_data['downloader']['extra_state']
|
||||
except Exception:
|
||||
ctx['ytdl_corrupt'] = True
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
def _write_ytdl_file(self, ctx):
|
||||
frag_index_stream, _ = self.sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
|
||||
try:
|
||||
downloader = {
|
||||
'current_fragment': {
|
||||
'index': ctx['fragment_index'],
|
||||
},
|
||||
}
|
||||
if 'extra_state' in ctx:
|
||||
downloader['extra_state'] = ctx['extra_state']
|
||||
if ctx.get('fragment_count') is not None:
|
||||
downloader['fragment_count'] = ctx['fragment_count']
|
||||
frag_index_stream.write(json.dumps({'downloader': downloader}))
|
||||
finally:
|
||||
frag_index_stream.close()
|
||||
|
||||
def _download_fragment(self, ctx, frag_url, info_dict, headers=None, request_data=None):
|
||||
fragment_filename = '%s-Frag%d' % (ctx['tmpfilename'], ctx['fragment_index'])
|
||||
fragment_info_dict = {
|
||||
'url': frag_url,
|
||||
'http_headers': headers or info_dict.get('http_headers'),
|
||||
'request_data': request_data,
|
||||
'ctx_id': ctx.get('ctx_id'),
|
||||
}
|
||||
success = ctx['dl'].download(fragment_filename, fragment_info_dict)
|
||||
if not success:
|
||||
return False, None
|
||||
if fragment_info_dict.get('filetime'):
|
||||
ctx['fragment_filetime'] = fragment_info_dict.get('filetime')
|
||||
ctx['fragment_filename_sanitized'] = fragment_filename
|
||||
return True, self._read_fragment(ctx)
|
||||
|
||||
def _read_fragment(self, ctx):
|
||||
down, frag_sanitized = self.sanitize_open(ctx['fragment_filename_sanitized'], 'rb')
|
||||
ctx['fragment_filename_sanitized'] = frag_sanitized
|
||||
frag_content = down.read()
|
||||
down.close()
|
||||
return frag_content
|
||||
|
||||
def _append_fragment(self, ctx, frag_content):
|
||||
try:
|
||||
ctx['dest_stream'].write(frag_content)
|
||||
ctx['dest_stream'].flush()
|
||||
finally:
|
||||
if self.__do_ytdl_file(ctx):
|
||||
self._write_ytdl_file(ctx)
|
||||
if not self.params.get('keep_fragments', False):
|
||||
os.remove(encodeFilename(ctx['fragment_filename_sanitized']))
|
||||
del ctx['fragment_filename_sanitized']
|
||||
|
||||
def _prepare_frag_download(self, ctx):
|
||||
if 'live' not in ctx:
|
||||
ctx['live'] = False
|
||||
if not ctx['live']:
|
||||
total_frags_str = '%d' % ctx['total_frags']
|
||||
ad_frags = ctx.get('ad_frags', 0)
|
||||
if ad_frags:
|
||||
total_frags_str += ' (not including %d ad)' % ad_frags
|
||||
else:
|
||||
total_frags_str = 'unknown (live)'
|
||||
self.to_screen(
|
||||
'[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
|
||||
self.report_destination(ctx['filename'])
|
||||
dl = HttpQuietDownloader(
|
||||
self.ydl,
|
||||
{
|
||||
'continuedl': True,
|
||||
'quiet': self.params.get('quiet'),
|
||||
'noprogress': True,
|
||||
'ratelimit': self.params.get('ratelimit'),
|
||||
'retries': self.params.get('retries', 0),
|
||||
'nopart': self.params.get('nopart', False),
|
||||
'test': self.params.get('test', False),
|
||||
}
|
||||
)
|
||||
tmpfilename = self.temp_name(ctx['filename'])
|
||||
open_mode = 'wb'
|
||||
resume_len = 0
|
||||
|
||||
# Establish possible resume length
|
||||
if os.path.isfile(encodeFilename(tmpfilename)):
|
||||
open_mode = 'ab'
|
||||
resume_len = os.path.getsize(encodeFilename(tmpfilename))
|
||||
|
||||
# Should be initialized before ytdl file check
|
||||
ctx.update({
|
||||
'tmpfilename': tmpfilename,
|
||||
'fragment_index': 0,
|
||||
})
|
||||
|
||||
if self.__do_ytdl_file(ctx):
|
||||
if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
|
||||
self._read_ytdl_file(ctx)
|
||||
is_corrupt = ctx.get('ytdl_corrupt') is True
|
||||
is_inconsistent = ctx['fragment_index'] > 0 and resume_len == 0
|
||||
if is_corrupt or is_inconsistent:
|
||||
message = (
|
||||
'.ytdl file is corrupt' if is_corrupt else
|
||||
'Inconsistent state of incomplete fragment download')
|
||||
self.report_warning(
|
||||
'%s. Restarting from the beginning ...' % message)
|
||||
ctx['fragment_index'] = resume_len = 0
|
||||
if 'ytdl_corrupt' in ctx:
|
||||
del ctx['ytdl_corrupt']
|
||||
self._write_ytdl_file(ctx)
|
||||
else:
|
||||
self._write_ytdl_file(ctx)
|
||||
assert ctx['fragment_index'] == 0
|
||||
|
||||
dest_stream, tmpfilename = self.sanitize_open(tmpfilename, open_mode)
|
||||
|
||||
ctx.update({
|
||||
'dl': dl,
|
||||
'dest_stream': dest_stream,
|
||||
'tmpfilename': tmpfilename,
|
||||
# Total complete fragments downloaded so far in bytes
|
||||
'complete_frags_downloaded_bytes': resume_len,
|
||||
})
|
||||
|
||||
def _start_frag_download(self, ctx, info_dict):
|
||||
resume_len = ctx['complete_frags_downloaded_bytes']
|
||||
total_frags = ctx['total_frags']
|
||||
ctx_id = ctx.get('ctx_id')
|
||||
# This dict stores the download progress, it's updated by the progress
|
||||
# hook
|
||||
state = {
|
||||
'status': 'downloading',
|
||||
'downloaded_bytes': resume_len,
|
||||
'fragment_index': ctx['fragment_index'],
|
||||
'fragment_count': total_frags,
|
||||
'filename': ctx['filename'],
|
||||
'tmpfilename': ctx['tmpfilename'],
|
||||
}
|
||||
|
||||
start = time.time()
|
||||
ctx.update({
|
||||
'started': start,
|
||||
'fragment_started': start,
|
||||
# Amount of fragment's bytes downloaded by the time of the previous
|
||||
# frag progress hook invocation
|
||||
'prev_frag_downloaded_bytes': 0,
|
||||
})
|
||||
|
||||
def frag_progress_hook(s):
|
||||
if s['status'] not in ('downloading', 'finished'):
|
||||
return
|
||||
|
||||
if ctx_id is not None and s.get('ctx_id') != ctx_id:
|
||||
return
|
||||
|
||||
state['max_progress'] = ctx.get('max_progress')
|
||||
state['progress_idx'] = ctx.get('progress_idx')
|
||||
|
||||
time_now = time.time()
|
||||
state['elapsed'] = time_now - start
|
||||
frag_total_bytes = s.get('total_bytes') or 0
|
||||
s['fragment_info_dict'] = s.pop('info_dict', {})
|
||||
if not ctx['live']:
|
||||
estimated_size = (
|
||||
(ctx['complete_frags_downloaded_bytes'] + frag_total_bytes)
|
||||
/ (state['fragment_index'] + 1) * total_frags)
|
||||
state['total_bytes_estimate'] = estimated_size
|
||||
|
||||
if s['status'] == 'finished':
|
||||
state['fragment_index'] += 1
|
||||
ctx['fragment_index'] = state['fragment_index']
|
||||
state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
|
||||
ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
|
||||
ctx['speed'] = state['speed'] = self.calc_speed(
|
||||
ctx['fragment_started'], time_now, frag_total_bytes)
|
||||
ctx['fragment_started'] = time.time()
|
||||
ctx['prev_frag_downloaded_bytes'] = 0
|
||||
else:
|
||||
frag_downloaded_bytes = s['downloaded_bytes']
|
||||
state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
|
||||
if not ctx['live']:
|
||||
state['eta'] = self.calc_eta(
|
||||
start, time_now, estimated_size - resume_len,
|
||||
state['downloaded_bytes'] - resume_len)
|
||||
ctx['speed'] = state['speed'] = self.calc_speed(
|
||||
ctx['fragment_started'], time_now, frag_downloaded_bytes)
|
||||
ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
|
||||
self._hook_progress(state, info_dict)
|
||||
|
||||
ctx['dl'].add_progress_hook(frag_progress_hook)
|
||||
|
||||
return start
|
||||
|
||||
def _finish_frag_download(self, ctx, info_dict):
|
||||
ctx['dest_stream'].close()
|
||||
if self.__do_ytdl_file(ctx):
|
||||
ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
|
||||
if os.path.isfile(ytdl_filename):
|
||||
os.remove(ytdl_filename)
|
||||
elapsed = time.time() - ctx['started']
|
||||
|
||||
if ctx['tmpfilename'] == '-':
|
||||
downloaded_bytes = ctx['complete_frags_downloaded_bytes']
|
||||
else:
|
||||
self.try_rename(ctx['tmpfilename'], ctx['filename'])
|
||||
if self.params.get('updatetime', True):
|
||||
filetime = ctx.get('fragment_filetime')
|
||||
if filetime:
|
||||
try:
|
||||
os.utime(ctx['filename'], (time.time(), filetime))
|
||||
except Exception:
|
||||
pass
|
||||
downloaded_bytes = os.path.getsize(encodeFilename(ctx['filename']))
|
||||
|
||||
self._hook_progress({
|
||||
'downloaded_bytes': downloaded_bytes,
|
||||
'total_bytes': downloaded_bytes,
|
||||
'filename': ctx['filename'],
|
||||
'status': 'finished',
|
||||
'elapsed': elapsed,
|
||||
'ctx_id': ctx.get('ctx_id'),
|
||||
'max_progress': ctx.get('max_progress'),
|
||||
'progress_idx': ctx.get('progress_idx'),
|
||||
}, info_dict)
|
||||
|
||||
def _prepare_external_frag_download(self, ctx):
|
||||
if 'live' not in ctx:
|
||||
ctx['live'] = False
|
||||
if not ctx['live']:
|
||||
total_frags_str = '%d' % ctx['total_frags']
|
||||
ad_frags = ctx.get('ad_frags', 0)
|
||||
if ad_frags:
|
||||
total_frags_str += ' (not including %d ad)' % ad_frags
|
||||
else:
|
||||
total_frags_str = 'unknown (live)'
|
||||
self.to_screen(
|
||||
'[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
|
||||
|
||||
tmpfilename = self.temp_name(ctx['filename'])
|
||||
|
||||
# Should be initialized before ytdl file check
|
||||
ctx.update({
|
||||
'tmpfilename': tmpfilename,
|
||||
'fragment_index': 0,
|
||||
})
|
||||
|
||||
def decrypter(self, info_dict):
|
||||
_key_cache = {}
|
||||
|
||||
def _get_key(url):
|
||||
if url not in _key_cache:
|
||||
_key_cache[url] = self.ydl.urlopen(self._prepare_url(info_dict, url)).read()
|
||||
return _key_cache[url]
|
||||
|
||||
def decrypt_fragment(fragment, frag_content):
|
||||
decrypt_info = fragment.get('decrypt_info')
|
||||
if not decrypt_info or decrypt_info['METHOD'] != 'AES-128':
|
||||
return frag_content
|
||||
iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', fragment['media_sequence'])
|
||||
decrypt_info['KEY'] = decrypt_info.get('KEY') or _get_key(info_dict.get('_decryption_key_url') or decrypt_info['URI'])
|
||||
# Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block
|
||||
# size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded,
|
||||
# not what it decrypts to.
|
||||
if self.params.get('test', False):
|
||||
return frag_content
|
||||
return unpad_pkcs7(aes_cbc_decrypt_bytes(frag_content, decrypt_info['KEY'], iv))
|
||||
|
||||
return decrypt_fragment
|
||||
|
||||
def download_and_append_fragments_multiple(self, *args, pack_func=None, finish_func=None):
|
||||
'''
|
||||
@params (ctx1, fragments1, info_dict1), (ctx2, fragments2, info_dict2), ...
|
||||
all args must be either tuple or list
|
||||
'''
|
||||
interrupt_trigger = [True]
|
||||
max_progress = len(args)
|
||||
if max_progress == 1:
|
||||
return self.download_and_append_fragments(*args[0], pack_func=pack_func, finish_func=finish_func)
|
||||
max_workers = self.params.get('concurrent_fragment_downloads', 1)
|
||||
if max_progress > 1:
|
||||
self._prepare_multiline_status(max_progress)
|
||||
|
||||
def thread_func(idx, ctx, fragments, info_dict, tpe):
|
||||
ctx['max_progress'] = max_progress
|
||||
ctx['progress_idx'] = idx
|
||||
return self.download_and_append_fragments(
|
||||
ctx, fragments, info_dict, pack_func=pack_func, finish_func=finish_func,
|
||||
tpe=tpe, interrupt_trigger=interrupt_trigger)
|
||||
|
||||
class FTPE(concurrent.futures.ThreadPoolExecutor):
|
||||
# has to stop this or it's going to wait on the worker thread itself
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
pass
|
||||
|
||||
spins = []
|
||||
if compat_os_name == 'nt':
|
||||
self.report_warning('Ctrl+C does not work on Windows when used with parallel threads. '
|
||||
'This is a known issue and patches are welcome')
|
||||
for idx, (ctx, fragments, info_dict) in enumerate(args):
|
||||
tpe = FTPE(math.ceil(max_workers / max_progress))
|
||||
job = tpe.submit(thread_func, idx, ctx, fragments, info_dict, tpe)
|
||||
spins.append((tpe, job))
|
||||
|
||||
result = True
|
||||
for tpe, job in spins:
|
||||
try:
|
||||
result = result and job.result()
|
||||
except KeyboardInterrupt:
|
||||
interrupt_trigger[0] = False
|
||||
finally:
|
||||
tpe.shutdown(wait=True)
|
||||
if not interrupt_trigger[0]:
|
||||
raise KeyboardInterrupt()
|
||||
return result
|
||||
|
||||
def download_and_append_fragments(
|
||||
self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None,
|
||||
tpe=None, interrupt_trigger=None):
|
||||
if not interrupt_trigger:
|
||||
interrupt_trigger = (True, )
|
||||
|
||||
fragment_retries = self.params.get('fragment_retries', 0)
|
||||
is_fatal = (
|
||||
((lambda _: False) if info_dict.get('is_live') else (lambda idx: idx == 0))
|
||||
if self.params.get('skip_unavailable_fragments', True) else (lambda _: True))
|
||||
|
||||
if not pack_func:
|
||||
pack_func = lambda frag_content, _: frag_content
|
||||
|
||||
def download_fragment(fragment, ctx):
|
||||
frag_index = ctx['fragment_index'] = fragment['frag_index']
|
||||
ctx['last_error'] = None
|
||||
if not interrupt_trigger[0]:
|
||||
return False, frag_index
|
||||
headers = info_dict.get('http_headers', {}).copy()
|
||||
byte_range = fragment.get('byte_range')
|
||||
if byte_range:
|
||||
headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
|
||||
|
||||
# Never skip the first fragment
|
||||
fatal = is_fatal(fragment.get('index') or (frag_index - 1))
|
||||
count, frag_content = 0, None
|
||||
while count <= fragment_retries:
|
||||
try:
|
||||
success, frag_content = self._download_fragment(ctx, fragment['url'], info_dict, headers)
|
||||
if not success:
|
||||
return False, frag_index
|
||||
break
|
||||
except (compat_urllib_error.HTTPError, http.client.IncompleteRead) as err:
|
||||
# Unavailable (possibly temporary) fragments may be served.
|
||||
# First we try to retry then either skip or abort.
|
||||
# See https://github.com/ytdl-org/youtube-dl/issues/10165,
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/10448).
|
||||
count += 1
|
||||
ctx['last_error'] = err
|
||||
if count <= fragment_retries:
|
||||
self.report_retry_fragment(err, frag_index, count, fragment_retries)
|
||||
except DownloadError:
|
||||
# Don't retry fragment if error occurred during HTTP downloading
|
||||
# itself since it has own retry settings
|
||||
if not fatal:
|
||||
break
|
||||
raise
|
||||
|
||||
if count > fragment_retries:
|
||||
if not fatal:
|
||||
return False, frag_index
|
||||
ctx['dest_stream'].close()
|
||||
self.report_error('Giving up after %s fragment retries' % fragment_retries)
|
||||
return False, frag_index
|
||||
return frag_content, frag_index
|
||||
|
||||
def append_fragment(frag_content, frag_index, ctx):
|
||||
if not frag_content:
|
||||
if not is_fatal(frag_index - 1):
|
||||
self.report_skip_fragment(frag_index, 'fragment not found')
|
||||
return True
|
||||
else:
|
||||
ctx['dest_stream'].close()
|
||||
self.report_error(
|
||||
'fragment %s not found, unable to continue' % frag_index)
|
||||
return False
|
||||
self._append_fragment(ctx, pack_func(frag_content, frag_index))
|
||||
return True
|
||||
|
||||
decrypt_fragment = self.decrypter(info_dict)
|
||||
|
||||
max_workers = math.ceil(
|
||||
self.params.get('concurrent_fragment_downloads', 1) / ctx.get('max_progress', 1))
|
||||
if can_threaded_download and max_workers > 1:
|
||||
|
||||
def _download_fragment(fragment):
|
||||
ctx_copy = ctx.copy()
|
||||
frag_content, frag_index = download_fragment(fragment, ctx_copy)
|
||||
return fragment, frag_content, frag_index, ctx_copy.get('fragment_filename_sanitized')
|
||||
|
||||
self.report_warning('The download speed shown is only of one thread. This is a known issue and patches are welcome')
|
||||
with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool:
|
||||
for fragment, frag_content, frag_index, frag_filename in pool.map(_download_fragment, fragments):
|
||||
if not interrupt_trigger[0]:
|
||||
break
|
||||
ctx['fragment_filename_sanitized'] = frag_filename
|
||||
ctx['fragment_index'] = frag_index
|
||||
result = append_fragment(decrypt_fragment(fragment, frag_content), frag_index, ctx)
|
||||
if not result:
|
||||
return False
|
||||
else:
|
||||
for fragment in fragments:
|
||||
if not interrupt_trigger[0]:
|
||||
break
|
||||
frag_content, frag_index = download_fragment(fragment, ctx)
|
||||
result = append_fragment(decrypt_fragment(fragment, frag_content), frag_index, ctx)
|
||||
if not result:
|
||||
return False
|
||||
|
||||
if finish_func is not None:
|
||||
ctx['dest_stream'].write(finish_func())
|
||||
ctx['dest_stream'].flush()
|
||||
self._finish_frag_download(ctx, info_dict)
|
||||
return True
|
||||
358
plugins/youtube_download/yt_dlp/downloader/hls.py
Normal file
358
plugins/youtube_download/yt_dlp/downloader/hls.py
Normal file
@@ -0,0 +1,358 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
import io
|
||||
import binascii
|
||||
|
||||
from ..downloader import get_suitable_downloader
|
||||
from .fragment import FragmentFD
|
||||
from .external import FFmpegFD
|
||||
|
||||
from ..compat import (
|
||||
compat_pycrypto_AES,
|
||||
compat_urlparse,
|
||||
)
|
||||
from ..utils import (
|
||||
parse_m3u8_attributes,
|
||||
update_url_query,
|
||||
bug_reports_message,
|
||||
)
|
||||
from .. import webvtt
|
||||
|
||||
|
||||
class HlsFD(FragmentFD):
|
||||
"""
|
||||
Download segments in a m3u8 manifest. External downloaders can take over
|
||||
the fragment downloads by supporting the 'm3u8_frag_urls' protocol and
|
||||
re-defining 'supports_manifest' function
|
||||
"""
|
||||
|
||||
FD_NAME = 'hlsnative'
|
||||
|
||||
@staticmethod
|
||||
def can_download(manifest, info_dict, allow_unplayable_formats=False):
|
||||
UNSUPPORTED_FEATURES = [
|
||||
# r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2]
|
||||
|
||||
# Live streams heuristic does not always work (e.g. geo restricted to Germany
|
||||
# http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
|
||||
# r'#EXT-X-MEDIA-SEQUENCE:(?!0$)', # live streams [3]
|
||||
|
||||
# This heuristic also is not correct since segments may not be appended as well.
|
||||
# Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite
|
||||
# no segments will definitely be appended to the end of the playlist.
|
||||
# r'#EXT-X-PLAYLIST-TYPE:EVENT', # media segments may be appended to the end of
|
||||
# # event media playlists [4]
|
||||
# r'#EXT-X-MAP:', # media initialization [5]
|
||||
# 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
|
||||
# 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
|
||||
# 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
|
||||
# 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
|
||||
# 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5
|
||||
]
|
||||
if not allow_unplayable_formats:
|
||||
UNSUPPORTED_FEATURES += [
|
||||
r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)', # encrypted streams [1]
|
||||
]
|
||||
|
||||
def check_results():
|
||||
yield not info_dict.get('is_live')
|
||||
for feature in UNSUPPORTED_FEATURES:
|
||||
yield not re.search(feature, manifest)
|
||||
return all(check_results())
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
man_url = info_dict['url']
|
||||
self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
|
||||
|
||||
urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
|
||||
man_url = urlh.geturl()
|
||||
s = urlh.read().decode('utf-8', 'ignore')
|
||||
|
||||
can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
|
||||
if can_download and not compat_pycrypto_AES and '#EXT-X-KEY:METHOD=AES-128' in s:
|
||||
if FFmpegFD.available():
|
||||
can_download, message = False, 'The stream has AES-128 encryption and pycryptodomex is not available'
|
||||
else:
|
||||
message = ('The stream has AES-128 encryption and neither ffmpeg nor pycryptodomex are available; '
|
||||
'Decryption will be performed natively, but will be extremely slow')
|
||||
if not can_download:
|
||||
has_drm = re.search('|'.join([
|
||||
r'#EXT-X-FAXS-CM:', # Adobe Flash Access
|
||||
r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://', # Apple FairPlay
|
||||
]), s)
|
||||
if has_drm and not self.params.get('allow_unplayable_formats'):
|
||||
self.report_error(
|
||||
'This video is DRM protected; Try selecting another format with --format or '
|
||||
'add --check-formats to automatically fallback to the next best format')
|
||||
return False
|
||||
message = message or 'Unsupported features have been detected'
|
||||
fd = FFmpegFD(self.ydl, self.params)
|
||||
self.report_warning(f'{message}; extraction will be delegated to {fd.get_basename()}')
|
||||
return fd.real_download(filename, info_dict)
|
||||
elif message:
|
||||
self.report_warning(message)
|
||||
|
||||
is_webvtt = info_dict['ext'] == 'vtt'
|
||||
if is_webvtt:
|
||||
real_downloader = None # Packing the fragments is not currently supported for external downloader
|
||||
else:
|
||||
real_downloader = get_suitable_downloader(
|
||||
info_dict, self.params, None, protocol='m3u8_frag_urls', to_stdout=(filename == '-'))
|
||||
if real_downloader and not real_downloader.supports_manifest(s):
|
||||
real_downloader = None
|
||||
if real_downloader:
|
||||
self.to_screen(
|
||||
'[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
|
||||
|
||||
def is_ad_fragment_start(s):
|
||||
return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
|
||||
or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',ad'))
|
||||
|
||||
def is_ad_fragment_end(s):
|
||||
return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s
|
||||
or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',segment'))
|
||||
|
||||
fragments = []
|
||||
|
||||
media_frags = 0
|
||||
ad_frags = 0
|
||||
ad_frag_next = False
|
||||
for line in s.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith('#'):
|
||||
if is_ad_fragment_start(line):
|
||||
ad_frag_next = True
|
||||
elif is_ad_fragment_end(line):
|
||||
ad_frag_next = False
|
||||
continue
|
||||
if ad_frag_next:
|
||||
ad_frags += 1
|
||||
continue
|
||||
media_frags += 1
|
||||
|
||||
ctx = {
|
||||
'filename': filename,
|
||||
'total_frags': media_frags,
|
||||
'ad_frags': ad_frags,
|
||||
}
|
||||
|
||||
if real_downloader:
|
||||
self._prepare_external_frag_download(ctx)
|
||||
else:
|
||||
self._prepare_and_start_frag_download(ctx, info_dict)
|
||||
|
||||
extra_state = ctx.setdefault('extra_state', {})
|
||||
|
||||
format_index = info_dict.get('format_index')
|
||||
extra_query = None
|
||||
extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
|
||||
if extra_param_to_segment_url:
|
||||
extra_query = compat_urlparse.parse_qs(extra_param_to_segment_url)
|
||||
i = 0
|
||||
media_sequence = 0
|
||||
decrypt_info = {'METHOD': 'NONE'}
|
||||
byte_range = {}
|
||||
discontinuity_count = 0
|
||||
frag_index = 0
|
||||
ad_frag_next = False
|
||||
for line in s.splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
if not line.startswith('#'):
|
||||
if format_index and discontinuity_count != format_index:
|
||||
continue
|
||||
if ad_frag_next:
|
||||
continue
|
||||
frag_index += 1
|
||||
if frag_index <= ctx['fragment_index']:
|
||||
continue
|
||||
frag_url = (
|
||||
line
|
||||
if re.match(r'^https?://', line)
|
||||
else compat_urlparse.urljoin(man_url, line))
|
||||
if extra_query:
|
||||
frag_url = update_url_query(frag_url, extra_query)
|
||||
|
||||
fragments.append({
|
||||
'frag_index': frag_index,
|
||||
'url': frag_url,
|
||||
'decrypt_info': decrypt_info,
|
||||
'byte_range': byte_range,
|
||||
'media_sequence': media_sequence,
|
||||
})
|
||||
media_sequence += 1
|
||||
|
||||
elif line.startswith('#EXT-X-MAP'):
|
||||
if format_index and discontinuity_count != format_index:
|
||||
continue
|
||||
if frag_index > 0:
|
||||
self.report_error(
|
||||
'Initialization fragment found after media fragments, unable to download')
|
||||
return False
|
||||
frag_index += 1
|
||||
map_info = parse_m3u8_attributes(line[11:])
|
||||
frag_url = (
|
||||
map_info.get('URI')
|
||||
if re.match(r'^https?://', map_info.get('URI'))
|
||||
else compat_urlparse.urljoin(man_url, map_info.get('URI')))
|
||||
if extra_query:
|
||||
frag_url = update_url_query(frag_url, extra_query)
|
||||
|
||||
fragments.append({
|
||||
'frag_index': frag_index,
|
||||
'url': frag_url,
|
||||
'decrypt_info': decrypt_info,
|
||||
'byte_range': byte_range,
|
||||
'media_sequence': media_sequence
|
||||
})
|
||||
media_sequence += 1
|
||||
|
||||
if map_info.get('BYTERANGE'):
|
||||
splitted_byte_range = map_info.get('BYTERANGE').split('@')
|
||||
sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end']
|
||||
byte_range = {
|
||||
'start': sub_range_start,
|
||||
'end': sub_range_start + int(splitted_byte_range[0]),
|
||||
}
|
||||
|
||||
elif line.startswith('#EXT-X-KEY'):
|
||||
decrypt_url = decrypt_info.get('URI')
|
||||
decrypt_info = parse_m3u8_attributes(line[11:])
|
||||
if decrypt_info['METHOD'] == 'AES-128':
|
||||
if 'IV' in decrypt_info:
|
||||
decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:].zfill(32))
|
||||
if not re.match(r'^https?://', decrypt_info['URI']):
|
||||
decrypt_info['URI'] = compat_urlparse.urljoin(
|
||||
man_url, decrypt_info['URI'])
|
||||
if extra_query:
|
||||
decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
|
||||
if decrypt_url != decrypt_info['URI']:
|
||||
decrypt_info['KEY'] = None
|
||||
|
||||
elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
|
||||
media_sequence = int(line[22:])
|
||||
elif line.startswith('#EXT-X-BYTERANGE'):
|
||||
splitted_byte_range = line[17:].split('@')
|
||||
sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end']
|
||||
byte_range = {
|
||||
'start': sub_range_start,
|
||||
'end': sub_range_start + int(splitted_byte_range[0]),
|
||||
}
|
||||
elif is_ad_fragment_start(line):
|
||||
ad_frag_next = True
|
||||
elif is_ad_fragment_end(line):
|
||||
ad_frag_next = False
|
||||
elif line.startswith('#EXT-X-DISCONTINUITY'):
|
||||
discontinuity_count += 1
|
||||
i += 1
|
||||
|
||||
# We only download the first fragment during the test
|
||||
if self.params.get('test', False):
|
||||
fragments = [fragments[0] if fragments else None]
|
||||
|
||||
if real_downloader:
|
||||
info_dict['fragments'] = fragments
|
||||
fd = real_downloader(self.ydl, self.params)
|
||||
# TODO: Make progress updates work without hooking twice
|
||||
# for ph in self._progress_hooks:
|
||||
# fd.add_progress_hook(ph)
|
||||
return fd.real_download(filename, info_dict)
|
||||
|
||||
if is_webvtt:
|
||||
def pack_fragment(frag_content, frag_index):
|
||||
output = io.StringIO()
|
||||
adjust = 0
|
||||
overflow = False
|
||||
mpegts_last = None
|
||||
for block in webvtt.parse_fragment(frag_content):
|
||||
if isinstance(block, webvtt.CueBlock):
|
||||
extra_state['webvtt_mpegts_last'] = mpegts_last
|
||||
if overflow:
|
||||
extra_state['webvtt_mpegts_adjust'] += 1
|
||||
overflow = False
|
||||
block.start += adjust
|
||||
block.end += adjust
|
||||
|
||||
dedup_window = extra_state.setdefault('webvtt_dedup_window', [])
|
||||
|
||||
ready = []
|
||||
|
||||
i = 0
|
||||
is_new = True
|
||||
while i < len(dedup_window):
|
||||
wcue = dedup_window[i]
|
||||
wblock = webvtt.CueBlock.from_json(wcue)
|
||||
i += 1
|
||||
if wblock.hinges(block):
|
||||
wcue['end'] = block.end
|
||||
is_new = False
|
||||
continue
|
||||
if wblock == block:
|
||||
is_new = False
|
||||
continue
|
||||
if wblock.end > block.start:
|
||||
continue
|
||||
ready.append(wblock)
|
||||
i -= 1
|
||||
del dedup_window[i]
|
||||
|
||||
if is_new:
|
||||
dedup_window.append(block.as_json)
|
||||
for block in ready:
|
||||
block.write_into(output)
|
||||
|
||||
# we only emit cues once they fall out of the duplicate window
|
||||
continue
|
||||
elif isinstance(block, webvtt.Magic):
|
||||
# take care of MPEG PES timestamp overflow
|
||||
if block.mpegts is None:
|
||||
block.mpegts = 0
|
||||
extra_state.setdefault('webvtt_mpegts_adjust', 0)
|
||||
block.mpegts += extra_state['webvtt_mpegts_adjust'] << 33
|
||||
if block.mpegts < extra_state.get('webvtt_mpegts_last', 0):
|
||||
overflow = True
|
||||
block.mpegts += 1 << 33
|
||||
mpegts_last = block.mpegts
|
||||
|
||||
if frag_index == 1:
|
||||
extra_state['webvtt_mpegts'] = block.mpegts or 0
|
||||
extra_state['webvtt_local'] = block.local or 0
|
||||
# XXX: block.local = block.mpegts = None ?
|
||||
else:
|
||||
if block.mpegts is not None and block.local is not None:
|
||||
adjust = (
|
||||
(block.mpegts - extra_state.get('webvtt_mpegts', 0))
|
||||
- (block.local - extra_state.get('webvtt_local', 0))
|
||||
)
|
||||
continue
|
||||
elif isinstance(block, webvtt.HeaderBlock):
|
||||
if frag_index != 1:
|
||||
# XXX: this should probably be silent as well
|
||||
# or verify that all segments contain the same data
|
||||
self.report_warning(bug_reports_message(
|
||||
'Discarding a %s block found in the middle of the stream; '
|
||||
'if the subtitles display incorrectly,'
|
||||
% (type(block).__name__)))
|
||||
continue
|
||||
block.write_into(output)
|
||||
|
||||
return output.getvalue().encode('utf-8')
|
||||
|
||||
def fin_fragments():
|
||||
dedup_window = extra_state.get('webvtt_dedup_window')
|
||||
if not dedup_window:
|
||||
return b''
|
||||
|
||||
output = io.StringIO()
|
||||
for cue in dedup_window:
|
||||
webvtt.CueBlock.from_json(cue).write_into(output)
|
||||
|
||||
return output.getvalue().encode('utf-8')
|
||||
|
||||
self.download_and_append_fragments(
|
||||
ctx, fragments, info_dict, pack_func=pack_fragment, finish_func=fin_fragments)
|
||||
else:
|
||||
return self.download_and_append_fragments(ctx, fragments, info_dict)
|
||||
386
plugins/youtube_download/yt_dlp/downloader/http.py
Normal file
386
plugins/youtube_download/yt_dlp/downloader/http.py
Normal file
@@ -0,0 +1,386 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import errno
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
import random
|
||||
import re
|
||||
|
||||
from .common import FileDownloader
|
||||
from ..compat import (
|
||||
compat_str,
|
||||
compat_urllib_error,
|
||||
)
|
||||
from ..utils import (
|
||||
ContentTooShortError,
|
||||
encodeFilename,
|
||||
int_or_none,
|
||||
sanitized_Request,
|
||||
ThrottledDownload,
|
||||
write_xattr,
|
||||
XAttrMetadataError,
|
||||
XAttrUnavailableError,
|
||||
)
|
||||
|
||||
|
||||
class HttpFD(FileDownloader):
|
||||
def real_download(self, filename, info_dict):
|
||||
url = info_dict['url']
|
||||
request_data = info_dict.get('request_data', None)
|
||||
|
||||
class DownloadContext(dict):
|
||||
__getattr__ = dict.get
|
||||
__setattr__ = dict.__setitem__
|
||||
__delattr__ = dict.__delitem__
|
||||
|
||||
ctx = DownloadContext()
|
||||
ctx.filename = filename
|
||||
ctx.tmpfilename = self.temp_name(filename)
|
||||
ctx.stream = None
|
||||
|
||||
# Do not include the Accept-Encoding header
|
||||
headers = {'Youtubedl-no-compression': 'True'}
|
||||
add_headers = info_dict.get('http_headers')
|
||||
if add_headers:
|
||||
headers.update(add_headers)
|
||||
|
||||
is_test = self.params.get('test', False)
|
||||
chunk_size = self._TEST_FILE_SIZE if is_test else (
|
||||
self.params.get('http_chunk_size')
|
||||
or info_dict.get('downloader_options', {}).get('http_chunk_size')
|
||||
or 0)
|
||||
|
||||
ctx.open_mode = 'wb'
|
||||
ctx.resume_len = 0
|
||||
ctx.data_len = None
|
||||
ctx.block_size = self.params.get('buffersize', 1024)
|
||||
ctx.start_time = time.time()
|
||||
ctx.chunk_size = None
|
||||
throttle_start = None
|
||||
|
||||
if self.params.get('continuedl', True):
|
||||
# Establish possible resume length
|
||||
if os.path.isfile(encodeFilename(ctx.tmpfilename)):
|
||||
ctx.resume_len = os.path.getsize(
|
||||
encodeFilename(ctx.tmpfilename))
|
||||
|
||||
ctx.is_resume = ctx.resume_len > 0
|
||||
|
||||
count = 0
|
||||
retries = self.params.get('retries', 0)
|
||||
|
||||
class SucceedDownload(Exception):
|
||||
pass
|
||||
|
||||
class RetryDownload(Exception):
|
||||
def __init__(self, source_error):
|
||||
self.source_error = source_error
|
||||
|
||||
class NextFragment(Exception):
|
||||
pass
|
||||
|
||||
def set_range(req, start, end):
|
||||
range_header = 'bytes=%d-' % start
|
||||
if end:
|
||||
range_header += compat_str(end)
|
||||
req.add_header('Range', range_header)
|
||||
|
||||
def establish_connection():
|
||||
ctx.chunk_size = (random.randint(int(chunk_size * 0.95), chunk_size)
|
||||
if not is_test and chunk_size else chunk_size)
|
||||
if ctx.resume_len > 0:
|
||||
range_start = ctx.resume_len
|
||||
if ctx.is_resume:
|
||||
self.report_resuming_byte(ctx.resume_len)
|
||||
ctx.open_mode = 'ab'
|
||||
elif ctx.chunk_size > 0:
|
||||
range_start = 0
|
||||
else:
|
||||
range_start = None
|
||||
ctx.is_resume = False
|
||||
range_end = range_start + ctx.chunk_size - 1 if ctx.chunk_size else None
|
||||
if range_end and ctx.data_len is not None and range_end >= ctx.data_len:
|
||||
range_end = ctx.data_len - 1
|
||||
has_range = range_start is not None
|
||||
ctx.has_range = has_range
|
||||
request = sanitized_Request(url, request_data, headers)
|
||||
if has_range:
|
||||
set_range(request, range_start, range_end)
|
||||
# Establish connection
|
||||
try:
|
||||
try:
|
||||
ctx.data = self.ydl.urlopen(request)
|
||||
except (compat_urllib_error.URLError, ) as err:
|
||||
# reason may not be available, e.g. for urllib2.HTTPError on python 2.6
|
||||
reason = getattr(err, 'reason', None)
|
||||
if isinstance(reason, socket.timeout):
|
||||
raise RetryDownload(err)
|
||||
raise err
|
||||
# When trying to resume, Content-Range HTTP header of response has to be checked
|
||||
# to match the value of requested Range HTTP header. This is due to a webservers
|
||||
# that don't support resuming and serve a whole file with no Content-Range
|
||||
# set in response despite of requested Range (see
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
|
||||
if has_range:
|
||||
content_range = ctx.data.headers.get('Content-Range')
|
||||
if content_range:
|
||||
content_range_m = re.search(r'bytes (\d+)-(\d+)?(?:/(\d+))?', content_range)
|
||||
# Content-Range is present and matches requested Range, resume is possible
|
||||
if content_range_m:
|
||||
if range_start == int(content_range_m.group(1)):
|
||||
content_range_end = int_or_none(content_range_m.group(2))
|
||||
content_len = int_or_none(content_range_m.group(3))
|
||||
accept_content_len = (
|
||||
# Non-chunked download
|
||||
not ctx.chunk_size
|
||||
# Chunked download and requested piece or
|
||||
# its part is promised to be served
|
||||
or content_range_end == range_end
|
||||
or content_len < range_end)
|
||||
if accept_content_len:
|
||||
ctx.data_len = content_len
|
||||
return
|
||||
# Content-Range is either not present or invalid. Assuming remote webserver is
|
||||
# trying to send the whole file, resume is not possible, so wiping the local file
|
||||
# and performing entire redownload
|
||||
self.report_unable_to_resume()
|
||||
ctx.resume_len = 0
|
||||
ctx.open_mode = 'wb'
|
||||
ctx.data_len = int_or_none(ctx.data.info().get('Content-length', None))
|
||||
return
|
||||
except (compat_urllib_error.HTTPError, ) as err:
|
||||
if err.code == 416:
|
||||
# Unable to resume (requested range not satisfiable)
|
||||
try:
|
||||
# Open the connection again without the range header
|
||||
ctx.data = self.ydl.urlopen(
|
||||
sanitized_Request(url, request_data, headers))
|
||||
content_length = ctx.data.info()['Content-Length']
|
||||
except (compat_urllib_error.HTTPError, ) as err:
|
||||
if err.code < 500 or err.code >= 600:
|
||||
raise
|
||||
else:
|
||||
# Examine the reported length
|
||||
if (content_length is not None
|
||||
and (ctx.resume_len - 100 < int(content_length) < ctx.resume_len + 100)):
|
||||
# The file had already been fully downloaded.
|
||||
# Explanation to the above condition: in issue #175 it was revealed that
|
||||
# YouTube sometimes adds or removes a few bytes from the end of the file,
|
||||
# changing the file size slightly and causing problems for some users. So
|
||||
# I decided to implement a suggested change and consider the file
|
||||
# completely downloaded if the file size differs less than 100 bytes from
|
||||
# the one in the hard drive.
|
||||
self.report_file_already_downloaded(ctx.filename)
|
||||
self.try_rename(ctx.tmpfilename, ctx.filename)
|
||||
self._hook_progress({
|
||||
'filename': ctx.filename,
|
||||
'status': 'finished',
|
||||
'downloaded_bytes': ctx.resume_len,
|
||||
'total_bytes': ctx.resume_len,
|
||||
}, info_dict)
|
||||
raise SucceedDownload()
|
||||
else:
|
||||
# The length does not match, we start the download over
|
||||
self.report_unable_to_resume()
|
||||
ctx.resume_len = 0
|
||||
ctx.open_mode = 'wb'
|
||||
return
|
||||
elif err.code < 500 or err.code >= 600:
|
||||
# Unexpected HTTP error
|
||||
raise
|
||||
raise RetryDownload(err)
|
||||
except socket.timeout as err:
|
||||
raise RetryDownload(err)
|
||||
except socket.error as err:
|
||||
if err.errno in (errno.ECONNRESET, errno.ETIMEDOUT):
|
||||
# Connection reset is no problem, just retry
|
||||
raise RetryDownload(err)
|
||||
raise
|
||||
|
||||
def download():
|
||||
nonlocal throttle_start
|
||||
data_len = ctx.data.info().get('Content-length', None)
|
||||
|
||||
# Range HTTP header may be ignored/unsupported by a webserver
|
||||
# (e.g. extractor/scivee.py, extractor/bambuser.py).
|
||||
# However, for a test we still would like to download just a piece of a file.
|
||||
# To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
|
||||
# block size when downloading a file.
|
||||
if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
|
||||
data_len = self._TEST_FILE_SIZE
|
||||
|
||||
if data_len is not None:
|
||||
data_len = int(data_len) + ctx.resume_len
|
||||
min_data_len = self.params.get('min_filesize')
|
||||
max_data_len = self.params.get('max_filesize')
|
||||
if min_data_len is not None and data_len < min_data_len:
|
||||
self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
|
||||
return False
|
||||
if max_data_len is not None and data_len > max_data_len:
|
||||
self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
|
||||
return False
|
||||
|
||||
byte_counter = 0 + ctx.resume_len
|
||||
block_size = ctx.block_size
|
||||
start = time.time()
|
||||
|
||||
# measure time over whole while-loop, so slow_down() and best_block_size() work together properly
|
||||
now = None # needed for slow_down() in the first loop run
|
||||
before = start # start measuring
|
||||
|
||||
def retry(e):
|
||||
to_stdout = ctx.tmpfilename == '-'
|
||||
if ctx.stream is not None:
|
||||
if not to_stdout:
|
||||
ctx.stream.close()
|
||||
ctx.stream = None
|
||||
ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
|
||||
raise RetryDownload(e)
|
||||
|
||||
while True:
|
||||
try:
|
||||
# Download and write
|
||||
data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
|
||||
# socket.timeout is a subclass of socket.error but may not have
|
||||
# errno set
|
||||
except socket.timeout as e:
|
||||
retry(e)
|
||||
except socket.error as e:
|
||||
# SSLError on python 2 (inherits socket.error) may have
|
||||
# no errno set but this error message
|
||||
if e.errno in (errno.ECONNRESET, errno.ETIMEDOUT) or getattr(e, 'message', None) == 'The read operation timed out':
|
||||
retry(e)
|
||||
raise
|
||||
|
||||
byte_counter += len(data_block)
|
||||
|
||||
# exit loop when download is finished
|
||||
if len(data_block) == 0:
|
||||
break
|
||||
|
||||
# Open destination file just in time
|
||||
if ctx.stream is None:
|
||||
try:
|
||||
ctx.stream, ctx.tmpfilename = self.sanitize_open(
|
||||
ctx.tmpfilename, ctx.open_mode)
|
||||
assert ctx.stream is not None
|
||||
ctx.filename = self.undo_temp_name(ctx.tmpfilename)
|
||||
self.report_destination(ctx.filename)
|
||||
except (OSError, IOError) as err:
|
||||
self.report_error('unable to open for writing: %s' % str(err))
|
||||
return False
|
||||
|
||||
if self.params.get('xattr_set_filesize', False) and data_len is not None:
|
||||
try:
|
||||
write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode('utf-8'))
|
||||
except (XAttrUnavailableError, XAttrMetadataError) as err:
|
||||
self.report_error('unable to set filesize xattr: %s' % str(err))
|
||||
|
||||
try:
|
||||
ctx.stream.write(data_block)
|
||||
except (IOError, OSError) as err:
|
||||
self.to_stderr('\n')
|
||||
self.report_error('unable to write data: %s' % str(err))
|
||||
return False
|
||||
|
||||
# Apply rate limit
|
||||
self.slow_down(start, now, byte_counter - ctx.resume_len)
|
||||
|
||||
# end measuring of one loop run
|
||||
now = time.time()
|
||||
after = now
|
||||
|
||||
# Adjust block size
|
||||
if not self.params.get('noresizebuffer', False):
|
||||
block_size = self.best_block_size(after - before, len(data_block))
|
||||
|
||||
before = after
|
||||
|
||||
# Progress message
|
||||
speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
|
||||
if ctx.data_len is None:
|
||||
eta = None
|
||||
else:
|
||||
eta = self.calc_eta(start, time.time(), ctx.data_len - ctx.resume_len, byte_counter - ctx.resume_len)
|
||||
|
||||
self._hook_progress({
|
||||
'status': 'downloading',
|
||||
'downloaded_bytes': byte_counter,
|
||||
'total_bytes': ctx.data_len,
|
||||
'tmpfilename': ctx.tmpfilename,
|
||||
'filename': ctx.filename,
|
||||
'eta': eta,
|
||||
'speed': speed,
|
||||
'elapsed': now - ctx.start_time,
|
||||
'ctx_id': info_dict.get('ctx_id'),
|
||||
}, info_dict)
|
||||
|
||||
if data_len is not None and byte_counter == data_len:
|
||||
break
|
||||
|
||||
if speed and speed < (self.params.get('throttledratelimit') or 0):
|
||||
# The speed must stay below the limit for 3 seconds
|
||||
# This prevents raising error when the speed temporarily goes down
|
||||
if throttle_start is None:
|
||||
throttle_start = now
|
||||
elif now - throttle_start > 3:
|
||||
if ctx.stream is not None and ctx.tmpfilename != '-':
|
||||
ctx.stream.close()
|
||||
raise ThrottledDownload()
|
||||
elif speed:
|
||||
throttle_start = None
|
||||
|
||||
if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len:
|
||||
ctx.resume_len = byte_counter
|
||||
# ctx.block_size = block_size
|
||||
raise NextFragment()
|
||||
|
||||
if ctx.stream is None:
|
||||
self.to_stderr('\n')
|
||||
self.report_error('Did not get any data blocks')
|
||||
return False
|
||||
if ctx.tmpfilename != '-':
|
||||
ctx.stream.close()
|
||||
|
||||
if data_len is not None and byte_counter != data_len:
|
||||
err = ContentTooShortError(byte_counter, int(data_len))
|
||||
if count <= retries:
|
||||
retry(err)
|
||||
raise err
|
||||
|
||||
self.try_rename(ctx.tmpfilename, ctx.filename)
|
||||
|
||||
# Update file modification time
|
||||
if self.params.get('updatetime', True):
|
||||
info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))
|
||||
|
||||
self._hook_progress({
|
||||
'downloaded_bytes': byte_counter,
|
||||
'total_bytes': byte_counter,
|
||||
'filename': ctx.filename,
|
||||
'status': 'finished',
|
||||
'elapsed': time.time() - ctx.start_time,
|
||||
'ctx_id': info_dict.get('ctx_id'),
|
||||
}, info_dict)
|
||||
|
||||
return True
|
||||
|
||||
while count <= retries:
|
||||
try:
|
||||
establish_connection()
|
||||
return download()
|
||||
except RetryDownload as e:
|
||||
count += 1
|
||||
if count <= retries:
|
||||
self.report_retry(e.source_error, count, retries)
|
||||
else:
|
||||
self.to_screen(f'[download] Got server HTTP error: {e.source_error}')
|
||||
continue
|
||||
except NextFragment:
|
||||
continue
|
||||
except SucceedDownload:
|
||||
return True
|
||||
|
||||
self.report_error('giving up after %s retries' % retries)
|
||||
return False
|
||||
289
plugins/youtube_download/yt_dlp/downloader/ism.py
Normal file
289
plugins/youtube_download/yt_dlp/downloader/ism.py
Normal file
@@ -0,0 +1,289 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import time
|
||||
import binascii
|
||||
import io
|
||||
|
||||
from .fragment import FragmentFD
|
||||
from ..compat import (
|
||||
compat_Struct,
|
||||
compat_urllib_error,
|
||||
)
|
||||
|
||||
|
||||
u8 = compat_Struct('>B')
|
||||
u88 = compat_Struct('>Bx')
|
||||
u16 = compat_Struct('>H')
|
||||
u1616 = compat_Struct('>Hxx')
|
||||
u32 = compat_Struct('>I')
|
||||
u64 = compat_Struct('>Q')
|
||||
|
||||
s88 = compat_Struct('>bx')
|
||||
s16 = compat_Struct('>h')
|
||||
s1616 = compat_Struct('>hxx')
|
||||
s32 = compat_Struct('>i')
|
||||
|
||||
unity_matrix = (s32.pack(0x10000) + s32.pack(0) * 3) * 2 + s32.pack(0x40000000)
|
||||
|
||||
TRACK_ENABLED = 0x1
|
||||
TRACK_IN_MOVIE = 0x2
|
||||
TRACK_IN_PREVIEW = 0x4
|
||||
|
||||
SELF_CONTAINED = 0x1
|
||||
|
||||
|
||||
def box(box_type, payload):
|
||||
return u32.pack(8 + len(payload)) + box_type + payload
|
||||
|
||||
|
||||
def full_box(box_type, version, flags, payload):
|
||||
return box(box_type, u8.pack(version) + u32.pack(flags)[1:] + payload)
|
||||
|
||||
|
||||
def write_piff_header(stream, params):
|
||||
track_id = params['track_id']
|
||||
fourcc = params['fourcc']
|
||||
duration = params['duration']
|
||||
timescale = params.get('timescale', 10000000)
|
||||
language = params.get('language', 'und')
|
||||
height = params.get('height', 0)
|
||||
width = params.get('width', 0)
|
||||
stream_type = params['stream_type']
|
||||
creation_time = modification_time = int(time.time())
|
||||
|
||||
ftyp_payload = b'isml' # major brand
|
||||
ftyp_payload += u32.pack(1) # minor version
|
||||
ftyp_payload += b'piff' + b'iso2' # compatible brands
|
||||
stream.write(box(b'ftyp', ftyp_payload)) # File Type Box
|
||||
|
||||
mvhd_payload = u64.pack(creation_time)
|
||||
mvhd_payload += u64.pack(modification_time)
|
||||
mvhd_payload += u32.pack(timescale)
|
||||
mvhd_payload += u64.pack(duration)
|
||||
mvhd_payload += s1616.pack(1) # rate
|
||||
mvhd_payload += s88.pack(1) # volume
|
||||
mvhd_payload += u16.pack(0) # reserved
|
||||
mvhd_payload += u32.pack(0) * 2 # reserved
|
||||
mvhd_payload += unity_matrix
|
||||
mvhd_payload += u32.pack(0) * 6 # pre defined
|
||||
mvhd_payload += u32.pack(0xffffffff) # next track id
|
||||
moov_payload = full_box(b'mvhd', 1, 0, mvhd_payload) # Movie Header Box
|
||||
|
||||
tkhd_payload = u64.pack(creation_time)
|
||||
tkhd_payload += u64.pack(modification_time)
|
||||
tkhd_payload += u32.pack(track_id) # track id
|
||||
tkhd_payload += u32.pack(0) # reserved
|
||||
tkhd_payload += u64.pack(duration)
|
||||
tkhd_payload += u32.pack(0) * 2 # reserved
|
||||
tkhd_payload += s16.pack(0) # layer
|
||||
tkhd_payload += s16.pack(0) # alternate group
|
||||
tkhd_payload += s88.pack(1 if stream_type == 'audio' else 0) # volume
|
||||
tkhd_payload += u16.pack(0) # reserved
|
||||
tkhd_payload += unity_matrix
|
||||
tkhd_payload += u1616.pack(width)
|
||||
tkhd_payload += u1616.pack(height)
|
||||
trak_payload = full_box(b'tkhd', 1, TRACK_ENABLED | TRACK_IN_MOVIE | TRACK_IN_PREVIEW, tkhd_payload) # Track Header Box
|
||||
|
||||
mdhd_payload = u64.pack(creation_time)
|
||||
mdhd_payload += u64.pack(modification_time)
|
||||
mdhd_payload += u32.pack(timescale)
|
||||
mdhd_payload += u64.pack(duration)
|
||||
mdhd_payload += u16.pack(((ord(language[0]) - 0x60) << 10) | ((ord(language[1]) - 0x60) << 5) | (ord(language[2]) - 0x60))
|
||||
mdhd_payload += u16.pack(0) # pre defined
|
||||
mdia_payload = full_box(b'mdhd', 1, 0, mdhd_payload) # Media Header Box
|
||||
|
||||
hdlr_payload = u32.pack(0) # pre defined
|
||||
if stream_type == 'audio': # handler type
|
||||
hdlr_payload += b'soun'
|
||||
hdlr_payload += u32.pack(0) * 3 # reserved
|
||||
hdlr_payload += b'SoundHandler\0' # name
|
||||
elif stream_type == 'video':
|
||||
hdlr_payload += b'vide'
|
||||
hdlr_payload += u32.pack(0) * 3 # reserved
|
||||
hdlr_payload += b'VideoHandler\0' # name
|
||||
elif stream_type == 'text':
|
||||
hdlr_payload += b'subt'
|
||||
hdlr_payload += u32.pack(0) * 3 # reserved
|
||||
hdlr_payload += b'SubtitleHandler\0' # name
|
||||
else:
|
||||
assert False
|
||||
mdia_payload += full_box(b'hdlr', 0, 0, hdlr_payload) # Handler Reference Box
|
||||
|
||||
if stream_type == 'audio':
|
||||
smhd_payload = s88.pack(0) # balance
|
||||
smhd_payload += u16.pack(0) # reserved
|
||||
media_header_box = full_box(b'smhd', 0, 0, smhd_payload) # Sound Media Header
|
||||
elif stream_type == 'video':
|
||||
vmhd_payload = u16.pack(0) # graphics mode
|
||||
vmhd_payload += u16.pack(0) * 3 # opcolor
|
||||
media_header_box = full_box(b'vmhd', 0, 1, vmhd_payload) # Video Media Header
|
||||
elif stream_type == 'text':
|
||||
media_header_box = full_box(b'sthd', 0, 0, b'') # Subtitle Media Header
|
||||
else:
|
||||
assert False
|
||||
minf_payload = media_header_box
|
||||
|
||||
dref_payload = u32.pack(1) # entry count
|
||||
dref_payload += full_box(b'url ', 0, SELF_CONTAINED, b'') # Data Entry URL Box
|
||||
dinf_payload = full_box(b'dref', 0, 0, dref_payload) # Data Reference Box
|
||||
minf_payload += box(b'dinf', dinf_payload) # Data Information Box
|
||||
|
||||
stsd_payload = u32.pack(1) # entry count
|
||||
|
||||
sample_entry_payload = u8.pack(0) * 6 # reserved
|
||||
sample_entry_payload += u16.pack(1) # data reference index
|
||||
if stream_type == 'audio':
|
||||
sample_entry_payload += u32.pack(0) * 2 # reserved
|
||||
sample_entry_payload += u16.pack(params.get('channels', 2))
|
||||
sample_entry_payload += u16.pack(params.get('bits_per_sample', 16))
|
||||
sample_entry_payload += u16.pack(0) # pre defined
|
||||
sample_entry_payload += u16.pack(0) # reserved
|
||||
sample_entry_payload += u1616.pack(params['sampling_rate'])
|
||||
|
||||
if fourcc == 'AACL':
|
||||
sample_entry_box = box(b'mp4a', sample_entry_payload)
|
||||
elif stream_type == 'video':
|
||||
sample_entry_payload += u16.pack(0) # pre defined
|
||||
sample_entry_payload += u16.pack(0) # reserved
|
||||
sample_entry_payload += u32.pack(0) * 3 # pre defined
|
||||
sample_entry_payload += u16.pack(width)
|
||||
sample_entry_payload += u16.pack(height)
|
||||
sample_entry_payload += u1616.pack(0x48) # horiz resolution 72 dpi
|
||||
sample_entry_payload += u1616.pack(0x48) # vert resolution 72 dpi
|
||||
sample_entry_payload += u32.pack(0) # reserved
|
||||
sample_entry_payload += u16.pack(1) # frame count
|
||||
sample_entry_payload += u8.pack(0) * 32 # compressor name
|
||||
sample_entry_payload += u16.pack(0x18) # depth
|
||||
sample_entry_payload += s16.pack(-1) # pre defined
|
||||
|
||||
codec_private_data = binascii.unhexlify(params['codec_private_data'].encode('utf-8'))
|
||||
if fourcc in ('H264', 'AVC1'):
|
||||
sps, pps = codec_private_data.split(u32.pack(1))[1:]
|
||||
avcc_payload = u8.pack(1) # configuration version
|
||||
avcc_payload += sps[1:4] # avc profile indication + profile compatibility + avc level indication
|
||||
avcc_payload += u8.pack(0xfc | (params.get('nal_unit_length_field', 4) - 1)) # complete representation (1) + reserved (11111) + length size minus one
|
||||
avcc_payload += u8.pack(1) # reserved (0) + number of sps (0000001)
|
||||
avcc_payload += u16.pack(len(sps))
|
||||
avcc_payload += sps
|
||||
avcc_payload += u8.pack(1) # number of pps
|
||||
avcc_payload += u16.pack(len(pps))
|
||||
avcc_payload += pps
|
||||
sample_entry_payload += box(b'avcC', avcc_payload) # AVC Decoder Configuration Record
|
||||
sample_entry_box = box(b'avc1', sample_entry_payload) # AVC Simple Entry
|
||||
else:
|
||||
assert False
|
||||
elif stream_type == 'text':
|
||||
if fourcc == 'TTML':
|
||||
sample_entry_payload += b'http://www.w3.org/ns/ttml\0' # namespace
|
||||
sample_entry_payload += b'\0' # schema location
|
||||
sample_entry_payload += b'\0' # auxilary mime types(??)
|
||||
sample_entry_box = box(b'stpp', sample_entry_payload)
|
||||
else:
|
||||
assert False
|
||||
else:
|
||||
assert False
|
||||
stsd_payload += sample_entry_box
|
||||
|
||||
stbl_payload = full_box(b'stsd', 0, 0, stsd_payload) # Sample Description Box
|
||||
|
||||
stts_payload = u32.pack(0) # entry count
|
||||
stbl_payload += full_box(b'stts', 0, 0, stts_payload) # Decoding Time to Sample Box
|
||||
|
||||
stsc_payload = u32.pack(0) # entry count
|
||||
stbl_payload += full_box(b'stsc', 0, 0, stsc_payload) # Sample To Chunk Box
|
||||
|
||||
stco_payload = u32.pack(0) # entry count
|
||||
stbl_payload += full_box(b'stco', 0, 0, stco_payload) # Chunk Offset Box
|
||||
|
||||
minf_payload += box(b'stbl', stbl_payload) # Sample Table Box
|
||||
|
||||
mdia_payload += box(b'minf', minf_payload) # Media Information Box
|
||||
|
||||
trak_payload += box(b'mdia', mdia_payload) # Media Box
|
||||
|
||||
moov_payload += box(b'trak', trak_payload) # Track Box
|
||||
|
||||
mehd_payload = u64.pack(duration)
|
||||
mvex_payload = full_box(b'mehd', 1, 0, mehd_payload) # Movie Extends Header Box
|
||||
|
||||
trex_payload = u32.pack(track_id) # track id
|
||||
trex_payload += u32.pack(1) # default sample description index
|
||||
trex_payload += u32.pack(0) # default sample duration
|
||||
trex_payload += u32.pack(0) # default sample size
|
||||
trex_payload += u32.pack(0) # default sample flags
|
||||
mvex_payload += full_box(b'trex', 0, 0, trex_payload) # Track Extends Box
|
||||
|
||||
moov_payload += box(b'mvex', mvex_payload) # Movie Extends Box
|
||||
stream.write(box(b'moov', moov_payload)) # Movie Box
|
||||
|
||||
|
||||
def extract_box_data(data, box_sequence):
|
||||
data_reader = io.BytesIO(data)
|
||||
while True:
|
||||
box_size = u32.unpack(data_reader.read(4))[0]
|
||||
box_type = data_reader.read(4)
|
||||
if box_type == box_sequence[0]:
|
||||
box_data = data_reader.read(box_size - 8)
|
||||
if len(box_sequence) == 1:
|
||||
return box_data
|
||||
return extract_box_data(box_data, box_sequence[1:])
|
||||
data_reader.seek(box_size - 8, 1)
|
||||
|
||||
|
||||
class IsmFD(FragmentFD):
|
||||
"""
|
||||
Download segments in a ISM manifest
|
||||
"""
|
||||
|
||||
FD_NAME = 'ism'
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
segments = info_dict['fragments'][:1] if self.params.get(
|
||||
'test', False) else info_dict['fragments']
|
||||
|
||||
ctx = {
|
||||
'filename': filename,
|
||||
'total_frags': len(segments),
|
||||
}
|
||||
|
||||
self._prepare_and_start_frag_download(ctx, info_dict)
|
||||
|
||||
extra_state = ctx.setdefault('extra_state', {
|
||||
'ism_track_written': False,
|
||||
})
|
||||
|
||||
fragment_retries = self.params.get('fragment_retries', 0)
|
||||
skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
|
||||
|
||||
frag_index = 0
|
||||
for i, segment in enumerate(segments):
|
||||
frag_index += 1
|
||||
if frag_index <= ctx['fragment_index']:
|
||||
continue
|
||||
count = 0
|
||||
while count <= fragment_retries:
|
||||
try:
|
||||
success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
|
||||
if not success:
|
||||
return False
|
||||
if not extra_state['ism_track_written']:
|
||||
tfhd_data = extract_box_data(frag_content, [b'moof', b'traf', b'tfhd'])
|
||||
info_dict['_download_params']['track_id'] = u32.unpack(tfhd_data[4:8])[0]
|
||||
write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
|
||||
extra_state['ism_track_written'] = True
|
||||
self._append_fragment(ctx, frag_content)
|
||||
break
|
||||
except compat_urllib_error.HTTPError as err:
|
||||
count += 1
|
||||
if count <= fragment_retries:
|
||||
self.report_retry_fragment(err, frag_index, count, fragment_retries)
|
||||
if count > fragment_retries:
|
||||
if skip_unavailable_fragments:
|
||||
self.report_skip_fragment(frag_index)
|
||||
continue
|
||||
self.report_error('giving up after %s fragment retries' % fragment_retries)
|
||||
return False
|
||||
|
||||
self._finish_frag_download(ctx, info_dict)
|
||||
|
||||
return True
|
||||
202
plugins/youtube_download/yt_dlp/downloader/mhtml.py
Normal file
202
plugins/youtube_download/yt_dlp/downloader/mhtml.py
Normal file
@@ -0,0 +1,202 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import io
|
||||
import quopri
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from .fragment import FragmentFD
|
||||
from ..utils import (
|
||||
escapeHTML,
|
||||
formatSeconds,
|
||||
srt_subtitles_timecode,
|
||||
urljoin,
|
||||
)
|
||||
from ..version import __version__ as YT_DLP_VERSION
|
||||
|
||||
|
||||
class MhtmlFD(FragmentFD):
|
||||
FD_NAME = 'mhtml'
|
||||
|
||||
_STYLESHEET = """\
|
||||
html, body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
height: 100vh;
|
||||
}
|
||||
|
||||
html {
|
||||
overflow-y: scroll;
|
||||
scroll-snap-type: y mandatory;
|
||||
}
|
||||
|
||||
body {
|
||||
scroll-snap-type: y mandatory;
|
||||
display: flex;
|
||||
flex-flow: column;
|
||||
}
|
||||
|
||||
body > figure {
|
||||
max-width: 100vw;
|
||||
max-height: 100vh;
|
||||
scroll-snap-align: center;
|
||||
}
|
||||
|
||||
body > figure > figcaption {
|
||||
text-align: center;
|
||||
height: 2.5em;
|
||||
}
|
||||
|
||||
body > figure > img {
|
||||
display: block;
|
||||
margin: auto;
|
||||
max-width: 100%;
|
||||
max-height: calc(100vh - 5em);
|
||||
}
|
||||
"""
|
||||
_STYLESHEET = re.sub(r'\s+', ' ', _STYLESHEET)
|
||||
_STYLESHEET = re.sub(r'\B \B|(?<=[\w\-]) (?=[^\w\-])|(?<=[^\w\-]) (?=[\w\-])', '', _STYLESHEET)
|
||||
|
||||
@staticmethod
|
||||
def _escape_mime(s):
|
||||
return '=?utf-8?Q?' + (b''.join(
|
||||
bytes((b,)) if b >= 0x20 else b'=%02X' % b
|
||||
for b in quopri.encodestring(s.encode('utf-8'), header=True)
|
||||
)).decode('us-ascii') + '?='
|
||||
|
||||
def _gen_cid(self, i, fragment, frag_boundary):
|
||||
return '%u.%s@yt-dlp.github.io.invalid' % (i, frag_boundary)
|
||||
|
||||
def _gen_stub(self, *, fragments, frag_boundary, title):
|
||||
output = io.StringIO()
|
||||
|
||||
output.write((
|
||||
'<!DOCTYPE html>'
|
||||
'<html>'
|
||||
'<head>'
|
||||
'' '<meta name="generator" content="yt-dlp {version}">'
|
||||
'' '<title>{title}</title>'
|
||||
'' '<style>{styles}</style>'
|
||||
'<body>'
|
||||
).format(
|
||||
version=escapeHTML(YT_DLP_VERSION),
|
||||
styles=self._STYLESHEET,
|
||||
title=escapeHTML(title)
|
||||
))
|
||||
|
||||
t0 = 0
|
||||
for i, frag in enumerate(fragments):
|
||||
output.write('<figure>')
|
||||
try:
|
||||
t1 = t0 + frag['duration']
|
||||
output.write((
|
||||
'<figcaption>Slide #{num}: {t0} – {t1} (duration: {duration})</figcaption>'
|
||||
).format(
|
||||
num=i + 1,
|
||||
t0=srt_subtitles_timecode(t0),
|
||||
t1=srt_subtitles_timecode(t1),
|
||||
duration=formatSeconds(frag['duration'], msec=True)
|
||||
))
|
||||
except (KeyError, ValueError, TypeError):
|
||||
t1 = None
|
||||
output.write((
|
||||
'<figcaption>Slide #{num}</figcaption>'
|
||||
).format(num=i + 1))
|
||||
output.write('<img src="cid:{cid}">'.format(
|
||||
cid=self._gen_cid(i, frag, frag_boundary)))
|
||||
output.write('</figure>')
|
||||
t0 = t1
|
||||
|
||||
return output.getvalue()
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
fragment_base_url = info_dict.get('fragment_base_url')
|
||||
fragments = info_dict['fragments'][:1] if self.params.get(
|
||||
'test', False) else info_dict['fragments']
|
||||
title = info_dict.get('title', info_dict['format_id'])
|
||||
origin = info_dict.get('webpage_url', info_dict['url'])
|
||||
|
||||
ctx = {
|
||||
'filename': filename,
|
||||
'total_frags': len(fragments),
|
||||
}
|
||||
|
||||
self._prepare_and_start_frag_download(ctx, info_dict)
|
||||
|
||||
extra_state = ctx.setdefault('extra_state', {
|
||||
'header_written': False,
|
||||
'mime_boundary': str(uuid.uuid4()).replace('-', ''),
|
||||
})
|
||||
|
||||
frag_boundary = extra_state['mime_boundary']
|
||||
|
||||
if not extra_state['header_written']:
|
||||
stub = self._gen_stub(
|
||||
fragments=fragments,
|
||||
frag_boundary=frag_boundary,
|
||||
title=title
|
||||
)
|
||||
|
||||
ctx['dest_stream'].write((
|
||||
'MIME-Version: 1.0\r\n'
|
||||
'From: <nowhere@yt-dlp.github.io.invalid>\r\n'
|
||||
'To: <nowhere@yt-dlp.github.io.invalid>\r\n'
|
||||
'Subject: {title}\r\n'
|
||||
'Content-type: multipart/related; '
|
||||
'' 'boundary="{boundary}"; '
|
||||
'' 'type="text/html"\r\n'
|
||||
'X.yt-dlp.Origin: {origin}\r\n'
|
||||
'\r\n'
|
||||
'--{boundary}\r\n'
|
||||
'Content-Type: text/html; charset=utf-8\r\n'
|
||||
'Content-Length: {length}\r\n'
|
||||
'\r\n'
|
||||
'{stub}\r\n'
|
||||
).format(
|
||||
origin=origin,
|
||||
boundary=frag_boundary,
|
||||
length=len(stub),
|
||||
title=self._escape_mime(title),
|
||||
stub=stub
|
||||
).encode('utf-8'))
|
||||
extra_state['header_written'] = True
|
||||
|
||||
for i, fragment in enumerate(fragments):
|
||||
if (i + 1) <= ctx['fragment_index']:
|
||||
continue
|
||||
|
||||
fragment_url = urljoin(fragment_base_url, fragment['path'])
|
||||
success, frag_content = self._download_fragment(ctx, fragment_url, info_dict)
|
||||
if not success:
|
||||
continue
|
||||
|
||||
mime_type = b'image/jpeg'
|
||||
if frag_content.startswith(b'\x89PNG\r\n\x1a\n'):
|
||||
mime_type = b'image/png'
|
||||
if frag_content.startswith((b'GIF87a', b'GIF89a')):
|
||||
mime_type = b'image/gif'
|
||||
if frag_content.startswith(b'RIFF') and frag_content[8:12] == 'WEBP':
|
||||
mime_type = b'image/webp'
|
||||
|
||||
frag_header = io.BytesIO()
|
||||
frag_header.write(
|
||||
b'--%b\r\n' % frag_boundary.encode('us-ascii'))
|
||||
frag_header.write(
|
||||
b'Content-ID: <%b>\r\n' % self._gen_cid(i, fragment, frag_boundary).encode('us-ascii'))
|
||||
frag_header.write(
|
||||
b'Content-type: %b\r\n' % mime_type)
|
||||
frag_header.write(
|
||||
b'Content-length: %u\r\n' % len(frag_content))
|
||||
frag_header.write(
|
||||
b'Content-location: %b\r\n' % fragment_url.encode('us-ascii'))
|
||||
frag_header.write(
|
||||
b'X.yt-dlp.Duration: %f\r\n' % fragment['duration'])
|
||||
frag_header.write(b'\r\n')
|
||||
self._append_fragment(
|
||||
ctx, frag_header.getvalue() + frag_content + b'\r\n')
|
||||
|
||||
ctx['dest_stream'].write(
|
||||
b'--%b--\r\n\r\n' % frag_boundary.encode('us-ascii'))
|
||||
self._finish_frag_download(ctx, info_dict)
|
||||
return True
|
||||
57
plugins/youtube_download/yt_dlp/downloader/niconico.py
Normal file
57
plugins/youtube_download/yt_dlp/downloader/niconico.py
Normal file
@@ -0,0 +1,57 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import threading
|
||||
|
||||
from .common import FileDownloader
|
||||
from ..downloader import get_suitable_downloader
|
||||
from ..extractor.niconico import NiconicoIE
|
||||
from ..utils import sanitized_Request
|
||||
|
||||
|
||||
class NiconicoDmcFD(FileDownloader):
|
||||
""" Downloading niconico douga from DMC with heartbeat """
|
||||
|
||||
FD_NAME = 'niconico_dmc'
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
self.to_screen('[%s] Downloading from DMC' % self.FD_NAME)
|
||||
|
||||
ie = NiconicoIE(self.ydl)
|
||||
info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
|
||||
|
||||
fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params)
|
||||
|
||||
success = download_complete = False
|
||||
timer = [None]
|
||||
heartbeat_lock = threading.Lock()
|
||||
heartbeat_url = heartbeat_info_dict['url']
|
||||
heartbeat_data = heartbeat_info_dict['data'].encode()
|
||||
heartbeat_interval = heartbeat_info_dict.get('interval', 30)
|
||||
|
||||
request = sanitized_Request(heartbeat_url, heartbeat_data)
|
||||
|
||||
def heartbeat():
|
||||
try:
|
||||
self.ydl.urlopen(request).read()
|
||||
except Exception:
|
||||
self.to_screen('[%s] Heartbeat failed' % self.FD_NAME)
|
||||
|
||||
with heartbeat_lock:
|
||||
if not download_complete:
|
||||
timer[0] = threading.Timer(heartbeat_interval, heartbeat)
|
||||
timer[0].start()
|
||||
|
||||
heartbeat_info_dict['ping']()
|
||||
self.to_screen('[%s] Heartbeat with %d second interval ...' % (self.FD_NAME, heartbeat_interval))
|
||||
try:
|
||||
heartbeat()
|
||||
if type(fd).__name__ == 'HlsFD':
|
||||
info_dict.update(ie._extract_m3u8_formats(info_dict['url'], info_dict['id'])[0])
|
||||
success = fd.real_download(filename, info_dict)
|
||||
finally:
|
||||
if heartbeat_lock:
|
||||
with heartbeat_lock:
|
||||
timer[0].cancel()
|
||||
download_complete = True
|
||||
return success
|
||||
217
plugins/youtube_download/yt_dlp/downloader/rtmp.py
Normal file
217
plugins/youtube_download/yt_dlp/downloader/rtmp.py
Normal file
@@ -0,0 +1,217 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from .common import FileDownloader
|
||||
from ..compat import compat_str
|
||||
from ..utils import (
|
||||
check_executable,
|
||||
encodeFilename,
|
||||
encodeArgument,
|
||||
get_exe_version,
|
||||
Popen,
|
||||
)
|
||||
|
||||
|
||||
def rtmpdump_version():
|
||||
return get_exe_version(
|
||||
'rtmpdump', ['--help'], r'(?i)RTMPDump\s*v?([0-9a-zA-Z._-]+)')
|
||||
|
||||
|
||||
class RtmpFD(FileDownloader):
|
||||
def real_download(self, filename, info_dict):
|
||||
def run_rtmpdump(args):
|
||||
start = time.time()
|
||||
resume_percent = None
|
||||
resume_downloaded_data_len = None
|
||||
proc = Popen(args, stderr=subprocess.PIPE)
|
||||
cursor_in_new_line = True
|
||||
proc_stderr_closed = False
|
||||
try:
|
||||
while not proc_stderr_closed:
|
||||
# read line from stderr
|
||||
line = ''
|
||||
while True:
|
||||
char = proc.stderr.read(1)
|
||||
if not char:
|
||||
proc_stderr_closed = True
|
||||
break
|
||||
if char in [b'\r', b'\n']:
|
||||
break
|
||||
line += char.decode('ascii', 'replace')
|
||||
if not line:
|
||||
# proc_stderr_closed is True
|
||||
continue
|
||||
mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line)
|
||||
if mobj:
|
||||
downloaded_data_len = int(float(mobj.group(1)) * 1024)
|
||||
percent = float(mobj.group(2))
|
||||
if not resume_percent:
|
||||
resume_percent = percent
|
||||
resume_downloaded_data_len = downloaded_data_len
|
||||
time_now = time.time()
|
||||
eta = self.calc_eta(start, time_now, 100 - resume_percent, percent - resume_percent)
|
||||
speed = self.calc_speed(start, time_now, downloaded_data_len - resume_downloaded_data_len)
|
||||
data_len = None
|
||||
if percent > 0:
|
||||
data_len = int(downloaded_data_len * 100 / percent)
|
||||
self._hook_progress({
|
||||
'status': 'downloading',
|
||||
'downloaded_bytes': downloaded_data_len,
|
||||
'total_bytes_estimate': data_len,
|
||||
'tmpfilename': tmpfilename,
|
||||
'filename': filename,
|
||||
'eta': eta,
|
||||
'elapsed': time_now - start,
|
||||
'speed': speed,
|
||||
}, info_dict)
|
||||
cursor_in_new_line = False
|
||||
else:
|
||||
# no percent for live streams
|
||||
mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line)
|
||||
if mobj:
|
||||
downloaded_data_len = int(float(mobj.group(1)) * 1024)
|
||||
time_now = time.time()
|
||||
speed = self.calc_speed(start, time_now, downloaded_data_len)
|
||||
self._hook_progress({
|
||||
'downloaded_bytes': downloaded_data_len,
|
||||
'tmpfilename': tmpfilename,
|
||||
'filename': filename,
|
||||
'status': 'downloading',
|
||||
'elapsed': time_now - start,
|
||||
'speed': speed,
|
||||
}, info_dict)
|
||||
cursor_in_new_line = False
|
||||
elif self.params.get('verbose', False):
|
||||
if not cursor_in_new_line:
|
||||
self.to_screen('')
|
||||
cursor_in_new_line = True
|
||||
self.to_screen('[rtmpdump] ' + line)
|
||||
if not cursor_in_new_line:
|
||||
self.to_screen('')
|
||||
return proc.wait()
|
||||
except BaseException: # Including KeyboardInterrupt
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
raise
|
||||
|
||||
url = info_dict['url']
|
||||
player_url = info_dict.get('player_url')
|
||||
page_url = info_dict.get('page_url')
|
||||
app = info_dict.get('app')
|
||||
play_path = info_dict.get('play_path')
|
||||
tc_url = info_dict.get('tc_url')
|
||||
flash_version = info_dict.get('flash_version')
|
||||
live = info_dict.get('rtmp_live', False)
|
||||
conn = info_dict.get('rtmp_conn')
|
||||
protocol = info_dict.get('rtmp_protocol')
|
||||
real_time = info_dict.get('rtmp_real_time', False)
|
||||
no_resume = info_dict.get('no_resume', False)
|
||||
continue_dl = self.params.get('continuedl', True)
|
||||
|
||||
self.report_destination(filename)
|
||||
tmpfilename = self.temp_name(filename)
|
||||
test = self.params.get('test', False)
|
||||
|
||||
# Check for rtmpdump first
|
||||
if not check_executable('rtmpdump', ['-h']):
|
||||
self.report_error('RTMP download detected but "rtmpdump" could not be run. Please install')
|
||||
return False
|
||||
|
||||
# Download using rtmpdump. rtmpdump returns exit code 2 when
|
||||
# the connection was interrupted and resuming appears to be
|
||||
# possible. This is part of rtmpdump's normal usage, AFAIK.
|
||||
basic_args = [
|
||||
'rtmpdump', '--verbose', '-r', url,
|
||||
'-o', tmpfilename]
|
||||
if player_url is not None:
|
||||
basic_args += ['--swfVfy', player_url]
|
||||
if page_url is not None:
|
||||
basic_args += ['--pageUrl', page_url]
|
||||
if app is not None:
|
||||
basic_args += ['--app', app]
|
||||
if play_path is not None:
|
||||
basic_args += ['--playpath', play_path]
|
||||
if tc_url is not None:
|
||||
basic_args += ['--tcUrl', tc_url]
|
||||
if test:
|
||||
basic_args += ['--stop', '1']
|
||||
if flash_version is not None:
|
||||
basic_args += ['--flashVer', flash_version]
|
||||
if live:
|
||||
basic_args += ['--live']
|
||||
if isinstance(conn, list):
|
||||
for entry in conn:
|
||||
basic_args += ['--conn', entry]
|
||||
elif isinstance(conn, compat_str):
|
||||
basic_args += ['--conn', conn]
|
||||
if protocol is not None:
|
||||
basic_args += ['--protocol', protocol]
|
||||
if real_time:
|
||||
basic_args += ['--realtime']
|
||||
|
||||
args = basic_args
|
||||
if not no_resume and continue_dl and not live:
|
||||
args += ['--resume']
|
||||
if not live and continue_dl:
|
||||
args += ['--skip', '1']
|
||||
|
||||
args = [encodeArgument(a) for a in args]
|
||||
|
||||
self._debug_cmd(args, exe='rtmpdump')
|
||||
|
||||
RD_SUCCESS = 0
|
||||
RD_FAILED = 1
|
||||
RD_INCOMPLETE = 2
|
||||
RD_NO_CONNECT = 3
|
||||
|
||||
started = time.time()
|
||||
|
||||
try:
|
||||
retval = run_rtmpdump(args)
|
||||
except KeyboardInterrupt:
|
||||
if not info_dict.get('is_live'):
|
||||
raise
|
||||
retval = RD_SUCCESS
|
||||
self.to_screen('\n[rtmpdump] Interrupted by user')
|
||||
|
||||
if retval == RD_NO_CONNECT:
|
||||
self.report_error('[rtmpdump] Could not connect to RTMP server.')
|
||||
return False
|
||||
|
||||
while retval in (RD_INCOMPLETE, RD_FAILED) and not test and not live:
|
||||
prevsize = os.path.getsize(encodeFilename(tmpfilename))
|
||||
self.to_screen('[rtmpdump] Downloaded %s bytes' % prevsize)
|
||||
time.sleep(5.0) # This seems to be needed
|
||||
args = basic_args + ['--resume']
|
||||
if retval == RD_FAILED:
|
||||
args += ['--skip', '1']
|
||||
args = [encodeArgument(a) for a in args]
|
||||
retval = run_rtmpdump(args)
|
||||
cursize = os.path.getsize(encodeFilename(tmpfilename))
|
||||
if prevsize == cursize and retval == RD_FAILED:
|
||||
break
|
||||
# Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
|
||||
if prevsize == cursize and retval == RD_INCOMPLETE and cursize > 1024:
|
||||
self.to_screen('[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
|
||||
retval = RD_SUCCESS
|
||||
break
|
||||
if retval == RD_SUCCESS or (test and retval == RD_INCOMPLETE):
|
||||
fsize = os.path.getsize(encodeFilename(tmpfilename))
|
||||
self.to_screen('[rtmpdump] Downloaded %s bytes' % fsize)
|
||||
self.try_rename(tmpfilename, filename)
|
||||
self._hook_progress({
|
||||
'downloaded_bytes': fsize,
|
||||
'total_bytes': fsize,
|
||||
'filename': filename,
|
||||
'status': 'finished',
|
||||
'elapsed': time.time() - started,
|
||||
}, info_dict)
|
||||
return True
|
||||
else:
|
||||
self.to_stderr('\n')
|
||||
self.report_error('rtmpdump exited with code %d' % retval)
|
||||
return False
|
||||
47
plugins/youtube_download/yt_dlp/downloader/rtsp.py
Normal file
47
plugins/youtube_download/yt_dlp/downloader/rtsp.py
Normal file
@@ -0,0 +1,47 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from .common import FileDownloader
|
||||
from ..utils import (
|
||||
check_executable,
|
||||
encodeFilename,
|
||||
)
|
||||
|
||||
|
||||
class RtspFD(FileDownloader):
|
||||
def real_download(self, filename, info_dict):
|
||||
url = info_dict['url']
|
||||
self.report_destination(filename)
|
||||
tmpfilename = self.temp_name(filename)
|
||||
|
||||
if check_executable('mplayer', ['-h']):
|
||||
args = [
|
||||
'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
|
||||
'-dumpstream', '-dumpfile', tmpfilename, url]
|
||||
elif check_executable('mpv', ['-h']):
|
||||
args = [
|
||||
'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
|
||||
else:
|
||||
self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install one')
|
||||
return False
|
||||
|
||||
self._debug_cmd(args)
|
||||
|
||||
retval = subprocess.call(args)
|
||||
if retval == 0:
|
||||
fsize = os.path.getsize(encodeFilename(tmpfilename))
|
||||
self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
|
||||
self.try_rename(tmpfilename, filename)
|
||||
self._hook_progress({
|
||||
'downloaded_bytes': fsize,
|
||||
'total_bytes': fsize,
|
||||
'filename': filename,
|
||||
'status': 'finished',
|
||||
}, info_dict)
|
||||
return True
|
||||
else:
|
||||
self.to_stderr('\n')
|
||||
self.report_error('%s exited with code %d' % (args[0], retval))
|
||||
return False
|
||||
62
plugins/youtube_download/yt_dlp/downloader/websocket.py
Normal file
62
plugins/youtube_download/yt_dlp/downloader/websocket.py
Normal file
@@ -0,0 +1,62 @@
|
||||
import os
|
||||
import signal
|
||||
import asyncio
|
||||
import threading
|
||||
|
||||
try:
|
||||
import websockets
|
||||
except (ImportError, SyntaxError):
|
||||
# websockets 3.10 on python 3.6 causes SyntaxError
|
||||
# See https://github.com/yt-dlp/yt-dlp/issues/2633
|
||||
has_websockets = False
|
||||
else:
|
||||
has_websockets = True
|
||||
|
||||
from .common import FileDownloader
|
||||
from .external import FFmpegFD
|
||||
|
||||
|
||||
class FFmpegSinkFD(FileDownloader):
|
||||
""" A sink to ffmpeg for downloading fragments in any form """
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
info_copy = info_dict.copy()
|
||||
info_copy['url'] = '-'
|
||||
|
||||
async def call_conn(proc, stdin):
|
||||
try:
|
||||
await self.real_connection(stdin, info_dict)
|
||||
except (BrokenPipeError, OSError):
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
stdin.flush()
|
||||
stdin.close()
|
||||
except OSError:
|
||||
pass
|
||||
os.kill(os.getpid(), signal.SIGINT)
|
||||
|
||||
class FFmpegStdinFD(FFmpegFD):
|
||||
@classmethod
|
||||
def get_basename(cls):
|
||||
return FFmpegFD.get_basename()
|
||||
|
||||
def on_process_started(self, proc, stdin):
|
||||
thread = threading.Thread(target=asyncio.run, daemon=True, args=(call_conn(proc, stdin), ))
|
||||
thread.start()
|
||||
|
||||
return FFmpegStdinFD(self.ydl, self.params or {}).download(filename, info_copy)
|
||||
|
||||
async def real_connection(self, sink, info_dict):
|
||||
""" Override this in subclasses """
|
||||
raise NotImplementedError('This method must be implemented by subclasses')
|
||||
|
||||
|
||||
class WebSocketFragmentFD(FFmpegSinkFD):
|
||||
async def real_connection(self, sink, info_dict):
|
||||
async with websockets.connect(info_dict['url'], extra_headers=info_dict.get('http_headers', {})) as ws:
|
||||
while True:
|
||||
recv = await ws.recv()
|
||||
if isinstance(recv, str):
|
||||
recv = recv.encode('utf8')
|
||||
sink.write(recv)
|
||||
236
plugins/youtube_download/yt_dlp/downloader/youtube_live_chat.py
Normal file
236
plugins/youtube_download/yt_dlp/downloader/youtube_live_chat.py
Normal file
@@ -0,0 +1,236 @@
|
||||
from __future__ import division, unicode_literals
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
from .fragment import FragmentFD
|
||||
from ..compat import compat_urllib_error
|
||||
from ..utils import (
|
||||
try_get,
|
||||
dict_get,
|
||||
int_or_none,
|
||||
RegexNotFoundError,
|
||||
)
|
||||
from ..extractor.youtube import YoutubeBaseInfoExtractor as YT_BaseIE
|
||||
|
||||
|
||||
class YoutubeLiveChatFD(FragmentFD):
|
||||
""" Downloads YouTube live chats fragment by fragment """
|
||||
|
||||
FD_NAME = 'youtube_live_chat'
|
||||
|
||||
def real_download(self, filename, info_dict):
|
||||
video_id = info_dict['video_id']
|
||||
self.to_screen('[%s] Downloading live chat' % self.FD_NAME)
|
||||
|
||||
fragment_retries = self.params.get('fragment_retries', 0)
|
||||
test = self.params.get('test', False)
|
||||
|
||||
ctx = {
|
||||
'filename': filename,
|
||||
'live': True,
|
||||
'total_frags': None,
|
||||
}
|
||||
|
||||
ie = YT_BaseIE(self.ydl)
|
||||
|
||||
start_time = int(time.time() * 1000)
|
||||
|
||||
def dl_fragment(url, data=None, headers=None):
|
||||
http_headers = info_dict.get('http_headers', {})
|
||||
if headers:
|
||||
http_headers = http_headers.copy()
|
||||
http_headers.update(headers)
|
||||
return self._download_fragment(ctx, url, info_dict, http_headers, data)
|
||||
|
||||
def parse_actions_replay(live_chat_continuation):
|
||||
offset = continuation_id = click_tracking_params = None
|
||||
processed_fragment = bytearray()
|
||||
for action in live_chat_continuation.get('actions', []):
|
||||
if 'replayChatItemAction' in action:
|
||||
replay_chat_item_action = action['replayChatItemAction']
|
||||
offset = int(replay_chat_item_action['videoOffsetTimeMsec'])
|
||||
processed_fragment.extend(
|
||||
json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n')
|
||||
if offset is not None:
|
||||
continuation = try_get(
|
||||
live_chat_continuation,
|
||||
lambda x: x['continuations'][0]['liveChatReplayContinuationData'], dict)
|
||||
if continuation:
|
||||
continuation_id = continuation.get('continuation')
|
||||
click_tracking_params = continuation.get('clickTrackingParams')
|
||||
self._append_fragment(ctx, processed_fragment)
|
||||
return continuation_id, offset, click_tracking_params
|
||||
|
||||
def try_refresh_replay_beginning(live_chat_continuation):
|
||||
# choose the second option that contains the unfiltered live chat replay
|
||||
refresh_continuation = try_get(
|
||||
live_chat_continuation,
|
||||
lambda x: x['header']['liveChatHeaderRenderer']['viewSelector']['sortFilterSubMenuRenderer']['subMenuItems'][1]['continuation']['reloadContinuationData'], dict)
|
||||
if refresh_continuation:
|
||||
# no data yet but required to call _append_fragment
|
||||
self._append_fragment(ctx, b'')
|
||||
refresh_continuation_id = refresh_continuation.get('continuation')
|
||||
offset = 0
|
||||
click_tracking_params = refresh_continuation.get('trackingParams')
|
||||
return refresh_continuation_id, offset, click_tracking_params
|
||||
return parse_actions_replay(live_chat_continuation)
|
||||
|
||||
live_offset = 0
|
||||
|
||||
def parse_actions_live(live_chat_continuation):
|
||||
nonlocal live_offset
|
||||
continuation_id = click_tracking_params = None
|
||||
processed_fragment = bytearray()
|
||||
for action in live_chat_continuation.get('actions', []):
|
||||
timestamp = self.parse_live_timestamp(action)
|
||||
if timestamp is not None:
|
||||
live_offset = timestamp - start_time
|
||||
# compatibility with replay format
|
||||
pseudo_action = {
|
||||
'replayChatItemAction': {'actions': [action]},
|
||||
'videoOffsetTimeMsec': str(live_offset),
|
||||
'isLive': True,
|
||||
}
|
||||
processed_fragment.extend(
|
||||
json.dumps(pseudo_action, ensure_ascii=False).encode('utf-8') + b'\n')
|
||||
continuation_data_getters = [
|
||||
lambda x: x['continuations'][0]['invalidationContinuationData'],
|
||||
lambda x: x['continuations'][0]['timedContinuationData'],
|
||||
]
|
||||
continuation_data = try_get(live_chat_continuation, continuation_data_getters, dict)
|
||||
if continuation_data:
|
||||
continuation_id = continuation_data.get('continuation')
|
||||
click_tracking_params = continuation_data.get('clickTrackingParams')
|
||||
timeout_ms = int_or_none(continuation_data.get('timeoutMs'))
|
||||
if timeout_ms is not None:
|
||||
time.sleep(timeout_ms / 1000)
|
||||
self._append_fragment(ctx, processed_fragment)
|
||||
return continuation_id, live_offset, click_tracking_params
|
||||
|
||||
def download_and_parse_fragment(url, frag_index, request_data=None, headers=None):
|
||||
count = 0
|
||||
while count <= fragment_retries:
|
||||
try:
|
||||
success, raw_fragment = dl_fragment(url, request_data, headers)
|
||||
if not success:
|
||||
return False, None, None, None
|
||||
try:
|
||||
data = ie.extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace'))
|
||||
except RegexNotFoundError:
|
||||
data = None
|
||||
if not data:
|
||||
data = json.loads(raw_fragment)
|
||||
live_chat_continuation = try_get(
|
||||
data,
|
||||
lambda x: x['continuationContents']['liveChatContinuation'], dict) or {}
|
||||
if info_dict['protocol'] == 'youtube_live_chat_replay':
|
||||
if frag_index == 1:
|
||||
continuation_id, offset, click_tracking_params = try_refresh_replay_beginning(live_chat_continuation)
|
||||
else:
|
||||
continuation_id, offset, click_tracking_params = parse_actions_replay(live_chat_continuation)
|
||||
elif info_dict['protocol'] == 'youtube_live_chat':
|
||||
continuation_id, offset, click_tracking_params = parse_actions_live(live_chat_continuation)
|
||||
return True, continuation_id, offset, click_tracking_params
|
||||
except compat_urllib_error.HTTPError as err:
|
||||
count += 1
|
||||
if count <= fragment_retries:
|
||||
self.report_retry_fragment(err, frag_index, count, fragment_retries)
|
||||
if count > fragment_retries:
|
||||
self.report_error('giving up after %s fragment retries' % fragment_retries)
|
||||
return False, None, None, None
|
||||
|
||||
self._prepare_and_start_frag_download(ctx, info_dict)
|
||||
|
||||
success, raw_fragment = dl_fragment(info_dict['url'])
|
||||
if not success:
|
||||
return False
|
||||
try:
|
||||
data = ie.extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace'))
|
||||
except RegexNotFoundError:
|
||||
return False
|
||||
continuation_id = try_get(
|
||||
data,
|
||||
lambda x: x['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation'])
|
||||
# no data yet but required to call _append_fragment
|
||||
self._append_fragment(ctx, b'')
|
||||
|
||||
ytcfg = ie.extract_ytcfg(video_id, raw_fragment.decode('utf-8', 'replace'))
|
||||
|
||||
if not ytcfg:
|
||||
return False
|
||||
api_key = try_get(ytcfg, lambda x: x['INNERTUBE_API_KEY'])
|
||||
innertube_context = try_get(ytcfg, lambda x: x['INNERTUBE_CONTEXT'])
|
||||
if not api_key or not innertube_context:
|
||||
return False
|
||||
visitor_data = try_get(innertube_context, lambda x: x['client']['visitorData'], str)
|
||||
if info_dict['protocol'] == 'youtube_live_chat_replay':
|
||||
url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay?key=' + api_key
|
||||
chat_page_url = 'https://www.youtube.com/live_chat_replay?continuation=' + continuation_id
|
||||
elif info_dict['protocol'] == 'youtube_live_chat':
|
||||
url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat?key=' + api_key
|
||||
chat_page_url = 'https://www.youtube.com/live_chat?continuation=' + continuation_id
|
||||
|
||||
frag_index = offset = 0
|
||||
click_tracking_params = None
|
||||
while continuation_id is not None:
|
||||
frag_index += 1
|
||||
request_data = {
|
||||
'context': innertube_context,
|
||||
'continuation': continuation_id,
|
||||
}
|
||||
if frag_index > 1:
|
||||
request_data['currentPlayerState'] = {'playerOffsetMs': str(max(offset - 5000, 0))}
|
||||
if click_tracking_params:
|
||||
request_data['context']['clickTracking'] = {'clickTrackingParams': click_tracking_params}
|
||||
headers = ie.generate_api_headers(ytcfg=ytcfg, visitor_data=visitor_data)
|
||||
headers.update({'content-type': 'application/json'})
|
||||
fragment_request_data = json.dumps(request_data, ensure_ascii=False).encode('utf-8') + b'\n'
|
||||
success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
|
||||
url, frag_index, fragment_request_data, headers)
|
||||
else:
|
||||
success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
|
||||
chat_page_url, frag_index)
|
||||
if not success:
|
||||
return False
|
||||
if test:
|
||||
break
|
||||
|
||||
self._finish_frag_download(ctx, info_dict)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def parse_live_timestamp(action):
|
||||
action_content = dict_get(
|
||||
action,
|
||||
['addChatItemAction', 'addLiveChatTickerItemAction', 'addBannerToLiveChatCommand'])
|
||||
if not isinstance(action_content, dict):
|
||||
return None
|
||||
item = dict_get(action_content, ['item', 'bannerRenderer'])
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
renderer = dict_get(item, [
|
||||
# text
|
||||
'liveChatTextMessageRenderer', 'liveChatPaidMessageRenderer',
|
||||
'liveChatMembershipItemRenderer', 'liveChatPaidStickerRenderer',
|
||||
# ticker
|
||||
'liveChatTickerPaidMessageItemRenderer',
|
||||
'liveChatTickerSponsorItemRenderer',
|
||||
# banner
|
||||
'liveChatBannerRenderer',
|
||||
])
|
||||
if not isinstance(renderer, dict):
|
||||
return None
|
||||
parent_item_getters = [
|
||||
lambda x: x['showItemEndpoint']['showLiveChatItemEndpoint']['renderer'],
|
||||
lambda x: x['contents'],
|
||||
]
|
||||
parent_item = try_get(renderer, parent_item_getters, dict)
|
||||
if parent_item:
|
||||
renderer = dict_get(parent_item, [
|
||||
'liveChatTextMessageRenderer', 'liveChatPaidMessageRenderer',
|
||||
'liveChatMembershipItemRenderer', 'liveChatPaidStickerRenderer',
|
||||
])
|
||||
if not isinstance(renderer, dict):
|
||||
return None
|
||||
return int_or_none(renderer.get('timestampUsec'), 1000)
|
||||
54
plugins/youtube_download/yt_dlp/extractor/__init__.py
Normal file
54
plugins/youtube_download/yt_dlp/extractor/__init__.py
Normal file
@@ -0,0 +1,54 @@
|
||||
import os
|
||||
|
||||
from ..utils import load_plugins
|
||||
|
||||
_LAZY_LOADER = False
|
||||
if not os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'):
|
||||
try:
|
||||
from .lazy_extractors import *
|
||||
from .lazy_extractors import _ALL_CLASSES
|
||||
_LAZY_LOADER = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
if not _LAZY_LOADER:
|
||||
from .extractors import *
|
||||
_ALL_CLASSES = [
|
||||
klass
|
||||
for name, klass in globals().items()
|
||||
if name.endswith('IE') and name != 'GenericIE'
|
||||
]
|
||||
_ALL_CLASSES.append(GenericIE)
|
||||
|
||||
_PLUGIN_CLASSES = load_plugins('extractor', 'IE', globals())
|
||||
_ALL_CLASSES = list(_PLUGIN_CLASSES.values()) + _ALL_CLASSES
|
||||
|
||||
|
||||
def gen_extractor_classes():
|
||||
""" Return a list of supported extractors.
|
||||
The order does matter; the first extractor matched is the one handling the URL.
|
||||
"""
|
||||
return _ALL_CLASSES
|
||||
|
||||
|
||||
def gen_extractors():
|
||||
""" Return a list of an instance of every supported extractor.
|
||||
The order does matter; the first extractor matched is the one handling the URL.
|
||||
"""
|
||||
return [klass() for klass in gen_extractor_classes()]
|
||||
|
||||
|
||||
def list_extractors(age_limit):
|
||||
"""
|
||||
Return a list of extractors that are suitable for the given age,
|
||||
sorted by extractor ID.
|
||||
"""
|
||||
|
||||
return sorted(
|
||||
filter(lambda ie: ie.is_suitable(age_limit), gen_extractors()),
|
||||
key=lambda ie: ie.IE_NAME.lower())
|
||||
|
||||
|
||||
def get_info_extractor(ie_name):
|
||||
"""Returns the info extractor class with the given ie_name"""
|
||||
return globals()[ie_name + 'IE']
|
||||
318
plugins/youtube_download/yt_dlp/extractor/abc.py
Normal file
318
plugins/youtube_download/yt_dlp/extractor/abc.py
Normal file
@@ -0,0 +1,318 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import re
|
||||
import time
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_str
|
||||
from ..utils import (
|
||||
dict_get,
|
||||
ExtractorError,
|
||||
js_to_json,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
str_or_none,
|
||||
try_get,
|
||||
unescapeHTML,
|
||||
update_url_query,
|
||||
)
|
||||
|
||||
|
||||
class ABCIE(InfoExtractor):
|
||||
IE_NAME = 'abc.net.au'
|
||||
_VALID_URL = r'https?://(?:www\.)?abc\.net\.au/(?:news|btn)/(?:[^/]+/){1,4}(?P<id>\d{5,})'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
|
||||
'md5': 'cb3dd03b18455a661071ee1e28344d9f',
|
||||
'info_dict': {
|
||||
'id': '5868334',
|
||||
'ext': 'mp4',
|
||||
'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
|
||||
'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
|
||||
},
|
||||
'skip': 'this video has expired',
|
||||
}, {
|
||||
'url': 'http://www.abc.net.au/news/2015-08-17/warren-entsch-introduces-same-sex-marriage-bill/6702326',
|
||||
'md5': '4ebd61bdc82d9a8b722f64f1f4b4d121',
|
||||
'info_dict': {
|
||||
'id': 'NvqvPeNZsHU',
|
||||
'ext': 'mp4',
|
||||
'upload_date': '20150816',
|
||||
'uploader': 'ABC News (Australia)',
|
||||
'description': 'Government backbencher Warren Entsch introduces a cross-party sponsored bill to legalise same-sex marriage, saying the bill is designed to promote "an inclusive Australia, not a divided one.". Read more here: http://ab.co/1Mwc6ef',
|
||||
'uploader_id': 'NewsOnABC',
|
||||
'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
|
||||
},
|
||||
'add_ie': ['Youtube'],
|
||||
'skip': 'Not accessible from Travis CI server',
|
||||
}, {
|
||||
'url': 'http://www.abc.net.au/news/2015-10-23/nab-lifts-interest-rates-following-westpac-and-cba/6880080',
|
||||
'md5': 'b96eee7c9edf4fc5a358a0252881cc1f',
|
||||
'info_dict': {
|
||||
'id': '6880080',
|
||||
'ext': 'mp3',
|
||||
'title': 'NAB lifts interest rates, following Westpac and CBA',
|
||||
'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://www.abc.net.au/news/2015-10-19/6866214',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.abc.net.au/btn/classroom/wwi-centenary/10527914',
|
||||
'info_dict': {
|
||||
'id': '10527914',
|
||||
'ext': 'mp4',
|
||||
'title': 'WWI Centenary',
|
||||
'description': 'md5:c2379ec0ca84072e86b446e536954546',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://www.abc.net.au/news/programs/the-world/2020-06-10/black-lives-matter-protests-spawn-support-for/12342074',
|
||||
'info_dict': {
|
||||
'id': '12342074',
|
||||
'ext': 'mp4',
|
||||
'title': 'Black Lives Matter protests spawn support for Papuans in Indonesia',
|
||||
'description': 'md5:2961a17dc53abc558589ccd0fb8edd6f',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://www.abc.net.au/btn/newsbreak/btn-newsbreak-20200814/12560476',
|
||||
'info_dict': {
|
||||
'id': 'tDL8Ld4dK_8',
|
||||
'ext': 'mp4',
|
||||
'title': 'Fortnite Banned From Apple and Google App Stores',
|
||||
'description': 'md5:a6df3f36ce8f816b74af4bd6462f5651',
|
||||
'upload_date': '20200813',
|
||||
'uploader': 'Behind the News',
|
||||
'uploader_id': 'behindthenews',
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
mobj = re.search(r'<a\s+href="(?P<url>[^"]+)"\s+data-duration="\d+"\s+title="Download audio directly">', webpage)
|
||||
if mobj:
|
||||
urls_info = mobj.groupdict()
|
||||
youtube = False
|
||||
video = False
|
||||
else:
|
||||
mobj = re.search(r'<a href="(?P<url>http://www\.youtube\.com/watch\?v=[^"]+)"><span><strong>External Link:</strong>',
|
||||
webpage)
|
||||
if mobj is None:
|
||||
mobj = re.search(r'<iframe width="100%" src="(?P<url>//www\.youtube-nocookie\.com/embed/[^?"]+)', webpage)
|
||||
if mobj:
|
||||
urls_info = mobj.groupdict()
|
||||
youtube = True
|
||||
video = True
|
||||
|
||||
if mobj is None:
|
||||
mobj = re.search(r'(?P<type>)"sources": (?P<json_data>\[[^\]]+\]),', webpage)
|
||||
if mobj is None:
|
||||
mobj = re.search(
|
||||
r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
|
||||
webpage)
|
||||
if mobj is None:
|
||||
expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?<span>(.+?)</span>', webpage, 'expired', None)
|
||||
if expired:
|
||||
raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True)
|
||||
raise ExtractorError('Unable to extract video urls')
|
||||
|
||||
urls_info = self._parse_json(
|
||||
mobj.group('json_data'), video_id, transform_source=js_to_json)
|
||||
youtube = mobj.group('type') == 'YouTube'
|
||||
video = mobj.group('type') == 'Video' or urls_info[0]['contentType'] == 'video/mp4'
|
||||
|
||||
if not isinstance(urls_info, list):
|
||||
urls_info = [urls_info]
|
||||
|
||||
if youtube:
|
||||
return self.playlist_result([
|
||||
self.url_result(url_info['url']) for url_info in urls_info])
|
||||
|
||||
formats = []
|
||||
for url_info in urls_info:
|
||||
height = int_or_none(url_info.get('height'))
|
||||
bitrate = int_or_none(url_info.get('bitrate'))
|
||||
width = int_or_none(url_info.get('width'))
|
||||
format_id = None
|
||||
mobj = re.search(r'_(?:(?P<height>\d+)|(?P<bitrate>\d+)k)\.mp4$', url_info['url'])
|
||||
if mobj:
|
||||
height_from_url = mobj.group('height')
|
||||
if height_from_url:
|
||||
height = height or int_or_none(height_from_url)
|
||||
width = width or int_or_none(url_info.get('label'))
|
||||
else:
|
||||
bitrate = bitrate or int_or_none(mobj.group('bitrate'))
|
||||
format_id = str_or_none(url_info.get('label'))
|
||||
formats.append({
|
||||
'url': url_info['url'],
|
||||
'vcodec': url_info.get('codec') if video else 'none',
|
||||
'width': width,
|
||||
'height': height,
|
||||
'tbr': bitrate,
|
||||
'filesize': int_or_none(url_info.get('filesize')),
|
||||
'format_id': format_id
|
||||
})
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._og_search_title(webpage),
|
||||
'formats': formats,
|
||||
'description': self._og_search_description(webpage),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
}
|
||||
|
||||
|
||||
class ABCIViewIE(InfoExtractor):
|
||||
IE_NAME = 'abc.net.au:iview'
|
||||
_VALID_URL = r'https?://iview\.abc\.net\.au/(?:[^/]+/)*video/(?P<id>[^/?#]+)'
|
||||
_GEO_COUNTRIES = ['AU']
|
||||
|
||||
# ABC iview programs are normally available for 14 days only.
|
||||
_TESTS = [{
|
||||
'url': 'https://iview.abc.net.au/show/gruen/series/11/video/LE1927H001S00',
|
||||
'md5': '67715ce3c78426b11ba167d875ac6abf',
|
||||
'info_dict': {
|
||||
'id': 'LE1927H001S00',
|
||||
'ext': 'mp4',
|
||||
'title': "Series 11 Ep 1",
|
||||
'series': "Gruen",
|
||||
'description': 'md5:52cc744ad35045baf6aded2ce7287f67',
|
||||
'upload_date': '20190925',
|
||||
'uploader_id': 'abc1',
|
||||
'timestamp': 1569445289,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
video_params = self._download_json(
|
||||
'https://iview.abc.net.au/api/programs/' + video_id, video_id)
|
||||
title = unescapeHTML(video_params.get('title') or video_params['seriesTitle'])
|
||||
stream = next(s for s in video_params['playlist'] if s.get('type') in ('program', 'livestream'))
|
||||
|
||||
house_number = video_params.get('episodeHouseNumber') or video_id
|
||||
path = '/auth/hls/sign?ts={0}&hn={1}&d=android-tablet'.format(
|
||||
int(time.time()), house_number)
|
||||
sig = hmac.new(
|
||||
b'android.content.res.Resources',
|
||||
path.encode('utf-8'), hashlib.sha256).hexdigest()
|
||||
token = self._download_webpage(
|
||||
'http://iview.abc.net.au{0}&sig={1}'.format(path, sig), video_id)
|
||||
|
||||
def tokenize_url(url, token):
|
||||
return update_url_query(url, {
|
||||
'hdnea': token,
|
||||
})
|
||||
|
||||
for sd in ('720', 'sd', 'sd-low'):
|
||||
sd_url = try_get(
|
||||
stream, lambda x: x['streams']['hls'][sd], compat_str)
|
||||
if not sd_url:
|
||||
continue
|
||||
formats = self._extract_m3u8_formats(
|
||||
tokenize_url(sd_url, token), video_id, 'mp4',
|
||||
entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
|
||||
if formats:
|
||||
break
|
||||
self._sort_formats(formats)
|
||||
|
||||
subtitles = {}
|
||||
src_vtt = stream.get('captions', {}).get('src-vtt')
|
||||
if src_vtt:
|
||||
subtitles['en'] = [{
|
||||
'url': src_vtt,
|
||||
'ext': 'vtt',
|
||||
}]
|
||||
|
||||
is_live = video_params.get('livestream') == '1'
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': video_params.get('description'),
|
||||
'thumbnail': video_params.get('thumbnail'),
|
||||
'duration': int_or_none(video_params.get('eventDuration')),
|
||||
'timestamp': parse_iso8601(video_params.get('pubDate'), ' '),
|
||||
'series': unescapeHTML(video_params.get('seriesTitle')),
|
||||
'series_id': video_params.get('seriesHouseNumber') or video_id[:7],
|
||||
'season_number': int_or_none(self._search_regex(
|
||||
r'\bSeries\s+(\d+)\b', title, 'season number', default=None)),
|
||||
'episode_number': int_or_none(self._search_regex(
|
||||
r'\bEp\s+(\d+)\b', title, 'episode number', default=None)),
|
||||
'episode_id': house_number,
|
||||
'uploader_id': video_params.get('channel'),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'is_live': is_live,
|
||||
}
|
||||
|
||||
|
||||
class ABCIViewShowSeriesIE(InfoExtractor):
|
||||
IE_NAME = 'abc.net.au:iview:showseries'
|
||||
_VALID_URL = r'https?://iview\.abc\.net\.au/show/(?P<id>[^/]+)(?:/series/\d+)?$'
|
||||
_GEO_COUNTRIES = ['AU']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://iview.abc.net.au/show/upper-middle-bogan',
|
||||
'info_dict': {
|
||||
'id': '124870-1',
|
||||
'title': 'Series 1',
|
||||
'description': 'md5:93119346c24a7c322d446d8eece430ff',
|
||||
'series': 'Upper Middle Bogan',
|
||||
'season': 'Series 1',
|
||||
'thumbnail': r're:^https?://cdn\.iview\.abc\.net\.au/thumbs/.*\.jpg$'
|
||||
},
|
||||
'playlist_count': 8,
|
||||
}, {
|
||||
'url': 'https://iview.abc.net.au/show/upper-middle-bogan',
|
||||
'info_dict': {
|
||||
'id': 'CO1108V001S00',
|
||||
'ext': 'mp4',
|
||||
'title': 'Series 1 Ep 1 I\'m A Swan',
|
||||
'description': 'md5:7b676758c1de11a30b79b4d301e8da93',
|
||||
'series': 'Upper Middle Bogan',
|
||||
'uploader_id': 'abc1',
|
||||
'upload_date': '20210630',
|
||||
'timestamp': 1625036400,
|
||||
},
|
||||
'params': {
|
||||
'noplaylist': True,
|
||||
'skip_download': 'm3u8',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
show_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, show_id)
|
||||
webpage_data = self._search_regex(
|
||||
r'window\.__INITIAL_STATE__\s*=\s*[\'"](.+?)[\'"]\s*;',
|
||||
webpage, 'initial state')
|
||||
video_data = self._parse_json(
|
||||
unescapeHTML(webpage_data).encode('utf-8').decode('unicode_escape'), show_id)
|
||||
video_data = video_data['route']['pageData']['_embedded']
|
||||
|
||||
highlight = try_get(video_data, lambda x: x['highlightVideo']['shareUrl'])
|
||||
if not self._yes_playlist(show_id, bool(highlight), video_label='highlight video'):
|
||||
return self.url_result(highlight, ie=ABCIViewIE.ie_key())
|
||||
|
||||
series = video_data['selectedSeries']
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'entries': [self.url_result(episode['shareUrl'])
|
||||
for episode in series['_embedded']['videoEpisodes']],
|
||||
'id': series.get('id'),
|
||||
'title': dict_get(series, ('title', 'displaySubtitle')),
|
||||
'description': series.get('description'),
|
||||
'series': dict_get(series, ('showTitle', 'displayTitle')),
|
||||
'season': dict_get(series, ('title', 'displaySubtitle')),
|
||||
'thumbnail': series.get('thumbnail'),
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user