Reworked the search logic

plugins/searcher/plugin.py

@@ -1,16 +1,17 @@
 # Python imports
-import os, multiprocessing, threading, subprocess, inspect, time, json
+import os, threading, subprocess, inspect, time, json, base64, shlex, select, signal, pickle
-from multiprocessing import Manager, Process
 
 # Lib imports
 import gi
 gi.require_version('Gtk', '3.0')
-from gi.repository import Gtk, GLib, GObject
+from gi.repository import Gtk, GLib
 
 # Application imports
 from plugins.plugin_base import PluginBase
 
 
+
+
 # NOTE: Threads WILL NOT die with parent's destruction.
 def threaded(fn):
     def wrapper(*args, **kwargs):
@@ -35,19 +36,21 @@ class FilePreviewWidget(Gtk.LinkButton):
 
 
 class GrepPreviewWidget(Gtk.Box):
-    def __init__(self, path, sub_keys, data):
+    def __init__(self, _path, sub_keys, data):
         super(GrepPreviewWidget, self).__init__()
         self.set_orientation(Gtk.Orientation.VERTICAL)
         self.line_color = "#e0cc64"
 
-
+        path   = base64.urlsafe_b64decode(_path.encode('utf-8')).decode('utf-8')
         _label = '/'.join( path.split("/")[-3:] )
         title  = Gtk.LinkButton.new_with_label(uri=f"file://{path}", label=_label)
 
         self.add(title)
         for key in sub_keys:
             line_num     = key
-            text         = data[key]
+            text = base64.urlsafe_b64decode(data[key].encode('utf-8')).decode('utf-8')
+
+
             box          = Gtk.Box()
             number_label = Gtk.Label()
             text_view    = Gtk.Label(label=text[:-1])
@@ -69,11 +72,7 @@ class GrepPreviewWidget(Gtk.Box):
         self.show_all()
 
 
-
+pause_fifo_update = False
-manager  = Manager()
-grep_result_set  = manager.dict()
-file_result_set  = manager.list()
-
 
 class Plugin(PluginBase):
     def __init__(self):
@@ -84,6 +83,9 @@ class Plugin(PluginBase):
                                            #       where self.name should not be needed for message comms
         self._GLADE_FILE       = f"{self.path}/search_dialog.glade"
 
+        self._files_fifo_file  = f"/tmp/search_files_fifo"
+        self._grep_fifo_file   = f"/tmp/grep_files_fifo"
+
         self._search_dialog    = None
         self._active_path      = None
         self._file_list        = None
@@ -116,113 +118,118 @@ class Plugin(PluginBase):
         self._search_dialog = self._builder.get_object("search_dialog")
         self._grep_list     = self._builder.get_object("grep_list")
         self._file_list     = self._builder.get_object("file_list")
+        self.fsearch        = self._builder.get_object("fsearch")
 
-        GObject.signal_new("update-file-ui-signal", self._search_dialog, GObject.SIGNAL_RUN_LAST, GObject.TYPE_PYOBJECT, (GObject.TYPE_PYOBJECT,))
+        self._event_system.subscribe("update-file-ui", self._load_file_ui)
-        self._search_dialog.connect("update-file-ui-signal", self._load_file_ui)
+        self._event_system.subscribe("update-grep-ui", self._load_grep_ui)
-        GObject.signal_new("update-grep-ui-signal", self._search_dialog, GObject.SIGNAL_RUN_LAST, GObject.TYPE_PYOBJECT, (GObject.TYPE_PYOBJECT,))
+
-        self._search_dialog.connect("update-grep-ui-signal", self._load_grep_ui)
+        if not os.path.exists(self._files_fifo_file):
+            os.mkfifo(self._files_fifo_file, 0o777)
+        if not os.path.exists(self._grep_fifo_file):
+            os.mkfifo(self._grep_fifo_file, 0o777)
+
+        self.run_files_fifo_thread()
+        self.run_grep_fifo_thread()
 
 
     @daemon_threaded
+    def run_files_fifo_thread(self):
+        with open(self._files_fifo_file) as fifo:
+            while True:
+                select.select([fifo],[],[fifo])
+                data = fifo.read()
+                GLib.idle_add(self._load_file_ui, data)
+
+    @daemon_threaded
+    def run_grep_fifo_thread(self):
+        with open(self._grep_fifo_file) as fifo:
+            while True:
+                select.select([fifo],[],[fifo])
+                data = fifo.read()
+                GLib.idle_add(self._load_grep_ui, data)
+
+
     def _show_grep_list_page(self, widget=None, eve=None):
-        self._event_system.emit("get_current_state
+        self._event_system.emit("get_current_state")
 
         state               = self._fm_state
         self._event_message = None
 
-        GLib.idle_add(self._process_queries, (state))
-
-    def _process_queries(self, state):
         self._active_path   = state.tab.get_current_directory()
         response            = self._search_dialog.run()
         self._search_dialog.hide()
 
 
     def _run_find_file_query(self, widget=None, eve=None):
-        if self._list_proc:
+        self._stop_find_file_query()
-            self._list_proc.terminate()
-            self._list_proc = None
-            time.sleep(.2)
-
-        del file_result_set[:]
-        self.clear_children(self._file_list)
 
         query = widget.get_text()
-        if query:
+        if not query in ("", None):
-            self._list_proc = multiprocessing.Process(self._do_list_search(self._active_path, query))
+            target_dir = shlex.quote( self._fm_state.tab.get_current_directory() )
-            self._list_proc.start()
+            # command = [f"{self.path}/search.sh", "-t", "file_search", "-d", f"{target_dir}", "-q", f"{query}"]
+            command = ["python", f"{self.path}/search.py", "-t", "file_search", "-d", f"{target_dir}", "-q", f"{query}"]
 
-    def _do_list_search(self, path, query):
+            process = subprocess.Popen(command, cwd=self.path, stdin=None, stdout=None, stderr=None)
-        self._file_traverse_path(path, query)
-        for target, file in file_result_set:
-            widget = FilePreviewWidget(target, file)
-            self._search_dialog.emit("update-file-ui-signal", (widget))
 
-    def _load_file_ui(self, parent=None, widget=None):
+    def _stop_find_file_query(self, widget=None, eve=None):
-        self._file_list.add(widget)
+        pause_fifo_update = True
 
-    def _file_traverse_path(self, path, query):
+        if self._list_proc:
-        try:
+            if self._list_proc.poll():
-            for file in os.listdir(path):
+                self._list_proc.send_signal(signal.SIGKILL)
-                target = os.path.join(path, file)
+                while self._list_proc.poll():
-                if os.path.isdir(target):
+                    pass
-                    self._file_traverse_path(target, query)
+
-                else:
+                self._list_proc = None
-                    if query.lower() in file.lower():
+            else:
-                        file_result_set.append([target, file])
+                self._list_proc = None
-        except Exception as e:
+
-            if debug:
+        self.clear_children(self._file_list)
-                print("Couldn't traverse to path. Might be permissions related...")
+        pause_fifo_update = False
 
 
     def _run_grep_query(self, widget=None, eve=None):
-        if self._grep_proc:
+        self._stop_grep_query()
-            self._grep_proc.terminate()
-            self._grep_proc = None
-            time.sleep(.2)
-
-        grep_result_set.clear()
-        self.clear_children(self._grep_list)
 
         query = widget.get_text()
-        if query:
+        if not query in ("", None):
-            self._grep_proc = multiprocessing.Process(self._do_grep_search(self._active_path, query))
+            target_dir = shlex.quote( self._fm_state.tab.get_current_directory() )
-            self._grep_proc.start()
+            command = ["python", f"{self.path}/search.py", "-t", "grep_search", "-d", f"{target_dir}", "-q", f"{query}"]
+            process = subprocess.Popen(command, cwd=self.path, stdin=None, stdout=None, stderr=None)
 
-    def _do_grep_search(self, path, query):
+    def _stop_grep_query(self, widget=None, eve=None):
-        self._grep_traverse_path(path, query)
+        pause_fifo_update = True
 
-        keys = grep_result_set.keys()
+        if self._grep_proc:
-        for key in keys:
+            if self._grep_proc.poll():
-            sub_keys = grep_result_set[key].keys()
+                self._grep_proc.send_signal(signal.SIGKILL)
-            widget   = GrepPreviewWidget(key, sub_keys, grep_result_set[key])
+                while self._grep_proc.poll():
-            self._search_dialog.emit("update-grep-ui-signal", (widget))
+                    pass
 
-    def _load_grep_ui(self, parent=None, widget=None):
+                self._grep_proc = None
-        self._grep_list.add(widget)
+            else:
+                self._grep_proc = None
 
-    def _grep_traverse_path(self, path, query):
+        self.clear_children(self._grep_list)
-        try:
+        pause_fifo_update = False
-            for file in os.listdir(path):
-                target = os.path.join(path, file)
-                if os.path.isdir(target):
-                    self._grep_traverse_path(target, query)
-                else:
-                    self._search_for_string(target, query)
-        except Exception as e:
-            if debug:
-                print("Couldn't traverse to path. Might be permissions related...")
 
-    def _search_for_string(self, file, query):
+
-        try:
+    def _load_file_ui(self, data):
-            with open(file, 'r') as fp:
+        if not data in ("", None) and not pause_fifo_update:
-                for i, line in enumerate(fp):
+            jdata  = json.loads( data )
-                    if query in line:
+            target = jdata[0]
-                        if f"{file}" in grep_result_set.keys():
+            file   = jdata[1]
-                            grep_result_set[f"{file}"][f"{i+1}"] = line
+
-                        else:
+            widget = FilePreviewWidget(target, file)
-                            grep_result_set[f"{file}"] = {}
+            self._file_list.add(widget)
-                            grep_result_set[f"{file}"] = {f"{i+1}": line}
+
-        except Exception as e:
+    def _load_grep_ui(self, data):
-            if debug:
+        if not data in ("", None) and not pause_fifo_update:
-                print("Couldn't read file. Might be binary or other cause...")
+            jdata = json.loads( data )
+            jkeys = jdata.keys()
+            for key in jkeys:
+                sub_keys    = jdata[key].keys()
+                grep_result = jdata[key]
+
+                widget = GrepPreviewWidget(key, sub_keys, grep_result)
+                self._grep_list.add(widget)

plugins/searcher/search.py

@@ -0,0 +1,103 @@
+#!/usr/bin/python3
+
+
+# Python imports
+import os, traceback, argparse, time, json, base64
+from setproctitle import setproctitle
+
+# Lib imports
+
+# Application imports
+
+
+
+_files_fifo_file  = f"/tmp/search_files_fifo"
+_grep_fifo_file   = f"/tmp/grep_files_fifo"
+
+filter = (".mkv", ".mp4", ".webm", ".avi", ".mov", ".m4v", ".mpg", ".mpeg", ".wmv", ".flv") + \
+            (".png", ".jpg", ".jpeg", ".gif", ".ico", ".tga", ".webp") + \
+            (".psf", ".mp3", ".ogg", ".flac", ".m4a")
+
+file_result_set = []
+
+
+def file_search(fifo, path, query):
+    try:
+        for file in os.listdir(path):
+            target = os.path.join(path, file)
+            if os.path.isdir(target):
+                file_search(fifo, target, query)
+            else:
+                if query.lower() in file.lower():
+                    # file_result_set.append([target, file])
+                    data = json.dumps([target, file])
+                    fifo.write(data)
+                    time.sleep(0.01)
+    except Exception as e:
+        print("Couldn't traverse to path. Might be permissions related...")
+        traceback.print_exc()
+
+def _search_for_string(file, query):
+    try:
+        b64_file = base64.urlsafe_b64encode(file.encode('utf-8')).decode('utf-8')
+        grep_result_set = {}
+
+        with open(file, 'r') as fp:
+            for i, line in enumerate(fp):
+                if query in line:
+                    b64_line = base64.urlsafe_b64encode(line.encode('utf-8')).decode('utf-8')
+
+                    if f"{b64_file}" in grep_result_set.keys():
+                        grep_result_set[f"{b64_file}"][f"{i+1}"] = b64_line
+                    else:
+                        grep_result_set[f"{b64_file}"] = {}
+                        grep_result_set[f"{b64_file}"] = {f"{i+1}": b64_line}
+
+            # NOTE: Push to fifo here after loop
+        with open(_grep_fifo_file, 'w') as fifo:
+            data = json.dumps(grep_result_set)
+            fifo.write(data)
+            time.sleep(0.05)
+    except Exception as e:
+        print("Couldn't read file. Might be binary or other cause...")
+        traceback.print_exc()
+
+
+def grep_search(path, query):
+    try:
+        for file in os.listdir(path):
+            target = os.path.join(path, file)
+            if os.path.isdir(target):
+                grep_search(target, query)
+            else:
+                if not target.lower().endswith(filter):
+                        _search_for_string(target, query)
+    except Exception as e:
+        print("Couldn't traverse to path. Might be permissions related...")
+        traceback.print_exc()
+
+def search(args):
+    if args.type == "file_search":
+       with open(_files_fifo_file, 'w') as fifo:
+           file_search(fifo, args.dir, args.query)
+
+    if args.type == "grep_search":
+        grep_search(args.dir, args.query)
+
+
+if __name__ == "__main__":
+    try:
+        setproctitle('SolarFM: File Search - Grepy')
+
+        parser = argparse.ArgumentParser()
+        # Add long and short arguments
+        parser.add_argument("--type", "-t", default=None, help="Type of search to do.")
+        parser.add_argument("--dir", "-d", default=None, help="Directory root for search type.")
+        parser.add_argument("--query", "-q", default=None, help="Query search is working against.")
+
+        # Read arguments (If any...)
+        args = parser.parse_args()
+        search(args)
+
+    except Exception as e:
+        traceback.print_exc()

plugins/searcher/search_dialog.glade

@@ -74,7 +74,7 @@
                 <property name="can-focus">False</property>
                 <property name="orientation">vertical</property>
                 <child>
-                  <object class="GtkSearchEntry">
+                  <object class="GtkSearchEntry" id="fsearch">
                     <property name="visible">True</property>
                     <property name="can-focus">True</property>
                     <property name="tooltip-text" translatable="yes">Query...</property>
@@ -83,6 +83,7 @@
                     <property name="primary-icon-sensitive">False</property>
                     <property name="placeholder-text" translatable="yes">Search for file...</property>
                     <signal name="search-changed" handler="_run_find_file_query" swapped="no"/>
+                    <signal name="stop-search" handler="_stop_find_file_query" swapped="no"/>
                   </object>
                   <packing>
                     <property name="expand">False</property>
@@ -149,6 +150,7 @@
                     <property name="primary-icon-sensitive">False</property>
                     <property name="placeholder-text" translatable="yes">Query string in file...</property>
                     <signal name="search-changed" handler="_run_grep_query" swapped="no"/>
+                    <signal name="stop-search" handler="_stop_grep_query" swapped="no"/>
                   </object>
                   <packing>
                     <property name="expand">False</property>