From e4e5e08cb44d7002f8892cb8456590dc2ab57ce6 Mon Sep 17 00:00:00 2001 From: itdominator <1itdominator@gmail.com> Date: Tue, 29 Nov 2022 22:34:25 -0600 Subject: [PATCH] Made searcher grep more like wrapper, addeed timestamp ignore --- plugins/disk_usage/plugin.py | 5 +- plugins/searcher/mixins/file_search_mixin.py | 7 +- plugins/searcher/mixins/grep_search_mixin.py | 3 + plugins/searcher/plugin.py | 32 +++--- plugins/searcher/utils/ipc_server.py | 20 ++-- plugins/searcher/utils/search.py | 107 ++++++------------- 6 files changed, 75 insertions(+), 99 deletions(-) diff --git a/plugins/disk_usage/plugin.py b/plugins/disk_usage/plugin.py index ec1c39d..65b9fa7 100644 --- a/plugins/disk_usage/plugin.py +++ b/plugins/disk_usage/plugin.py @@ -76,9 +76,10 @@ class Plugin(PluginBase): path = self._fm_state.tab.get_current_directory() # NOTE: -h = human readable, -d = depth asigned to 1 command = ["du", "-h", "-d", "1", path] - proc = subprocess.Popen(command, stdout=subprocess.PIPE) + proc = subprocess.Popen(command, stdout=subprocess.PIPE, encoding="utf-8") raw_data = proc.communicate()[0] - data = raw_data.decode("utf-8").strip() # NOTE: Will return data AFTER completion (if any) + # NOTE: Will return data AFTER completion (if any) + data = raw_data.strip() parts = data.split("\n") # NOTE: Last entry is curret dir. Move to top of list and pop off... diff --git a/plugins/searcher/mixins/file_search_mixin.py b/plugins/searcher/mixins/file_search_mixin.py index a72be13..6837b20 100644 --- a/plugins/searcher/mixins/file_search_mixin.py +++ b/plugins/searcher/mixins/file_search_mixin.py @@ -4,6 +4,7 @@ import subprocess import signal import json import shlex +from datetime import datetime # Lib imports import gi @@ -38,8 +39,10 @@ class FileSearchMixin: @daemon_threaded def _handle_find_file_query(self, widget=None, eve=None, query=None): # NOTE: Freeze IPC consumption - self.pause_fifo_update = True - self.search_query = "" + self.pause_fifo_update = True + self.search_query = "" + dt = datetime.now() + self.fsearch_time_stamp = datetime.timestamp(dt) # NOTE: Get timestamp # NOTE: Kill the former process if self._list_proc: diff --git a/plugins/searcher/mixins/grep_search_mixin.py b/plugins/searcher/mixins/grep_search_mixin.py index 9f7f1d9..3174cd0 100644 --- a/plugins/searcher/mixins/grep_search_mixin.py +++ b/plugins/searcher/mixins/grep_search_mixin.py @@ -5,6 +5,7 @@ import subprocess import signal import json import shlex +from datetime import datetime libgcc_s = ctypes.CDLL('libgcc_s.so.1') # Lib imports @@ -42,6 +43,8 @@ class GrepSearchMixin: # NOTE: Freeze IPC consumption self.pause_fifo_update = True self.grep_query = "" + dt = datetime.now() + self.grep_time_stamp = datetime.timestamp(dt) # NOTE: Get timestamp # NOTE: Kill the former process if self._grep_proc: diff --git a/plugins/searcher/plugin.py b/plugins/searcher/plugin.py index 774a113..391fe1b 100644 --- a/plugins/searcher/plugin.py +++ b/plugins/searcher/plugin.py @@ -37,23 +37,25 @@ class Plugin(IPCServer, FileSearchMixin, GrepSearchMixin, PluginBase): def __init__(self): super().__init__() - self.path = os.path.dirname(os.path.realpath(__file__)) - self.name = "Search" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus - # where self.name should not be needed for message comms - self._GLADE_FILE = f"{self.path}/search_dialog.glade" + self.path = os.path.dirname(os.path.realpath(__file__)) + self.name = "Search" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus + # where self.name should not be needed for message comms + self._GLADE_FILE = f"{self.path}/search_dialog.glade" - self._search_dialog = None - self._active_path = None - self.file_list_parent = None - self.grep_list_parent = None - self._file_list = None - self._grep_list = None - self._grep_proc = None - self._list_proc = None - self.pause_fifo_update = False self.update_list_ui_buffer = () - self.grep_query = "" - self.search_query = "" + self._search_dialog = None + self._active_path = None + self.file_list_parent = None + self.grep_list_parent = None + self._file_list = None + self._grep_list = None + self._grep_proc = None + self._list_proc = None + self.pause_fifo_update = False + self.grep_time_stamp = None + self.fsearch_time_stamp = None + self.grep_query = "" + self.search_query = "" def run(self): diff --git a/plugins/searcher/utils/ipc_server.py b/plugins/searcher/utils/ipc_server.py index e4ca2aa..b8b0b14 100644 --- a/plugins/searcher/utils/ipc_server.py +++ b/plugins/searcher/utils/ipc_server.py @@ -60,14 +60,22 @@ class IPCServer: msg = conn.recv() if "SEARCH|" in msg: - file = msg.split("SEARCH|")[1].strip() - if file: - GLib.idle_add(self._load_file_ui, file, priority=GLib.PRIORITY_LOW) + ts, file = msg.split("SEARCH|")[1].strip().split("|", 1) + try: + timestamp = float(ts) + if timestamp > self.fsearch_time_stamp and file: + GLib.idle_add(self._load_file_ui, file, priority=GLib.PRIORITY_LOW) + except Exception as e: + ... if "GREP|" in msg: - data = msg.split("GREP|")[1].strip() - if data: - GLib.idle_add(self._load_grep_ui, data, priority=GLib.PRIORITY_LOW) + ts, data = msg.split("GREP|")[1].strip().split("|", 1) + try: + timestamp = float(ts) + if timestamp > self.grep_time_stamp and data: + GLib.idle_add(self._load_grep_ui, data, priority=GLib.PRIORITY_LOW) + except Exception as e: + ... conn.close() diff --git a/plugins/searcher/utils/search.py b/plugins/searcher/utils/search.py index 5d833be..a61eb12 100755 --- a/plugins/searcher/utils/search.py +++ b/plugins/searcher/utils/search.py @@ -5,11 +5,11 @@ import os import traceback import argparse -import threading +import subprocess import json import base64 import time -import pickle +from datetime import datetime from setproctitle import setproctitle from multiprocessing.connection import Client @@ -26,18 +26,10 @@ _ipc_authkey = b'' + bytes(f'solarfm-search_grep-ipc', 'utf-8') filter = (".cpp", ".css", ".c", ".go", ".html", ".htm", ".java", ".js", ".json", ".lua", ".md", ".py", ".rs", ".toml", ".xml", ".pom") + \ (".txt", ".text", ".sh", ".cfg", ".conf", ".log") - -# NOTE: Threads WILL NOT die with parent's destruction. -def threaded(fn): - def wrapper(*args, **kwargs): - threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start() - return wrapper - -# NOTE: Threads WILL die with parent's destruction. -def daemon_threaded(fn): - def wrapper(*args, **kwargs): - threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start() - return wrapper +# NOTE: Create timestamp of when this launched. Is used in IPC to see if +# we are stale and that new call didn't fully kill this or older processes. +dt = datetime.now() +ts = datetime.timestamp(dt) def send_ipc_message(message) -> None: @@ -55,84 +47,51 @@ def file_search(path, query): for file in _files: if query in file.lower(): target = os.path.join(_path, file) - data = f"SEARCH|{json.dumps([target, file])}" + data = f"SEARCH|{ts}|{json.dumps([target, file])}" send_ipc_message(data) except Exception as e: print("Couldn't traverse to path. Might be permissions related...") traceback.print_exc() -def _search_for_string(file, query): - b64_file = base64.urlsafe_b64encode(file.encode('utf-8')).decode('utf-8') - grep_result_set = {} - padding = 15 - with open(file, 'rb') as fp: - # NOTE: I know there's an issue if there's a very large file with content - # all on one line will lower and dupe it. And, yes, it will only - # return one instance from the file. - try: - for i, raw in enumerate(fp): - line = None - llower = raw.lower() - if not query in llower: - continue +def grep_search(target=None, query=None): + if not query or not target: + return - if len(raw) > 72: - start = 0 - end = len(raw) - 1 - index = llower.index(query) - sindex = llower.index(query) - 15 if index >= 15 else abs(start - index) - index - eindex = sindex + 15 if end > (index + 15) else abs(index - end) + index - line = raw[sindex:eindex] - else: - line = raw + # NOTE: -n = provide line numbers, -R = Search recursive in given target + # -i = insensitive, -F = don't do regex parsing. (Treat as raw string) + command = ["grep", "-n", "-R", "-i", "-F", query, target] + proc = subprocess.Popen(command, stdout=subprocess.PIPE, encoding="utf-8") + raw_data = proc.communicate()[0].strip() + proc_data = raw_data.split("\n") # NOTE: Will return data AFTER completion (if any) + collection = {} - b64_line = base64.urlsafe_b64encode(line).decode('utf-8') - if f"{b64_file}" in grep_result_set.keys(): - grep_result_set[f"{b64_file}"][f"{i+1}"] = b64_line - else: - grep_result_set[f"{b64_file}"] = {} - grep_result_set[f"{b64_file}"] = {f"{i+1}": b64_line} + for line in proc_data: + file, line_no, data = line.split(":", 2) + b64_file = base64.urlsafe_b64encode(file.encode('utf-8')).decode('utf-8') + b64_data = base64.urlsafe_b64encode(data.encode('utf-8')).decode('utf-8') - except Exception as e: - ... + if b64_file in collection.keys(): + collection[f"{b64_file}"][f"{line_no}"] = b64_data + else: + collection[f"{b64_file}"] = {} + collection[f"{b64_file}"] = { f"{line_no}": b64_data} - try: - data = f"GREP|{json.dumps(grep_result_set)}" - send_ipc_message(data) - except Exception as e: - ... - - - -@daemon_threaded -def _search_for_string_threaded(file, query): - _search_for_string(file, query) - -def grep_search(path, query): try: - for file in os.listdir(path): - target = os.path.join(path, file) - if os.path.isdir(target): - grep_search(target, query) - else: - if target.lower().endswith(filter): - size = os.path.getsize(target) - if not size > 5000: - _search_for_string(target, query) - else: - _search_for_string_threaded(target, query) - + data = f"GREP|{ts}|{json.dumps(collection, separators=(',', ':'), indent=4)}" + send_ipc_message(data) except Exception as e: - print("Couldn't traverse to path. Might be permissions related...") - traceback.print_exc() + ... + + collection = {} + def search(args): if args.type == "file_search": file_search(args.dir, args.query.lower()) if args.type == "grep_search": - grep_search(args.dir, args.query.lower().encode("utf-8")) + grep_search(args.dir, args.query.encode("utf-8")) if __name__ == "__main__":