itdominator
/
SolarFM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Made searcher grep more like wrapper, addeed timestamp ignore

This commit is contained in:
itdominator 2022-11-29 22:34:25 -06:00
parent f2090a7d46
commit e4e5e08cb4
6 changed files with 75 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
plugins/disk_usage/plugin.py
View File

@ -76,9 +76,10 @@ class Plugin(PluginBase):
path = self._fm_state.tab.get_current_directory()
# NOTE: -h = human readable, -d = depth asigned to 1
command = ["du", "-h", "-d", "1", path]
proc = subprocess.Popen(command, stdout=subprocess.PIPE)
proc = subprocess.Popen(command, stdout=subprocess.PIPE, encoding="utf-8")
raw_data = proc.communicate()[0]
data = raw_data.decode("utf-8").strip() # NOTE: Will return data AFTER completion (if any)
# NOTE: Will return data AFTER completion (if any)
data = raw_data.strip()
parts = data.split("\n")
# NOTE: Last entry is curret dir. Move to top of list and pop off...

7
plugins/searcher/mixins/file_search_mixin.py
View File

@ -4,6 +4,7 @@ import subprocess
import signal
import json
import shlex
from datetime import datetime
# Lib imports
import gi
@ -38,8 +39,10 @@ class FileSearchMixin:
@daemon_threaded
def _handle_find_file_query(self, widget=None, eve=None, query=None):
# NOTE: Freeze IPC consumption
self.pause_fifo_update = True
self.search_query = ""
self.pause_fifo_update = True
self.search_query = ""
dt = datetime.now()
self.fsearch_time_stamp = datetime.timestamp(dt) # NOTE: Get timestamp
# NOTE: Kill the former process
if self._list_proc:

3
plugins/searcher/mixins/grep_search_mixin.py
View File

@ -5,6 +5,7 @@ import subprocess
import signal
import json
import shlex
from datetime import datetime
libgcc_s = ctypes.CDLL('libgcc_s.so.1')
# Lib imports
@ -42,6 +43,8 @@ class GrepSearchMixin:
# NOTE: Freeze IPC consumption
self.pause_fifo_update = True
self.grep_query = ""
dt = datetime.now()
self.grep_time_stamp = datetime.timestamp(dt) # NOTE: Get timestamp
# NOTE: Kill the former process
if self._grep_proc:

32
plugins/searcher/plugin.py
View File

@ -37,23 +37,25 @@ class Plugin(IPCServer, FileSearchMixin, GrepSearchMixin, PluginBase):
def __init__(self):
super().__init__()
self.path = os.path.dirname(os.path.realpath(__file__))
self.name = "Search" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
# where self.name should not be needed for message comms
self._GLADE_FILE = f"{self.path}/search_dialog.glade"
self.path = os.path.dirname(os.path.realpath(__file__))
self.name = "Search" # NOTE: Need to remove after establishing private bidirectional 1-1 message bus
# where self.name should not be needed for message comms
self._GLADE_FILE = f"{self.path}/search_dialog.glade"
self._search_dialog = None
self._active_path = None
self.file_list_parent = None
self.grep_list_parent = None
self._file_list = None
self._grep_list = None
self._grep_proc = None
self._list_proc = None
self.pause_fifo_update = False
self.update_list_ui_buffer = ()
self.grep_query = ""
self.search_query = ""
self._search_dialog = None
self._active_path = None
self.file_list_parent = None
self.grep_list_parent = None
self._file_list = None
self._grep_list = None
self._grep_proc = None
self._list_proc = None
self.pause_fifo_update = False
self.grep_time_stamp = None
self.fsearch_time_stamp = None
self.grep_query = ""
self.search_query = ""
def run(self):

20
plugins/searcher/utils/ipc_server.py
View File

@ -60,14 +60,22 @@ class IPCServer:
msg = conn.recv()
if "SEARCH|" in msg:
file = msg.split("SEARCH|")[1].strip()
if file:
GLib.idle_add(self._load_file_ui, file, priority=GLib.PRIORITY_LOW)
ts, file = msg.split("SEARCH|")[1].strip().split("|", 1)
try:
timestamp = float(ts)
if timestamp > self.fsearch_time_stamp and file:
GLib.idle_add(self._load_file_ui, file, priority=GLib.PRIORITY_LOW)
except Exception as e:
...
if "GREP|" in msg:
data = msg.split("GREP|")[1].strip()
if data:
GLib.idle_add(self._load_grep_ui, data, priority=GLib.PRIORITY_LOW)
ts, data = msg.split("GREP|")[1].strip().split("|", 1)
try:
timestamp = float(ts)
if timestamp > self.grep_time_stamp and data:
GLib.idle_add(self._load_grep_ui, data, priority=GLib.PRIORITY_LOW)
except Exception as e:
...
conn.close()

107
plugins/searcher/utils/search.py
View File

@ -5,11 +5,11 @@
import os
import traceback
import argparse
import threading
import subprocess
import json
import base64
import time
import pickle
from datetime import datetime
from setproctitle import setproctitle
from multiprocessing.connection import Client
@ -26,18 +26,10 @@ _ipc_authkey = b'' + bytes(f'solarfm-search_grep-ipc', 'utf-8')
filter = (".cpp", ".css", ".c", ".go", ".html", ".htm", ".java", ".js", ".json", ".lua", ".md", ".py", ".rs", ".toml", ".xml", ".pom") + \
(".txt", ".text", ".sh", ".cfg", ".conf", ".log")
# NOTE: Threads WILL NOT die with parent's destruction.
def threaded(fn):
def wrapper(*args, **kwargs):
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=False).start()
return wrapper
# NOTE: Threads WILL die with parent's destruction.
def daemon_threaded(fn):
def wrapper(*args, **kwargs):
threading.Thread(target=fn, args=args, kwargs=kwargs, daemon=True).start()
return wrapper
# NOTE: Create timestamp of when this launched. Is used in IPC to see if
# we are stale and that new call didn't fully kill this or older processes.
dt = datetime.now()
ts = datetime.timestamp(dt)
def send_ipc_message(message) -> None:
@ -55,84 +47,51 @@ def file_search(path, query):
for file in _files:
if query in file.lower():
target = os.path.join(_path, file)
data = f"SEARCH|{json.dumps([target, file])}"
data = f"SEARCH|{ts}|{json.dumps([target, file])}"
send_ipc_message(data)
except Exception as e:
print("Couldn't traverse to path. Might be permissions related...")
traceback.print_exc()
def _search_for_string(file, query):
b64_file = base64.urlsafe_b64encode(file.encode('utf-8')).decode('utf-8')
grep_result_set = {}
padding = 15
with open(file, 'rb') as fp:
# NOTE: I know there's an issue if there's a very large file with content
# all on one line will lower and dupe it. And, yes, it will only
# return one instance from the file.
try:
for i, raw in enumerate(fp):
line = None
llower = raw.lower()
if not query in llower:
continue
def grep_search(target=None, query=None):
if not query or not target:
return
if len(raw) > 72:
start = 0
end = len(raw) - 1
index = llower.index(query)
sindex = llower.index(query) - 15 if index >= 15 else abs(start - index) - index
eindex = sindex + 15 if end > (index + 15) else abs(index - end) + index
line = raw[sindex:eindex]
else:
line = raw
# NOTE: -n = provide line numbers, -R = Search recursive in given target
# -i = insensitive, -F = don't do regex parsing. (Treat as raw string)
command = ["grep", "-n", "-R", "-i", "-F", query, target]
proc = subprocess.Popen(command, stdout=subprocess.PIPE, encoding="utf-8")
raw_data = proc.communicate()[0].strip()
proc_data = raw_data.split("\n") # NOTE: Will return data AFTER completion (if any)
collection = {}
b64_line = base64.urlsafe_b64encode(line).decode('utf-8')
if f"{b64_file}" in grep_result_set.keys():
grep_result_set[f"{b64_file}"][f"{i+1}"] = b64_line
else:
grep_result_set[f"{b64_file}"] = {}
grep_result_set[f"{b64_file}"] = {f"{i+1}": b64_line}
for line in proc_data:
file, line_no, data = line.split(":", 2)
b64_file = base64.urlsafe_b64encode(file.encode('utf-8')).decode('utf-8')
b64_data = base64.urlsafe_b64encode(data.encode('utf-8')).decode('utf-8')
except Exception as e:
...
if b64_file in collection.keys():
collection[f"{b64_file}"][f"{line_no}"] = b64_data
else:
collection[f"{b64_file}"] = {}
collection[f"{b64_file}"] = { f"{line_no}": b64_data}
try:
data = f"GREP|{json.dumps(grep_result_set)}"
send_ipc_message(data)
except Exception as e:
...
@daemon_threaded
def _search_for_string_threaded(file, query):
_search_for_string(file, query)
def grep_search(path, query):
try:
for file in os.listdir(path):
target = os.path.join(path, file)
if os.path.isdir(target):
grep_search(target, query)
else:
if target.lower().endswith(filter):
size = os.path.getsize(target)
if not size > 5000:
_search_for_string(target, query)
else:
_search_for_string_threaded(target, query)
data = f"GREP|{ts}|{json.dumps(collection, separators=(',', ':'), indent=4)}"
send_ipc_message(data)
except Exception as e:
print("Couldn't traverse to path. Might be permissions related...")
traceback.print_exc()
...
collection = {}
def search(args):
if args.type == "file_search":
file_search(args.dir, args.query.lower())
if args.type == "grep_search":
grep_search(args.dir, args.query.lower().encode("utf-8"))
grep_search(args.dir, args.query.encode("utf-8"))
if __name__ == "__main__":