feat: ServerModelManager WIP

dzgui/controllers/mc.py

@@ -43,8 +43,9 @@ from dzgui.config import update
 from dzgui.config.query import lookup
 from dzgui.config.userprefs import UserPrefs
 from dzgui.controllers.emitter import Emitter
-from dzgui.model.contextmenu import ContextMenuManager
+from dzgui.managers.contextmenu import ContextMenuManager
 from dzgui.model.filtered_model import FilteredModelManager
+from dzgui.model.servers import ServerModelManager
 from dzgui.model.model_factory import ModelFactory
 from dzgui.util import strings
 from dzgui.util._json import read_json, write_json
@@ -111,7 +112,6 @@ class Controller(GObject.GObject):
         self.prefs: UserPrefs
         self.cleanup_func: StoredFunc = None
 
-        #self.model_man = ModelManager()
         self.emitter = Emitter()
         self.emitter.connect("map_selection_changed", self._on_map_selection_changed)
         self.emitter.connect("check_toggled", self._on_check_toggled)
@@ -481,7 +481,7 @@ class Controller(GObject.GObject):
         # FIXME: refiltration should occur in thread
         model.append(self.insert_record[0])
         filtered = filter_man.filter(FilterMode.INITIAL)
-        treeview.set_model(filter_man.get_model())
+        treeview.set_model(filter_man.get_proxy_model())
         # TODO: update statusbar
         context = self.get_active_context()
         # TODO: adding a row may update available maps
@@ -640,6 +640,7 @@ class Controller(GObject.GObject):
                 sel.select_path(path)
 
     # TODO: make as method of tree?
+    # TODO: could have a LocalModManager that accepts button enums
     def uncolorize_mods(self) -> None:
         model = self.get_mod_store()
         for mod in model:
@@ -856,7 +857,7 @@ class Controller(GObject.GObject):
         self, filter_man: "FilteredModelManager", mode: FilterMode, label: str
     ) -> None:
         filter_man.filter(mode, label)
-        self.to_insert = filter_man.get_model()
+        self.to_insert = filter_man.get_proxy_model()
         print("filtering threaded")
         self.cleanup_func = StoredFunc(self.cleanup_on_success)
 
@@ -872,31 +873,33 @@ class Controller(GObject.GObject):
         return self.filter_man
 
     def set_filter_man(self, filter_man: "FilteredModelManager") -> None:
+        # TODO: used when staging filter man outside of thread
         self.filter_man = filter_man
 
-    def populate_model(self) -> None:
+    def populate_model(self, tv: Gtk.TreeView) -> None:
         # NOTE: prepare GTK objects outside of thread
-        treeview = self.get_active_treeview()
+        #if tv.is_loaded() is True:
-        if treeview.is_loaded() is True:
+        #    self.emitter.emit("servers_loaded", tv.get_enum())
-            self.emitter.emit("servers_loaded", treeview.get_enum())
+        #    return
-            return
 
-        func, jobs = treeview.get_query_func()
+        #func, jobs = tv.get_query_func()
-        if func is None:
+        ServerModelManager(self, tv, first_iteration=True)
-            self.emitter.emit("servers_loaded", treeview.get_enum())
+
-            treeview.set_model(None)
+        #if func is None:
-            return
+        #    self.emitter.emit("servers_loaded", tv.get_enum())
+        #    tv.set_model(None)
+        #    return
 
         # TODO: clear ephemeral model if necessary
         # manager = treeview.get_filter_man()
         # manager.clear_model()
 
-        self.first_iteration = True
+        # self.first_iteration = True
-        self.pending_jobs = jobs
+        # self.pending_jobs = jobs
-        # TODO: get filter manager a priori and set it so that it can be accessed out of thread
+        # # TODO: get filter manager a priori and set it so that it can be accessed out of thread
-        filter_man = treeview.get_filter_man()
+        # filter_man = treeview.get_filter_man()
-        self.set_filter_man(filter_man)
+        # self.set_filter_man(filter_man)
-        self.run_query_func(func)
+        # self.run_query_func(func)
 
     def get_favorite(self) -> tuple[str, str] | tuple[None, None]:
         fav = str(self.query_config(Preferences.FAV_LBL))
@@ -950,11 +953,16 @@ class Controller(GObject.GObject):
 
     def _on_servers_loaded(self, emitter: "Emitter", tab: "ServerTab") -> None:
         # NOTE: workaround for GTK bug where fullscreen causes headers to vanish when model is None
+        # TODO: this should be internal to servers page
         state = self.has_server_model()
         tv = self.get_active_treeview()
         tv.set_headers_visible(state)
         tv.set_headers_clickable(state)
 
+    def has_server_model(self) -> bool:
+        treeview = self.get_active_treeview()
+        return treeview.get_model() is not None
+
     def _on_check_toggled(self, emitter: Emitter, label: str, state: bool) -> None:
         map_man = self.get_map_man()
         map_man.set_filter(label, state)
@@ -971,10 +979,6 @@ class Controller(GObject.GObject):
         map_man.set_selected_map(selection)
         self.refilter_model(FilterMode.MAP)
 
-    def has_server_model(self) -> bool:
-        treeview = self.get_active_treeview()
-        return treeview.get_model() is not None
-
     def hide_widgets_on_init(self) -> None:
         self.mediator.grid.conpan.set_lan_visible(False)
         self.mediator.grid.right_panel.sel_panel.hide()

dzgui/managers/contextmenu.py

@@ -3,7 +3,7 @@ import logging
 from typing import TYPE_CHECKING
 
 from dzgui.const.enum import ContextMenu, Preferences
-from dzgui.model.thread_man import ThreadingManager
+from dzgui.managers.thread_man import ThreadingManager
 from dzgui.util import strings
 from dzgui.util.open_links import open_workshop_page
 

dzgui/managers/thread_man.py

@@ -47,6 +47,7 @@ class StoredFunc:
 class ThreadingManager:
     def __init__(self, parent: Gtk.Window) -> None:
         self.parent = parent
+        self.jobs = 1
         self.cleanup_func = None
 
     def call_on_thread(self, dialog_str: str, func: StoredFunc) -> None:
@@ -54,11 +55,20 @@ class ThreadingManager:
             func.call()
             GLib.idle_add(self._destroy_on_idle)
 
-        self.wait_dialog = WaitDialog(self.parent, dialog_str)
+        self.wait_dialog = WaitDialog(self.parent, dialog_str, jobs=self.jobs)
         self.wait_dialog.show_all()
         thread = threading.Thread(target=callback)
         thread.start()
 
+    def set_job_count(self, jobs: int) -> None:
+        self.jobs = jobs
+
+    def increment_dialog(self) -> None:
+        GLib.idle_add(self.wait_dialog.increment)
+
+    def increment_dialog_with_str(self, text: str) -> None:
+        GLib.idle_add(lambda: self.wait_dialog.increment(text))
+
     def set_cleanup_func(self, func: StoredFunc) -> None:
         if type(func) not in (StoredFunc, type(None)):
             msg = f"Callback function '{func}' is not of type StoredFunc or None"

dzgui/model/filtered_model.py

@@ -53,7 +53,7 @@ class FilteredModelManager:
         self.filter_cache = {}
         self.ping_cache: dict[str, int] = {}
 
-        self.ephemeral_model: ListStore = None
+        self.proxy_model: ListStore = None
 
         self.control_model: list = None
         self.filtered: list = None
@@ -68,13 +68,13 @@ class FilteredModelManager:
         self.keyword_filter = keyword
 
     def append_row(self, row: list) -> None:
-        self.ephemeral_model.append(row)
+        self.proxy_model.append(row)
 
-    def clear_model(self) -> None:
+    def clear_proxy_model(self) -> None:
-        self.ephemeral_model.clear()
+        self.proxy_model.clear()
 
-    def get_model(self) -> ListStore:
+    def get_proxy_model(self) -> ListStore:
-        return self.ephemeral_model
+        return self.proxy_model
 
     def new_model_from_class(self, cls: type) -> ListStore:
         store = ListStore(*[ftype for field, ftype in cls.__annotations__.items()])
@@ -89,7 +89,7 @@ class FilteredModelManager:
 
         if filters in self.filter_cache:
             cache = self.filter_cache[filters]
-            self.set_model(cache[0])
+            self.set_proxy_model(cache[0])
             self.set_filtered(cache[1])
             return
 
@@ -128,11 +128,13 @@ class FilteredModelManager:
         if len(rows) > 0:
             rows = self.sort_rows(rows)
             for i, row in enumerate(rows):
+                # TODO: consider overriding append() method of Gtk.ListStore
+                # check Gtk source code
                 clone.insert_with_values(i, tuple(range(0, n_cols)), row)
                 #clone.append(row)
 
         self.set_cache(filters, clone, rows)
-        self.set_model(clone)
+        self.set_proxy_model(clone)
         return clone
 
     def sort_rows(self, rows: list) -> list:
@@ -266,11 +268,11 @@ class FilteredModelManager:
     def get_filtered(self) -> list:
         return self.filtered
 
-    def set_model(self, model: ListStore | None) -> None:
+    def set_proxy_model(self, model: ListStore | None) -> None:
         """
-        ListStore representation of the model
+        ListStore representation of the raw model after filtration
         """
-        self.ephemeral_model = model
+        self.proxy_model = model
 
     def set_control(self, rows: list) -> None:
         """
@@ -281,20 +283,20 @@ class FilteredModelManager:
     def get_control(self) -> list:
         return self.control_model
 
-    @deprecated("Legacy code")
+    #@deprecated("Legacy code")
-    def set_success(self, result: bool) -> None:
+    #def set_success(self, result: bool) -> None:
-        self.success = result
+    #    self.success = result
 
-    @deprecated("Legacy code")
+    #@deprecated("Legacy code")
-    def get_success(self) -> bool:
+    #def get_success(self) -> bool:
-        return self.success
+    #    return self.success
 
-    # NOTE: used when adding/removing rows in-situ in the ephemeral model
+    # NOTE: used when adding/removing rows in-situ in the proxy model
     # and syncing changes to control model, but ignored for player count/ping updates
     # cf. remove_from_history(), remove_server()
     # NOTE: this can most likely be simplified for v7
     def wipe_cache(self, full=False) -> None:
-        self.set_success(True)
+        # self.set_success(True)
         self.filtered = None
         self.filter_cache = {}
         self.ping_cache = {}

dzgui/model/servers.py

@@ -0,0 +1,324 @@
+from datetime import datetime
+import logging
+
+from concurrent.futures import wait, as_completed
+from concurrent.futures import ThreadPoolExecutor
+
+import dzgui.api.servers as Servers
+from dzgui.const.constants import (
+    APPID_DAYZ,
+    APPID_DAYZ_EXP,
+)
+from dzgui.const.enum import FilterMode, Preferences, ServerTab
+from dzgui.managers.thread_man import call_on_thread, StoredFunc, ThreadingManager
+from dzgui.util.strings import dialog
+from dzgui.views.dialogs.generic import ExceptionDialog
+
+from typing import Optional, TYPE_CHECKING
+
+import gi
+
+gi.require_version("Gtk", "3.0")
+from gi.repository import Gtk, GLib  # noqa E402
+
+
+if TYPE_CHECKING:
+    from dzgui.controllers.mc import Controller
+    from dzgui.controllers.emitter import Emitter
+    from dzgui.model.filtered_model import FilteredModelManager
+
+logger = logging.getLogger(__name__)
+
+class ServerModelManager:
+    def __init__(self, controller: "Controller", tv: Gtk.TreeView, first_iteration=False) -> None:
+
+        enum = tv.get_enum()
+        self.controller = controller
+        self.emitter = controller.get_emitter()
+        if tv.is_loaded():
+            self.emitter.emit("servers_loaded", enum)
+
+        self.tv = tv
+        self.jobs = 1
+
+        self.filter_man = tv.get_filter_man()
+        self.thread_man = ThreadingManager(parent=controller)
+
+        # TODO: if first iteration, clear filter man control model
+        # literal first load: iteration 1
+        # refresh: iteration 1 (wipe model)
+        # filter: iteration N+1
+        self.first_iteration = first_iteration
+
+        match enum:
+            case ServerTab.BROWSER:
+                # NOTE: extra DAYZ_EXP param
+                self.thread_man.set_job_count(len(Servers.params) + 1)
+                self._dump_api()
+            case ServerTab.SAVED:
+                self._dump_favorites()
+            case ServerTab.RECENT:
+                # TODO: get row count
+                self._dump_history()
+            case ServerTab.LAN:
+                self._dump_lan()
+
+    @call_on_thread(dialog.fetching)
+    def _dump_api(self) -> None:
+        key = self.controller.query_config(Preferences.STEAM)
+        job = Servers.query_api
+        params = Servers.params
+        servers = []
+        with ThreadPoolExecutor() as executor:
+            futures = [executor.submit(job, key, APPID_DAYZ, param) for param in params]
+            for future in as_completed(futures):
+                try:
+                    self.thread_man.increment_dialog()
+                    res = future.result(timeout=3)
+                    if res.status != 200 or not res.parsed:
+                        self.thread_man.set_cleanup_func(StoredFunc(self._cleanup_on_failure))
+                        return
+                    j = res.json
+                    servers += j["response"]["servers"]
+                except Exception as e:
+                    # TODO: could store exception in cleanup func
+                    logger.critical(e)
+                    self.thread_man.set_cleanup_func(StoredFunc(self._cleanup_on_failure))
+                    return
+
+        # NOTE: This step is allowed to fail, since this metadata is incidental
+        res = Servers.query_api(key, APPID_DAYZ_EXP, "")
+        if res.status == 200 and res.parsed is True:
+            j = res.json
+            servers += j["response"]["servers"]
+
+        # TODO: strings
+        #self.thread_man.increment_dialog_with_str("Unpacking servers")
+        # TODO: try/except
+        parsed = Servers.parse_json(servers)
+        self._push_data_success(parsed, FilterMode.INITIAL)
+
+    # TODO: strings
+    @call_on_thread("scanning LAN ports")
+    def _dump_lan(self, port: int, early_abort: bool) -> None:
+        servers = []
+        ports = range(1, 256)
+
+        event = threading.Event()
+        with ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(Servers.test_ip, i, port, event) for i in ports
+            ]
+            for future in as_completed(futures):
+                try:
+                    res = future.result(timeout=0.5)
+                    if res is not None and early_abort is True:
+                        # NOTE: first non-empty hit, flag pending threads to close
+                        event.set()
+                        servers.append(res)
+                        self.cleanup_func = StoredFunc(self._cleanup_on_success)
+                        return
+                    if res is None:
+                        continue
+                    servers.append(res)
+                except Exception as e:
+                    logger.critical(e)
+                    self.cleanup_func = StoredFunc(self._cleanup_on_failure)
+            if len(servers) == 0:
+                self.cleanup_func = StoredFunc(self._cleanup_on_failure)
+                return
+        parsed = Servers.parse_json(servers)
+        self._push_data_success(parsed, FilterMode.INITIAL)
+
+    # TODO: strings
+    @call_on_thread("dumping ips")
+    def _dump_ips(self, ips: list[str]) -> None:
+        # NOTE: block malformed records (TODO: add github issue no.)
+        # TODO: sanitize ip list at config time, drop this
+        ips = [ip for ip in ips if len(ip.split(":")) == 3 and ip.split(":")[2] != ""]
+        with ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(
+                    Servers.query_direct,
+                    ip.split(":")[0],
+                    int(ip.split(":")[2]),
+                )
+                for ip in ips
+            ]
+            servers = []
+            for future in as_completed(futures):
+                res = future.result()
+                self.thread_man.increment_dialog()
+                # NOTE: failing entries are culled
+                if res is None:
+                    continue
+                servers.append(res)
+                if len(servers) == 0:
+                    self.cleanup_func = StoredFunc(self._cleanup_on_failure)
+                    return
+
+        # NOTE: 1 extra progress bar pass for parsing
+        parsed = Servers.parse_json(servers)
+        self._push_data_success(parsed, FilterMode.INITIAL)
+
+    def _query_ip_id(self, addr: str) -> None:
+        # NOTE: Battlemetrics
+        if addr.isdigit():
+            # FIXME:
+            config = self.controller.get_prefs().paths.config
+            resolved = map_id_to_record(config, addr)
+            res = Servers.query_direct(resolved.ip, resolved.qport)
+        else:
+            record = addr.split(":")
+            ip, qport = record[0], record[1]
+            res = Servers.query_direct(ip, int(qport))
+        return res
+
+    # TODO: strings
+    @call_on_thread("querying address")
+    def _connect_by_id_or_ip(self, addr: str) -> None:
+        res = self.query_ip_id(addr)
+
+    @call_on_thread("querying address")
+    def _add_by_id_or_ip(self, addr: str) -> None:
+        res = self.query_ip_id(addr)
+        # TODO: investigate this
+        if res is None:
+            self.set_cleanup_func(StoredFunc(self._cleanup_on_failure))
+            return
+        # NOTE: single record insertion
+        self.insert_record = Servers.parse_json([res])
+        # TODO: add into saved servers file
+        # TODO: update saved servers model
+        # NOTE: this can be called from other tabs--if current focus is not ServerTab.SAVED, update label only
+        # TODO: saved servers might not be loaded yet, in which case should just update local file only
+        # TODO: perform simple equality comparison of self.tv.get_enum() == ServerTab.SAVED
+        # FIXME: filter man is saved on a per tab basis, so this will mismatch
+        self.set_cleanup_func(StoredFunc(self._cleanup_on_insert))
+
+    def _dump_history(self) -> None:
+        history = self.controller.get_prefs().paths.history
+        # TODO: customize statusbar to mention how records are added after connecting
+        try:
+            with open(history, "r") as f:
+                rows = [row.rstrip("\n") for row in f]
+        except OSError:
+            self.cleanup_func = StoredFunc(self._cleanup_on_failure, False)
+            return
+        if len(rows) == 0:
+            self.cleanup_func = StoredFunc(self._cleanup_on_failure, False)
+            return
+        self.thread_man.set_job_count(len(rows))
+        self._dump_ips(rows)
+
+    def _dump_favorites(self) -> None:
+        ips = self.controller.query_config(Preferences.IP_LIST)
+        self.thread_man.set_job_count(len(ips))
+        # TODO: customize statusbar to mention how records can be added via contextmenu
+        if len(ips) == 0:
+            # FIXME: this is not a failure, just a quiet exit with custom statusbar
+            # TODO: add custom statusbar parameters
+            self.cleanup_func = StoredFunc(self._cleanup_on_failure, False)
+            return
+        self._dump_ips(ips)
+
+    def _cleanup_on_insert(self) -> None:
+        filter_man = self.get_filter_man()
+        model = filter_man.get_control()
+
+        # FIXME: this is a single row insertion,
+        # but refiltration should occur in thread for consistency/scalability
+        model.append(self.insert_record[0])
+        proxy = filter_man.filter(FilterMode.INITIAL)
+
+        # TODO: get proxy model out of thread
+        proxy = self.get_filter_man().get_proxy_model()
+        self.tv.set_model(proxy)
+
+        # TODO: update statusbar
+        context = self.tv.get_enum()
+        # TODO: adding a row may update available maps
+        # TODO: if all filters are already applied, strange behavior may occur
+        # -> need to insert and update per current filters
+        self.emitter.emit("servers_loaded", context)
+
+    def _cleanup_on_success(self) -> None:
+        self.pending_jobs = 1
+        self.tv.set_loaded(True)
+        self.tv.set_model(self.to_insert)
+
+        # TODO: this will allow history and saved tab to emit signals to statusbar
+        # CHORE: test if treeview's sort method inserts row at the correct index
+        # inserting a row serializes file on disk, updates control model for that tab, and
+        # reapplies filters to ephemeral model; since filters are applied, in-situ insertion might not be necessary
+        # TODO: will be inserted out of order
+        #self.to_insert.connect("row-inserted", lambda *args: print("row inserted into model"))
+
+        # TODO: signals or other approach to deferring map
+        # model insertion after thread closes
+        # cf. servers_loaded signal
+
+        # TODO: servers_loaded vs servers_reloaded
+        context = self.tv.get_enum()
+        self.emitter.emit("servers_loaded", context)
+
+        # CHORE: this is placeholder logic
+        if self.first_iteration:
+            map_man = self.tv.get_map_man()
+            map_man.set_unique_maps(self.new_maps)
+            self.emitter.emit("servers_loaded_init")
+            self.first_iteration = False
+            self.new_maps = None
+
+        self.tv.grab_focus()
+
+    def _cleanup_on_failure(self, show_dialog=True) -> None:
+        self.treeview.set_loaded(True)
+        map_man = self.treeview.get_map_man()
+
+        # TODO: disable map, keyword, and filter widgets if model is None
+        # -> signal driven (servers_empty)
+        # TODO: what if refresh action occurred and failed, and the old model is still valid?
+        # skip the step below if refresh action failed
+        # do not wipe control model in this case
+        # e.g. if treeview.is_refresh():
+        # revert old model
+        # wipe refresh state to False
+
+        self.treeview.set_model(None)
+        self.treeview.grab_focus()
+
+        map_man.set_unique_maps(None)
+        context = self.treeview.get_enum()
+
+        # TODO: distinguish signals, e.g. "servers_failed_to_load", "servers_loaded_empty"
+        # customize statusbar accordingly
+        self.emitter.emit("servers_loaded", context)
+        if show_dialog:
+            dialog = ExceptionDialog(self, strings.api_warn_msg)
+            dialog.run()
+
+    # TODO: break into initial dump and refilter modes, can drop filtermode kwarg
+    # and stop pushing empty data
+    def _push_data_success(self, data: tuple, mode: Optional[FilterMode]) -> None:
+        # FIXME: calls treeview read methods in thread
+        # treeview = self.get_active_treeview()
+        # manager = treeview.get_filter_man()
+        manager = self._get_filter_man()
+
+        if data is None:
+            self.to_insert = None
+        else:
+            if mode == FilterMode.INITIAL:
+                manager.set_control(data)
+            manager.filter(mode)
+            self.to_insert = manager.get_proxy_model()
+            # TODO: pre parse maps
+            u_maps = set([row[1] for row in data])
+            self.new_maps = sorted(u_maps)
+
+        self.thread_man.set_cleanup_func(StoredFunc(self._cleanup_on_success))
+
+    def _get_filter_man(self) -> "FilteredModelManager":
+        return self.filter_man
+

dzgui/views/base.py

@@ -152,20 +152,18 @@ class OuterWindow(Gtk.Window):
 
         MainController.set_resolution(self)
         self.show_all()
-
         css.load_css()
 
-        # TODO: first run
         MainController.open_page(NotebookPage.SERVERS)
-        MainController.hide_widgets_on_init()
+        self.grid.hide_widgets_on_init()
 
         # TODO: POC, trigger page change here
         MainController.loaded = True
 
-        # TODO: debug flag
+        # TODO: debug flag, perhaps drop
         if MainController.get_prefs().is_debug:
             return
-        MainController.populate_model()
+        MainController.populate_model(MainController.get_active_treeview())
 
     def _on_delete_event(self, window: "OuterWindow", event: Gdk.EventKey) -> None:
         self.halt_proc_and_quit()
@@ -352,6 +350,10 @@ class Grid(Gtk.Grid):
 
         self.emitter.connect("server_page_toggled", self.toggle_filter_panels)
 
+    def hide_widgets_on_init(self) -> None:
+        self.conpan.set_lan_visible(False)
+        self.right_panel.sel_panel.hide()
+
     def _shrink(self, button: Gtk.Button) -> None:
         tv = MainController.get_active_treeview()
         tv.shrink_to_fit()

dzgui/views/pages/servers.py

@@ -100,11 +100,12 @@ class ServerNotebook(Gtk.ScrolledWindow):
         if label is None:
             return
 
-        # TODO: strings
+        # TODO: strings, tab activity notifier
         text = label.strip("*")
         self.notebook.set_tab_label_text(child, text)
+
         # NOTE: spawns a thread
-        self.controller.populate_model()
+        self.controller.populate_model(self.get_active_treeview())
 
     def get_active_treeview(self) -> ServerTreeView:
         index = self.notebook.get_current_page()

dzgui/views/trees/tree_servers.py

@@ -48,7 +48,7 @@ class ServerTreeView(ContextMixin, TreeView):
         self.query_func: Callable = None
 
         self.filter_man = FilteredModelManager(controller)
-        model = self.filter_man.get_model()
+        model = self.filter_man.get_proxy_model()
         self.set_model(model)
 
         # NOTE: each tab context has its own unique maps