feat: lazy load WIP

dzgui/api/servers.py

@@ -124,7 +124,7 @@ def parse_json(json: list) -> list:
             view = strings.filter_3pp
 
         if "external" in r:
-            provider = strings.filter_unoffiical
+            provider = strings.filter_unofficial
         else:
             provider = strings.filter_official
 
@@ -396,16 +396,16 @@ def query_api(key: str, appid: int, param: str) -> Res:
         "limit": LIMIT,
         "key": key,
     }
-
     try:
-        res = requests.get(STEAM_SERVERS, params=payload, timeout=REQUEST_TIMEOUT)
+        res = requests.get(STEAM_SERVERS, params=payload, timeout=5)
         res.raise_for_status()
         parsed = True
+        status = 200
         data = res.json()
     except Exception:
-        status = res.status_code
         parsed = False
         data = None
+        status = 403
     finally:
         return Res(status, parsed, data)
 

dzgui/const/constants.py

@@ -4,7 +4,7 @@ UDP_PORT = 27016
 VM_FILE = "/proc/sys/vm/max_map_count"
 MIN_COUNT = 1048576
 
-REQUEST_TIMEOUT = 1
+REQUEST_TIMEOUT = 5
 
 APPNAME_DAYZ = "DayZ"
 APPNAME_DAYZ_EXP = "DayZ Experimental"

dzgui/controllers/mc.py

@@ -4,10 +4,13 @@ import threading
 import traceback
 from warnings import deprecated
 
+from concurrent.futures import wait
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Any, Callable, Literal, TYPE_CHECKING
 
 import dzgui.api.pefile as PeFile
+import dzgui.api.servers as Servers
 import dzgui.util._json as JSON  # noqa
 from dzgui.views.dialogs.generic import ExceptionDialog
 
@@ -353,6 +356,42 @@ class Controller(GObject.GObject):
 
         self.mediator.notebook.set_page_by_enum(button.opens)
 
+    def _dump_api(self):
+        key = self.query_config(Preferences.STEAM)
+        job = Servers.query_api
+        params = Servers.params
+        serv = []
+        with ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(job, key, APPID_DAYZ, param)
+                for param in params
+            ]
+            wait(futures)
+            for future in futures:
+                res = future.result()
+                if res.status != 200 or not res.parsed:
+                    # ModelManager.set_store(None)
+                    # ModelManager.set_success(False)
+                    # GLib.idle_add(self._filter_cleanup)
+                    print("status error")
+                    return
+                j = res.json
+                serv += j["response"]["servers"]
+
+        res = Servers.query_api(key, APPID_DAYZ_EXP, "")
+        if res.status == 200 and res.parsed is True:
+            j = res.json
+            serv += j["response"]["servers"]
+
+        parsed = Servers.parse_json(serv)
+        self.data = parsed
+        self.get_func_data()
+        self.destroy_on_idle()
+        # try:
+        # except Exception as e:
+        #     print(e)
+        # return parsed
+
     def get_help_row(self) -> str:
         tv = self.mediator.menu
         model = self.get_help_store()
@@ -538,20 +577,6 @@ class Controller(GObject.GObject):
         self.data = (
             ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
             ["BAR", "a", "a", "a", 1, 1, 1, "172.111.51.156:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
-            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
         )
         # TODO: refer to prior implementation--should need to load data into model while in thread
         self.get_func_data()
@@ -562,6 +587,16 @@ class Controller(GObject.GObject):
         time.sleep(1)
         self.data = (
             ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "172.111.51.156:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
+            ["BAR", "a", "a", "a", 1, 1, 1, "185.207.214.16:2302", 0, 0, "a", False],
         )
         # TODO: refer to prior implementation--should need to load data into model while in thread
         self.get_func_data()
@@ -572,8 +607,11 @@ class Controller(GObject.GObject):
             # NOTE: do not insert model until main thread is idle
             # TODO: grab from model manager and insert entire model
             # TODO: do not start refresh button count until load finished
+            # TODO: append rows into filtered model
+            # when complete, destroy dialog and set model
             for row in data:
                 manager.append_row(row)
+            treeview.set_model(manager.get_model())
             treeview.set_loaded(True)
             self.mediator.statusbar.emit("server_page_changed", context)
             treeview.grab_focus()
@@ -699,6 +737,7 @@ class Controller(GObject.GObject):
             return
         manager = treeview.get_filter_man()
         # TODO: this may lag?
+        treeview.set_model(None)
         manager.clear_model()
         self.set_callback(None, None)
         self.call_on_thread(func)

dzgui/model/filtered_model.py

@@ -46,17 +46,23 @@ class FilteredModelManager:
         self.ping_cache: dict[str, int] = {}
 
         self.model = self.new_model_from_class(ServerColumns)
+        self.real_model = self.new_model_from_class(ServerColumns)
 
         self.control_model = None
         self.filtered = None
         self.success = True
 
     def append_row(self, row: list) -> None:
-        self.model.append(row)
+        self.real_model.append(row)
+        # NOTE: append null rows to thin client
+        self.model.append()
 
     def clear_model(self) -> None:
         self.model.clear()
 
+    def get_real_model(self) -> ListStore:
+        return self.real_model
+
     def get_model(self) -> ListStore:
         return self.model
 

dzgui/views/pages/servers.py

@@ -34,8 +34,9 @@ class ServerNotebook(Gtk.ScrolledWindow):
         )
         self.lan = ServerTreeView(controller, ServerTab.LAN, ContextMenuGroup.SCAN_LAN)
 
-        self.browser.set_query_func(self.controller.dump_test_1)
+        # self.browser.set_query_func(self.controller.dump_test_2)
         self.lan.set_query_func(self.controller.dump_test_2)
+        self.browser.set_query_func(self.controller._dump_api)
 
         tabs = [
             (self.browser, server_labels.browser),

dzgui/views/trees/tree_servers.py

@@ -1,6 +1,6 @@
 import logging
 import multiprocessing
-from typing import Self
+from typing import Any, Self
 
 from dzgui.const.enum import ContextMenu, ContextMenuGroup, FilterMode, ServerTab
 from dzgui.api.servers import Record
@@ -76,6 +76,7 @@ class ServerTreeView(TreeView):
             renderer = Gtk.CellRendererText()
             column = Gtk.TreeViewColumn(column_title, renderer, text=i)
             column.set_sizing(Gtk.TreeViewColumnSizing.FIXED)
+            column.set_cell_data_func(renderer, self._lazy_load, func_data=i)
             column.set_resizable(True)
             column.set_sort_column_id(i)
 
@@ -335,6 +336,8 @@ class ServerTreeView(TreeView):
         print(self.get_value_at_index(0))
 
     def _parent_selection_changed(self, base_class: TreeView, sel: Gtk.TreeSelection):
+        # TODO: lazy load
+        # https://stackoverflow.com/questions/3164262/lazy-loaded-list-view-in-gtk
         self.terminate_process()
         self.emit("distcalc_started")
 
@@ -348,6 +351,7 @@ class ServerTreeView(TreeView):
         select = self.get_selection()
         sels = select.get_selected_rows()
         (model, pathlist) = sels
+        # TODO: clean up per lazy load logic
         if len(pathlist) < 1:
             return None
         path = pathlist[0]
@@ -355,6 +359,8 @@ class ServerTreeView(TreeView):
         if not model:
             return None
         addr = model[path][7]
+        if addr is None:
+            return
         qport = model[path][8]
         ip = addr.split(":")[0]
         gameport = int(addr.split(":")[1])
@@ -365,3 +371,23 @@ class ServerTreeView(TreeView):
 
     def set_loaded(self, status: bool) -> None:
         self.loaded = status
+
+    def _lazy_load(
+        self,
+        column: Gtk.TreeViewColumn,
+        cell: Gtk.CellRendererText,
+        model: Gtk.TreeModel,
+        it: Gtk.TreeIter,
+        col_index: int,
+    ) -> Any:
+        # TODO: initially load empty rows
+        # TODO: if cell is in visible range
+        path = model.get_path(it)
+        row_index = path.get_indices()[0]
+        start, end = self.get_visible_range()
+        if row_index >= start[0] <= end[0]:
+            real_model = self.filter_man.get_real_model()
+            value = real_model[row_index][col_index]
+            cell.set_property("text", str(value))
+        else:
+            return