diff --git a/dzgui/app_init.py b/dzgui/app_init.py index 3ed89d2..3502040 100644 --- a/dzgui/app_init.py +++ b/dzgui/app_init.py @@ -4,7 +4,7 @@ import shutil from typing import TYPE_CHECKING -from dzgui.const.constants import APP_NAME, APP_NAME_LOWER +from dzgui.const.constants import APP_NAME from dzgui.const.enum import Preferences from dzgui.config.query import lookup from dzgui.config.userprefs import UserPrefs @@ -21,6 +21,7 @@ from dzgui.strings import boot # from dzgui.util.map_count import get_map_count from dzgui.util.deck import is_steam_deck, is_game_mode from dzgui.util.localize import set_locale +from dzgui.util.redact import RedactionFilter, REDACTION_PATTERNS from dzgui.util.strings import init from dzgui.views.base import App @@ -40,13 +41,19 @@ def make_parents(path: "Path") -> None: def setup_logger(log_path: "Path") -> None: + # TODO: put in consts? _format = ( "%(asctime)s␞%(levelname)s␞%(filename)s::%(funcName)s::%(lineno)s␞%(message)s" ) + fh = logging.FileHandler(log_path) formatter = logging.Formatter(_format) fh.setFormatter(formatter) fh.setLevel(logging.DEBUG) + + _filter = RedactionFilter(patterns=REDACTION_PATTERNS) + fh.addFilter(_filter) + logger.setLevel(logging.DEBUG) logger.addHandler(fh) diff --git a/dzgui/model/model_factory.py b/dzgui/model/model_factory.py index daefeb3..7dfffbf 100644 --- a/dzgui/model/model_factory.py +++ b/dzgui/model/model_factory.py @@ -2,7 +2,6 @@ from dataclasses import dataclass from typing import Any, Sequence, TYPE_CHECKING from dzgui.const.enum import HELP_MENU_ROWS -from dzgui.util.redact import redact_log from dzgui.util.strings import delimiter import gi @@ -134,9 +133,7 @@ class ModelFactory: with open(path, "r") as f: lines = [line.split(delimiter) for line in f.read().splitlines()] for record in lines: - # NOTE: strips PII and API keys - clean = redact_log(record) - store.append(clean) + store.append(record) return store def new_model_from_class(self, cls: type) -> FastInsertListStore: diff --git a/dzgui/util/diag.py b/dzgui/util/diag.py index d027687..77342e3 100644 --- a/dzgui/util/diag.py +++ b/dzgui/util/diag.py @@ -1,3 +1,4 @@ +import logging import os import platform @@ -9,7 +10,9 @@ from dzgui.const.constants import APP_NAME from dzgui.const.enum import Preferences from dzgui.config.query import lookup from dzgui.init.prefix import get_version -from dzgui.util.redact import redact +from dzgui.util.redact import redact_home + +logger = logging.getLogger(APP_NAME) def get_cpu_model() -> str: @@ -41,12 +44,16 @@ def write_diagnostic(config: Path, outfile: Path) -> None: # TODO: test availability on other distros date = datetime.now().isoformat() - distro = platform.freedesktop_os_release()["ID_LIKE"] + try: + distro = platform.freedesktop_os_release()["ID_LIKE"] + except Exception as e: + logger.warn(e) + distro = "Unknown" + kernel = os.uname().release cpu = get_cpu_model() version = get_version() - branch = lookup(config, Preferences.BRANCH) debug = lookup(config, Preferences.DEBUG) install = lookup(config, Preferences.INSTALL) default = lookup(config, Preferences.DEFAULT) @@ -54,8 +61,8 @@ def write_diagnostic(config: Path, outfile: Path) -> None: steam_path = Path(default) workshop_path = get_local_mod_path(steam_path) - steam_redacted = redact(default) - workshop_redacted = redact(str(workshop_path)) + steam_redacted = redact_home(default) + workshop_redacted = redact_home(str(workshop_path)) mods = get_local_mod_ids(steam_path) mods_pretty = print_mods(mods) @@ -65,7 +72,7 @@ def write_diagnostic(config: Path, outfile: Path) -> None: # FIXME: extraneous newlines in lists of mods template = f"""\ - {APP_NAME} version {version} ({branch}) + {APP_NAME} version {version} Date: {date} =============================== Distribution: {distro} diff --git a/dzgui/util/redact.py b/dzgui/util/redact.py index 94d4bfb..2cc57af 100644 --- a/dzgui/util/redact.py +++ b/dzgui/util/redact.py @@ -1,20 +1,29 @@ +import logging import re +from typing import Literal -def redact(text: str) -> str: - r = r"(/home/)([^/])*" - cleaned = re.sub(r, r"/home/REDACTED", text) +api_filter = r"(.*&key=)([^&]*)(.*)" +home_filter = r"(/home/)([^\s'\/]*)(.*)" +REDACTED = r"\1REDACTED\3" +REDACTION_PATTERNS = [api_filter, home_filter] + + +def redact_home(text: str) -> str: + pat = re.compile(home_filter) + cleaned = pat.sub(REDACTED, text) return cleaned -def redact_log(record: list) -> list[str]: - """ - requests library includes Steam API key in URL params - """ - clean = [] - for item in record: - if "&key=" in item: - pat = r"(.*&key=)(\S+)(.*)" - scrubbed = re.sub(pat, r"\1REDACTED\3", item) - clean.append(scrubbed) - else: - clean.append(item) - return clean + +class RedactionFilter(logging.Filter): + def __init__(self, patterns: list[str] | None = None) -> None: + super().__init__() + self._patterns = [re.compile(pat) for pat in (patterns or [])] + + def filter(self, record: logging.LogRecord) -> Literal[True]: + for pattern in self._patterns: + try: + record.msg = pattern.sub(REDACTED, record.msg) + except TypeError: + exception_text = f"{type(record.msg).__name__}: {record.msg}" + record.msg = pattern.sub(REDACTED, exception_text) + return True diff --git a/dzgui/views/pages/options.py b/dzgui/views/pages/options.py index aea0417..aea6250 100644 --- a/dzgui/views/pages/options.py +++ b/dzgui/views/pages/options.py @@ -288,6 +288,7 @@ class Options(Gtk.Box): def _on_api_change_failed(self, emitter: "Emitter") -> None: self.old_entry.set_text(self.old_text) + # TODO: use popover dialog = ExceptionDialog(self.controller, errors.api_validation_error) dialog.run() diff --git a/pyproject.toml b/pyproject.toml index 3b3615e..0972486 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,6 +110,7 @@ markers = [ "mods: tests mod metadata/link creation", "pefile: validate PE files", "post_install: requires a completed installation", + "redact: log redaction mechanisms", "slow: long-running tests", "webtest: checks remote endpoints" ] diff --git a/tests/test_bare_conf_files.py b/tests/test_bare_conf_files.py index 9235390..f55765e 100644 --- a/tests/test_bare_conf_files.py +++ b/tests/test_bare_conf_files.py @@ -4,7 +4,6 @@ import os from dzgui.app_init import copy_bare_configs from dzgui.config.xdg import get_xdg_paths, parse_filepaths -from pathlib import Path CONF_STRING = "DZGUI_CONF\n" @@ -27,8 +26,7 @@ def state_files(): @pytest.fixture -def xdg_paths(): - paths = [] +def xdg_paths(monkeypatch): routes = { "XDG_CONFIG_HOME": "", "XDG_STATE_HOME": "", @@ -39,7 +37,7 @@ def xdg_paths(): tmp = tempfile.TemporaryDirectory(delete=False) routes[route] = tmp.name for k, v in routes.items(): - os.environ[k] = v + monkeypatch.setenv(k, v) env = get_xdg_paths() return parse_filepaths(env) diff --git a/tests/test_config.py b/tests/test_config.py index 72c0460..3beaf03 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -95,8 +95,3 @@ def test_missing_values(unset_values): j = convert.rc2json(unset_values) j = json.loads(j) assert j.keys() == config_boilerplate.keys() - - - - -# TODO: test that when a config file is created from scratch, it contains all values diff --git a/tests/test_log_redaction.py b/tests/test_log_redaction.py new file mode 100644 index 0000000..a96058c --- /dev/null +++ b/tests/test_log_redaction.py @@ -0,0 +1,54 @@ +import logging +import pytest + +from dzgui.util.redact import RedactionFilter, REDACTION_PATTERNS + + +class RecordsListHandler(logging.Handler): + def __init__(self) -> None: + super().__init__() + self.records_list = [] + + def emit(self, record: logging.LogRecord) -> None: + self.records_list.append(record) + + def pop(self) -> None: + return self.records_list[-1].msg + + +@pytest.mark.redact +@pytest.mark.parametrize( + "log_error, expect", + [ + ("/home/SENSITIVE_USERNAME/subdir", "/home/REDACTED/subdir"), + ( + "https://url.com/?api&key=SENSITIVE_KEY&results=10", + "https://url.com/?api&key=REDACTED&results=10", + ), + ( + "https://url.com/?api&key=SENSITIVE_KEY", + "https://url.com/?api&key=REDACTED", + ), + ( + "Error in directory: '/home/SENSITIVE_USERNAME/'", + "Error in directory: '/home/REDACTED/'", + ), + ( + "Error in directory: '/home/SENSITIVE_USERNAME'", + "Error in directory: '/home/REDACTED'", + ), + ], +) +def test_log_redaction(log_error: str, expect: str) -> None: + logger = logging.getLogger("TEST") + + handler = RecordsListHandler() + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + + _filter = RedactionFilter(patterns=REDACTION_PATTERNS) + logger.addFilter(_filter) + + logger.critical(log_error) + redacted = handler.pop() + assert expect == redacted diff --git a/tests/test_remote_api.py b/tests/test_remote_api.py index 1cd4989..e47a4d6 100644 --- a/tests/test_remote_api.py +++ b/tests/test_remote_api.py @@ -20,7 +20,6 @@ def test_steam(config): key = config["steam_api"] assert probe.test_steam_api(key) - def test_bm(config): key = config["bm_api"] assert probe.test_bm_api(key)