Merge pull request #216 from aclist/feat/fetch_ipdb

feat: auto fetch geolocation records

CHANGELOG.md

@@ -1,18 +1,28 @@
 # Changelog
 
-## [5.7.0-beta.13] 2024-04-17
+## [5.8.0-beta.1] 2025-06-05
+## Added
+- Automatically fetch geolocation records
+## Fixed
+- Corrected erroneous 2024 date in prior changelog entries
+
+## [5.7.1-beta.1] 2025-04-17
+## Changed
+- Updated geolocation records
+
+## [5.7.0-beta.13] 2025-04-17
 ## Fixed
 - Updated checksum for UI helper file
 
-## [5.7.0-beta.12] 2024-04-04
+## [5.7.0-beta.12] 2025-04-04
 ## Dropped
 - Removed extraneous pre-boot API checks that could cause error messages to be printed if the user had not set up an API key yet
 
-## [5.7.0-beta.11] 2024-03-20
+## [5.7.0-beta.11] 2025-03-20
 ## Fixed
 - Reduce startup time when testing remote APIs
 
-## [5.7.0-beta.10] 2024-03-15
+## [5.7.0-beta.10] 2025-03-15
 ## Added
 - Restore application size on subsequent startup
 ## Fixed
@@ -23,44 +33,44 @@
 - Packed filter checkbox buttons into a 3x3 grid
 - Updated IP database to 2025-03 records
 
-## [5.7.0-beta.9] 2024-03-04
+## [5.7.0-beta.9] 2025-03-04
 ## Fixed
 - Livonia server results being dropped from batch queries
 
-## [5.7.0-beta.8] 2024-02-10
+## [5.7.0-beta.8] 2025-02-10
 ## Changed
 - Drop launch flag and check for invocation through Steam automatically
 
-## [5.7.0-beta.7] 2024-02-09
+## [5.7.0-beta.7] 2025-02-09
 ## Changed
 - Update IP database records for 2025-02
 
-## [5.7.0-beta.6] 2024-01-10
+## [5.7.0-beta.6] 2025-01-10
 ## Fixed
 - Resolve regression introduced with IP resolution feature in 13c6813
 
-## [5.7.0-beta.5] 2024-01-10
+## [5.7.0-beta.5] 2025-01-10
 ## Changed
 - Stricter redacting of usernames (again)
 
-## [5.7.0-beta.4] 2024-01-10
+## [5.7.0-beta.4] 2025-01-10
 ## Fixed
 - Clerical hotfix
 
-## [5.7.0-beta.3] 2024-01-10
+## [5.7.0-beta.3] 2025-01-10
 ## Changed
 - Support legacy jq syntax for Ubuntu variants
 
-## [5.7.0-beta.2] 2024-01-07
+## [5.7.0-beta.2] 2025-01-07
 ## Fixed
 - Stricter redacting of usernames
 - Omit nested directories when traversing symlinks
 
-## [5.7.0-beta.1] 2024-01-07
+## [5.7.0-beta.1] 2025-01-07
 ### Changed
 - Normalize version number
 
-## [5.6.0-beta.21] 2024-01-06
+## [5.6.0-beta.21] 2025-01-06
 ### Added
 - Add in-app documentation link to Codeberg mirror
 - Hover tooltips to most buttons

dzgui.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -o pipefail
 
-version=5.7.1-beta.1
+version=5.8.1-beta.1
 
 #CONSTANTS
 aid=221100
@@ -49,10 +49,9 @@ freedesktop_path="$HOME/.local/share/applications"
 
 #HELPERS
 ui_helper="$helpers_path/ui.py"
-geo_file="$helpers_path/ips.csv"
 km_helper="$helpers_path/latlon"
-sums_path="$helpers_path/sums.md5"
 func_helper="$helpers_path/funcs"
+geo_helper="$helpers_path/ips.csv"
 
 #REMOTE
 remote_host=gh
@@ -63,7 +62,6 @@ stable_url="$url_prefix/dzgui"
 testing_url="$url_prefix/testing"
 releases_url="https://github.com/$author/$repo/releases/download/browser"
 km_helper_url="$releases_url/latlon"
-geo_file_url="$releases_url/ips.csv.gz"
 
 
 set_im_module(){
@@ -637,18 +635,8 @@ fetch_helpers_by_sum(){
     done
     return 0
 }
-fetch_geo_file(){
+fetch_km_helper(){
-    # for binary releases
-    local geo_sum="83bf7655a4197214b4be73dfa9a7b319"
     local km_sum="b038fdb8f655798207bd28de3a004706"
-    local gzip="$helpers_path/ips.csv.gz"
-    if [[ ! -f $geo_file  ]] || [[ $(get_hash $geo_file) != $geo_sum ]]; then
-        local res=$(get_response_code "$geo_file_url")
-        [[ $res -ne 200 ]] && raise_error_and_quit "Remote resource unavailable: '$geo_file_url'"
-        curl -Ls "$geo_file_url" > "$gzip"
-        #force overwrite
-        gunzip -f "$gzip"
-    fi
     if [[ ! -f $km_helper ]] || [[ $(get_hash $km_helper) != $km_sum ]]; then
         local res=$(get_response_code "$km_helper_url")
         [[ $res -ne 200 ]] && raise_error_and_quit "Remote resource unavailable: '$km_helper_url'"
@@ -656,10 +644,119 @@ fetch_geo_file(){
         chmod +x "$km_helper"
     fi
 }
+get_response_code(){
+    local url="$1"
+    curl -Ls -I -o /dev/null -w "%{http_code}" "$url"
+}
+raise_error_and_quit(){
+    echo "$1"
+    exit 1
+}
+fetch_ip_db(){
+    parse_dl_url(){
+        curl -Ls "$url" \
+        | grep "csv.gz" \
+        | awk -F"['']" '{print $2}'
+    }
+
+    parse_dl_url_date(){
+        local url="$1"
+        <<< "$url" \
+        awk -F/ '{print $5}' \
+        | awk -F"dbip-city-lite-" '{print $2}' \
+        | awk -F'.csv.gz' '{print $1}'
+    }
+
+    fetch(){
+        logger INFO "Triggering fetch routine"
+        local url="$1"
+
+        curl -Ls "$url" > "$base_file"
+        if [[ $? -ne 0 ]]; then
+            logger WARN "Abnormal exit while accessing '$url'"
+            return
+        fi
+
+        gunzip -f "$base_file"
+        if [[ $? -ne 0 ]]; then
+            rm "$base_file"
+            logger WARN "Abnormal exit while unpacking gzip '$base_file'"
+            return
+        fi
+
+        < "$extracted_file" grep -vE "^[a-z0-9]{4}:" | grep -v "::" > "$ip_file"
+        if [[ $? -ne 0 ]]; then
+            logger WARN "Abnormal exit while parsing IPs"
+            return
+        fi
+        rm "${state_path}/${month}.csv"
+
+        readarray -t records < <(cat $ip_file | awk -F, 'NR==1 {print $1} END {print $1}')
+        if [[ ${records[0]} != "0.0.0.0" ]] && [[ ${records[1]} != "224.0.0.0" ]]; then
+            logger WARN "Anchor records missing in '$ip_file', possibly malformed"
+            rm "$ip_file"
+            return
+        fi
+
+        mv "$ip_file" "$geo_helper"
+        if [[ $? -ne 0 ]]; then
+            logger WARN "Abnormal exit while moving '$ip_file' to '$helpers_path'"
+            rm "$ip_file"
+        fi
+
+        echo "$month" > "$month_file"
+        logger INFO "Wrote '$month' to stub '$month_file'"
+        logger INFO "Updated '$ip_file'"
+    }
+
+    local url="https://db-ip.com/db/download/ip-to-city-lite"
+    local month_file="${state_path}/.month"
+    local ip_file="${state_path}/ips.csv"
+
+    # test main url
+    local res=$(get_response_code "$url")
+    if [[ $res -ne 200 ]]; then
+        logger WARN "Failed to retrieve remote resource: '$url' ($res)"
+        return
+    fi
+    logger INFO "Resolved remote URL: '$url'"
+
+    # test dl url
+    local dl_url="$(parse_dl_url)"
+    local res=$(get_response_code "$dl_url")
+    if [[ $res -ne 200 ]]; then
+        logger WARN "Remote resource unavailable: '$dl_url' ($res)"
+        return
+    fi
+    logger INFO "Resolved download URL: '$dl_url'"
+
+    local month=$(parse_dl_url_date "$dl_url")
+    local base_file="${state_path}/${month}.csv.gz"
+    local extracted_file="${state_path}/${month}.csv"
+
+    # no stub file
+    if [[ ! -f $month_file ]]; then
+        logger WARN "No stub file '$month_file' present"
+        fetch "$dl_url"
+        return
+    fi
+
+    # local needs update
+    local last_month=$(< "$month_file")
+    if [[ $last_month != "$month" ]]; then
+        logger WARN "Local stub '$last_month' does not match remote stub '$month'"
+        fetch "$dl_url"
+        return
+    fi
+
+    # if stub is same date, abort
+    logger INFO "Local stub '$last_month' is identical to remote, skipping"
+}
 fetch_helpers(){
     fetch_a2s
     fetch_dzq
-    fetch_geo_file
+    fetch_km_helper
+    fetch_ip_db
     fetch_helpers_by_sum
     [[ ! -f $share_path/icon.png ]] && freedesktop_dirs
     fetch_icons
@@ -899,7 +996,6 @@ test_connection(){
         stable_url="$url_prefix/dzgui"
         testing_url="$url_prefix/testing"
         km_helper_url="$releases_url/latlon"
-        geo_file_url="$releases_url/ips.csv.gz"
     fi
 }
 legacy_cols(){