Editing out the already moved & commented out Connection Controller pieces

.gitignore

@@ -1,3 +1,4 @@
+.dev
 prototype_client.py
 .idea
 .venv

assets/yaml_mappings/locations.yaml

@@ -0,0 +1,54 @@
+fields:
+  - name: country_code
+    path: ['country', 'code']
+    required: true
+
+  - name: code # this is city code
+    path: ['code']
+    required: true
+
+  - name: id
+    path: ['id']
+
+  - name: country_name
+    path: ['country', 'name']
+
+  - name: name # this is CITY name
+    path: ['name']
+
+  - name: time_zone
+    path: ['time_zone', 'code']
+
+  - name: operator_id
+    path: ['operator_id']
+    required: true
+
+  - name: provider_name
+    path: ['provider', 'name']
+
+  - name: is_proxy_capable
+    path: ['is_proxy_capable']
+
+  - name: is_wireguard_capable
+    path: ['is_wireguard_capable']
+    required: true
+
+  - name: is_hysteria2_capable
+    path: ['is_hysteria2_capable']
+
+  - name: is_vless_capable
+    path: ['is_vless_capable']
+
+
+# original version:
+  # locations.append((
+  # location['country']['code'],
+  # location['code'],
+  # location['id'],
+  # location['country']['name'],
+  # location['name'],
+  # location['time_zone']['code'],
+  # location['operator_id'],
+  # location['provider']['name'],
+  # location['is_proxy_capable'],
+  # location['is_wireguard_capable']))

assets/yaml_mappings/mapping_one.yaml

@@ -0,0 +1,8 @@
+fields:
+  - name: group_a_code
+    path: ['group_a', 'code']
+  - name: code
+    path: ['code']
+  - name: id
+    path: ['id']
+    required: true

assets/yaml_mappings/operators.yaml

@@ -0,0 +1,19 @@
+fields:
+  - name: id
+    path: ['id']
+    required: true
+
+  - name: name
+    path: ['name']
+
+  - name: public_key # this is ed25519
+    path: ['public_key']
+
+  - name: nostr_public_key
+    path: ['nostr_public_key']
+
+  - name: nostr_profile_reference
+    path: ['nostr_profile_reference']
+
+  - name: nostr_attestation_event_reference
+    path: ['nostr_attestation', 'event_reference']

change_log.md

@@ -0,0 +1,38 @@
+# Major Change Log: 
+
+
+# 2.4.3  
+### Systemwide Killswitch Introduced for Enable  
+
+### July 23, 2026  
+Separation of performing subprocess commands, parsing them for error strings, and handling those errors as enum types.  Systemwide error handling was introduced via a wrapper to all systemwide functions requiring the sudo scripts.  
+<br/>
+
+### July 20, 2026  
+Introduced IP-interface-based killswitch support for systemwide wireguard, on enable only.  This version is stable and working for enable only, with a "test" script in the fileslot. And fixed GUI's object save creation. Additionally rolled back Connection data models to previous non-enum version.
+<br/>
+
+# 2.4.2  
+### Surgery on ConnectionController  
+Huge separation of ConnectionController & ProfileController into naked function modules for enabling a connection, registering wireguard keys, coordinating proxy configs, subscriptions, endpoint verification, and the logistical flow.  Additionally enums were used for type checking, instead of polymorphic functions.  This version is stable and confirmed working for systemwide and session wireguard, but not yet proxies.  
+<br/>
+
+# 2.4.1
+### Enums for Profile types.
+July 18, 2026
+Introduce Base profile types on load, which gradually will switch over on save/edits. Transition Connections to using Enum instead of raw strings.  End goal here is to transition ConnectionController to use the Enums.  
+<br/>
+
+# 2.4.0  
+July 15, 2026  
+Robust Database Error Handling & Migrations System.  For New Client upgrades (Operational Errors) & Old Clients with New JSON keys from an API (TypeErrors).  We also added an SQL generic handling wrapper for use across any project or function.
+<br/>
+
+# 2.3.9  
+July 11, 2026  
+Features: Revamp of GET/POST API requests for both Tor and Clearweb.  
+Why: Move towards more clear Tor API error feedback.  And reduce redundancy.  
+Status: Stable, works for both clear/tor and GET/POST.  
+Still Needs: UI feedback, DNS Tor improvements
+<br/>
+

core/Constants.py

@@ -5,6 +5,16 @@ import os
 
 @dataclass(frozen=True)
 class Constants:
+
+    DB_VERSION_THIS_APP_WANTS = 1
+
+    # Fallback for development (running outside AppImage)
+    fallback_non_appimage = os.path.dirname(os.path.abspath(__file__))
+
+    # appimage home
+    APPDIR_HOME: Final[str] = os.environ.get('APPDIR', fallback_non_appimage)
+    print(f"APPDIR_HOME is {APPDIR_HOME}")
+    
     # ticketing group:
     TICKET_API_BASE_URL: Final[str] = os.environ.get(
         "TICKET_API_BASE_URL", "https://ticket.hydraveil.net"
@@ -55,3 +65,8 @@ class Constants:
 
     HV_SESSION_STATE_HOME: Final[str] = f'{HV_STATE_HOME}/sessions'
     HV_TOR_STATE_HOME: Final[str] = f'{HV_STATE_HOME}/tor'
+
+    # ── killswitch / dns wrappers ─────────────────────────────────────────────
+    KILLSWITCH_WRAPPER:         Final[str] = os.environ.get(
+        'KILLSWITCH_WRAPPER', '/opt/hydra-veil/killswitch'
+    )

core/assets/sudo_scripts/dns

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ "$EUID" -ne 0 ]]; then
+    echo "[killswitch] ERROR: must be run as root" >&2
+    exit 1
+fi
+
+ACTION="${1:-}"
+IFACE="${2:-}"
+DNS="${3:-}"
+
+
+_check_prereqs() {
+        [[ -z "$DNS" ]] && { echo "Error: DNS IP required" >&2; exit 1; }
+        
+        # ✓ Check it's a valid IPv4 address:
+        if ! [[ "$DNS" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then
+            echo "Error: Invalid IP: $DNS" >&2
+            exit 1
+        fi
+                
+        # ✓ checks interface exists before configuring
+        if ! ip link show "$IFACE" &>/dev/null; then
+            echo "Error: Interface $IFACE not found" >&2
+            exit 1
+        fi
+}
+
+_set_dns() {
+    resolvectl dns "$IFACE" "$DNS" || {
+        echo "ERROR: Failed to set DNS to $DNS" >&2
+        return 1
+    }
+    resolvectl domain "$IFACE" '~.' || {
+        echo "ERROR: Failed to set domain routing" >&2
+        return 1
+    }
+    resolvectl default-route "$IFACE" true || {
+        echo "ERROR: Failed to set default route" >&2
+        return 1
+    }
+}
+
+if [[ "$ACTION" == "set" ]]; then
+    _check_prereqs
+    _set_dns
+fi
+
+# if [[ "$ACTION" == "revert" ]]; then
+#     resolvectl revert "$IFACE" 2>/dev/null || true
+
+# fi
+
+if [[ "$ACTION" == "revert" ]]; then
+    resolvectl revert "$IFACE" || true
+fi

core/assets/sudo_scripts/firewall

@@ -0,0 +1,120 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ "$EUID" -ne 0 ]]; then
+    echo "[killswitch] ERROR: must be run as root" >&2
+    exit 1
+fi
+
+ACTION="${1:-}"
+SERVER_IP="${2:-}"
+TUNNEL_IF="${3:-}"
+INTERNAL_SUBNET="${4:-}"
+TABLE="hydraveil"
+
+if [[ "$ACTION" != "arm" && "$ACTION" != "disarm" && "$ACTION" != "status" ]]; then
+    echo "[killswitch] ERROR: invalid action '$ACTION'. Usage: arm <server_ip> <tunnel_if> [internal_subnet] | disarm | status" >&2
+    exit 1
+fi
+
+_default_iface() {
+    ip route show default 2>/dev/null | awk 'NR==1{print $5}'
+}
+
+_log() {
+    local level="$1"; shift
+    echo "[killswitch] [$level] $* — $(date '+%Y-%m-%d %H:%M:%S')" >&2
+}
+
+if [[ "$ACTION" == "status" ]]; then
+    if nft list table inet "$TABLE" &>/dev/null 2>&1; then
+        echo "armed"
+    else
+        echo "disarmed"
+    fi
+    exit 0
+fi
+
+if [[ "$ACTION" == "disarm" ]]; then
+    if nft list table inet "$TABLE" &>/dev/null 2>&1; then
+        nft delete table inet "$TABLE"
+        _log "INFO" "disarmed"
+    else
+        _log "INFO" "already disarmed"
+    fi
+    exit 0
+fi
+
+[[ -z "$SERVER_IP" ]]  && { _log "ERROR" "server_ip required for arm"; exit 1; }
+[[ -z "$TUNNEL_IF" ]]  && { _log "ERROR" "tunnel_if required for arm"; exit 1; }
+
+if ! [[ "$SERVER_IP" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
+    _log "ERROR" "invalid IPv4: $SERVER_IP"
+    exit 1
+fi
+
+IFS='.' read -r o1 o2 o3 o4 <<< "$SERVER_IP"
+for oct in "$o1" "$o2" "$o3" "$o4"; do
+    if (( oct > 255 )); then
+        _log "ERROR" "invalid IPv4 octet ($oct) in $SERVER_IP"
+        exit 1
+    fi
+done
+
+WAN_IFACE=$(_default_iface)
+if [[ -z "$WAN_IFACE" ]]; then
+    _log "ERROR" "could not detect primary network interface"
+    exit 1
+fi
+
+INTERNAL_RULE=""
+if [[ -n "$INTERNAL_SUBNET" ]]; then
+    INTERNAL_RULE="        ip daddr ${INTERNAL_SUBNET} accept"
+fi
+
+nft list table inet "$TABLE" &>/dev/null 2>&1 && nft delete table inet "$TABLE" || true
+
+nft -f - << NFTEOF
+table inet ${TABLE} {
+    chain output {
+        type filter hook output priority filter; policy drop;
+        oifname "lo" accept
+        ether type arp drop
+        ip6 daddr != ::1 drop
+        ip daddr 224.0.0.0/4 drop
+        ip daddr 255.255.255.255 drop
+        ip daddr 192.168.1.0/24 drop
+        
+        oifname "${WAN_IFACE}" ip daddr ${SERVER_IP} accept
+        oifname "${TUNNEL_IF}" accept        
+        log prefix "hydraveil-drop " drop
+    }
+
+    chain input {
+        type filter hook input priority filter; policy drop;
+        iifname "lo" accept
+        
+        ether type arp drop
+        ip6 saddr != ::1 drop
+        ip daddr 224.0.0.0/4 drop
+        ip daddr 255.255.255.255 drop
+
+        ct state established,related accept
+        iifname "${WAN_IFACE}" ip saddr ${SERVER_IP} accept
+        iifname "${TUNNEL_IF}" accept
+        
+        log prefix "hydraveil-drop " drop
+    }
+    chain forward {
+        type filter hook forward priority filter; policy drop;
+    }
+}
+NFTEOF
+
+
+_log "INFO" "armed — wan=${WAN_IFACE} server=${SERVER_IP} tunnel=${TUNNEL_IF} internal=${INTERNAL_SUBNET:-none}"
+exit 0
+
+
+
+

core/assets/sudo_scripts/setup.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+ORIGINAL_FOLDER="${ORIGINAL_FOLDER:-$(pwd)}"
+TARGET_FOLDER="${TARGET_FOLDER:-/opt/hydra-veil}"
+LINUX_USER="${LINUX_USER:-$SUDO_USER}"
+
+add_sudoers_rule() {
+    sudo tee /etc/sudoers.d/zzzzzzzzzzzzzzzzzzz > /dev/null <<EOF
+${LINUX_USER} ALL=(root) NOPASSWD: /opt/hydra-veil/firewall
+${LINUX_USER} ALL=(root) NOPASSWD: /opt/hydra-veil/dns
+EOF
+}
+
+# Check if running as root
+if [[ $EUID -ne 0 ]]; then
+   echo "Error: This script must be run as root (use sudo)" >&2
+   exit 1
+fi
+
+# Validate environment variables
+if [[ -z "$ORIGINAL_FOLDER" ]]; then
+   echo "Error: ORIGINAL_FOLDER environment variable not set" >&2
+   exit 1
+fi
+
+if [[ -z "$TARGET_FOLDER" ]]; then
+   echo "Error: TARGET_FOLDER environment variable not set" >&2
+   exit 1
+fi
+
+# Validate that source folder exists
+if [[ ! -d "$ORIGINAL_FOLDER" ]]; then
+   echo "Error: ORIGINAL_FOLDER does not exist: $ORIGINAL_FOLDER" >&2
+   exit 1
+fi
+
+# Create target folder if it doesn't exist
+if [[ ! -d "$TARGET_FOLDER" ]]; then
+   echo "Creating target folder: $TARGET_FOLDER"
+   mkdir -p "$TARGET_FOLDER"
+fi
+
+# Get the setup script's own filename, because we'll exclude it from being copied,
+SCRIPT_NAME="$(basename "$0")"
+
+echo "Starting file copy from $ORIGINAL_FOLDER to $TARGET_FOLDER"
+echo "Script name to exclude: $SCRIPT_NAME"
+echo ""
+
+# Counter for tracking progress
+COPIED=0
+SKIPPED=0
+
+# Loop through all files in source folder
+while IFS= read -r -d '' file; do
+   filename="$(basename "$file")"
+   
+   # Skip if this is the script itself
+   if [[ "$filename" == "$SCRIPT_NAME" ]]; then
+      echo "⊘ Skipping: $filename (this script)"
+      ((SKIPPED++))
+      continue
+   fi
+   
+   # Copy the file
+   if cp "$file" "$TARGET_FOLDER/"; then
+      echo "✓ Copied: $filename"
+      ((COPIED++))
+      
+      # Set permissions to 755
+      if chmod 755 "$TARGET_FOLDER/$filename"; then
+         echo "  └─ chmod 755 applied"
+      else
+         echo "  └─ WARNING: chmod 755 failed for $filename" >&2
+      fi
+   else
+      echo "✗ ERROR: Failed to copy $filename" >&2
+      ((SKIPPED++))
+   fi
+   
+done < <(find "$ORIGINAL_FOLDER" -maxdepth 1 -type f -print0)
+
+echo ""
+echo "=========================================="
+echo "Copy complete!"
+echo "Files copied: $COPIED"
+echo "Files skipped: $SKIPPED"
+echo "Target folder: $TARGET_FOLDER"
+echo "=========================================="
+
+# Try to apply the rules, if not emit 1
+if add_sudoers_rule; then
+    echo "Operation successful"
+else
+    echo "Operation failed - sudoers rule not applied"
+    exit 1
+fi
+
+exit 0
+
+
+

core/assets/yaml_mappings/locations.yaml

@@ -0,0 +1,54 @@
+fields:
+  - name: country_code
+    path: ['country', 'code']
+    required: true
+
+  - name: code # this is city code
+    path: ['code']
+    required: true
+
+  - name: id
+    path: ['id']
+
+  - name: country_name
+    path: ['country', 'name']
+
+  - name: name # this is CITY name
+    path: ['name']
+
+  - name: time_zone
+    path: ['time_zone', 'code']
+
+  - name: operator_id
+    path: ['operator_id']
+    required: true
+
+  - name: provider_name
+    path: ['provider', 'name']
+
+  - name: is_proxy_capable
+    path: ['is_proxy_capable']
+
+  - name: is_wireguard_capable
+    path: ['is_wireguard_capable']
+    required: true
+
+  - name: is_hysteria2_capable
+    path: ['is_hysteria2_capable']
+
+  - name: is_vless_capable
+    path: ['is_vless_capable']
+
+
+# original version:
+  # locations.append((
+  # location['country']['code'],
+  # location['code'],
+  # location['id'],
+  # location['country']['name'],
+  # location['name'],
+  # location['time_zone']['code'],
+  # location['operator_id'],
+  # location['provider']['name'],
+  # location['is_proxy_capable'],
+  # location['is_wireguard_capable']))

core/assets/yaml_mappings/mapping_one.yaml

@@ -0,0 +1,8 @@
+fields:
+  - name: group_a_code
+    path: ['group_a', 'code']
+  - name: code
+    path: ['code']
+  - name: id
+    path: ['id']
+    required: true

core/assets/yaml_mappings/operators.yaml

@@ -0,0 +1,19 @@
+fields:
+  - name: id
+    path: ['id']
+    required: true
+
+  - name: name
+    path: ['name']
+
+  - name: public_key # this is ed25519
+    path: ['public_key']
+
+  - name: nostr_public_key
+    path: ['nostr_public_key']
+
+  - name: nostr_profile_reference
+    path: ['nostr_profile_reference']
+
+  - name: nostr_attestation_event_reference
+    path: ['nostr_attestation', 'event_reference']

core/controllers/ClientController.py

@@ -1,11 +1,23 @@
+# new sync refactor:
+from core.models.manage.session_management import init_session, close_session
+from core.services.sync.sync_service import coordinate_cache_sync, save_metadata
+from core.services.sync.orm_methods.sync_one_orm import sync_one_orm_model
+
+
+from core.errors.logger import logger
+
+# ORM models that can be sync'ed:
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+
+
+# prior versions:
 from core.Constants import Constants
 from core.Errors import UnknownClientPathError, UnknownClientVersionError, CommandNotFoundError
 from core.controllers.ApplicationController import ApplicationController
 from core.controllers.ApplicationVersionController import ApplicationVersionController
 from core.controllers.ClientVersionController import ClientVersionController
 from core.controllers.ConfigurationController import ConfigurationController
-from core.controllers.LocationController import LocationController
-from core.controllers.OperatorController import OperatorController
 from core.controllers.SubscriptionPlanController import SubscriptionPlanController
 from core.observers.ClientObserver import ClientObserver
 from core.observers.ConnectionObserver import ConnectionObserver
@@ -15,7 +27,13 @@ import pathlib
 import re
 import shutil
 import subprocess
+import time
 
+def evaluate_errors(final_result: dict, client_observer: Optional[ClientObserver] = None) -> str:
+    if not final_result["success"] and client_observer:
+        category = final_result["category"]
+        client_observer.notify('synchronizing', f'{category} Error! Check error logs')
+        time.sleep(2) # so they can see it.
 
 class ClientController:
 
@@ -43,9 +61,85 @@ class ClientController:
 
     @staticmethod
     def sync(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None):
+        if client_observer is not None:
+            client_observer.notify('synchronizing', "Fetching list of new data ..")
 
+        result = coordinate_cache_sync(client_observer, connection_observer)
+
+        # Outright Error:
+        if not result["success"]:
+            error_msg = result["error"]
+            if client_observer is not None:
+                client_observer.notify('synchronizing', f'Error! {error_msg}')
+                return
+
+        # Same:
+        changed_tables = result["changed_tables"]
+        if not changed_tables:
+            if client_observer is not None:
+                client_observer.notify('synchronized')
+                return
+
+        # We only make it past this point if there's New Data
+
+        # flag for after the save,
+        data_was_saved = False
+
+        # Fetch and update the real data (no longer metadata)...
+
+        # =================== ORM BASED MODELS ==================
+        """
+        Note: for the new ORM based models,
+        it does the Tor/system check in the API call itself.
+        """
+
+        if "locations" in changed_tables:
+            logger.info("Sync of Locations")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Locations List..')
+
+            final_result = sync_one_orm_model(Location, "locations")
+            evaluate_errors(final_result)
+
+
+        if "operators" in changed_tables:
+            logger.info("Sync of Operators")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Operators List..')
+
+            final_result_two = sync_one_orm_model(Operator, "operators")
+            evaluate_errors(final_result_two)
+
+        # =================== MANUAL-SQL BASED MODELS ==================
+        try:
             from core.controllers.ConnectionController import ConnectionController
-        ConnectionController.with_preferred_connection(task=ClientController.__sync, client_observer=client_observer, connection_observer=connection_observer)
+            ConnectionController.with_preferred_connection(task=ClientController.__sync, changed_tables=changed_tables, client_observer=client_observer, connection_observer=connection_observer)
+
+            # We set the flag to true,
+            # the reason we use a flag, and don't just save it right here,
+            # is because we want to isolate the success (or failure) of the real data,
+            # from the potential failure of the ORM session metadata.
+            data_was_saved = True
+
+        except:
+            # sync failed here,
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetch Failed, but you can use old data.')
+        finally:
+            if data_was_saved:
+                filtered_metadata = result["filtered_metadata"] # from the top of the function
+                save_successful = save_metadata(filtered_metadata) # the "save_data" function is inside sync_service
+
+                if client_observer is None:
+                    logger.error("Error: No client_observer to update the UI, the final part of the sync function skipped")
+                    return # can't update their UI
+
+                if save_successful:
+                    logger.info("Metadata Saved Successfully")
+                    client_observer.notify('synchronized', "Fetch & Save Complete!")
+                else:
+                    client_observer.notify('synchronizing', "Saving List of Metadata Failed.")
+
 
     @staticmethod
     def update(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None):
@@ -61,32 +155,43 @@ class ClientController:
 
         return path
 
+
     @staticmethod
-    def __sync(client_observer: Optional[ClientObserver] = None, proxies: Optional[dict] = None):
+    def __sync(changed_tables: list, client_observer: Optional[ClientObserver] = None, proxies: Optional[dict] = None):
 
+        if "applications" in changed_tables:
+            logger.info("Sync applications..")
             if client_observer is not None:
-            client_observer.notify('synchronizing')
-
+                client_observer.notify('synchronizing', 'Fetching Browser List..')
             # noinspection PyProtectedMember
             ApplicationController._sync(proxies=proxies)
+
+        if "application_versions" in changed_tables:
+            logger.info("Sync Application Versions..")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Browser Version List..')
             # noinspection PyProtectedMember
             ApplicationVersionController._sync(proxies=proxies)
+
+        if "client_version" in changed_tables:
+            logger.info("Sync of client version")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Client Version List..')
             # noinspection PyProtectedMember
             ClientVersionController._sync(proxies=proxies)
-        # noinspection PyProtectedMember
-        OperatorController._sync(proxies=proxies)
-        # noinspection PyProtectedMember
-        LocationController._sync(proxies=proxies)
+
+        if "subscriptions" in changed_tables:
+            logger.info("Sync of Subscriptions")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Subscription List..')
             # noinspection PyProtectedMember
             SubscriptionPlanController._sync(proxies=proxies)
 
         ConfigurationController.update_last_synced_at()
 
-        if client_observer is not None:
-            client_observer.notify('synchronized')
+        logger.info("Real Data Fetch Completed Successfully")
 
     @staticmethod
-
     def __update(client_observer: Optional[ClientObserver] = None, proxies: Optional[dict] = None):
 
         if ClientController.can_be_updated():

core/controllers/ConfigurationController.py

@@ -91,3 +91,15 @@ class ConfigurationController:
     @staticmethod
     def update_or_create(configuration):
         configuration.save()
+
+    @staticmethod
+    def change_firewall(new_value):
+        configuration = ConfigurationController.get_or_new()
+        configuration.firewall = new_value
+        configuration.save()
+
+    @staticmethod
+    def change_dns(new_value):
+        configuration = ConfigurationController.get_or_new()
+        configuration.dns = new_value
+        configuration.save()

core/controllers/ConnectionController.py

@@ -1,10 +1,19 @@
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+from core.services.networking.general_connection_tools.testing_evaluating import await_connection, system_uses_wireguard_interface, await_network_interface
+from core.services.networking.systemwide.systemwide_wireguard import establish_system_connection, terminate_system_connection
+from core.services.keys_and_verifications.endpoint_verification import verify_wireguard_endpoint
+
+
 from collections.abc import Callable
 from core.Constants import Constants
 from core.Errors import InvalidSubscriptionError, MissingSubscriptionError, ConnectionUnprotectedError, ConnectionTerminationError, CommandNotFoundError
 from core.controllers.ConfigurationController import ConfigurationController
-from core.controllers.ProfileController import ProfileController
+# from core.controllers.ProfileController import ProfileController
 from core.controllers.SessionStateController import SessionStateController
 from core.controllers.SystemStateController import SystemStateController
+from core.models.BaseProfile import ProfileType
 from core.models.session.SessionProfile import SessionProfile
 from core.models.system.SystemProfile import SystemProfile
 from core.models.system.SystemState import SystemState
@@ -17,18 +26,26 @@ from subprocess import CalledProcessError
 from typing import Union, Optional, Any
 import os
 import random
-import re
 import shutil
 import subprocess
-import sys
 import tempfile
 import time
+from enum import Enum
+# import sys
+# import re
 
 
 class ConnectionController:
 
     @staticmethod
     def with_preferred_connection(*args, task: Callable[..., Any], connection_observer: Optional[ConnectionObserver] = None, **kwargs):
+        """
+        This function does a task with a preferred connection type.
+
+        It is currently being CALLED UPON with keyword based kwargs only, because any args would execute immediately.
+
+        However, the *args keeps it open to future uses.
+        """
 
         connection = ConfigurationController.get_connection()
 
@@ -41,81 +58,6 @@ class ConnectionController:
         else:
             return None
 
-    @staticmethod
-    def establish_connection(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = (), connection_observer: Optional[ConnectionObserver] = None):
-
-        connection = profile.connection
-
-        if connection.needs_proxy_configuration() and not profile.has_proxy_configuration():
-
-            if profile.has_subscription():
-
-                if not profile.subscription.has_been_activated():
-                    ProfileController.activate_subscription(profile, connection_observer=connection_observer)
-
-                proxy_configuration = ConnectionController.with_preferred_connection(profile.subscription.billing_code, task=WebServiceApiService.get_proxy_configuration, connection_observer=connection_observer)
-
-                if proxy_configuration is None:
-                    raise InvalidSubscriptionError()
-
-                profile.attach_proxy_configuration(proxy_configuration)
-
-            else:
-                raise MissingSubscriptionError()
-
-        if connection.needs_wireguard_configuration() and not profile.has_wireguard_configuration():
-
-            if profile.has_subscription():
-
-                if not profile.subscription.has_been_activated():
-                    ProfileController.activate_subscription(profile, connection_observer=connection_observer)
-
-                ProfileController.register_wireguard_session(profile, connection_observer=connection_observer)
-
-            else:
-
-                if profile.is_system_profile():
-
-                    if ConnectionController.system_uses_wireguard_interface() and SystemStateController.exists():
-
-                        try:
-                            ConnectionController.terminate_system_connection()
-                        except ConnectionTerminationError:
-                            pass
-
-                raise MissingSubscriptionError()
-
-        if profile.is_session_profile():
-
-            try:
-                return ConnectionController.establish_session_connection(profile, ignore=ignore, connection_observer=connection_observer)
-
-            except ConnectionError:
-
-                if ConnectionController.__should_renegotiate(profile):
-
-                    ProfileController.register_wireguard_session(profile, connection_observer=connection_observer)
-                    return ConnectionController.establish_session_connection(profile, ignore=ignore, connection_observer=connection_observer)
-
-                else:
-                    raise ConnectionError('The connection could not be established.')
-
-        if profile.is_system_profile():
-
-            try:
-                return ConnectionController.establish_system_connection(profile, ignore=ignore, connection_observer=connection_observer)
-
-            except ConnectionError:
-
-                if ConnectionController.__should_renegotiate(profile):
-
-                    ProfileController.register_wireguard_session(profile, connection_observer=connection_observer)
-                    return ConnectionController.establish_system_connection(profile, ignore=ignore, connection_observer=connection_observer)
-
-                else:
-                    raise ConnectionError('The connection could not be established.')
-
-        return None
 
     @staticmethod
     def establish_session_connection(profile: SessionProfile, ignore: tuple[type[Exception]] = (), connection_observer: Optional[ConnectionObserver] = None):
@@ -126,25 +68,28 @@ class ConnectionController:
         port_number = None
         proxy_port_number = None
 
+        # this is a check from SessionConnection of if there's a systemwide with mask
         if profile.connection.is_unprotected():
 
-            if not ConnectionController.system_uses_wireguard_interface():
+            if not system_uses_wireguard_interface():
 
                 if not ConnectionUnprotectedError in ignore:
                     raise ConnectionUnprotectedError('Connection unprotected while the system is not using a WireGuard interface.')
                 else:
+                    from core.controllers.ProfileController import ProfileController
                     ProfileController.disable(profile)
 
         if profile.connection.code == 'tor':
 
             port_number = ConnectionService.get_random_available_port_number()
             ConnectionController.establish_tor_session_connection(port_number, connection_observer=connection_observer)
+
             session_state.network_port_numbers.tor.append(port_number)
 
         elif profile.connection.code == 'wireguard':
 
             if ConfigurationController.get_endpoint_verification_enabled():
-                ProfileController.verify_wireguard_endpoint(profile, ignore=ignore)
+                verify_wireguard_endpoint(profile, ignore=ignore)
 
             port_number = ConnectionService.get_random_available_port_number()
             ConnectionController.establish_wireguard_session_connection(profile, session_directory, port_number)
@@ -159,55 +104,16 @@ class ConnectionController:
             session_state.network_port_numbers.proxy.append(proxy_port_number)
 
         if not profile.connection.is_unprotected():
-            ConnectionController.await_connection(proxy_port_number or port_number, connection_observer=connection_observer)
+            await_connection(proxy_port_number or port_number, connection_observer=connection_observer)
 
         SessionStateController.update_or_create(session_state)
 
         return proxy_port_number or port_number
 
-    @staticmethod
-    def establish_system_connection(profile: SystemProfile, ignore: tuple[type[Exception]] = (), connection_observer: Optional[ConnectionObserver] = None):
-
-        if ConfigurationController.get_endpoint_verification_enabled():
-            ProfileController.verify_wireguard_endpoint(profile, ignore=ignore)
-
-        try:
-            ConnectionController.__establish_system_connection(profile, connection_observer)
-
-        except ConnectionError:
-
-            try:
-                ConnectionController.terminate_system_connection()
-            except ConnectionTerminationError:
-                pass
-
-            raise ConnectionError('The connection could not be established.')
-
-        except CalledProcessError:
-
-            try:
-                ConnectionController.terminate_system_connection()
-            except ConnectionTerminationError:
-                pass
-
-            try:
-                ConnectionController.__establish_system_connection(profile, connection_observer)
-
-            except (ConnectionError, CalledProcessError):
-
-                try:
-                    ConnectionController.terminate_system_connection()
-                except ConnectionTerminationError:
-                    pass
-
-                raise ConnectionError('The connection could not be established.')
-
-        ConnectionController.terminate_tor_connection()
-        time.sleep(1.0)
 
     @staticmethod
     def establish_tor_connection(connection_observer: Optional[ConnectionObserver] = None):
-
+        try:
             tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
             tor_module.start_service(connection_observer)
 
@@ -215,18 +121,27 @@ class ConnectionController:
 
                 for port_number in session_state.network_port_numbers.tor:
                     tor_module.create_session(port_number)
+        except:
+            if connection_observer is not None:
+                connection_observer.notify('custom_message', "Tor Can't Initialize")
 
-    @staticmethod
-    def terminate_tor_connection():
+    # @staticmethod
+    # def terminate_tor_connection():
 
-        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
-        tor_module.stop_service()
+    #     tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+    #     tor_module.stop_service()
 
     @staticmethod
     def establish_tor_session_connection(port_number: int, connection_observer: Optional[ConnectionObserver] = None):
-
+        try:
             tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
             tor_module.create_session(port_number, connection_observer)
+        except TorServiceInitializationError as e:
+            logger.error(f"TorServiceInitializationError. Tor Can't Start: {e}")
+            if connection_observer is not None:
+                connection_observer.notify('custom_message', "Tor Can't Initialize")
+        except Exception as e:
+            logger.error(f"Tor Can't Start: {e}")
 
     @staticmethod
     def terminate_tor_session_connection(port_number: int):
@@ -288,26 +203,9 @@ class ConnectionController:
 
         return subprocess.Popen(('proxychains4', '-f', proxychains_configuration_file_path, 'microsocks', '-p', str(proxy_port_number)), stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
 
-    @staticmethod
-    def terminate_system_connection():
 
-        if shutil.which('nmcli') is None:
-            raise CommandNotFoundError('nmcli')
-
-        if SystemStateController.exists():
-
-            process = subprocess.Popen(('nmcli', 'connection', 'delete', 'wg'), stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
-            completed_successfully = not bool(os.waitpid(process.pid, 0)[1] >> 8)
-
-            if completed_successfully or not ConnectionController.system_uses_wireguard_interface():
-
-                subprocess.run(('nmcli', 'connection', 'delete', 'hv-ipv6-sink'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-                ConnectionController.terminate_tor_connection()
-                SystemState.dissolve()
-
-            else:
-                raise ConnectionTerminationError('The connection could not be terminated.')
 
+    # Stays here for Tor based only. Will move with Tor later.
     @staticmethod
     def get_proxies(port_number: int):
 
@@ -316,107 +214,6 @@ class ConnectionController:
             https=f'socks5h://127.0.0.1:{port_number}'
         )
 
-    @staticmethod
-    def await_connection(port_number: Optional[int] = None, connection_observer: Optional[ConnectionObserver] = None):
-
-        if port_number is None:
-            ConnectionController.await_network_interface()
-
-        for retry_count in range(Constants.MAX_CONNECTION_ATTEMPTS):
-
-            if connection_observer is not None:
-
-                connection_observer.notify('connecting', dict(
-                    retry_interval=Constants.CONNECTION_RETRY_INTERVAL,
-                    maximum_number_of_attempts=Constants.MAX_CONNECTION_ATTEMPTS,
-                    attempt_count=retry_count + 1
-                ))
-
-            try:
-
-                ConnectionController.__test_connection(port_number)
-                return
-
-            except ConnectionError:
-
-                time.sleep(Constants.CONNECTION_RETRY_INTERVAL)
-                retry_count += 1
-
-        raise ConnectionError('The connection could not be established.')
-
-    @staticmethod
-    def await_network_interface():
-
-        network_interface_is_activated = False
-
-        retry_interval = .5
-        maximum_number_of_attempts = 10
-        attempt = 0
-
-        while not network_interface_is_activated and attempt < maximum_number_of_attempts:
-
-            time.sleep(retry_interval)
-
-            network_interface_is_activated = ConnectionController.system_uses_wireguard_interface()
-            attempt += 1
-
-        if not network_interface_is_activated:
-            raise ConnectionError('The network interface could not be activated.')
-
-    @staticmethod
-    def system_uses_wireguard_interface():
-
-        if shutil.which('ip') is None:
-            raise CommandNotFoundError('ip')
-
-        process = subprocess.Popen(('ip', 'route', 'get', '192.0.2.1'), stdout=subprocess.PIPE)
-        process_output = str(process.stdout.read())
-
-        return bool(re.search('dev wg', str(process_output)))
-
-    @staticmethod
-    def __establish_system_connection(profile: SystemProfile, connection_observer: Optional[ConnectionObserver] = None):
-
-        if shutil.which('dbus-send') is None:
-            raise CommandNotFoundError('dbus-send')
-
-        if shutil.which('nmcli') is None:
-            raise CommandNotFoundError('nmcli')
-
-        ConnectionController.terminate_system_connection()
-
-        try:
-            process_output = subprocess.check_output(('nmcli', 'connection', 'import', '--temporary', 'type', 'wireguard', 'file', profile.get_wireguard_configuration_path()), text=True)
-        except CalledProcessError:
-            raise ConnectionError('The connection could not be established.')
-
-        try:
-
-            connection_id = (m := re.search(r'(?<=\()([a-f0-9-]+?)(?=\))', process_output)) and m.group(1)
-            ipv6_method = subprocess.check_output(('nmcli', '-g', 'ipv6.method', 'connection', 'show', connection_id), text=True).strip()
-
-        except CalledProcessError:
-            raise ConnectionError('The connection could not be established.')
-
-        if ipv6_method in ('disabled', 'ignore'):
-
-            try:
-                subprocess.run(('dbus-send', '--system', '--print-reply', '--dest=org.freedesktop.NetworkManager', '/org/freedesktop/NetworkManager', 'org.freedesktop.DBus.Properties.Set', 'string:org.freedesktop.NetworkManager', 'string:ConnectivityCheckEnabled', 'variant:boolean:false'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
-            except CalledProcessError:
-                raise ConnectionError('The connection could not be established.')
-
-            try:
-                subprocess.run(('nmcli', 'connection', 'add', 'type', 'dummy', 'save', 'no', 'con-name', 'hv-ipv6-sink', 'ifname', 'hvipv6sink0', 'ipv6.method', 'manual', 'ipv6.addresses', 'fd7a:fd4b:54e3:077c::/64', 'ipv6.gateway', 'fd7a:fd4b:54e3:077c::1', 'ipv6.dns', '::1', 'ipv6.route-metric', '72'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
-            except CalledProcessError:
-                raise ConnectionError('The connection could not be established.')
-
-        SystemStateController.create(profile.id)
-
-        try:
-            ConnectionController.await_connection(connection_observer=connection_observer)
-
-        except ConnectionError:
-            raise ConnectionError('The connection could not be established.')
 
     @staticmethod
     def __with_tor_connection(*args, task: Callable[..., Any], connection_observer: Optional[ConnectionObserver] = None, **kwargs):
@@ -424,62 +221,9 @@ class ConnectionController:
         port_number = ConnectionService.get_random_available_port_number()
         ConnectionController.establish_tor_session_connection(port_number, connection_observer=connection_observer)
 
-        ConnectionController.await_connection(port_number, connection_observer=connection_observer)
+        await_connection(port_number, connection_observer=connection_observer)
         task_output = task(*args, proxies=ConnectionController.get_proxies(port_number), **kwargs)
 
         ConnectionController.terminate_tor_session_connection(port_number)
 
         return task_output
-
-    @staticmethod
-    def __test_connection(port_number: Optional[int] = None, timeout: float = 4.0):
-
-        request_urls = [Constants.PING_URL]
-        proxies = None
-
-        if os.environ.get('PING_URL') is None:
-
-            request_urls.extend([
-                'https://hc1.simplifiedprivacy.net',
-                'https://hc2.simplifiedprivacy.org',
-                'https://hc3.hydraveil.net'
-            ])
-
-            random.shuffle(request_urls)
-
-        if port_number is not None:
-            proxies = ConnectionController.get_proxies(port_number)
-
-        for request_url in request_urls:
-
-            command = [
-                sys.executable, '-u', '-c', 'import requests, sys\n'
-                'try:\n'
-                f'    response = requests.get(\'{request_url}\', proxies={proxies}, timeout={timeout})\n'
-                '    response.raise_for_status(); print(response.text)\n'
-                'except requests.exceptions.RequestException:\n'
-                '    sys.exit(1)'
-            ]
-
-            try:
-
-                _response = subprocess.check_output(command, text=True, timeout=timeout)
-                return None
-
-            except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
-                pass
-
-        raise ConnectionError('The connection could not be established.')
-
-    @staticmethod
-    def __should_renegotiate(profile: Union[SessionProfile, SystemProfile]):
-
-        if not profile.has_subscription():
-            raise MissingSubscriptionError()
-
-        if profile.connection.needs_wireguard_configuration() and profile.has_wireguard_configuration():
-
-            if profile.subscription.has_been_activated():
-                return True
-
-        return False

core/controllers/LocationController.py

@@ -1,22 +1,62 @@
-from core.models.Location import Location
-from core.services.WebServiceApiService import WebServiceApiService
-from typing import Optional
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.errors.logger import logger
 
+from core.models.manage.session_management import get_session
+from typing import Optional
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.orm_calls.location_calls import execute_location_sql
 
 class LocationController:
 
     @staticmethod
-    def get(country_code: str, code: str):
-        return Location.find(country_code, code)
+    def get(country_code: str, city_code: str):
+        location_object = execute_location_sql(country_code, city_code)
+        if location_object.valid:
+            return location_object.data
+        else:
+            critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}"
+            logger.error(critical_error)
+            print(critical_error)
+            return None
+
+
+        # with get_session() as session:
+        #     location_object = session.execute(
+        #         select(Location)
+        #         .where((Location.country_code == country_code) & (Location.code == city_code))
+        #         .options(joinedload(Location.operator))
+        #     ).scalar_one_or_none()
+
+        # return location_object
+
+    # legacy:
+    # Location.find(country_code, code)
+
+            
 
     @staticmethod
     def get_all():
-        return Location.all()
+        with get_session() as session:
+            all_records = session.execute(
+                select(Location)
+                .options(joinedload(Location.operator))
+            ).scalars().all()
+            return all_records
 
-    @staticmethod
-    def _sync(proxies: Optional[dict] = None):
+    # legacy:
+    # return Location.all()
 
-        locations = WebServiceApiService.get_locations(proxies)
 
-        Location.truncate()
-        Location.save_many(locations)
+# Deprecated legacy sync,
+
+# from core.services.WebServiceApiService import WebServiceApiService
+    # @staticmethod
+    # def _sync(proxies: Optional[dict] = None):
+
+    #     locations = WebServiceApiService.get_locations(proxies)
+
+    #     Location.truncate()
+    #     Location.save_many(locations)

core/controllers/OperatorController.py

@@ -1,22 +0,0 @@
-from core.models.Operator import Operator
-from core.services.WebServiceApiService import WebServiceApiService
-from typing import Optional
-
-
-class OperatorController:
-
-    @staticmethod
-    def get(id: int):
-        return Operator.find_by_id(id)
-
-    @staticmethod
-    def get_all():
-        return Operator.all()
-
-    @staticmethod
-    def _sync(proxies: Optional[dict] = None):
-
-        operators = WebServiceApiService.get_operators(proxies)
-
-        Operator.truncate()
-        Operator.save_many(operators)

core/controllers/ProfileController.py

@@ -1,3 +1,8 @@
+from core.services.networking.systemwide.systemwide_wireguard import terminate_system_connection
+from core.services.networking.general_connection_tools.testing_evaluating import system_uses_wireguard_interface
+from core.services.networking.general_connection_tools.connection_enable import establish_connection
+# from core.services.networking.systemwide.systemwide_utils import get_firewall_setting, get_dns_setting
+
 from core.Errors import InvalidSubscriptionError, MissingSubscriptionError, ConnectionTerminationError, ProfileActivationError, ProfileDeactivationError, MissingLocationError, ConnectionUnprotectedError, EndpointVerificationError, ProfileStateConflictError
 from core.controllers.ApplicationController import ApplicationController
 from core.controllers.ApplicationVersionController import ApplicationVersionController
@@ -35,29 +40,33 @@ class ProfileController:
         if profile_observer is not None:
             profile_observer.notify('created', profile)
 
-    @staticmethod
-    def update(profile: Union[SessionProfile, SystemProfile], profile_observer: ProfileObserver = None):
-
-        profile.save()
-
-        if profile_observer is not None:
-            profile_observer.notify('updated', profile)
 
     @staticmethod
-    def enable(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = (), pristine: bool = False, asynchronous: bool = False, profile_observer: ProfileObserver = None, application_version_observer: ApplicationVersionObserver = None, connection_observer: ConnectionObserver = None):
+    def enable(
+        profile: Union[SessionProfile, SystemProfile],
+        ignore: tuple[type[Exception]] = (),
+        pristine: bool = False,
+        asynchronous: bool = False,
+        profile_observer: ProfileObserver = None,
+        application_version_observer: ApplicationVersionObserver = None,
+        connection_observer: ConnectionObserver = None):
 
         from core.controllers.ConnectionController import ConnectionController
 
+        # =========== ALREADY ENABLED ============
         if ProfileController.is_enabled(profile):
-
             if not ProfileStateConflictError in ignore:
                 raise ProfileStateConflictError('The profile is already enabled or its session was not properly terminated.')
             else:
                 ProfileController.disable(profile)
 
+        # =========== PRISTINE ============
         if pristine:
             profile.delete_data()
 
+        # ============================================================================
+        # SESSION
+        # ============================================================================
         if profile.is_session_profile():
 
             application_version = profile.application_version
@@ -66,7 +75,7 @@ class ProfileController:
                 ApplicationVersionController.install(application_version, application_version_observer=application_version_observer, connection_observer=connection_observer)
 
             try:
-                port_number = ConnectionController.establish_connection(profile, ignore=ignore, connection_observer=connection_observer)
+                port_number = establish_connection(profile, ignore=ignore, connection_observer=connection_observer)
             except ConnectionError:
                 raise ProfileActivationError('The profile could not be enabled.')
 
@@ -75,10 +84,12 @@ class ProfileController:
 
             ApplicationController.launch(application_version, profile, port_number, asynchronous=asynchronous, profile_observer=profile_observer)
 
+        # ============================================================================
+        # SYSTEMWIDE
+        # ============================================================================
         if profile.is_system_profile():
-
             try:
-                ConnectionController.establish_connection(profile, ignore=ignore, connection_observer=connection_observer)
+                establish_connection(profile, ignore=ignore, connection_observer=connection_observer)
             except ConnectionError:
                 raise ProfileActivationError('The profile could not be enabled.')
 
@@ -122,7 +133,7 @@ class ProfileController:
                     raise ProfileDeactivationError('The profile could not be disabled.')
 
                 try:
-                    ConnectionController.terminate_system_connection()
+                    terminate_system_connection()
                 except ConnectionTerminationError:
                     raise ProfileDeactivationError('The profile could not be disabled.')
 
@@ -184,7 +195,7 @@ class ProfileController:
             system_state = SystemStateController.get()
 
             if system_state is not None and system_state.profile_id is profile.id:
-                return ConnectionController.system_uses_wireguard_interface()
+                return system_uses_wireguard_interface()
 
         return False
 
@@ -218,28 +229,28 @@ class ProfileController:
     def has_proxy_configuration(profile: Union[SessionProfile, SystemProfile]):
         profile.has_proxy_configuration()
 
-    @staticmethod
-    def register_wireguard_session(profile: Union[SessionProfile, SystemProfile], connection_observer: Optional[ConnectionObserver] = None):
+    # @staticmethod
+    # def register_wireguard_session(profile: Union[SessionProfile, SystemProfile], connection_observer: Optional[ConnectionObserver] = None):
 
-        from core.controllers.ConnectionController import ConnectionController
+    #     from core.controllers.ConnectionController import ConnectionController
 
-        if not profile.has_subscription():
-            raise MissingSubscriptionError()
+    #     if not profile.has_subscription():
+    #         raise MissingSubscriptionError()
 
-        if not profile.has_location():
-            raise MissingLocationError()
+    #     if not profile.has_location():
+    #         raise MissingLocationError()
 
-        wireguard_keys = ProfileController.__generate_wireguard_keys()
+    #     wireguard_keys = ProfileController.__generate_wireguard_keys()
 
-        wireguard_configuration = ConnectionController.with_preferred_connection(profile.location.country_code, profile.location.code, profile.subscription.billing_code, wireguard_keys.get('public'), task=WebServiceApiService.post_wireguard_session, connection_observer=connection_observer)
+    #     wireguard_configuration = ConnectionController.with_preferred_connection(profile.location.country_code, profile.location.code, profile.subscription.billing_code, wireguard_keys.get('public'), task=WebServiceApiService.post_wireguard_session, connection_observer=connection_observer)
 
-        if wireguard_configuration is None:
-            raise InvalidSubscriptionError()
+    #     if wireguard_configuration is None:
+    #         raise InvalidSubscriptionError()
 
-        expression = re.compile(r'^(PrivateKey =)\s?$', re.MULTILINE)
-        wireguard_configuration = re.sub(expression, r'\1 ' + wireguard_keys.get('private'), wireguard_configuration)
+    #     expression = re.compile(r'^(PrivateKey =)\s?$', re.MULTILINE)
+    #     wireguard_configuration = re.sub(expression, r'\1 ' + wireguard_keys.get('private'), wireguard_configuration)
 
-        profile.attach_wireguard_configuration(wireguard_configuration)
+    #     profile.attach_wireguard_configuration(wireguard_configuration)
 
     @staticmethod
     def get_wireguard_configuration_path(profile: Union[SessionProfile, SystemProfile]):
@@ -249,65 +260,65 @@ class ProfileController:
     def has_wireguard_configuration(profile: Union[SessionProfile, SystemProfile]):
         return profile.has_wireguard_configuration()
 
-    @staticmethod
-    def verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = ()):
+    # @staticmethod
+    # def verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = ()):
 
-        try:
-            ProfileController.__verify_wireguard_endpoint(profile)
+    #     try:
+    #         ProfileController.__verify_wireguard_endpoint(profile)
 
-        except EndpointVerificationError as error:
+    #     except EndpointVerificationError as error:
 
-            if not EndpointVerificationError in ignore:
+    #         if not EndpointVerificationError in ignore:
 
-                profile.address_security_incident()
-                raise error
+    #             profile.address_security_incident()
+    #             raise error
 
-    @staticmethod
-    def __verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile]):
+    # @staticmethod
+    # def __verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile]):
 
-        from cryptography.hazmat.primitives.asymmetric import ed25519
-        import base64
+    #     from cryptography.hazmat.primitives.asymmetric import ed25519
+    #     import base64
 
-        signature = profile.get_wireguard_configuration_metadata('Signature')
-        wireguard_public_keys = profile.get_wireguard_public_keys()
-        operator = profile.location.operator
+    #     signature = profile.get_wireguard_configuration_metadata('Signature')
+    #     wireguard_public_keys = profile.get_wireguard_public_keys()
+    #     operator = profile.location.operator
 
-        if signature is None:
-            raise EndpointVerificationError('The WireGuard endpoint\'s signature could not be determined.')
+    #     if signature is None:
+    #         raise EndpointVerificationError('The WireGuard endpoint\'s signature could not be determined.')
 
-        if not wireguard_public_keys:
-            raise EndpointVerificationError('The WireGuard endpoint\'s public key could not be determined.')
+    #     if not wireguard_public_keys:
+    #         raise EndpointVerificationError('The WireGuard endpoint\'s public key could not be determined.')
 
-        if operator is None:
-            raise EndpointVerificationError('The WireGuard endpoint\'s operator could not be determined.')
+    #     if operator is None:
+    #         raise EndpointVerificationError('The WireGuard endpoint\'s operator could not be determined.')
 
-        try:
+    #     try:
 
-            operator_public_key = ed25519.Ed25519PublicKey.from_public_bytes(bytes.fromhex(operator.public_key))
+    #         operator_public_key = ed25519.Ed25519PublicKey.from_public_bytes(bytes.fromhex(operator.public_key))
 
-            for wireguard_public_key in wireguard_public_keys:
-                operator_public_key.verify(base64.b64decode(signature), wireguard_public_key.encode('utf-8'))
+    #         for wireguard_public_key in wireguard_public_keys:
+    #             operator_public_key.verify(base64.b64decode(signature), wireguard_public_key.encode('utf-8'))
 
-        except Exception:
-            raise EndpointVerificationError('The WireGuard endpoint could not be verified.')
+    #     except Exception:
+    #         raise EndpointVerificationError('The WireGuard endpoint could not be verified.')
 
-    @staticmethod
-    def __generate_wireguard_keys():
+    # @staticmethod
+    # def __generate_wireguard_keys():
 
-        from cryptography.hazmat.primitives import serialization
-        from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey
+    #     from cryptography.hazmat.primitives import serialization
+    #     from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey
 
-        raw_private_key = X25519PrivateKey.generate()
+    #     raw_private_key = X25519PrivateKey.generate()
 
-        public_key = raw_private_key.public_key().public_bytes(
-            encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw
-        )
+    #     public_key = raw_private_key.public_key().public_bytes(
+    #         encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw
+    #     )
 
-        private_key = raw_private_key.private_bytes(
-            encoding=serialization.Encoding.Raw, format=serialization.PrivateFormat.Raw, encryption_algorithm=serialization.NoEncryption()
-        )
+    #     private_key = raw_private_key.private_bytes(
+    #         encoding=serialization.Encoding.Raw, format=serialization.PrivateFormat.Raw, encryption_algorithm=serialization.NoEncryption()
+    #     )
 
-        return dict(
-            private=base64.b64encode(private_key).decode(),
-            public=base64.b64encode(public_key).decode()
-        )
+    #     return dict(
+    #         private=base64.b64encode(private_key).decode(),
+    #         public=base64.b64encode(public_key).decode()
+    #     )

core/controllers/SubscriptionPlanController.py

@@ -9,10 +9,23 @@ class SubscriptionPlanController:
 
     @staticmethod
     def get(connection: Union[SessionConnection, SystemConnection], duration: int):
+        """
+        Called by:
+            GUI in worker.py
+
+        Purpose:
+            confirm the subscription's length is valid
+        """
         return SubscriptionPlan.find(connection, duration)
 
     @staticmethod
     def get_all(connection: Optional[Union[SessionConnection, SystemConnection]] = None):
+        """
+        Not used.  Good candidate to be cut.
+
+        GUI's create_interface_elements
+        inside duration selection page actually has hardcoded amounts
+        """
         return SubscriptionPlan.all(connection)
 
     @staticmethod

core/controllers/SystemStateController.py

@@ -12,8 +12,10 @@ class SystemStateController:
         return SystemState.exists()
 
     @staticmethod
-    def create(profile_id):
-        return SystemState(profile_id).save()
+    def create(profile_id: int, firewalled: bool, dns_set: bool) -> SystemState:
+        current_state = SystemState(profile_id, firewalled, dns_set)
+        current_state.save()
+        return current_state
 
     @staticmethod
     def update_or_create(system_state):

core/controllers/profile_state/update_profile.py

@@ -0,0 +1,117 @@
+# utils
+from core.errors.logger import logger
+from core.models.Result import Result, ResultError
+
+# JSON Models
+from core.models.BaseProfile import BaseProfile as Profile
+from core.models.Subscription import Subscription
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+
+# ORM Models
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.models.orm_calls.location_calls import get_profile_location_data
+
+SYSTEMWIDE_CHOICES = ['wireguard', 'hysteria2', 'vless']
+SESSION_CHOICES = ['wireguard', 'tor', 'proxy']
+
+"""
+Steps:
+1) Looks up the profile by id number
+2) Filters what values to save, using the ORM for some such as location
+3) Saves it with the abstract class' json serialization methods, which use the ORM methods
+
+Returns:
+    Result Object
+
+Called by:
+    GUI's editor_page
+
+Raises Errors:
+    False
+"""
+
+def update_profile(
+    profile_id: int,
+    key: str,
+    new_value: str) -> Result:
+
+    final_data = None
+    logger.info(f"[UPDATE PROFILE] Recieved profile_id {profile_id}, key {key}, and new_value {new_value}.")
+
+    if not key or not new_value:
+        return Result(valid=False, data=f"Invalid Inputs of {key} and {new_value}", error_type=ResultError.INVALID_INPUT)
+
+    profile = Profile.find_by_id(profile_id)
+    if not profile:
+        error_msg = f"Invalid profile id of {profile_id}"
+        return Result(valid=False, message=error_msg, error_type=ResultError.INVALID_INPUT)
+
+    # logger.info(f"[UPDATE PROFILE] Found a relevant profile {profile_id}.")
+
+    if key == 'dimentions':
+        profile.resolution = new_value
+
+    elif key == 'name':
+        profile.name = new_value
+
+    # ============= CONNECTION TYPE =============
+    elif key == 'connection':
+        if new_value == 'tor':
+            profile.connection.code = new_value
+            profile.connection.masked = True
+
+        elif new_value == 'just proxy':
+            profile.connection.code = 'system'
+            profile.connection.masked = True
+        else:
+            error_msg = 'System wide profiles not supported atm'
+            return Result(valid=False, message=error_msg, error_type=ResultError.NOT_SUPPORTED)
+
+    # ============= BROWSER =============
+    elif key == 'browser':
+        browser_type, browser_version = new_value.split(':', 1)
+        profile.application_version.application_code = browser_type
+        profile.application_version.version_number = browser_version
+
+    elif key == 'protocol':
+        # ============= SYSTEMWIDE =============
+        if profile.connection == 'system-wide':
+            if new_value in SYSTEMWIDE_CHOICES:
+                profile.connection.code = new_value
+                final_data = "edit_session"
+            else:
+                error_msg = f"{new_value} is not a systemwide choice"
+                return Result(valid=False, message=error_msg, error_type=ResultError.NOT_SUPPORTED)
+
+        # ============= SESSION =============
+        else:
+            if new_value in SESSION_CHOICES:
+                profile.connection.code = new_value
+                if new_value == 'wireguard':
+                    profile.connection.masked = False
+
+    # ============= LOCATION =============
+    elif key == "location":
+        country_code, city_code = new_value.split("_")
+        location = get_profile_location_data(   # SQLAlchemy
+            country_code=country_code,
+            city_code=city_code
+        )
+        # SQLAlchemy foreign key assignment — fills in id, timezone, operator, etc.
+        profile.location = location
+        
+        # Subscription gets wiped on a location change, and is outside ORM
+        profile.subscription = None
+    else:
+        return Result(valid=False, message="Invalid Value to Edit", error_type=ResultError.INVALID_INPUT)
+
+    logger.info("[UPDATE PROFILE] Passing to Profile model to save..")
+    try:
+        profile.save()
+        logger.info("[UPDATE PROFILE] Save worked, passing to the GUI the Result..")
+        return Result(valid=True, data=final_data)
+    except:
+        error_msg = "Profile save failed"
+        return Result(valid=False, message=error_msg, error_type=ResultError.UNKNOWN)

core/controllers/tickets/TicketPayController.py

@@ -9,16 +9,25 @@ from core.models.invoice.TicketInvoice import TicketInvoice
 from core.services.prepare_tickets.get_pub_key import get_pub_key
 from core.observers.BaseObserver import BaseObserver
 from core.services.payment_phase.save_and_send_intitial_billing import save_and_send_intitial_billing
-from core.services.payment_phase.check_if_paid import _check_if_paid
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
+
 from core.services.prepare_tickets.ticket_tracker import does_ticket_tracker_exist
-from core.services.networking.send_data_to_server import send_data_to_server
+
+# from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+
+
 from core.services.networking.make_url import make_url
-from core.utils.confirm_its_a_valid_key_choice import confirm_its_a_valid_key_choice
+# from core.utils.confirm_its_a_valid_key_choice import confirm_its_a_valid_key_choice
 from core.services.helpers.valid_profile_quantity import valid_profile_quantity
 from core.errors.exceptions import *
 from core.errors.logger import logger
 from core.controllers.tickets.TicketSyncController import sync_ticket_prices
 
+from core.services.payment_phase.do_we_have_billing_id import do_we_have_billing_id
+
 """
 Inputs: Which plan (key), which crypto, and how many profiles
 
@@ -47,6 +56,12 @@ def initiate_payment(
                 invoice_data_object.add_error_code("already_exists")
                 return invoice_data_object
 
+            billing_id = do_we_have_billing_id()
+            if billing_id:
+                invoice_data_object.add_error_code("billing_code_exists")
+                invoice_data_object.temp_billing_code = billing_id
+                return invoice_data_object
+
         rejected_choices = [None, "", False]
 
         if how_many_profiles in rejected_choices:
@@ -67,18 +82,6 @@ def initiate_payment(
             invoice_data_object.add_error_code("no_keyplan")
             return invoice_data_object
 
-        # confirm the key choice is among the choices from their sync file,
-        # and if not, then sync again, and try the results from that new file,
-        is_valid_key = confirm_its_a_valid_key_choice(which_key, ticket_observer)
-        if not is_valid_key:
-            sync_results = sync_ticket_prices(ticket_observer, connection_observer)
-            second_try_to_match = confirm_its_a_valid_key_choice(
-                which_key, ticket_observer
-            )
-            if not second_try_to_match:
-                invoice_data_object.add_error_code("invalid_key")
-                return invoice_data_object
-
         # get & save the public key:
         public_key_results = get_pub_key(which_key, connection_observer, "local")
 
@@ -116,6 +119,14 @@ def initiate_payment(
             payload, connection_observer, invoice_data_object
         )
 
+        if isinstance(result, ApiResponse):
+            logger.error(result.message)
+            if not result.valid:
+                invoice_data_object.add_error_code("connection_error")
+                return invoice_data_object
+            else:
+                logger.error("Critical Error with TicketPayController recieving a valid ApiResponse object.")
+
         if result == False or result == None:
             invoice_data_object.add_error_code("failed_save")
 
@@ -173,8 +184,25 @@ def check_if_paid(
         url = make_url(which_endpoint)
 
         # literally send:
-        reply = send_data_to_server(payload, url, connection_observer)
+        api_reply_object = send_data_to_server(payload, url, connection_observer)
 
-        logger.debug(f"inside ticketpay controller the reply is {reply}")
+        if not api_reply_object.valid:
+            api_reply_object = solve_api_problems(
+                api_reply_object=api_reply_object,
+                get_or_post="post",
+                url=url,
+                payload=payload,
+                connection_observer=connection_observer,
+                client_observer=None
+            )
+            if not api_reply_object.valid:
+                # 2nd try, return the error of why we have no payload:
+                error_msg = f"Connection/API Error: {api_reply_object.message}"
+                logger.error(f"[TICKET PayController] 2nd Post Request inside ticketpay controller had a {error_msg}")
+                return {"valid": False, "message": error_msg}
+
+        # return the payload with GUI/CLI to interpret results:
+        reply_dict = api_reply_object.data
+        logger.debug(f"[TICKET PayController] We have a valid reply from the API inside ticketpay controller of {reply_dict}")
+        return reply_dict
 
-        return reply

core/controllers/tickets/TicketPrepController.py

@@ -51,6 +51,9 @@ def prepare_tickets(
         ticket_observer.notify("failed_input", None)
         return {"valid": False, "error_code": "failed_input"}
 
+    notification = "Preparing Cryptography Locally"
+    ticket_observer.notify("preparing", subject=notification)
+
     # ok now we have the pre-reqs, let's use this high level orchestrator,
     prep_results = ticket_prep_orchestrator(
         how_many_profiles, ticket_observer, connection_observer

core/controllers/tickets/TicketSyncController.py

@@ -7,13 +7,16 @@ if TYPE_CHECKING:
 
 from core.Constants import Constants
 from core.observers.BaseObserver import BaseObserver
-from core.services.networking.get_data_from_server import get_data_from_server
-from core.services.helpers.save_sync_results import save_sync_results
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
 from core.errors.logger import logger
 
 
 def sync_ticket_prices(
-    ticket_observer: TicketObserver, connection_observer: ConnectionObserver
+    ticket_observer: TicketObserver,
+    connection_observer: ConnectionObserver
 ) -> dict:
     notification = f"Connecting to get Ticket Pricing..."
     ticket_observer.notify("connecting", subject=notification)
@@ -29,16 +32,24 @@ def sync_ticket_prices(
 
     url = f"{base_url}/sync"
     try:
-        sync_results = get_data_from_server(url, connection_observer)
+        api_result = get_data_from_api(url, None, connection_observer)
 
-        if sync_results in rejected_list:
+        if not api_result.valid:
+            error_msg = api_result.message
+            logger.error(f"[TICKET SYNC Controller] There's an issue with the sync of endpoint {url} the API Reply: {error_msg}")
+            api_result = solve_api_problems(
+            api_reply_object=api_result,
+            get_or_post="get",
+            url=url,
+            payload=None,
+            connection_observer=connection_observer,
+            client_observer=None
+            )
+            # 2nd try:
+            if not api_result.valid:
                 return {"valid": False, "error_code": "sync_failed"}
 
-        logger.debug(f"Inside the sync controller, sync_results is: {sync_results}")
+        return {"valid": True, "data": api_result.data}
+
     except:
         return {"valid": False, "error_code": "sync_failed"}
-
-    did_it_save = save_sync_results(sync_results)
-    logger.debug(f"Inside the sync controller, did_it_save is {did_it_save}")
-
-    return sync_results

core/controllers/tickets/UseTicketController.py

@@ -8,6 +8,8 @@ if TYPE_CHECKING:
 from core.Constants import Constants
 from core.observers.BaseObserver import BaseObserver
 from core.services.using_tickets.use_ticket_orchestrator import use_ticket_orchestrator
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
 from core.services.prepare_tickets.ticket_tracker import (
     get_data_for_a_single_ticket,
     does_ticket_tracker_exist,
@@ -111,7 +113,7 @@ def get_unused_tickets(ticket_observer: TicketObserver) -> dict:
     does_the_file_exist, path_of_file = does_ticket_tracker_exist()
     if does_the_file_exist == False:
         error_msg = f"The ticket tracker organizer file does not exist.  Check the folder {path_of_file}"
-        ticket_observer.notify("failed_input", subject=error_msg)
+        # ticket_observer.notify("failed_input", subject=error_msg)
         return {"valid": False, "message": error_msg}
 
     # Use the Model:
@@ -168,6 +170,11 @@ def use_ticket(
     ticket_observer.notify("connecting", "Connecting..")
     reply = use_ticket_orchestrator(which_ticket, which_location, connection_observer)
 
+    if isinstance(reply, ApiResponse):
+        error_msg = reply.message
+        reply_as_dict = {"valid": False, "message": f"API Failed: {error_msg}"}
+        return reply_as_dict
+    else:
         return reply
 
 

core/errors/exceptions.py

@@ -1,3 +1,29 @@
+from core.models.Result import Result, ResultError
+
+class SudoScript(Exception):
+    """There are issues with the sudo scripts"""
+    def __init__(self, result: Result):
+        self.result = result
+        if result.message and result.message is not None:
+            error_msg = result.message
+        else:
+            error_msg = result.error_type
+        super().__init__(error_msg)
+
+class MissingPreReqs(Exception):
+    """User can't take an action because they lack the prereqs"""
+    def __init__(self, result: Result):
+        self.result = result
+        if result.message and result.message is not None:
+            error_msg = result.message
+        else:
+            error_msg = result.error_type
+        super().__init__(error_msg)
+
+
+class TorServiceInitializationError(Exception):
+    pass
+
 class ApplicationError(Exception):
     pass
 

core/models/BaseConnection.py

@@ -1,17 +1,21 @@
 from dataclasses import dataclass
 from dataclasses_json import dataclass_json
 
-
 @dataclass_json
 @dataclass
 class BaseConnection:
     code: str
 
+    # Called by: ConnectionController
+    # it uses it for basic type checks on wireguard code
     def needs_wireguard_configuration(self):
         return self.code == 'wireguard'
 
+    # Called By SubscriptionPlan
     def is_session_connection(self):
         return type(self).__name__ == 'SessionConnection'
 
+    # Not called. Dead code
     def is_system_connection(self):
         return type(self).__name__ == 'SystemConnection'
+

core/models/BaseProfile.py

@@ -1,11 +1,29 @@
+from core.models.manage.session_management import get_session
+from core.errors.logger import logger
+
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
+
 from abc import ABC, abstractmethod
 from core.Constants import Constants
 from core.Helpers import write_atomically
-from core.models.Location import Location
+
+from core.models.manage.wrapper import safe_db_operation, WrapperRollback
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+
+# If switching to enums for Connection:
+# from core.models.system.SystemConnection import SystemConnection, SystemConnectionTypes
+# from core.models.session.SessionConnection import SessionConnection, SessionConnectionTypes
+
+from core.controllers.LocationController import LocationController
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+
 from core.models.Subscription import Subscription
 from core.models.session.ApplicationVersion import ApplicationVersion
 from dataclasses import dataclass, field, asdict
 from dataclasses_json import config, Exclude, dataclass_json
+
 from pathlib import Path
 from typing import Optional, Self
 import json
@@ -13,7 +31,34 @@ import os
 import re
 import shutil
 import tempfile
+from sqlalchemy.orm import Session
+from enum import Enum
 
+@safe_db_operation
+def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
+    location_object = session.execute(
+        select(Location)
+        .where((Location.country_code == country_code) & (Location.code == city_code))
+        .options(joinedload(Location.operator))
+    ).scalar_one_or_none()
+    return location_object
+
+
+def get_profile_location_data(country_code: str, city_code: str) -> Location:
+    # with get_session() as session:
+    api_reply_object = execute_location_sql(country_code, city_code)
+    if api_reply_object.valid:
+        data = api_reply_object.data
+        location_obj = api_reply_object.data
+        return location_obj
+    else:
+        logger.error(f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}")
+        return None
+
+
+class ProfileType(str, Enum):
+    SESSION = "session"
+    SYSTEM = "system"
 
 @dataclass_json
 @dataclass
@@ -23,7 +68,11 @@ class BaseProfile(ABC):
     )
     name: str
     subscription: Optional[Subscription]
-    location: Optional[Location]
+    type: ProfileType
+    location: Optional[Location] = field(metadata=config(exclude=Exclude.ALWAYS))  # SQLAlchemy object
+
+    # legacy version included it to be serialized, but now it's an SQLAlchemy object.
+    # location: Optional[Location]
 
     @abstractmethod
     def get_wireguard_configuration_path(self):
@@ -51,9 +100,21 @@ class BaseProfile(ABC):
     def is_system_profile(self):
         return type(self).__name__ == 'SystemProfile'
 
-    def save(self: Self):
 
-        config_file_contents = f'{self.to_json(indent=4)}\n'
+    def save(self: Self):
+        # === SERIALIZATION ===
+        config_dict = self.to_dict()
+
+        # === LOCATION ===
+        location_dict = self.location.convert_to_dict() # this is from SQLAlchemy, and not JSON-models, that's why it's separate.
+        if self.location:
+            config_dict["location"] = location_dict
+
+        # === FILE I/O ===
+        config_file_contents = json.dumps(config_dict, indent=4) + '\n'
+
+        # legacy version:
+        # config_file_contents = f'{self.to_json(indent=4)}\n'
 
         os.makedirs(self.get_config_path(), exist_ok=True)
         os.makedirs(self.get_data_path(), exist_ok=True)
@@ -146,6 +207,7 @@ class BaseProfile(ABC):
 
         return list([key for key, value in asdict(self).items() if value != asdict(reference).get(key)])
 
+
     @staticmethod
     def find_by_id(id: int):
 
@@ -161,18 +223,34 @@ class BaseProfile(ABC):
 
         profile['id'] = id
 
+        profiles_location = profile['location']
+
         if profile['location'] is not None:
+            # =========== GET COUNTRY & LOCATION ===========
+            if isinstance(profiles_location, dict):
+                try:
+                    country_code = profile['location']['country_code']
+                    city_code = profile['location']['code']
+                except:
+                    logger.error(f"CRITICAL ERROR! Can't find country code or city for profile id {profile['id']} inside BaseProfile")
+                    return
+            else:
+                # potentially coming from SQLAlchemy ALREADY:
+                country_code = profile.location.country_code
+                city_code = profile.location.code
 
-            location = Location.find(profile['location']['country_code'] or None, profile['location']['code'] or None)
+            # =========== GET DATA USING THAT COUNTRY & LOCATION ===========
+            location_dict = get_profile_location_data(country_code, city_code)
 
-            if location is not None:
+            if location_dict:
+                profile['location'] = location_dict
 
-                if profile['location'].get('time_zone') is not None:
-                    location.time_zone = profile['location']['time_zone']
+            # this needs error handling if there's no location or malconformed config.
 
-                profile['location'] = location
 
+        # =========== SESSION ===========
         if 'application_version' in profile:
+            profile['type'] = ProfileType.SESSION
 
             if profile['application_version'] is not None:
                 application_version = ApplicationVersion.find(profile['application_version']['application_code'] or None, profile['application_version']['version_number'] or None)
@@ -184,7 +262,10 @@ class BaseProfile(ABC):
             # noinspection PyUnresolvedReferences
             profile = SessionProfile.from_dict(profile)
 
+
+        # =========== SYSTEM ===========
         else:
+            profile['type'] = ProfileType.SYSTEM
 
             from core.models.system.SystemProfile import SystemProfile
             # noinspection PyUnresolvedReferences
@@ -192,6 +273,7 @@ class BaseProfile(ABC):
 
         return profile
 
+
     @staticmethod
     def exists(id: int):
         return re.match(r'^\d{1,2}$', str(id)) and os.path.isfile(f'{BaseProfile.__get_config_path(id)}/config.json')
@@ -224,3 +306,10 @@ class BaseProfile(ABC):
     @staticmethod
     def __get_data_path(id: int):
         return f'{Constants.HV_PROFILE_DATA_HOME}/{str(id)}'
+
+
+# legacy phased out since SQLAlchemy handles the time_zone.
+# if location is not None:
+#     if profile['location'].get('time_zone') is not None:
+#         location.time_zone = profile['location']['time_zone']
+#     profile['location'] = location

core/models/Configuration.py

@@ -1,3 +1,5 @@
+from core.errors.logger import logger
+
 from core.Constants import Constants
 from core.Helpers import write_atomically
 from dataclasses import dataclass, field
@@ -47,6 +49,30 @@ class Configuration:
         )
     )
 
+    firewall: Optional[bool] = field(
+        default=False,
+        metadata=config(
+            undefined=dataclasses_json.Undefined.EXCLUDE,
+            exclude=lambda value: value is None
+        )
+    )
+
+    dns: Optional[bool] = field(
+        default=False,
+        metadata=config(
+            undefined=dataclasses_json.Undefined.EXCLUDE,
+            exclude=lambda value: value is None
+        )
+    )
+
+    did_sudo_setup: Optional[bool] = field(
+        default=False,
+        metadata=config(
+            undefined=dataclasses_json.Undefined.EXCLUDE,
+            exclude=lambda value: value is None
+        )
+    )
+
     def save(self: Self):
 
         config_file_contents = f'{self.to_json(indent=4)}\n'
@@ -82,3 +108,31 @@ class Configuration:
     def _from_iso_format(datetime_string: str):
         date_string = datetime_string.replace('Z', '+00:00')
         return datetime.fromisoformat(date_string)
+
+
+def read_config():
+    try:
+        config_file_contents = open(f'{Constants.HV_CONFIG_HOME}/config.json', 'r').read()
+    except FileNotFoundError:
+        return None
+
+    try:
+        configuration = json.loads(config_file_contents)
+    except ValueError:
+        logger.error(f"[CONFIG] Can't load JSON config")
+    
+    return configuration
+
+def get_setting(looking_for):
+    config = read_config()
+    if not config:
+        logger.error(f"[CONFIG] Can't load the entire config")
+        return None
+
+    if looking_for not in config:
+        logger.error(f"[CONFIG] What you want isn't in the config")
+        return None
+ 
+    result = config[looking_for]
+
+    return result

core/models/DatabaseOperation.py

@@ -0,0 +1,51 @@
+
+
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional, Any
+
+class DBErrorType(Enum):
+    """Classified error categories."""
+    SUCCESS = "success"
+    NEED_MIGRATION = "need_migration"
+    MIGRATION_FAILED = "migration_failed"
+    OLD_CLIENT_NEW_API = "old_client_new_api"
+    MALFORMED_SQL = "malformed_sql_file"
+    MISSING_SQL = "missing_sql_file"
+    MISSING_DEPENDENCY = "missing_dependency"
+    INTEGRITY_ERROR = "integrity_error"
+    PERMISSION_ERROR = "permission_error"
+    FILESYSTEM_FULL = "filesystem_full"
+    DATABASE_LOCKED = "database_locked"
+    CORRUPTED_DATABASE = "corrupted_database"
+    PYTHON_MODEL_STRUCTURE = "python_model_structure"
+    UNKNOWN = "unknown"
+
+@dataclass
+class DatabaseOperation:
+    valid: bool
+    error_type: DBErrorType = DBErrorType.SUCCESS
+    data: Optional[Any] = None
+    message: Optional[str] = None
+    tried_migration: bool = False
+    tried_filtered: bool = False
+    
+    def is_recoverable(self) -> bool:
+        """Can the caller attempt a retry or manual fix?"""
+        return self.db_error_type in {
+            DBErrorType.OLD_CLIENT_NEW_API,
+            DBErrorType.NEED_MIGRATION
+        }
+    
+    def user_message(self) -> str:
+        """Human-readable error for the UI."""
+        messages = {
+            DBErrorType.SUCCESS: "Operation completed successfully.",
+            DBErrorType.NEED_MIGRATION: "Database schema needs update. Contact admin.",
+            DBErrorType.OLD_CLIENT_NEW_API: "Client version incompatible. Please upgrade.",
+            DBErrorType.MALFORMED_SQL: "Migration file is corrupted. Contact admin.",
+            DBErrorType.MISSING_SQL: "Migration file missing. Contact admin.",
+            DBErrorType.MISSING_DEPENDENCY: "Database dependency missing. Contact admin.",
+            DBErrorType.UNKNOWN: f"Unexpected error: {self.message}",
+        }
+        return messages.get(self.error_type, "Unknown error")

core/models/Location.py

@@ -1,115 +0,0 @@
-from core.models.Model import Model
-from core.models.Operator import Operator
-from dataclasses import dataclass, field
-from dataclasses_json import config, Exclude
-from typing import Optional
-
-_table_name: str = 'locations'
-
-_table_definition: str = """
-    'id' int UNIQUE,
-    'country_code' varchar,
-    'country_name' varchar,
-    'code' varchar,
-    'name' varchar,
-    'time_zone' varchar,
-    'operator_id' int,
-    'provider_name' varchar,
-    'is_proxy_capable' bool,
-    'is_wireguard_capable' bool,
-    UNIQUE(code, country_code)
-"""
-
-
-@dataclass
-class Location(Model):
-    country_code: str
-    code: str
-    id: Optional[int] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    country_name: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    name: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    time_zone: Optional[str] = None
-    operator_id: Optional[int] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    provider_name: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    is_proxy_capable: Optional[bool] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    is_wireguard_capable: Optional[bool] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    operator: Optional[Operator] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    available: Optional[bool] = field(
-        default=False,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-
-    def __post_init__(self):
-        self.operator = Operator.find_by_id(self.operator_id)
-        self.available = self.exists(self.country_code, self.code)
-
-        if isinstance(self.is_proxy_capable, int):
-            self.is_proxy_capable = bool(self.is_proxy_capable)
-
-        if isinstance(self.is_wireguard_capable, int):
-            self.is_wireguard_capable = bool(self.is_wireguard_capable)
-
-    def is_available(self):
-        return self.exists(self.country_code, self.code)
-
-    @staticmethod
-    def find_by_id(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM locations WHERE id = ? LIMIT 1', Location.factory, [id])
-
-    @staticmethod
-    def find(country_code: str, code: str):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM locations WHERE country_code = ? AND code = ? LIMIT 1', Location.factory, [country_code, code])
-
-    @staticmethod
-    def exists(country_code: str, code: str):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_exists('SELECT * FROM locations WHERE country_code = ? AND code = ?', [country_code, code])
-
-    @staticmethod
-    def all():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_all('SELECT * FROM locations', Location.factory)
-
-    @staticmethod
-    def truncate():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True)
-
-    @staticmethod
-    def save_many(locations):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        Model._insert_many('INSERT INTO locations VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', Location.tuple_factory, locations)
-
-    @staticmethod
-    def factory(cursor, row):
-        local_fields = [column[0] for column in cursor.description]
-        return Location(**{key: value for key, value in zip(local_fields, row)})
-
-    @staticmethod
-    def tuple_factory(location):
-        return location.id, location.country_code, location.country_name, location.code, location.name, location.time_zone, location.operator_id, location.provider_name, location.is_proxy_capable, location.is_wireguard_capable

core/models/Operator.py

@@ -1,56 +0,0 @@
-from core.models.Model import Model
-from dataclasses import dataclass
-
-_table_name: str = 'operators'
-
-_table_definition: str = """
-    'id' int UNIQUE,
-    'name' varchar,
-    'public_key' varchar,
-    'nostr_public_key' varchar,
-    'nostr_profile_reference' varchar,
-    'nostr_attestation_event_reference' varchar
-"""
-
-
-@dataclass
-class Operator(Model):
-    id: int
-    name: str
-    public_key: str
-    nostr_public_key: str
-    nostr_profile_reference: str
-    nostr_attestation_event_reference: str
-
-    @staticmethod
-    def find_by_id(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM operators WHERE id = ? LIMIT 1', Operator.factory, [id])
-
-    @staticmethod
-    def exists(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_exists('SELECT * FROM operators WHERE id = ?', [id])
-
-    @staticmethod
-    def all():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_all('SELECT * FROM operators', Operator.factory)
-
-    @staticmethod
-    def truncate():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True)
-
-    @staticmethod
-    def save_many(operators):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        Model._insert_many('INSERT INTO operators VALUES(?, ?, ?, ?, ?, ?)', Operator.tuple_factory, operators)
-
-    @staticmethod
-    def factory(cursor, row):
-        local_fields = [column[0] for column in cursor.description]
-        return Operator(**{key: value for key, value in zip(local_fields, row)})
-
-    @staticmethod
-    def tuple_factory(operator):
-        return operator.id, operator.name, operator.public_key, operator.nostr_public_key, operator.nostr_profile_reference, operator.nostr_attestation_event_reference

core/models/Result.py

@@ -0,0 +1,40 @@
+
+
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional, Any
+
+class ResultError(Enum):
+    """Classified error categories."""
+    SUCCESS = "success"
+    NOT_SUPPORTED = "NOT_SUPPORTED"
+    INVALID_INPUT = "invalid_input"
+    MISSING_FILE = "missing_file"
+    MISSING_DEPENDENCY = "missing_dependency"
+    CONNECTION = "connection"
+    DATABASE = "database"
+    PERMISSION = "permission"
+    SUBSCRIPTION = "subscription"
+    NMCLI = "nmcli_issues"
+    FIREWALL = "firewall"
+    CLIENT_DNS = "client_dns"
+    EXTERNAL_DNS = "external_dns"
+    INTERFACE = "interface"
+    TIMEOUT = "timeout"
+    UNKNOWN = "unknown"
+
+@dataclass
+class Result():
+    valid: bool
+    error_type: ResultError = ResultError.SUCCESS
+    data: Optional[Any] = None
+    message: Optional[str] = None
+    goal: Optional[str] = None
+    
+    def user_message(self) -> str:
+        """Human-readable error for the UI."""
+        messages = {
+            DBErrorType.SUCCESS: "Operation completed successfully.",
+            DBErrorType.UNKNOWN: f"Error: {self.message}",
+        }
+        return messages.get(self.error_type, "Unknown error")

core/models/manage/clear_sql_model.py

@@ -0,0 +1,97 @@
+
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.manage.wrapper import safe_db_operation
+from core.errors.logger import logger
+
+from typing import Type
+from sqlalchemy.orm import Session
+
+
+
+@safe_db_operation
+def _drop_sql_model_wrapped(model_class: Type, session=None) -> DatabaseOperation:
+    """
+    Purpose:
+        Drop the entire table
+
+    Confusion:
+        It gets the session from the wrapper
+        The wrapper converts the returning dictionary into DatabaseOperation.data
+    """
+    model_class.__table__.drop(bind=session.bind)
+    session.commit()
+    return {
+        'dropped': model_class.__name__,
+        'action': 'drop_fallback'
+    }
+
+
+@safe_db_operation
+def _clear_sql_model_wrapped(model_class: Type, session=None, **filters) -> DatabaseOperation:
+    """
+    Delete all records from a SQLAlchemy model table, optionally filtered.
+    Preserves the table structure.
+    
+    Args:
+        model_class: The SQLAlchemy model class to clear
+        session: Database session (injected by @safe_db_operation decorator)
+        **filters: Optional column=value filters for selective deletion
+                   e.g., clear_sql_model(User, status='inactive')
+    
+    Returns:
+        DatabaseOperation with deletion count and model name in the .data
+    
+    Raises:
+        AttributeError: If a filter column doesn't exist on the model
+    """
+    query = session.query(model_class)
+    
+    # Validate and apply filters
+    for column_name, value in filters.items():
+        if not hasattr(model_class, column_name):
+            raise AttributeError(
+                f"Model {model_class.__name__} has no column '{column_name}'"
+            )
+        column = getattr(model_class, column_name)
+        query = query.filter(column == value)
+
+    # Delete and commit    
+    deleted_count = query.delete(synchronize_session=False)
+    session.commit()
+    
+    return {
+        'deleted_rows': deleted_count,
+        'model': model_class.__name__,
+        'filters_applied': filters if filters else None
+    }
+
+
+
+def clear_sql_model(model_class: Type, **filters) -> dict:
+    """
+    Purpose:
+        Clear table; fallback to DROP if clear fails
+
+    Role:
+        Public handler that unwraps DatabaseOperation
+    """
+    result = _clear_sql_model_wrapped(model_class, **filters)
+    
+    if result.valid:
+        return result.data
+    else:
+        logger.warning(
+            f"Failed to clear {model_class.__name__} "
+            f"({result.error_type}), attempting DROP as fallback..."
+        )
+        
+        drop_result = _drop_sql_model_wrapped(model_class)
+        
+        if drop_result.valid:
+            return drop_result.data
+        else:
+            logger.error(
+                f"[CLEAR SQL MODEL] Both clear and drop failed for {model_class.__name__}: {drop_result.message}"
+            )
+            return False
+    

core/models/manage/denormalize.py

@@ -0,0 +1,74 @@
+import yaml
+
+"""
+This function extracts the value from a nested dictionary,
+By looping through each layer. And comparing the current dictionary's key to pre-made YAML key mappings.
+"""
+def extract_from_nested(obj, yaml_keys, default=None):
+    # loop through each key from the YAML mapping:
+    for each_yaml_key in yaml_keys:
+
+        # Check if each layer down of the 'object' is a dictionary with that key,
+        if isinstance(obj, dict) and each_yaml_key in obj:
+            # then extract the value for that key, and replace the object placeholder itself with it. 
+            obj = obj[each_yaml_key]
+
+            # this has the effect of going deeper into the nesting on the next round of the loop,
+
+        # But if it's not a dictionary or doesn't have the key,
+        else:
+            # then we hit a dead end,
+            return default
+
+    # We finished looping through all the keys of valid dictionaries, and extracted the values,
+    return obj
+
+"""
+This function removes unnecessary nesting,
+by comparing it to a pre-made YAML mapping
+"""
+def denormalize(data, mapping_file):
+    final_results = []
+
+    # prep the mapping config:
+    with open(mapping_file) as f:
+        mapping = yaml.safe_load(f)
+
+    # Extract out the 'data' variable if it exists, otherwise just use the data directly
+    extracted_data = data.get('data', data)
+
+    # go through the data
+    for each_item in extracted_data:
+
+        # setup temp flags/containers
+        each_denormalized_piece = {}
+        valid = True
+
+        # go through mapping:
+        for field in mapping['fields']:
+            """
+            Extract the values based on the YAML mapping
+            field['path'] - Gets a Python list of the keys from the YAML mapping
+            extract_from_nested - This function checks it's actually a dictionary and extracts it
+            """
+            value = extract_from_nested(each_item, field['path'])
+ 
+            # Store the extracted value. If it wasn't there, then it's None
+            each_denormalized_piece[field['name']] = value
+
+            # if it's required & not there, ditch it,
+            if field.get('required') and value is None:
+                print(f"It's required and value is none, skipping {value}")
+                valid = False
+                break
+
+        # Only add items where all required fields were present
+        if valid:
+            final_results.append(each_denormalized_piece)
+
+    # Return all successfully denormalized items
+    return final_results
+
+
+# Isolated testing:
+# items = denormalize(response.json(), 'example.yaml')

core/models/manage/get_from_model.py

@@ -0,0 +1,35 @@
+from core.models.manage.session_management import get_session
+from typing import Type, List, Dict, Any
+from sqlalchemy.exc import SQLAlchemyError
+
+def get_from_model(model_class: Type, **filters) -> List[Dict[str, Any]]:
+    """
+    Retrieve data from a model.  Keeping this generic for reuse
+    
+    Args:
+        model_class: The ORM model class
+        **filters: Optional column=value pairs to filter by (e.g., id=1, name='John')
+    
+    Returns:
+        List of dictionaries representing matching rows
+    """
+    with get_session() as session:
+        try:
+            query = session.query(model_class)
+            
+            # Apply filters dynamically
+            if filters:
+                for column, value in filters.items():
+                    if hasattr(model_class, column):
+                        query = query.filter(getattr(model_class, column) == value)
+            
+            # Convert to list of dicts
+            results = [
+                {c.name: getattr(row, c.name) for c in row.__table__.columns}
+                for row in query.all()
+            ]
+            return results
+        
+        except SQLAlchemyError as e:
+            raise ValueError(f"Query failed: {e}")
+

core/models/manage/insert.py

@@ -0,0 +1,115 @@
+
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import create_engine
+from typing import Type, Dict, Any
+from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from sqlalchemy.orm import Session
+
+from core.errors.logger import logger
+from core.models.manage.session_management import get_session
+from core.models.orm_models.Base import BaseModel
+
+from core.models.manage.wrapper import safe_db_operation, WrapperRollback
+from core.models.DatabaseOperation import DatabaseOperation
+
+# all models it knows how to do:
+from core.models.orm_models.CachedSync import CachedSync
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.models.orm_models.EncryptedProxy import EncryptedProxy
+
+# This is the public interface,
+def insert_into_model(model_class: Type, all_data: dict | list, override=False) -> DatabaseOperation:
+    """
+    Purpose:
+        Generic ORM insert for any model.  Keeping it generic for reuse.
+    
+    Args:
+        model_class: The ORM model class
+        all_data: Dictionary or list of dictionaries with data matching the model
+        override: If True, wipe the table before inserting
+    
+    Returns:
+        DatabaseOperation object with true/false
+    """
+    logger.info(f"All the public data preparing to be inserted is {all_data}")
+
+    # Normalize to list for uniform handling
+    data_list = all_data if isinstance(all_data, list) else [all_data]
+
+    # Call the wrapped function with normalized data
+    return _wrapped_insert(model_class=model_class, data_list=data_list, override=override)
+
+
+@safe_db_operation
+def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session, override=False) -> DatabaseOperation:
+    """Insert items. Doesn't manage session."""
+
+    if override:
+        logger.info(f"First, WIPING the pre-existing data for {model_class.__name__}. Are you sure you intended to completely delete the old data?")
+        session.query(model_class).delete()
+
+    logger.info(f"Starting insert for {model_class.__name__}")
+
+    try:
+        for each_json in data_list:
+            instance = model_class(**each_json)
+            session.add(instance)
+        session.commit()  # Commits both delete + inserts atomically
+        return DatabaseOperation(valid=True)
+
+    except TypeError as e:
+        """
+        This is triggered when an Old client was sent unknown fields from the API in the JSON,
+        And it's handled inside the wrapped function because it needs access to the variables used.
+        """
+        session.rollback()
+
+        valid_fields = {col.name for col in model_class.__table__.columns}
+        filtered_data_list = [
+            {k: v for k, v in each_json.items() if k in valid_fields}
+            for each_json in data_list
+        ]
+        
+        dropped_fields = set().union(*(set(d.keys()) - valid_fields for d in data_list))
+        if dropped_fields:
+            logger.warning(f"Old Client Dropped unknown fields: {dropped_fields}")
+        
+        try:
+            for filtered_json in filtered_data_list:
+                instance = model_class(**filtered_json)
+                session.add(instance)
+            session.commit()
+            
+            # the fix worked,
+            return DatabaseOperation(
+                valid=True,
+                tried_filtered=True,
+                message=f"Inserted after filtering {dropped_fields}"
+            )
+        
+        except Exception as retry_error:
+            # Signal wrapper to rollback
+            raise WrapperRollback(DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.OLD_CLIENT_NEW_API,
+                tried_filtered=True,
+                message=str(f"Wrapped Insert Failed on Retry of a TypeError: {retry_error}")
+            ))
+
+
+# =================== Isolation Test ======================
+"""
+Spoof Failure of an operational error to trigger/test a migration
+"""
+    # from sqlalchemy import text
+
+    # # 1. Drop a column the model expects
+    # if model_class == Location:
+    #     with get_session() as session:
+    #         print("dropping provider_name from location..")
+    #         session.execute(text("ALTER TABLE locations DROP COLUMN provider_name"))
+    #         session.commit()
+    #         print("dropped provider_name from location!!!")
+
+    # # 2. Now insert will trigger OperationalError

core/models/manage/migrations.py

@@ -0,0 +1,148 @@
+# custom
+from core.errors.logger import logger
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.Constants import Constants
+
+from core.models.manage.session_management import create_ALL_tables
+# from core.models.orm_models.Location import Location
+# from core.models.orm_models.Operator import Operator
+# from core.models.orm_models.CachedSync import CachedSync
+# from core.models.orm_models.EncryptedProxy import EncryptedProxy
+
+# generic
+from pathlib import Path
+from sqlalchemy import Column, Integer, String, create_engine, inspect, text
+from sqlalchemy.orm import declarative_base, Session
+from sqlalchemy.exc import OperationalError, DBAPIError
+from sqlalchemy import inspect
+from sqlalchemy import text
+
+"""
+Purpose:
+    Sync SQL database schema to current Python model definitions.
+
+Situation Used:
+    Python Models have new values missing from the same SQL database
+
+Metaphor:
+    This replaces something like alembic, by doing it directly
+
+Confusion:
+    This may be confusing because we're using an ORM in general,
+    but doing migrations manually.
+    
+Why:
+    The reason is because alembic is good for servers, but not clients,
+    due to a large amount of boilerplate for migrations.
+"""
+
+def get_model_columns(model_class):
+    """
+    Extract all columns directly from a model class.
+    Bypasses metadata, caching, and all SQLAlchemy indirection.
+    """
+    try:
+        mapper = inspect(model_class)
+        columns = {}
+        
+        for column in mapper.columns:
+            columns[column.name] = {
+                'type': str(column.type),
+                'nullable': column.nullable,
+                'default': column.default,
+                'primary_key': column.primary_key,
+            }
+        
+        return columns
+    except Exception as e:
+        raise ValueError(f"Failed to inspect {model_class}: {e}")
+
+
+def migrate_sql() -> DatabaseOperation:
+    from core.models.manage.session_management import engine
+    from core.models.orm_models.Base import BaseModel, Base
+    try:
+        from core.models.orm_models.Location import Location
+        from core.models.orm_models.Operator import Operator
+        from core.models.orm_models.CachedSync import CachedSync
+        from core.models.orm_models.EncryptedProxy import EncryptedProxy
+        MODELS = [Location, Operator, CachedSync, EncryptedProxy]
+        logger.info(f"[MIGRATION] Tables loaded")
+    except:
+        logger.error(f"[MIGRATION] Could not load models.  Critical failure.")
+
+    changes_made = []
+    
+    # Step 1: Extract column definitions directly from each model
+    model_schema = {}
+    for model_class in MODELS:
+        table_name = model_class.__tablename__
+        try:
+            model_schema[table_name] = get_model_columns(model_class)
+        except ValueError as e:
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.PYTHON_MODEL_STRUCTURE,
+                message=e
+            )
+        logger.info(f"[MIGRATION] Model '{table_name}': {list(model_schema[table_name].keys())}")
+    
+    # Step 2: Compare to database schema
+    inspector = inspect(engine)
+    db_tables = set(inspector.get_table_names())
+        
+    for table_name, expected_columns in model_schema.items():
+        if table_name not in db_tables:
+            changes_made.append(table_name)
+            logger.info(f"[MIGRATION] Table '{table_name}' is MISSING from database")
+        else:
+            db_columns = {col['name']: col for col in inspector.get_columns(table_name)}
+            
+            for col_name, col_info in expected_columns.items():
+                if col_name not in db_columns:
+                    changes_made.append(col_name)
+                    logger.info(f"[MIGRATION] Column '{col_name}' is MISSING from table '{table_name}'")
+    
+    # Step 3: Perform the actual migration
+    if changes_made:
+        logger.info("[MIGRATION] Changes detected. Running schema update...")
+        Base.metadata.create_all(engine)
+        
+        try:
+            with engine.begin() as conn:
+                for table_name, expected_columns in model_schema.items():
+                    inspector = inspect(engine)
+                    
+                    if table_name in inspector.get_table_names():
+                        db_columns = {col['name'] for col in inspector.get_columns(table_name)}
+                        
+                        for col_name, col_info in expected_columns.items():
+                            if col_name not in db_columns:
+                                col_type = col_info['type']
+                                nullable = "NULL" if col_info['nullable'] else "NOT NULL"
+                                print(f"[MIGRATION] Adding column '{col_name}' to '{table_name}'")
+                                conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {nullable}"))
+
+        except (OperationalError, DBAPIError) as e:
+            logger.error(f"[MIGRATION] Failed: {str(e)}")
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.UNKNOWN,
+                message=f"Schema sync failed: {str(e)}"
+            )
+
+        logger.info(f"[MIGRATION] Completed! Schema synced successfully with these changes: {changes_made}")
+        return DatabaseOperation(
+            valid=True,
+            data=changes_made,
+            message=f"Synced these tables and columns: {'; '.join(changes_made)}"
+        )
+
+    else:
+        no_migration_needed = "Schema already matches models.  No migration needed."
+        logger.info(f"[MIGRATION] {no_migration_needed}")
+        return DatabaseOperation(
+            valid=True,
+            message=no_migration_needed
+        )
+

core/models/manage/session_management.py

@@ -0,0 +1,164 @@
+from core.errors.logger import logger
+from core.models.orm_models.Base import BaseModel
+from core.models.manage.version_check import insert_new_version
+from core.Constants import Constants
+
+
+from sqlalchemy import create_engine, inspect
+from sqlalchemy.orm import sessionmaker
+import os
+from pathlib import Path
+
+"""
+Note:
+At the bottom it initializes the Session and sets it up outside the function loose.
+"""
+
+# ============================================================================
+# PATH & INITIALIZATION
+# ============================================================================
+
+def get_path():
+    """Returns XDG Base Directory path or ~/.local/share/hydra-veil"""
+    xdg_data = os.getenv("XDG_DATA_HOME")
+    if xdg_data:
+        return Path(xdg_data) / "hydra-veil"
+    return Path.home() / ".local" / "share" / "hydra-veil"
+
+def does_it_exist(filepath):
+    if filepath.exists():
+        return True
+    else:
+        return False
+
+# ============================================================================
+# GLOBAL STATE
+# ============================================================================
+
+system_path = get_path()
+database_path = system_path / "storage.db"
+engine = None
+Session = None
+_session = None
+
+# ============================================================================
+# ENGINE & SESSION MANAGEMENT
+# ============================================================================
+
+def _reinitialize_engine_and_session():
+    """
+    Recreate the global engine and Session factory.
+    Call this after deleting the database to sync globals with the new DB file.
+    """
+    global engine, Session
+    
+    system_path.mkdir(parents=True, exist_ok=True)
+    engine = create_engine(f"sqlite:///{database_path}")
+    Session = sessionmaker(bind=engine)
+
+
+def init_session():
+    """Initialize the global _session from the global Session factory."""
+    global _session
+    if Session is None:
+        raise RuntimeError("Session factory not initialized. Call _reinitialize_engine_and_session() first.")
+    _session = Session()
+
+
+def get_session():
+    """Return the global _session or raise RuntimeError."""
+    if _session is None:
+        raise RuntimeError("Session not initialized. Call init_session() first.")
+    return _session
+
+
+def close_session():
+    """Close and reset the global _session."""
+    global _session
+    if _session is not None:
+        _session.close()
+    _session = None
+
+
+# ============================================================================
+# DATABASE OPERATIONS
+# ============================================================================
+
+def create_ONLY_db_version_table():
+    """Create only the database_version table using the global engine."""
+    if engine is None:
+        raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
+    
+    from core.models.orm_models.DatabaseVersion import database_version
+    
+    try:
+        database_version.create(engine, checkfirst=True)
+        logger.info("[DB MANAGEMENT] database_version table created.")
+        
+        # CRITICAL FIX: Populate the table immediately
+        session = get_session()
+        existing_row = session.query(database_version).first()
+        
+        if not existing_row:
+            logger.info("[DB MANAGEMENT] Table was empty. Initializing with current app version.")
+            insert_new_version(session, Constants.DB_VERSION_THIS_APP_WANTS)
+        else:
+            logger.info("[DB MANAGEMENT] Table already has a version row.")
+            
+    except Exception as e:
+        logger.error(f"[DB MANAGEMENT] Failed to create/initialize database_version table: {e}")
+
+
+# def create_ONLY_db_version_table():
+#     """Create only the database_version table using the global engine."""
+#     if engine is None:
+#         raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
+    
+#     from core.models.orm_models.DatabaseVersion import database_version
+#     try:
+#         database_version.create(engine)
+#     except:
+#         logger.info("[DB MANAGEMENT] database_version already exists, but was attempted to be made again by calling create_ONLY_db_version_table.")
+
+#     # if using without checking or try except blocks:
+#     # database_version.create(engine, checkfirst=True)
+
+
+def does_db_version_table_exist():
+    if engine is None:
+        raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
+
+    inspector = inspect(engine)
+    table_exists = inspector.has_table("database_version")
+    return table_exists
+
+
+def create_ALL_tables():
+    """Create all tables from BaseModel.metadata using the global engine."""
+    if engine is None:
+        raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
+
+    from core.models.orm_models.Location import Location
+    from core.models.orm_models.Operator import Operator
+    from core.models.orm_models.CachedSync import CachedSync
+    from core.models.orm_models.EncryptedProxy import EncryptedProxy
+    from core.models.SubscriptionPlan import SubscriptionPlan
+    from core.models.session.ApplicationVersion import ApplicationVersion
+    
+    try:
+        BaseModel.metadata.create_all(engine, checkfirst=True)
+        logger.info("[DB MANAGEMENT] All Tables have been successfully created.")
+        return True
+    except:
+        logger.error("[DB MANAGEMENT] Fatal Error with creating all tables in the create_ALL_tables function of session management.")
+        return False
+
+
+# ============================================================================
+# STARTUP: Initialize engine and session on import
+# ============================================================================
+
+_reinitialize_engine_and_session()
+logger.info("[DB MANAGEMENT] Started Engine in Session Management Module..")
+init_session()
+logger.info("[DB MANAGEMENT] Initialized Session")

core/models/manage/version_check.py

@@ -0,0 +1,455 @@
+from core.models.orm_models.DatabaseVersion import database_version
+from core.Constants import Constants
+from core.errors.logger import logger
+
+# generic
+from sqlalchemy import exc, text
+from sqlalchemy.orm import Session
+from typing import Optional
+
+
+def get_database_version(session: Session) -> Optional[int]:
+    """
+    Purpose:
+        Retrieve the current database version from the single-row version table.
+
+    Rank:
+        Helper
+
+    Called By:
+        check_database_compatibility
+
+    Features:
+        Handles errors with multiple items, failed loads, and other conditions.
+
+    Returns:
+        int: The database version, or None if retrieval fails.
+    """
+    func_name = "get_database_version"
+    
+    # Validate session
+    if session is None:
+        logger.error(f"{func_name}: Session is None")
+        return None
+    
+    if not isinstance(session, Session):
+        logger.error(f"{func_name}: Invalid session type: {type(session)}")
+        return None
+    
+    try:
+        # Attempt to query the table
+        logger.debug(f"{func_name}: Attempting to query database_version table")
+        result = session.query(database_version).all()
+        
+        # Handle empty table
+        if not result:
+            logger.warning(f"{func_name}: Table is empty, no version row found")
+            return None
+        
+        # Handle multiple rows (shouldn't happen, but log it)
+        if len(result) > 1:
+            logger.warning(
+                f"{func_name}: Expected 1 row, found {len(result)}. "
+                "Table should contain only a single row. Returning first version."
+            )
+        
+        # Extract version from first row
+        version_row = result[0]
+        version = version_row.version
+        
+        # Type validation
+        if not isinstance(version, int):
+            logger.error(
+                f"{func_name}: Version column type mismatch. "
+                f"Expected int, got {type(version).__name__}: {version}"
+            )
+            return None
+        
+        if version < 0:
+            logger.warning(
+                f"{func_name}: Version is negative ({version}). "
+                "This may indicate corrupted data."
+            )
+        
+        logger.info(f"{func_name}: Successfully retrieved version: {version}")
+        return version
+    
+    except exc.NoSuchTableError:
+        logger.error(
+            f"{func_name}: Table 'database_version' does not exist. "
+            "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
+        )
+        return None
+    
+    except exc.OperationalError as e:
+        logger.error(
+            f"{func_name}: Database operational error (connection, locked, or permissions). "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except exc.DatabaseError as e:
+        logger.error(
+            f"{func_name}: General database error. "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except exc.StatementError as e:
+        logger.error(
+            f"{func_name}: SQL statement error. "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except AttributeError as e:
+        logger.error(
+            f"{func_name}: Row does not have 'version' attribute. "
+            f"Verify table schema matches definition. Details: {str(e)}"
+        )
+        return None
+    
+    except TypeError as e:
+        logger.error(
+            f"{func_name}: Type error when accessing version. "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except Exception as e:
+        logger.critical(
+            f"{func_name}: Unexpected exception type {type(e).__name__}. "
+            f"Details: {str(e)}",
+            exc_info=True
+        )
+        return None
+
+
+def insert_new_version(session: Session, new_version: int) -> bool:
+    """
+    Purpose:
+        Insert (or replace) the database version. Clears existing rows and inserts new version.
+
+    Rank:
+        Helper
+
+    Called By:
+        GUI's startup script in __main__
+
+    Args:
+        session: SQLAlchemy session
+        new_version: The version number to insert (must be a positive integer)
+    
+    Returns:
+        bool: True if successful, False otherwise.
+        does NOT raise errors.
+    """
+    func_name = "insert_new_version" # for logging purposes
+    
+    # Validate session
+    if session is None:
+        logger.error(f"{func_name}: Session is None")
+        return False
+    
+    if not isinstance(session, Session):
+        logger.error(f"{func_name}: Invalid session type: {type(session)}")
+        return False
+    
+    # Validate new_version parameter
+    if new_version is None:
+        logger.error(f"{func_name}: new_version is None")
+        return False
+    
+    if not isinstance(new_version, int):
+        logger.error(
+            f"{func_name}: new_version must be an integer, got {type(new_version).__name__}: {new_version}"
+        )
+        return False
+    
+    if new_version < 0:
+        logger.error(
+            f"{func_name}: new_version must be non-negative, got {new_version}"
+        )
+        return False
+    
+    try:
+        logger.debug(f"{func_name}: Starting transaction for version insert/replace")
+        
+        # Check if table exists before proceeding
+        try:
+            session.query(database_version).limit(1).all()
+        except exc.NoSuchTableError:
+            logger.error(
+                f"{func_name}: Table 'database_version' does not exist. "
+                "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
+            )
+            return False
+        
+        # Delete existing rows
+        logger.debug(f"{func_name}: Deleting existing version rows")
+        deleted_count = session.query(database_version).delete()
+        logger.debug(f"{func_name}: Deleted {deleted_count} existing row(s)")
+        
+        if deleted_count > 1:
+            logger.warning(
+                f"{func_name}: Deleted {deleted_count} rows. "
+                "Table should contain only a single row at any time."
+            )
+        
+        # Actually Insert the new version
+        logger.debug(f"{func_name}: Inserting new version {new_version}")
+        insert_stmt = database_version.insert().values(version=new_version)
+        session.execute(insert_stmt)
+        
+        # Commit transaction
+        session.commit()
+        logger.info(f"{func_name}: Successfully inserted version {new_version}")
+        return True
+    
+    except exc.NoSuchTableError:
+        logger.error(
+            f"{func_name}: Table 'database_version' does not exist. "
+            "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
+        )
+        session.rollback()
+        return False
+    
+    except exc.IntegrityError as e:
+        logger.error(
+            f"{func_name}: Integrity constraint violation (e.g., primary key conflict). "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.OperationalError as e:
+        logger.error(
+            f"{func_name}: Database operational error (connection, locked, or permissions). "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.DatabaseError as e:
+        logger.error(
+            f"{func_name}: General database error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.StatementError as e:
+        logger.error(
+            f"{func_name}: SQL statement error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except ValueError as e:
+        logger.error(
+            f"{func_name}: Value error during insert. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except Exception as e:
+        logger.critical(
+            f"{func_name}: Unexpected exception type {type(e).__name__}. "
+            f"Details: {str(e)}",
+            exc_info=True
+        )
+        session.rollback()
+        return False
+
+"""Spits back a string based on the reason and error."""
+def get_custom_message(reason: str, compatability_dict: dict) -> str:
+    """
+    Called By:
+        GUI's startup script in __main__
+    """
+    if reason == "upgrade":
+        custom_error = "You just upgraded HydraVeil, so let's upgrade your database to handle the new data types.  This will NOT harm your existing profiles or browser sessions."
+    elif reason == "old_app":
+        custom_error = "Upgrade Time! You're using an old version of HydraVeil, with a newer version of the database.  This means the older app would crash trying to handle the newer data.  This will NOT harm your existing profiles or browser sessions."
+
+    else:
+        error_msg = compatability_dict.get("error", "Unknown reason.")
+        custom_error = f"There was an error with upgrading your database version.  Please tell customer support: {error_msg}."
+
+    return custom_error
+
+# ============================================================================
+# ORCHESTRATOR
+# ============================================================================
+def check_database_compatibility(session: Session) -> dict:
+    """
+    Purpose:
+        Evaluates if the user's existing database is incompatible with the app's version.
+
+    Rank:
+        Orchestrator
+
+    Called By:
+        GUI's startup script in __main__
+
+    Returns:
+        Dictionary of True/False, with reason
+        NO error raises.
+    """
+    db_version_you_have = get_database_version(session)
+   
+    if db_version_you_have is None:
+        print("Nothing there")
+        # First run, corrupted, or empty
+        return {
+            "result": True,
+            "reason": "no_database" # Allow initialization still, since there's nothing to delete.
+        }
+
+    logger.info(f"[DB MANAGEMENT] The Database version this APP WANTS is {Constants.DB_VERSION_THIS_APP_WANTS}")
+    logger.info(f"[DB MANAGEMENT] The Database version this device currently has is {db_version_you_have}")
+
+    if Constants.DB_VERSION_THIS_APP_WANTS == db_version_you_have:
+        logger.info(f"[DB MANAGEMENT] Great, the database versions match exactly.")
+        return {
+            "result": True,
+            "reason": None,
+        }
+
+    # BAD!! Old version,
+    elif Constants.DB_VERSION_THIS_APP_WANTS < db_version_you_have:
+        logger.info(f"[DB MANAGEMENT] The App version is older, this is NOT compatabile")
+        # make them wipe the tables, or switch versions of the app
+        return {
+           "result": False,
+            "reason": "old_app",
+            "old_db_version": db_version_you_have
+        }
+
+    # Upgrade User:
+    elif Constants.DB_VERSION_THIS_APP_WANTS > db_version_you_have:
+        logger.info(f"[DB MANAGEMENT] The user has an upgraded App, and an older database")
+        return {
+            "result": False,
+            "reason": "upgrade",
+            "old_db_version": db_version_you_have
+        }
+
+    # this is an error
+    else:
+        logger.error(f"[DB MANAGEMENT] Unknown Error inside version check module with the King Orchestration")
+        return {
+            "result": False,
+            "reason": "error",
+            "old_db_version": db_version_you_have,
+            "error": "unknown result"
+        }
+
+
+# DEAD CODE. Not used.  But COULD be in the future.
+def delete_sync_metadata(session: Session) -> bool:
+    """
+    Called By:
+        Nobody, dead code.  I originally was going to use this but cut it's use.
+
+    Purpose:
+        Clears existing rows of metadata
+
+    Rank:
+        Helper
+
+    Args:
+        session: SQLAlchemy session
+    
+    Returns:
+        bool: True if successful, False otherwise.
+        does NOT raise errors.
+    """
+    func_name = "delete_sync_metadata" # for logging purposes
+    
+    # Validate session
+    if session is None:
+        logger.error(f"{func_name}: Session is None")
+        return False
+    
+    if not isinstance(session, Session):
+        logger.error(f"{func_name}: Invalid session type: {type(session)}")
+        return False
+    
+    try:
+        logger.debug(f"[{func_name}]: Starting transaction for metatable entry delete")
+        
+        # Check if table exists before proceeding
+        try:
+            session.query(database_version).limit(1).all()
+        except exc.NoSuchTableError:
+            logger.info(
+                f"{func_name}: Table 'database_version' does not exist, this is good because we wanted to clear it."
+            )
+            return True # TRUE! this is great that it's gone already.
+        
+        # Delete existing rows
+        logger.debug(f"{func_name}: Deleting existing version rows..")
+        deleted_count = session.query(database_version).delete()
+        logger.debug(f"{func_name}: Deleted {deleted_count} existing row(s)")
+        
+        if deleted_count > 1:
+            logger.warning(
+                f"{func_name}: Deleted {deleted_count} rows. "
+                "Table should contain only a single row at any time."
+            )
+                
+        # Commit transaction
+        session.commit()
+        return True
+        
+    except exc.IntegrityError as e:
+        logger.error(
+            f"{func_name}: Integrity constraint violation (e.g., primary key conflict). "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.OperationalError as e:
+        logger.info(
+            f"{func_name}: Sync metadata never existed. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return True # this is good
+    
+    except exc.DatabaseError as e:
+        logger.error(
+            f"{func_name}: General database error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.StatementError as e:
+        logger.error(
+            f"{func_name}: SQL statement error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except ValueError as e:
+        logger.error(
+            f"{func_name}: Value error during metadata deletion."
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except Exception as e:
+        logger.critical(
+            f"{func_name}: Unexpected exception type {type(e).__name__}. "
+            f"Details: {str(e)}",
+            exc_info=True
+        )
+        session.rollback()
+        return False

core/models/manage/wrapper.py

@@ -0,0 +1,118 @@
+# custom
+from core.errors.logger import logger
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.manage.migrations import migrate_sql
+from core.models.manage.session_management import get_session
+from core.Constants import Constants
+
+# generic
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import (
+    OperationalError,
+    IntegrityError,
+    SQLAlchemyError
+)
+from typing import Type, Dict, Any, Callable, TypeVar
+from functools import wraps
+from enum import Enum
+
+"""
+Purpose:
+    This is a wrapper for other SQL functions to catch errors and do solutions/fixes.
+
+Used/Called by:
+    insert_into_model
+
+Returns:
+    DatabaseOperation Objects
+"""
+
+def safe_db_operation(func):
+    def wrapper(*args, **kwargs):
+
+        # get the Session for the function it's wrapping,
+        session = get_session()
+
+        try:
+            # passes the session to the function,
+            result = func(*args, session=session, **kwargs)
+            
+            # If func already returns DatabaseOperation, pass it through
+            if isinstance(result, DatabaseOperation):
+                return result
+            
+            # Otherwise, wrap success
+            return DatabaseOperation(valid=True, data=result)
+        
+        except OperationalError as e:
+            logger.error(f"Schema mismatch in {func.__name__}: {e}")
+            session.rollback()
+            
+            try:
+                logger.info("Attempting migration...")
+                migrate_sql()
+
+                # try again:
+                result = func(*args, session=session, **kwargs)
+                return DatabaseOperation(
+                    valid=True,
+                    data=result,
+                    tried_migration=True,
+                    message="Recovered via migration"
+                )
+            except Exception as retry_error: # migration failed:
+                logger.error(f"Migration failed with error: {str(retry_error)}", exc_info=True)
+                session.rollback()
+                return DatabaseOperation(
+                    valid=False,
+                    error_type=DBErrorType.MIGRATION_FAILED,
+                    message=str(retry_error)
+                )
+
+        # This is really TypeErrors, passed from the insert function.
+        except WrapperRollback as e:
+            session.rollback()
+            return e.database_operation
+
+        except IntegrityError as e:
+            error_msg = f"IntegrityError caught: {e.orig}"
+            logger.error(error_msg)
+            session.rollback()
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.INTEGRITY_ERROR,
+                message=error_msg
+            )
+
+        except SQLAlchemyError as e:
+            error_msg = f"SQLAlchemyError caught: {e}"
+            logger.error(error_msg)
+            session.rollback()
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.UNKNOWN,
+                message=str(e)
+            )
+
+        except Exception as e:
+            # Catch-all for unexpected errors
+            session.rollback()
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.UNKNOWN,
+                message=str(e)
+            )
+    
+    return wrapper
+
+
+"""
+If a wrapped function needs to pass up an error to this wrapper,
+so that the wrapper can roll back sessions,
+
+Then it uses this error exception and passes the DatabaseOperation object.
+"""
+class WrapperRollback(Exception):
+    def __init__(self, database_operation: DatabaseOperation):
+        self.database_operation = database_operation
+

core/models/orm_calls/location_calls.py

@@ -0,0 +1,31 @@
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.models.manage.wrapper import safe_db_operation, WrapperRollback
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.errors.logger import logger
+
+# generic
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import Session
+
+@safe_db_operation
+def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
+    location_object = session.execute(
+        select(Location)
+        .where((Location.country_code == country_code) & (Location.code == city_code))
+        .options(joinedload(Location.operator))
+    ).scalar_one_or_none()
+    return location_object
+
+
+def get_profile_location_data(country_code: str, city_code: str) -> Location:
+    location_object = execute_location_sql(country_code, city_code)
+    if location_object.valid:
+        return location_object.data
+    else:
+        critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}"
+        logger.error(critical_error)
+        print(critical_error)
+        return None
+

core/models/orm_models/Base.py

@@ -0,0 +1,19 @@
+# base.py
+from sqlalchemy.orm import declarative_base
+from sqlalchemy import inspect
+
+Base = declarative_base()
+
+"""
+This mapper exists so the children classes
+have the ability to print human-readable strings
+as dictionaries for the UI.
+"""
+
+class BaseModel(Base):
+    __abstract__ = True
+    
+    def __repr__(self):
+        mapper = inspect(self.__class__)
+        fields = ', '.join(f'{col.name}={getattr(self, col.name)}' for col in mapper.columns)
+        return f'{self.__class__.__name__}({fields})'

core/models/orm_models/CachedSync.py

@@ -0,0 +1,24 @@
+from sqlalchemy import Integer, String, ForeignKey
+from sqlalchemy.orm import declarative_base, mapped_column, Mapped, relationship
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import Base
+
+"""
+This is one of the SQLAlchemy Models from the refactor.
+
+That’s why it lacks functions to get or insert data.
+"""
+
+class CachedSync(Base):
+    __tablename__ = 'cached_sync'
+
+    # version of the cached sync itself is the primary key
+    version: Mapped[int] = mapped_column(Integer, primary_key=True)
+
+    applications: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)
+    application_versions: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)
+    client_version: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)
+    operators: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)
+    locations: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)
+    subscriptions: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)

core/models/orm_models/DatabaseVersion.py

@@ -0,0 +1,11 @@
+from sqlalchemy import Column, Integer, MetaData, Table
+
+# Separate metadata for the compatibility table only
+compat_metadata = MetaData()
+
+database_version = Table(
+    'database_version',
+    compat_metadata,
+    Column('version', Integer, primary_key=True)
+)
+

core/models/orm_models/EncryptedProxy.py

@@ -0,0 +1,40 @@
+from sqlalchemy import Integer, String, ForeignKey
+from sqlalchemy.orm import declarative_base, mapped_column
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import Base
+from core.models.orm_models.Operator import Operator
+from core.models.orm_models.Location import Location
+
+class EncryptedProxy(Base):
+    __tablename__ = 'encryptedproxies'
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+    protocol_type: Mapped[str] = mapped_column(String, nullable=False) # changed from name "type"
+
+    username: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    password: Mapped[str] = mapped_column(String, nullable=False)
+
+    links: Mapped[str] = mapped_column(String, nullable=False)
+    subscription_url: Mapped[str] = mapped_column(String, nullable=False)
+
+    # Foreign key column
+    operator_id: Mapped[Optional[int]] = mapped_column(
+        Integer, 
+        ForeignKey("operators.id"), 
+        nullable=True, 
+        default=None
+    )
+
+    # Foreign key column
+    location_id: Mapped[Optional[int]] = mapped_column(
+        Integer, 
+        ForeignKey("locations.id"), 
+        nullable=True, 
+        default=None
+    )
+
+    operator_domain: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    operator_hysteria2_host: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    operator_vless_host: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    server_ip: Mapped[str] = mapped_column(String, nullable=True, default=None) # pre-resolved IP — avoids DNS leak at connect time

core/models/orm_models/Location.py

@@ -0,0 +1,73 @@
+
+from sqlalchemy import Integer, String, ForeignKey
+from sqlalchemy.orm import declarative_base, mapped_column, Mapped, relationship
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import BaseModel
+from core.models.orm_models.Operator import Operator
+
+
+class Location(BaseModel):
+    __tablename__ = 'locations'
+
+    # model primary key, but only used for business lookups
+    id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
+
+    # BUSINESS logic primary keys:
+    country_code: Mapped[str] = mapped_column(String, nullable=False)
+    code: Mapped[str] = mapped_column(String, unique=True, nullable=False)
+
+    # country name:
+    country_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+
+    # this is CITY name:
+    name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+
+    time_zone: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+    
+    # Foreign key column
+    operator_id: Mapped[Optional[int]] = mapped_column(
+        Integer, 
+        ForeignKey("operators.id"), 
+        nullable=True, 
+        default=None
+    )
+    
+    # Relationship with selectin eager loading
+    operator: Mapped[Optional["Operator"]] = relationship(
+        "Operator", 
+        lazy="selectin"
+    )
+
+    provider_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+    available: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_proxy_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_wireguard_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_hysteria2_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_vless_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+
+    def convert_to_dict(self):
+        return {
+            "country_code": self.country_code,
+            "code": self.code,
+            "time_zone": self.time_zone
+        }
+
+
+# to use:
+# Lookup by location/city codes
+# record = session.get(Location, (country_code, code))
+
+# example data:
+"""
+id: 7
+country_code: is
+country_name: Iceland
+code: 1
+name: Capital Region
+time_zone: Atlantic/Reykjavik
+operator_id: 6
+provider_name: FlokiNET
+is_proxy_capable: 0
+is_wireguard_capable: 1
+"""

core/models/orm_models/Operator.py

@@ -0,0 +1,18 @@
+from sqlalchemy import Integer, String
+from sqlalchemy.orm import declarative_base, mapped_column
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import BaseModel
+
+class Operator(BaseModel):
+    __tablename__ = 'operators'
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+    name: Mapped[str] = mapped_column(String, nullable=False)
+    public_key: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    nostr_public_key: Mapped[str] = mapped_column(String, nullable=False)
+    nostr_profile_reference: Mapped[str] = mapped_column(String, nullable=False)
+    nostr_attestation_event_reference: Mapped[str] = mapped_column(String, nullable=False)
+
+    # legacy:
+    # operator: Mapped[str] = mapped_column(String, nullable=False)

core/models/session/SessionConnection.py

@@ -1,7 +1,6 @@
 from core.models.BaseConnection import BaseConnection
 from dataclasses import dataclass
 
-
 @dataclass
 class SessionConnection(BaseConnection):
     masked: bool = False
@@ -11,8 +10,43 @@ class SessionConnection(BaseConnection):
         if self.code not in ('system', 'tor', 'wireguard'):
             raise ValueError('Invalid connection code.')
 
+
+    # Called by: ConnectionController
+    # doesn't even make sense, it's checking if the Session is code system. this will always be false.
     def is_unprotected(self):
         return self.code == 'system' and self.masked is False
 
+    # Called by: SessionProfile.determine_timezone 
     def needs_proxy_configuration(self):
         return self.masked is True
+
+
+# Potential refactor
+
+# from dataclasses import dataclass, field
+# from dataclasses_json import dataclass_json, config
+# from enum import Enum
+
+# class SessionConnectionTypes(str, Enum):
+#     WIREGUARD = "wireguard"
+#     TOR = "tor"
+
+# @dataclass_json
+# @dataclass
+# class SessionConnection:
+#     code: SessionConnectionTypes = field(
+#         metadata=config(
+#             encoder=lambda x: x.value,  # Convert enum to string on save
+#             decoder=lambda x: SessionConnectionTypes(x)  # Convert string to enum on load
+#         )
+#     )
+#     masked: bool = False
+
+#     def __post_init__(self):
+#         # Convert string to enum for deserialization
+#         if isinstance(self.code, str):
+#             self.code = SessionConnectionTypes(self.code)
+        
+#         # Validation
+#         if self.code not in SessionConnectionTypes:
+#             raise ValueError(f'Invalid code: {self.code}')

core/models/session/SessionProfile.py

@@ -16,7 +16,7 @@ import shutil
 class SessionProfile(BaseProfile):
     resolution: str
     application_version: Optional[ApplicationVersion]
-    connection: Optional[SessionConnection]
+    connection: Optional[SessionConnection] = None
 
     def has_connection(self):
         return self.connection is not None

core/models/system/SystemConnection.py

@@ -1,15 +1,29 @@
 from core.models.BaseConnection import BaseConnection
 from dataclasses import dataclass
 
-
+# legacy:
 @dataclass
 class SystemConnection (BaseConnection):
 
     def __post_init__(self):
 
-        if self.code != 'wireguard':
+        if self.code not in ('vless', 'hysteria2', 'wireguard'):
             raise ValueError('Invalid connection code.')
 
     @staticmethod
     def needs_proxy_configuration():
         return False
+
+# Potential refactor:
+# from enum import Enum
+# from typing import Literal
+
+# class SystemConnectionTypes(str, Enum):
+#     WIREGUARD = "wireguard"
+#     HYSTERIA2 = "hysteria2"
+#     VLESS = "vless"
+
+# @dataclass
+# class SystemConnection(BaseConnection):
+#     code: SystemConnectionTypes
+#     masked: Literal[False] = False

core/models/system/SystemProfile.py

@@ -14,7 +14,9 @@ class SystemProfile(BaseProfile):
     connection: Optional[SystemConnection]
 
     def get_system_config_path(self):
-        return self.__get_system_config_path(self.id)
+        filepath = self.__get_system_config_path(self.id)
+        the_id = self.id
+        return filepath
 
     def save(self):
 
@@ -48,10 +50,15 @@ class SystemProfile(BaseProfile):
             raise ProfileModificationError('The WireGuard configuration could not be attached.')
 
     def get_wireguard_configuration_path(self):
-        return f'{self.get_system_config_path()}/wg.conf'
+        filepath = f'{self.get_system_config_path()}/wg.conf'
+        return filepath
 
     def has_wireguard_configuration(self):
-        return os.path.isfile(f'{self.get_system_config_path()}/wg.conf')
+        filepath = f'{self.get_system_config_path()}/wg.conf'
+        if os.path.isdir(os.path.dirname(filepath)):
+            return os.path.isfile(filepath)
+        else:
+            return False
 
     def address_security_incident(self):
 
@@ -59,7 +66,6 @@ class SystemProfile(BaseProfile):
         self.__delete_wireguard_configuration()
 
     def delete(self):
-
         try:
             self.__delete_wireguard_configuration()
         except ProfileModificationError:
@@ -68,11 +74,13 @@ class SystemProfile(BaseProfile):
         if shutil.which('pkexec') is None:
             raise CommandNotFoundError('pkexec')
 
-        process = subprocess.Popen(('pkexec', 'rm', '-d', self.get_system_config_path()))
+        try:
+            process = subprocess.run(('pkexec', 'rm', '-rf', self.get_system_config_path()))
             completed_successfully = not bool(os.waitpid(process.pid, 0)[1] >> 8)
-
             if not completed_successfully:
                 raise ProfileDeletionError('The profile could not be deleted.')
+        except:
+            print("skipping the delete of the WG folder.")
 
         super().delete()
 
@@ -83,12 +91,19 @@ class SystemProfile(BaseProfile):
             if shutil.which('pkexec') is None:
                 raise CommandNotFoundError('pkexec')
 
-            process = subprocess.Popen(('pkexec', 'rm', '-d', self.get_wireguard_configuration_path()))
+            try:
+                process = subprocess.run(('pkexec', 'rm', '-rf', self.get_wireguard_configuration_path()), check=True)
+
                 completed_successfully = not bool(os.waitpid(process.pid, 0)[1] >> 8)
+            except subprocess.CalledProcessError as e:
+                    completed_successfully = True
+            except:
+                completed_successfully = True
 
             if not completed_successfully:
                 raise ProfileModificationError('The WireGuard configuration could not be deleted.')
 
     @staticmethod
     def __get_system_config_path(id: int):
-        return f'{Constants.HV_SYSTEM_PROFILE_CONFIG_PATH}/{str(id)}'
+        config_path = f'{Constants.HV_SYSTEM_PROFILE_CONFIG_PATH}/{str(id)}'
+        return config_path

core/models/system/SystemState.py

@@ -11,6 +11,8 @@ import pathlib
 @dataclass
 class SystemState:
     profile_id: int
+    firewalled: bool
+    dns_set: bool
 
     def save(self: Self):
 

core/services/WebServiceApiService.py

@@ -1,7 +1,7 @@
 from core.Constants import Constants
 from core.models.ClientVersion import ClientVersion
-from core.models.Location import Location
-from core.models.Operator import Operator
+# from core.models.Location import Location
+# from core.models.Operator import Operator
 from core.models.Subscription import Subscription
 from core.models.SubscriptionPlan import SubscriptionPlan
 from core.models.invoice.Invoice import Invoice
@@ -212,3 +212,15 @@ class WebServiceApiService:
             headers = None
 
         return requests.post(Constants.SP_API_BASE_URL + path, headers=headers, json=body, proxies=proxies)
+
+    @staticmethod
+    def get_cached_sync(proxies: Optional[dict] = None):
+
+        from requests.status_codes import codes as status_codes
+
+        response = WebServiceApiService.__get('/cachedsync', None, proxies)
+
+        if response.status_code == status_codes.OK:
+            return response.json()
+        else:
+            return None

core/services/failed_verification/is_the_key_to_blame.py

@@ -11,7 +11,9 @@ from core.services.prepare_tickets.get_public_key_by_config import get_public_ke
 from core.services.prepare_tickets.get_pub_key import key_is_in_valid_format, get_pub_key
 from core.services.failed_verification.test_if_new_key_works import test_if_new_key_works
 from core.services.networking.make_url import make_url
-from core.services.networking.get_data_from_server import get_data_from_server
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+
+
 # utils
 from core.utils.basic_operations.write_or_read_from_json import get_value_from_json_file
 from core.utils.basic_operations.write_string_to_text_file import write_string_to_text_file
@@ -70,7 +72,7 @@ def get_new_pubkey_from_api(connection_observer: ConnectionObserver) -> dict | N
     url = make_url(which_key_plan)
 
     # the result of this is a python dictionary with single '
-    api_results = get_data_from_server(url, connection_observer)
+    api_results = get_data_from_server(url, None, connection_observer)
 
     if "data" in api_results:
         new_public_key = api_results["data"]
@@ -96,7 +98,7 @@ def is_the_key_to_blame(
         # invalid key
         notification = f"Invalid Numbers on the public_key"
         ticket_observer.notify("preparing", subject=notification)
-        return {"valid": True, "message": "invalid_key"}
+        return {"valid": False, "message": "invalid_key"}
 
     new_public_key = get_new_pubkey_from_api(connection_observer)
 
@@ -121,7 +123,7 @@ def is_the_key_to_blame(
     if not result_of_comparison:
         error_msg = "New key is the SAME as the old one."
         ticket_observer.notify("preparing", subject=error_msg)
-        return {"valid": False, "message": "same"}
+        return {"valid": False, "comparison": "same"}
 
     status_update = "New key is DIFFERENT from the old one!"
     ticket_observer.notify("preparing", subject=status_update)
@@ -142,9 +144,9 @@ def is_the_key_to_blame(
 
     if quantity_results > 0:
         logger.debug("Therefore, the new key works.")
-        return {"valid": True, "comparison": "different", "matters": False}
+        return {"valid": True, "comparison": "different"}
     else:
         logger.debug(
             "Therefore, the new key doesn't help.  It is different, but also produces invalid blind signatures."
         )
-        return {"valid": False, "comparison": "different", "matters": False}
+        return {"valid": False, "comparison": "different"}

core/services/helpers/get_plan_data.py

@@ -5,13 +5,15 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 # services
 from core.services.networking.make_url import make_url
-from core.services.networking.send_data_to_server import send_data_to_server
+# from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
 
 
 # use temp billing to get the plan details
 def get_plan_data(
     temp_billing_code: str, connection_observer: ConnectionObserver
-) -> dict:
+) -> dict | ApiResponse:
     which_endpoint = "/plan"
     url = make_url(which_endpoint)
     payload = {"temp_billing_code": temp_billing_code}

core/services/helpers/manage_assets.py

@@ -0,0 +1,53 @@
+from core.Constants import Constants
+from core.errors.logger import logger
+
+# generic
+from pathlib import Path
+import shutil
+import os
+
+def assets_folder_setup():
+    initial_appimage_assets = f"{Constants.APPDIR_HOME}/assets"
+    current_assets_folder = f"{Constants.HV_DATA_HOME}/assets"
+    if os.path.exists(current_assets_folder):
+        return True
+    return copy_folders(initial_appimage_assets, current_assets_folder)
+    
+
+
+def sudo_assets_folder_setup() -> bool:
+    # initial_appimage_assets = f"{Constants.APPDIR_HOME}/assets/sudo_scripts"
+    initial_appimage_assets = f"/home/a/VPN/code/DEV/sp-hydra-veil-core/core/assets/sudo_scripts"
+    current_assets_folder = f"{Constants.HOME}/Downloads/hydraveil_sudo_scripts"
+    return copy_folders(initial_appimage_assets, current_assets_folder)
+
+
+def copy_folders(initial_appimage_assets: str, current_assets_folder: str) -> bool:
+    try:
+        os.makedirs(current_assets_folder, exist_ok=True)
+        for item in os.listdir(initial_appimage_assets):
+            src = os.path.join(initial_appimage_assets, item)
+            dst = os.path.join(current_assets_folder, item)
+            if os.path.isdir(src):
+                shutil.copytree(src, dst)
+            else:
+                shutil.copy2(src, dst)
+        logger.info(f"[MANAGE ASSETS] Successfully copied over the folder.")
+        return True
+    except Exception as e:
+        logger.error(f"[MANAGE ASSETS] initial_appimage_assets is {initial_appimage_assets}")
+        logger.error(f"[MANAGE ASSETS] current_assets_folder is {current_assets_folder}")
+        logger.error(f"[MANAGE ASSETS] ERROR: {e}")
+        return False
+
+
+def cleanup_setup_dir():
+    """After install completes"""
+    delete_this_folder = f"{Constants.HOME}/Downloads/hydraveil_sudo_scripts"
+    try:
+        shutil.rmtree(delete_this_folder)
+        logger.info(f"[MANAGE ASSETS] Cleaned up {delete_this_folder}")
+        return True
+    except Exception as e:
+        logger.error(f"[MANAGE ASSETS] We could not delete {delete_this_folder} because {e}")
+        return False

core/services/helpers/setup_sudo_scripts.py

@@ -0,0 +1,67 @@
+from core.services.helpers.manage_assets import sudo_assets_folder_setup
+from core.utils.basic_operations.confirm_files_exist import confirm_files_and_folders_exist
+from core.models.Result import Result, ResultError
+from core.Constants import Constants
+from core.errors.logger import logger
+
+import subprocess
+from pathlib import Path
+import shutil
+import os
+from subprocess import CalledProcessError
+import copy
+
+FILES_TO_CHECK = ["setup.sh", "firewall", "dns"]
+TARGET_FOLDER = "/opt/hydra-veil"
+
+def auto_install_sudo_script() -> Result:
+    setup_dir = Path(f"{Constants.HOME}/Downloads/hydraveil_sudo_scripts")
+    setup_script = Path(setup_dir, 'setup.sh')
+    original_linux_user = os.getlogin()
+
+    # Make sure the folder exists
+    files_exist = confirm_files_and_folders_exist(
+        folder_path=setup_dir, files_to_check=FILES_TO_CHECK
+    )
+
+    if not files_exist:
+        copied = sudo_assets_folder_setup()
+        if not copied:
+            return Result(valid=False, message="Could not copy scripts to begin.")
+
+    try:
+        result = subprocess.run(
+            ['pkexec', 'env', f'ORIGINAL_FOLDER={setup_dir}', f'TARGET_FOLDER={TARGET_FOLDER}', f'LINUX_USER={original_linux_user}', 'bash', setup_script],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode == 0:
+            logger.info("Sudo Setup Ran. Now testing..")
+            return Result(valid=True)
+        else:
+            logger.error(f"Sudo setup setup failed: {result.stderr.strip()}")
+            return Result(valid=False, message=result.stderr.strip())
+    except CalledProcessError as e:
+        return Result(valid=False, message=e)
+    except Exception as e:
+        return Result(valid=False, message=e)
+
+def test_if_in_sudo_folder() -> Result:
+    final_sudo_folder = Path("/opt/hydra-veil/")
+    _files_to_check = FILES_TO_CHECK.copy() # to not affect the other functions.
+    
+    # setup script itself doesn't get copied over:
+    _files_to_check.remove("setup.sh")
+
+    # Now verify it copied them:
+    sudo_files_copied = confirm_files_and_folders_exist(
+        folder_path=final_sudo_folder, files_to_check=_files_to_check
+    )
+
+    if sudo_files_copied:
+        return Result(valid=True, message="Sudo Script Setup Completed Successfully!")
+    else:
+        error_msg = "We failed to find the files in the sudo folder."
+        logger.error(error_msg)
+        return Result(valid=False, message=error_msg)
+

core/services/keys_and_verifications/endpoint_verification.py

@@ -0,0 +1,48 @@
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+from core.models.BaseProfile import BaseProfile as Profile
+from core.Errors import InvalidSubscriptionError, MissingSubscriptionError, ConnectionTerminationError, ProfileActivationError, ProfileDeactivationError, MissingLocationError, ConnectionUnprotectedError, EndpointVerificationError, ProfileStateConflictError
+from typing import Union, Optional
+import base64
+
+
+def verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = ()):
+
+    try:
+        __verify_wireguard_endpoint(profile)
+
+    except EndpointVerificationError as error:
+
+        if not EndpointVerificationError in ignore:
+
+            profile.address_security_incident()
+            raise error
+
+def __verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile]):
+
+    from cryptography.hazmat.primitives.asymmetric import ed25519
+    import base64
+
+    signature = profile.get_wireguard_configuration_metadata('Signature')
+    wireguard_public_keys = profile.get_wireguard_public_keys()
+    operator = profile.location.operator
+
+    if signature is None:
+        raise EndpointVerificationError('The WireGuard endpoint\'s signature could not be determined.')
+
+    if not wireguard_public_keys:
+        raise EndpointVerificationError('The WireGuard endpoint\'s public key could not be determined.')
+
+    if operator is None:
+        raise EndpointVerificationError('The WireGuard endpoint\'s operator could not be determined.')
+
+    try:
+
+        operator_public_key = ed25519.Ed25519PublicKey.from_public_bytes(bytes.fromhex(operator.public_key))
+
+        for wireguard_public_key in wireguard_public_keys:
+            operator_public_key.verify(base64.b64decode(signature), wireguard_public_key.encode('utf-8'))
+
+    except Exception:
+        raise EndpointVerificationError('The WireGuard endpoint could not be verified.')
+

core/services/keys_and_verifications/wireguard_keys.py

@@ -0,0 +1,66 @@
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+from core.Errors import MissingSubscriptionError, MissingLocationError, InvalidSubscriptionError
+from core.services.WebServiceApiService import WebServiceApiService
+from core.controllers.ConnectionController import ConnectionController
+from core.observers.ConnectionObserver import ConnectionObserver
+from typing import Union, Optional
+import base64
+import re
+
+def register_wireguard_session(
+    profile: Union[SessionProfile, SystemProfile], 
+    connection_observer: Optional[ConnectionObserver] = None
+):
+    """Register a WireGuard session for the given profile."""
+    
+    if not profile.has_subscription():
+        raise MissingSubscriptionError()
+    
+    if not profile.has_location():
+        raise MissingLocationError()
+    
+    wireguard_keys = _generate_wireguard_keys()
+    
+    wireguard_configuration = ConnectionController.with_preferred_connection(
+        profile.location.country_code, 
+        profile.location.code, 
+        profile.subscription.billing_code, 
+        wireguard_keys.get('public'), 
+        task=WebServiceApiService.post_wireguard_session, 
+        connection_observer=connection_observer
+    )
+    
+    if wireguard_configuration is None:
+        raise InvalidSubscriptionError()
+    
+    wireguard_configuration = _inject_private_key(wireguard_configuration, wireguard_keys.get('private'))
+    profile.attach_wireguard_configuration(wireguard_configuration)
+
+
+def _generate_wireguard_keys() -> dict:
+    """Generate WireGuard public/private key pair."""
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey
+
+    raw_private_key = X25519PrivateKey.generate()
+
+    public_key = raw_private_key.public_key().public_bytes(
+        encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw
+    )
+
+    private_key = raw_private_key.private_bytes(
+        encoding=serialization.Encoding.Raw, format=serialization.PrivateFormat.Raw, encryption_algorithm=serialization.NoEncryption()
+    )
+
+    return dict(
+        private=base64.b64encode(private_key).decode(),
+        public=base64.b64encode(public_key).decode()
+    )
+
+
+def _inject_private_key(config: str, private_key: str) -> str:
+    """Inject private key into WireGuard config."""
+    expression = re.compile(r'^(PrivateKey =)\s?$', re.MULTILINE)
+    return re.sub(expression, r'\1 ' + private_key, config)
+

core/services/networking/api_requests/ApiResponseModel.py

@@ -0,0 +1,48 @@
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional, Any
+
+class ErrorType(Enum):
+    """Classified error categories."""
+    SUCCESS = "success"
+    TOR_NOT_WORKING = "tor_not_working"
+    TOR_DNS_BLOCKED = "tor_dns_blocked"
+    DNS_RESOLUTION = "dns_resolution"
+    QUAD9_DNS_RESOLUTION = "quad9_dns_resolution"
+    NO_INTERNET = "no_internet"
+    NETWORK_ERROR = "network_error"
+    INVALID_INPUT = "invalid_input"
+    RATE_LIMITED = "rate_limited"
+    SERVER_ERROR = "server_error"
+    DEVELOPER_ERROR = "developer_error"
+    UNKNOWN = "unknown"
+
+@dataclass
+class ApiResponse:
+    """Unified response from all API operations."""
+    valid: bool
+    error_type: Optional[ErrorType] = None
+    data: Optional[Any] = None
+    message: Optional[str] = None
+    retry_now: bool = False
+    retry_later: bool = False
+    tor: bool = False
+    ip_address: str = None
+    ask_clearweb: bool = None
+
+    def to_dict(self) -> dict:
+        """Convert to dict for backwards compatibility."""
+        return {
+            "valid": self.valid,
+            "error_code": self.error_type.value if self.error_type else None,
+            "data": self.data,
+            "message": self.message,
+            "retry_now": self.retry_now,
+            "retry_later": self.retry_later,
+            "tor": self.tor,
+            "ask_clearweb": self.ask_clearweb
+        }
+    
+    def get(self, key: str, default=None):
+        """Dict-like access for backwards compatibility."""
+        return getattr(self, key, default)

core/services/networking/api_requests/step1_get_or_post.py

@@ -0,0 +1,102 @@
+from core.services.networking.api_requests.step2_execute import _execute_tor_request, _execute_regular_request
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
+from core.services.networking.api_requests.subtools.get_connection_type import get_connection_type
+from core.services.networking.api_requests.subtools.replace_http_with_https import replace_http_with_https
+
+from core.Constants import Constants
+from core.observers.BaseObserver import BaseObserver
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+from typing import Optional
+
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+
+def get_data_from_api(
+    full_request_url: str,
+    client_observer: Optional[ClientObserver] = None,
+    connection_observer: Optional[ConnectionObserver] = None,
+    clearweb_resolved_ip: str = None,
+) -> ApiResponse:
+    """
+    GET wrapper. Desktop passes observer for updates; Android passes None.
+    """
+    if not full_request_url:
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.INVALID_INPUT,
+            message="Input URL is not properly configured"
+        )
+    
+    connection_type = get_connection_type()
+    
+    if connection_type == "tor":
+        # Tor: can push intermediate updates (desktop-only)
+        result_object = _execute_tor_request(
+            "get",
+            full_request_url,
+            None, # get request has no payload
+            connection_observer,
+            client_observer,
+            timeout=30,
+            clearweb_resolved_ip=clearweb_resolved_ip
+        )
+    else:
+        # Regular, no tor:
+        result_object = _execute_regular_request(
+            "get",
+            full_request_url,
+            None, # get request has no payload
+            connection_observer,
+            client_observer,
+            timeout=15
+        )
+
+    if not result_object.valid and client_observer:
+        error_msg = result_object.message
+        client_observer.notify('synchronizing', f"Connection Issue: {error_msg}")
+    
+    return result_object
+
+
+def send_data_to_server(
+    payload: dict,
+    original_url: str,
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+    clearweb_resolved_ip: str = None,
+) -> ApiResponse:
+    """
+    POST wrapper. Same pattern as GET.
+    """
+    url = replace_http_with_https(original_url)
+    connection_type = get_connection_type()
+    
+    if connection_type == "tor":
+        result_object = _execute_tor_request(
+            "post",
+            url,
+            payload,
+            connection_observer,
+            client_observer,
+            timeout=10,
+            clearweb_resolved_ip=clearweb_resolved_ip
+        )
+    else:
+        result_object = _execute_regular_request(
+            "post",
+            url,
+            payload,
+            connection_observer,
+            client_observer,
+            timeout=5
+        )
+    
+    # Push final error to UI if provided
+    if not result_object.valid and client_observer:
+        error_msg = result_object.message
+        client_observer.notify('synchronizing', f"Connection Issue: {error_msg}")
+
+    return result_object

core/services/networking/api_requests/step2_execute.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import requests
+
+from core.Constants import Constants
+
+# Networking Group
+from core.services.networking.api_requests.step3_classify_http import classify_http_response
+from core.services.networking.api_requests.step4_error_classifier import classify_request_error
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.subtools.custom_dns import prep_url_with_known_ip, load_custom_dns_resolver
+
+# tor
+from essentials.modules.TorModule import TorModule
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+
+from core.observers.ClientObserver import ClientObserver
+
+# errors
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+# generic
+import json, os
+from typing import Any
+import time
+
+
+
+def _execute_tor_request(
+    method: str,
+    url: str,
+    payload: Optional[dict],
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+    timeout: int = 30,
+    clearweb_resolved_ip: str = None # if DNS via Tor is blocked
+) -> ApiResponse:
+    """
+    Tor request with optional intermediate UI feedback (desktop-only).
+    """
+    import requests
+    port_number = None
+    tor_module = None
+    
+    try:
+        if client_observer:
+            status_update = "Initializing Tor connection..."
+            client_observer.notify('synchronizing', status_update)
+
+        port_number = ConnectionService.get_random_available_port_number()
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+        
+        if client_observer:
+            status_update = f"Executing {method.upper()} via Tor..."
+            client_observer.notify('synchronizing', status_update)
+
+        proxies = {"http": f"socks5h://127.0.0.1:{port_number}",
+                   "https": f"socks5h://127.0.0.1:{port_number}"}
+
+        # ================= OPTION: DNS OUTSIDE TOR. (IF DNS VIA TOR IS BLOCKED) ===============
+        if clearweb_resolved_ip:
+            CustomHostnameAdapter = load_custom_dns_resolver()
+
+            url_with_ip, original_hostname = prep_url_with_known_ip(clearweb_resolved_ip, url)
+
+            # Set up session with custom adapter for DNS resolution:
+            session = requests.Session()
+            session.mount('https://', CustomHostnameAdapter(original_hostname))
+            session.headers.update({"Host": original_hostname})
+
+            # Make the request to the raw IP, but tell the server it's the original domain
+            if method.lower() == "get":
+                response = session.get(url_with_ip, proxies=proxies, timeout=timeout, verify=True)
+            else:
+                response = requests.post(url_with_ip, json=payload, proxies=proxies, timeout=timeout, verify=True)
+
+        else:
+            # ================= USING DNS VIA TOR (normal-use) ===============
+            if method.lower() == "get":
+                response = requests.get(url, proxies=proxies, timeout=timeout)
+            else:
+                response = requests.post(url, json=payload, proxies=proxies, timeout=timeout)
+
+        # ================= Working Response ===============
+        result = classify_http_response(response)
+        
+    except Exception as e:
+        logger.debug(f"Tor {method.upper()} failed inside _execute_tor_request: {type(e).__name__}: {e}")
+        result = classify_request_error(url, "tor", connection_observer)
+    
+    finally:
+        if tor_module and port_number:
+            try:
+                tor_module.destroy_session(port_number)
+            except Exception as e:
+                logger.warning(f"Error destroying Tor session: {e}")
+    
+    return result
+
+
+def _execute_regular_request(
+    method: str,
+    url: str,
+    payload: Optional[dict],
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[client_observer] = None,
+    timeout: int = 20,
+) -> ApiResponse:
+    """
+    Regular (non-Tor) request. NO observer coupling.
+    Cross-platform reusable: desktop UI and Android/Kivy both consume ApiResponse.
+    """
+    import requests
+
+    try:
+        logger.debug(f"Executing {method.upper()} to {url}")
+        
+        if method.lower() == "get":
+            response = requests.get(url, timeout=timeout)
+        else:
+            response = requests.post(url, json=payload, timeout=timeout)
+        
+        return classify_http_response(response)
+    
+    except Exception as e:
+        logger.debug(f"{method.upper()} request failed: {type(e).__name__}: {e}")
+        return classify_request_error(url, "regular", connection_observer)

core/services/networking/api_requests/step3_classify_http.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import requests
+
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+import json
+
+def classify_http_response(response) -> ApiResponse:
+    """
+    Classify HTTP response status and extract error info.
+    Replaces evaluate_response().
+    """
+    import requests
+
+    if not isinstance(response, requests.Response):
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.NETWORK_ERROR,
+            message="Invalid response object"
+        )
+
+    print(f"We're inside classify http testing the status code of the response: {response.status_code}")
+
+    # Success case
+    if 200 <= response.status_code < 300:
+        try:
+            data = response.json()
+            return ApiResponse(valid=True, data=data)
+        except ValueError:
+            return ApiResponse(
+                valid=True,
+                data=response.text
+            )
+    
+    # Rate limiting
+    if response.status_code == 429:
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.RATE_LIMITED,
+            message="Rate limit exceeded",
+            retry_now=True
+        )
+    
+    # Client errors (4xx)
+    if 400 <= response.status_code < 500:
+        try:
+            resp_json = response.json()
+            error_msg = resp_json.get("message") or resp_json.get("error") or resp_json.get("error_code")
+        except ValueError:
+            error_msg = response.text
+        
+        error_map = {
+            404: ErrorType.INVALID_INPUT,
+            400: ErrorType.INVALID_INPUT,
+            401: ErrorType.INVALID_INPUT,
+            403: ErrorType.INVALID_INPUT,
+            405: ErrorType.INVALID_INPUT,
+        }
+        error_type = error_map.get(response.status_code, ErrorType.INVALID_INPUT)
+        
+        return ApiResponse(
+            valid=False,
+            error_type=error_type,
+            message=error_msg or f"Client error {response.status_code}"
+        )
+    
+    # Server errors (5xx)
+    if response.status_code >= 500:
+        retry_now = response.status_code in (502, 503, 504)
+        retry_later = response.status_code in (500, 501, 505)
+        
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.SERVER_ERROR,
+            message=f"Server error {response.status_code}",
+            retry_now=retry_now,
+            retry_later=retry_later
+        )
+    
+    return ApiResponse(
+        valid=False,
+        error_type=ErrorType.UNKNOWN,
+        message=f"Unexpected status {response.status_code}"
+    )

core/services/networking/api_requests/step4_error_classifier.py

@@ -0,0 +1,91 @@
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+
+
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
+# diagnosis services
+from core.services.networking.api_requests.subtools.internet_test import do_we_have_internet
+from core.services.networking.api_requests.subtools.dns_evaluation import is_dns_problem, dns_works_via_tor
+from core.services.networking.api_requests.subtools.extract_domain import extract_domain
+from core.services.networking.api_requests.subtools.is_tor_working import is_tor_working
+
+
+# errors
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+from typing import Optional
+
+
+def classify_request_error(
+    url: str,
+    connection_type: str,
+    connection_observer: ConnectionObserver,
+    client_observer: Optional[ClientObserver] = None,
+) -> ApiResponse:
+    """
+    Classify all error requests,
+    Tor Connection.
+    DNS
+    Internet Connection
+    """
+    domain_only = extract_domain(url)
+    
+    if connection_type == "tor":
+        # Tor-specific diagnostics
+        if not is_tor_working(connection_observer):
+            logger.debug("Tor is not working")
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.TOR_NOT_WORKING,
+                tor=True,
+                message="Tor connection failed"
+            )
+
+        ip_via_tor = dns_works_via_tor(domain_only, connection_observer)
+
+        if not ip_via_tor:
+            logger.debug("DNS resolution via Tor failed")
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.DNS_RESOLUTION,
+                tor=True,
+                message=f"Cannot resolve {domain_only} via Tor"
+            )
+        else:
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.DNS_RESOLUTION,
+                tor=True,
+                ip_address=ip_via_tor,
+                message=f"For Unknown reasons, we can't connect, but we can resolve {domain_only} to {ip_via_tor} via Tor"
+            )
+
+    else:
+        # Regular (non-Tor) diagnostics
+        if not do_we_have_internet():
+            logger.error("No internet connection")
+            return ApiResponse(
+                valid=False,
+                tor=False,
+                error_type=ErrorType.NO_INTERNET,
+                message="No internet connectivity"
+            )
+        
+        if is_dns_problem(domain_only, connection_observer, client_observer):
+            logger.error("Local DNS resolution failed")
+            return ApiResponse(
+                valid=False,
+                tor=False,
+                error_type=ErrorType.DNS_RESOLUTION,
+                message=f"Cannot resolve {domain_only}"
+            )
+    
+    # Fallback: unknown error
+    logger.error(f"Unknown error for {connection_type} request to {domain_only}")
+    return ApiResponse(
+        valid=False,
+        error_type=ErrorType.UNKNOWN,
+        message="Request failed for unknown reason"
+    )

core/services/networking/api_requests/step5_solve_api_problems.py

@@ -0,0 +1,60 @@
+from core.errors.logger import logger
+from core.Constants import Constants
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+from core.observers.ClientObserver import ClientObserver
+
+# tools to solve:
+from core.services.networking.api_requests.subtools.direct_dns_tools import get_DNS_then_use_it
+
+# generic
+from typing import Optional
+
+
+def solve_api_problems(
+    api_reply_object: ApiResponse,
+    get_or_post: str,
+    url: str,
+    payload: str = None,
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+) -> ApiResponse:
+
+    if api_reply_object.valid:
+        logger.debug(f"[API SOLVER] The API call worked, so there's no reason to try to solve it.")
+        return api_reply_object
+
+    reason_for_error = api_reply_object.error_type
+
+    cant_be_solved = [ErrorType.QUAD9_DNS_RESOLUTION, ErrorType.UNKNOWN, ErrorType.NO_INTERNET]
+
+    if reason_for_error in cant_be_solved:
+        logger.debug(f"[API SOLVER] This can't be solved if the reason is {reason_for_error}")
+        return api_reply_object
+    
+    elif reason_for_error == ErrorType.DEVELOPER_ERROR:
+        logger.error(f"[API SOLVER] The developer made an error {api_reply_object.message}")
+        return api_reply_object
+
+    elif reason_for_error == ErrorType.DNS_RESOLUTION:
+        logger.debug(f"[API SOLVER] We're solving a DNS resolution error {api_reply_object.message}")
+        if api_reply_object.tor:
+            dns_fix = get_DNS_then_use_it(get_or_post, url, payload, api_reply_object.ip_address, connection_observer, client_observer)
+            return dns_fix
+        else:
+            logger.error(f"[API SOLVER] We have NOT yet setup clearweb DNS solutions")
+            return api_reply_object
+
+    elif reason_for_error == ErrorType.TOR_NOT_WORKING:
+        api_reply_object.ask_clearweb = True
+        return api_reply_object
+
+    elif reason_for_error == ErrorType.NETWORK_ERROR:
+        if api_reply_object.tor:
+            api_reply_object.ask_clearweb = True
+        return api_reply_object
+
+    else:
+        logger.error(f"[API SOLVER] Unknown reason {reason_for_error} with message: {api_reply_object.message}")
+        return api_reply_object

core/services/networking/api_requests/subtools/custom_dns.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import requests
+
+# errors
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+from urllib.parse import urlparse, urlunparse
+
+def prep_url_with_known_ip(known_ip: str, original_url: str):
+    """
+    Purpose:
+        Prepares URLs for custom DNS checking
+
+    Rank:
+        Helper
+    
+    Called by:
+        step2_execute's _execute_tor_request
+
+    Raises Errors?
+        False
+    """
+
+    # Parse the original URL to extract the hostname
+    parsed = urlparse(original_url)
+    original_hostname = parsed.hostname
+
+    # Reconstruct the URL with the IP instead of the domain
+    url_with_ip = original_url.replace(parsed.hostname, known_ip)
+
+    return url_with_ip, original_hostname
+
+
+def load_custom_dns_resolver():
+    from requests.adapters import HTTPAdapter
+    from urllib3.util.ssl_ import create_urllib3_context
+    import ssl
+    import certifi
+
+    class CustomHostnameAdapter(HTTPAdapter):
+        """
+        Purpose:
+            If Tor (or ANY) DNS server is blocked, we can go directly to the IP,
+            and use this module to verify the SSL against local certs.
+
+        Method:
+            Adapter that connects to an IP but verifies the SSL cert against the original hostname (via SNI).
+            then uses that with the proxy, essentially intercepting/modifying the proxy behavior
+
+        Rank:
+            Coordinator for the SSL verification Task,
+            But the caller is orchestrating the actual API calls.
+
+        Called by:
+            step2_execute's _execute_tor_request
+
+        Raises Errors?
+            False
+        """
+        def __init__(self, original_hostname, *args, **kwargs):
+            logger.debug(f"\n[ADAPTER INIT] Creating CustomHostnameAdapter")
+            logger.debug(f"  original_hostname: {original_hostname}")
+            self.original_hostname = original_hostname
+            super().__init__(*args, **kwargs)
+            logger.debug(f"[ADAPTER INIT] ✓ Adapter initialized")
+
+        def init_poolmanager(self, *args, **kwargs):
+            logger.debug(f"\n[POOLMANAGER INIT] Creating standard pool manager (for direct connections)")
+            ctx = create_urllib3_context()
+            ctx.check_hostname = True
+            ctx.verify_mode = ssl.CERT_REQUIRED
+            ctx.load_verify_locations(certifi.where())
+            
+            kwargs['ssl_context'] = ctx
+            logger.debug(f"  ✓ Standard pool manager configured")
+            return super().init_poolmanager(*args, **kwargs)
+
+        def proxy_manager_for(self, proxy, **proxy_kwargs):
+            """
+            This is called when making requests through a proxy.
+            This is where we inject the custom SSL context.
+            """
+            logger.debug(f"\n[PROXY MANAGER] Creating proxy manager")
+            logger.debug(f"  proxy: {proxy}")
+            logger.debug(f"  original_hostname (for verification): {self.original_hostname}")
+            
+            ctx = create_urllib3_context()
+            ctx.check_hostname = True
+            ctx.verify_mode = ssl.CERT_REQUIRED
+            ctx.load_verify_locations(certifi.where())
+            
+            # KEY: Set server_hostname on the context for SNI
+            logger.debug(f"  Setting server_hostname for SNI: {self.original_hostname}")
+            
+            proxy_kwargs['ssl_context'] = ctx
+            proxy_kwargs['server_hostname'] = self.original_hostname
+            
+            logger.debug(f"  ✓ Proxy manager will verify cert against: {self.original_hostname}")
+            
+            result = super().proxy_manager_for(proxy, **proxy_kwargs)
+            logger.debug(f"  ✓ Proxy manager created successfully")
+            return result
+
+    return CustomHostnameAdapter
+

core/services/networking/api_requests/subtools/custom_httpx_dns_resolver.py

@@ -0,0 +1,46 @@
+import json
+import httpx
+import ssl
+import certifi
+from unittest.mock import patch
+import httpcore
+
+def create_httpx_client_with_custom_dns(hostname_to_ip_map: dict, custom_proxy: str):
+    """
+    Purpose:
+        Creates an httpx.Client that connects to IPs while verifying certificates
+        against the original hostnames (via SNI).
+    """
+    
+    ssl_context = ssl.create_default_context(cafile=certifi.where())
+    ssl_context.check_hostname = True
+    ssl_context.verify_mode = ssl.CERT_REQUIRED
+    
+    # Store the original wrap_socket method
+    original_wrap_socket = ssl.SSLContext.wrap_socket
+    
+    def patched_wrap_socket(self, sock, *args, **kwargs):
+        # Inject server_hostname for IPs in our map
+        for hostname, ip in hostname_to_ip_map.items():
+            try:
+                peer_addr = sock.getpeername()
+                if peer_addr[0] == ip:
+                    kwargs['server_hostname'] = hostname
+                    break
+            except OSError:
+                pass
+        return original_wrap_socket(self, sock, *args, **kwargs)
+    
+    # Apply the patch PERMANENTLY
+    ssl.SSLContext.wrap_socket = patched_wrap_socket
+    
+    # Build mounts
+    transport = httpx.HTTPTransport(verify=ssl_context, http2=True)
+    mounts = {}
+    for hostname, ip in hostname_to_ip_map.items():
+        mounts[f"https://{hostname}"] = transport
+        mounts[f"https://{ip}"] = transport
+    
+    # Create and return client
+    client = httpx.Client(mounts=mounts, http2=True, proxy=custom_proxy)
+    return client

core/services/networking/api_requests/subtools/direct_dns_tools.py

@@ -0,0 +1,346 @@
+from core.errors.logger import logger
+from core.Constants import Constants
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.subtools.custom_httpx_dns_resolver import create_httpx_client_with_custom_dns
+from core.services.networking.api_requests.subtools.extract_domain import extract_domain, swap_domain_for_ip
+
+# tor
+from essentials.modules.TorModule import TorModule
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+from core.observers.ClientObserver import ClientObserver
+
+# pip install pip install httpx[http2]
+# pip install httpx[socks]
+import httpx
+import dns.message
+import dns.name
+import dns.rdatatype
+import base64
+
+import json
+from typing import Optional
+
+
+def get_DNS_then_use_it(
+    get_or_post: str,
+    full_url: str,
+    payload: str = None,
+    ip_address: str = None,
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+) -> ApiResponse:
+
+    if not full_url:
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.DEVELOPER_ERROR,
+            tor=True,
+            message="Dev Error! Invalid Inputs to Function"
+        )
+
+    # ================= Setup Tor ===============
+    if client_observer:
+        status_update = "Evading Tor DNS block..."
+        client_observer.notify('synchronizing', status_update)
+
+    port_number = None
+    tor_module = None
+    client = None
+
+    try:
+        port_number = ConnectionService.get_random_available_port_number()
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+
+        tor_proxy = f"socks5h://127.0.0.1:{port_number}"
+
+        logger.debug(f"[DNS TOOLS] Tor Proxy Setup on Port {port_number}")
+
+        # ================= Do Quad9 Query ===============
+        domain = extract_domain(full_url)
+        logger.debug(f"[DNS TOOLS] We'll be doing a DNS lookup to the domain {domain}..")
+
+        if not ip_address:
+            ip_address = quad9_proxy_dns_lookup(domain, tor_proxy)
+
+            if not ip_address:
+                return ApiResponse(
+                    valid=False,
+                    error_type=ErrorType.QUAD9_DNS_RESOLUTION,
+                    tor=True,
+                    message=f"Cannot resolve {domain} via Tor or Quad9"
+                )
+            
+            logger.debug(f"[DNS TOOLS] We got an IP address from Quad9 for {domain} of: {ip_address}")
+
+        # ================= Use Custom DNS ===============
+        client = create_httpx_client_with_custom_dns({domain: ip_address}, tor_proxy)
+        url_with_ip = swap_domain_for_ip(domain, full_url, ip_address)
+        print(f"url_with_ip is {url_with_ip}")
+
+        print(f"we are doing a {get_or_post} request")
+
+        if get_or_post == "get":
+             response = client.get(url_with_ip, headers={"Host": domain})
+        elif get_or_post == "post":
+            if payload:
+                response = client.post(url_with_ip, json=payload, headers={"Host": domain})
+            else:
+                return ApiResponse(
+                    valid=False,
+                    error_type=ErrorType.DEVELOPER_ERROR,
+                    ip_address=ip_address,
+                    tor=True,
+                    message=f"Dev Error! No Payload for POST request"
+                )
+        else:
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.DEVELOPER_ERROR,
+                ip_address=ip_address,
+                tor=True,
+                message=f"Dev Error! Invalid Input of get_or_post: {get_or_post}"
+            )
+
+        # print(response.json())
+
+        if not response:
+            error_msg = f"Could not connect to {domain} via {ip_address}"
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.NETWORK_ERROR,
+                ip_address=ip_address,
+                tor=True,
+                message=error_msg
+            )
+
+        response_status_code = response.status_code
+
+        if 200 <= response_status_code < 300:
+            try:
+                data = response.json()
+                return ApiResponse(
+                    valid=True,
+                    error_type=ErrorType.TOR_DNS_BLOCKED,
+                    ip_address=ip_address,
+                    tor=True,
+                    data=data
+                )
+            except ValueError:
+                print("We could not get the value from it.")
+                return ApiResponse(
+                    valid=True,
+                    error_type=ErrorType.SERVER_ERROR,
+                    ip_address=ip_address,                    
+                    tor=True,
+                    data=response.text
+                )
+        else:
+            error_msg = f"Server replied with {response_status_code} at {ip_address}"
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.SERVER_ERROR,
+                ip_address=ip_address,                    
+                tor=True,
+                message=error_msg
+            )
+
+    except Exception as e:
+        error_msg = f"Tor failed: {type(e).__name__}: {e}"
+        logger.error(f"[DNS] {error_msg}")
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.NETWORK_ERROR,
+            tor=True,
+            message=error_msg
+        )
+
+    finally:
+        if client:
+            client.close()
+
+        if tor_module and port_number:
+            try:
+                tor_module.destroy_session(port_number)
+            except Exception as e:
+                logger.warning(f"[DNS] Error destroying Tor session: {e}")
+            
+
+
+
+# This version is passed a "generic" proxy with external Tor:
+def quad9_proxy_dns_lookup(
+    domain: str,
+    custom_proxy: str,
+    timeout: int = 10,
+) -> str:
+
+    logger.debug("Doing a Proxy Quad9 DNS lookup")
+
+    # ================= Use Proxy for Quad9 ===============
+
+    qname = dns.name.from_text(domain)
+    query = dns.message.make_query(qname, dns.rdatatype.A)
+    wire_format = query.to_wire()
+    encoded = base64.urlsafe_b64encode(wire_format).decode().rstrip('=')
+
+    # Use httpx with HTTP/2
+    with httpx.Client(http2=True, proxy=custom_proxy, timeout=timeout) as client:
+        response = client.get(
+            "https://dns.quad9.net/dns-query",
+            params={"dns": encoded},
+            headers={"Accept": "application/dns-message"}
+        )
+        
+        logger.debug(f"[QUAD9-DNS] Status: {response.status_code}")
+        logger.debug(f"[QUAD9-DNS] Headers: {response.headers}")
+        
+        if response.status_code == 200:
+            # ================= Filter Results ===============
+            result = dns.message.from_wire(response.content)
+            logger.debug(f"[QUAD9-DNS] Quad9 Returned {result}")
+
+            ip_address = None
+            for rrset in result.answer:
+                for rr in rrset:
+                    if rr.rdtype == dns.rdatatype.A:
+                        ip_address = rr.address  # Get just the IP address
+                        break
+                if ip_address:
+                    break
+
+            return ip_address
+        else:
+            logger.error(f"Quad9's Invalid Response Body: {response.text[:500]}")
+            if client_observer:
+                status_update = f"Quad9 Failed to Resolve {domain}"
+                client_observer.notify('synchronizing', status_update)
+            return False
+
+
+
+# This version does Tor & the lookup same time.
+def quad9_tor_dns_lookup(
+    domain: str,
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+    timeout: int = 10,
+) -> str:
+
+    import httpx
+    import dns.message
+    import dns.name
+    import dns.rdatatype
+    import base64
+
+    logger.debug("Doing a Tor Quad9 DNS lookup")
+
+    # ================= Setup Tor ===============
+    if client_observer:
+        status_update = "Initializing Tor connection..."
+        client_observer.notify('synchronizing', status_update)
+
+    port_number = None
+    tor_module = None
+
+    try:
+        port_number = ConnectionService.get_random_available_port_number()
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+
+        tor_proxy = f"socks5h://127.0.0.1:{port_number}"
+
+        logger.debug(f"[QUAD9-DNS] Tor Proxy Setup on Port {port_number}")
+
+        # ================= Use Proxy for Quad9 ===============
+
+        qname = dns.name.from_text(domain)
+        query = dns.message.make_query(qname, dns.rdatatype.A)
+        wire_format = query.to_wire()
+        encoded = base64.urlsafe_b64encode(wire_format).decode().rstrip('=')
+
+        # Use httpx with HTTP/2
+        with httpx.Client(http2=True, proxy=tor_proxy, timeout=timeout) as client:
+            response = client.get(
+                "https://dns.quad9.net/dns-query",
+                params={"dns": encoded},
+                headers={"Accept": "application/dns-message"}
+            )
+            
+            logger.debug(f"[QUAD9-DNS] Status: {response.status_code}")
+            logger.debug(f"[QUAD9-DNS] Headers: {response.headers}")
+            
+            if response.status_code == 200:
+                # ================= Filter Results ===============
+                result = dns.message.from_wire(response.content)
+                logger.debug(f"[QUAD9-DNS] Quad9 Returned {result}")
+
+                ip_address = None
+                for rrset in result.answer:
+                    for rr in rrset:
+                        if rr.rdtype == dns.rdatatype.A:
+                            ip_address = rr.address  # Get just the IP address
+                            break
+                    if ip_address:
+                        break
+
+                return ip_address
+            else:
+                logger.error(f"Quad9's Invalid Response Body: {response.text[:500]}")
+                if client_observer:
+                    status_update = f"Quad9 Failed to Resolve {domain}"
+                    client_observer.notify('synchronizing', status_update)
+                return False
+
+    except Exception as e:
+        logger.debug(f"[QUAD9-DNS] Tor Quad9 GET failed: {type(e).__name__}: {e}")
+        return None
+
+    finally:
+        if tor_module and port_number:
+            try:
+                tor_module.destroy_session(port_number)
+            except Exception as e:
+                logger.warning(f"[QUAD9-DNS] Error destroying Tor session: {e}")
+    
+    return result
+
+
+
+def quad9_clearweb_dns_lookup(domain: str) -> str:
+    import httpx
+    import dns.message
+    import dns.name
+    import dns.rdatatype
+    import base64
+
+    logger.debug("Doing a Clearweb Quad9 DNS lookup")
+
+    qname = dns.name.from_text(domain)
+    query = dns.message.make_query(qname, dns.rdatatype.A)
+    wire_format = query.to_wire()
+    encoded = base64.urlsafe_b64encode(wire_format).decode().rstrip('=')
+
+    try:
+        # Use httpx with HTTP/2
+        with httpx.Client(http2=True) as client:
+            response = client.get(
+                "https://dns.quad9.net/dns-query",
+                params={"dns": encoded},
+                headers={"Accept": "application/dns-message"}
+            )
+            
+            logger.debug(f"Status: {response.status_code}")
+            logger.debug(f"Headers: {response.headers}")
+            
+            if response.status_code == 200:
+                result = dns.message.from_wire(response.content)
+                logger.debug(f"Quad9 Returned {result}")
+                return result
+            else:
+                logger.error(f"Quad9 Body: {response.text[:500]}")
+                return False
+    except:
+        logger.error("Regular Quad9 request failed.")
+        return None

core/services/networking/api_requests/subtools/dns_evaluation.py

@@ -0,0 +1,133 @@
+# for tor:
+from essentials.modules.TorModule import TorModule
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+
+from core.observers.ClientObserver import ClientObserver
+from core.Constants import Constants
+from core.services.networking.api_requests.subtools.extract_domain import extract_domain
+from core.errors.logger import logger
+from core.errors.exceptions import *
+
+# import dns.resolver
+# import dns.exception
+# import dns.rdatatype
+from typing import Optional
+import os
+import socket
+import socks
+
+import subprocess
+
+
+"""
+Check if there's a DNS problem with a domain.
+Returns True if DNS fails (problem exists), False if DNS resolves successfully.
+"""
+
+def is_dns_problem(url: str) -> bool:
+    """Regular DNS check for non-Tor domains"""
+    domain = extract_domain(url)
+    try:
+        socket.gethostbyname(domain)
+        return False  # DNS resolved successfully
+    except socket.gaierror:
+        return True  # DNS resolution failed
+    except Exception:
+        return True
+
+
+
+def dns_works_via_tor(domain, connection_observer, client_observer=None):
+    """
+    Purpose:
+        Check if domain's resolver is blocking Tor by querying via proxychains.
+
+    Method:
+        Dynamically configures proxychains with the given port.
+        Uses this to query the DNS via dig.
+
+    Dependencies:
+        Proxychains being installed.
+
+    Called by:
+        step4_error_classifier's classify_request_error
+    
+    Raises Errors?
+        False
+    """
+
+    port_number = ConnectionService.get_random_available_port_number()
+    logger.debug(f"[DNS PROBLEM OF TOR SERVICE] Using Tor on port number {port_number}")
+    if client_observer is not None:
+        client_observer.notify('synchronizing', f"Using Tor on Port {port_number}..")
+
+    try:
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+        logger.debug(f"[DNS PROBLEM OF TOR SERVICE] Tor Started a Session")
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Initialized Tor..")
+
+    except TorServiceInitializationError as e:
+        error_msg = f"[DNS PROBLEM OF TOR SERVICE] Tor failed to start: {e}"
+        tor_module.destroy_session(port_number)
+        logger.error(error_msg, exc_info=True)
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Failed to Initialize")
+
+    proxies = {
+        "http": f"socks5h://127.0.0.1:{port_number}",
+        "https": f"socks5h://127.0.0.1:{port_number}",
+    }
+    config_path = f"{Constants.HV_CONFIG_HOME}/proxychains_dynamic.conf"
+    
+    try:
+        # Write proxychains config on the fly
+        os.makedirs(os.path.dirname(config_path), exist_ok=True)
+        with open(config_path, 'w') as f:
+            f.write(f"""
+strict_chain
+proxy_dns
+tcp_read_time_out 10000
+tcp_connect_time_out 10000
+
+[ProxyList]
+socks5 127.0.0.1 {port_number}
+        """)
+            f.flush()
+            os.fsync(f.fileno())  # Force write to disk
+
+        print("running command...")
+        # Run dig through proxychains with the dynamic config
+        try:
+            result = subprocess.run(
+                ['proxychains4', '-f', config_path, 'dig', domain, '+short'],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+            logger.debug(f"result: {result}")
+        except:
+            logger.error(f"[DNS PROBLEM] Proxychains not installed on this computer")
+            return False
+
+        output = result.stdout.strip()
+        logger.debug(f"output of stripping the IP out of the result: {output}")
+        
+        if not output or result.returncode != 0:
+            logger.debug(f"[DNS PROBLEM] {domain}: No response")
+            return False
+        
+        logger.debug(f"[DNS OK] {domain}: {output}")
+        return output
+
+    except subprocess.TimeoutExpired:
+        logger.debug(f"[DNS PROBLEM] {domain}: Timeout")
+        return False
+    except Exception as e:
+        logger.debug(f"[DNS PROBLEM] {domain}: {type(e).__name__}: {e}")
+        return False
+    finally:
+        tor_module.destroy_session(port_number)
+

core/services/networking/api_requests/subtools/extract_domain.py

@@ -0,0 +1,54 @@
+# used in the 2nd function, not the first,
+import re
+
+def extract_domain(url: str) -> str:
+    """
+    Purpose:
+        Extract domain from URL, stripping protocol and path.
+        If exact TLD match isn't found, tries to return the best guess.
+    """
+    common_domains = [".com", ".net", ".org", ".is"]
+
+    # Remove protocol
+    if url.startswith("https://"):
+        url = url[8:]
+    elif url.startswith("http://"):
+        url = url[7:]
+
+    # Find the earliest TLD in the string
+    earliest_pos = len(url)
+    earliest_tld_len = 0
+
+    for tld in common_domains:
+        pos = url.find(tld)
+        if pos != -1 and pos < earliest_pos:
+            earliest_pos = pos
+            earliest_tld_len = len(tld)
+
+    # If a valid TLD was found, return domain + TLD
+    if earliest_tld_len > 0:
+        return url[: earliest_pos + earliest_tld_len]
+
+    # Fallback: return everything before the first slash (best guess)
+    return url.split("/")[0]
+
+
+def swap_domain_for_ip(raw_domain: str, full_url: str, ip_address: str):
+    """
+    Purpose:
+        Switch the real domain for the raw IP address,
+        But keep the rest of the domain intact.
+
+    Rank:
+        Helper
+    
+    Raises Errors?
+        False
+    """
+    # Escape the domain in case it has regex special characters (like dots)
+    escaped_domain = re.escape(raw_domain)
+    
+    # Replace only the first instance of the domain with the IP
+    url_with_ip = re.sub(escaped_domain, ip_address, full_url, count=1)
+    
+    return url_with_ip

core/services/networking/api_requests/subtools/replace_http_with_https.py

@@ -0,0 +1,7 @@
+
+
+
+def replace_http_with_https(url):
+    if url.startswith("http://"):
+        return url.replace("http://", "https://", 1)
+    return url

core/services/networking/evaluate_networking_problem.py

@@ -1,76 +0,0 @@
-from __future__ import annotations
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from essentials.observers.ConnectionObserver import ConnectionObserver
-# services
-from core.services.networking.internet_test import do_we_have_internet
-from core.services.networking.is_dns_problem import is_dns_problem, is_dns_problem_via_tor
-from core.services.networking.extract_domain import extract_domain
-from core.services.networking.is_tor_working import is_tor_working
-# errors
-from core.errors.exceptions import *
-from core.errors.logger import logger
-
-#######################################################
-############### START : REGULAR SYSTEM  ###############
-#######################################################
-
-
-def evaluate_regular_networking_problem(
-    url: str,
-    connection_observer: ConnectionObserver,
-    port_number: int | None = None,
-    proxies: dict | None = None,
-) -> dict:
-
-    domain_only = extract_domain(url)
-
-    if not do_we_have_internet():
-        return {"valid": False, "error_code": "no_internet"}
-
-    if is_dns_problem(domain_only):
-        logger.debug("This is a DNS issue.")
-        return {"valid": False, "error_code": "dns_issue"}
-
-    return {"valid": False, "error_code": "unknown"}
-    #######################################################
-    ############### END: REGULAR SYSTEM  ###############
-    #######################################################
-
-    #######################################################
-    ###############      START: TOR         ###############
-    #######################################################
-
-
-def evaluate_tor_networking_problem(
-    url: str,
-    connection_observer: ConnectionObserver,
-) -> dict:
-
-    domain_only = extract_domain(url)
-    logger.debug(f"Evaluating a networking problem for a Tor connection")
-
-    logger.debug("checking if Tor works against the official Tor Project API..")
-    if not is_tor_working(connection_observer):
-        return {"valid": False, "error_code": "tor_issue"}
-    else:
-        tor_fine = f"While there were connection issues, but Tor is working fine!"
-        logger.error(tor_fine, exc_info=True)
-        logger.debug(tor_fine)
-
-    logger.debug("Check the original DNS via Tor...")
-
-    if is_dns_problem_via_tor(domain_only, connection_observer):
-        return {"valid": False, "error_code": "dns_issue"}
-    else:
-        dns_fine = f"There were connection issues, but the DNS for {domain_only} via Tor is working fine!"
-        logger.error(dns_fine, exc_info=True)
-        logger.debug(dns_fine)
-
-    # can't solve it:
-    return {"valid": False, "error_code": "unknown"}
-
-    #######################################################
-    ###############      END: TOR           ###############
-    #######################################################

core/services/networking/extract_domain.py

@@ -1,31 +0,0 @@
-"""
-Extract domain from URL, stripping protocol and path.
-If exact TLD match isn't found, tries to return the best guess.
-"""
-
-
-def extract_domain(url: str) -> str:
-    common_domains = [".com", ".net", ".org", ".is"]
-
-    # Remove protocol
-    if url.startswith("https://"):
-        url = url[8:]
-    elif url.startswith("http://"):
-        url = url[7:]
-
-    # Find the earliest TLD in the string
-    earliest_pos = len(url)
-    earliest_tld_len = 0
-
-    for tld in common_domains:
-        pos = url.find(tld)
-        if pos != -1 and pos < earliest_pos:
-            earliest_pos = pos
-            earliest_tld_len = len(tld)
-
-    # If a valid TLD was found, return domain + TLD
-    if earliest_tld_len > 0:
-        return url[: earliest_pos + earliest_tld_len]
-
-    # Fallback: return everything before the first slash (best guess)
-    return url.split("/")[0]

core/services/networking/general_connection_tools/config_tools.py

@@ -0,0 +1,35 @@
+from core.Constants import Constants
+import os
+
+def extract_endpoint_ip(profile_id: str) -> str | bool:
+    try:
+        profile_path = Constants.HV_PROFILE_CONFIG_HOME + f'/{profile_id}'
+        wg_conf_path = f'{profile_path}/wg.conf.bak'
+        print(f"wg_conf_path is {wg_conf_path}")
+        if not os.path.exists(wg_conf_path):
+            return None
+        with open(wg_conf_path, 'r') as f:
+            content = f.read()
+        for line in content.split('\n'):
+            if line.strip().startswith('Endpoint = '):
+                endpoint = line.strip().split(' = ')[1]
+                return endpoint.split(':')[0]
+        return None
+    except Exception:
+        return None
+
+def extract_dns(profile_id: str) -> str | bool:
+    try:
+        profile_path = Constants.HV_PROFILE_CONFIG_HOME + f'/{profile_id}'
+        wg_conf_path = f'{profile_path}/wg.conf.bak'
+        print(f"wg_conf_path is {wg_conf_path}")
+        if not os.path.exists(wg_conf_path):
+            return None
+        with open(wg_conf_path, 'r') as f:
+            content = f.read()
+        for line in content.split('\n'):
+            if line.strip().startswith('DNS = '):
+                return line.strip().split(' = ')[1]
+        return None
+    except Exception:
+        return None

core/services/networking/general_connection_tools/connection_enable.py

@@ -0,0 +1,134 @@
+from core.services.networking.general_connection_tools.testing_evaluating import system_uses_wireguard_interface
+from core.services.networking.systemwide.systemwide_wireguard import establish_system_connection, terminate_system_connection
+from core.services.keys_and_verifications.wireguard_keys import register_wireguard_session
+from core.services.subscriptions.subscriptions import activate_subscription
+
+# If refactored to enums:
+# from core.models.session.SessionConnection import SessionConnectionTypes
+# from core.models.system.SystemConnection import SystemConnectionTypes
+# wireguard_types = [SystemConnectionTypes.WIREGUARD, SessionConnectionTypes.WIREGUARD]
+
+
+from core.errors.logger import logger
+from core.errors.exceptions import *
+from typing import Union, Optional, Callable
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+from core.Errors import ConnectionTerminationError, InvalidSubscriptionError
+from core.services.WebServiceApiService import WebServiceApiService
+from core.controllers.ConnectionController import ConnectionController
+from core.models.BaseProfile import ProfileType
+from core.observers.ConnectionObserver import ConnectionObserver
+from core.controllers.SystemStateController import SystemStateController
+
+def establish_connection(
+    profile: Union[SessionProfile, SystemProfile], 
+    ignore: tuple[type[Exception]] = (), 
+    connection_observer: Optional[ConnectionObserver] = None,
+):
+    """Establish a connection for the given profile."""
+    
+    logger.info(f"[CONNECTION] Checking subscription..")
+    activate_subscription(profile, connection_observer)
+
+    logger.info(f"[CONNECTION] Checking proxy configuration..")
+    _ensure_proxy_configured(profile, connection_observer)
+
+    logger.info(f"[CONNECTION] Checking Wireguard configuration..")
+    _ensure_wireguard_configured(profile, connection_observer)
+    
+    establish_fn = {
+        ProfileType.SESSION: ConnectionController.establish_session_connection,
+        ProfileType.SYSTEM: establish_system_connection,
+    }[profile.type]
+    
+    return _establish_with_renegotiation(profile, establish_fn, ignore, connection_observer)
+
+
+def _ensure_proxy_configured(
+    profile: Union[SessionProfile, SystemProfile],
+    connection_observer: Optional[ConnectionObserver],
+):
+    """Setup proxy config if needed."""
+    if not profile.connection.needs_proxy_configuration():
+        return
+    
+    if not profile.has_proxy_configuration():
+        logger.info(f"[CONNECTION] You need a new proxy configuration")
+        activate_subscription(profile, connection_observer)
+
+        logger.info(f"[CONNECTION] Getting a proxy config from web service..")
+        proxy_config = ConnectionController.with_preferred_connection(
+            profile.subscription.billing_code, 
+            task=WebServiceApiService.get_proxy_configuration,
+            connection_observer=connection_observer
+        )
+        
+        if proxy_config is None:
+            raise InvalidSubscriptionError()
+        
+        profile.attach_proxy_configuration(proxy_config)
+
+
+def _ensure_wireguard_configured(
+    profile: Union[SessionProfile, SystemProfile],
+    connection_observer: Optional[ConnectionObserver],
+):
+    """Setup WireGuard config if needed."""
+
+    logger.info(f"[CONNECTION] Checking if this profile needs a wg config.  The profile.connection.code is {profile.connection.code}")
+    if profile.connection.code != "wireguard":
+        logger.info(f"[CONNECTION] This profile {profile.id} does NOT need a wireguard config")
+        return
+    
+    if profile.type == ProfileType.SYSTEM:
+        if system_uses_wireguard_interface() and SystemStateController.exists():
+            logger.info(f"[CONNECTION] There is a systemwide profile that is already is active.")
+            try:
+                terminate_system_connection()
+            except ConnectionTerminationError as e:
+                logger.error(f"[CONNECTION] Previous Systemwide Wireguard connection could not be disabled. {e}")
+
+    activate_subscription(profile, connection_observer)
+    logger.info(f"[CONNECTION] Checking if the profile already has a wg config..")
+    if not profile.has_wireguard_configuration():
+        logger.info(f"[CONNECTION] Profile does NOT have WG keys and it needs it.  Registering a NEW wg session..")
+        register_wireguard_session(profile, connection_observer=connection_observer)
+
+
+def _establish_with_renegotiation(
+    profile: Union[SessionProfile, SystemProfile],
+    establish_fn: Callable,
+    ignore: tuple[type[Exception]],
+    connection_observer: Optional[ConnectionObserver],
+):
+    """Attempt connection, renegotiate WireGuard once if needed."""
+    try:
+        logger.info(f"[CONNECTION] Connecting..")
+        return establish_fn(profile, ignore=ignore, connection_observer=connection_observer)
+    except ConnectionError:
+        if __should_renegotiate(profile):
+            logger.info(f"[CONNECTION] Renegotiating a new wg key session with API..")    
+            register_wireguard_session(profile, connection_observer=connection_observer)
+            return establish_fn(profile, ignore=ignore, connection_observer=connection_observer)
+        raise ConnectionError('The connection could not be established.')
+
+
+def __should_renegotiate(profile: Union[SessionProfile, SystemProfile]):
+
+    if not profile.has_subscription():
+        raise MissingSubscriptionError()
+
+    if profile.connection.code == "wireguard" and profile.has_wireguard_configuration():
+        if profile.subscription.has_been_activated():
+            return True
+    else:
+        return False
+
+
+
+# Enums
+# if profile.type == ProfileType.SYSTEM:
+#      print("This is a system profile")
+#  elif profile.type == ProfileType.SESSION:
+#      print("This is a session profile")

core/services/networking/general_connection_tools/testing_evaluating.py

@@ -0,0 +1,125 @@
+from core.Constants import Constants
+from core.Errors import CommandNotFoundError
+from core.observers.ConnectionObserver import ConnectionObserver
+from essentials.modules.TorModule import TorModule
+from typing import Optional
+import os
+import random
+import re
+import shutil
+import subprocess
+import sys
+import time
+
+
+def terminate_tor_connection():
+
+    tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+    tor_module.stop_service()
+
+
+def await_network_interface():
+
+    network_interface_is_activated = False
+
+    retry_interval = .5
+    maximum_number_of_attempts = 10
+    attempt = 0
+
+    while not network_interface_is_activated and attempt < maximum_number_of_attempts:
+
+        time.sleep(retry_interval)
+
+        network_interface_is_activated = system_uses_wireguard_interface()
+        attempt += 1
+
+    if not network_interface_is_activated:
+        raise ConnectionError('The network interface could not be activated.')
+
+
+def system_uses_wireguard_interface():
+
+    if shutil.which('ip') is None:
+        raise CommandNotFoundError('ip')
+
+    process = subprocess.Popen(('ip', 'route', 'get', '192.0.2.1'), stdout=subprocess.PIPE)
+    process_output = str(process.stdout.read())
+
+    return bool(re.search('dev wg', str(process_output)))
+
+
+def __test_connection(port_number: Optional[int] = None, timeout: float = 4.0):
+
+    request_urls = [Constants.PING_URL]
+    proxies = None
+
+    if os.environ.get('PING_URL') is None:
+
+        request_urls.extend([
+            'https://hc1.simplifiedprivacy.net',
+            'https://hc2.simplifiedprivacy.org',
+            'https://hc3.hydraveil.net'
+        ])
+
+        random.shuffle(request_urls)
+
+    if port_number is not None:
+        proxies = get_proxies(port_number)
+
+    for request_url in request_urls:
+
+        command = [
+            sys.executable, '-u', '-c', 'import requests, sys\n'
+            'try:\n'
+            f'    response = requests.get(\'{request_url}\', proxies={proxies}, timeout={timeout})\n'
+            '    response.raise_for_status(); print(response.text)\n'
+            'except requests.exceptions.RequestException:\n'
+            '    sys.exit(1)'
+        ]
+
+        try:
+
+            _response = subprocess.check_output(command, text=True, timeout=timeout)
+            return None
+
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+            pass
+
+    raise ConnectionError('The connection could not be established.')
+
+
+# Kept a duplicate in connection controller, because Tor uses it. and circular imports are an issue for now. Refactor later.
+def get_proxies(port_number: int):
+
+    return dict(
+        http=f'socks5h://127.0.0.1:{port_number}',
+        https=f'socks5h://127.0.0.1:{port_number}'
+    )
+
+
+def await_connection(port_number: Optional[int] = None, connection_observer: Optional[ConnectionObserver] = None):
+
+    if port_number is None:
+        await_network_interface()
+
+    for retry_count in range(Constants.MAX_CONNECTION_ATTEMPTS):
+
+        if connection_observer is not None:
+
+            connection_observer.notify('connecting', dict(
+                retry_interval=Constants.CONNECTION_RETRY_INTERVAL,
+                maximum_number_of_attempts=Constants.MAX_CONNECTION_ATTEMPTS,
+                attempt_count=retry_count + 1
+            ))
+
+        try:
+
+            __test_connection(port_number)
+            return
+
+        except ConnectionError:
+
+            time.sleep(Constants.CONNECTION_RETRY_INTERVAL)
+            retry_count += 1
+
+    raise ConnectionError('The connection could not be established.')

core/services/networking/is_dns_problem.py

@@ -1,56 +0,0 @@
-# for tor:
-from essentials.modules.TorModule import TorModule
-from essentials.observers.ConnectionObserver import ConnectionObserver
-from essentials.services.ConnectionService import ConnectionService
-import json, os
-
-# for both:
-from core.services.networking.extract_domain import extract_domain
-from core.errors.logger import logger
-import socket
-import socks
-
-"""
-Check if there's a DNS problem with a domain.
-Returns True if DNS fails (problem exists), False if DNS resolves successfully.
-"""
-
-
-def is_dns_problem(url: str) -> bool:
-    domain = extract_domain(url)
-    try:
-        socket.gethostbyname(domain)
-        return False  # DNS resolved successfully, no problem
-    except socket.gaierror:
-        return True  # DNS resolution failed, problem exists
-    except Exception:
-        return True  # Any other error is treated as a DNS problem
-
-
-def is_dns_problem_via_tor(
-    domain: str,
-    connection_observer: ConnectionObserver | None = None,
-) -> bool:
-    logger.debug("We've triggered EVALUATING IF it's a DNS problem via Tor")
-
-    port_number = ConnectionService.get_random_available_port_number()
-    tor_module = TorModule(os.path.expanduser("~/sp-tor-test"))
-    tor_module.create_session(port_number, connection_observer)
-
-    # Set up SOCKS5 proxy using the same port as the existing Tor proxy,
-    socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", port_number)
-    socket.socket = socks.socksocket
-
-    # DNS query check
-    try:
-        logger.debug("checking dns via socket..")
-        socket.gethostbyname(domain)
-
-        tor_module.destroy_session(port_number)
-
-        return False  # DNS resolved successfully, no problem
-
-    except socket.gaierror:
-        return True  # DNS resolution failed, problem exists
-    except Exception:
-        return True  # Any other error is treated as a DNS problem

core/services/networking/legacy/get_data_from_server.py

@@ -1,8 +1,13 @@
 from __future__ import annotations
 from typing import TYPE_CHECKING
 
-if TYPE_CHECKING:
+# if TYPE_CHECKING:
+from core.observers.ClientObserver import ClientObserver
 from essentials.observers.ConnectionObserver import ConnectionObserver
+
+from core.observers.ClientObserver import ClientObserver
+from core.services.networking.evaluate_networking_problem import evaluate_tor_networking_problem
+
 # services
 from core.services.networking.get_connection_type import get_connection_type
 from core.services.networking.use_tor import tor_get_request
@@ -14,13 +19,13 @@ import traceback
 
 
 # Generic GET request to an endpoint, filtered by the user's preference of connection type (Tor or Not)
-def get_data_from_server(url: str, connection_observer: ConnectionObserver) -> dict:
+def get_data_from_server(url: str, connection_observer: ConnectionObserver, client_observer: Optional[ClientObserver] = None) -> dict:
     try:
         connection_type = get_connection_type()
         logger.debug(f"connection type is: {connection_type}")
 
         if connection_type == "tor":
-            response = tor_get_request(url, connection_observer)
+            response = tor_get_request(url, connection_observer, client_observer)
 
         else:
             response = regular_get_request(url, connection_observer)

core/services/networking/legacy/regular_get_request.py

@@ -15,6 +15,7 @@ def regular_get_request(url: str, connection_observer: ConnectionObserver):
     import requests
 
     try:
+        print(f"we are doing a regular GET request to: {url}")
         response = requests.get(url)
 
         # Raises HTTPError for 4xx/5xx

core/services/networking/make_url.py

@@ -5,8 +5,5 @@ from core.Constants import Constants
 def make_url(which_endpoint: str) -> str:
     domain = Constants.TICKET_API_BASE_URL
 
-    # hardcode option:
-    # domain = "https://ticket.hydraveil.net"
-
     final_url = f"{domain}/{which_endpoint}"
     return final_url

core/services/networking/systemwide/dns.py

@@ -0,0 +1,68 @@
+# custom
+from core.services.networking.systemwide.systemwide_errors import systemwide_hell_raiser
+from core.utils.basic_operations.wrap_with import wrap_with
+from core.utils.basic_operations.run_generic_command import run_generic_command
+from core.Constants import Constants
+from core.models.Result import Result, ResultError
+from core.errors.logger import logger
+
+DNS_SCRIPT = f"/opt/hydra-veil/dns"
+
+# requires script to have sudo
+@wrap_with(systemwide_hell_raiser)
+def set_on(network_interface: str, dns_should_be: str) -> Result:
+    command = ["sudo", DNS_SCRIPT, "set", network_interface, dns_should_be]
+    human_readable_goal = "Set DNS"
+    return run_generic_command(command, human_readable_goal)
+
+
+# requires script to have sudo
+@wrap_with(systemwide_hell_raiser)
+def revert() -> Result:
+    command = ["sudo", DNS_SCRIPT, "revert"]
+    human_readable_goal = "DNS revert"
+    return run_generic_command(command, human_readable_goal)
+
+
+# does NOT need sudo
+def is_systemd_enabled() -> Result:
+    command = ["systemctl", "is-enabled", "systemd-resolved"]
+    human_readable_goal = "Find out if SystemD is enabled"
+    result = run_generic_command(command, human_readable_goal, timeout=5)
+    if result.valid:
+        if "enabled" in result.data:
+            return Result(valid=True)
+        else:
+            logger.error("Systemctl is not even enabled.")
+            return Result(valid=False) # this would be true in the original object just because the command ran.
+    else:
+        return result # send the error itself.
+
+
+# does NOT need sudo
+def get_resolvctl_data(which_command: str) -> Result:
+    command = ['resolvectl', which_command]
+    human_readable_goal = "Get Resolvctl Data"
+    return run_generic_command(command, human_readable_goal, timeout=5)
+
+
+# These are the raw Linux systemctl commands:
+"""
+# is it enabled?
+systemctl is-enabled systemd-resolved
+
+# socket exists
+-S /run/systemd/resolve/io.systemd.Resolve
+
+# Shows DNS servers per interface. Should spit back what was put in it for that interface.
+resolvectl dns
+
+# Shows search domains per interface. should show ~. for that interface
+resolvectl domain
+
+# Shows which is the default. yes/no for each
+resolvectl default-route
+
+# Test DNS:
+resolvectl query example.com
+"""

core/services/networking/systemwide/dns_tools/SearchResult.py

@@ -0,0 +1,33 @@
+
+
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional, Any
+
+"""
+Purpose:
+    Searching for values in strings from the system's CLI,
+    In it's current use context, systemctl
+"""
+
+class SearchError(Enum):
+    """Classified error categories."""
+    SUCCESS = "success"
+    NO_INPUT = "no_input"
+    INVALID_INPUT = "invalid_input"
+    TARGET_MISSING = "target_missing"
+    TARGET_NO_VALUE = "target_no_value"
+    VALUE_DOESNT_MATCH = "value_doesnt_match"
+    COMPETITORS_MATCH = "competitors_match"
+    COMMAND_DIDNT_FINISH = "command_didnt_finish"
+    UNKNOWN = "unknown"
+
+@dataclass
+class SearchResult():
+    valid: bool
+    error_type: SearchError = SearchError.SUCCESS
+    which_command_was_run: Optional[str] = None
+    target_value: str = None
+    competitors_count: Optional[int] = 0
+    competitors_list: Optional[Any] = None
+    message: Optional[str] = None

core/services/networking/systemwide/dns_tools/process_dns_result.py

@@ -0,0 +1,124 @@
+from core.services.networking.systemwide.dns_tools.SearchResult import SearchResult, SearchError
+from core.services.networking.systemwide.dns_tools.search_resolv_output import evaluating_resolv_output
+from core.services.networking.systemwide import dns
+from core.models.Result import Result, ResultError
+from core.errors.logger import logger
+
+# This is for the future to potentially give errors to the end user:
+def turn_command_into_readable_string(which_command) -> str:
+    if which_command == "default-route":
+        human_readable_string = "The Systemctl's DNS Default Route"
+    elif which_command == "domain":
+        human_readable_string = "The Systemctl's DNS Domain, for which domains are routed to it"
+    elif which_command == "dns":
+        human_readable_string = "The Systemctl's Default DNS IP address, which should match the VPN configuration file"
+    else:
+        human_readable_string = "Unknown error"
+    return human_readable_string
+
+
+def process_ONE_dns_result(which_command: str, target_interface: str) -> SearchResult:
+    # get the output
+    resolvctl_output = dns.get_resolvctl_data(which_command)
+
+    if not resolvctl_output.valid:
+        return SearchResult(valid=False, error_type=SearchError.COMMAND_DIDNT_FINISH, message=resolvctl_output.message)
+
+    # search that output:
+    return evaluating_resolv_output(which_command=which_command, target_interface=target_interface, resolv_output=resolvctl_output.data)
+
+
+def process_ALL_dns_results(target_interface: str) -> Result:
+
+    list_of_commands = ["domain", "default-route", "dns"]
+    list_of_results_as_objects = []
+    list_of_results_as_binary = []
+
+    # ================= START LOOP =====================
+    for each_command in list_of_commands:
+        each_SearchResult = process_ONE_dns_result(
+            which_command=each_command,
+            target_interface=target_interface,
+        )
+        # add the command to the SearchResult here, so it's only declared once,
+        each_SearchResult.which_command_was_run = each_command
+
+        list_of_results_as_objects.append(each_SearchResult)
+        list_of_results_as_binary.append(each_SearchResult.valid)
+
+    # ================= END LOOP =====================
+    return list_of_results_as_objects, list_of_results_as_binary
+
+
+def get_quantity_of_false(list_of_results_as_binary: list) -> int:
+    quantity_of_false = 0
+    for each_result in list_of_results_as_binary:
+        if each_result is False:
+            quantity_of_false =+ 1
+    return quantity_of_false
+
+
+def orchestrate_dns_check(target_interface: str) -> Result:
+    """
+    Purpose:
+        Coordinates DNS checks for resolvctl output, with actionable results
+
+    Rank:
+        Module Orchestrator
+
+    Called by:
+        systemwide_wireguard.__establish_system_connection
+    """
+
+    list_of_results_as_objects, list_of_results_as_binary = process_ALL_dns_results(
+        target_interface=target_interface,
+    )
+
+    quantity_of_checks = len(list_of_results_as_binary)
+    quantity_of_false = get_quantity_of_false(list_of_results_as_binary)
+
+    if quantity_of_false == 0:
+        logger.info(f"[DNS Orchestrator] Congrats, all {quantity_of_checks} worked.")
+        return Result(valid=True)
+
+    else:
+        logger.info(f"[DNS Orchestrator] Some of the DNS checks did NOT work.  But this might be because the tunnel is down.")
+
+        targets_not_found = 0
+
+        for each_object in list_of_results_as_objects:
+            if each_object.error_type == SearchError.TARGET_MISSING:
+                target_not_found = target_not_found + 1
+        
+        return Result(valid=False, data=targets_not_found)
+
+
+
+
+
+# NOT YET INTEGRATED. This is for potentially showing errors to the end-user:
+
+# if search_results.valid:
+#     return Result(valid=True)
+
+# resolvctl_name = turn_command_into_readable_string(which_command)
+# reason = search_results.error_type
+# not_leak_msg = "This is NOT a leak, because the firewall is still blocking all queries outside the VPN tunnel.  But for queries to realize they are being blocked, it may cause (very small) speed delays on domain lookups.  Do you want to reconnect now or just continue as is?"
+
+# if reason == SearchError.COMPETITORS_MATCH:
+#     error_log_msg = f"{resolvctl_name} had interfaces using DNS other than the VPN."
+#     logger.error(error_log_msg)
+#     final_human_message = f"{error_log_msg} {not_leak_msg}"
+#     return Result(valid=False, error_type=ResultError.CLIENT_DNS, message=final_human_message)
+
+# elif reason == SearchError.TARGET_NO_VALUE:
+#     error_log_msg = f"{resolvctl_name} wasn't able to be set to the VPN."
+#     logger.error(error_log_msg)
+#     final_human_message = f"{error_log_msg} {not_leak_msg}"
+#     return Result(valid=False, error_type=ResultError.CLIENT_DNS, message=final_human_message)
+
+# else:
+#     error_log_msg = "There was an Unknown error in setting your default DNS to the VPN."
+#     logger.error(error_log_msg)
+#     final_human_message = f"{error_log_msg} {not_leak_msg}"
+#     return Result(valid=False, error_type=ResultError.UNKNOWN, message=final_human_message)

core/services/networking/systemwide/dns_tools/search_resolv_output.py

@@ -0,0 +1,78 @@
+from core.services.networking.systemwide.dns_tools.SearchResult import SearchResult, SearchError
+from core.errors.logger import logger
+import re
+
+def get_pattern_and_phrase(
+    which_command: str,
+    ip_address: str = "10.13.0.1"
+)-> str:
+    """figure out which re pattern & phrase to use, based on the search command"""
+
+    if which_command == "domain":
+        target_phrase = "~."
+        pattern = r'\(([^)]+)\):\s*(\S*)?'
+ 
+    elif which_command == "default-route":
+        target_phrase = "yes"
+        pattern = r'\(([^)]+)\):\s*(yes|no)'
+ 
+    elif which_command == "dns":
+        target_phrase = ip_address
+        pattern = r'\(([^)]+)\):\s*([\d.]+)'
+ 
+    else:
+        target_phrase = None
+        pattern = None
+    return pattern, target_phrase
+
+def evaluating_resolv_output(which_command: str, target_interface: str, resolv_output: str) -> SearchResult:
+    if target_interface not in resolv_output:
+        return SearchResult(valid=False, error_type=SearchError.TARGET_MISSING)
+
+    # ============= PREPARE SEARCH =============
+    pattern, target_phrase = get_pattern_and_phrase(which_command)
+    if not pattern or not target_phrase:
+        return SearchResult(valid=False, error_type=SearchError.UNKNOWN, message=f"invalid which_command of {which_command}")
+
+    # Find all entries with pattern (name): value
+    matches = re.findall(pattern, resolv_output)
+
+    # Separate target from competitors
+    target_has_match = False
+    competitors_with_match = []
+    target_actually_had = None
+
+    # ============= LOOP =============
+    for name, value in matches:
+        if value == target_phrase:
+            if name == target_interface:
+                target_has_match = True
+            else:
+                competitors_with_match.append(name)
+
+        else:
+            # Check if Target has the wrong value
+            if name == target_interface:
+                target_actually_had = value
+
+    # ============= EVALUATE =============
+    competitors_match = len(competitors_with_match) > 0
+    # logger.info(f"Target has {target_phrase}: {target_has_match}")
+    # logger.info(f"Competitors have {target_phrase}: {competitors_match}")
+    # logger.info(f"Competitor list: {competitors_with_match}")
+
+    # target has it, and nobody else:
+    if target_has_match and not competitors_match:
+        return SearchResult(valid=True, target_value=target_phrase)
+
+    if target_has_match and competitors_match:
+        return SearchResult(valid=False, error_type=SearchError.COMPETITORS_MATCH, target_value=target_phrase)
+
+    if not target_has_match:
+        # target has wrong value:
+        if target_actually_had:
+            return SearchResult(valid=False, error_type=SearchError.VALUE_DOESNT_MATCH, competitors_list=competitors_with_match, target_value=target_actually_had)
+        # target has no value
+        else:
+            return SearchResult(valid=False, error_type=SearchError.TARGET_NO_VALUE, competitors_list=competitors_with_match, target_value=target_actually_had)
+

core/services/networking/systemwide/killswitch.py

@@ -0,0 +1,37 @@
+from core.services.networking.systemwide.systemwide_errors import systemwide_hell_raiser
+from core.utils.basic_operations.wrap_with import wrap_with
+from core.utils.basic_operations.run_generic_command import run_generic_command
+from core.models.Result import Result, ResultError
+from core.errors.logger import logger
+import subprocess
+
+KILLSWITCH_WRAPPER = "/opt/hydra-veil/firewall" # needs chmod 755
+
+@wrap_with(systemwide_hell_raiser)
+def arm(server_ip: str, tunnel_if: str, internal_subnet: str = None) -> Result:
+    command = ["sudo", KILLSWITCH_WRAPPER, "arm", server_ip, tunnel_if]
+    if internal_subnet:
+        command.append(internal_subnet)
+    human_readable_goal = "Arming the Firewall"
+    return run_generic_command(command, human_readable_goal)
+
+@wrap_with(systemwide_hell_raiser)
+def disarm() -> bool:
+    command = ["sudo", KILLSWITCH_WRAPPER, "disarm"]
+    human_readable_goal = "Disarming the Firewall"
+    return run_generic_command(command, human_readable_goal)
+
+def status() -> bool:
+    command = ["sudo", KILLSWITCH_WRAPPER, "status"]
+    human_readable_goal = "Get the Firewall's status"
+    result = run_generic_command(command, human_readable_goal, timeout=5)
+    if result.valid:
+        if "armed" in result.data.strip():
+            return True
+        else:
+            logger.error("[FIREWALL STATUS] It is currently NOT armed.")
+            return False
+    else:
+        # process error with running the command itself,
+        # send the error object with details.
+        return result

core/services/networking/systemwide/systemwide_errors.py

@@ -0,0 +1,54 @@
+from core.errors.exceptions import SudoScript, MissingPreReqs
+from core.models.Result import Result, ResultError
+from core.errors.logger import logger
+
+"""
+Purpose:
+    Raises Errors for known problems with the systemwide connections
+
+Kinds of Raises:
+    SudoScript
+    MissingPreReqs
+"""
+
+def systemwide_hell_raiser(result: Result) -> Result:
+    if result.valid:
+        return result
+    
+    logger.error(f"[HELL RAISER] We were trying to {result.goal} but got an error of {result.error_type} with a human message of {result.message}")
+
+    if result.error_type == ResultError.MISSING_FILE:
+        logger.error(f"[HELL RAISER] We are missing the setup scripts for {result.goal}")
+        result.message = f"You're missing the setup scripts for {result.goal}.  Re-do the install, or check the setup you did."
+        raise SudoScript(result)
+
+    elif result.error_type == ResultError.PERMISSION:
+        error_msg = f"There's improper permissions on the scripts for {result.goal}.  Re-do the install, or check the setup you did."
+        result.message = error_msg
+        logger.error(f"[HELL RAISER] {error_msg}")
+        raise SudoScript(result)
+
+    elif result.error_type == ResultError.MISSING_DEPENDENCY:
+        error_output = result.message
+        list_of_dependencies = ["nmcli", "resolvconf", "nftables", "nft"]
+        what_is_missing = []
+
+        for each_dependency in list_of_dependencies:
+            if each_dependency in error_output:
+                what_is_missing.append(each_dependency)
+
+        result.message = f"You're missing the prereq packages of {what_is_missing}.  Re-do the install, or check the setup you did."
+        raise MissingPreReqs(result)
+
+    elif result.error_type == ResultError.INTERFACE:
+        logger.info("The Interface is not found.")
+        # might not be an error, if it's already down,
+        return result
+
+    elif result.error_type == ResultError.UNKNOWN:
+        logger.info(f"Unknown error type for {result.goal} with {result.message} and potentially output of {result.data}.")
+        return result
+
+    else:
+        logger.info("Unknown Result object")
+        return result

core/services/networking/systemwide/systemwide_utils.py

@@ -0,0 +1,68 @@
+
+from core.models.Result import Result, ResultError
+import re
+import shutil
+from core.Errors import CommandNotFoundError
+from core.models.Configuration import get_setting
+from core.errors.logger import logger
+
+
+def get_dns_setting() -> bool:
+    dns_setting = get_setting("dns")
+    logger.info(f"DNS Setting is {dns_setting}")
+    if dns_setting is None:
+        return False
+    return dns_setting
+
+
+def get_firewall_setting() -> bool:
+    firewall_setting = get_setting("firewall")
+    logger.info(f"Firewall Setting is {firewall_setting}")
+    if firewall_setting is None:
+        return False
+    return firewall_setting
+
+
+
+def check_system_prereqs():
+    if shutil.which('dbus-send') is None:
+        raise CommandNotFoundError('dbus-send')
+
+    if shutil.which('nmcli') is None:
+        raise CommandNotFoundError('nmcli')
+
+
+
+def extract_wg_interface_name(text: str) -> Result:
+    """
+    Purpose:
+        Extracts the wg interface name, which should be 'wg'
+
+    Pattern:
+        "Connection" followed by text in single quotes
+
+    Example:
+        "Connection 'wg' (jgfdgfdjgfdjfdg"
+    """
+
+    pattern = r"Connection '([^']*)'"
+    
+    match = re.search(pattern, text)
+    
+    if match:
+        # Return what's between the quotes
+        result = match.group(1)
+        return Result(valid=True, data=result)
+    else:
+        error_msg = "Error: String doesn't contain 'Connection' and text within single quotes"
+        return Result(valid=True, error_type=ResultError.NMCLI, message=error_msg)
+
+
+
+
+
+
+
+######### ISOLATION TESTING ########
+# result1 = extract_connection_name("Connection 'wg' (jgfdgfdjgfdjfdg")
+# print(result1)  # Output: wg

core/services/networking/systemwide/systemwide_wireguard.py

@@ -0,0 +1,269 @@
+from core.services.networking.general_connection_tools.testing_evaluating import await_connection, system_uses_wireguard_interface, terminate_tor_connection
+from core.services.keys_and_verifications.endpoint_verification import verify_wireguard_endpoint
+from core.models.Result import Result, ResultError
+from core.services.networking.systemwide.systemwide_utils import extract_wg_interface_name, check_system_prereqs, get_firewall_setting, get_dns_setting
+# from core.services.networking.general_connection_tools.config_tools import extract_endpoint_ip, extract_dns
+from core.services.networking.systemwide import killswitch
+from core.services.networking.systemwide.dns_tools.process_dns_result import orchestrate_dns_check
+from core.services.networking.systemwide.dns_tools.SearchResult import SearchResult, SearchError
+from core.services.networking.systemwide.wireguard.nmcli_tools import setup_nmcli_connection, setup_ipv6_sinkhole
+from core.services.networking.systemwide.wireguard.wg_firewall_dns import set_dns, revert_dns, turn_on_firewall, check_and_kill_firewall
+from core.errors.logger import logger
+
+from core.Constants import Constants
+from core.Errors import ConnectionTerminationError, CommandNotFoundError
+from core.controllers.ConfigurationController import ConfigurationController
+from core.controllers.SystemStateController import SystemStateController
+from core.models.BaseProfile import ProfileType
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+from core.models.system.SystemState import SystemState
+from core.observers.ConnectionObserver import ConnectionObserver
+from subprocess import CalledProcessError
+from typing import Optional
+import os
+import re
+import shutil
+import subprocess
+import time
+
+
+def terminate_system_connection(
+    firewall_setting: Optional[bool] = get_firewall_setting(),
+    dns_setting: Optional[bool] = get_dns_setting()
+):
+
+    if shutil.which('nmcli') is None:
+        raise CommandNotFoundError('nmcli')
+
+    # ====== FIREWALL =======
+    if firewall_setting:
+        check_and_kill_firewall() # on failure, this raises errors, which then bubble up
+
+    # ====== DNS =======
+    if dns_setting:
+        reverting_dns_worked = revert_dns()
+
+    # ====== NMCLI =======
+    if SystemStateController.exists():
+        process = subprocess.Popen(('nmcli', 'connection', 'delete', 'wg'), stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+        completed_successfully = not bool(os.waitpid(process.pid, 0)[1] >> 8)
+
+        if completed_successfully or not system_uses_wireguard_interface():
+
+            subprocess.run(('nmcli', 'connection', 'delete', 'hv-ipv6-sink'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            terminate_tor_connection()
+
+            # confirm check on dns:
+            # did_dns_revert = confirm_dns_is_reverted('wg')
+            # print(f"did_dns_revert is {did_dns_revert}")
+
+            # finally end that profile's JSON state:
+            SystemState.dissolve()
+
+        else:
+            raise ConnectionTerminationError('The connection could not be terminated.')
+
+
+
+def __establish_system_connection(
+    profile: SystemProfile,
+    firewall_setting: bool = False,
+    dns_setting: bool = False,
+    connection_observer: Optional[ConnectionObserver] = None):
+    """
+    Purpose:
+        Orchestrates the full flow of the connection.
+
+    Steps:
+    1) Check Prereqs
+    2) Disable any pre-existing system connections
+    3) Turn on WG via nmcli
+    4) Turn on IPv6 sinkhole
+    5) Turn on Firewall (if enabled)
+    6) Set DNS (if enabled), and test it.
+    7) Setup State (JSON file)
+    8) Confirm Firewall is on (if enabled).  Raises errors if not.
+    9) Testing, confirming, displaying to the end user.
+    """
+
+    check_system_prereqs()
+    logger.info("[CONNECT] Terming existing connections..")
+    terminate_system_connection(firewall_setting, dns_setting)
+
+    # ============= INITIAL NMCLI CONNECTION =============
+    wg_config_path = profile.get_wireguard_configuration_path()
+    nmcli_result = setup_nmcli_connection(wg_config_path)
+
+    if nmcli_result.valid:
+        process_output = nmcli_result.data
+        logger.info("[CONNECT] Successfully turned on WG with nmcli")
+    else:
+        logger.error(f"[SYSTEMWIDE WG] Critical issue, the initial nmcli command failed {nmcli_result.message}.")
+        return nmcli_result
+
+    # ============= IPv6 SINKHOLE =============
+    sinkhole_result = setup_ipv6_sinkhole(process_output)
+    if not sinkhole_result:
+        raise ConnectionError('IPv6 Could not be protected.')
+
+    # ============= SETUP STATE =============
+    # Even if firewall is off, we want to save the fact we turned on the VPN tunnel, before we raise errors.
+
+    logger.info(f"Setting State JSON..")
+    current_state = SystemStateController.create(
+        profile_id=profile.id,
+        firewalled=firewall_setting, # intended
+        dns_set=dns_setting # intended
+    )
+    
+    # ============= PREP SETTINGS =============
+    if firewall_setting or dns_setting:
+        wg_interface_result = extract_wg_interface_name(process_output)
+        if not wg_interface_result.valid:
+            error_msg = f"WG interface is in the wrong format.  This is the raw output: {process_output}"
+            logger.error(f"[SYSTEMWIDE WG] Critical issue, the {error_msg}")
+            raise ConnectionError(error_msg)
+            # or handle the interface being wrong?
+
+        wg_interface_name = wg_interface_result.data
+
+    # ============= FIREWALL =============
+    firewall_tracker = False
+    if firewall_setting:
+        logger.info("Firewall setting is enabled, let's turn it on..")
+        turn_on_result = turn_on_firewall(
+            profile_id=str(profile.id),
+            wg_interface_name=wg_interface_name
+        )
+        logger.info(f"Results of firewall turn on are: {turn_on_result.valid}")
+        if turn_on_result.valid:
+            firewall_tracker = True
+
+    # ============= CONFIRM ON FIREWALL =============
+    # if firewall_setting:
+        logger.info(f"Evaluating Firewall Situation after we tried to turn on it on.")
+        firewall_status = killswitch.status()
+        logger.info(f"""
+The firewall setting is on.
+The result of arming it, {turn_on_result.valid}
+The result of the status {firewall_status}
+        """)
+        if firewall_status:
+            logger.info("Firewall is on. All good.")
+        else:
+            logger.error("Firewall FAILED TO ARM")  # DON'T TRY TO RENEGOTIATE WITH DEAD INTERNET !!
+
+            # raise ConnectionError(f'Firewall failed to arm!! {turn_on_result.message}.') # if this triggers, it tries to renegotiate with dead internet.
+
+    # ============= DNS =============
+    if dns_setting:
+        set_dns_result = set_dns(str(profile.id), wg_interface_name)
+        logger.info(f"[CONNECT] The result of our setting the DNS is {set_dns_result.valid}")
+        if not set_dns_result.valid:
+            logger.error(f"[CONNECT] Setting the DNS DIDN'T work. Here's the reason given: {set_dns_result.message}")
+
+        # now check:
+        test_dns_result = orchestrate_dns_check(wg_interface_name)
+        if test_dns_result.valid:
+            did_dns_work = True
+            logger.info("[CONNECT] All of the DNS Checks Worked")
+    else:
+        did_dns_work = False
+        logger.info("Skipping DNS setup, because it's off in settings")
+
+    # ============= TESTING =============
+    logger.info("If we made it this far, we have nmcli on, ipv6 sinkhole on, firewall on.")
+    try:
+        await_connection(connection_observer=connection_observer)
+
+    except ConnectionError:
+        raise ConnectionError('The connection could not be established.')
+
+    # ============= UPDATE STATE =============
+    current_state.firewalled = firewall_tracker
+    current_state.dns_set = did_dns_work
+    SystemStateController.update_or_create(current_state)
+
+    return Result(valid=True)
+
+
+def evaluate_connection_result(connection_result: Result):
+    if not connection_result.valid:
+        logger.error(f"[SYSTEMWIDE WG] Critical issue, could not establish a connection: {connection_result.message}")
+        try:
+            terminate_system_connection(firewall_setting, True)
+        except ConnectionTerminationError:
+            pass
+
+
+def establish_system_connection(
+    profile: SystemProfile,
+    ignore: tuple[type[Exception]] = (),
+    connection_observer: Optional[ConnectionObserver] = None):
+    """
+    Purpose:
+        Calls the private function with the full flow, so it can be retried on failure.
+
+    Steps)
+    1) Verify Endpoint's encryption
+    2) Pass to private function
+    3) Retry on failure
+    """
+
+    # ================= VERIFY ENDPOINT =================
+    if ConfigurationController.get_endpoint_verification_enabled():
+        verify_wireguard_endpoint(profile, ignore=ignore)
+
+    # ================= SETTINGS =================
+    firewall_setting = get_firewall_setting()
+    # dns_setting = True # for testing
+    dns_setting = get_dns_setting()
+
+    # ================= CONNECT WG ================
+    try:
+        connection_result = __establish_system_connection(
+            profile=profile,
+            firewall_setting=firewall_setting,
+            dns_setting=dns_setting,
+            connection_observer=connection_observer
+        )
+        evaluate_connection_result(connection_result)
+
+    except ConnectionError:
+        # ================= RETRY ON FAILURE =================
+        try:
+            terminate_system_connection(firewall_setting, dns_setting)
+        except ConnectionTerminationError:
+            pass
+
+        raise ConnectionError('The connection could not be established.')
+
+    except CalledProcessError:
+
+        try:
+            terminate_system_connection(firewall_setting, dns_setting)
+        except ConnectionTerminationError:
+            pass
+
+        # trying again..
+        try:
+            connection_result = __establish_system_connection(
+            profile=profile,
+            firewall_setting=firewall_setting,
+            dns_setting=dns_setting,
+            connection_observer=connection_observer
+            )
+            evaluate_connection_result(connection_result)
+
+        except (ConnectionError, CalledProcessError):
+
+            try:
+                terminate_system_connection(firewall_setting, dns_setting)
+            except ConnectionTerminationError:
+                pass
+
+            raise ConnectionError('The connection could not be established.')
+
+    terminate_tor_connection()
+    time.sleep(1.0)

core/services/networking/systemwide/wireguard/nmcli_tools.py

@@ -0,0 +1,42 @@
+from core.models.Result import Result, ResultError
+from subprocess import CalledProcessError
+import os
+import re
+import shutil
+import subprocess
+
+
+def setup_nmcli_connection(wg_config_path: str) -> Result:
+    try:
+        process_output = subprocess.check_output(('nmcli', 'connection', 'import', '--temporary', 'type', 'wireguard', 'file', wg_config_path), text=True)
+        return Result(valid=True, data=process_output)
+
+    except CalledProcessError as e:
+        return Result(valid=False, error_type=ResultError.NMCLI, message=f"Called Process error with nmcli: {e}")
+        # raise ConnectionError('The connection could not be established.')
+
+
+def setup_ipv6_sinkhole(process_output: str) -> Result:
+    error_info = "with nmcli setting up the IPv6 sinkhole, error is "
+    try:
+        connection_id = (m := re.search(r'(?<=\()([a-f0-9-]+?)(?=\))', process_output)) and m.group(1)
+        ipv6_method = subprocess.check_output(('nmcli', '-g', 'ipv6.method', 'connection', 'show', connection_id), text=True).strip()
+    except CalledProcessError as e:
+        return Result(valid=False, error_type=ResultError.NMCLI, message=f"Called Process error {error_info}: {e}")
+        # raise ConnectionError('The connection could not be established.')
+
+    if ipv6_method in ('disabled', 'ignore'):
+        try:
+            subprocess.run(('dbus-send', '--system', '--print-reply', '--dest=org.freedesktop.NetworkManager', '/org/freedesktop/NetworkManager', 'org.freedesktop.DBus.Properties.Set', 'string:org.freedesktop.NetworkManager', 'string:ConnectivityCheckEnabled', 'variant:boolean:false'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
+        except CalledProcessError as e:
+            return Result(valid=False, error_type=ResultError.NMCLI, message=f"Called Process error {error_info}: {e}")
+            # raise ConnectionError('The connection could not be established.')
+
+        try:
+            subprocess.run(('nmcli', 'connection', 'add', 'type', 'dummy', 'save', 'no', 'con-name', 'hv-ipv6-sink', 'ifname', 'hvipv6sink0', 'ipv6.method', 'manual', 'ipv6.addresses', 'fd7a:fd4b:54e3:077c::/64', 'ipv6.gateway', 'fd7a:fd4b:54e3:077c::1', 'ipv6.dns', '::1', 'ipv6.route-metric', '72'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
+            return Result(valid=True)
+
+        except CalledProcessError:
+            return Result(valid=False, error_type=ResultError.NMCLI, message=f"Called Process error {error_info}")
+            # raise ConnectionError('The connection could not be established.')
+

core/services/networking/systemwide/wireguard/wg_firewall_dns.py

@@ -0,0 +1,105 @@
+from core.models.Result import Result, ResultError
+from core.services.networking.systemwide import killswitch, dns
+from core.services.networking.general_connection_tools.config_tools import extract_endpoint_ip, extract_dns
+from core.errors.logger import logger
+
+def set_dns(profile_id: str, network_interface: str) -> Result:
+    do_they_even_use_systemd = dns.is_systemd_enabled()
+
+    if not do_they_even_use_systemd.valid:
+        error_msg = "You don't use SystemD, so we are skipping setting your DNS."
+        logger.info(error_msg)
+        return Result(valid=True, message=error_msg) # true to not prompt errors
+
+    dns_should_be = extract_dns(profile_id)
+
+    logger.info(f"[CONNECT] About to set the DNS for {network_interface}..")
+    return dns.set_on(network_interface=network_interface, dns_should_be=dns_should_be)
+
+
+
+def revert_dns() -> Result:
+    do_they_even_use_systemd = dns.is_systemd_enabled()
+
+    if not do_they_even_use_systemd.valid:
+        error_msg = "You don't use SystemD, so we are skipping setting your DNS."
+        logger.info(error_msg)
+        return Result(valid=True, message=error_msg) # true to not prompt errors
+
+    logger.info("Reverting DNS Settings..")
+    turn_off = dns.revert()
+    logger.info(f"The result of the turn off is {turn_off.valid}")
+    if not turn_off.valid:
+        logger.error(f"Critical DNS Error! Could NOT turn off DNS. Error type: {turn_off.error_type} with {turn_off.message}")
+    return turn_off.valid
+
+
+
+def turn_on_firewall(profile_id: str, wg_interface_name: str) -> Result:
+    """
+    Purpose:
+        This is a wireguard specific function, using the WG interface,
+        but the underlying killswitch.arm function it uses is protocol neutral.
+
+    Steps:
+    1) Get the WG interface from the process output
+    2) Get the server IP from the backup WG config
+    3) Use the killswitch's arm
+    """
+    logger.info(f"[SYSTEMWIDE WG] Turning on Firewall..")
+
+    server_ip_address = extract_endpoint_ip(profile_id)
+    if wg_interface_name != 'wg':
+        logger.info(f"Note: The WG interface is NOT WG.  It's {wg_interface_name}.  Be careful about what we kill switch compared to what goes up.  Although we kill the nftable rule regardless on down.")
+
+    return killswitch.arm(
+        server_ip=server_ip_address,
+        tunnel_if=wg_interface_name,
+        internal_subnet=None
+    )
+
+
+# This raises errors on failure.
+def check_and_kill_firewall():
+    firewall_on = killswitch.status() # produces a boolean
+    logger.info(f"The Firewall is currently: {firewall_on}")
+
+    # is it even armed? if not, get lost,
+    if firewall_on:
+        # if it's armed, kill it,
+        logger.info(f"Killing the firewall..")
+        disable_result_object = killswitch.disarm() # produces a Result Object
+
+        # did that work?
+        if disable_result_object.valid:
+            logger.info(f"Success! We disabled the firewall correctly.")
+            return
+        else:
+            error_msg = f"Firewall can't be disabled: {disable_result_object.message}"
+            logger.error(error_msg)
+
+            # if the systemwide_raiser didn't catch it with the reason, then..
+            raise ConnectionTerminationError(error_msg)
+    else:
+        logger.info("We are skipping disabling the firewall, because it's already off.")
+
+
+# used in dead code at bottom:
+# from core.services.networking.systemwide.dns_tools.process_dns_result import orchestrate_dns_check
+
+# This is dead code right now
+# def confirm_dns_is_reverted(wg_interface_name: str = 'wg'):
+#     do_they_even_use_systemd = dns.is_systemd_enabled()
+
+#     if not do_they_even_use_systemd:
+#         return True
+
+#     check_on_dns = orchestrate_dns_check(wg_interface_name)
+
+#     if check_on_dns.valid:
+#         logger.error("CRITICAL DNS PROBLEM! We turned WG off, but the DNS check shows it's still on.  Trying again..")
+#         return False
+#     else:
+#         logger.info("We turned WG off, and the DNS check shows it reverted also. This is great.")
+#         return True
+

core/services/networking/use_tor.py

@@ -7,6 +7,9 @@ if TYPE_CHECKING:
 from essentials.modules.TorModule import TorModule
 from essentials.observers.ConnectionObserver import ConnectionObserver
 from essentials.services.ConnectionService import ConnectionService
+
+from core.observers.ClientObserver import ClientObserver
+
 # services
 from core.services.networking.evaluate_networking_problem import evaluate_tor_networking_problem
 # errors
@@ -15,54 +18,119 @@ from core.errors.logger import logger
 # generic
 import json, os
 from typing import Any
+from core.Constants import Constants
+import time
 
-
-def tor_get_request(custom_url: str, connection_observer: ConnectionObserver) -> dict:
+def tor_get_request(custom_url: str, connection_observer: ConnectionObserver, client_observer: Optional[ClientObserver] = None) -> dict:
     import requests
 
+    # ============================================================================
+    # INITIALIZE TOR
+    # ============================================================================
     port_number = ConnectionService.get_random_available_port_number()
-    logger.debug(f"Using Tor on port number {port_number}")
+    logger.debug(f"[USE TOR SERVICE] Using Tor on port number {port_number}")
+    if client_observer is not None:
+        client_observer.notify('synchronizing', f"Using Tor on Port {port_number}..")
 
     try:
-        tor_module = TorModule(os.path.expanduser("~/sp-tor-test"))
-
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
         tor_module.create_session(port_number, connection_observer)
+        logger.debug(f"[USE TOR SERVICE] Tor Started a Session")
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Initialized Tor..")
+
+    except TorServiceInitializationError as e:
+        error_msg = f"[USE TOR SERVICE] Tor failed to start: {e}"
+        tor_module.destroy_session(port_number)
+        logger.error(error_msg, exc_info=True)
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Failed to Initialize")
+
+        problem_results = evaluate_tor_networking_problem(
+            custom_url, connection_observer, client_observer
+        )
+        return problem_results
+    except Exception as e:
+        logger.error(f"[USE TOR SERVICE] Unexpected Tor error: {e}")
+        tor_module.destroy_session(port_number)
+        problem_results = evaluate_tor_networking_problem(
+            custom_url, connection_observer
+        )
+        return problem_results
 
     proxies = {
         "http": f"socks5h://127.0.0.1:{port_number}",
         "https": f"socks5h://127.0.0.1:{port_number}",
     }
 
-        logger.debug(f"Doing Request through Tor via port {port_number}")
-        response = requests.get(custom_url, proxies=proxies)
-        logger.debug("Request through Tor successful.")
+    # ============================================================================
+    # USE TOR AS PROXY
+    # ============================================================================
+    try:
+        logger.debug(f"[TOR GET REQUEST] Using Tor on port number {port_number}")
+        response = requests.get(custom_url, proxies=proxies, timeout=5)
+        logger.debug("[TOR GET REQUEST] Request through Tor Successful!")
 
-        # if it crashes from the above error check, then it won't destroy the proxy,
+        # if it crashes from an error in trying to get a JSON out of it, then we'd skip destroying the Tor session,
+        # so we'll destroy it now first,
+        logger.debug("[TOR GET REQUEST] DESTROY SESSION!")
         tor_module.destroy_session(port_number)
 
-        # return response
-
+        # Now that it's safe, get a JSON out of it, and return the response
         dictonary_of_response = response.json()
+        logger.debug(f"[TOR GET REQUEST] Got Data out of the reply. {dictonary_of_response}")
         final_reply = {"valid": True, "data": dictonary_of_response}
         return final_reply
 
     except requests.exceptions.ConnectionError:
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Connection Error")
+
+        tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
-            custom_url, connection_observer, port_number, proxies
+            custom_url, connection_observer
         )
         return problem_results
 
     except requests.exceptions.Timeout:
         logger.debug("Connection timed out")
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Connection Timeout")
+
+        tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
-            custom_url, connection_observer, port_number
+            custom_url, connection_observer
         )
         return problem_results
 
     except requests.exceptions.HTTPError:
-        logger.debug(f"HTTP error: {response.status_code}")
+        tor_http_error = f"Tor HTTP error: {response.status_code}"
+        logger.debug(tor_http_error)
+        if client_observer is not None:
+            client_observer.notify('synchronizing', tor_http_error)
+
+        tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
-            custom_url, connection_observer, port_number
+            custom_url, connection_observer
+        )
+        return problem_results
+    except requests.exceptions.ProxyError:
+        error_msg = "[USE TOR SERVICE] Requests Proxy Error"
+        tor_module.destroy_session(port_number)
+        logger.error(error_msg, exc_info=True)
+        problem_results = evaluate_tor_networking_problem(
+            custom_url, connection_observer
+        )
+        return problem_results
+
+    except Exception as e:
+        logger.error(f"[USE TOR SERVICE] Unexpected Tor error: {e}")
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Error: {e}")
+
+        tor_module.destroy_session(port_number)
+        problem_results = evaluate_tor_networking_problem(
+            custom_url, connection_observer
         )
         return problem_results
 
@@ -79,41 +147,63 @@ def tor_post_request(
         "status": "unable_to_get"
     }  # incase this goes to the except block, it's "defined"
 
+    # ============================================================================
+    # INITIALIZE TOR
+    # ============================================================================
     port_number = ConnectionService.get_random_available_port_number()
     logger.debug(f"Using Tor on port number {port_number}")
 
     try:
-        tor_module = TorModule(os.path.expanduser("~/sp-tor-test"))
-
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
         tor_module.create_session(port_number, connection_observer)
+    except TorServiceInitializationError as e:
+        error_msg = f"[USE TOR SERVICE] Tor failed to start: {e}"
+        tor_module.destroy_session(port_number)
+        logger.error(error_msg, exc_info=True)
+        problem_results = evaluate_tor_networking_problem(
+            custom_url, connection_observer
+        )
+        return problem_results
+    except Exception as e:
+        logger.error(f"[USE TOR SERVICE] Unexpected Tor error: {e}")
+        tor_module.destroy_session(port_number)
+        problem_results = evaluate_tor_networking_problem(
+            custom_url, connection_observer
+        )
+        return problem_results
 
     proxies = {
         "http": f"socks5h://127.0.0.1:{port_number}",
         "https": f"socks5h://127.0.0.1:{port_number}",
     }
-
     logger.debug(f"Sending data through Tor via port {port_number}")
 
+    # ============================================================================
+    # USE TOR AS PROXY
+    # ============================================================================
+    try:
         response = requests.post(custom_url, json=payload, proxies=proxies)
-        logger.debug("Sending data through Tor successful.")
+        logger.debug("[USE TOR SERVICE] Sent data through Tor successfully.  Now deleting the session..")
 
         tor_module.destroy_session(port_number)
 
         return response
 
     except requests.exceptions.ConnectionError:
-        error_msg = "There was a Connection Error with the Tor Module."
+        error_msg = "[USE TOR SERVICE] There was a Connection Error with the Use Tor Module."
         logger.error(error_msg, exc_info=True)
         logger.debug(error_msg)
+        tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
             custom_url, connection_observer
         )
         return problem_results
 
     except requests.exceptions.Timeout:
-        error_msg = "Connection timed out"
+        error_msg = "[USE TOR SERVICE] Connection timed out"
         logger.error(error_msg, exc_info=True)
         logger.debug(error_msg)
+        tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
             custom_url, connection_observer
         )
@@ -121,10 +211,28 @@ def tor_post_request(
 
     except requests.exceptions.HTTPError:
         if isinstance(response, requests.Response):
-            error_msg = f"HTTP error: {response.status_code}"
+            error_msg = f"[USE TOR SERVICE] HTTP error: {response.status_code}"
             logger.error(error_msg, exc_info=True)
             logger.debug(error_msg)
 
+        tor_module.destroy_session(port_number)
+        problem_results = evaluate_tor_networking_problem(
+            custom_url, connection_observer
+        )
+        return problem_results
+
+    except requests.exceptions.ProxyError:
+        error_msg = "[USE TOR SERVICE] Requests Proxy Error"
+        logger.error(error_msg, exc_info=True)
+        tor_module.destroy_session(port_number)
+        problem_results = evaluate_tor_networking_problem(
+            custom_url, connection_observer
+        )
+        return problem_results
+
+    except Exception as e:
+        logger.error(f"[USE TOR SERVICE] Unexpected Tor error: {e}")
+        tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
             custom_url, connection_observer
         )

core/services/payment_phase/check_if_paid.py

@@ -1,31 +0,0 @@
-from __future__ import annotations
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from essentials.observers.ConnectionObserver import ConnectionObserver
-# services:
-from core.services.networking.send_data_to_server import send_data_to_server
-from core.services.networking.make_url import make_url
-# errors:
-from core.errors.exceptions import *
-from core.errors.logger import logger
-
-
-# it's private because it assumes it's being called from the controller, (with a JSON)
-def _check_if_paid(payload: dict, connection_observer: ConnectionObserver) -> str:
-    # prep endpoint:
-    which_endpoint = "check_paid"
-    url = make_url(which_endpoint)
-
-    # literally send:
-    reply = send_data_to_server(payload, url, connection_observer)
-
-    if "valid" in reply:
-        valid_status = reply.get("valid")
-        if valid_status == True:
-            return "paid"
-        else:
-            return "not_paid"
-    else:
-        error_msg = f"When checking if paid, the Server returned invalid data or it never sent.  The reply is {reply}"
-        raise InvalidData(error_msg)

core/services/payment_phase/do_we_have_billing_id.py

@@ -0,0 +1,13 @@
+from core.utils.basic_operations.write_or_read_from_json import get_value_from_json_file
+from core.Constants import Constants
+
+def do_we_have_billing_id() -> str | None:
+    try:
+        billing_folder = Constants.HV_TICKETING_CONFIG_HOME
+        filepath = f"{billing_folder}/billing_choices.json"
+
+        billing_id = get_value_from_json_file(filepath, "temp_billing_code")
+        return billing_id
+
+    except:
+        return None

core/services/payment_phase/save_and_send_intitial_billing.py

@@ -5,7 +5,13 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 #
 from core.models.invoice.TicketInvoice import TicketInvoice
-from core.services.networking.send_data_to_server import send_data_to_server
+
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+# from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
+
 from core.services.networking.make_url import make_url
 from core.services.payment_phase.extract_payment_details import extract_payment_details
 from core.services.payment_phase.save_billing_choices import save_billing_choices
@@ -18,7 +24,22 @@ def save_and_send_intitial_billing(
     payload: dict,
     connection_observer: ConnectionObserver,
     invoice_data_object: TicketInvoice,
-) -> TicketInvoice:
+) -> TicketInvoice | ApiResponse:
+
+    """
+    Purpose:
+        Does a POST request to get updated billing information.
+
+    Returns:
+        TicketInvoice on Success, with the modified values.
+        ApiResponse on Failure, with why the API failed.
+
+    Called by:
+        TicketPayController
+
+    Errors:
+        No. Does NOT raise errors.
+    """
 
     # Save choices:
     save_billing_choices(payload)
@@ -26,9 +47,23 @@ def save_and_send_intitial_billing(
     # send them:
     which_endpoint = "start_payment"
     url = make_url(which_endpoint)
-    reply = send_data_to_server(payload, url, connection_observer)
+    api_reply_object = send_data_to_server(payload, url, connection_observer)
+
+    if not api_reply_object.valid:
+        api_reply_object = solve_api_problems(
+            api_reply_object=api_reply_object,
+            get_or_post="post",
+            url=url,
+            payload=payload,
+            connection_observer=connection_observer,
+            client_observer=None
+        )
+        if not api_reply_object.valid:
+            return api_reply_object
+
+    reply_dict_data = api_reply_object.data
 
     # extract values from server's reply:
-    modified_data_object = extract_payment_details(reply, invoice_data_object)
+    modified_data_object = extract_payment_details(reply_dict_data, invoice_data_object)
 
     return modified_data_object