Editing out the already moved & commented out Connection Controller pieces

change_log.md

@@ -0,0 +1,38 @@
+# Major Change Log: 
+
+
+# 2.4.3  
+### Systemwide Killswitch Introduced for Enable  
+
+### July 23, 2026  
+Separation of performing subprocess commands, parsing them for error strings, and handling those errors as enum types.  Systemwide error handling was introduced via a wrapper to all systemwide functions requiring the sudo scripts.  
+<br/>
+
+### July 20, 2026  
+Introduced IP-interface-based killswitch support for systemwide wireguard, on enable only.  This version is stable and working for enable only, with a "test" script in the fileslot. And fixed GUI's object save creation. Additionally rolled back Connection data models to previous non-enum version.
+<br/>
+
+# 2.4.2  
+### Surgery on ConnectionController  
+Huge separation of ConnectionController & ProfileController into naked function modules for enabling a connection, registering wireguard keys, coordinating proxy configs, subscriptions, endpoint verification, and the logistical flow.  Additionally enums were used for type checking, instead of polymorphic functions.  This version is stable and confirmed working for systemwide and session wireguard, but not yet proxies.  
+<br/>
+
+# 2.4.1
+### Enums for Profile types.
+July 18, 2026
+Introduce Base profile types on load, which gradually will switch over on save/edits. Transition Connections to using Enum instead of raw strings.  End goal here is to transition ConnectionController to use the Enums.  
+<br/>
+
+# 2.4.0  
+July 15, 2026  
+Robust Database Error Handling & Migrations System.  For New Client upgrades (Operational Errors) & Old Clients with New JSON keys from an API (TypeErrors).  We also added an SQL generic handling wrapper for use across any project or function.
+<br/>
+
+# 2.3.9  
+July 11, 2026  
+Features: Revamp of GET/POST API requests for both Tor and Clearweb.  
+Why: Move towards more clear Tor API error feedback.  And reduce redundancy.  
+Status: Stable, works for both clear/tor and GET/POST.  
+Still Needs: UI feedback, DNS Tor improvements
+<br/>
+

core/Constants.py

@@ -8,6 +8,13 @@ class Constants:
 
     DB_VERSION_THIS_APP_WANTS = 1
 
+    # Fallback for development (running outside AppImage)
+    fallback_non_appimage = os.path.dirname(os.path.abspath(__file__))
+
+    # appimage home
+    APPDIR_HOME: Final[str] = os.environ.get('APPDIR', fallback_non_appimage)
+    print(f"APPDIR_HOME is {APPDIR_HOME}")
+    
     # ticketing group:
     TICKET_API_BASE_URL: Final[str] = os.environ.get(
         "TICKET_API_BASE_URL", "https://ticket.hydraveil.net"
@@ -58,3 +65,8 @@ class Constants:
 
     HV_SESSION_STATE_HOME: Final[str] = f'{HV_STATE_HOME}/sessions'
     HV_TOR_STATE_HOME: Final[str] = f'{HV_STATE_HOME}/tor'
+
+    # ── killswitch / dns wrappers ─────────────────────────────────────────────
+    KILLSWITCH_WRAPPER:         Final[str] = os.environ.get(
+        'KILLSWITCH_WRAPPER', '/opt/hydra-veil/killswitch'
+    )

core/assets/sudo_scripts/dns

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ "$EUID" -ne 0 ]]; then
+    echo "[killswitch] ERROR: must be run as root" >&2
+    exit 1
+fi
+
+ACTION="${1:-}"
+IFACE="${2:-}"
+DNS="${3:-}"
+
+
+_check_prereqs() {
+        [[ -z "$DNS" ]] && { echo "Error: DNS IP required" >&2; exit 1; }
+        
+        # ✓ Check it's a valid IPv4 address:
+        if ! [[ "$DNS" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then
+            echo "Error: Invalid IP: $DNS" >&2
+            exit 1
+        fi
+                
+        # ✓ checks interface exists before configuring
+        if ! ip link show "$IFACE" &>/dev/null; then
+            echo "Error: Interface $IFACE not found" >&2
+            exit 1
+        fi
+}
+
+_set_dns() {
+    resolvectl dns "$IFACE" "$DNS" || {
+        echo "ERROR: Failed to set DNS to $DNS" >&2
+        return 1
+    }
+    resolvectl domain "$IFACE" '~.' || {
+        echo "ERROR: Failed to set domain routing" >&2
+        return 1
+    }
+    resolvectl default-route "$IFACE" true || {
+        echo "ERROR: Failed to set default route" >&2
+        return 1
+    }
+}
+
+if [[ "$ACTION" == "set" ]]; then
+    _check_prereqs
+    _set_dns
+fi
+
+# if [[ "$ACTION" == "revert" ]]; then
+#     resolvectl revert "$IFACE" 2>/dev/null || true
+
+# fi
+
+if [[ "$ACTION" == "revert" ]]; then
+    resolvectl revert "$IFACE" || true
+fi

core/assets/sudo_scripts/firewall

@@ -0,0 +1,120 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ "$EUID" -ne 0 ]]; then
+    echo "[killswitch] ERROR: must be run as root" >&2
+    exit 1
+fi
+
+ACTION="${1:-}"
+SERVER_IP="${2:-}"
+TUNNEL_IF="${3:-}"
+INTERNAL_SUBNET="${4:-}"
+TABLE="hydraveil"
+
+if [[ "$ACTION" != "arm" && "$ACTION" != "disarm" && "$ACTION" != "status" ]]; then
+    echo "[killswitch] ERROR: invalid action '$ACTION'. Usage: arm <server_ip> <tunnel_if> [internal_subnet] | disarm | status" >&2
+    exit 1
+fi
+
+_default_iface() {
+    ip route show default 2>/dev/null | awk 'NR==1{print $5}'
+}
+
+_log() {
+    local level="$1"; shift
+    echo "[killswitch] [$level] $* — $(date '+%Y-%m-%d %H:%M:%S')" >&2
+}
+
+if [[ "$ACTION" == "status" ]]; then
+    if nft list table inet "$TABLE" &>/dev/null 2>&1; then
+        echo "armed"
+    else
+        echo "disarmed"
+    fi
+    exit 0
+fi
+
+if [[ "$ACTION" == "disarm" ]]; then
+    if nft list table inet "$TABLE" &>/dev/null 2>&1; then
+        nft delete table inet "$TABLE"
+        _log "INFO" "disarmed"
+    else
+        _log "INFO" "already disarmed"
+    fi
+    exit 0
+fi
+
+[[ -z "$SERVER_IP" ]]  && { _log "ERROR" "server_ip required for arm"; exit 1; }
+[[ -z "$TUNNEL_IF" ]]  && { _log "ERROR" "tunnel_if required for arm"; exit 1; }
+
+if ! [[ "$SERVER_IP" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
+    _log "ERROR" "invalid IPv4: $SERVER_IP"
+    exit 1
+fi
+
+IFS='.' read -r o1 o2 o3 o4 <<< "$SERVER_IP"
+for oct in "$o1" "$o2" "$o3" "$o4"; do
+    if (( oct > 255 )); then
+        _log "ERROR" "invalid IPv4 octet ($oct) in $SERVER_IP"
+        exit 1
+    fi
+done
+
+WAN_IFACE=$(_default_iface)
+if [[ -z "$WAN_IFACE" ]]; then
+    _log "ERROR" "could not detect primary network interface"
+    exit 1
+fi
+
+INTERNAL_RULE=""
+if [[ -n "$INTERNAL_SUBNET" ]]; then
+    INTERNAL_RULE="        ip daddr ${INTERNAL_SUBNET} accept"
+fi
+
+nft list table inet "$TABLE" &>/dev/null 2>&1 && nft delete table inet "$TABLE" || true
+
+nft -f - << NFTEOF
+table inet ${TABLE} {
+    chain output {
+        type filter hook output priority filter; policy drop;
+        oifname "lo" accept
+        ether type arp drop
+        ip6 daddr != ::1 drop
+        ip daddr 224.0.0.0/4 drop
+        ip daddr 255.255.255.255 drop
+        ip daddr 192.168.1.0/24 drop
+        
+        oifname "${WAN_IFACE}" ip daddr ${SERVER_IP} accept
+        oifname "${TUNNEL_IF}" accept        
+        log prefix "hydraveil-drop " drop
+    }
+
+    chain input {
+        type filter hook input priority filter; policy drop;
+        iifname "lo" accept
+        
+        ether type arp drop
+        ip6 saddr != ::1 drop
+        ip daddr 224.0.0.0/4 drop
+        ip daddr 255.255.255.255 drop
+
+        ct state established,related accept
+        iifname "${WAN_IFACE}" ip saddr ${SERVER_IP} accept
+        iifname "${TUNNEL_IF}" accept
+        
+        log prefix "hydraveil-drop " drop
+    }
+    chain forward {
+        type filter hook forward priority filter; policy drop;
+    }
+}
+NFTEOF
+
+
+_log "INFO" "armed — wan=${WAN_IFACE} server=${SERVER_IP} tunnel=${TUNNEL_IF} internal=${INTERNAL_SUBNET:-none}"
+exit 0
+
+
+
+

core/assets/sudo_scripts/setup.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+ORIGINAL_FOLDER="${ORIGINAL_FOLDER:-$(pwd)}"
+TARGET_FOLDER="${TARGET_FOLDER:-/opt/hydra-veil}"
+LINUX_USER="${LINUX_USER:-$SUDO_USER}"
+
+add_sudoers_rule() {
+    sudo tee /etc/sudoers.d/zzzzzzzzzzzzzzzzzzz > /dev/null <<EOF
+${LINUX_USER} ALL=(root) NOPASSWD: /opt/hydra-veil/firewall
+${LINUX_USER} ALL=(root) NOPASSWD: /opt/hydra-veil/dns
+EOF
+}
+
+# Check if running as root
+if [[ $EUID -ne 0 ]]; then
+   echo "Error: This script must be run as root (use sudo)" >&2
+   exit 1
+fi
+
+# Validate environment variables
+if [[ -z "$ORIGINAL_FOLDER" ]]; then
+   echo "Error: ORIGINAL_FOLDER environment variable not set" >&2
+   exit 1
+fi
+
+if [[ -z "$TARGET_FOLDER" ]]; then
+   echo "Error: TARGET_FOLDER environment variable not set" >&2
+   exit 1
+fi
+
+# Validate that source folder exists
+if [[ ! -d "$ORIGINAL_FOLDER" ]]; then
+   echo "Error: ORIGINAL_FOLDER does not exist: $ORIGINAL_FOLDER" >&2
+   exit 1
+fi
+
+# Create target folder if it doesn't exist
+if [[ ! -d "$TARGET_FOLDER" ]]; then
+   echo "Creating target folder: $TARGET_FOLDER"
+   mkdir -p "$TARGET_FOLDER"
+fi
+
+# Get the setup script's own filename, because we'll exclude it from being copied,
+SCRIPT_NAME="$(basename "$0")"
+
+echo "Starting file copy from $ORIGINAL_FOLDER to $TARGET_FOLDER"
+echo "Script name to exclude: $SCRIPT_NAME"
+echo ""
+
+# Counter for tracking progress
+COPIED=0
+SKIPPED=0
+
+# Loop through all files in source folder
+while IFS= read -r -d '' file; do
+   filename="$(basename "$file")"
+   
+   # Skip if this is the script itself
+   if [[ "$filename" == "$SCRIPT_NAME" ]]; then
+      echo "⊘ Skipping: $filename (this script)"
+      ((SKIPPED++))
+      continue
+   fi
+   
+   # Copy the file
+   if cp "$file" "$TARGET_FOLDER/"; then
+      echo "✓ Copied: $filename"
+      ((COPIED++))
+      
+      # Set permissions to 755
+      if chmod 755 "$TARGET_FOLDER/$filename"; then
+         echo "  └─ chmod 755 applied"
+      else
+         echo "  └─ WARNING: chmod 755 failed for $filename" >&2
+      fi
+   else
+      echo "✗ ERROR: Failed to copy $filename" >&2
+      ((SKIPPED++))
+   fi
+   
+done < <(find "$ORIGINAL_FOLDER" -maxdepth 1 -type f -print0)
+
+echo ""
+echo "=========================================="
+echo "Copy complete!"
+echo "Files copied: $COPIED"
+echo "Files skipped: $SKIPPED"
+echo "Target folder: $TARGET_FOLDER"
+echo "=========================================="
+
+# Try to apply the rules, if not emit 1
+if add_sudoers_rule; then
+    echo "Operation successful"
+else
+    echo "Operation failed - sudoers rule not applied"
+    exit 1
+fi
+
+exit 0
+
+
+

core/assets/yaml_mappings/locations.yaml

@@ -0,0 +1,54 @@
+fields:
+  - name: country_code
+    path: ['country', 'code']
+    required: true
+
+  - name: code # this is city code
+    path: ['code']
+    required: true
+
+  - name: id
+    path: ['id']
+
+  - name: country_name
+    path: ['country', 'name']
+
+  - name: name # this is CITY name
+    path: ['name']
+
+  - name: time_zone
+    path: ['time_zone', 'code']
+
+  - name: operator_id
+    path: ['operator_id']
+    required: true
+
+  - name: provider_name
+    path: ['provider', 'name']
+
+  - name: is_proxy_capable
+    path: ['is_proxy_capable']
+
+  - name: is_wireguard_capable
+    path: ['is_wireguard_capable']
+    required: true
+
+  - name: is_hysteria2_capable
+    path: ['is_hysteria2_capable']
+
+  - name: is_vless_capable
+    path: ['is_vless_capable']
+
+
+# original version:
+  # locations.append((
+  # location['country']['code'],
+  # location['code'],
+  # location['id'],
+  # location['country']['name'],
+  # location['name'],
+  # location['time_zone']['code'],
+  # location['operator_id'],
+  # location['provider']['name'],
+  # location['is_proxy_capable'],
+  # location['is_wireguard_capable']))

core/assets/yaml_mappings/mapping_one.yaml

@@ -0,0 +1,8 @@
+fields:
+  - name: group_a_code
+    path: ['group_a', 'code']
+  - name: code
+    path: ['code']
+  - name: id
+    path: ['id']
+    required: true

core/assets/yaml_mappings/operators.yaml

@@ -0,0 +1,19 @@
+fields:
+  - name: id
+    path: ['id']
+    required: true
+
+  - name: name
+    path: ['name']
+
+  - name: public_key # this is ed25519
+    path: ['public_key']
+
+  - name: nostr_public_key
+    path: ['nostr_public_key']
+
+  - name: nostr_profile_reference
+    path: ['nostr_profile_reference']
+
+  - name: nostr_attestation_event_reference
+    path: ['nostr_attestation', 'event_reference']

core/controllers/ClientController.py

@@ -2,6 +2,8 @@
 from core.models.manage.session_management import init_session, close_session
 from core.services.sync.sync_service import coordinate_cache_sync, save_metadata
 from core.services.sync.orm_methods.sync_one_orm import sync_one_orm_model
+
+
 from core.errors.logger import logger
 
 # ORM models that can be sync'ed:
@@ -16,8 +18,6 @@ from core.controllers.ApplicationController import ApplicationController
 from core.controllers.ApplicationVersionController import ApplicationVersionController
 from core.controllers.ClientVersionController import ClientVersionController
 from core.controllers.ConfigurationController import ConfigurationController
-# from core.controllers.LocationController import LocationController
-# from core.controllers.OperatorController import OperatorController
 from core.controllers.SubscriptionPlanController import SubscriptionPlanController
 from core.observers.ClientObserver import ClientObserver
 from core.observers.ConnectionObserver import ConnectionObserver
@@ -27,7 +27,13 @@ import pathlib
 import re
 import shutil
 import subprocess
+import time
 
+def evaluate_errors(final_result: dict, client_observer: Optional[ClientObserver] = None) -> str:
+    if not final_result["success"] and client_observer:
+        category = final_result["category"]
+        client_observer.notify('synchronizing', f'{category} Error! Check error logs')
+        time.sleep(2) # so they can see it.
 
 class ClientController:
 
@@ -55,13 +61,10 @@ class ClientController:
 
     @staticmethod
     def sync(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None):
-
         if client_observer is not None:
             client_observer.notify('synchronizing', "Fetching list of new data ..")
 
-        result = coordinate_cache_sync(ClientObserver, ConnectionObserver)
+        result = coordinate_cache_sync(client_observer, connection_observer)
-
-        # logger.info(f"We got a Result from the API of {result}")
 
         # Outright Error:
         if not result["success"]:
@@ -96,6 +99,8 @@ class ClientController:
                 client_observer.notify('synchronizing', 'Fetching Locations List..')
 
             final_result = sync_one_orm_model(Location, "locations")
+            evaluate_errors(final_result)
+
 
         if "operators" in changed_tables:
             logger.info("Sync of Operators")
@@ -103,6 +108,7 @@ class ClientController:
                 client_observer.notify('synchronizing', 'Fetching Operators List..')
 
             final_result_two = sync_one_orm_model(Operator, "operators")
+            evaluate_errors(final_result_two)
 
         # =================== MANUAL-SQL BASED MODELS ==================
         try:

core/controllers/ConfigurationController.py

@@ -91,3 +91,15 @@ class ConfigurationController:
     @staticmethod
     def update_or_create(configuration):
         configuration.save()
+
+    @staticmethod
+    def change_firewall(new_value):
+        configuration = ConfigurationController.get_or_new()
+        configuration.firewall = new_value
+        configuration.save()
+
+    @staticmethod
+    def change_dns(new_value):
+        configuration = ConfigurationController.get_or_new()
+        configuration.dns = new_value
+        configuration.save()

core/controllers/ConnectionController.py

@@ -1,13 +1,19 @@
 from core.errors.exceptions import *
 from core.errors.logger import logger
 
+from core.services.networking.general_connection_tools.testing_evaluating import await_connection, system_uses_wireguard_interface, await_network_interface
+from core.services.networking.systemwide.systemwide_wireguard import establish_system_connection, terminate_system_connection
+from core.services.keys_and_verifications.endpoint_verification import verify_wireguard_endpoint
+
+
 from collections.abc import Callable
 from core.Constants import Constants
 from core.Errors import InvalidSubscriptionError, MissingSubscriptionError, ConnectionUnprotectedError, ConnectionTerminationError, CommandNotFoundError
 from core.controllers.ConfigurationController import ConfigurationController
-from core.controllers.ProfileController import ProfileController
+# from core.controllers.ProfileController import ProfileController
 from core.controllers.SessionStateController import SessionStateController
 from core.controllers.SystemStateController import SystemStateController
+from core.models.BaseProfile import ProfileType
 from core.models.session.SessionProfile import SessionProfile
 from core.models.system.SystemProfile import SystemProfile
 from core.models.system.SystemState import SystemState
@@ -20,12 +26,13 @@ from subprocess import CalledProcessError
 from typing import Union, Optional, Any
 import os
 import random
-import re
 import shutil
 import subprocess
-import sys
 import tempfile
 import time
+from enum import Enum
+# import sys
+# import re
 
 
 class ConnectionController:
@@ -51,85 +58,6 @@ class ConnectionController:
         else:
             return None
 
-    @staticmethod
-    def establish_connection(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = (), connection_observer: Optional[ConnectionObserver] = None):
-
-        connection = profile.connection
-
-        # needs_proxy_configuration comes from the child connection models
-        # for the system it returns false blindly. for the session is checks if the connection type is masked.
-        if connection.needs_proxy_configuration() and not profile.has_proxy_configuration():
-
-            if profile.has_subscription():
-
-                if not profile.subscription.has_been_activated():
-                    ProfileController.activate_subscription(profile, connection_observer=connection_observer)
-
-                proxy_configuration = ConnectionController.with_preferred_connection(profile.subscription.billing_code, task=WebServiceApiService.get_proxy_configuration, connection_observer=connection_observer)
-
-                if proxy_configuration is None:
-                    raise InvalidSubscriptionError()
-
-                profile.attach_proxy_configuration(proxy_configuration)
-
-            else:
-                raise MissingSubscriptionError()
-
-        # The needs_wireguard_configuration comes from the connection model, and can be transitioned to get it directly from the object's data
-        # The has_wireguard_configuration comes from each polymorph object doing an os check on if the wg config exists
-        if connection.needs_wireguard_configuration() and not profile.has_wireguard_configuration():
-
-            if profile.has_subscription():
-
-                if not profile.subscription.has_been_activated():
-                    ProfileController.activate_subscription(profile, connection_observer=connection_observer)
-
-                ProfileController.register_wireguard_session(profile, connection_observer=connection_observer)
-
-            else:
-
-                if profile.is_system_profile():
-
-                    if ConnectionController.system_uses_wireguard_interface() and SystemStateController.exists():
-
-                        try:
-                            ConnectionController.terminate_system_connection()
-                        except ConnectionTerminationError:
-                            pass
-
-                raise MissingSubscriptionError()
-
-        if profile.is_session_profile():
-
-            try:
-                return ConnectionController.establish_session_connection(profile, ignore=ignore, connection_observer=connection_observer)
-
-            except ConnectionError:
-
-                if ConnectionController.__should_renegotiate(profile):
-
-                    ProfileController.register_wireguard_session(profile, connection_observer=connection_observer)
-                    return ConnectionController.establish_session_connection(profile, ignore=ignore, connection_observer=connection_observer)
-
-                else:
-                    raise ConnectionError('The connection could not be established.')
-
-        if profile.is_system_profile():
-
-            try:
-                return ConnectionController.establish_system_connection(profile, ignore=ignore, connection_observer=connection_observer)
-
-            except ConnectionError:
-
-                if ConnectionController.__should_renegotiate(profile):
-
-                    ProfileController.register_wireguard_session(profile, connection_observer=connection_observer)
-                    return ConnectionController.establish_system_connection(profile, ignore=ignore, connection_observer=connection_observer)
-
-                else:
-                    raise ConnectionError('The connection could not be established.')
-
-        return None
 
     @staticmethod
     def establish_session_connection(profile: SessionProfile, ignore: tuple[type[Exception]] = (), connection_observer: Optional[ConnectionObserver] = None):
@@ -143,11 +71,12 @@ class ConnectionController:
         # this is a check from SessionConnection of if there's a systemwide with mask
         if profile.connection.is_unprotected():
 
-            if not ConnectionController.system_uses_wireguard_interface():
+            if not system_uses_wireguard_interface():
 
                 if not ConnectionUnprotectedError in ignore:
                     raise ConnectionUnprotectedError('Connection unprotected while the system is not using a WireGuard interface.')
                 else:
+                    from core.controllers.ProfileController import ProfileController
                     ProfileController.disable(profile)
 
         if profile.connection.code == 'tor':
@@ -160,7 +89,7 @@ class ConnectionController:
         elif profile.connection.code == 'wireguard':
 
             if ConfigurationController.get_endpoint_verification_enabled():
-                ProfileController.verify_wireguard_endpoint(profile, ignore=ignore)
+                verify_wireguard_endpoint(profile, ignore=ignore)
 
             port_number = ConnectionService.get_random_available_port_number()
             ConnectionController.establish_wireguard_session_connection(profile, session_directory, port_number)
@@ -175,55 +104,16 @@ class ConnectionController:
             session_state.network_port_numbers.proxy.append(proxy_port_number)
 
         if not profile.connection.is_unprotected():
-            ConnectionController.await_connection(proxy_port_number or port_number, connection_observer=connection_observer)
+            await_connection(proxy_port_number or port_number, connection_observer=connection_observer)
 
         SessionStateController.update_or_create(session_state)
 
         return proxy_port_number or port_number
 
-    @staticmethod
-    def establish_system_connection(profile: SystemProfile, ignore: tuple[type[Exception]] = (), connection_observer: Optional[ConnectionObserver] = None):
-
-        if ConfigurationController.get_endpoint_verification_enabled():
-            ProfileController.verify_wireguard_endpoint(profile, ignore=ignore)
-
-        try:
-            ConnectionController.__establish_system_connection(profile, connection_observer)
-
-        except ConnectionError:
-
-            try:
-                ConnectionController.terminate_system_connection()
-            except ConnectionTerminationError:
-                pass
-
-            raise ConnectionError('The connection could not be established.')
-
-        except CalledProcessError:
-
-            try:
-                ConnectionController.terminate_system_connection()
-            except ConnectionTerminationError:
-                pass
-
-            try:
-                ConnectionController.__establish_system_connection(profile, connection_observer)
-
-            except (ConnectionError, CalledProcessError):
-
-                try:
-                    ConnectionController.terminate_system_connection()
-                except ConnectionTerminationError:
-                    pass
-
-                raise ConnectionError('The connection could not be established.')
-
-        ConnectionController.terminate_tor_connection()
-        time.sleep(1.0)
 
     @staticmethod
     def establish_tor_connection(connection_observer: Optional[ConnectionObserver] = None):
-
+        try:
             tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
             tor_module.start_service(connection_observer)
 
@@ -231,12 +121,15 @@ class ConnectionController:
 
                 for port_number in session_state.network_port_numbers.tor:
                     tor_module.create_session(port_number)
+        except:
+            if connection_observer is not None:
+                connection_observer.notify('custom_message', "Tor Can't Initialize")
 
-    @staticmethod
+    # @staticmethod
-    def terminate_tor_connection():
+    # def terminate_tor_connection():
 
-        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+    #     tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
-        tor_module.stop_service()
+    #     tor_module.stop_service()
 
     @staticmethod
     def establish_tor_session_connection(port_number: int, connection_observer: Optional[ConnectionObserver] = None):
@@ -245,6 +138,8 @@ class ConnectionController:
             tor_module.create_session(port_number, connection_observer)
         except TorServiceInitializationError as e:
             logger.error(f"TorServiceInitializationError. Tor Can't Start: {e}")
+            if connection_observer is not None:
+                connection_observer.notify('custom_message', "Tor Can't Initialize")
         except Exception as e:
             logger.error(f"Tor Can't Start: {e}")
 
@@ -308,26 +203,9 @@ class ConnectionController:
 
         return subprocess.Popen(('proxychains4', '-f', proxychains_configuration_file_path, 'microsocks', '-p', str(proxy_port_number)), stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
 
-    @staticmethod
-    def terminate_system_connection():
 
-        if shutil.which('nmcli') is None:
-            raise CommandNotFoundError('nmcli')
-
-        if SystemStateController.exists():
-
-            process = subprocess.Popen(('nmcli', 'connection', 'delete', 'wg'), stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
-            completed_successfully = not bool(os.waitpid(process.pid, 0)[1] >> 8)
-
-            if completed_successfully or not ConnectionController.system_uses_wireguard_interface():
-
-                subprocess.run(('nmcli', 'connection', 'delete', 'hv-ipv6-sink'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-                ConnectionController.terminate_tor_connection()
-                SystemState.dissolve()
-
-            else:
-                raise ConnectionTerminationError('The connection could not be terminated.')
 
+    # Stays here for Tor based only. Will move with Tor later.
     @staticmethod
     def get_proxies(port_number: int):
 
@@ -336,107 +214,6 @@ class ConnectionController:
             https=f'socks5h://127.0.0.1:{port_number}'
         )
 
-    @staticmethod
-    def await_connection(port_number: Optional[int] = None, connection_observer: Optional[ConnectionObserver] = None):
-
-        if port_number is None:
-            ConnectionController.await_network_interface()
-
-        for retry_count in range(Constants.MAX_CONNECTION_ATTEMPTS):
-
-            if connection_observer is not None:
-
-                connection_observer.notify('connecting', dict(
-                    retry_interval=Constants.CONNECTION_RETRY_INTERVAL,
-                    maximum_number_of_attempts=Constants.MAX_CONNECTION_ATTEMPTS,
-                    attempt_count=retry_count + 1
-                ))
-
-            try:
-
-                ConnectionController.__test_connection(port_number)
-                return
-
-            except ConnectionError:
-
-                time.sleep(Constants.CONNECTION_RETRY_INTERVAL)
-                retry_count += 1
-
-        raise ConnectionError('The connection could not be established.')
-
-    @staticmethod
-    def await_network_interface():
-
-        network_interface_is_activated = False
-
-        retry_interval = .5
-        maximum_number_of_attempts = 10
-        attempt = 0
-
-        while not network_interface_is_activated and attempt < maximum_number_of_attempts:
-
-            time.sleep(retry_interval)
-
-            network_interface_is_activated = ConnectionController.system_uses_wireguard_interface()
-            attempt += 1
-
-        if not network_interface_is_activated:
-            raise ConnectionError('The network interface could not be activated.')
-
-    @staticmethod
-    def system_uses_wireguard_interface():
-
-        if shutil.which('ip') is None:
-            raise CommandNotFoundError('ip')
-
-        process = subprocess.Popen(('ip', 'route', 'get', '192.0.2.1'), stdout=subprocess.PIPE)
-        process_output = str(process.stdout.read())
-
-        return bool(re.search('dev wg', str(process_output)))
-
-    @staticmethod
-    def __establish_system_connection(profile: SystemProfile, connection_observer: Optional[ConnectionObserver] = None):
-
-        if shutil.which('dbus-send') is None:
-            raise CommandNotFoundError('dbus-send')
-
-        if shutil.which('nmcli') is None:
-            raise CommandNotFoundError('nmcli')
-
-        ConnectionController.terminate_system_connection()
-
-        try:
-            process_output = subprocess.check_output(('nmcli', 'connection', 'import', '--temporary', 'type', 'wireguard', 'file', profile.get_wireguard_configuration_path()), text=True)
-        except CalledProcessError:
-            raise ConnectionError('The connection could not be established.')
-
-        try:
-
-            connection_id = (m := re.search(r'(?<=\()([a-f0-9-]+?)(?=\))', process_output)) and m.group(1)
-            ipv6_method = subprocess.check_output(('nmcli', '-g', 'ipv6.method', 'connection', 'show', connection_id), text=True).strip()
-
-        except CalledProcessError:
-            raise ConnectionError('The connection could not be established.')
-
-        if ipv6_method in ('disabled', 'ignore'):
-
-            try:
-                subprocess.run(('dbus-send', '--system', '--print-reply', '--dest=org.freedesktop.NetworkManager', '/org/freedesktop/NetworkManager', 'org.freedesktop.DBus.Properties.Set', 'string:org.freedesktop.NetworkManager', 'string:ConnectivityCheckEnabled', 'variant:boolean:false'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
-            except CalledProcessError:
-                raise ConnectionError('The connection could not be established.')
-
-            try:
-                subprocess.run(('nmcli', 'connection', 'add', 'type', 'dummy', 'save', 'no', 'con-name', 'hv-ipv6-sink', 'ifname', 'hvipv6sink0', 'ipv6.method', 'manual', 'ipv6.addresses', 'fd7a:fd4b:54e3:077c::/64', 'ipv6.gateway', 'fd7a:fd4b:54e3:077c::1', 'ipv6.dns', '::1', 'ipv6.route-metric', '72'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
-            except CalledProcessError:
-                raise ConnectionError('The connection could not be established.')
-
-        SystemStateController.create(profile.id)
-
-        try:
-            ConnectionController.await_connection(connection_observer=connection_observer)
-
-        except ConnectionError:
-            raise ConnectionError('The connection could not be established.')
 
     @staticmethod
     def __with_tor_connection(*args, task: Callable[..., Any], connection_observer: Optional[ConnectionObserver] = None, **kwargs):
@@ -444,62 +221,9 @@ class ConnectionController:
         port_number = ConnectionService.get_random_available_port_number()
         ConnectionController.establish_tor_session_connection(port_number, connection_observer=connection_observer)
 
-        ConnectionController.await_connection(port_number, connection_observer=connection_observer)
+        await_connection(port_number, connection_observer=connection_observer)
         task_output = task(*args, proxies=ConnectionController.get_proxies(port_number), **kwargs)
 
         ConnectionController.terminate_tor_session_connection(port_number)
 
         return task_output
-
-    @staticmethod
-    def __test_connection(port_number: Optional[int] = None, timeout: float = 4.0):
-
-        request_urls = [Constants.PING_URL]
-        proxies = None
-
-        if os.environ.get('PING_URL') is None:
-
-            request_urls.extend([
-                'https://hc1.simplifiedprivacy.net',
-                'https://hc2.simplifiedprivacy.org',
-                'https://hc3.hydraveil.net'
-            ])
-
-            random.shuffle(request_urls)
-
-        if port_number is not None:
-            proxies = ConnectionController.get_proxies(port_number)
-
-        for request_url in request_urls:
-
-            command = [
-                sys.executable, '-u', '-c', 'import requests, sys\n'
-                'try:\n'
-                f'    response = requests.get(\'{request_url}\', proxies={proxies}, timeout={timeout})\n'
-                '    response.raise_for_status(); print(response.text)\n'
-                'except requests.exceptions.RequestException:\n'
-                '    sys.exit(1)'
-            ]
-
-            try:
-
-                _response = subprocess.check_output(command, text=True, timeout=timeout)
-                return None
-
-            except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
-                pass
-
-        raise ConnectionError('The connection could not be established.')
-
-    @staticmethod
-    def __should_renegotiate(profile: Union[SessionProfile, SystemProfile]):
-
-        if not profile.has_subscription():
-            raise MissingSubscriptionError()
-
-        if profile.connection.needs_wireguard_configuration() and profile.has_wireguard_configuration():
-
-            if profile.subscription.has_been_activated():
-                return True
-
-        return False

core/controllers/LocationController.py

@@ -1,23 +1,36 @@
 from core.models.orm_models.Location import Location
 from core.models.orm_models.Operator import Operator
+from core.errors.logger import logger
 
 from core.models.manage.session_management import get_session
 from typing import Optional
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.orm_calls.location_calls import execute_location_sql
 
 class LocationController:
 
     @staticmethod
     def get(country_code: str, city_code: str):
-        with get_session() as session:
+        location_object = execute_location_sql(country_code, city_code)
-            location_object = session.execute(
+        if location_object.valid:
-                select(Location)
+            return location_object.data
-                .where((Location.country_code == country_code) & (Location.code == city_code))
+        else:
-                .options(joinedload(Location.operator))
+            critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}"
-            ).scalar_one_or_none()
+            logger.error(critical_error)
+            print(critical_error)
+            return None
 
-        return location_object
+
+        # with get_session() as session:
+        #     location_object = session.execute(
+        #         select(Location)
+        #         .where((Location.country_code == country_code) & (Location.code == city_code))
+        #         .options(joinedload(Location.operator))
+        #     ).scalar_one_or_none()
+
+        # return location_object
 
     # legacy:
     # Location.find(country_code, code)

core/controllers/ProfileController.py

@@ -1,3 +1,8 @@
+from core.services.networking.systemwide.systemwide_wireguard import terminate_system_connection
+from core.services.networking.general_connection_tools.testing_evaluating import system_uses_wireguard_interface
+from core.services.networking.general_connection_tools.connection_enable import establish_connection
+# from core.services.networking.systemwide.systemwide_utils import get_firewall_setting, get_dns_setting
+
 from core.Errors import InvalidSubscriptionError, MissingSubscriptionError, ConnectionTerminationError, ProfileActivationError, ProfileDeactivationError, MissingLocationError, ConnectionUnprotectedError, EndpointVerificationError, ProfileStateConflictError
 from core.controllers.ApplicationController import ApplicationController
 from core.controllers.ApplicationVersionController import ApplicationVersionController
@@ -35,29 +40,33 @@ class ProfileController:
         if profile_observer is not None:
             profile_observer.notify('created', profile)
 
-    @staticmethod
-    def update(profile: Union[SessionProfile, SystemProfile], profile_observer: ProfileObserver = None):
-
-        profile.save()
-
-        if profile_observer is not None:
-            profile_observer.notify('updated', profile)
 
     @staticmethod
-    def enable(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = (), pristine: bool = False, asynchronous: bool = False, profile_observer: ProfileObserver = None, application_version_observer: ApplicationVersionObserver = None, connection_observer: ConnectionObserver = None):
+    def enable(
+        profile: Union[SessionProfile, SystemProfile],
+        ignore: tuple[type[Exception]] = (),
+        pristine: bool = False,
+        asynchronous: bool = False,
+        profile_observer: ProfileObserver = None,
+        application_version_observer: ApplicationVersionObserver = None,
+        connection_observer: ConnectionObserver = None):
 
         from core.controllers.ConnectionController import ConnectionController
 
+        # =========== ALREADY ENABLED ============
         if ProfileController.is_enabled(profile):
-
             if not ProfileStateConflictError in ignore:
                 raise ProfileStateConflictError('The profile is already enabled or its session was not properly terminated.')
             else:
                 ProfileController.disable(profile)
 
+        # =========== PRISTINE ============
         if pristine:
             profile.delete_data()
 
+        # ============================================================================
+        # SESSION
+        # ============================================================================
         if profile.is_session_profile():
 
             application_version = profile.application_version
@@ -66,7 +75,7 @@ class ProfileController:
                 ApplicationVersionController.install(application_version, application_version_observer=application_version_observer, connection_observer=connection_observer)
 
             try:
-                port_number = ConnectionController.establish_connection(profile, ignore=ignore, connection_observer=connection_observer)
+                port_number = establish_connection(profile, ignore=ignore, connection_observer=connection_observer)
             except ConnectionError:
                 raise ProfileActivationError('The profile could not be enabled.')
 
@@ -75,10 +84,12 @@ class ProfileController:
 
             ApplicationController.launch(application_version, profile, port_number, asynchronous=asynchronous, profile_observer=profile_observer)
 
+        # ============================================================================
+        # SYSTEMWIDE
+        # ============================================================================
         if profile.is_system_profile():
-
             try:
-                ConnectionController.establish_connection(profile, ignore=ignore, connection_observer=connection_observer)
+                establish_connection(profile, ignore=ignore, connection_observer=connection_observer)
             except ConnectionError:
                 raise ProfileActivationError('The profile could not be enabled.')
 
@@ -122,7 +133,7 @@ class ProfileController:
                     raise ProfileDeactivationError('The profile could not be disabled.')
 
                 try:
-                    ConnectionController.terminate_system_connection()
+                    terminate_system_connection()
                 except ConnectionTerminationError:
                     raise ProfileDeactivationError('The profile could not be disabled.')
 
@@ -184,7 +195,7 @@ class ProfileController:
             system_state = SystemStateController.get()
 
             if system_state is not None and system_state.profile_id is profile.id:
-                return ConnectionController.system_uses_wireguard_interface()
+                return system_uses_wireguard_interface()
 
         return False
 
@@ -218,28 +229,28 @@ class ProfileController:
     def has_proxy_configuration(profile: Union[SessionProfile, SystemProfile]):
         profile.has_proxy_configuration()
 
-    @staticmethod
+    # @staticmethod
-    def register_wireguard_session(profile: Union[SessionProfile, SystemProfile], connection_observer: Optional[ConnectionObserver] = None):
+    # def register_wireguard_session(profile: Union[SessionProfile, SystemProfile], connection_observer: Optional[ConnectionObserver] = None):
 
-        from core.controllers.ConnectionController import ConnectionController
+    #     from core.controllers.ConnectionController import ConnectionController
 
-        if not profile.has_subscription():
+    #     if not profile.has_subscription():
-            raise MissingSubscriptionError()
+    #         raise MissingSubscriptionError()
 
-        if not profile.has_location():
+    #     if not profile.has_location():
-            raise MissingLocationError()
+    #         raise MissingLocationError()
 
-        wireguard_keys = ProfileController.__generate_wireguard_keys()
+    #     wireguard_keys = ProfileController.__generate_wireguard_keys()
 
-        wireguard_configuration = ConnectionController.with_preferred_connection(profile.location.country_code, profile.location.code, profile.subscription.billing_code, wireguard_keys.get('public'), task=WebServiceApiService.post_wireguard_session, connection_observer=connection_observer)
+    #     wireguard_configuration = ConnectionController.with_preferred_connection(profile.location.country_code, profile.location.code, profile.subscription.billing_code, wireguard_keys.get('public'), task=WebServiceApiService.post_wireguard_session, connection_observer=connection_observer)
 
-        if wireguard_configuration is None:
+    #     if wireguard_configuration is None:
-            raise InvalidSubscriptionError()
+    #         raise InvalidSubscriptionError()
 
-        expression = re.compile(r'^(PrivateKey =)\s?$', re.MULTILINE)
+    #     expression = re.compile(r'^(PrivateKey =)\s?$', re.MULTILINE)
-        wireguard_configuration = re.sub(expression, r'\1 ' + wireguard_keys.get('private'), wireguard_configuration)
+    #     wireguard_configuration = re.sub(expression, r'\1 ' + wireguard_keys.get('private'), wireguard_configuration)
 
-        profile.attach_wireguard_configuration(wireguard_configuration)
+    #     profile.attach_wireguard_configuration(wireguard_configuration)
 
     @staticmethod
     def get_wireguard_configuration_path(profile: Union[SessionProfile, SystemProfile]):
@@ -249,65 +260,65 @@ class ProfileController:
     def has_wireguard_configuration(profile: Union[SessionProfile, SystemProfile]):
         return profile.has_wireguard_configuration()
 
-    @staticmethod
+    # @staticmethod
-    def verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = ()):
+    # def verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = ()):
 
-        try:
+    #     try:
-            ProfileController.__verify_wireguard_endpoint(profile)
+    #         ProfileController.__verify_wireguard_endpoint(profile)
 
-        except EndpointVerificationError as error:
+    #     except EndpointVerificationError as error:
 
-            if not EndpointVerificationError in ignore:
+    #         if not EndpointVerificationError in ignore:
 
-                profile.address_security_incident()
+    #             profile.address_security_incident()
-                raise error
+    #             raise error
 
-    @staticmethod
+    # @staticmethod
-    def __verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile]):
+    # def __verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile]):
 
-        from cryptography.hazmat.primitives.asymmetric import ed25519
+    #     from cryptography.hazmat.primitives.asymmetric import ed25519
-        import base64
+    #     import base64
 
-        signature = profile.get_wireguard_configuration_metadata('Signature')
+    #     signature = profile.get_wireguard_configuration_metadata('Signature')
-        wireguard_public_keys = profile.get_wireguard_public_keys()
+    #     wireguard_public_keys = profile.get_wireguard_public_keys()
-        operator = profile.location.operator
+    #     operator = profile.location.operator
 
-        if signature is None:
+    #     if signature is None:
-            raise EndpointVerificationError('The WireGuard endpoint\'s signature could not be determined.')
+    #         raise EndpointVerificationError('The WireGuard endpoint\'s signature could not be determined.')
 
-        if not wireguard_public_keys:
+    #     if not wireguard_public_keys:
-            raise EndpointVerificationError('The WireGuard endpoint\'s public key could not be determined.')
+    #         raise EndpointVerificationError('The WireGuard endpoint\'s public key could not be determined.')
 
-        if operator is None:
+    #     if operator is None:
-            raise EndpointVerificationError('The WireGuard endpoint\'s operator could not be determined.')
+    #         raise EndpointVerificationError('The WireGuard endpoint\'s operator could not be determined.')
 
-        try:
+    #     try:
 
-            operator_public_key = ed25519.Ed25519PublicKey.from_public_bytes(bytes.fromhex(operator.public_key))
+    #         operator_public_key = ed25519.Ed25519PublicKey.from_public_bytes(bytes.fromhex(operator.public_key))
 
-            for wireguard_public_key in wireguard_public_keys:
+    #         for wireguard_public_key in wireguard_public_keys:
-                operator_public_key.verify(base64.b64decode(signature), wireguard_public_key.encode('utf-8'))
+    #             operator_public_key.verify(base64.b64decode(signature), wireguard_public_key.encode('utf-8'))
 
-        except Exception:
+    #     except Exception:
-            raise EndpointVerificationError('The WireGuard endpoint could not be verified.')
+    #         raise EndpointVerificationError('The WireGuard endpoint could not be verified.')
 
-    @staticmethod
+    # @staticmethod
-    def __generate_wireguard_keys():
+    # def __generate_wireguard_keys():
 
-        from cryptography.hazmat.primitives import serialization
+    #     from cryptography.hazmat.primitives import serialization
-        from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey
+    #     from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey
 
-        raw_private_key = X25519PrivateKey.generate()
+    #     raw_private_key = X25519PrivateKey.generate()
 
-        public_key = raw_private_key.public_key().public_bytes(
+    #     public_key = raw_private_key.public_key().public_bytes(
-            encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw
+    #         encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw
-        )
+    #     )
 
-        private_key = raw_private_key.private_bytes(
+    #     private_key = raw_private_key.private_bytes(
-            encoding=serialization.Encoding.Raw, format=serialization.PrivateFormat.Raw, encryption_algorithm=serialization.NoEncryption()
+    #         encoding=serialization.Encoding.Raw, format=serialization.PrivateFormat.Raw, encryption_algorithm=serialization.NoEncryption()
-        )
+    #     )
 
-        return dict(
+    #     return dict(
-            private=base64.b64encode(private_key).decode(),
+    #         private=base64.b64encode(private_key).decode(),
-            public=base64.b64encode(public_key).decode()
+    #         public=base64.b64encode(public_key).decode()
-        )
+    #     )

core/controllers/SystemStateController.py

@@ -12,8 +12,10 @@ class SystemStateController:
         return SystemState.exists()
 
     @staticmethod
-    def create(profile_id):
+    def create(profile_id: int, firewalled: bool, dns_set: bool) -> SystemState:
-        return SystemState(profile_id).save()
+        current_state = SystemState(profile_id, firewalled, dns_set)
+        current_state.save()
+        return current_state
 
     @staticmethod
     def update_or_create(system_state):

core/controllers/profile_state/update_profile.py

@@ -0,0 +1,117 @@
+# utils
+from core.errors.logger import logger
+from core.models.Result import Result, ResultError
+
+# JSON Models
+from core.models.BaseProfile import BaseProfile as Profile
+from core.models.Subscription import Subscription
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+
+# ORM Models
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.models.orm_calls.location_calls import get_profile_location_data
+
+SYSTEMWIDE_CHOICES = ['wireguard', 'hysteria2', 'vless']
+SESSION_CHOICES = ['wireguard', 'tor', 'proxy']
+
+"""
+Steps:
+1) Looks up the profile by id number
+2) Filters what values to save, using the ORM for some such as location
+3) Saves it with the abstract class' json serialization methods, which use the ORM methods
+
+Returns:
+    Result Object
+
+Called by:
+    GUI's editor_page
+
+Raises Errors:
+    False
+"""
+
+def update_profile(
+    profile_id: int,
+    key: str,
+    new_value: str) -> Result:
+
+    final_data = None
+    logger.info(f"[UPDATE PROFILE] Recieved profile_id {profile_id}, key {key}, and new_value {new_value}.")
+
+    if not key or not new_value:
+        return Result(valid=False, data=f"Invalid Inputs of {key} and {new_value}", error_type=ResultError.INVALID_INPUT)
+
+    profile = Profile.find_by_id(profile_id)
+    if not profile:
+        error_msg = f"Invalid profile id of {profile_id}"
+        return Result(valid=False, message=error_msg, error_type=ResultError.INVALID_INPUT)
+
+    # logger.info(f"[UPDATE PROFILE] Found a relevant profile {profile_id}.")
+
+    if key == 'dimentions':
+        profile.resolution = new_value
+
+    elif key == 'name':
+        profile.name = new_value
+
+    # ============= CONNECTION TYPE =============
+    elif key == 'connection':
+        if new_value == 'tor':
+            profile.connection.code = new_value
+            profile.connection.masked = True
+
+        elif new_value == 'just proxy':
+            profile.connection.code = 'system'
+            profile.connection.masked = True
+        else:
+            error_msg = 'System wide profiles not supported atm'
+            return Result(valid=False, message=error_msg, error_type=ResultError.NOT_SUPPORTED)
+
+    # ============= BROWSER =============
+    elif key == 'browser':
+        browser_type, browser_version = new_value.split(':', 1)
+        profile.application_version.application_code = browser_type
+        profile.application_version.version_number = browser_version
+
+    elif key == 'protocol':
+        # ============= SYSTEMWIDE =============
+        if profile.connection == 'system-wide':
+            if new_value in SYSTEMWIDE_CHOICES:
+                profile.connection.code = new_value
+                final_data = "edit_session"
+            else:
+                error_msg = f"{new_value} is not a systemwide choice"
+                return Result(valid=False, message=error_msg, error_type=ResultError.NOT_SUPPORTED)
+
+        # ============= SESSION =============
+        else:
+            if new_value in SESSION_CHOICES:
+                profile.connection.code = new_value
+                if new_value == 'wireguard':
+                    profile.connection.masked = False
+
+    # ============= LOCATION =============
+    elif key == "location":
+        country_code, city_code = new_value.split("_")
+        location = get_profile_location_data(   # SQLAlchemy
+            country_code=country_code,
+            city_code=city_code
+        )
+        # SQLAlchemy foreign key assignment — fills in id, timezone, operator, etc.
+        profile.location = location
+        
+        # Subscription gets wiped on a location change, and is outside ORM
+        profile.subscription = None
+    else:
+        return Result(valid=False, message="Invalid Value to Edit", error_type=ResultError.INVALID_INPUT)
+
+    logger.info("[UPDATE PROFILE] Passing to Profile model to save..")
+    try:
+        profile.save()
+        logger.info("[UPDATE PROFILE] Save worked, passing to the GUI the Result..")
+        return Result(valid=True, data=final_data)
+    except:
+        error_msg = "Profile save failed"
+        return Result(valid=False, message=error_msg, error_type=ResultError.UNKNOWN)

core/controllers/tickets/TicketPayController.py

@@ -9,9 +9,16 @@ from core.models.invoice.TicketInvoice import TicketInvoice
 from core.services.prepare_tickets.get_pub_key import get_pub_key
 from core.observers.BaseObserver import BaseObserver
 from core.services.payment_phase.save_and_send_intitial_billing import save_and_send_intitial_billing
-from core.services.payment_phase.check_if_paid import _check_if_paid
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
+
 from core.services.prepare_tickets.ticket_tracker import does_ticket_tracker_exist
-from core.services.networking.send_data_to_server import send_data_to_server
+
+# from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+
+
 from core.services.networking.make_url import make_url
 # from core.utils.confirm_its_a_valid_key_choice import confirm_its_a_valid_key_choice
 from core.services.helpers.valid_profile_quantity import valid_profile_quantity
@@ -112,6 +119,14 @@ def initiate_payment(
             payload, connection_observer, invoice_data_object
         )
 
+        if isinstance(result, ApiResponse):
+            logger.error(result.message)
+            if not result.valid:
+                invoice_data_object.add_error_code("connection_error")
+                return invoice_data_object
+            else:
+                logger.error("Critical Error with TicketPayController recieving a valid ApiResponse object.")
+
         if result == False or result == None:
             invoice_data_object.add_error_code("failed_save")
 
@@ -169,8 +184,25 @@ def check_if_paid(
         url = make_url(which_endpoint)
 
         # literally send:
-        reply = send_data_to_server(payload, url, connection_observer)
+        api_reply_object = send_data_to_server(payload, url, connection_observer)
 
-        logger.debug(f"inside ticketpay controller the reply is {reply}")
+        if not api_reply_object.valid:
+            api_reply_object = solve_api_problems(
+                api_reply_object=api_reply_object,
+                get_or_post="post",
+                url=url,
+                payload=payload,
+                connection_observer=connection_observer,
+                client_observer=None
+            )
+            if not api_reply_object.valid:
+                # 2nd try, return the error of why we have no payload:
+                error_msg = f"Connection/API Error: {api_reply_object.message}"
+                logger.error(f"[TICKET PayController] 2nd Post Request inside ticketpay controller had a {error_msg}")
+                return {"valid": False, "message": error_msg}
+
+        # return the payload with GUI/CLI to interpret results:
+        reply_dict = api_reply_object.data
+        logger.debug(f"[TICKET PayController] We have a valid reply from the API inside ticketpay controller of {reply_dict}")
+        return reply_dict
 
-        return reply

core/controllers/tickets/TicketSyncController.py

@@ -7,12 +7,16 @@ if TYPE_CHECKING:
 
 from core.Constants import Constants
 from core.observers.BaseObserver import BaseObserver
-from core.services.networking.get_data_from_server import get_data_from_server
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
 from core.errors.logger import logger
 
 
 def sync_ticket_prices(
-    ticket_observer: TicketObserver, connection_observer: ConnectionObserver
+    ticket_observer: TicketObserver,
+    connection_observer: ConnectionObserver
 ) -> dict:
     notification = f"Connecting to get Ticket Pricing..."
     ticket_observer.notify("connecting", subject=notification)
@@ -28,13 +32,24 @@ def sync_ticket_prices(
 
     url = f"{base_url}/sync"
     try:
-        sync_results = get_data_from_server(url, connection_observer)
+        api_result = get_data_from_api(url, None, connection_observer)
 
-        if sync_results in rejected_list:
+        if not api_result.valid:
+            error_msg = api_result.message
+            logger.error(f"[TICKET SYNC Controller] There's an issue with the sync of endpoint {url} the API Reply: {error_msg}")
+            api_result = solve_api_problems(
+            api_reply_object=api_result,
+            get_or_post="get",
+            url=url,
+            payload=None,
+            connection_observer=connection_observer,
+            client_observer=None
+            )
+            # 2nd try:
+            if not api_result.valid:
                 return {"valid": False, "error_code": "sync_failed"}
 
-        logger.debug(f"Inside the sync controller, sync_results is: {sync_results}")
+        return {"valid": True, "data": api_result.data}
+
     except:
         return {"valid": False, "error_code": "sync_failed"}
-
-    return sync_results

core/controllers/tickets/UseTicketController.py

@@ -8,6 +8,8 @@ if TYPE_CHECKING:
 from core.Constants import Constants
 from core.observers.BaseObserver import BaseObserver
 from core.services.using_tickets.use_ticket_orchestrator import use_ticket_orchestrator
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
 from core.services.prepare_tickets.ticket_tracker import (
     get_data_for_a_single_ticket,
     does_ticket_tracker_exist,
@@ -111,7 +113,7 @@ def get_unused_tickets(ticket_observer: TicketObserver) -> dict:
     does_the_file_exist, path_of_file = does_ticket_tracker_exist()
     if does_the_file_exist == False:
         error_msg = f"The ticket tracker organizer file does not exist.  Check the folder {path_of_file}"
-        ticket_observer.notify("failed_input", subject=error_msg)
+        # ticket_observer.notify("failed_input", subject=error_msg)
         return {"valid": False, "message": error_msg}
 
     # Use the Model:
@@ -168,6 +170,11 @@ def use_ticket(
     ticket_observer.notify("connecting", "Connecting..")
     reply = use_ticket_orchestrator(which_ticket, which_location, connection_observer)
 
+    if isinstance(reply, ApiResponse):
+        error_msg = reply.message
+        reply_as_dict = {"valid": False, "message": f"API Failed: {error_msg}"}
+        return reply_as_dict
+    else:
         return reply
 
 

core/errors/exceptions.py

@@ -1,3 +1,25 @@
+from core.models.Result import Result, ResultError
+
+class SudoScript(Exception):
+    """There are issues with the sudo scripts"""
+    def __init__(self, result: Result):
+        self.result = result
+        if result.message and result.message is not None:
+            error_msg = result.message
+        else:
+            error_msg = result.error_type
+        super().__init__(error_msg)
+
+class MissingPreReqs(Exception):
+    """User can't take an action because they lack the prereqs"""
+    def __init__(self, result: Result):
+        self.result = result
+        if result.message and result.message is not None:
+            error_msg = result.message
+        else:
+            error_msg = result.error_type
+        super().__init__(error_msg)
+
 
 class TorServiceInitializationError(Exception):
     pass

core/models/BaseConnection.py

@@ -1,19 +1,21 @@
 from dataclasses import dataclass
 from dataclasses_json import dataclass_json
 
-
 @dataclass_json
 @dataclass
 class BaseConnection:
     code: str
 
-    # called by: connection controller for basic type checks on wireguard code
+    # Called by: ConnectionController
+    # it uses it for basic type checks on wireguard code
     def needs_wireguard_configuration(self):
         return self.code == 'wireguard'
 
+    # Called By SubscriptionPlan
     def is_session_connection(self):
         return type(self).__name__ == 'SessionConnection'
 
+    # Not called. Dead code
     def is_system_connection(self):
         return type(self).__name__ == 'SystemConnection'
 

core/models/BaseProfile.py

@@ -1,4 +1,6 @@
 from core.models.manage.session_management import get_session
+from core.errors.logger import logger
+
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 
@@ -6,7 +8,13 @@ from abc import ABC, abstractmethod
 from core.Constants import Constants
 from core.Helpers import write_atomically
 
-# from core.models.Location import Location
+from core.models.manage.wrapper import safe_db_operation, WrapperRollback
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+
+# If switching to enums for Connection:
+# from core.models.system.SystemConnection import SystemConnection, SystemConnectionTypes
+# from core.models.session.SessionConnection import SessionConnection, SessionConnectionTypes
+
 from core.controllers.LocationController import LocationController
 from core.models.orm_models.Location import Location
 from core.models.orm_models.Operator import Operator
@@ -23,7 +31,34 @@ import os
 import re
 import shutil
 import tempfile
+from sqlalchemy.orm import Session
+from enum import Enum
 
+@safe_db_operation
+def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
+    location_object = session.execute(
+        select(Location)
+        .where((Location.country_code == country_code) & (Location.code == city_code))
+        .options(joinedload(Location.operator))
+    ).scalar_one_or_none()
+    return location_object
+
+
+def get_profile_location_data(country_code: str, city_code: str) -> Location:
+    # with get_session() as session:
+    api_reply_object = execute_location_sql(country_code, city_code)
+    if api_reply_object.valid:
+        data = api_reply_object.data
+        location_obj = api_reply_object.data
+        return location_obj
+    else:
+        logger.error(f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}")
+        return None
+
+
+class ProfileType(str, Enum):
+    SESSION = "session"
+    SYSTEM = "system"
 
 @dataclass_json
 @dataclass
@@ -33,6 +68,7 @@ class BaseProfile(ABC):
     )
     name: str
     subscription: Optional[Subscription]
+    type: ProfileType
     location: Optional[Location] = field(metadata=config(exclude=Exclude.ALWAYS))  # SQLAlchemy object
 
     # legacy version included it to be serialized, but now it's an SQLAlchemy object.
@@ -64,13 +100,17 @@ class BaseProfile(ABC):
     def is_system_profile(self):
         return type(self).__name__ == 'SystemProfile'
 
-    def save(self: Self):
-        config_dict = self.to_dict()  # Get dict, not JSON string
-        location_dict = self.location.to_dict() # this is from SQLAlchemy, and not JSON-models, that's why it's separate.
 
+    def save(self: Self):
+        # === SERIALIZATION ===
+        config_dict = self.to_dict()
+
+        # === LOCATION ===
+        location_dict = self.location.convert_to_dict() # this is from SQLAlchemy, and not JSON-models, that's why it's separate.
         if self.location:
             config_dict["location"] = location_dict
 
+        # === FILE I/O ===
         config_file_contents = json.dumps(config_dict, indent=4) + '\n'
 
         # legacy version:
@@ -186,42 +226,31 @@ class BaseProfile(ABC):
         profiles_location = profile['location']
 
         if profile['location'] is not None:
+            # =========== GET COUNTRY & LOCATION ===========
             if isinstance(profiles_location, dict):
                 try:
                     country_code = profile['location']['country_code']
                     city_code = profile['location']['code']
                 except:
-                    print("ERROR! CANT FIND country code or city")
+                    logger.error(f"CRITICAL ERROR! Can't find country code or city for profile id {profile['id']} inside BaseProfile")
                     return
             else:
-                # potentially coming from SQLAlchemy:
+                # potentially coming from SQLAlchemy ALREADY:
                 country_code = profile.location.country_code
                 city_code = profile.location.code
 
-            with get_session() as session:
+            # =========== GET DATA USING THAT COUNTRY & LOCATION ===========
-                location_object = session.execute(
+            location_dict = get_profile_location_data(country_code, city_code)
-                    select(Location)
-                    .where((Location.country_code == country_code) & (Location.code == city_code))
-                    .options(joinedload(Location.operator))
-                ).scalar_one_or_none()
 
-
+            if location_dict:
-                if location_object:
+                profile['location'] = location_dict
-                    profile['location'] = location_object
 
             # this needs error handling if there's no location or malconformed config.
 
 
-            # legacy phased out since SQLAlchemy handles the time_zone.
+        # =========== SESSION ===========
-
-            # if location is not None:
-
-            #     if profile['location'].get('time_zone') is not None:
-            #         location.time_zone = profile['location']['time_zone']
-
-            #     profile['location'] = location
-
         if 'application_version' in profile:
+            profile['type'] = ProfileType.SESSION
 
             if profile['application_version'] is not None:
                 application_version = ApplicationVersion.find(profile['application_version']['application_code'] or None, profile['application_version']['version_number'] or None)
@@ -233,7 +262,10 @@ class BaseProfile(ABC):
             # noinspection PyUnresolvedReferences
             profile = SessionProfile.from_dict(profile)
 
+
+        # =========== SYSTEM ===========
         else:
+            profile['type'] = ProfileType.SYSTEM
 
             from core.models.system.SystemProfile import SystemProfile
             # noinspection PyUnresolvedReferences
@@ -241,6 +273,7 @@ class BaseProfile(ABC):
 
         return profile
 
+
     @staticmethod
     def exists(id: int):
         return re.match(r'^\d{1,2}$', str(id)) and os.path.isfile(f'{BaseProfile.__get_config_path(id)}/config.json')
@@ -273,3 +306,10 @@ class BaseProfile(ABC):
     @staticmethod
     def __get_data_path(id: int):
         return f'{Constants.HV_PROFILE_DATA_HOME}/{str(id)}'
+
+
+# legacy phased out since SQLAlchemy handles the time_zone.
+# if location is not None:
+#     if profile['location'].get('time_zone') is not None:
+#         location.time_zone = profile['location']['time_zone']
+#     profile['location'] = location

core/models/Configuration.py

@@ -1,3 +1,5 @@
+from core.errors.logger import logger
+
 from core.Constants import Constants
 from core.Helpers import write_atomically
 from dataclasses import dataclass, field
@@ -47,6 +49,30 @@ class Configuration:
         )
     )
 
+    firewall: Optional[bool] = field(
+        default=False,
+        metadata=config(
+            undefined=dataclasses_json.Undefined.EXCLUDE,
+            exclude=lambda value: value is None
+        )
+    )
+
+    dns: Optional[bool] = field(
+        default=False,
+        metadata=config(
+            undefined=dataclasses_json.Undefined.EXCLUDE,
+            exclude=lambda value: value is None
+        )
+    )
+
+    did_sudo_setup: Optional[bool] = field(
+        default=False,
+        metadata=config(
+            undefined=dataclasses_json.Undefined.EXCLUDE,
+            exclude=lambda value: value is None
+        )
+    )
+
     def save(self: Self):
 
         config_file_contents = f'{self.to_json(indent=4)}\n'
@@ -82,3 +108,31 @@ class Configuration:
     def _from_iso_format(datetime_string: str):
         date_string = datetime_string.replace('Z', '+00:00')
         return datetime.fromisoformat(date_string)
+
+
+def read_config():
+    try:
+        config_file_contents = open(f'{Constants.HV_CONFIG_HOME}/config.json', 'r').read()
+    except FileNotFoundError:
+        return None
+
+    try:
+        configuration = json.loads(config_file_contents)
+    except ValueError:
+        logger.error(f"[CONFIG] Can't load JSON config")
+    
+    return configuration
+
+def get_setting(looking_for):
+    config = read_config()
+    if not config:
+        logger.error(f"[CONFIG] Can't load the entire config")
+        return None
+
+    if looking_for not in config:
+        logger.error(f"[CONFIG] What you want isn't in the config")
+        return None
+ 
+    result = config[looking_for]
+
+    return result

core/models/DatabaseOperation.py

@@ -0,0 +1,51 @@
+
+
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional, Any
+
+class DBErrorType(Enum):
+    """Classified error categories."""
+    SUCCESS = "success"
+    NEED_MIGRATION = "need_migration"
+    MIGRATION_FAILED = "migration_failed"
+    OLD_CLIENT_NEW_API = "old_client_new_api"
+    MALFORMED_SQL = "malformed_sql_file"
+    MISSING_SQL = "missing_sql_file"
+    MISSING_DEPENDENCY = "missing_dependency"
+    INTEGRITY_ERROR = "integrity_error"
+    PERMISSION_ERROR = "permission_error"
+    FILESYSTEM_FULL = "filesystem_full"
+    DATABASE_LOCKED = "database_locked"
+    CORRUPTED_DATABASE = "corrupted_database"
+    PYTHON_MODEL_STRUCTURE = "python_model_structure"
+    UNKNOWN = "unknown"
+
+@dataclass
+class DatabaseOperation:
+    valid: bool
+    error_type: DBErrorType = DBErrorType.SUCCESS
+    data: Optional[Any] = None
+    message: Optional[str] = None
+    tried_migration: bool = False
+    tried_filtered: bool = False
+    
+    def is_recoverable(self) -> bool:
+        """Can the caller attempt a retry or manual fix?"""
+        return self.db_error_type in {
+            DBErrorType.OLD_CLIENT_NEW_API,
+            DBErrorType.NEED_MIGRATION
+        }
+    
+    def user_message(self) -> str:
+        """Human-readable error for the UI."""
+        messages = {
+            DBErrorType.SUCCESS: "Operation completed successfully.",
+            DBErrorType.NEED_MIGRATION: "Database schema needs update. Contact admin.",
+            DBErrorType.OLD_CLIENT_NEW_API: "Client version incompatible. Please upgrade.",
+            DBErrorType.MALFORMED_SQL: "Migration file is corrupted. Contact admin.",
+            DBErrorType.MISSING_SQL: "Migration file missing. Contact admin.",
+            DBErrorType.MISSING_DEPENDENCY: "Database dependency missing. Contact admin.",
+            DBErrorType.UNKNOWN: f"Unexpected error: {self.message}",
+        }
+        return messages.get(self.error_type, "Unknown error")

core/models/Result.py

@@ -0,0 +1,40 @@
+
+
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional, Any
+
+class ResultError(Enum):
+    """Classified error categories."""
+    SUCCESS = "success"
+    NOT_SUPPORTED = "NOT_SUPPORTED"
+    INVALID_INPUT = "invalid_input"
+    MISSING_FILE = "missing_file"
+    MISSING_DEPENDENCY = "missing_dependency"
+    CONNECTION = "connection"
+    DATABASE = "database"
+    PERMISSION = "permission"
+    SUBSCRIPTION = "subscription"
+    NMCLI = "nmcli_issues"
+    FIREWALL = "firewall"
+    CLIENT_DNS = "client_dns"
+    EXTERNAL_DNS = "external_dns"
+    INTERFACE = "interface"
+    TIMEOUT = "timeout"
+    UNKNOWN = "unknown"
+
+@dataclass
+class Result():
+    valid: bool
+    error_type: ResultError = ResultError.SUCCESS
+    data: Optional[Any] = None
+    message: Optional[str] = None
+    goal: Optional[str] = None
+    
+    def user_message(self) -> str:
+        """Human-readable error for the UI."""
+        messages = {
+            DBErrorType.SUCCESS: "Operation completed successfully.",
+            DBErrorType.UNKNOWN: f"Error: {self.message}",
+        }
+        return messages.get(self.error_type, "Unknown error")

core/models/manage/clear_sql_model.py

@@ -0,0 +1,97 @@
+
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.manage.wrapper import safe_db_operation
+from core.errors.logger import logger
+
+from typing import Type
+from sqlalchemy.orm import Session
+
+
+
+@safe_db_operation
+def _drop_sql_model_wrapped(model_class: Type, session=None) -> DatabaseOperation:
+    """
+    Purpose:
+        Drop the entire table
+
+    Confusion:
+        It gets the session from the wrapper
+        The wrapper converts the returning dictionary into DatabaseOperation.data
+    """
+    model_class.__table__.drop(bind=session.bind)
+    session.commit()
+    return {
+        'dropped': model_class.__name__,
+        'action': 'drop_fallback'
+    }
+
+
+@safe_db_operation
+def _clear_sql_model_wrapped(model_class: Type, session=None, **filters) -> DatabaseOperation:
+    """
+    Delete all records from a SQLAlchemy model table, optionally filtered.
+    Preserves the table structure.
+    
+    Args:
+        model_class: The SQLAlchemy model class to clear
+        session: Database session (injected by @safe_db_operation decorator)
+        **filters: Optional column=value filters for selective deletion
+                   e.g., clear_sql_model(User, status='inactive')
+    
+    Returns:
+        DatabaseOperation with deletion count and model name in the .data
+    
+    Raises:
+        AttributeError: If a filter column doesn't exist on the model
+    """
+    query = session.query(model_class)
+    
+    # Validate and apply filters
+    for column_name, value in filters.items():
+        if not hasattr(model_class, column_name):
+            raise AttributeError(
+                f"Model {model_class.__name__} has no column '{column_name}'"
+            )
+        column = getattr(model_class, column_name)
+        query = query.filter(column == value)
+
+    # Delete and commit    
+    deleted_count = query.delete(synchronize_session=False)
+    session.commit()
+    
+    return {
+        'deleted_rows': deleted_count,
+        'model': model_class.__name__,
+        'filters_applied': filters if filters else None
+    }
+
+
+
+def clear_sql_model(model_class: Type, **filters) -> dict:
+    """
+    Purpose:
+        Clear table; fallback to DROP if clear fails
+
+    Role:
+        Public handler that unwraps DatabaseOperation
+    """
+    result = _clear_sql_model_wrapped(model_class, **filters)
+    
+    if result.valid:
+        return result.data
+    else:
+        logger.warning(
+            f"Failed to clear {model_class.__name__} "
+            f"({result.error_type}), attempting DROP as fallback..."
+        )
+        
+        drop_result = _drop_sql_model_wrapped(model_class)
+        
+        if drop_result.valid:
+            return drop_result.data
+        else:
+            logger.error(
+                f"[CLEAR SQL MODEL] Both clear and drop failed for {model_class.__name__}: {drop_result.message}"
+            )
+            return False
+    

core/models/manage/insert.py

@@ -1,19 +1,27 @@
-from core.errors.logger import logger
+
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy import create_engine
 from typing import Type, Dict, Any
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from sqlalchemy.orm import Session
+
+from core.errors.logger import logger
 from core.models.manage.session_management import get_session
 from core.models.orm_models.Base import BaseModel
 
+from core.models.manage.wrapper import safe_db_operation, WrapperRollback
+from core.models.DatabaseOperation import DatabaseOperation
+
 # all models it knows how to do:
 from core.models.orm_models.CachedSync import CachedSync
 from core.models.orm_models.Location import Location
 from core.models.orm_models.Operator import Operator
 from core.models.orm_models.EncryptedProxy import EncryptedProxy
 
-def insert_into_model(model_class: Type, all_data: dict | list, override=False) -> bool:
+# This is the public interface,
+def insert_into_model(model_class: Type, all_data: dict | list, override=False) -> DatabaseOperation:
     """
+    Purpose:
         Generic ORM insert for any model.  Keeping it generic for reuse.
     
     Args:
@@ -22,45 +30,86 @@ def insert_into_model(model_class: Type, all_data: dict | list, override=False)
         override: If True, wipe the table before inserting
     
     Returns:
-        True if successful
+        DatabaseOperation object with true/false
     """
-
     logger.info(f"All the public data preparing to be inserted is {all_data}")
 
-    with get_session() as session:
-        # Step 1: Wipe the table if override=True
-        if override:
-            logger.info(f"First, WIPING the pre-existing data for {model_class.__name__}. Are you sure you intended to completely delete the old data?")
-            session.query(model_class).delete()
-            session.commit()
-
-        logger.info(f"Starting insert for {model_class.__name__}")
-
     # Normalize to list for uniform handling
     data_list = all_data if isinstance(all_data, list) else [all_data]
 
+    # Call the wrapped function with normalized data
+    return _wrapped_insert(model_class=model_class, data_list=data_list, override=override)
+
+
+@safe_db_operation
+def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session, override=False) -> DatabaseOperation:
+    """Insert items. Doesn't manage session."""
+
+    if override:
+        logger.info(f"First, WIPING the pre-existing data for {model_class.__name__}. Are you sure you intended to completely delete the old data?")
+        session.query(model_class).delete()
+
+    logger.info(f"Starting insert for {model_class.__name__}")
+
     try:
         for each_json in data_list:
             instance = model_class(**each_json)
             session.add(instance)
-            
+        session.commit()  # Commits both delete + inserts atomically
-            session.commit()
+        return DatabaseOperation(valid=True)
-            logger.info(f"Completed! SQL Insertion Successful following the {model_class.__name__} Model’s Type Rules")
-            return True
 
     except TypeError as e:
-            print(f"TypeError caught: {e}")
+        """
+        This is triggered when an Old client was sent unknown fields from the API in the JSON,
+        And it's handled inside the wrapped function because it needs access to the variables used.
+        """
         session.rollback()
-            raise ValueError(f"Invalid fields for {model_class.__name__}: {e}")
+
-        except IntegrityError as e:
+        valid_fields = {col.name for col in model_class.__table__.columns}
-            print(f"IntegrityError caught: {e.orig}")
+        filtered_data_list = [
-            session.rollback()
+            {k: v for k, v in each_json.items() if k in valid_fields}
-            raise ValueError(f"Constraint violation: {e.orig}")
+            for each_json in data_list
-        except SQLAlchemyError as e:
+        ]
-            print(f"SQLAlchemyError caught: {e}")
+        
-            session.rollback()
+        dropped_fields = set().union(*(set(d.keys()) - valid_fields for d in data_list))
-            raise
+        if dropped_fields:
-        except Exception as e:
+            logger.warning(f"Old Client Dropped unknown fields: {dropped_fields}")
-            print(f"Generic exception caught: {e}")
+        
-            session.rollback()
+        try:
-            raise e
+            for filtered_json in filtered_data_list:
+                instance = model_class(**filtered_json)
+                session.add(instance)
+            session.commit()
+            
+            # the fix worked,
+            return DatabaseOperation(
+                valid=True,
+                tried_filtered=True,
+                message=f"Inserted after filtering {dropped_fields}"
+            )
+        
+        except Exception as retry_error:
+            # Signal wrapper to rollback
+            raise WrapperRollback(DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.OLD_CLIENT_NEW_API,
+                tried_filtered=True,
+                message=str(f"Wrapped Insert Failed on Retry of a TypeError: {retry_error}")
+            ))
+
+
+# =================== Isolation Test ======================
+"""
+Spoof Failure of an operational error to trigger/test a migration
+"""
+    # from sqlalchemy import text
+
+    # # 1. Drop a column the model expects
+    # if model_class == Location:
+    #     with get_session() as session:
+    #         print("dropping provider_name from location..")
+    #         session.execute(text("ALTER TABLE locations DROP COLUMN provider_name"))
+    #         session.commit()
+    #         print("dropped provider_name from location!!!")
+
+    # # 2. Now insert will trigger OperationalError

core/models/manage/migrations.py

@@ -0,0 +1,148 @@
+# custom
+from core.errors.logger import logger
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.Constants import Constants
+
+from core.models.manage.session_management import create_ALL_tables
+# from core.models.orm_models.Location import Location
+# from core.models.orm_models.Operator import Operator
+# from core.models.orm_models.CachedSync import CachedSync
+# from core.models.orm_models.EncryptedProxy import EncryptedProxy
+
+# generic
+from pathlib import Path
+from sqlalchemy import Column, Integer, String, create_engine, inspect, text
+from sqlalchemy.orm import declarative_base, Session
+from sqlalchemy.exc import OperationalError, DBAPIError
+from sqlalchemy import inspect
+from sqlalchemy import text
+
+"""
+Purpose:
+    Sync SQL database schema to current Python model definitions.
+
+Situation Used:
+    Python Models have new values missing from the same SQL database
+
+Metaphor:
+    This replaces something like alembic, by doing it directly
+
+Confusion:
+    This may be confusing because we're using an ORM in general,
+    but doing migrations manually.
+    
+Why:
+    The reason is because alembic is good for servers, but not clients,
+    due to a large amount of boilerplate for migrations.
+"""
+
+def get_model_columns(model_class):
+    """
+    Extract all columns directly from a model class.
+    Bypasses metadata, caching, and all SQLAlchemy indirection.
+    """
+    try:
+        mapper = inspect(model_class)
+        columns = {}
+        
+        for column in mapper.columns:
+            columns[column.name] = {
+                'type': str(column.type),
+                'nullable': column.nullable,
+                'default': column.default,
+                'primary_key': column.primary_key,
+            }
+        
+        return columns
+    except Exception as e:
+        raise ValueError(f"Failed to inspect {model_class}: {e}")
+
+
+def migrate_sql() -> DatabaseOperation:
+    from core.models.manage.session_management import engine
+    from core.models.orm_models.Base import BaseModel, Base
+    try:
+        from core.models.orm_models.Location import Location
+        from core.models.orm_models.Operator import Operator
+        from core.models.orm_models.CachedSync import CachedSync
+        from core.models.orm_models.EncryptedProxy import EncryptedProxy
+        MODELS = [Location, Operator, CachedSync, EncryptedProxy]
+        logger.info(f"[MIGRATION] Tables loaded")
+    except:
+        logger.error(f"[MIGRATION] Could not load models.  Critical failure.")
+
+    changes_made = []
+    
+    # Step 1: Extract column definitions directly from each model
+    model_schema = {}
+    for model_class in MODELS:
+        table_name = model_class.__tablename__
+        try:
+            model_schema[table_name] = get_model_columns(model_class)
+        except ValueError as e:
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.PYTHON_MODEL_STRUCTURE,
+                message=e
+            )
+        logger.info(f"[MIGRATION] Model '{table_name}': {list(model_schema[table_name].keys())}")
+    
+    # Step 2: Compare to database schema
+    inspector = inspect(engine)
+    db_tables = set(inspector.get_table_names())
+        
+    for table_name, expected_columns in model_schema.items():
+        if table_name not in db_tables:
+            changes_made.append(table_name)
+            logger.info(f"[MIGRATION] Table '{table_name}' is MISSING from database")
+        else:
+            db_columns = {col['name']: col for col in inspector.get_columns(table_name)}
+            
+            for col_name, col_info in expected_columns.items():
+                if col_name not in db_columns:
+                    changes_made.append(col_name)
+                    logger.info(f"[MIGRATION] Column '{col_name}' is MISSING from table '{table_name}'")
+    
+    # Step 3: Perform the actual migration
+    if changes_made:
+        logger.info("[MIGRATION] Changes detected. Running schema update...")
+        Base.metadata.create_all(engine)
+        
+        try:
+            with engine.begin() as conn:
+                for table_name, expected_columns in model_schema.items():
+                    inspector = inspect(engine)
+                    
+                    if table_name in inspector.get_table_names():
+                        db_columns = {col['name'] for col in inspector.get_columns(table_name)}
+                        
+                        for col_name, col_info in expected_columns.items():
+                            if col_name not in db_columns:
+                                col_type = col_info['type']
+                                nullable = "NULL" if col_info['nullable'] else "NOT NULL"
+                                print(f"[MIGRATION] Adding column '{col_name}' to '{table_name}'")
+                                conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {nullable}"))
+
+        except (OperationalError, DBAPIError) as e:
+            logger.error(f"[MIGRATION] Failed: {str(e)}")
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.UNKNOWN,
+                message=f"Schema sync failed: {str(e)}"
+            )
+
+        logger.info(f"[MIGRATION] Completed! Schema synced successfully with these changes: {changes_made}")
+        return DatabaseOperation(
+            valid=True,
+            data=changes_made,
+            message=f"Synced these tables and columns: {'; '.join(changes_made)}"
+        )
+
+    else:
+        no_migration_needed = "Schema already matches models.  No migration needed."
+        logger.info(f"[MIGRATION] {no_migration_needed}")
+        return DatabaseOperation(
+            valid=True,
+            message=no_migration_needed
+        )
+

core/models/manage/session_management.py

@@ -1,5 +1,9 @@
 from core.errors.logger import logger
 from core.models.orm_models.Base import BaseModel
+from core.models.manage.version_check import insert_new_version
+from core.Constants import Constants
+
+
 from sqlalchemy import create_engine, inspect
 from sqlalchemy.orm import sessionmaker
 import os
@@ -86,13 +90,38 @@ def create_ONLY_db_version_table():
         raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
     
     from core.models.orm_models.DatabaseVersion import database_version
-    try:
-        database_version.create(engine)
-    except:
-        logger.info("[DB MANAGEMENT] database_version already exists, but was attempted to be made again by calling create_ONLY_db_version_table.")
     
-    # if using without checking or try except blocks:
+    try:
-    # database_version.create(engine, checkfirst=True)
+        database_version.create(engine, checkfirst=True)
+        logger.info("[DB MANAGEMENT] database_version table created.")
+        
+        # CRITICAL FIX: Populate the table immediately
+        session = get_session()
+        existing_row = session.query(database_version).first()
+        
+        if not existing_row:
+            logger.info("[DB MANAGEMENT] Table was empty. Initializing with current app version.")
+            insert_new_version(session, Constants.DB_VERSION_THIS_APP_WANTS)
+        else:
+            logger.info("[DB MANAGEMENT] Table already has a version row.")
+            
+    except Exception as e:
+        logger.error(f"[DB MANAGEMENT] Failed to create/initialize database_version table: {e}")
+
+
+# def create_ONLY_db_version_table():
+#     """Create only the database_version table using the global engine."""
+#     if engine is None:
+#         raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
+    
+#     from core.models.orm_models.DatabaseVersion import database_version
+#     try:
+#         database_version.create(engine)
+#     except:
+#         logger.info("[DB MANAGEMENT] database_version already exists, but was attempted to be made again by calling create_ONLY_db_version_table.")
+
+#     # if using without checking or try except blocks:
+#     # database_version.create(engine, checkfirst=True)
 
 
 def does_db_version_table_exist():

core/models/manage/version_check.py

@@ -2,7 +2,6 @@ from core.models.orm_models.DatabaseVersion import database_version
 from core.Constants import Constants
 from core.errors.logger import logger
 
-
 # generic
 from sqlalchemy import exc, text
 from sqlalchemy.orm import Session
@@ -271,9 +270,10 @@ def get_custom_message(reason: str, compatability_dict: dict) -> str:
         GUI's startup script in __main__
     """
     if reason == "upgrade":
-        custom_error = "You just upgraded HydraVeil, so let's upgrade your database to handle the new data types."
+        custom_error = "You just upgraded HydraVeil, so let's upgrade your database to handle the new data types.  This will NOT harm your existing profiles or browser sessions."
     elif reason == "old_app":
-        custom_error = "You're using an old version of HydraVeil, with a newer version of the database.  This means the older app would crash trying to handle the newer data."
+        custom_error = "Upgrade Time! You're using an old version of HydraVeil, with a newer version of the database.  This means the older app would crash trying to handle the newer data.  This will NOT harm your existing profiles or browser sessions."
+
     else:
         error_msg = compatability_dict.get("error", "Unknown reason.")
         custom_error = f"There was an error with upgrading your database version.  Please tell customer support: {error_msg}."
@@ -346,3 +346,110 @@ def check_database_compatibility(session: Session) -> dict:
             "old_db_version": db_version_you_have,
             "error": "unknown result"
         }
+
+
+# DEAD CODE. Not used.  But COULD be in the future.
+def delete_sync_metadata(session: Session) -> bool:
+    """
+    Called By:
+        Nobody, dead code.  I originally was going to use this but cut it's use.
+
+    Purpose:
+        Clears existing rows of metadata
+
+    Rank:
+        Helper
+
+    Args:
+        session: SQLAlchemy session
+    
+    Returns:
+        bool: True if successful, False otherwise.
+        does NOT raise errors.
+    """
+    func_name = "delete_sync_metadata" # for logging purposes
+    
+    # Validate session
+    if session is None:
+        logger.error(f"{func_name}: Session is None")
+        return False
+    
+    if not isinstance(session, Session):
+        logger.error(f"{func_name}: Invalid session type: {type(session)}")
+        return False
+    
+    try:
+        logger.debug(f"[{func_name}]: Starting transaction for metatable entry delete")
+        
+        # Check if table exists before proceeding
+        try:
+            session.query(database_version).limit(1).all()
+        except exc.NoSuchTableError:
+            logger.info(
+                f"{func_name}: Table 'database_version' does not exist, this is good because we wanted to clear it."
+            )
+            return True # TRUE! this is great that it's gone already.
+        
+        # Delete existing rows
+        logger.debug(f"{func_name}: Deleting existing version rows..")
+        deleted_count = session.query(database_version).delete()
+        logger.debug(f"{func_name}: Deleted {deleted_count} existing row(s)")
+        
+        if deleted_count > 1:
+            logger.warning(
+                f"{func_name}: Deleted {deleted_count} rows. "
+                "Table should contain only a single row at any time."
+            )
+                
+        # Commit transaction
+        session.commit()
+        return True
+        
+    except exc.IntegrityError as e:
+        logger.error(
+            f"{func_name}: Integrity constraint violation (e.g., primary key conflict). "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.OperationalError as e:
+        logger.info(
+            f"{func_name}: Sync metadata never existed. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return True # this is good
+    
+    except exc.DatabaseError as e:
+        logger.error(
+            f"{func_name}: General database error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.StatementError as e:
+        logger.error(
+            f"{func_name}: SQL statement error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except ValueError as e:
+        logger.error(
+            f"{func_name}: Value error during metadata deletion."
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except Exception as e:
+        logger.critical(
+            f"{func_name}: Unexpected exception type {type(e).__name__}. "
+            f"Details: {str(e)}",
+            exc_info=True
+        )
+        session.rollback()
+        return False

core/models/manage/wrapper.py

@@ -0,0 +1,118 @@
+# custom
+from core.errors.logger import logger
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.manage.migrations import migrate_sql
+from core.models.manage.session_management import get_session
+from core.Constants import Constants
+
+# generic
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import (
+    OperationalError,
+    IntegrityError,
+    SQLAlchemyError
+)
+from typing import Type, Dict, Any, Callable, TypeVar
+from functools import wraps
+from enum import Enum
+
+"""
+Purpose:
+    This is a wrapper for other SQL functions to catch errors and do solutions/fixes.
+
+Used/Called by:
+    insert_into_model
+
+Returns:
+    DatabaseOperation Objects
+"""
+
+def safe_db_operation(func):
+    def wrapper(*args, **kwargs):
+
+        # get the Session for the function it's wrapping,
+        session = get_session()
+
+        try:
+            # passes the session to the function,
+            result = func(*args, session=session, **kwargs)
+            
+            # If func already returns DatabaseOperation, pass it through
+            if isinstance(result, DatabaseOperation):
+                return result
+            
+            # Otherwise, wrap success
+            return DatabaseOperation(valid=True, data=result)
+        
+        except OperationalError as e:
+            logger.error(f"Schema mismatch in {func.__name__}: {e}")
+            session.rollback()
+            
+            try:
+                logger.info("Attempting migration...")
+                migrate_sql()
+
+                # try again:
+                result = func(*args, session=session, **kwargs)
+                return DatabaseOperation(
+                    valid=True,
+                    data=result,
+                    tried_migration=True,
+                    message="Recovered via migration"
+                )
+            except Exception as retry_error: # migration failed:
+                logger.error(f"Migration failed with error: {str(retry_error)}", exc_info=True)
+                session.rollback()
+                return DatabaseOperation(
+                    valid=False,
+                    error_type=DBErrorType.MIGRATION_FAILED,
+                    message=str(retry_error)
+                )
+
+        # This is really TypeErrors, passed from the insert function.
+        except WrapperRollback as e:
+            session.rollback()
+            return e.database_operation
+
+        except IntegrityError as e:
+            error_msg = f"IntegrityError caught: {e.orig}"
+            logger.error(error_msg)
+            session.rollback()
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.INTEGRITY_ERROR,
+                message=error_msg
+            )
+
+        except SQLAlchemyError as e:
+            error_msg = f"SQLAlchemyError caught: {e}"
+            logger.error(error_msg)
+            session.rollback()
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.UNKNOWN,
+                message=str(e)
+            )
+
+        except Exception as e:
+            # Catch-all for unexpected errors
+            session.rollback()
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.UNKNOWN,
+                message=str(e)
+            )
+    
+    return wrapper
+
+
+"""
+If a wrapped function needs to pass up an error to this wrapper,
+so that the wrapper can roll back sessions,
+
+Then it uses this error exception and passes the DatabaseOperation object.
+"""
+class WrapperRollback(Exception):
+    def __init__(self, database_operation: DatabaseOperation):
+        self.database_operation = database_operation
+

core/models/orm_calls/location_calls.py

@@ -0,0 +1,31 @@
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.models.manage.wrapper import safe_db_operation, WrapperRollback
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.errors.logger import logger
+
+# generic
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import Session
+
+@safe_db_operation
+def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
+    location_object = session.execute(
+        select(Location)
+        .where((Location.country_code == country_code) & (Location.code == city_code))
+        .options(joinedload(Location.operator))
+    ).scalar_one_or_none()
+    return location_object
+
+
+def get_profile_location_data(country_code: str, city_code: str) -> Location:
+    location_object = execute_location_sql(country_code, city_code)
+    if location_object.valid:
+        return location_object.data
+    else:
+        critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}"
+        logger.error(critical_error)
+        print(critical_error)
+        return None
+

core/models/orm_models/Location.py

@@ -46,7 +46,7 @@ class Location(BaseModel):
     is_hysteria2_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
     is_vless_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
 
-    def to_dict(self):
+    def convert_to_dict(self):
         return {
             "country_code": self.country_code,
             "code": self.code,

core/models/session/SessionConnection.py

@@ -1,7 +1,6 @@
 from core.models.BaseConnection import BaseConnection
 from dataclasses import dataclass
 
-
 @dataclass
 class SessionConnection(BaseConnection):
     masked: bool = False
@@ -11,10 +10,43 @@ class SessionConnection(BaseConnection):
         if self.code not in ('system', 'tor', 'wireguard'):
             raise ValueError('Invalid connection code.')
 
-    # called by connection controller
+
+    # Called by: ConnectionController
     # doesn't even make sense, it's checking if the Session is code system. this will always be false.
     def is_unprotected(self):
         return self.code == 'system' and self.masked is False
 
+    # Called by: SessionProfile.determine_timezone 
     def needs_proxy_configuration(self):
         return self.masked is True
+
+
+# Potential refactor
+
+# from dataclasses import dataclass, field
+# from dataclasses_json import dataclass_json, config
+# from enum import Enum
+
+# class SessionConnectionTypes(str, Enum):
+#     WIREGUARD = "wireguard"
+#     TOR = "tor"
+
+# @dataclass_json
+# @dataclass
+# class SessionConnection:
+#     code: SessionConnectionTypes = field(
+#         metadata=config(
+#             encoder=lambda x: x.value,  # Convert enum to string on save
+#             decoder=lambda x: SessionConnectionTypes(x)  # Convert string to enum on load
+#         )
+#     )
+#     masked: bool = False
+
+#     def __post_init__(self):
+#         # Convert string to enum for deserialization
+#         if isinstance(self.code, str):
+#             self.code = SessionConnectionTypes(self.code)
+        
+#         # Validation
+#         if self.code not in SessionConnectionTypes:
+#             raise ValueError(f'Invalid code: {self.code}')

core/models/session/SessionProfile.py

@@ -16,7 +16,7 @@ import shutil
 class SessionProfile(BaseProfile):
     resolution: str
     application_version: Optional[ApplicationVersion]
-    connection: Optional[SessionConnection]
+    connection: Optional[SessionConnection] = None
 
     def has_connection(self):
         return self.connection is not None

core/models/system/SystemConnection.py

@@ -1,11 +1,10 @@
 from core.models.BaseConnection import BaseConnection
 from dataclasses import dataclass
 
-
+# legacy:
 @dataclass
 class SystemConnection (BaseConnection):
 
-
     def __post_init__(self):
 
         if self.code not in ('vless', 'hysteria2', 'wireguard'):
@@ -14,3 +13,17 @@ class SystemConnection (BaseConnection):
     @staticmethod
     def needs_proxy_configuration():
         return False
+
+# Potential refactor:
+# from enum import Enum
+# from typing import Literal
+
+# class SystemConnectionTypes(str, Enum):
+#     WIREGUARD = "wireguard"
+#     HYSTERIA2 = "hysteria2"
+#     VLESS = "vless"
+
+# @dataclass
+# class SystemConnection(BaseConnection):
+#     code: SystemConnectionTypes
+#     masked: Literal[False] = False

core/models/system/SystemState.py

@@ -11,6 +11,8 @@ import pathlib
 @dataclass
 class SystemState:
     profile_id: int
+    firewalled: bool
+    dns_set: bool
 
     def save(self: Self):
 

core/services/failed_verification/is_the_key_to_blame.py

@@ -11,7 +11,9 @@ from core.services.prepare_tickets.get_public_key_by_config import get_public_ke
 from core.services.prepare_tickets.get_pub_key import key_is_in_valid_format, get_pub_key
 from core.services.failed_verification.test_if_new_key_works import test_if_new_key_works
 from core.services.networking.make_url import make_url
-from core.services.networking.get_data_from_server import get_data_from_server
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+
+
 # utils
 from core.utils.basic_operations.write_or_read_from_json import get_value_from_json_file
 from core.utils.basic_operations.write_string_to_text_file import write_string_to_text_file
@@ -70,7 +72,7 @@ def get_new_pubkey_from_api(connection_observer: ConnectionObserver) -> dict | N
     url = make_url(which_key_plan)
 
     # the result of this is a python dictionary with single '
-    api_results = get_data_from_server(url, connection_observer)
+    api_results = get_data_from_server(url, None, connection_observer)
 
     if "data" in api_results:
         new_public_key = api_results["data"]

core/services/helpers/get_plan_data.py

@@ -5,13 +5,15 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 # services
 from core.services.networking.make_url import make_url
-from core.services.networking.send_data_to_server import send_data_to_server
+# from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
 
 
 # use temp billing to get the plan details
 def get_plan_data(
     temp_billing_code: str, connection_observer: ConnectionObserver
-) -> dict:
+) -> dict | ApiResponse:
     which_endpoint = "/plan"
     url = make_url(which_endpoint)
     payload = {"temp_billing_code": temp_billing_code}

core/services/helpers/manage_assets.py

@@ -0,0 +1,53 @@
+from core.Constants import Constants
+from core.errors.logger import logger
+
+# generic
+from pathlib import Path
+import shutil
+import os
+
+def assets_folder_setup():
+    initial_appimage_assets = f"{Constants.APPDIR_HOME}/assets"
+    current_assets_folder = f"{Constants.HV_DATA_HOME}/assets"
+    if os.path.exists(current_assets_folder):
+        return True
+    return copy_folders(initial_appimage_assets, current_assets_folder)
+    
+
+
+def sudo_assets_folder_setup() -> bool:
+    # initial_appimage_assets = f"{Constants.APPDIR_HOME}/assets/sudo_scripts"
+    initial_appimage_assets = f"/home/a/VPN/code/DEV/sp-hydra-veil-core/core/assets/sudo_scripts"
+    current_assets_folder = f"{Constants.HOME}/Downloads/hydraveil_sudo_scripts"
+    return copy_folders(initial_appimage_assets, current_assets_folder)
+
+
+def copy_folders(initial_appimage_assets: str, current_assets_folder: str) -> bool:
+    try:
+        os.makedirs(current_assets_folder, exist_ok=True)
+        for item in os.listdir(initial_appimage_assets):
+            src = os.path.join(initial_appimage_assets, item)
+            dst = os.path.join(current_assets_folder, item)
+            if os.path.isdir(src):
+                shutil.copytree(src, dst)
+            else:
+                shutil.copy2(src, dst)
+        logger.info(f"[MANAGE ASSETS] Successfully copied over the folder.")
+        return True
+    except Exception as e:
+        logger.error(f"[MANAGE ASSETS] initial_appimage_assets is {initial_appimage_assets}")
+        logger.error(f"[MANAGE ASSETS] current_assets_folder is {current_assets_folder}")
+        logger.error(f"[MANAGE ASSETS] ERROR: {e}")
+        return False
+
+
+def cleanup_setup_dir():
+    """After install completes"""
+    delete_this_folder = f"{Constants.HOME}/Downloads/hydraveil_sudo_scripts"
+    try:
+        shutil.rmtree(delete_this_folder)
+        logger.info(f"[MANAGE ASSETS] Cleaned up {delete_this_folder}")
+        return True
+    except Exception as e:
+        logger.error(f"[MANAGE ASSETS] We could not delete {delete_this_folder} because {e}")
+        return False

core/services/helpers/setup_sudo_scripts.py

@@ -0,0 +1,67 @@
+from core.services.helpers.manage_assets import sudo_assets_folder_setup
+from core.utils.basic_operations.confirm_files_exist import confirm_files_and_folders_exist
+from core.models.Result import Result, ResultError
+from core.Constants import Constants
+from core.errors.logger import logger
+
+import subprocess
+from pathlib import Path
+import shutil
+import os
+from subprocess import CalledProcessError
+import copy
+
+FILES_TO_CHECK = ["setup.sh", "firewall", "dns"]
+TARGET_FOLDER = "/opt/hydra-veil"
+
+def auto_install_sudo_script() -> Result:
+    setup_dir = Path(f"{Constants.HOME}/Downloads/hydraveil_sudo_scripts")
+    setup_script = Path(setup_dir, 'setup.sh')
+    original_linux_user = os.getlogin()
+
+    # Make sure the folder exists
+    files_exist = confirm_files_and_folders_exist(
+        folder_path=setup_dir, files_to_check=FILES_TO_CHECK
+    )
+
+    if not files_exist:
+        copied = sudo_assets_folder_setup()
+        if not copied:
+            return Result(valid=False, message="Could not copy scripts to begin.")
+
+    try:
+        result = subprocess.run(
+            ['pkexec', 'env', f'ORIGINAL_FOLDER={setup_dir}', f'TARGET_FOLDER={TARGET_FOLDER}', f'LINUX_USER={original_linux_user}', 'bash', setup_script],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode == 0:
+            logger.info("Sudo Setup Ran. Now testing..")
+            return Result(valid=True)
+        else:
+            logger.error(f"Sudo setup setup failed: {result.stderr.strip()}")
+            return Result(valid=False, message=result.stderr.strip())
+    except CalledProcessError as e:
+        return Result(valid=False, message=e)
+    except Exception as e:
+        return Result(valid=False, message=e)
+
+def test_if_in_sudo_folder() -> Result:
+    final_sudo_folder = Path("/opt/hydra-veil/")
+    _files_to_check = FILES_TO_CHECK.copy() # to not affect the other functions.
+    
+    # setup script itself doesn't get copied over:
+    _files_to_check.remove("setup.sh")
+
+    # Now verify it copied them:
+    sudo_files_copied = confirm_files_and_folders_exist(
+        folder_path=final_sudo_folder, files_to_check=_files_to_check
+    )
+
+    if sudo_files_copied:
+        return Result(valid=True, message="Sudo Script Setup Completed Successfully!")
+    else:
+        error_msg = "We failed to find the files in the sudo folder."
+        logger.error(error_msg)
+        return Result(valid=False, message=error_msg)
+

core/services/keys_and_verifications/endpoint_verification.py

@@ -0,0 +1,48 @@
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+from core.models.BaseProfile import BaseProfile as Profile
+from core.Errors import InvalidSubscriptionError, MissingSubscriptionError, ConnectionTerminationError, ProfileActivationError, ProfileDeactivationError, MissingLocationError, ConnectionUnprotectedError, EndpointVerificationError, ProfileStateConflictError
+from typing import Union, Optional
+import base64
+
+
+def verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile], ignore: tuple[type[Exception]] = ()):
+
+    try:
+        __verify_wireguard_endpoint(profile)
+
+    except EndpointVerificationError as error:
+
+        if not EndpointVerificationError in ignore:
+
+            profile.address_security_incident()
+            raise error
+
+def __verify_wireguard_endpoint(profile: Union[SessionProfile, SystemProfile]):
+
+    from cryptography.hazmat.primitives.asymmetric import ed25519
+    import base64
+
+    signature = profile.get_wireguard_configuration_metadata('Signature')
+    wireguard_public_keys = profile.get_wireguard_public_keys()
+    operator = profile.location.operator
+
+    if signature is None:
+        raise EndpointVerificationError('The WireGuard endpoint\'s signature could not be determined.')
+
+    if not wireguard_public_keys:
+        raise EndpointVerificationError('The WireGuard endpoint\'s public key could not be determined.')
+
+    if operator is None:
+        raise EndpointVerificationError('The WireGuard endpoint\'s operator could not be determined.')
+
+    try:
+
+        operator_public_key = ed25519.Ed25519PublicKey.from_public_bytes(bytes.fromhex(operator.public_key))
+
+        for wireguard_public_key in wireguard_public_keys:
+            operator_public_key.verify(base64.b64decode(signature), wireguard_public_key.encode('utf-8'))
+
+    except Exception:
+        raise EndpointVerificationError('The WireGuard endpoint could not be verified.')
+

core/services/keys_and_verifications/wireguard_keys.py

@@ -0,0 +1,66 @@
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+from core.Errors import MissingSubscriptionError, MissingLocationError, InvalidSubscriptionError
+from core.services.WebServiceApiService import WebServiceApiService
+from core.controllers.ConnectionController import ConnectionController
+from core.observers.ConnectionObserver import ConnectionObserver
+from typing import Union, Optional
+import base64
+import re
+
+def register_wireguard_session(
+    profile: Union[SessionProfile, SystemProfile], 
+    connection_observer: Optional[ConnectionObserver] = None
+):
+    """Register a WireGuard session for the given profile."""
+    
+    if not profile.has_subscription():
+        raise MissingSubscriptionError()
+    
+    if not profile.has_location():
+        raise MissingLocationError()
+    
+    wireguard_keys = _generate_wireguard_keys()
+    
+    wireguard_configuration = ConnectionController.with_preferred_connection(
+        profile.location.country_code, 
+        profile.location.code, 
+        profile.subscription.billing_code, 
+        wireguard_keys.get('public'), 
+        task=WebServiceApiService.post_wireguard_session, 
+        connection_observer=connection_observer
+    )
+    
+    if wireguard_configuration is None:
+        raise InvalidSubscriptionError()
+    
+    wireguard_configuration = _inject_private_key(wireguard_configuration, wireguard_keys.get('private'))
+    profile.attach_wireguard_configuration(wireguard_configuration)
+
+
+def _generate_wireguard_keys() -> dict:
+    """Generate WireGuard public/private key pair."""
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey
+
+    raw_private_key = X25519PrivateKey.generate()
+
+    public_key = raw_private_key.public_key().public_bytes(
+        encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw
+    )
+
+    private_key = raw_private_key.private_bytes(
+        encoding=serialization.Encoding.Raw, format=serialization.PrivateFormat.Raw, encryption_algorithm=serialization.NoEncryption()
+    )
+
+    return dict(
+        private=base64.b64encode(private_key).decode(),
+        public=base64.b64encode(public_key).decode()
+    )
+
+
+def _inject_private_key(config: str, private_key: str) -> str:
+    """Inject private key into WireGuard config."""
+    expression = re.compile(r'^(PrivateKey =)\s?$', re.MULTILINE)
+    return re.sub(expression, r'\1 ' + private_key, config)
+

core/services/networking/api_requests/ApiResponseModel.py

@@ -0,0 +1,48 @@
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional, Any
+
+class ErrorType(Enum):
+    """Classified error categories."""
+    SUCCESS = "success"
+    TOR_NOT_WORKING = "tor_not_working"
+    TOR_DNS_BLOCKED = "tor_dns_blocked"
+    DNS_RESOLUTION = "dns_resolution"
+    QUAD9_DNS_RESOLUTION = "quad9_dns_resolution"
+    NO_INTERNET = "no_internet"
+    NETWORK_ERROR = "network_error"
+    INVALID_INPUT = "invalid_input"
+    RATE_LIMITED = "rate_limited"
+    SERVER_ERROR = "server_error"
+    DEVELOPER_ERROR = "developer_error"
+    UNKNOWN = "unknown"
+
+@dataclass
+class ApiResponse:
+    """Unified response from all API operations."""
+    valid: bool
+    error_type: Optional[ErrorType] = None
+    data: Optional[Any] = None
+    message: Optional[str] = None
+    retry_now: bool = False
+    retry_later: bool = False
+    tor: bool = False
+    ip_address: str = None
+    ask_clearweb: bool = None
+
+    def to_dict(self) -> dict:
+        """Convert to dict for backwards compatibility."""
+        return {
+            "valid": self.valid,
+            "error_code": self.error_type.value if self.error_type else None,
+            "data": self.data,
+            "message": self.message,
+            "retry_now": self.retry_now,
+            "retry_later": self.retry_later,
+            "tor": self.tor,
+            "ask_clearweb": self.ask_clearweb
+        }
+    
+    def get(self, key: str, default=None):
+        """Dict-like access for backwards compatibility."""
+        return getattr(self, key, default)

core/services/networking/api_requests/step1_get_or_post.py

@@ -0,0 +1,102 @@
+from core.services.networking.api_requests.step2_execute import _execute_tor_request, _execute_regular_request
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
+from core.services.networking.api_requests.subtools.get_connection_type import get_connection_type
+from core.services.networking.api_requests.subtools.replace_http_with_https import replace_http_with_https
+
+from core.Constants import Constants
+from core.observers.BaseObserver import BaseObserver
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+from typing import Optional
+
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+
+def get_data_from_api(
+    full_request_url: str,
+    client_observer: Optional[ClientObserver] = None,
+    connection_observer: Optional[ConnectionObserver] = None,
+    clearweb_resolved_ip: str = None,
+) -> ApiResponse:
+    """
+    GET wrapper. Desktop passes observer for updates; Android passes None.
+    """
+    if not full_request_url:
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.INVALID_INPUT,
+            message="Input URL is not properly configured"
+        )
+    
+    connection_type = get_connection_type()
+    
+    if connection_type == "tor":
+        # Tor: can push intermediate updates (desktop-only)
+        result_object = _execute_tor_request(
+            "get",
+            full_request_url,
+            None, # get request has no payload
+            connection_observer,
+            client_observer,
+            timeout=30,
+            clearweb_resolved_ip=clearweb_resolved_ip
+        )
+    else:
+        # Regular, no tor:
+        result_object = _execute_regular_request(
+            "get",
+            full_request_url,
+            None, # get request has no payload
+            connection_observer,
+            client_observer,
+            timeout=15
+        )
+
+    if not result_object.valid and client_observer:
+        error_msg = result_object.message
+        client_observer.notify('synchronizing', f"Connection Issue: {error_msg}")
+    
+    return result_object
+
+
+def send_data_to_server(
+    payload: dict,
+    original_url: str,
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+    clearweb_resolved_ip: str = None,
+) -> ApiResponse:
+    """
+    POST wrapper. Same pattern as GET.
+    """
+    url = replace_http_with_https(original_url)
+    connection_type = get_connection_type()
+    
+    if connection_type == "tor":
+        result_object = _execute_tor_request(
+            "post",
+            url,
+            payload,
+            connection_observer,
+            client_observer,
+            timeout=10,
+            clearweb_resolved_ip=clearweb_resolved_ip
+        )
+    else:
+        result_object = _execute_regular_request(
+            "post",
+            url,
+            payload,
+            connection_observer,
+            client_observer,
+            timeout=5
+        )
+    
+    # Push final error to UI if provided
+    if not result_object.valid and client_observer:
+        error_msg = result_object.message
+        client_observer.notify('synchronizing', f"Connection Issue: {error_msg}")
+
+    return result_object

core/services/networking/api_requests/step2_execute.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import requests
+
+from core.Constants import Constants
+
+# Networking Group
+from core.services.networking.api_requests.step3_classify_http import classify_http_response
+from core.services.networking.api_requests.step4_error_classifier import classify_request_error
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.subtools.custom_dns import prep_url_with_known_ip, load_custom_dns_resolver
+
+# tor
+from essentials.modules.TorModule import TorModule
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+
+from core.observers.ClientObserver import ClientObserver
+
+# errors
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+# generic
+import json, os
+from typing import Any
+import time
+
+
+
+def _execute_tor_request(
+    method: str,
+    url: str,
+    payload: Optional[dict],
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+    timeout: int = 30,
+    clearweb_resolved_ip: str = None # if DNS via Tor is blocked
+) -> ApiResponse:
+    """
+    Tor request with optional intermediate UI feedback (desktop-only).
+    """
+    import requests
+    port_number = None
+    tor_module = None
+    
+    try:
+        if client_observer:
+            status_update = "Initializing Tor connection..."
+            client_observer.notify('synchronizing', status_update)
+
+        port_number = ConnectionService.get_random_available_port_number()
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+        
+        if client_observer:
+            status_update = f"Executing {method.upper()} via Tor..."
+            client_observer.notify('synchronizing', status_update)
+
+        proxies = {"http": f"socks5h://127.0.0.1:{port_number}",
+                   "https": f"socks5h://127.0.0.1:{port_number}"}
+
+        # ================= OPTION: DNS OUTSIDE TOR. (IF DNS VIA TOR IS BLOCKED) ===============
+        if clearweb_resolved_ip:
+            CustomHostnameAdapter = load_custom_dns_resolver()
+
+            url_with_ip, original_hostname = prep_url_with_known_ip(clearweb_resolved_ip, url)
+
+            # Set up session with custom adapter for DNS resolution:
+            session = requests.Session()
+            session.mount('https://', CustomHostnameAdapter(original_hostname))
+            session.headers.update({"Host": original_hostname})
+
+            # Make the request to the raw IP, but tell the server it's the original domain
+            if method.lower() == "get":
+                response = session.get(url_with_ip, proxies=proxies, timeout=timeout, verify=True)
+            else:
+                response = requests.post(url_with_ip, json=payload, proxies=proxies, timeout=timeout, verify=True)
+
+        else:
+            # ================= USING DNS VIA TOR (normal-use) ===============
+            if method.lower() == "get":
+                response = requests.get(url, proxies=proxies, timeout=timeout)
+            else:
+                response = requests.post(url, json=payload, proxies=proxies, timeout=timeout)
+
+        # ================= Working Response ===============
+        result = classify_http_response(response)
+        
+    except Exception as e:
+        logger.debug(f"Tor {method.upper()} failed inside _execute_tor_request: {type(e).__name__}: {e}")
+        result = classify_request_error(url, "tor", connection_observer)
+    
+    finally:
+        if tor_module and port_number:
+            try:
+                tor_module.destroy_session(port_number)
+            except Exception as e:
+                logger.warning(f"Error destroying Tor session: {e}")
+    
+    return result
+
+
+def _execute_regular_request(
+    method: str,
+    url: str,
+    payload: Optional[dict],
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[client_observer] = None,
+    timeout: int = 20,
+) -> ApiResponse:
+    """
+    Regular (non-Tor) request. NO observer coupling.
+    Cross-platform reusable: desktop UI and Android/Kivy both consume ApiResponse.
+    """
+    import requests
+
+    try:
+        logger.debug(f"Executing {method.upper()} to {url}")
+        
+        if method.lower() == "get":
+            response = requests.get(url, timeout=timeout)
+        else:
+            response = requests.post(url, json=payload, timeout=timeout)
+        
+        return classify_http_response(response)
+    
+    except Exception as e:
+        logger.debug(f"{method.upper()} request failed: {type(e).__name__}: {e}")
+        return classify_request_error(url, "regular", connection_observer)

core/services/networking/api_requests/step3_classify_http.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import requests
+
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+import json
+
+def classify_http_response(response) -> ApiResponse:
+    """
+    Classify HTTP response status and extract error info.
+    Replaces evaluate_response().
+    """
+    import requests
+
+    if not isinstance(response, requests.Response):
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.NETWORK_ERROR,
+            message="Invalid response object"
+        )
+
+    print(f"We're inside classify http testing the status code of the response: {response.status_code}")
+
+    # Success case
+    if 200 <= response.status_code < 300:
+        try:
+            data = response.json()
+            return ApiResponse(valid=True, data=data)
+        except ValueError:
+            return ApiResponse(
+                valid=True,
+                data=response.text
+            )
+    
+    # Rate limiting
+    if response.status_code == 429:
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.RATE_LIMITED,
+            message="Rate limit exceeded",
+            retry_now=True
+        )
+    
+    # Client errors (4xx)
+    if 400 <= response.status_code < 500:
+        try:
+            resp_json = response.json()
+            error_msg = resp_json.get("message") or resp_json.get("error") or resp_json.get("error_code")
+        except ValueError:
+            error_msg = response.text
+        
+        error_map = {
+            404: ErrorType.INVALID_INPUT,
+            400: ErrorType.INVALID_INPUT,
+            401: ErrorType.INVALID_INPUT,
+            403: ErrorType.INVALID_INPUT,
+            405: ErrorType.INVALID_INPUT,
+        }
+        error_type = error_map.get(response.status_code, ErrorType.INVALID_INPUT)
+        
+        return ApiResponse(
+            valid=False,
+            error_type=error_type,
+            message=error_msg or f"Client error {response.status_code}"
+        )
+    
+    # Server errors (5xx)
+    if response.status_code >= 500:
+        retry_now = response.status_code in (502, 503, 504)
+        retry_later = response.status_code in (500, 501, 505)
+        
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.SERVER_ERROR,
+            message=f"Server error {response.status_code}",
+            retry_now=retry_now,
+            retry_later=retry_later
+        )
+    
+    return ApiResponse(
+        valid=False,
+        error_type=ErrorType.UNKNOWN,
+        message=f"Unexpected status {response.status_code}"
+    )

core/services/networking/api_requests/step4_error_classifier.py

@@ -0,0 +1,91 @@
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+
+
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
+# diagnosis services
+from core.services.networking.api_requests.subtools.internet_test import do_we_have_internet
+from core.services.networking.api_requests.subtools.dns_evaluation import is_dns_problem, dns_works_via_tor
+from core.services.networking.api_requests.subtools.extract_domain import extract_domain
+from core.services.networking.api_requests.subtools.is_tor_working import is_tor_working
+
+
+# errors
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+from typing import Optional
+
+
+def classify_request_error(
+    url: str,
+    connection_type: str,
+    connection_observer: ConnectionObserver,
+    client_observer: Optional[ClientObserver] = None,
+) -> ApiResponse:
+    """
+    Classify all error requests,
+    Tor Connection.
+    DNS
+    Internet Connection
+    """
+    domain_only = extract_domain(url)
+    
+    if connection_type == "tor":
+        # Tor-specific diagnostics
+        if not is_tor_working(connection_observer):
+            logger.debug("Tor is not working")
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.TOR_NOT_WORKING,
+                tor=True,
+                message="Tor connection failed"
+            )
+
+        ip_via_tor = dns_works_via_tor(domain_only, connection_observer)
+
+        if not ip_via_tor:
+            logger.debug("DNS resolution via Tor failed")
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.DNS_RESOLUTION,
+                tor=True,
+                message=f"Cannot resolve {domain_only} via Tor"
+            )
+        else:
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.DNS_RESOLUTION,
+                tor=True,
+                ip_address=ip_via_tor,
+                message=f"For Unknown reasons, we can't connect, but we can resolve {domain_only} to {ip_via_tor} via Tor"
+            )
+
+    else:
+        # Regular (non-Tor) diagnostics
+        if not do_we_have_internet():
+            logger.error("No internet connection")
+            return ApiResponse(
+                valid=False,
+                tor=False,
+                error_type=ErrorType.NO_INTERNET,
+                message="No internet connectivity"
+            )
+        
+        if is_dns_problem(domain_only, connection_observer, client_observer):
+            logger.error("Local DNS resolution failed")
+            return ApiResponse(
+                valid=False,
+                tor=False,
+                error_type=ErrorType.DNS_RESOLUTION,
+                message=f"Cannot resolve {domain_only}"
+            )
+    
+    # Fallback: unknown error
+    logger.error(f"Unknown error for {connection_type} request to {domain_only}")
+    return ApiResponse(
+        valid=False,
+        error_type=ErrorType.UNKNOWN,
+        message="Request failed for unknown reason"
+    )

core/services/networking/api_requests/step5_solve_api_problems.py

@@ -0,0 +1,60 @@
+from core.errors.logger import logger
+from core.Constants import Constants
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+from core.observers.ClientObserver import ClientObserver
+
+# tools to solve:
+from core.services.networking.api_requests.subtools.direct_dns_tools import get_DNS_then_use_it
+
+# generic
+from typing import Optional
+
+
+def solve_api_problems(
+    api_reply_object: ApiResponse,
+    get_or_post: str,
+    url: str,
+    payload: str = None,
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+) -> ApiResponse:
+
+    if api_reply_object.valid:
+        logger.debug(f"[API SOLVER] The API call worked, so there's no reason to try to solve it.")
+        return api_reply_object
+
+    reason_for_error = api_reply_object.error_type
+
+    cant_be_solved = [ErrorType.QUAD9_DNS_RESOLUTION, ErrorType.UNKNOWN, ErrorType.NO_INTERNET]
+
+    if reason_for_error in cant_be_solved:
+        logger.debug(f"[API SOLVER] This can't be solved if the reason is {reason_for_error}")
+        return api_reply_object
+    
+    elif reason_for_error == ErrorType.DEVELOPER_ERROR:
+        logger.error(f"[API SOLVER] The developer made an error {api_reply_object.message}")
+        return api_reply_object
+
+    elif reason_for_error == ErrorType.DNS_RESOLUTION:
+        logger.debug(f"[API SOLVER] We're solving a DNS resolution error {api_reply_object.message}")
+        if api_reply_object.tor:
+            dns_fix = get_DNS_then_use_it(get_or_post, url, payload, api_reply_object.ip_address, connection_observer, client_observer)
+            return dns_fix
+        else:
+            logger.error(f"[API SOLVER] We have NOT yet setup clearweb DNS solutions")
+            return api_reply_object
+
+    elif reason_for_error == ErrorType.TOR_NOT_WORKING:
+        api_reply_object.ask_clearweb = True
+        return api_reply_object
+
+    elif reason_for_error == ErrorType.NETWORK_ERROR:
+        if api_reply_object.tor:
+            api_reply_object.ask_clearweb = True
+        return api_reply_object
+
+    else:
+        logger.error(f"[API SOLVER] Unknown reason {reason_for_error} with message: {api_reply_object.message}")
+        return api_reply_object

core/services/networking/api_requests/subtools/custom_dns.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import requests
+
+# errors
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+from urllib.parse import urlparse, urlunparse
+
+def prep_url_with_known_ip(known_ip: str, original_url: str):
+    """
+    Purpose:
+        Prepares URLs for custom DNS checking
+
+    Rank:
+        Helper
+    
+    Called by:
+        step2_execute's _execute_tor_request
+
+    Raises Errors?
+        False
+    """
+
+    # Parse the original URL to extract the hostname
+    parsed = urlparse(original_url)
+    original_hostname = parsed.hostname
+
+    # Reconstruct the URL with the IP instead of the domain
+    url_with_ip = original_url.replace(parsed.hostname, known_ip)
+
+    return url_with_ip, original_hostname
+
+
+def load_custom_dns_resolver():
+    from requests.adapters import HTTPAdapter
+    from urllib3.util.ssl_ import create_urllib3_context
+    import ssl
+    import certifi
+
+    class CustomHostnameAdapter(HTTPAdapter):
+        """
+        Purpose:
+            If Tor (or ANY) DNS server is blocked, we can go directly to the IP,
+            and use this module to verify the SSL against local certs.
+
+        Method:
+            Adapter that connects to an IP but verifies the SSL cert against the original hostname (via SNI).
+            then uses that with the proxy, essentially intercepting/modifying the proxy behavior
+
+        Rank:
+            Coordinator for the SSL verification Task,
+            But the caller is orchestrating the actual API calls.
+
+        Called by:
+            step2_execute's _execute_tor_request
+
+        Raises Errors?
+            False
+        """
+        def __init__(self, original_hostname, *args, **kwargs):
+            logger.debug(f"\n[ADAPTER INIT] Creating CustomHostnameAdapter")
+            logger.debug(f"  original_hostname: {original_hostname}")
+            self.original_hostname = original_hostname
+            super().__init__(*args, **kwargs)
+            logger.debug(f"[ADAPTER INIT] ✓ Adapter initialized")
+
+        def init_poolmanager(self, *args, **kwargs):
+            logger.debug(f"\n[POOLMANAGER INIT] Creating standard pool manager (for direct connections)")
+            ctx = create_urllib3_context()
+            ctx.check_hostname = True
+            ctx.verify_mode = ssl.CERT_REQUIRED
+            ctx.load_verify_locations(certifi.where())
+            
+            kwargs['ssl_context'] = ctx
+            logger.debug(f"  ✓ Standard pool manager configured")
+            return super().init_poolmanager(*args, **kwargs)
+
+        def proxy_manager_for(self, proxy, **proxy_kwargs):
+            """
+            This is called when making requests through a proxy.
+            This is where we inject the custom SSL context.
+            """
+            logger.debug(f"\n[PROXY MANAGER] Creating proxy manager")
+            logger.debug(f"  proxy: {proxy}")
+            logger.debug(f"  original_hostname (for verification): {self.original_hostname}")
+            
+            ctx = create_urllib3_context()
+            ctx.check_hostname = True
+            ctx.verify_mode = ssl.CERT_REQUIRED
+            ctx.load_verify_locations(certifi.where())
+            
+            # KEY: Set server_hostname on the context for SNI
+            logger.debug(f"  Setting server_hostname for SNI: {self.original_hostname}")
+            
+            proxy_kwargs['ssl_context'] = ctx
+            proxy_kwargs['server_hostname'] = self.original_hostname
+            
+            logger.debug(f"  ✓ Proxy manager will verify cert against: {self.original_hostname}")
+            
+            result = super().proxy_manager_for(proxy, **proxy_kwargs)
+            logger.debug(f"  ✓ Proxy manager created successfully")
+            return result
+
+    return CustomHostnameAdapter
+

core/services/networking/api_requests/subtools/custom_httpx_dns_resolver.py

@@ -0,0 +1,46 @@
+import json
+import httpx
+import ssl
+import certifi
+from unittest.mock import patch
+import httpcore
+
+def create_httpx_client_with_custom_dns(hostname_to_ip_map: dict, custom_proxy: str):
+    """
+    Purpose:
+        Creates an httpx.Client that connects to IPs while verifying certificates
+        against the original hostnames (via SNI).
+    """
+    
+    ssl_context = ssl.create_default_context(cafile=certifi.where())
+    ssl_context.check_hostname = True
+    ssl_context.verify_mode = ssl.CERT_REQUIRED
+    
+    # Store the original wrap_socket method
+    original_wrap_socket = ssl.SSLContext.wrap_socket
+    
+    def patched_wrap_socket(self, sock, *args, **kwargs):
+        # Inject server_hostname for IPs in our map
+        for hostname, ip in hostname_to_ip_map.items():
+            try:
+                peer_addr = sock.getpeername()
+                if peer_addr[0] == ip:
+                    kwargs['server_hostname'] = hostname
+                    break
+            except OSError:
+                pass
+        return original_wrap_socket(self, sock, *args, **kwargs)
+    
+    # Apply the patch PERMANENTLY
+    ssl.SSLContext.wrap_socket = patched_wrap_socket
+    
+    # Build mounts
+    transport = httpx.HTTPTransport(verify=ssl_context, http2=True)
+    mounts = {}
+    for hostname, ip in hostname_to_ip_map.items():
+        mounts[f"https://{hostname}"] = transport
+        mounts[f"https://{ip}"] = transport
+    
+    # Create and return client
+    client = httpx.Client(mounts=mounts, http2=True, proxy=custom_proxy)
+    return client

core/services/networking/api_requests/subtools/direct_dns_tools.py

@@ -0,0 +1,346 @@
+from core.errors.logger import logger
+from core.Constants import Constants
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.subtools.custom_httpx_dns_resolver import create_httpx_client_with_custom_dns
+from core.services.networking.api_requests.subtools.extract_domain import extract_domain, swap_domain_for_ip
+
+# tor
+from essentials.modules.TorModule import TorModule
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+from core.observers.ClientObserver import ClientObserver
+
+# pip install pip install httpx[http2]
+# pip install httpx[socks]
+import httpx
+import dns.message
+import dns.name
+import dns.rdatatype
+import base64
+
+import json
+from typing import Optional
+
+
+def get_DNS_then_use_it(
+    get_or_post: str,
+    full_url: str,
+    payload: str = None,
+    ip_address: str = None,
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+) -> ApiResponse:
+
+    if not full_url:
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.DEVELOPER_ERROR,
+            tor=True,
+            message="Dev Error! Invalid Inputs to Function"
+        )
+
+    # ================= Setup Tor ===============
+    if client_observer:
+        status_update = "Evading Tor DNS block..."
+        client_observer.notify('synchronizing', status_update)
+
+    port_number = None
+    tor_module = None
+    client = None
+
+    try:
+        port_number = ConnectionService.get_random_available_port_number()
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+
+        tor_proxy = f"socks5h://127.0.0.1:{port_number}"
+
+        logger.debug(f"[DNS TOOLS] Tor Proxy Setup on Port {port_number}")
+
+        # ================= Do Quad9 Query ===============
+        domain = extract_domain(full_url)
+        logger.debug(f"[DNS TOOLS] We'll be doing a DNS lookup to the domain {domain}..")
+
+        if not ip_address:
+            ip_address = quad9_proxy_dns_lookup(domain, tor_proxy)
+
+            if not ip_address:
+                return ApiResponse(
+                    valid=False,
+                    error_type=ErrorType.QUAD9_DNS_RESOLUTION,
+                    tor=True,
+                    message=f"Cannot resolve {domain} via Tor or Quad9"
+                )
+            
+            logger.debug(f"[DNS TOOLS] We got an IP address from Quad9 for {domain} of: {ip_address}")
+
+        # ================= Use Custom DNS ===============
+        client = create_httpx_client_with_custom_dns({domain: ip_address}, tor_proxy)
+        url_with_ip = swap_domain_for_ip(domain, full_url, ip_address)
+        print(f"url_with_ip is {url_with_ip}")
+
+        print(f"we are doing a {get_or_post} request")
+
+        if get_or_post == "get":
+             response = client.get(url_with_ip, headers={"Host": domain})
+        elif get_or_post == "post":
+            if payload:
+                response = client.post(url_with_ip, json=payload, headers={"Host": domain})
+            else:
+                return ApiResponse(
+                    valid=False,
+                    error_type=ErrorType.DEVELOPER_ERROR,
+                    ip_address=ip_address,
+                    tor=True,
+                    message=f"Dev Error! No Payload for POST request"
+                )
+        else:
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.DEVELOPER_ERROR,
+                ip_address=ip_address,
+                tor=True,
+                message=f"Dev Error! Invalid Input of get_or_post: {get_or_post}"
+            )
+
+        # print(response.json())
+
+        if not response:
+            error_msg = f"Could not connect to {domain} via {ip_address}"
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.NETWORK_ERROR,
+                ip_address=ip_address,
+                tor=True,
+                message=error_msg
+            )
+
+        response_status_code = response.status_code
+
+        if 200 <= response_status_code < 300:
+            try:
+                data = response.json()
+                return ApiResponse(
+                    valid=True,
+                    error_type=ErrorType.TOR_DNS_BLOCKED,
+                    ip_address=ip_address,
+                    tor=True,
+                    data=data
+                )
+            except ValueError:
+                print("We could not get the value from it.")
+                return ApiResponse(
+                    valid=True,
+                    error_type=ErrorType.SERVER_ERROR,
+                    ip_address=ip_address,                    
+                    tor=True,
+                    data=response.text
+                )
+        else:
+            error_msg = f"Server replied with {response_status_code} at {ip_address}"
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.SERVER_ERROR,
+                ip_address=ip_address,                    
+                tor=True,
+                message=error_msg
+            )
+
+    except Exception as e:
+        error_msg = f"Tor failed: {type(e).__name__}: {e}"
+        logger.error(f"[DNS] {error_msg}")
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.NETWORK_ERROR,
+            tor=True,
+            message=error_msg
+        )
+
+    finally:
+        if client:
+            client.close()
+
+        if tor_module and port_number:
+            try:
+                tor_module.destroy_session(port_number)
+            except Exception as e:
+                logger.warning(f"[DNS] Error destroying Tor session: {e}")
+            
+
+
+
+# This version is passed a "generic" proxy with external Tor:
+def quad9_proxy_dns_lookup(
+    domain: str,
+    custom_proxy: str,
+    timeout: int = 10,
+) -> str:
+
+    logger.debug("Doing a Proxy Quad9 DNS lookup")
+
+    # ================= Use Proxy for Quad9 ===============
+
+    qname = dns.name.from_text(domain)
+    query = dns.message.make_query(qname, dns.rdatatype.A)
+    wire_format = query.to_wire()
+    encoded = base64.urlsafe_b64encode(wire_format).decode().rstrip('=')
+
+    # Use httpx with HTTP/2
+    with httpx.Client(http2=True, proxy=custom_proxy, timeout=timeout) as client:
+        response = client.get(
+            "https://dns.quad9.net/dns-query",
+            params={"dns": encoded},
+            headers={"Accept": "application/dns-message"}
+        )
+        
+        logger.debug(f"[QUAD9-DNS] Status: {response.status_code}")
+        logger.debug(f"[QUAD9-DNS] Headers: {response.headers}")
+        
+        if response.status_code == 200:
+            # ================= Filter Results ===============
+            result = dns.message.from_wire(response.content)
+            logger.debug(f"[QUAD9-DNS] Quad9 Returned {result}")
+
+            ip_address = None
+            for rrset in result.answer:
+                for rr in rrset:
+                    if rr.rdtype == dns.rdatatype.A:
+                        ip_address = rr.address  # Get just the IP address
+                        break
+                if ip_address:
+                    break
+
+            return ip_address
+        else:
+            logger.error(f"Quad9's Invalid Response Body: {response.text[:500]}")
+            if client_observer:
+                status_update = f"Quad9 Failed to Resolve {domain}"
+                client_observer.notify('synchronizing', status_update)
+            return False
+
+
+
+# This version does Tor & the lookup same time.
+def quad9_tor_dns_lookup(
+    domain: str,
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+    timeout: int = 10,
+) -> str:
+
+    import httpx
+    import dns.message
+    import dns.name
+    import dns.rdatatype
+    import base64
+
+    logger.debug("Doing a Tor Quad9 DNS lookup")
+
+    # ================= Setup Tor ===============
+    if client_observer:
+        status_update = "Initializing Tor connection..."
+        client_observer.notify('synchronizing', status_update)
+
+    port_number = None
+    tor_module = None
+
+    try:
+        port_number = ConnectionService.get_random_available_port_number()
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+
+        tor_proxy = f"socks5h://127.0.0.1:{port_number}"
+
+        logger.debug(f"[QUAD9-DNS] Tor Proxy Setup on Port {port_number}")
+
+        # ================= Use Proxy for Quad9 ===============
+
+        qname = dns.name.from_text(domain)
+        query = dns.message.make_query(qname, dns.rdatatype.A)
+        wire_format = query.to_wire()
+        encoded = base64.urlsafe_b64encode(wire_format).decode().rstrip('=')
+
+        # Use httpx with HTTP/2
+        with httpx.Client(http2=True, proxy=tor_proxy, timeout=timeout) as client:
+            response = client.get(
+                "https://dns.quad9.net/dns-query",
+                params={"dns": encoded},
+                headers={"Accept": "application/dns-message"}
+            )
+            
+            logger.debug(f"[QUAD9-DNS] Status: {response.status_code}")
+            logger.debug(f"[QUAD9-DNS] Headers: {response.headers}")
+            
+            if response.status_code == 200:
+                # ================= Filter Results ===============
+                result = dns.message.from_wire(response.content)
+                logger.debug(f"[QUAD9-DNS] Quad9 Returned {result}")
+
+                ip_address = None
+                for rrset in result.answer:
+                    for rr in rrset:
+                        if rr.rdtype == dns.rdatatype.A:
+                            ip_address = rr.address  # Get just the IP address
+                            break
+                    if ip_address:
+                        break
+
+                return ip_address
+            else:
+                logger.error(f"Quad9's Invalid Response Body: {response.text[:500]}")
+                if client_observer:
+                    status_update = f"Quad9 Failed to Resolve {domain}"
+                    client_observer.notify('synchronizing', status_update)
+                return False
+
+    except Exception as e:
+        logger.debug(f"[QUAD9-DNS] Tor Quad9 GET failed: {type(e).__name__}: {e}")
+        return None
+
+    finally:
+        if tor_module and port_number:
+            try:
+                tor_module.destroy_session(port_number)
+            except Exception as e:
+                logger.warning(f"[QUAD9-DNS] Error destroying Tor session: {e}")
+    
+    return result
+
+
+
+def quad9_clearweb_dns_lookup(domain: str) -> str:
+    import httpx
+    import dns.message
+    import dns.name
+    import dns.rdatatype
+    import base64
+
+    logger.debug("Doing a Clearweb Quad9 DNS lookup")
+
+    qname = dns.name.from_text(domain)
+    query = dns.message.make_query(qname, dns.rdatatype.A)
+    wire_format = query.to_wire()
+    encoded = base64.urlsafe_b64encode(wire_format).decode().rstrip('=')
+
+    try:
+        # Use httpx with HTTP/2
+        with httpx.Client(http2=True) as client:
+            response = client.get(
+                "https://dns.quad9.net/dns-query",
+                params={"dns": encoded},
+                headers={"Accept": "application/dns-message"}
+            )
+            
+            logger.debug(f"Status: {response.status_code}")
+            logger.debug(f"Headers: {response.headers}")
+            
+            if response.status_code == 200:
+                result = dns.message.from_wire(response.content)
+                logger.debug(f"Quad9 Returned {result}")
+                return result
+            else:
+                logger.error(f"Quad9 Body: {response.text[:500]}")
+                return False
+    except:
+        logger.error("Regular Quad9 request failed.")
+        return None

core/services/networking/api_requests/subtools/dns_evaluation.py

@@ -0,0 +1,133 @@
+# for tor:
+from essentials.modules.TorModule import TorModule
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+
+from core.observers.ClientObserver import ClientObserver
+from core.Constants import Constants
+from core.services.networking.api_requests.subtools.extract_domain import extract_domain
+from core.errors.logger import logger
+from core.errors.exceptions import *
+
+# import dns.resolver
+# import dns.exception
+# import dns.rdatatype
+from typing import Optional
+import os
+import socket
+import socks
+
+import subprocess
+
+
+"""
+Check if there's a DNS problem with a domain.
+Returns True if DNS fails (problem exists), False if DNS resolves successfully.
+"""
+
+def is_dns_problem(url: str) -> bool:
+    """Regular DNS check for non-Tor domains"""
+    domain = extract_domain(url)
+    try:
+        socket.gethostbyname(domain)
+        return False  # DNS resolved successfully
+    except socket.gaierror:
+        return True  # DNS resolution failed
+    except Exception:
+        return True
+
+
+
+def dns_works_via_tor(domain, connection_observer, client_observer=None):
+    """
+    Purpose:
+        Check if domain's resolver is blocking Tor by querying via proxychains.
+
+    Method:
+        Dynamically configures proxychains with the given port.
+        Uses this to query the DNS via dig.
+
+    Dependencies:
+        Proxychains being installed.
+
+    Called by:
+        step4_error_classifier's classify_request_error
+    
+    Raises Errors?
+        False
+    """
+
+    port_number = ConnectionService.get_random_available_port_number()
+    logger.debug(f"[DNS PROBLEM OF TOR SERVICE] Using Tor on port number {port_number}")
+    if client_observer is not None:
+        client_observer.notify('synchronizing', f"Using Tor on Port {port_number}..")
+
+    try:
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+        logger.debug(f"[DNS PROBLEM OF TOR SERVICE] Tor Started a Session")
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Initialized Tor..")
+
+    except TorServiceInitializationError as e:
+        error_msg = f"[DNS PROBLEM OF TOR SERVICE] Tor failed to start: {e}"
+        tor_module.destroy_session(port_number)
+        logger.error(error_msg, exc_info=True)
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Failed to Initialize")
+
+    proxies = {
+        "http": f"socks5h://127.0.0.1:{port_number}",
+        "https": f"socks5h://127.0.0.1:{port_number}",
+    }
+    config_path = f"{Constants.HV_CONFIG_HOME}/proxychains_dynamic.conf"
+    
+    try:
+        # Write proxychains config on the fly
+        os.makedirs(os.path.dirname(config_path), exist_ok=True)
+        with open(config_path, 'w') as f:
+            f.write(f"""
+strict_chain
+proxy_dns
+tcp_read_time_out 10000
+tcp_connect_time_out 10000
+
+[ProxyList]
+socks5 127.0.0.1 {port_number}
+        """)
+            f.flush()
+            os.fsync(f.fileno())  # Force write to disk
+
+        print("running command...")
+        # Run dig through proxychains with the dynamic config
+        try:
+            result = subprocess.run(
+                ['proxychains4', '-f', config_path, 'dig', domain, '+short'],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+            logger.debug(f"result: {result}")
+        except:
+            logger.error(f"[DNS PROBLEM] Proxychains not installed on this computer")
+            return False
+
+        output = result.stdout.strip()
+        logger.debug(f"output of stripping the IP out of the result: {output}")
+        
+        if not output or result.returncode != 0:
+            logger.debug(f"[DNS PROBLEM] {domain}: No response")
+            return False
+        
+        logger.debug(f"[DNS OK] {domain}: {output}")
+        return output
+
+    except subprocess.TimeoutExpired:
+        logger.debug(f"[DNS PROBLEM] {domain}: Timeout")
+        return False
+    except Exception as e:
+        logger.debug(f"[DNS PROBLEM] {domain}: {type(e).__name__}: {e}")
+        return False
+    finally:
+        tor_module.destroy_session(port_number)
+

core/services/networking/api_requests/subtools/extract_domain.py

@@ -0,0 +1,54 @@
+# used in the 2nd function, not the first,
+import re
+
+def extract_domain(url: str) -> str:
+    """
+    Purpose:
+        Extract domain from URL, stripping protocol and path.
+        If exact TLD match isn't found, tries to return the best guess.
+    """
+    common_domains = [".com", ".net", ".org", ".is"]
+
+    # Remove protocol
+    if url.startswith("https://"):
+        url = url[8:]
+    elif url.startswith("http://"):
+        url = url[7:]
+
+    # Find the earliest TLD in the string
+    earliest_pos = len(url)
+    earliest_tld_len = 0
+
+    for tld in common_domains:
+        pos = url.find(tld)
+        if pos != -1 and pos < earliest_pos:
+            earliest_pos = pos
+            earliest_tld_len = len(tld)
+
+    # If a valid TLD was found, return domain + TLD
+    if earliest_tld_len > 0:
+        return url[: earliest_pos + earliest_tld_len]
+
+    # Fallback: return everything before the first slash (best guess)
+    return url.split("/")[0]
+
+
+def swap_domain_for_ip(raw_domain: str, full_url: str, ip_address: str):
+    """
+    Purpose:
+        Switch the real domain for the raw IP address,
+        But keep the rest of the domain intact.
+
+    Rank:
+        Helper
+    
+    Raises Errors?
+        False
+    """
+    # Escape the domain in case it has regex special characters (like dots)
+    escaped_domain = re.escape(raw_domain)
+    
+    # Replace only the first instance of the domain with the IP
+    url_with_ip = re.sub(escaped_domain, ip_address, full_url, count=1)
+    
+    return url_with_ip

core/services/networking/api_requests/subtools/replace_http_with_https.py

@@ -0,0 +1,7 @@
+
+
+
+def replace_http_with_https(url):
+    if url.startswith("http://"):
+        return url.replace("http://", "https://", 1)
+    return url

core/services/networking/evaluate_networking_problem.py

@@ -1,76 +0,0 @@
-from __future__ import annotations
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from essentials.observers.ConnectionObserver import ConnectionObserver
-# services
-from core.services.networking.internet_test import do_we_have_internet
-from core.services.networking.is_dns_problem import is_dns_problem, is_dns_problem_via_tor
-from core.services.networking.extract_domain import extract_domain
-from core.services.networking.is_tor_working import is_tor_working
-# errors
-from core.errors.exceptions import *
-from core.errors.logger import logger
-
-#######################################################
-############### START : REGULAR SYSTEM  ###############
-#######################################################
-
-
-def evaluate_regular_networking_problem(
-    url: str,
-    connection_observer: ConnectionObserver,
-    port_number: int | None = None,
-    proxies: dict | None = None,
-) -> dict:
-
-    domain_only = extract_domain(url)
-
-    if not do_we_have_internet():
-        return {"valid": False, "error_code": "no_internet"}
-
-    if is_dns_problem(domain_only):
-        logger.debug("This is a DNS issue.")
-        return {"valid": False, "error_code": "dns_issue"}
-
-    return {"valid": False, "error_code": "unknown"}
-    #######################################################
-    ############### END: REGULAR SYSTEM  ###############
-    #######################################################
-
-    #######################################################
-    ###############      START: TOR         ###############
-    #######################################################
-
-
-def evaluate_tor_networking_problem(
-    url: str,
-    connection_observer: ConnectionObserver,
-) -> dict:
-
-    domain_only = extract_domain(url)
-    logger.debug(f"Evaluating a networking problem for a Tor connection")
-
-    logger.debug("checking if Tor works against the official Tor Project API..")
-    if not is_tor_working(connection_observer):
-        return {"valid": False, "error_code": "tor_issue"}
-    else:
-        tor_fine = f"While there were connection issues, but Tor is working fine!"
-        logger.error(tor_fine, exc_info=True)
-        logger.debug(tor_fine)
-
-    logger.debug("Check the original DNS via Tor...")
-
-    if is_dns_problem_via_tor(domain_only, connection_observer):
-        return {"valid": False, "error_code": "dns_issue"}
-    else:
-        dns_fine = f"There were connection issues, but the DNS for {domain_only} via Tor is working fine!"
-        logger.error(dns_fine, exc_info=True)
-        logger.debug(dns_fine)
-
-    # can't solve it:
-    return {"valid": False, "error_code": "unknown"}
-
-    #######################################################
-    ###############      END: TOR           ###############
-    #######################################################

core/services/networking/extract_domain.py

@@ -1,31 +0,0 @@
-"""
-Extract domain from URL, stripping protocol and path.
-If exact TLD match isn't found, tries to return the best guess.
-"""
-
-
-def extract_domain(url: str) -> str:
-    common_domains = [".com", ".net", ".org", ".is"]
-
-    # Remove protocol
-    if url.startswith("https://"):
-        url = url[8:]
-    elif url.startswith("http://"):
-        url = url[7:]
-
-    # Find the earliest TLD in the string
-    earliest_pos = len(url)
-    earliest_tld_len = 0
-
-    for tld in common_domains:
-        pos = url.find(tld)
-        if pos != -1 and pos < earliest_pos:
-            earliest_pos = pos
-            earliest_tld_len = len(tld)
-
-    # If a valid TLD was found, return domain + TLD
-    if earliest_tld_len > 0:
-        return url[: earliest_pos + earliest_tld_len]
-
-    # Fallback: return everything before the first slash (best guess)
-    return url.split("/")[0]

core/services/networking/general_connection_tools/config_tools.py

@@ -0,0 +1,35 @@
+from core.Constants import Constants
+import os
+
+def extract_endpoint_ip(profile_id: str) -> str | bool:
+    try:
+        profile_path = Constants.HV_PROFILE_CONFIG_HOME + f'/{profile_id}'
+        wg_conf_path = f'{profile_path}/wg.conf.bak'
+        print(f"wg_conf_path is {wg_conf_path}")
+        if not os.path.exists(wg_conf_path):
+            return None
+        with open(wg_conf_path, 'r') as f:
+            content = f.read()
+        for line in content.split('\n'):
+            if line.strip().startswith('Endpoint = '):
+                endpoint = line.strip().split(' = ')[1]
+                return endpoint.split(':')[0]
+        return None
+    except Exception:
+        return None
+
+def extract_dns(profile_id: str) -> str | bool:
+    try:
+        profile_path = Constants.HV_PROFILE_CONFIG_HOME + f'/{profile_id}'
+        wg_conf_path = f'{profile_path}/wg.conf.bak'
+        print(f"wg_conf_path is {wg_conf_path}")
+        if not os.path.exists(wg_conf_path):
+            return None
+        with open(wg_conf_path, 'r') as f:
+            content = f.read()
+        for line in content.split('\n'):
+            if line.strip().startswith('DNS = '):
+                return line.strip().split(' = ')[1]
+        return None
+    except Exception:
+        return None

core/services/networking/general_connection_tools/connection_enable.py

@@ -0,0 +1,134 @@
+from core.services.networking.general_connection_tools.testing_evaluating import system_uses_wireguard_interface
+from core.services.networking.systemwide.systemwide_wireguard import establish_system_connection, terminate_system_connection
+from core.services.keys_and_verifications.wireguard_keys import register_wireguard_session
+from core.services.subscriptions.subscriptions import activate_subscription
+
+# If refactored to enums:
+# from core.models.session.SessionConnection import SessionConnectionTypes
+# from core.models.system.SystemConnection import SystemConnectionTypes
+# wireguard_types = [SystemConnectionTypes.WIREGUARD, SessionConnectionTypes.WIREGUARD]
+
+
+from core.errors.logger import logger
+from core.errors.exceptions import *
+from typing import Union, Optional, Callable
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+from core.Errors import ConnectionTerminationError, InvalidSubscriptionError
+from core.services.WebServiceApiService import WebServiceApiService
+from core.controllers.ConnectionController import ConnectionController
+from core.models.BaseProfile import ProfileType
+from core.observers.ConnectionObserver import ConnectionObserver
+from core.controllers.SystemStateController import SystemStateController
+
+def establish_connection(
+    profile: Union[SessionProfile, SystemProfile], 
+    ignore: tuple[type[Exception]] = (), 
+    connection_observer: Optional[ConnectionObserver] = None,
+):
+    """Establish a connection for the given profile."""
+    
+    logger.info(f"[CONNECTION] Checking subscription..")
+    activate_subscription(profile, connection_observer)
+
+    logger.info(f"[CONNECTION] Checking proxy configuration..")
+    _ensure_proxy_configured(profile, connection_observer)
+
+    logger.info(f"[CONNECTION] Checking Wireguard configuration..")
+    _ensure_wireguard_configured(profile, connection_observer)
+    
+    establish_fn = {
+        ProfileType.SESSION: ConnectionController.establish_session_connection,
+        ProfileType.SYSTEM: establish_system_connection,
+    }[profile.type]
+    
+    return _establish_with_renegotiation(profile, establish_fn, ignore, connection_observer)
+
+
+def _ensure_proxy_configured(
+    profile: Union[SessionProfile, SystemProfile],
+    connection_observer: Optional[ConnectionObserver],
+):
+    """Setup proxy config if needed."""
+    if not profile.connection.needs_proxy_configuration():
+        return
+    
+    if not profile.has_proxy_configuration():
+        logger.info(f"[CONNECTION] You need a new proxy configuration")
+        activate_subscription(profile, connection_observer)
+
+        logger.info(f"[CONNECTION] Getting a proxy config from web service..")
+        proxy_config = ConnectionController.with_preferred_connection(
+            profile.subscription.billing_code, 
+            task=WebServiceApiService.get_proxy_configuration,
+            connection_observer=connection_observer
+        )
+        
+        if proxy_config is None:
+            raise InvalidSubscriptionError()
+        
+        profile.attach_proxy_configuration(proxy_config)
+
+
+def _ensure_wireguard_configured(
+    profile: Union[SessionProfile, SystemProfile],
+    connection_observer: Optional[ConnectionObserver],
+):
+    """Setup WireGuard config if needed."""
+
+    logger.info(f"[CONNECTION] Checking if this profile needs a wg config.  The profile.connection.code is {profile.connection.code}")
+    if profile.connection.code != "wireguard":
+        logger.info(f"[CONNECTION] This profile {profile.id} does NOT need a wireguard config")
+        return
+    
+    if profile.type == ProfileType.SYSTEM:
+        if system_uses_wireguard_interface() and SystemStateController.exists():
+            logger.info(f"[CONNECTION] There is a systemwide profile that is already is active.")
+            try:
+                terminate_system_connection()
+            except ConnectionTerminationError as e:
+                logger.error(f"[CONNECTION] Previous Systemwide Wireguard connection could not be disabled. {e}")
+
+    activate_subscription(profile, connection_observer)
+    logger.info(f"[CONNECTION] Checking if the profile already has a wg config..")
+    if not profile.has_wireguard_configuration():
+        logger.info(f"[CONNECTION] Profile does NOT have WG keys and it needs it.  Registering a NEW wg session..")
+        register_wireguard_session(profile, connection_observer=connection_observer)
+
+
+def _establish_with_renegotiation(
+    profile: Union[SessionProfile, SystemProfile],
+    establish_fn: Callable,
+    ignore: tuple[type[Exception]],
+    connection_observer: Optional[ConnectionObserver],
+):
+    """Attempt connection, renegotiate WireGuard once if needed."""
+    try:
+        logger.info(f"[CONNECTION] Connecting..")
+        return establish_fn(profile, ignore=ignore, connection_observer=connection_observer)
+    except ConnectionError:
+        if __should_renegotiate(profile):
+            logger.info(f"[CONNECTION] Renegotiating a new wg key session with API..")    
+            register_wireguard_session(profile, connection_observer=connection_observer)
+            return establish_fn(profile, ignore=ignore, connection_observer=connection_observer)
+        raise ConnectionError('The connection could not be established.')
+
+
+def __should_renegotiate(profile: Union[SessionProfile, SystemProfile]):
+
+    if not profile.has_subscription():
+        raise MissingSubscriptionError()
+
+    if profile.connection.code == "wireguard" and profile.has_wireguard_configuration():
+        if profile.subscription.has_been_activated():
+            return True
+    else:
+        return False
+
+
+
+# Enums
+# if profile.type == ProfileType.SYSTEM:
+#      print("This is a system profile")
+#  elif profile.type == ProfileType.SESSION:
+#      print("This is a session profile")

core/services/networking/general_connection_tools/testing_evaluating.py

@@ -0,0 +1,125 @@
+from core.Constants import Constants
+from core.Errors import CommandNotFoundError
+from core.observers.ConnectionObserver import ConnectionObserver
+from essentials.modules.TorModule import TorModule
+from typing import Optional
+import os
+import random
+import re
+import shutil
+import subprocess
+import sys
+import time
+
+
+def terminate_tor_connection():
+
+    tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+    tor_module.stop_service()
+
+
+def await_network_interface():
+
+    network_interface_is_activated = False
+
+    retry_interval = .5
+    maximum_number_of_attempts = 10
+    attempt = 0
+
+    while not network_interface_is_activated and attempt < maximum_number_of_attempts:
+
+        time.sleep(retry_interval)
+
+        network_interface_is_activated = system_uses_wireguard_interface()
+        attempt += 1
+
+    if not network_interface_is_activated:
+        raise ConnectionError('The network interface could not be activated.')
+
+
+def system_uses_wireguard_interface():
+
+    if shutil.which('ip') is None:
+        raise CommandNotFoundError('ip')
+
+    process = subprocess.Popen(('ip', 'route', 'get', '192.0.2.1'), stdout=subprocess.PIPE)
+    process_output = str(process.stdout.read())
+
+    return bool(re.search('dev wg', str(process_output)))
+
+
+def __test_connection(port_number: Optional[int] = None, timeout: float = 4.0):
+
+    request_urls = [Constants.PING_URL]
+    proxies = None
+
+    if os.environ.get('PING_URL') is None:
+
+        request_urls.extend([
+            'https://hc1.simplifiedprivacy.net',
+            'https://hc2.simplifiedprivacy.org',
+            'https://hc3.hydraveil.net'
+        ])
+
+        random.shuffle(request_urls)
+
+    if port_number is not None:
+        proxies = get_proxies(port_number)
+
+    for request_url in request_urls:
+
+        command = [
+            sys.executable, '-u', '-c', 'import requests, sys\n'
+            'try:\n'
+            f'    response = requests.get(\'{request_url}\', proxies={proxies}, timeout={timeout})\n'
+            '    response.raise_for_status(); print(response.text)\n'
+            'except requests.exceptions.RequestException:\n'
+            '    sys.exit(1)'
+        ]
+
+        try:
+
+            _response = subprocess.check_output(command, text=True, timeout=timeout)
+            return None
+
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+            pass
+
+    raise ConnectionError('The connection could not be established.')
+
+
+# Kept a duplicate in connection controller, because Tor uses it. and circular imports are an issue for now. Refactor later.
+def get_proxies(port_number: int):
+
+    return dict(
+        http=f'socks5h://127.0.0.1:{port_number}',
+        https=f'socks5h://127.0.0.1:{port_number}'
+    )
+
+
+def await_connection(port_number: Optional[int] = None, connection_observer: Optional[ConnectionObserver] = None):
+
+    if port_number is None:
+        await_network_interface()
+
+    for retry_count in range(Constants.MAX_CONNECTION_ATTEMPTS):
+
+        if connection_observer is not None:
+
+            connection_observer.notify('connecting', dict(
+                retry_interval=Constants.CONNECTION_RETRY_INTERVAL,
+                maximum_number_of_attempts=Constants.MAX_CONNECTION_ATTEMPTS,
+                attempt_count=retry_count + 1
+            ))
+
+        try:
+
+            __test_connection(port_number)
+            return
+
+        except ConnectionError:
+
+            time.sleep(Constants.CONNECTION_RETRY_INTERVAL)
+            retry_count += 1
+
+    raise ConnectionError('The connection could not be established.')

core/services/networking/is_dns_problem.py

@@ -1,56 +0,0 @@
-# for tor:
-from essentials.modules.TorModule import TorModule
-from essentials.observers.ConnectionObserver import ConnectionObserver
-from essentials.services.ConnectionService import ConnectionService
-import json, os
-
-# for both:
-from core.services.networking.extract_domain import extract_domain
-from core.errors.logger import logger
-import socket
-import socks
-
-"""
-Check if there's a DNS problem with a domain.
-Returns True if DNS fails (problem exists), False if DNS resolves successfully.
-"""
-
-
-def is_dns_problem(url: str) -> bool:
-    domain = extract_domain(url)
-    try:
-        socket.gethostbyname(domain)
-        return False  # DNS resolved successfully, no problem
-    except socket.gaierror:
-        return True  # DNS resolution failed, problem exists
-    except Exception:
-        return True  # Any other error is treated as a DNS problem
-
-
-def is_dns_problem_via_tor(
-    domain: str,
-    connection_observer: ConnectionObserver | None = None,
-) -> bool:
-    logger.debug("We've triggered EVALUATING IF it's a DNS problem via Tor")
-
-    port_number = ConnectionService.get_random_available_port_number()
-    tor_module = TorModule(os.path.expanduser("~/sp-tor-test"))
-    tor_module.create_session(port_number, connection_observer)
-
-    # Set up SOCKS5 proxy using the same port as the existing Tor proxy,
-    socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", port_number)
-    socket.socket = socks.socksocket
-
-    # DNS query check
-    try:
-        logger.debug("checking dns via socket..")
-        socket.gethostbyname(domain)
-
-        tor_module.destroy_session(port_number)
-
-        return False  # DNS resolved successfully, no problem
-
-    except socket.gaierror:
-        return True  # DNS resolution failed, problem exists
-    except Exception:
-        return True  # Any other error is treated as a DNS problem

core/services/networking/legacy/get_data_from_server.py

@@ -1,8 +1,13 @@
 from __future__ import annotations
 from typing import TYPE_CHECKING
 
-if TYPE_CHECKING:
+# if TYPE_CHECKING:
+from core.observers.ClientObserver import ClientObserver
 from essentials.observers.ConnectionObserver import ConnectionObserver
+
+from core.observers.ClientObserver import ClientObserver
+from core.services.networking.evaluate_networking_problem import evaluate_tor_networking_problem
+
 # services
 from core.services.networking.get_connection_type import get_connection_type
 from core.services.networking.use_tor import tor_get_request
@@ -14,13 +19,13 @@ import traceback
 
 
 # Generic GET request to an endpoint, filtered by the user's preference of connection type (Tor or Not)
-def get_data_from_server(url: str, connection_observer: ConnectionObserver) -> dict:
+def get_data_from_server(url: str, connection_observer: ConnectionObserver, client_observer: Optional[ClientObserver] = None) -> dict:
     try:
         connection_type = get_connection_type()
         logger.debug(f"connection type is: {connection_type}")
 
         if connection_type == "tor":
-            response = tor_get_request(url, connection_observer)
+            response = tor_get_request(url, connection_observer, client_observer)
 
         else:
             response = regular_get_request(url, connection_observer)

core/services/networking/systemwide/dns.py

@@ -0,0 +1,68 @@
+# custom
+from core.services.networking.systemwide.systemwide_errors import systemwide_hell_raiser
+from core.utils.basic_operations.wrap_with import wrap_with
+from core.utils.basic_operations.run_generic_command import run_generic_command
+from core.Constants import Constants
+from core.models.Result import Result, ResultError
+from core.errors.logger import logger
+
+DNS_SCRIPT = f"/opt/hydra-veil/dns"
+
+# requires script to have sudo
+@wrap_with(systemwide_hell_raiser)
+def set_on(network_interface: str, dns_should_be: str) -> Result:
+    command = ["sudo", DNS_SCRIPT, "set", network_interface, dns_should_be]
+    human_readable_goal = "Set DNS"
+    return run_generic_command(command, human_readable_goal)
+
+
+# requires script to have sudo
+@wrap_with(systemwide_hell_raiser)
+def revert() -> Result:
+    command = ["sudo", DNS_SCRIPT, "revert"]
+    human_readable_goal = "DNS revert"
+    return run_generic_command(command, human_readable_goal)
+
+
+# does NOT need sudo
+def is_systemd_enabled() -> Result:
+    command = ["systemctl", "is-enabled", "systemd-resolved"]
+    human_readable_goal = "Find out if SystemD is enabled"
+    result = run_generic_command(command, human_readable_goal, timeout=5)
+    if result.valid:
+        if "enabled" in result.data:
+            return Result(valid=True)
+        else:
+            logger.error("Systemctl is not even enabled.")
+            return Result(valid=False) # this would be true in the original object just because the command ran.
+    else:
+        return result # send the error itself.
+
+
+# does NOT need sudo
+def get_resolvctl_data(which_command: str) -> Result:
+    command = ['resolvectl', which_command]
+    human_readable_goal = "Get Resolvctl Data"
+    return run_generic_command(command, human_readable_goal, timeout=5)
+
+
+# These are the raw Linux systemctl commands:
+"""
+# is it enabled?
+systemctl is-enabled systemd-resolved
+
+# socket exists
+-S /run/systemd/resolve/io.systemd.Resolve
+
+# Shows DNS servers per interface. Should spit back what was put in it for that interface.
+resolvectl dns
+
+# Shows search domains per interface. should show ~. for that interface
+resolvectl domain
+
+# Shows which is the default. yes/no for each
+resolvectl default-route
+
+# Test DNS:
+resolvectl query example.com
+"""

core/services/networking/systemwide/dns_tools/SearchResult.py

@@ -0,0 +1,33 @@
+
+
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional, Any
+
+"""
+Purpose:
+    Searching for values in strings from the system's CLI,
+    In it's current use context, systemctl
+"""
+
+class SearchError(Enum):
+    """Classified error categories."""
+    SUCCESS = "success"
+    NO_INPUT = "no_input"
+    INVALID_INPUT = "invalid_input"
+    TARGET_MISSING = "target_missing"
+    TARGET_NO_VALUE = "target_no_value"
+    VALUE_DOESNT_MATCH = "value_doesnt_match"
+    COMPETITORS_MATCH = "competitors_match"
+    COMMAND_DIDNT_FINISH = "command_didnt_finish"
+    UNKNOWN = "unknown"
+
+@dataclass
+class SearchResult():
+    valid: bool
+    error_type: SearchError = SearchError.SUCCESS
+    which_command_was_run: Optional[str] = None
+    target_value: str = None
+    competitors_count: Optional[int] = 0
+    competitors_list: Optional[Any] = None
+    message: Optional[str] = None

core/services/networking/systemwide/dns_tools/process_dns_result.py

@@ -0,0 +1,124 @@
+from core.services.networking.systemwide.dns_tools.SearchResult import SearchResult, SearchError
+from core.services.networking.systemwide.dns_tools.search_resolv_output import evaluating_resolv_output
+from core.services.networking.systemwide import dns
+from core.models.Result import Result, ResultError
+from core.errors.logger import logger
+
+# This is for the future to potentially give errors to the end user:
+def turn_command_into_readable_string(which_command) -> str:
+    if which_command == "default-route":
+        human_readable_string = "The Systemctl's DNS Default Route"
+    elif which_command == "domain":
+        human_readable_string = "The Systemctl's DNS Domain, for which domains are routed to it"
+    elif which_command == "dns":
+        human_readable_string = "The Systemctl's Default DNS IP address, which should match the VPN configuration file"
+    else:
+        human_readable_string = "Unknown error"
+    return human_readable_string
+
+
+def process_ONE_dns_result(which_command: str, target_interface: str) -> SearchResult:
+    # get the output
+    resolvctl_output = dns.get_resolvctl_data(which_command)
+
+    if not resolvctl_output.valid:
+        return SearchResult(valid=False, error_type=SearchError.COMMAND_DIDNT_FINISH, message=resolvctl_output.message)
+
+    # search that output:
+    return evaluating_resolv_output(which_command=which_command, target_interface=target_interface, resolv_output=resolvctl_output.data)
+
+
+def process_ALL_dns_results(target_interface: str) -> Result:
+
+    list_of_commands = ["domain", "default-route", "dns"]
+    list_of_results_as_objects = []
+    list_of_results_as_binary = []
+
+    # ================= START LOOP =====================
+    for each_command in list_of_commands:
+        each_SearchResult = process_ONE_dns_result(
+            which_command=each_command,
+            target_interface=target_interface,
+        )
+        # add the command to the SearchResult here, so it's only declared once,
+        each_SearchResult.which_command_was_run = each_command
+
+        list_of_results_as_objects.append(each_SearchResult)
+        list_of_results_as_binary.append(each_SearchResult.valid)
+
+    # ================= END LOOP =====================
+    return list_of_results_as_objects, list_of_results_as_binary
+
+
+def get_quantity_of_false(list_of_results_as_binary: list) -> int:
+    quantity_of_false = 0
+    for each_result in list_of_results_as_binary:
+        if each_result is False:
+            quantity_of_false =+ 1
+    return quantity_of_false
+
+
+def orchestrate_dns_check(target_interface: str) -> Result:
+    """
+    Purpose:
+        Coordinates DNS checks for resolvctl output, with actionable results
+
+    Rank:
+        Module Orchestrator
+
+    Called by:
+        systemwide_wireguard.__establish_system_connection
+    """
+
+    list_of_results_as_objects, list_of_results_as_binary = process_ALL_dns_results(
+        target_interface=target_interface,
+    )
+
+    quantity_of_checks = len(list_of_results_as_binary)
+    quantity_of_false = get_quantity_of_false(list_of_results_as_binary)
+
+    if quantity_of_false == 0:
+        logger.info(f"[DNS Orchestrator] Congrats, all {quantity_of_checks} worked.")
+        return Result(valid=True)
+
+    else:
+        logger.info(f"[DNS Orchestrator] Some of the DNS checks did NOT work.  But this might be because the tunnel is down.")
+
+        targets_not_found = 0
+
+        for each_object in list_of_results_as_objects:
+            if each_object.error_type == SearchError.TARGET_MISSING:
+                target_not_found = target_not_found + 1
+        
+        return Result(valid=False, data=targets_not_found)
+
+
+
+
+
+# NOT YET INTEGRATED. This is for potentially showing errors to the end-user:
+
+# if search_results.valid:
+#     return Result(valid=True)
+
+# resolvctl_name = turn_command_into_readable_string(which_command)
+# reason = search_results.error_type
+# not_leak_msg = "This is NOT a leak, because the firewall is still blocking all queries outside the VPN tunnel.  But for queries to realize they are being blocked, it may cause (very small) speed delays on domain lookups.  Do you want to reconnect now or just continue as is?"
+
+# if reason == SearchError.COMPETITORS_MATCH:
+#     error_log_msg = f"{resolvctl_name} had interfaces using DNS other than the VPN."
+#     logger.error(error_log_msg)
+#     final_human_message = f"{error_log_msg} {not_leak_msg}"
+#     return Result(valid=False, error_type=ResultError.CLIENT_DNS, message=final_human_message)
+
+# elif reason == SearchError.TARGET_NO_VALUE:
+#     error_log_msg = f"{resolvctl_name} wasn't able to be set to the VPN."
+#     logger.error(error_log_msg)
+#     final_human_message = f"{error_log_msg} {not_leak_msg}"
+#     return Result(valid=False, error_type=ResultError.CLIENT_DNS, message=final_human_message)
+
+# else:
+#     error_log_msg = "There was an Unknown error in setting your default DNS to the VPN."
+#     logger.error(error_log_msg)
+#     final_human_message = f"{error_log_msg} {not_leak_msg}"
+#     return Result(valid=False, error_type=ResultError.UNKNOWN, message=final_human_message)

core/services/networking/systemwide/dns_tools/search_resolv_output.py

@@ -0,0 +1,78 @@
+from core.services.networking.systemwide.dns_tools.SearchResult import SearchResult, SearchError
+from core.errors.logger import logger
+import re
+
+def get_pattern_and_phrase(
+    which_command: str,
+    ip_address: str = "10.13.0.1"
+)-> str:
+    """figure out which re pattern & phrase to use, based on the search command"""
+
+    if which_command == "domain":
+        target_phrase = "~."
+        pattern = r'\(([^)]+)\):\s*(\S*)?'
+ 
+    elif which_command == "default-route":
+        target_phrase = "yes"
+        pattern = r'\(([^)]+)\):\s*(yes|no)'
+ 
+    elif which_command == "dns":
+        target_phrase = ip_address
+        pattern = r'\(([^)]+)\):\s*([\d.]+)'
+ 
+    else:
+        target_phrase = None
+        pattern = None
+    return pattern, target_phrase
+
+def evaluating_resolv_output(which_command: str, target_interface: str, resolv_output: str) -> SearchResult:
+    if target_interface not in resolv_output:
+        return SearchResult(valid=False, error_type=SearchError.TARGET_MISSING)
+
+    # ============= PREPARE SEARCH =============
+    pattern, target_phrase = get_pattern_and_phrase(which_command)
+    if not pattern or not target_phrase:
+        return SearchResult(valid=False, error_type=SearchError.UNKNOWN, message=f"invalid which_command of {which_command}")
+
+    # Find all entries with pattern (name): value
+    matches = re.findall(pattern, resolv_output)
+
+    # Separate target from competitors
+    target_has_match = False
+    competitors_with_match = []
+    target_actually_had = None
+
+    # ============= LOOP =============
+    for name, value in matches:
+        if value == target_phrase:
+            if name == target_interface:
+                target_has_match = True
+            else:
+                competitors_with_match.append(name)
+
+        else:
+            # Check if Target has the wrong value
+            if name == target_interface:
+                target_actually_had = value
+
+    # ============= EVALUATE =============
+    competitors_match = len(competitors_with_match) > 0
+    # logger.info(f"Target has {target_phrase}: {target_has_match}")
+    # logger.info(f"Competitors have {target_phrase}: {competitors_match}")
+    # logger.info(f"Competitor list: {competitors_with_match}")
+
+    # target has it, and nobody else:
+    if target_has_match and not competitors_match:
+        return SearchResult(valid=True, target_value=target_phrase)
+
+    if target_has_match and competitors_match:
+        return SearchResult(valid=False, error_type=SearchError.COMPETITORS_MATCH, target_value=target_phrase)
+
+    if not target_has_match:
+        # target has wrong value:
+        if target_actually_had:
+            return SearchResult(valid=False, error_type=SearchError.VALUE_DOESNT_MATCH, competitors_list=competitors_with_match, target_value=target_actually_had)
+        # target has no value
+        else:
+            return SearchResult(valid=False, error_type=SearchError.TARGET_NO_VALUE, competitors_list=competitors_with_match, target_value=target_actually_had)
+

core/services/networking/systemwide/killswitch.py

@@ -0,0 +1,37 @@
+from core.services.networking.systemwide.systemwide_errors import systemwide_hell_raiser
+from core.utils.basic_operations.wrap_with import wrap_with
+from core.utils.basic_operations.run_generic_command import run_generic_command
+from core.models.Result import Result, ResultError
+from core.errors.logger import logger
+import subprocess
+
+KILLSWITCH_WRAPPER = "/opt/hydra-veil/firewall" # needs chmod 755
+
+@wrap_with(systemwide_hell_raiser)
+def arm(server_ip: str, tunnel_if: str, internal_subnet: str = None) -> Result:
+    command = ["sudo", KILLSWITCH_WRAPPER, "arm", server_ip, tunnel_if]
+    if internal_subnet:
+        command.append(internal_subnet)
+    human_readable_goal = "Arming the Firewall"
+    return run_generic_command(command, human_readable_goal)
+
+@wrap_with(systemwide_hell_raiser)
+def disarm() -> bool:
+    command = ["sudo", KILLSWITCH_WRAPPER, "disarm"]
+    human_readable_goal = "Disarming the Firewall"
+    return run_generic_command(command, human_readable_goal)
+
+def status() -> bool:
+    command = ["sudo", KILLSWITCH_WRAPPER, "status"]
+    human_readable_goal = "Get the Firewall's status"
+    result = run_generic_command(command, human_readable_goal, timeout=5)
+    if result.valid:
+        if "armed" in result.data.strip():
+            return True
+        else:
+            logger.error("[FIREWALL STATUS] It is currently NOT armed.")
+            return False
+    else:
+        # process error with running the command itself,
+        # send the error object with details.
+        return result

core/services/networking/systemwide/systemwide_errors.py

@@ -0,0 +1,54 @@
+from core.errors.exceptions import SudoScript, MissingPreReqs
+from core.models.Result import Result, ResultError
+from core.errors.logger import logger
+
+"""
+Purpose:
+    Raises Errors for known problems with the systemwide connections
+
+Kinds of Raises:
+    SudoScript
+    MissingPreReqs
+"""
+
+def systemwide_hell_raiser(result: Result) -> Result:
+    if result.valid:
+        return result
+    
+    logger.error(f"[HELL RAISER] We were trying to {result.goal} but got an error of {result.error_type} with a human message of {result.message}")
+
+    if result.error_type == ResultError.MISSING_FILE:
+        logger.error(f"[HELL RAISER] We are missing the setup scripts for {result.goal}")
+        result.message = f"You're missing the setup scripts for {result.goal}.  Re-do the install, or check the setup you did."
+        raise SudoScript(result)
+
+    elif result.error_type == ResultError.PERMISSION:
+        error_msg = f"There's improper permissions on the scripts for {result.goal}.  Re-do the install, or check the setup you did."
+        result.message = error_msg
+        logger.error(f"[HELL RAISER] {error_msg}")
+        raise SudoScript(result)
+
+    elif result.error_type == ResultError.MISSING_DEPENDENCY:
+        error_output = result.message
+        list_of_dependencies = ["nmcli", "resolvconf", "nftables", "nft"]
+        what_is_missing = []
+
+        for each_dependency in list_of_dependencies:
+            if each_dependency in error_output:
+                what_is_missing.append(each_dependency)
+
+        result.message = f"You're missing the prereq packages of {what_is_missing}.  Re-do the install, or check the setup you did."
+        raise MissingPreReqs(result)
+
+    elif result.error_type == ResultError.INTERFACE:
+        logger.info("The Interface is not found.")
+        # might not be an error, if it's already down,
+        return result
+
+    elif result.error_type == ResultError.UNKNOWN:
+        logger.info(f"Unknown error type for {result.goal} with {result.message} and potentially output of {result.data}.")
+        return result
+
+    else:
+        logger.info("Unknown Result object")
+        return result

core/services/networking/systemwide/systemwide_utils.py

@@ -0,0 +1,68 @@
+
+from core.models.Result import Result, ResultError
+import re
+import shutil
+from core.Errors import CommandNotFoundError
+from core.models.Configuration import get_setting
+from core.errors.logger import logger
+
+
+def get_dns_setting() -> bool:
+    dns_setting = get_setting("dns")
+    logger.info(f"DNS Setting is {dns_setting}")
+    if dns_setting is None:
+        return False
+    return dns_setting
+
+
+def get_firewall_setting() -> bool:
+    firewall_setting = get_setting("firewall")
+    logger.info(f"Firewall Setting is {firewall_setting}")
+    if firewall_setting is None:
+        return False
+    return firewall_setting
+
+
+
+def check_system_prereqs():
+    if shutil.which('dbus-send') is None:
+        raise CommandNotFoundError('dbus-send')
+
+    if shutil.which('nmcli') is None:
+        raise CommandNotFoundError('nmcli')
+
+
+
+def extract_wg_interface_name(text: str) -> Result:
+    """
+    Purpose:
+        Extracts the wg interface name, which should be 'wg'
+
+    Pattern:
+        "Connection" followed by text in single quotes
+
+    Example:
+        "Connection 'wg' (jgfdgfdjgfdjfdg"
+    """
+
+    pattern = r"Connection '([^']*)'"
+    
+    match = re.search(pattern, text)
+    
+    if match:
+        # Return what's between the quotes
+        result = match.group(1)
+        return Result(valid=True, data=result)
+    else:
+        error_msg = "Error: String doesn't contain 'Connection' and text within single quotes"
+        return Result(valid=True, error_type=ResultError.NMCLI, message=error_msg)
+
+
+
+
+
+
+
+######### ISOLATION TESTING ########
+# result1 = extract_connection_name("Connection 'wg' (jgfdgfdjgfdjfdg")
+# print(result1)  # Output: wg

core/services/networking/systemwide/systemwide_wireguard.py

@@ -0,0 +1,269 @@
+from core.services.networking.general_connection_tools.testing_evaluating import await_connection, system_uses_wireguard_interface, terminate_tor_connection
+from core.services.keys_and_verifications.endpoint_verification import verify_wireguard_endpoint
+from core.models.Result import Result, ResultError
+from core.services.networking.systemwide.systemwide_utils import extract_wg_interface_name, check_system_prereqs, get_firewall_setting, get_dns_setting
+# from core.services.networking.general_connection_tools.config_tools import extract_endpoint_ip, extract_dns
+from core.services.networking.systemwide import killswitch
+from core.services.networking.systemwide.dns_tools.process_dns_result import orchestrate_dns_check
+from core.services.networking.systemwide.dns_tools.SearchResult import SearchResult, SearchError
+from core.services.networking.systemwide.wireguard.nmcli_tools import setup_nmcli_connection, setup_ipv6_sinkhole
+from core.services.networking.systemwide.wireguard.wg_firewall_dns import set_dns, revert_dns, turn_on_firewall, check_and_kill_firewall
+from core.errors.logger import logger
+
+from core.Constants import Constants
+from core.Errors import ConnectionTerminationError, CommandNotFoundError
+from core.controllers.ConfigurationController import ConfigurationController
+from core.controllers.SystemStateController import SystemStateController
+from core.models.BaseProfile import ProfileType
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+from core.models.system.SystemState import SystemState
+from core.observers.ConnectionObserver import ConnectionObserver
+from subprocess import CalledProcessError
+from typing import Optional
+import os
+import re
+import shutil
+import subprocess
+import time
+
+
+def terminate_system_connection(
+    firewall_setting: Optional[bool] = get_firewall_setting(),
+    dns_setting: Optional[bool] = get_dns_setting()
+):
+
+    if shutil.which('nmcli') is None:
+        raise CommandNotFoundError('nmcli')
+
+    # ====== FIREWALL =======
+    if firewall_setting:
+        check_and_kill_firewall() # on failure, this raises errors, which then bubble up
+
+    # ====== DNS =======
+    if dns_setting:
+        reverting_dns_worked = revert_dns()
+
+    # ====== NMCLI =======
+    if SystemStateController.exists():
+        process = subprocess.Popen(('nmcli', 'connection', 'delete', 'wg'), stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+        completed_successfully = not bool(os.waitpid(process.pid, 0)[1] >> 8)
+
+        if completed_successfully or not system_uses_wireguard_interface():
+
+            subprocess.run(('nmcli', 'connection', 'delete', 'hv-ipv6-sink'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            terminate_tor_connection()
+
+            # confirm check on dns:
+            # did_dns_revert = confirm_dns_is_reverted('wg')
+            # print(f"did_dns_revert is {did_dns_revert}")
+
+            # finally end that profile's JSON state:
+            SystemState.dissolve()
+
+        else:
+            raise ConnectionTerminationError('The connection could not be terminated.')
+
+
+
+def __establish_system_connection(
+    profile: SystemProfile,
+    firewall_setting: bool = False,
+    dns_setting: bool = False,
+    connection_observer: Optional[ConnectionObserver] = None):
+    """
+    Purpose:
+        Orchestrates the full flow of the connection.
+
+    Steps:
+    1) Check Prereqs
+    2) Disable any pre-existing system connections
+    3) Turn on WG via nmcli
+    4) Turn on IPv6 sinkhole
+    5) Turn on Firewall (if enabled)
+    6) Set DNS (if enabled), and test it.
+    7) Setup State (JSON file)
+    8) Confirm Firewall is on (if enabled).  Raises errors if not.
+    9) Testing, confirming, displaying to the end user.
+    """
+
+    check_system_prereqs()
+    logger.info("[CONNECT] Terming existing connections..")
+    terminate_system_connection(firewall_setting, dns_setting)
+
+    # ============= INITIAL NMCLI CONNECTION =============
+    wg_config_path = profile.get_wireguard_configuration_path()
+    nmcli_result = setup_nmcli_connection(wg_config_path)
+
+    if nmcli_result.valid:
+        process_output = nmcli_result.data
+        logger.info("[CONNECT] Successfully turned on WG with nmcli")
+    else:
+        logger.error(f"[SYSTEMWIDE WG] Critical issue, the initial nmcli command failed {nmcli_result.message}.")
+        return nmcli_result
+
+    # ============= IPv6 SINKHOLE =============
+    sinkhole_result = setup_ipv6_sinkhole(process_output)
+    if not sinkhole_result:
+        raise ConnectionError('IPv6 Could not be protected.')
+
+    # ============= SETUP STATE =============
+    # Even if firewall is off, we want to save the fact we turned on the VPN tunnel, before we raise errors.
+
+    logger.info(f"Setting State JSON..")
+    current_state = SystemStateController.create(
+        profile_id=profile.id,
+        firewalled=firewall_setting, # intended
+        dns_set=dns_setting # intended
+    )
+    
+    # ============= PREP SETTINGS =============
+    if firewall_setting or dns_setting:
+        wg_interface_result = extract_wg_interface_name(process_output)
+        if not wg_interface_result.valid:
+            error_msg = f"WG interface is in the wrong format.  This is the raw output: {process_output}"
+            logger.error(f"[SYSTEMWIDE WG] Critical issue, the {error_msg}")
+            raise ConnectionError(error_msg)
+            # or handle the interface being wrong?
+
+        wg_interface_name = wg_interface_result.data
+
+    # ============= FIREWALL =============
+    firewall_tracker = False
+    if firewall_setting:
+        logger.info("Firewall setting is enabled, let's turn it on..")
+        turn_on_result = turn_on_firewall(
+            profile_id=str(profile.id),
+            wg_interface_name=wg_interface_name
+        )
+        logger.info(f"Results of firewall turn on are: {turn_on_result.valid}")
+        if turn_on_result.valid:
+            firewall_tracker = True
+
+    # ============= CONFIRM ON FIREWALL =============
+    # if firewall_setting:
+        logger.info(f"Evaluating Firewall Situation after we tried to turn on it on.")
+        firewall_status = killswitch.status()
+        logger.info(f"""
+The firewall setting is on.
+The result of arming it, {turn_on_result.valid}
+The result of the status {firewall_status}
+        """)
+        if firewall_status:
+            logger.info("Firewall is on. All good.")
+        else:
+            logger.error("Firewall FAILED TO ARM")  # DON'T TRY TO RENEGOTIATE WITH DEAD INTERNET !!
+
+            # raise ConnectionError(f'Firewall failed to arm!! {turn_on_result.message}.') # if this triggers, it tries to renegotiate with dead internet.
+
+    # ============= DNS =============
+    if dns_setting:
+        set_dns_result = set_dns(str(profile.id), wg_interface_name)
+        logger.info(f"[CONNECT] The result of our setting the DNS is {set_dns_result.valid}")
+        if not set_dns_result.valid:
+            logger.error(f"[CONNECT] Setting the DNS DIDN'T work. Here's the reason given: {set_dns_result.message}")
+
+        # now check:
+        test_dns_result = orchestrate_dns_check(wg_interface_name)
+        if test_dns_result.valid:
+            did_dns_work = True
+            logger.info("[CONNECT] All of the DNS Checks Worked")
+    else:
+        did_dns_work = False
+        logger.info("Skipping DNS setup, because it's off in settings")
+
+    # ============= TESTING =============
+    logger.info("If we made it this far, we have nmcli on, ipv6 sinkhole on, firewall on.")
+    try:
+        await_connection(connection_observer=connection_observer)
+
+    except ConnectionError:
+        raise ConnectionError('The connection could not be established.')
+
+    # ============= UPDATE STATE =============
+    current_state.firewalled = firewall_tracker
+    current_state.dns_set = did_dns_work
+    SystemStateController.update_or_create(current_state)
+
+    return Result(valid=True)
+
+
+def evaluate_connection_result(connection_result: Result):
+    if not connection_result.valid:
+        logger.error(f"[SYSTEMWIDE WG] Critical issue, could not establish a connection: {connection_result.message}")
+        try:
+            terminate_system_connection(firewall_setting, True)
+        except ConnectionTerminationError:
+            pass
+
+
+def establish_system_connection(
+    profile: SystemProfile,
+    ignore: tuple[type[Exception]] = (),
+    connection_observer: Optional[ConnectionObserver] = None):
+    """
+    Purpose:
+        Calls the private function with the full flow, so it can be retried on failure.
+
+    Steps)
+    1) Verify Endpoint's encryption
+    2) Pass to private function
+    3) Retry on failure
+    """
+
+    # ================= VERIFY ENDPOINT =================
+    if ConfigurationController.get_endpoint_verification_enabled():
+        verify_wireguard_endpoint(profile, ignore=ignore)
+
+    # ================= SETTINGS =================
+    firewall_setting = get_firewall_setting()
+    # dns_setting = True # for testing
+    dns_setting = get_dns_setting()
+
+    # ================= CONNECT WG ================
+    try:
+        connection_result = __establish_system_connection(
+            profile=profile,
+            firewall_setting=firewall_setting,
+            dns_setting=dns_setting,
+            connection_observer=connection_observer
+        )
+        evaluate_connection_result(connection_result)
+
+    except ConnectionError:
+        # ================= RETRY ON FAILURE =================
+        try:
+            terminate_system_connection(firewall_setting, dns_setting)
+        except ConnectionTerminationError:
+            pass
+
+        raise ConnectionError('The connection could not be established.')
+
+    except CalledProcessError:
+
+        try:
+            terminate_system_connection(firewall_setting, dns_setting)
+        except ConnectionTerminationError:
+            pass
+
+        # trying again..
+        try:
+            connection_result = __establish_system_connection(
+            profile=profile,
+            firewall_setting=firewall_setting,
+            dns_setting=dns_setting,
+            connection_observer=connection_observer
+            )
+            evaluate_connection_result(connection_result)
+
+        except (ConnectionError, CalledProcessError):
+
+            try:
+                terminate_system_connection(firewall_setting, dns_setting)
+            except ConnectionTerminationError:
+                pass
+
+            raise ConnectionError('The connection could not be established.')
+
+    terminate_tor_connection()
+    time.sleep(1.0)

core/services/networking/systemwide/wireguard/nmcli_tools.py

@@ -0,0 +1,42 @@
+from core.models.Result import Result, ResultError
+from subprocess import CalledProcessError
+import os
+import re
+import shutil
+import subprocess
+
+
+def setup_nmcli_connection(wg_config_path: str) -> Result:
+    try:
+        process_output = subprocess.check_output(('nmcli', 'connection', 'import', '--temporary', 'type', 'wireguard', 'file', wg_config_path), text=True)
+        return Result(valid=True, data=process_output)
+
+    except CalledProcessError as e:
+        return Result(valid=False, error_type=ResultError.NMCLI, message=f"Called Process error with nmcli: {e}")
+        # raise ConnectionError('The connection could not be established.')
+
+
+def setup_ipv6_sinkhole(process_output: str) -> Result:
+    error_info = "with nmcli setting up the IPv6 sinkhole, error is "
+    try:
+        connection_id = (m := re.search(r'(?<=\()([a-f0-9-]+?)(?=\))', process_output)) and m.group(1)
+        ipv6_method = subprocess.check_output(('nmcli', '-g', 'ipv6.method', 'connection', 'show', connection_id), text=True).strip()
+    except CalledProcessError as e:
+        return Result(valid=False, error_type=ResultError.NMCLI, message=f"Called Process error {error_info}: {e}")
+        # raise ConnectionError('The connection could not be established.')
+
+    if ipv6_method in ('disabled', 'ignore'):
+        try:
+            subprocess.run(('dbus-send', '--system', '--print-reply', '--dest=org.freedesktop.NetworkManager', '/org/freedesktop/NetworkManager', 'org.freedesktop.DBus.Properties.Set', 'string:org.freedesktop.NetworkManager', 'string:ConnectivityCheckEnabled', 'variant:boolean:false'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
+        except CalledProcessError as e:
+            return Result(valid=False, error_type=ResultError.NMCLI, message=f"Called Process error {error_info}: {e}")
+            # raise ConnectionError('The connection could not be established.')
+
+        try:
+            subprocess.run(('nmcli', 'connection', 'add', 'type', 'dummy', 'save', 'no', 'con-name', 'hv-ipv6-sink', 'ifname', 'hvipv6sink0', 'ipv6.method', 'manual', 'ipv6.addresses', 'fd7a:fd4b:54e3:077c::/64', 'ipv6.gateway', 'fd7a:fd4b:54e3:077c::1', 'ipv6.dns', '::1', 'ipv6.route-metric', '72'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
+            return Result(valid=True)
+
+        except CalledProcessError:
+            return Result(valid=False, error_type=ResultError.NMCLI, message=f"Called Process error {error_info}")
+            # raise ConnectionError('The connection could not be established.')
+

core/services/networking/systemwide/wireguard/wg_firewall_dns.py

@@ -0,0 +1,105 @@
+from core.models.Result import Result, ResultError
+from core.services.networking.systemwide import killswitch, dns
+from core.services.networking.general_connection_tools.config_tools import extract_endpoint_ip, extract_dns
+from core.errors.logger import logger
+
+def set_dns(profile_id: str, network_interface: str) -> Result:
+    do_they_even_use_systemd = dns.is_systemd_enabled()
+
+    if not do_they_even_use_systemd.valid:
+        error_msg = "You don't use SystemD, so we are skipping setting your DNS."
+        logger.info(error_msg)
+        return Result(valid=True, message=error_msg) # true to not prompt errors
+
+    dns_should_be = extract_dns(profile_id)
+
+    logger.info(f"[CONNECT] About to set the DNS for {network_interface}..")
+    return dns.set_on(network_interface=network_interface, dns_should_be=dns_should_be)
+
+
+
+def revert_dns() -> Result:
+    do_they_even_use_systemd = dns.is_systemd_enabled()
+
+    if not do_they_even_use_systemd.valid:
+        error_msg = "You don't use SystemD, so we are skipping setting your DNS."
+        logger.info(error_msg)
+        return Result(valid=True, message=error_msg) # true to not prompt errors
+
+    logger.info("Reverting DNS Settings..")
+    turn_off = dns.revert()
+    logger.info(f"The result of the turn off is {turn_off.valid}")
+    if not turn_off.valid:
+        logger.error(f"Critical DNS Error! Could NOT turn off DNS. Error type: {turn_off.error_type} with {turn_off.message}")
+    return turn_off.valid
+
+
+
+def turn_on_firewall(profile_id: str, wg_interface_name: str) -> Result:
+    """
+    Purpose:
+        This is a wireguard specific function, using the WG interface,
+        but the underlying killswitch.arm function it uses is protocol neutral.
+
+    Steps:
+    1) Get the WG interface from the process output
+    2) Get the server IP from the backup WG config
+    3) Use the killswitch's arm
+    """
+    logger.info(f"[SYSTEMWIDE WG] Turning on Firewall..")
+
+    server_ip_address = extract_endpoint_ip(profile_id)
+    if wg_interface_name != 'wg':
+        logger.info(f"Note: The WG interface is NOT WG.  It's {wg_interface_name}.  Be careful about what we kill switch compared to what goes up.  Although we kill the nftable rule regardless on down.")
+
+    return killswitch.arm(
+        server_ip=server_ip_address,
+        tunnel_if=wg_interface_name,
+        internal_subnet=None
+    )
+
+
+# This raises errors on failure.
+def check_and_kill_firewall():
+    firewall_on = killswitch.status() # produces a boolean
+    logger.info(f"The Firewall is currently: {firewall_on}")
+
+    # is it even armed? if not, get lost,
+    if firewall_on:
+        # if it's armed, kill it,
+        logger.info(f"Killing the firewall..")
+        disable_result_object = killswitch.disarm() # produces a Result Object
+
+        # did that work?
+        if disable_result_object.valid:
+            logger.info(f"Success! We disabled the firewall correctly.")
+            return
+        else:
+            error_msg = f"Firewall can't be disabled: {disable_result_object.message}"
+            logger.error(error_msg)
+
+            # if the systemwide_raiser didn't catch it with the reason, then..
+            raise ConnectionTerminationError(error_msg)
+    else:
+        logger.info("We are skipping disabling the firewall, because it's already off.")
+
+
+# used in dead code at bottom:
+# from core.services.networking.systemwide.dns_tools.process_dns_result import orchestrate_dns_check
+
+# This is dead code right now
+# def confirm_dns_is_reverted(wg_interface_name: str = 'wg'):
+#     do_they_even_use_systemd = dns.is_systemd_enabled()
+
+#     if not do_they_even_use_systemd:
+#         return True
+
+#     check_on_dns = orchestrate_dns_check(wg_interface_name)
+
+#     if check_on_dns.valid:
+#         logger.error("CRITICAL DNS PROBLEM! We turned WG off, but the DNS check shows it's still on.  Trying again..")
+#         return False
+#     else:
+#         logger.info("We turned WG off, and the DNS check shows it reverted also. This is great.")
+#         return True
+

core/services/networking/use_tor.py

@@ -7,6 +7,9 @@ if TYPE_CHECKING:
 from essentials.modules.TorModule import TorModule
 from essentials.observers.ConnectionObserver import ConnectionObserver
 from essentials.services.ConnectionService import ConnectionService
+
+from core.observers.ClientObserver import ClientObserver
+
 # services
 from core.services.networking.evaluate_networking_problem import evaluate_tor_networking_problem
 # errors
@@ -18,7 +21,7 @@ from typing import Any
 from core.Constants import Constants
 import time
 
-def tor_get_request(custom_url: str, connection_observer: ConnectionObserver) -> dict:
+def tor_get_request(custom_url: str, connection_observer: ConnectionObserver, client_observer: Optional[ClientObserver] = None) -> dict:
     import requests
 
     # ============================================================================
@@ -26,18 +29,25 @@ def tor_get_request(custom_url: str, connection_observer: ConnectionObserver) ->
     # ============================================================================
     port_number = ConnectionService.get_random_available_port_number()
     logger.debug(f"[USE TOR SERVICE] Using Tor on port number {port_number}")
+    if client_observer is not None:
+        client_observer.notify('synchronizing', f"Using Tor on Port {port_number}..")
 
     try:
         tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
         tor_module.create_session(port_number, connection_observer)
         logger.debug(f"[USE TOR SERVICE] Tor Started a Session")
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Initialized Tor..")
 
     except TorServiceInitializationError as e:
         error_msg = f"[USE TOR SERVICE] Tor failed to start: {e}"
         tor_module.destroy_session(port_number)
         logger.error(error_msg, exc_info=True)
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Failed to Initialize")
+
         problem_results = evaluate_tor_networking_problem(
-            custom_url, connection_observer
+            custom_url, connection_observer, client_observer
         )
         return problem_results
     except Exception as e:
@@ -73,6 +83,9 @@ def tor_get_request(custom_url: str, connection_observer: ConnectionObserver) ->
         return final_reply
 
     except requests.exceptions.ConnectionError:
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Connection Error")
+
         tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
             custom_url, connection_observer
@@ -81,6 +94,9 @@ def tor_get_request(custom_url: str, connection_observer: ConnectionObserver) ->
 
     except requests.exceptions.Timeout:
         logger.debug("Connection timed out")
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Connection Timeout")
+
         tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
             custom_url, connection_observer
@@ -88,7 +104,11 @@ def tor_get_request(custom_url: str, connection_observer: ConnectionObserver) ->
         return problem_results
 
     except requests.exceptions.HTTPError:
-        logger.debug(f"HTTP error: {response.status_code}")
+        tor_http_error = f"Tor HTTP error: {response.status_code}"
+        logger.debug(tor_http_error)
+        if client_observer is not None:
+            client_observer.notify('synchronizing', tor_http_error)
+
         tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
             custom_url, connection_observer
@@ -105,6 +125,9 @@ def tor_get_request(custom_url: str, connection_observer: ConnectionObserver) ->
 
     except Exception as e:
         logger.error(f"[USE TOR SERVICE] Unexpected Tor error: {e}")
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Error: {e}")
+
         tor_module.destroy_session(port_number)
         problem_results = evaluate_tor_networking_problem(
             custom_url, connection_observer

core/services/payment_phase/check_if_paid.py

@@ -1,31 +0,0 @@
-from __future__ import annotations
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from essentials.observers.ConnectionObserver import ConnectionObserver
-# services:
-from core.services.networking.send_data_to_server import send_data_to_server
-from core.services.networking.make_url import make_url
-# errors:
-from core.errors.exceptions import *
-from core.errors.logger import logger
-
-
-# it's private because it assumes it's being called from the controller, (with a JSON)
-def _check_if_paid(payload: dict, connection_observer: ConnectionObserver) -> str:
-    # prep endpoint:
-    which_endpoint = "check_paid"
-    url = make_url(which_endpoint)
-
-    # literally send:
-    reply = send_data_to_server(payload, url, connection_observer)
-
-    if "valid" in reply:
-        valid_status = reply.get("valid")
-        if valid_status == True:
-            return "paid"
-        else:
-            return "not_paid"
-    else:
-        error_msg = f"When checking if paid, the Server returned invalid data or it never sent.  The reply is {reply}"
-        raise InvalidData(error_msg)

core/services/payment_phase/save_and_send_intitial_billing.py

@@ -5,7 +5,13 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 #
 from core.models.invoice.TicketInvoice import TicketInvoice
-from core.services.networking.send_data_to_server import send_data_to_server
+
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+# from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
+
 from core.services.networking.make_url import make_url
 from core.services.payment_phase.extract_payment_details import extract_payment_details
 from core.services.payment_phase.save_billing_choices import save_billing_choices
@@ -18,7 +24,22 @@ def save_and_send_intitial_billing(
     payload: dict,
     connection_observer: ConnectionObserver,
     invoice_data_object: TicketInvoice,
-) -> TicketInvoice:
+) -> TicketInvoice | ApiResponse:
+
+    """
+    Purpose:
+        Does a POST request to get updated billing information.
+
+    Returns:
+        TicketInvoice on Success, with the modified values.
+        ApiResponse on Failure, with why the API failed.
+
+    Called by:
+        TicketPayController
+
+    Errors:
+        No. Does NOT raise errors.
+    """
 
     # Save choices:
     save_billing_choices(payload)
@@ -26,9 +47,23 @@ def save_and_send_intitial_billing(
     # send them:
     which_endpoint = "start_payment"
     url = make_url(which_endpoint)
-    reply = send_data_to_server(payload, url, connection_observer)
+    api_reply_object = send_data_to_server(payload, url, connection_observer)
+
+    if not api_reply_object.valid:
+        api_reply_object = solve_api_problems(
+            api_reply_object=api_reply_object,
+            get_or_post="post",
+            url=url,
+            payload=payload,
+            connection_observer=connection_observer,
+            client_observer=None
+        )
+        if not api_reply_object.valid:
+            return api_reply_object
+
+    reply_dict_data = api_reply_object.data
 
     # extract values from server's reply:
-    modified_data_object = extract_payment_details(reply, invoice_data_object)
+    modified_data_object = extract_payment_details(reply_dict_data, invoice_data_object)
 
     return modified_data_object

core/services/prepare_tickets/get_pub_key.py

@@ -6,7 +6,8 @@ if TYPE_CHECKING:
 from typing import Any
 # services
 from core.services.networking.make_url import make_url
-from core.services.networking.get_data_from_server import get_data_from_server
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+
 # utils
 from core.utils.basic_operations.does_file_exist import does_file_exist
 from core.utils.basic_operations.write_string_to_text_file import write_string_to_text_file

core/services/prepare_tickets/get_public_key_by_config.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 # services & helpers
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
 from core.services.prepare_tickets.get_pub_key import get_pub_key
 from core.services.prepare_tickets.get_pub_key import key_is_in_valid_format
 from core.services.helpers.get_plan_data import get_plan_data
@@ -55,14 +56,27 @@ def get_public_key_by_config(connection_observer: ConnectionObserver) -> dict:
 
         # Now we're assuming we have a temp billing code,
         # So we can use that to get the key from the server,
-        reply = get_plan_data(temp_billing_code, connection_observer)
+        api_reply_object = get_plan_data(temp_billing_code, connection_observer)
 
-        if reply in list_of_failures:
+        complete_failure_msg = {
-            return {
                 "status": False,
                 "message": f"Issues with both finding your local key plan and even connecting to the server for it. Please check {filepath}",
         }
 
+        if not api_reply_object.valid:
+            return complete_failure_msg
+        
+        reply = api_reply_object.data
+
+        if reply in list_of_failures:
+            return complete_failure_msg
+
+        if not instance(reply, dict):
+            return {
+                "status": False,
+                "message": f"Server returned an invalid format, and even accessing via local files. Please check {filepath}",
+            }
+
         # from the server's reply, get the key
         which_key = filter_data(reply, "which_key")
         if which_key in list_of_failures:
@@ -112,13 +126,27 @@ def get_public_key_from_LOCAL_files_only(
                     "message": f"Issue with finding key plan or billing code. Please check {filepath}",
                 }
 
-            reply = get_plan_data(temp_billing_code, connection_observer)
+            api_reply_object = get_plan_data(temp_billing_code, connection_observer)
-            if reply in list_of_failures:
+
-                return {
+            complete_failure_msg = {
                     "status": False,
                     "message": f"Issues with both finding your local key plan and even connecting to the server for it. Please check {filepath}",
             }
 
+            if not api_reply_object.valid:
+                return complete_failure_msg
+            
+            reply = api_reply_object.data
+
+            if reply in list_of_failures:
+                return complete_failure_msg
+
+            if not instance(reply, dict):
+                return {
+                    "status": False,
+                    "message": f"Server returned an invalid format, and even accessing via local files. Please check {filepath}",
+                }
+
             which_key = filter_data(reply, "which_key")
             if which_key in list_of_failures:
                 return {

core/services/prepare_tickets/send_blind_commitments.py

@@ -5,7 +5,10 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
     from core.observers.TicketObserver import TicketObserver
 # services
-from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
 from core.services.networking.make_url import make_url
 from core.services.helpers.get_which_billing_key import get_which_billing_key
 # utils
@@ -44,9 +47,35 @@ def send_blind_commitments(
         # send it:
         which_endpoint = "sign"
         url = make_url(which_endpoint)
-        reply = send_data_to_server(payload, url, connection_observer)
+        api_reply_object = send_data_to_server(payload, url, connection_observer)
+
+        # did API call work?
+        if not api_reply_object.valid:
+            final_error_msg = f"[SEND BLIND COMMITMENTS] First Post request failed {api_reply_object.message}"
+            logger.error(final_error_msg)
+            api_reply_object = solve_api_problems(
+                api_reply_object=api_reply_object,
+                get_or_post="post",
+                url=url,
+                payload=payload,
+                connection_observer=connection_observer,
+                client_observer=None
+            )
+            if not api_reply_object.valid:
+                logger.error(f"[SEND BLIND COMMITMENTS] Second Post request failed. {api_reply_object.message}")
+                ticket_observer.notify("error", subject=api_reply_object.message)
+                return None
+
+        # assuming it worked, extract the data:
+        reply = api_reply_object.data
+
+        # is it in valid format?
+        if not isinstance(reply, dict):
+            final_error_msg = f"API returned an invalid format {reply}"
+            logger.error(final_error_msg)
+            ticket_observer.notify("error", subject=final_error_msg)
+            return None
 
-        # did it work?
         if reply.get("valid") == True:
             # this 'signed_data' variable is a list,
             signed_data = reply.get("signed_data")

core/services/subscriptions/subscriptions.py

@@ -0,0 +1,61 @@
+from typing import Union, Optional
+from core.models.session.SessionProfile import SessionProfile
+from core.models.system.SystemProfile import SystemProfile
+from core.Errors import MissingSubscriptionError, InvalidSubscriptionError
+from core.services.WebServiceApiService import WebServiceApiService
+from core.controllers.ConnectionController import ConnectionController
+from core.observers.ConnectionObserver import ConnectionObserver
+
+def activate_subscription(
+    profile: Union[SessionProfile, SystemProfile], 
+    connection_observer: Optional[ConnectionObserver] = None
+) -> bool:
+    """
+    Purpose:
+        Ensure subscription is ready.
+
+    Features:
+        Idempotent.
+        Checks local first
+
+    Confusion:
+        This returns True both if was already active or just activated.
+
+    Returns:
+        Returns True if subscription was already active.
+        Returns True if subscription was just activated.
+
+    Errors:
+        True.
+        Raises if subscription is missing or invalid. (not false)
+    """
+    
+    if not profile.has_subscription():
+        raise MissingSubscriptionError()
+    
+    # Already activated—nothing to do
+    if profile.subscription.has_been_activated():
+        return True
+    
+    # Fetch and activate
+    subscription = ConnectionController.with_preferred_connection(
+        profile.subscription.billing_code, 
+        task=WebServiceApiService.get_subscription, 
+        connection_observer=connection_observer
+    )
+    
+    if subscription is None:
+        raise InvalidSubscriptionError()
+    
+    profile.subscription = subscription
+    profile.save()
+    return True
+
+
+def is_subscription_ready(profile: Union[SessionProfile, SystemProfile]) -> bool:
+    """Check if subscription exists and is activated (no side effects)."""
+    return (
+        profile.has_subscription() 
+        and profile.subscription.has_been_activated()
+    )
+

core/services/sync/get_data_from_api.py

@@ -1,78 +0,0 @@
-from core.Constants import Constants
-from core.observers.BaseObserver import BaseObserver
-from core.services.networking.get_data_from_server import get_data_from_server
-from core.errors.logger import logger
-from core.observers.ClientObserver import ClientObserver
-from core.observers.ConnectionObserver import ConnectionObserver
-from typing import Optional
-
-
-# this can be spoofed with the isolation testing comment below
-
-def get_data_from_api(
-    endpoint: str,
-    client_observer: Optional[ClientObserver] = None,
-    connection_observer: Optional[ConnectionObserver] = None
-) -> dict:
-    logger.info("Syncing with the API in get_data_from_api...")
-
-    # Get and validate the base URL
-    rejected_list = [None, False, ""]
-    base_url = Constants.SP_API_BASE_URL
-    if base_url in rejected_list:
-        error_msg = "Invalid base URL from configuration"
-        logger.error(error_msg)
-        return {"success": False, "data": None, "error": error_msg}
-
-    # Construct the full endpoint URL
-    url = f"{base_url}/{endpoint}"
-    logger.debug(f"API endpoint: {url}")
-
-    try:
-        logger.debug("Fetching data from API server...")
-        sync_results = get_data_from_server(url, connection_observer)
-
-        if sync_results in rejected_list:
-            error_msg = "API returned no data"
-            logger.error(error_msg)
-            return {"success": False, "data": None, "error": error_msg}
-
-        logger.debug(f"Raw API response: {sync_results}")
-
-        final_result = sync_results.get("data", None)
-
-        if final_result:
-            logger.info("Successfully retrieved sync versions metadata from API")
-            return {"success": True, "data": final_result, "error": None}
-        else:
-            error_msg = f"API response missing 'data' field at the end of get_data_from_api. This is the response from the other module: {sync_results}"
-            logger.error(error_msg)
-            return {"success": False, "data": None, "error": error_msg}
-
-    except Exception as e:
-        error_msg = f"API fetch failed: {str(e)}"
-        logger.error(error_msg)
-        return {"success": False, "data": None, "error": error_msg}
-
-
-
-
-
-
-# ============== ISOLATION TESTING ==================
-# SPOOF ISOLATION TEST:
-# def _get_sync_cache_from_api(
-#     client_observer: Optional[ClientObserver] = None,
-#     connection_observer: Optional[ConnectionObserver] = None
-# ) -> dict:
-#     final_result = {
-#         "version": 9,
-#         "applications": 5,
-#         "application_versions": 4,
-#         "client_version": 3,
-#         "operators": 2,
-#         "locations": 3,
-#         "subscriptions": 1
-#     }
-#     return {"success": True, "data": final_result, "error": None}
-

core/services/sync/orm_methods/sync_one_orm.py

@@ -1,11 +1,22 @@
-from core.services.sync.get_data_from_api import get_data_from_api
+# API
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
+# Database
+from core.models.DatabaseOperation import DatabaseOperation
 from core.models.manage.insert import insert_into_model
 from core.models.manage.denormalize import denormalize
 from core.models.orm_models.Base import Base
+
+# observers & loiggers
 from core.observers.ClientObserver import ClientObserver
 from core.observers.ConnectionObserver import ConnectionObserver
 from core.errors.logger import logger
+
+# basic utils
 from core.utils.basic_operations.get_parent_directory import get_parent_directory
+from core.Constants import Constants
 
 from typing import Optional
 import json
@@ -17,22 +28,34 @@ def sync_one_orm_model(
     connection_observer: Optional[ConnectionObserver] = None
 ) -> dict:
 
-    api_result = get_data_from_api(which_endpoint, ClientObserver, ConnectionObserver)
+    full_request_url = f"{Constants.SP_API_BASE_URL}/{which_endpoint}"
+    api_result = get_data_from_api(full_request_url, client_observer, connection_observer)
 
-    if not api_result.get("success"):
+    if not api_result.valid:
-        error_msg = api_result.get("error", "Unknown API error")
+        error_msg = api_result.message
-        logger.error(f"API sync failed for {which_endpoint} with: {error_msg}")
+        logger.error(f"[ONE ORM SYNC] There's an issue with the sync of endpoint {which_endpoint} the API Reply: {error_msg}")
+        api_result = solve_api_problems(
+        api_reply_object=api_result,
+        get_or_post="get",
+        url=full_request_url,
+        payload=None,
+        connection_observer=connection_observer,
+        client_observer=client_observer
+        )
+        # 2nd try:
+        if not api_result.valid:
             return {
                 "success": False,
-            "error": error_msg
+                "category": "API",
+                "error": api_result.message
             }
 
-    new_data = api_result.get("data")
+    new_data = api_result.data
 
     # prep data (denormalize)
-    parent_directory = get_parent_directory()
+    current_assets_folder = f"{Constants.HV_DATA_HOME}/assets"
     yaml_filename = f"{which_endpoint}.yaml"
-    full_yaml_path = f"{parent_directory}/assets/yaml_mappings/{yaml_filename}"
+    full_yaml_path = f"{current_assets_folder}/yaml_mappings/{yaml_filename}"
 
     denormalized_data = denormalize(new_data, str(full_yaml_path))
 
@@ -40,25 +63,21 @@ def sync_one_orm_model(
         # Debug Pretty print with indentation
         # print(json.dumps(denormalized_data, indent=2))
 
-        try:
+        database_operation_object = insert_into_model(which_model, denormalized_data, True)
-            did_it_work = insert_into_model(which_model, denormalized_data, True)
+
-            if did_it_work:
+        if database_operation_object.valid:
             return {
                 "success": True
             }
         else:
             return {
                 "success": False,
-                    "error": f"We got the data from the endpoint {which_endpoint}, but failed to insert into the model.  That raw data is {denormalized_data}"
+                "category": "SQL",
+                "error": f"SQL Error: {database_operation_object.error_type}.  We got the data from the endpoint {which_endpoint}, but failed to insert into the model.  That raw data is {denormalized_data}"
             }
-        except:
-            return {
-                "success": False,
-                "error": f"The except triggered on a sync_one_orm_model's failure to insert into the model with the endpoint {which_endpoint}.  We got the raw data of {denormalized_data}"
-            }
-
     else:
         return {
             "success": False,
+            "category": "Data",
             "error": f"We had issues with denormalizing the data from the server.  That data was {new_data}."
         }

core/services/sync/sync_service.py

@@ -1,7 +1,11 @@
 
 # comparisons & api calls:
 from core.services.sync.compare_tables import compare_tables
-from core.services.sync.get_data_from_api import get_data_from_api
+# from core.services.sync.get_data_from_api import get_data_from_api
+
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
 
 # ORM for metadata:
 from core.models.manage.session_management import get_session
@@ -147,21 +151,38 @@ def coordinate_cache_sync(
         - 'filtered_metadata': the metadata from the server after removing keys not in the ORM
     """
     
-    api_result = get_data_from_api("cachedsync", ClientObserver, ConnectionObserver)
+    full_request_url = f"{Constants.SP_API_BASE_URL}/cachedsync"
+    api_result = get_data_from_api(full_request_url, client_observer, connection_observer)
 
+    # ================== SOLVE API CALL PROBLEMS  ==================
     # we trust the 'get_data_from_api' function to give us a dictionary:
-    if not api_result.get("success"):
+    if not api_result.valid:
-        error_msg = api_result.get("error", "Unknown API error")
+        error_msg = api_result.message
-        logger.error(f"API sync failed: {error_msg}")
+        logger.error(f"[SYNC SERVICE] There's an issue with the API Reply: {error_msg}")
+        api_result = solve_api_problems(
+        api_reply_object=api_result,
+        get_or_post="get",
+        url=full_request_url,
+        payload=None,
+        connection_observer=connection_observer,
+        client_observer=client_observer
+        )
+        # 2nd try:
+        if not api_result.valid:
             return {
                 "success": False,
                 "changed_tables": [],
                 "new_model_types": [],
-            "error": error_msg,
+                "error": api_result.message,
             }
 
-    # Extract the data from the client's own trusted function,
+    # ================== VALIDATE DATA  ==================
-    new_data = api_result.get("data")
+
+    # Extract the data from our own trusted function without type checking,
+    new_data = api_result.data
+
+    if new_data:
+        logger.debug(f"We got an API response with some data, let's check if it's a valid format...")
 
     # Validate the API's payload, (we don't trust the API's structure)
     if not isinstance(new_data, dict):

core/services/using_tickets/send_unblinded.py

@@ -3,8 +3,12 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
+
 # services & helpers
-from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
 from core.services.networking.make_url import make_url
 from core.services.helpers.get_which_billing_key import get_which_billing_key
 # utils
@@ -20,7 +24,7 @@ def send_unblinded_ticket_to_server(
     which_ticket: int,
     which_location: str,
     connection_observer: ConnectionObserver,
-) -> dict:
+) -> dict|ApiResponse:
 
     try:
         ticket_data = get_raw_string(which_ticket, "unblinded_final_ticket")
@@ -46,9 +50,20 @@ def send_unblinded_ticket_to_server(
         # send it:
         which_endpoint = "validate"
         url = make_url(which_endpoint)
-        reply = send_data_to_server(payload, url, connection_observer)
+        api_reply_object = send_data_to_server(payload, url, connection_observer)
 
-        return reply
+        # This is if the API failed, not if it's not a valid ticket:
+        if not api_reply_object.valid:
+            api_reply_object = solve_api_problems(
+                api_reply_object=api_reply_object,
+                get_or_post="post",
+                url=url,
+                payload=payload,
+                connection_observer=connection_observer,
+                client_observer=None
+            )
+
+        return api_reply_object
 
     except Exception as e:
         human_readable_error_msg = f"The send_unblinded_ticket_to_server function's try-except block failed for ticket {which_ticket}.  Returning False..."

core/services/using_tickets/use_ticket_orchestrator.py

@@ -5,6 +5,8 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 ##############
 from core.services.using_tickets.send_unblinded import send_unblinded_ticket_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
 from core.utils.basic_operations.write_or_read_from_json import (
     update_value_in_json_with_two_values,
 )
@@ -21,25 +23,35 @@ def use_ticket_orchestrator(
     which_ticket: int,
     which_location: str,
     connection_observer: ConnectionObserver,
-) -> dict:
+) -> dict | ApiResponse:
 
     # prep the ticket tracker:
     billing_folder = Constants.HV_TICKETING_CONFIG_HOME
     ticket_tracker_path = f"{billing_folder}/ticket_tracker.json"
 
     # send to server:
-    reply = send_unblinded_ticket_to_server(
+    api_reply_object = send_unblinded_ticket_to_server(
         which_ticket, which_location, connection_observer
     )
 
-    logger.debug(f"reply is {reply}")
+    if not api_reply_object.valid:
+        return api_reply_object
 
-    if "valid" not in reply:
+    raw_data_dict = api_reply_object.data
+
+    # logger.debug(f"[USE TICKET ORCHESTRATOR] We have received a raw dictionary of {raw_data_dict}")
+
+    if not isinstance(raw_data_dict, dict):
+        logger.debug(f"[USE TICKET ORCHESTRATOR] Invalid Data format returned from the API server.")
         return {"valid": False, "message": "invalid_format"}
-    is_it_valid = reply.get("valid")
 
-    if "billing_code" in reply:
+    if "valid" not in raw_data_dict:
-        billing_code = reply.get("billing_code", None)
+        logger.debug(f"[USE TICKET ORCHESTRATOR] Invalid Data format returned from the API server.")
+        return {"valid": False, "message": "invalid_format"}
+    is_it_valid = raw_data_dict.get("valid")
+
+    if "billing_code" in raw_data_dict:
+        billing_code = raw_data_dict.get("billing_code", None)
 
     """
     the reason this 'billing_code' check is a seperate function (and before the validity test),
@@ -47,6 +59,8 @@ def use_ticket_orchestrator(
     """
 
     if is_it_valid:
+        logger.debug(f"[USE TICKET ORCHESTRATOR] The API server returned a valid billing code.")
+
         # update the ticket tracker:
         status_update = update_value_in_json_with_two_values(
             ticket_tracker_path, which_ticket, "status", "used"
@@ -63,12 +77,14 @@ def use_ticket_orchestrator(
             )
         return {"valid": True, "billing_code": billing_code}
 
-    if "message" not in reply:
+    if "message" not in raw_data_dict:
         return {"valid": False, "message": "invalid_format"}
 
-    message = reply.get("message")
+    message = raw_data_dict.get("message")
 
     if message == "already_used":
+        logger.debug(f"[USE TICKET ORCHESTRATOR] This billing code has already been used, but it is still potentially still valid.")
+
         # update the ticket tracker to reflect that it's already used with that billing code:
         status_update = update_value_in_json_with_two_values(
             ticket_tracker_path, which_ticket, "status", "used"
@@ -84,7 +100,7 @@ def use_ticket_orchestrator(
                 subscription_update, which_ticket, billing_code, billing_folder
             )
 
-    return reply
+    return raw_data_dict
 
 
 def make_sure_sub_saved(

core/utils/basic_operations/confirm_files_exist.py

@@ -0,0 +1,20 @@
+from core.errors.logger import logger
+from pathlib import Path
+
+def confirm_files_and_folders_exist(folder_path, files_to_check):
+    if folder_path.is_dir():
+        logger.info(f"Folder exists: {folder_path}")
+                
+        # Check each file
+        for filename in files_to_check:
+            file_path = folder_path / filename
+            if file_path.is_file():
+                logger.info(f"✓ {filename} exists")
+                return True
+            else:
+                logger.error(f"✗ {filename} NOT found")
+                return False
+    else:
+        logger.error("Folder does not exist")
+        return False
+

core/utils/basic_operations/run_generic_command.py

@@ -0,0 +1,66 @@
+from core.models.Result import Result, ResultError
+from core.errors.logger import logger
+
+import subprocess
+import os
+
+_SUDO_KW = {
+    "stdout": subprocess.PIPE,
+    "stderr": subprocess.PIPE,
+    "stdin": subprocess.DEVNULL,
+    "env": {**os.environ, "SUDO_ASKPASS": "/bin/false"},
+    "text": True,
+}
+
+
+def parse_errors(error_output: str) -> ResultError:
+    # Interface missing
+    if "No such device" in error_output:
+        error_enum = ResultError.INTERFACE
+    # Permission error
+    elif "No permissions" in error_output:
+        error_enum = ResultError.PERMISSION
+    elif "sudo: a password is required" or "password is required" in error_output:
+        error_enum = ResultError.PERMISSION
+    elif "command not found" in error_output:
+        error_enum = ResultError.MISSING_DEPENDENCY
+    else:
+        error_enum = ResultError.UNKNOWN
+    return error_enum
+
+
+def run_generic_command(
+    command: list,
+    human_readable_goal: str,
+    timeout: float | None = None
+) -> Result:
+
+    output_data = None # incase command has nothing.
+    
+    # Add -n flag to sudo for non-interactive mode (no prompts)
+    if command and command[0] == "sudo":
+        if "-n" not in command:
+            command = [command[0], "-n"] + command[1:]
+
+    try:
+        result = subprocess.run(
+            command,
+            **_SUDO_KW,
+            timeout=timeout
+        )
+        output_data = result.stdout
+        if result.returncode == 0:
+            logger.info(f"{human_readable_goal} was successful")
+            return Result(valid=True, data=output_data)
+        else:
+            error_output = result.stderr.strip()
+            error_enum = parse_errors(error_output)
+            logger.error(f"{human_readable_goal} Failed, error type is {error_enum}.")
+            return Result(valid=False, error_type=error_enum, goal=human_readable_goal, message=error_output)
+    except subprocess.TimeoutExpired:
+        return Result(valid=False, error_type=ResultError.TIMEOUT, goal=human_readable_goal, data=output_data, message="Command timed out")
+    except FileNotFoundError:
+        return Result(valid=False, error_type=ResultError.MISSING_FILE, goal=human_readable_goal, data=output_data, message="Script is not found")
+    except Exception as e:
+        return Result(valid=False, error_type=ResultError.UNKNOWN, goal=human_readable_goal, data=output_data, message=e)
+

core/utils/basic_operations/wrap_with.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+from typing import TypeVar, Callable
+import functools
+
+"""
+Purpose:
+    Wraps one function with another
+
+Steps:
+    1) Takes the original function it's wrapping around, and does it
+    2) Passes that into the second function `wrapper_fn`
+    3) Returns the resulting object type
+
+Assumes:
+    Same object/type for both functions.
+"""
+
+T = TypeVar('T')  # Input type (what the original function returns)
+U = TypeVar('U')  # Output type (what wrapper_fn returns)
+
+def wrap_with(wrapper_fn: Callable[[T], U]) -> Callable:
+    def decorator(func: Callable[..., T]) -> Callable[..., U]:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> U:
+            result = func(*args, **kwargs)
+            return wrapper_fn(result)
+        return wrapper
+    return decorator
+
+
+# def wrap_with(wrapper_fn: Callable[[Result], Result]) -> Callable:
+#     def decorator(func: Callable[..., Result]) -> Callable[..., Result]:
+#         @functools.wraps(func)
+#         def wrapper(*args, **kwargs) -> Result:
+#             result = func(*args, **kwargs)
+#             processed_result = wrapper_fn(result)  # ← Any function plugged in
+#             return processed_result
+#         return wrapper
+#     return decorator
+

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sp-hydra-veil-core"
-version = "2.3.8"
+version = "2.4.3"
 authors = [
     { name = "Simplified Privacy" },
 ]
@@ -18,7 +18,7 @@ dependencies = [
     "pysocks ~= 1.7.1",
     "python-dateutil ~= 2.9.0.post0",
     "requests ~= 2.32.5",
-    "sp-essentials ~= 1.2.0",
+    "sp-essentials ~= 1.2.1",
     "annotated-types==0.7.0",
     "certifi==2026.4.22",
     "charset-normalizer==3.4.7",
@@ -45,6 +45,9 @@ dependencies = [
     "typing-inspection==0.4.2",
     "typing_extensions==4.15.0",
     "urllib3==2.6.3",
+    "httpx[http2]==0.28.1",
+    "dnspython==2.8.0",
+    "socksio==1.0.0",
 ]
 
 [project.urls]
@@ -56,4 +59,4 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
-packages = ["core"]
+packages = ["core"]-