Revamp of GET/POST API requests for both Tor and Clearweb. Why: Move towards more clear Tor API error feedback. And reduce redundancy. Status: Stable, works for both clear/tor and GET/POST. Still Needs: UI feedback, DNS Tor improvements

change_log.md

@@ -0,0 +1,9 @@
+# Major Change Log: 
+ 
+# 2.3.9  
+July 11, 2026  
+Features: Revamp of GET/POST API requests for both Tor and Clearweb.  
+Why: Move towards more clear Tor API error feedback.  And reduce redundancy.  
+Status: Stable, works for both clear/tor and GET/POST.  
+<br/>
+

core/controllers/tickets/TicketPayController.py

@@ -9,9 +9,16 @@ from core.models.invoice.TicketInvoice import TicketInvoice
 from core.services.prepare_tickets.get_pub_key import get_pub_key
 from core.observers.BaseObserver import BaseObserver
 from core.services.payment_phase.save_and_send_intitial_billing import save_and_send_intitial_billing
-from core.services.payment_phase.check_if_paid import _check_if_paid
+# from core.services.payment_phase.check_if_paid import _check_if_paid
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
+
 from core.services.prepare_tickets.ticket_tracker import does_ticket_tracker_exist
-from core.services.networking.send_data_to_server import send_data_to_server
+
+# from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+
+
 from core.services.networking.make_url import make_url
 # from core.utils.confirm_its_a_valid_key_choice import confirm_its_a_valid_key_choice
 from core.services.helpers.valid_profile_quantity import valid_profile_quantity
@@ -112,6 +119,14 @@ def initiate_payment(
             payload, connection_observer, invoice_data_object
         )
 
+        if isinstance(result, ApiResponse):
+            logger.error(result.message)
+            if not result.valid:
+                invoice_data_object.add_error_code("connection_error")
+                return invoice_data_object
+            else:
+                logger.error("Critical Error with TicketPayController recieving a valid ApiResponse object.")
+
         if result == False or result == None:
             invoice_data_object.add_error_code("failed_save")
 
@@ -169,8 +184,17 @@ def check_if_paid(
         url = make_url(which_endpoint)
 
         # literally send:
-        reply = send_data_to_server(payload, url, connection_observer)
+        api_reply_object = send_data_to_server(payload, url, connection_observer)
 
-        logger.debug(f"inside ticketpay controller the reply is {reply}")
+        # return the payload with GUI/CLI to interpret results:
+        if api_reply_object.valid:
+            reply_dict = api_reply_object.data
+            logger.debug(f"inside ticketpay controller the reply is {reply_dict}")
+            return reply_dict
 
-        return reply
+        # return the error of why we have no payload:
+        else:
+            error_msg = api_reply_object.message
+            error_msg = f"Connection/API Error: {error_msg}"
+            logger.error(f"inside ticketpay controller there was an a {error_msg}")
+            return {"valid": False, "message": error_msg}

core/controllers/tickets/TicketSyncController.py

@@ -7,12 +7,15 @@ if TYPE_CHECKING:
 
 from core.Constants import Constants
 from core.observers.BaseObserver import BaseObserver
-from core.services.networking.get_data_from_server import get_data_from_server
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
 from core.errors.logger import logger
 
 
 def sync_ticket_prices(
-    ticket_observer: TicketObserver, connection_observer: ConnectionObserver
+    ticket_observer: TicketObserver,
+    connection_observer: ConnectionObserver
 ) -> dict:
     notification = f"Connecting to get Ticket Pricing..."
     ticket_observer.notify("connecting", subject=notification)
@@ -28,13 +31,12 @@ def sync_ticket_prices(
 
     url = f"{base_url}/sync"
     try:
-        sync_results = get_data_from_server(url, connection_observer)
+        sync_results = get_data_from_api(url, None, connection_observer)
 
-        if sync_results in rejected_list:
+        if sync_results.valid:
+            return {"valid": True, "data": sync_results.data}
+        else:
             return {"valid": False, "error_code": "sync_failed"}
 
-        logger.debug(f"Inside the sync controller, sync_results is: {sync_results}")
     except:
         return {"valid": False, "error_code": "sync_failed"}
-
-    return sync_results

core/controllers/tickets/UseTicketController.py

@@ -8,6 +8,8 @@ if TYPE_CHECKING:
 from core.Constants import Constants
 from core.observers.BaseObserver import BaseObserver
 from core.services.using_tickets.use_ticket_orchestrator import use_ticket_orchestrator
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
 from core.services.prepare_tickets.ticket_tracker import (
     get_data_for_a_single_ticket,
     does_ticket_tracker_exist,
@@ -168,7 +170,12 @@ def use_ticket(
     ticket_observer.notify("connecting", "Connecting..")
     reply = use_ticket_orchestrator(which_ticket, which_location, connection_observer)
 
-    return reply
+    if isinstance(reply, ApiResponse):
+        error_msg = reply.message
+        reply_as_dict = {"valid": False, "message": f"API Failed: {error_msg}"}
+        return reply_as_dict
+    else:
+        return reply
 
 
 def pick_a_random_ticket(ticket_observer: TicketObserver) -> dict:

core/services/failed_verification/is_the_key_to_blame.py

@@ -11,7 +11,9 @@ from core.services.prepare_tickets.get_public_key_by_config import get_public_ke
 from core.services.prepare_tickets.get_pub_key import key_is_in_valid_format, get_pub_key
 from core.services.failed_verification.test_if_new_key_works import test_if_new_key_works
 from core.services.networking.make_url import make_url
-from core.services.networking.get_data_from_server import get_data_from_server
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+
+
 # utils
 from core.utils.basic_operations.write_or_read_from_json import get_value_from_json_file
 from core.utils.basic_operations.write_string_to_text_file import write_string_to_text_file
@@ -70,7 +72,7 @@ def get_new_pubkey_from_api(connection_observer: ConnectionObserver) -> dict | N
     url = make_url(which_key_plan)
 
     # the result of this is a python dictionary with single '
-    api_results = get_data_from_server(url, connection_observer)
+    api_results = get_data_from_server(url, None, connection_observer)
 
     if "data" in api_results:
         new_public_key = api_results["data"]

core/services/helpers/get_plan_data.py

@@ -5,13 +5,15 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 # services
 from core.services.networking.make_url import make_url
-from core.services.networking.send_data_to_server import send_data_to_server
+# from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
 
 
 # use temp billing to get the plan details
 def get_plan_data(
     temp_billing_code: str, connection_observer: ConnectionObserver
-) -> dict:
+) -> dict | ApiResponse:
     which_endpoint = "/plan"
     url = make_url(which_endpoint)
     payload = {"temp_billing_code": temp_billing_code}

core/services/networking/api_requests/ApiResponseModel.py

@@ -0,0 +1,40 @@
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional, Any
+
+class ErrorType(Enum):
+    """Classified error categories."""
+    SUCCESS = "success"
+    TOR_NOT_WORKING = "tor_not_working"
+    DNS_RESOLUTION = "dns_resolution"
+    NO_INTERNET = "no_internet"
+    NETWORK_ERROR = "network_error"
+    INVALID_INPUT = "invalid_input"
+    RATE_LIMITED = "rate_limited"
+    SERVER_ERROR = "server_error"
+    UNKNOWN = "unknown"
+
+@dataclass
+class ApiResponse:
+    """Unified response from all API operations."""
+    valid: bool
+    error_type: Optional[ErrorType] = None
+    data: Optional[Any] = None
+    message: Optional[str] = None
+    retry_now: bool = False
+    retry_later: bool = False
+    
+    def to_dict(self) -> dict:
+        """Convert to dict for backwards compatibility."""
+        return {
+            "valid": self.valid,
+            "error_code": self.error_type.value if self.error_type else None,
+            "data": self.data,
+            "message": self.message,
+            "retry_now": self.retry_now,
+            "retry_later": self.retry_later,
+        }
+    
+    def get(self, key: str, default=None):
+        """Dict-like access for backwards compatibility."""
+        return getattr(self, key, default)

core/services/networking/api_requests/step1_get_or_post.py

@@ -0,0 +1,98 @@
+from core.services.networking.api_requests.step2_execute import _execute_tor_request, _execute_regular_request
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
+from core.services.networking.api_requests.subtools.get_connection_type import get_connection_type
+from core.services.networking.api_requests.subtools.replace_http_with_https import replace_http_with_https
+
+from core.Constants import Constants
+from core.observers.BaseObserver import BaseObserver
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+from typing import Optional
+
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+
+def get_data_from_api(
+    full_request_url: str,
+    client_observer: Optional[ClientObserver] = None,
+    connection_observer: Optional[ConnectionObserver] = None,
+) -> ApiResponse:
+    """
+    GET wrapper. Desktop passes observer for updates; Android passes None.
+    """
+    if not full_request_url:
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.INVALID_INPUT,
+            message="Input URL is not properly configured"
+        )
+    
+    connection_type = get_connection_type()
+    
+    if connection_type == "tor":
+        # Tor: can push intermediate updates (desktop-only)
+        result_object = _execute_tor_request(
+            "get",
+            full_request_url,
+            None, # get request has no payload
+            connection_observer,
+            client_observer,
+            timeout=10
+        )
+    else:
+        # Regular, no tor:
+        result_object = _execute_regular_request(
+            "get",
+            full_request_url,
+            None, # get request has no payload
+            connection_observer,
+            client_observer,
+            timeout=5
+        )
+
+    if not result_object.valid and client_observer:
+        error_msg = result_object.message
+        client_observer.notify('synchronizing', f"Connection Issue: {error_msg}")
+    
+    return result_object
+
+
+def send_data_to_server(
+    payload: dict,
+    original_url: str,
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+) -> ApiResponse:
+    """
+    POST wrapper. Same pattern as GET.
+    """
+    url = replace_http_with_https(original_url)
+    connection_type = get_connection_type()
+    
+    if connection_type == "tor":
+        result_object = _execute_tor_request(
+            "post",
+            url,
+            payload,
+            connection_observer,
+            client_observer,
+            timeout=10
+        )
+    else:
+        result_object = _execute_regular_request(
+            "post",
+            url,
+            payload,
+            connection_observer,
+            client_observer,
+            timeout=5
+        )
+    
+    # Push final error to UI if provided
+    if not result_object.valid and client_observer:
+        error_msg = result_object.message
+        client_observer.notify('synchronizing', f"Connection Issue: {error_msg}")
+    
+    return result_object

core/services/networking/api_requests/step2_execute.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import requests
+
+
+from core.Constants import Constants
+
+# Networking Group
+from core.services.networking.api_requests.step3_classify_http import classify_http_response
+from core.services.networking.api_requests.step4_error_classifier import classify_request_error
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
+# tor
+from essentials.modules.TorModule import TorModule
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+
+from core.observers.ClientObserver import ClientObserver
+
+# errors
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+# generic
+import json, os
+from typing import Any
+import time
+
+def _execute_tor_request(
+    method: str,
+    url: str,
+    payload: Optional[dict],
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[ClientObserver] = None,
+    timeout: int = 6,
+) -> ApiResponse:
+    """
+    Tor request with optional intermediate UI feedback (desktop-only).
+    """
+    import requests
+    port_number = None
+    tor_module = None
+    
+    try:
+        if client_observer:
+            status_update = "Initializing Tor connection..."
+            client_observer.notify('synchronizing', status_update)
+
+        port_number = ConnectionService.get_random_available_port_number()
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+        
+        if client_observer:
+            status_update = f"Executing {method.upper()} via Tor..."
+            client_observer.notify('synchronizing', status_update)
+
+        proxies = {"http": f"socks5h://127.0.0.1:{port_number}",
+                   "https": f"socks5h://127.0.0.1:{port_number}"}
+        
+        if method.lower() == "get":
+            response = requests.get(url, proxies=proxies, timeout=timeout)
+        else:
+            response = requests.post(url, json=payload, proxies=proxies, timeout=timeout)
+        
+        result = classify_http_response(response)
+        
+    except Exception as e:
+        logger.debug(f"Tor {method.upper()} failed inside _execute_tor_request: {type(e).__name__}: {e}")
+        result = classify_request_error(url, "tor", connection_observer)
+    
+    finally:
+        if tor_module and port_number:
+            try:
+                tor_module.destroy_session(port_number)
+            except Exception as e:
+                logger.warning(f"Error destroying Tor session: {e}")
+    
+    return result
+
+
+def _execute_regular_request(
+    method: str,
+    url: str,
+    payload: Optional[dict],
+    connection_observer: Optional[ConnectionObserver] = None,
+    client_observer: Optional[client_observer] = None,
+    timeout: int = 5,
+) -> ApiResponse:
+    """
+    Regular (non-Tor) request. NO observer coupling.
+    Cross-platform reusable: desktop UI and Android/Kivy both consume ApiResponse.
+    """
+    import requests
+
+    try:
+        logger.debug(f"Executing {method.upper()} to {url}")
+        
+        if method.lower() == "get":
+            response = requests.get(url, timeout=timeout)
+        else:
+            response = requests.post(url, json=payload, timeout=timeout)
+        
+        return classify_http_response(response)
+    
+    except Exception as e:
+        logger.debug(f"{method.upper()} request failed: {type(e).__name__}: {e}")
+        return classify_request_error(url, "regular", connection_observer)

core/services/networking/api_requests/step3_classify_http.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    import requests
+
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+import json
+
+def classify_http_response(response) -> ApiResponse:
+    """
+    Classify HTTP response status and extract error info.
+    Replaces evaluate_response().
+    """
+    import requests
+
+    print("We're inside classify http")
+
+    if not isinstance(response, requests.Response):
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.NETWORK_ERROR,
+            message="Invalid response object"
+        )
+
+    print(f"We're inside classify http testing the status code of the response: {response.status_code}")
+
+    # Success case
+    if 200 <= response.status_code < 300:
+        try:
+            data = response.json()
+            return ApiResponse(valid=True, data=data)
+        except ValueError:
+            return ApiResponse(
+                valid=True,
+                data=response.text
+            )
+    
+    # Rate limiting
+    if response.status_code == 429:
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.RATE_LIMITED,
+            message="Rate limit exceeded",
+            retry_now=True
+        )
+    
+    # Client errors (4xx)
+    if 400 <= response.status_code < 500:
+        try:
+            resp_json = response.json()
+            error_msg = resp_json.get("message") or resp_json.get("error") or resp_json.get("error_code")
+        except ValueError:
+            error_msg = response.text
+        
+        error_map = {
+            404: ErrorType.INVALID_INPUT,
+            400: ErrorType.INVALID_INPUT,
+            401: ErrorType.INVALID_INPUT,
+            403: ErrorType.INVALID_INPUT,
+            405: ErrorType.INVALID_INPUT,
+        }
+        error_type = error_map.get(response.status_code, ErrorType.INVALID_INPUT)
+        
+        return ApiResponse(
+            valid=False,
+            error_type=error_type,
+            message=error_msg or f"Client error {response.status_code}"
+        )
+    
+    # Server errors (5xx)
+    if response.status_code >= 500:
+        retry_now = response.status_code in (502, 503, 504)
+        retry_later = response.status_code in (500, 501, 505)
+        
+        return ApiResponse(
+            valid=False,
+            error_type=ErrorType.SERVER_ERROR,
+            message=f"Server error {response.status_code}",
+            retry_now=retry_now,
+            retry_later=retry_later
+        )
+    
+    return ApiResponse(
+        valid=False,
+        error_type=ErrorType.UNKNOWN,
+        message=f"Unexpected status {response.status_code}"
+    )

core/services/networking/api_requests/step4_error_classifier.py

@@ -0,0 +1,81 @@
+# from __future__ import annotations
+# from typing import TYPE_CHECKING
+
+# if TYPE_CHECKING:
+#     from essentials.observers.ConnectionObserver import ConnectionObserver
+
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+
+
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
+# diagnosis services
+from core.services.networking.api_requests.subtools.internet_test import do_we_have_internet
+from core.services.networking.api_requests.subtools.is_dns_problem import is_dns_problem, is_dns_problem_via_tor
+from core.services.networking.api_requests.subtools.extract_domain import extract_domain
+from core.services.networking.api_requests.subtools.is_tor_working import is_tor_working
+
+
+# errors
+from core.errors.exceptions import *
+from core.errors.logger import logger
+
+from typing import Optional
+
+
+def classify_request_error(
+    url: str,
+    connection_type: str,
+    connection_observer: ConnectionObserver,
+    client_observer: Optional[ClientObserver] = None,
+) -> ApiResponse:
+    """
+    Unified diagnostic tree for all request failures.
+    Replaces evaluate_tor_networking_problem() and evaluate_regular_networking_problem().
+    """
+    domain_only = extract_domain(url)
+    
+    if connection_type == "tor":
+        # Tor-specific diagnostics
+        if not is_tor_working(connection_observer):
+            logger.debug("Tor is not working")
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.TOR_NOT_WORKING,
+                message="Tor connection failed"
+            )
+
+        logger.debug("Skipping DNS resolution via Tor due to PySocks support.")
+        # if is_dns_problem_via_tor(domain_only, connection_observer):
+        #     logger.debug("DNS resolution via Tor failed")
+        #     return ApiResponse(
+        #         valid=False,
+        #         error_type=ErrorType.DNS_RESOLUTION,
+        #         message=f"Cannot resolve {domain_only} via Tor"
+        #     )
+    else:
+        # Regular (non-Tor) diagnostics
+        if not do_we_have_internet():
+            logger.error("No internet connection")
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.NO_INTERNET,
+                message="No internet connectivity"
+            )
+        
+        if is_dns_problem(domain_only, connection_observer, client_observer):
+            logger.error("Local DNS resolution failed")
+            return ApiResponse(
+                valid=False,
+                error_type=ErrorType.DNS_RESOLUTION,
+                message=f"Cannot resolve {domain_only}"
+            )
+    
+    # Fallback: unknown error
+    logger.error(f"Unknown error for {connection_type} request to {domain_only}")
+    return ApiResponse(
+        valid=False,
+        error_type=ErrorType.UNKNOWN,
+        message="Request failed for unknown reason"
+    )

core/services/networking/api_requests/subtools/is_dns_problem.py

@@ -0,0 +1,87 @@
+# for tor:
+from essentials.modules.TorModule import TorModule
+from essentials.observers.ConnectionObserver import ConnectionObserver
+from essentials.services.ConnectionService import ConnectionService
+
+from core.observers.ClientObserver import ClientObserver
+from core.Constants import Constants
+
+import json, os
+from typing import Optional
+
+from core.services.networking.api_requests.subtools.extract_domain import extract_domain
+from core.errors.logger import logger
+import socket
+import socks
+
+"""
+Check if there's a DNS problem with a domain.
+Returns True if DNS fails (problem exists), False if DNS resolves successfully.
+"""
+
+
+def is_dns_problem(url: str) -> bool:
+    domain = extract_domain(url)
+    try:
+        socket.gethostbyname(domain)
+        return False  # DNS resolved successfully, no problem
+    except socket.gaierror:
+        return True  # DNS resolution failed, problem exists
+    except Exception:
+        return True  # Any other error is treated as a DNS problem
+
+
+def is_dns_problem_via_tor(
+    domain: str,
+    connection_observer: ConnectionObserver | None = None,
+    client_observer: Optional[ClientObserver] = None,
+) -> bool:
+    logger.debug("[DNS PROBLEM OF TOR SERVICE] We've triggered EVALUATING IF it's a DNS problem via Tor")
+
+    port_number = ConnectionService.get_random_available_port_number()
+    logger.debug(f"[DNS PROBLEM OF TOR SERVICE] Using Tor on port number {port_number}")
+    if client_observer is not None:
+        client_observer.notify('synchronizing', f"Using Tor on Port {port_number}..")
+
+    try:
+        tor_module = TorModule(Constants.HV_TOR_STATE_HOME)
+        tor_module.create_session(port_number, connection_observer)
+        logger.debug(f"[DNS PROBLEM OF TOR SERVICE] Tor Started a Session")
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Initialized Tor..")
+
+    except TorServiceInitializationError as e:
+        error_msg = f"[DNS PROBLEM OF TOR SERVICE] Tor failed to start: {e}"
+        tor_module.destroy_session(port_number)
+        logger.error(error_msg, exc_info=True)
+        if client_observer is not None:
+            client_observer.notify('synchronizing', f"Tor Failed to Initialize")
+
+    # proxies = {
+    #     "http": f"socks5h://127.0.0.1:{port_number}",
+    #     "https": f"socks5h://127.0.0.1:{port_number}",
+    # }
+
+    logger.debug(f"[DNS PROBLEM OF TOR SERVICE] About to try to apply the proxy but may get a PySocks error...")
+    try:
+        # Set up SOCKS5 proxy using the same port as the existing Tor proxy,
+        socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", port_number)
+        socket.socket = socks.socksocket
+    except:
+        logger.error(f"[DNS PROBLEM OF TOR SERVICE] Error with setting up the socks5 proxy using socks.set_deault0")
+        tor_module.destroy_session(port_number)
+        return True
+       
+    # DNS query check
+    try:
+        logger.debug("checking dns via socket..")
+        socket.gethostbyname(domain)
+
+        tor_module.destroy_session(port_number)
+
+        return False  # DNS resolved successfully, no problem
+
+    except socket.gaierror:
+        return True  # DNS resolution failed, problem exists
+    except Exception:
+        return True  # Any other error is treated as a DNS problem

core/services/networking/api_requests/subtools/replace_http_with_https.py

@@ -0,0 +1,7 @@
+
+
+
+def replace_http_with_https(url):
+    if url.startswith("http://"):
+        return url.replace("http://", "https://", 1)
+    return url

core/services/networking/evaluate_networking_problem.py

@@ -1,76 +0,0 @@
-from __future__ import annotations
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from essentials.observers.ConnectionObserver import ConnectionObserver
-# services
-from core.services.networking.internet_test import do_we_have_internet
-from core.services.networking.is_dns_problem import is_dns_problem, is_dns_problem_via_tor
-from core.services.networking.extract_domain import extract_domain
-from core.services.networking.is_tor_working import is_tor_working
-# errors
-from core.errors.exceptions import *
-from core.errors.logger import logger
-
-#######################################################
-############### START : REGULAR SYSTEM  ###############
-#######################################################
-
-
-def evaluate_regular_networking_problem(
-    url: str,
-    connection_observer: ConnectionObserver,
-    client_observer: Optional[ClientObserver] = None
-) -> dict:
-
-    domain_only = extract_domain(url)
-
-    if not do_we_have_internet():
-        return {"valid": False, "error_code": "no_internet"}
-
-    if is_dns_problem(domain_only):
-        logger.debug("This is a DNS issue.")
-        return {"valid": False, "error_code": "dns_issue"}
-
-    return {"valid": False, "error_code": "unknown"}
-    #######################################################
-    ############### END: REGULAR SYSTEM  ###############
-    #######################################################
-
-    #######################################################
-    ###############      START: TOR         ###############
-    #######################################################
-
-
-def evaluate_tor_networking_problem(
-    url: str,
-    connection_observer: ConnectionObserver,
-    client_observer: Optional[ClientObserver] = None,
-) -> dict:
-
-    domain_only = extract_domain(url)
-    logger.debug(f"Evaluating a networking problem for a Tor connection")
-
-    logger.debug("checking if Tor works against the official Tor Project API..")
-    if not is_tor_working(connection_observer):
-        return {"valid": False, "error_code": "tor_issue"}
-    else:
-        tor_fine = f"While there were connection issues, but Tor is working fine!"
-        logger.error(tor_fine, exc_info=True)
-        logger.debug(tor_fine)
-
-    logger.debug("Check the original DNS via Tor...")
-
-    if is_dns_problem_via_tor(domain_only, connection_observer):
-        return {"valid": False, "error_code": "dns_issue"}
-    else:
-        dns_fine = f"There were connection issues, but the DNS for {domain_only} via Tor is working fine!"
-        logger.error(dns_fine, exc_info=True)
-        logger.debug(dns_fine)
-
-    # can't solve it:
-    return {"valid": False, "error_code": "unknown"}
-
-    #######################################################
-    ###############      END: TOR           ###############
-    #######################################################

core/services/networking/is_dns_problem.py

@@ -1,56 +0,0 @@
-# for tor:
-from essentials.modules.TorModule import TorModule
-from essentials.observers.ConnectionObserver import ConnectionObserver
-from essentials.services.ConnectionService import ConnectionService
-import json, os
-
-# for both:
-from core.services.networking.extract_domain import extract_domain
-from core.errors.logger import logger
-import socket
-import socks
-
-"""
-Check if there's a DNS problem with a domain.
-Returns True if DNS fails (problem exists), False if DNS resolves successfully.
-"""
-
-
-def is_dns_problem(url: str) -> bool:
-    domain = extract_domain(url)
-    try:
-        socket.gethostbyname(domain)
-        return False  # DNS resolved successfully, no problem
-    except socket.gaierror:
-        return True  # DNS resolution failed, problem exists
-    except Exception:
-        return True  # Any other error is treated as a DNS problem
-
-
-def is_dns_problem_via_tor(
-    domain: str,
-    connection_observer: ConnectionObserver | None = None,
-) -> bool:
-    logger.debug("We've triggered EVALUATING IF it's a DNS problem via Tor")
-
-    port_number = ConnectionService.get_random_available_port_number()
-    tor_module = TorModule(os.path.expanduser("~/sp-tor-test"))
-    tor_module.create_session(port_number, connection_observer)
-
-    # Set up SOCKS5 proxy using the same port as the existing Tor proxy,
-    socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", port_number)
-    socket.socket = socks.socksocket
-
-    # DNS query check
-    try:
-        logger.debug("checking dns via socket..")
-        socket.gethostbyname(domain)
-
-        tor_module.destroy_session(port_number)
-
-        return False  # DNS resolved successfully, no problem
-
-    except socket.gaierror:
-        return True  # DNS resolution failed, problem exists
-    except Exception:
-        return True  # Any other error is treated as a DNS problem

core/services/payment_phase/check_if_paid.py

@@ -1,31 +0,0 @@
-from __future__ import annotations
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from essentials.observers.ConnectionObserver import ConnectionObserver
-# services:
-from core.services.networking.send_data_to_server import send_data_to_server
-from core.services.networking.make_url import make_url
-# errors:
-from core.errors.exceptions import *
-from core.errors.logger import logger
-
-
-# it's private because it assumes it's being called from the controller, (with a JSON)
-def _check_if_paid(payload: dict, connection_observer: ConnectionObserver) -> str:
-    # prep endpoint:
-    which_endpoint = "check_paid"
-    url = make_url(which_endpoint)
-
-    # literally send:
-    reply = send_data_to_server(payload, url, connection_observer)
-
-    if "valid" in reply:
-        valid_status = reply.get("valid")
-        if valid_status == True:
-            return "paid"
-        else:
-            return "not_paid"
-    else:
-        error_msg = f"When checking if paid, the Server returned invalid data or it never sent.  The reply is {reply}"
-        raise InvalidData(error_msg)

core/services/payment_phase/save_and_send_intitial_billing.py

@@ -5,7 +5,12 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 #
 from core.models.invoice.TicketInvoice import TicketInvoice
-from core.services.networking.send_data_to_server import send_data_to_server
+
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+# from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
+
 from core.services.networking.make_url import make_url
 from core.services.payment_phase.extract_payment_details import extract_payment_details
 from core.services.payment_phase.save_billing_choices import save_billing_choices
@@ -18,7 +23,22 @@ def save_and_send_intitial_billing(
     payload: dict,
     connection_observer: ConnectionObserver,
     invoice_data_object: TicketInvoice,
-) -> TicketInvoice:
+) -> TicketInvoice | ApiResponse:
+
+    """
+    Purpose:
+        Does a POST request to get updated billing information.
+
+    Returns:
+        TicketInvoice on Success, with the modified values.
+        ApiResponse on Failure, with why the API failed.
+
+    Called by:
+        TicketPayController
+
+    Errors:
+        No. Does NOT raise errors.
+    """
 
     # Save choices:
     save_billing_choices(payload)
@@ -26,9 +46,14 @@ def save_and_send_intitial_billing(
     # send them:
     which_endpoint = "start_payment"
     url = make_url(which_endpoint)
-    reply = send_data_to_server(payload, url, connection_observer)
+    api_reply_object = send_data_to_server(payload, url, connection_observer)
+
+    if not api_reply_object.valid:
+        return api_reply_object
+
+    reply_dict_data = api_reply_object.data
 
     # extract values from server's reply:
-    modified_data_object = extract_payment_details(reply, invoice_data_object)
+    modified_data_object = extract_payment_details(reply_dict_data, invoice_data_object)
 
     return modified_data_object

core/services/prepare_tickets/get_pub_key.py

@@ -6,7 +6,8 @@ if TYPE_CHECKING:
 from typing import Any
 # services
 from core.services.networking.make_url import make_url
-from core.services.networking.get_data_from_server import get_data_from_server
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+
 # utils
 from core.utils.basic_operations.does_file_exist import does_file_exist
 from core.utils.basic_operations.write_string_to_text_file import write_string_to_text_file

core/services/prepare_tickets/get_public_key_by_config.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 # services & helpers
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
 from core.services.prepare_tickets.get_pub_key import get_pub_key
 from core.services.prepare_tickets.get_pub_key import key_is_in_valid_format
 from core.services.helpers.get_plan_data import get_plan_data
@@ -55,12 +56,25 @@ def get_public_key_by_config(connection_observer: ConnectionObserver) -> dict:
 
         # Now we're assuming we have a temp billing code,
         # So we can use that to get the key from the server,
-        reply = get_plan_data(temp_billing_code, connection_observer)
+        api_reply_object = get_plan_data(temp_billing_code, connection_observer)
 
-        if reply in list_of_failures:
+        complete_failure_msg = {
-            return {
                 "status": False,
                 "message": f"Issues with both finding your local key plan and even connecting to the server for it. Please check {filepath}",
+        }
+
+        if not api_reply_object.valid:
+            return complete_failure_msg
+        
+        reply = api_reply_object.data
+
+        if reply in list_of_failures:
+            return complete_failure_msg
+
+        if not instance(reply, dict):
+            return {
+                "status": False,
+                "message": f"Server returned an invalid format, and even accessing via local files. Please check {filepath}",
             }
 
         # from the server's reply, get the key
@@ -112,11 +126,25 @@ def get_public_key_from_LOCAL_files_only(
                     "message": f"Issue with finding key plan or billing code. Please check {filepath}",
                 }
 
-            reply = get_plan_data(temp_billing_code, connection_observer)
+            api_reply_object = get_plan_data(temp_billing_code, connection_observer)
-            if reply in list_of_failures:
+
-                return {
+            complete_failure_msg = {
                     "status": False,
                     "message": f"Issues with both finding your local key plan and even connecting to the server for it. Please check {filepath}",
+            }
+
+            if not api_reply_object.valid:
+                return complete_failure_msg
+            
+            reply = api_reply_object.data
+
+            if reply in list_of_failures:
+                return complete_failure_msg
+
+            if not instance(reply, dict):
+                return {
+                    "status": False,
+                    "message": f"Server returned an invalid format, and even accessing via local files. Please check {filepath}",
                 }
 
             which_key = filter_data(reply, "which_key")

core/services/prepare_tickets/send_blind_commitments.py

@@ -5,7 +5,9 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
     from core.observers.TicketObserver import TicketObserver
 # services
-from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
 from core.services.networking.make_url import make_url
 from core.services.helpers.get_which_billing_key import get_which_billing_key
 # utils
@@ -44,9 +46,26 @@ def send_blind_commitments(
         # send it:
         which_endpoint = "sign"
         url = make_url(which_endpoint)
-        reply = send_data_to_server(payload, url, connection_observer)
+        api_reply_object = send_data_to_server(payload, url, connection_observer)
+
+        # did API call work?
+        if not api_reply_object.valid:
+            error_msg = api_reply_object.message
+            final_error_msg = f"API/Connection Error {error_msg}"
+            logger.error(final_error_msg)
+            ticket_observer.notify("error", subject=final_error_msg)
+            return None
+
+        # assuming it worked, extract the data:
+        reply = api_reply_object.data
+
+        # is it in valid format?
+        if not isinstance(reply, dict):
+            final_error_msg = f"API returned an invalid format {reply}"
+            logger.error(final_error_msg)
+            ticket_observer.notify("error", subject=final_error_msg)
+            return None
 
-        # did it work?
         if reply.get("valid") == True:
             # this 'signed_data' variable is a list,
             signed_data = reply.get("signed_data")

core/services/sync/get_data_from_api.py

@@ -1,79 +0,0 @@
-from core.Constants import Constants
-from core.observers.BaseObserver import BaseObserver
-from core.services.networking.get_data_from_server import get_data_from_server
-from core.errors.logger import logger
-from core.observers.ClientObserver import ClientObserver
-from core.observers.ConnectionObserver import ConnectionObserver
-from typing import Optional
-
-
-# this can be spoofed with the isolation testing comment below
-
-def get_data_from_api(
-    endpoint: str,
-    client_observer: Optional[ClientObserver] = None,
-    connection_observer: Optional[ConnectionObserver] = None
-) -> dict:
-    logger.info("Syncing with the API in get_data_from_api...")
-
-    # Get and validate the base URL
-    rejected_list = [None, False, ""]
-    base_url = Constants.SP_API_BASE_URL
-    if base_url in rejected_list:
-        error_msg = "Invalid base URL from configuration"
-        logger.error(error_msg)
-        return {"success": False, "data": None, "error": error_msg}
-
-    # Construct the full endpoint URL
-    url = f"{base_url}/{endpoint}"
-    logger.debug(f"API endpoint: {url}")
-
-    try:
-        logger.debug("Fetching data from API server...")
-        sync_results = get_data_from_server(url, connection_observer, client_observer)
-
-        if sync_results in rejected_list:
-            error_msg = "API returned no data"
-            logger.error(error_msg)
-            return {"success": False, "data": None, "error": error_msg}
-
-        logger.debug(f"Raw API response: {sync_results}")
-
-        final_result = sync_results.get("data", None)
-        final_result = sync_results.get("data", None)
-
-        if final_result:
-            logger.info("Successfully retrieved sync versions metadata from API")
-            return {"success": True, "data": final_result, "error": None}
-        else:
-            error_msg = f"API response missing 'data' field at the end of get_data_from_api. This is the response from the other module: {sync_results}"
-            logger.error(error_msg)
-            return {"success": False, "data": None, "error": error_msg}
-
-    except Exception as e:
-        error_msg = f"API fetch failed: {str(e)}"
-        logger.error(error_msg)
-        return {"success": False, "data": None, "error": error_msg}
-
-
-
-
-
-
-# ============== ISOLATION TESTING ==================
-# SPOOF ISOLATION TEST:
-# def _get_sync_cache_from_api(
-#     client_observer: Optional[ClientObserver] = None,
-#     connection_observer: Optional[ConnectionObserver] = None
-# ) -> dict:
-#     final_result = {
-#         "version": 9,
-#         "applications": 5,
-#         "application_versions": 4,
-#         "client_version": 3,
-#         "operators": 2,
-#         "locations": 3,
-#         "subscriptions": 1
-#     }
-#     return {"success": True, "data": final_result, "error": None}
-

core/services/sync/orm_methods/sync_one_orm.py

@@ -1,4 +1,7 @@
-from core.services.sync.get_data_from_api import get_data_from_api
+# from core.services.sync.get_data_from_api import get_data_from_api
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse
+
 from core.models.manage.insert import insert_into_model
 from core.models.manage.denormalize import denormalize
 from core.models.orm_models.Base import Base
@@ -6,6 +9,7 @@ from core.observers.ClientObserver import ClientObserver
 from core.observers.ConnectionObserver import ConnectionObserver
 from core.errors.logger import logger
 from core.utils.basic_operations.get_parent_directory import get_parent_directory
+from core.Constants import Constants
 
 from typing import Optional
 import json
@@ -17,17 +21,18 @@ def sync_one_orm_model(
     connection_observer: Optional[ConnectionObserver] = None
 ) -> dict:
 
-    api_result = get_data_from_api(which_endpoint, ClientObserver, ConnectionObserver)
+    full_request_url = f"{Constants.SP_API_BASE_URL}/{which_endpoint}"
+    api_result = get_data_from_api(full_request_url, ClientObserver, ConnectionObserver)
 
-    if not api_result.get("success"):
+    if not api_result.valid:
-        error_msg = api_result.get("error", "Unknown API error")
+        error_msg = api_result.message
         logger.error(f"API sync failed for {which_endpoint} with: {error_msg}")
         return {
             "success": False,
             "error": error_msg
         }
 
-    new_data = api_result.get("data")
+    new_data = api_result.data
 
     # prep data (denormalize)
     parent_directory = get_parent_directory()

core/services/sync/sync_service.py

@@ -1,7 +1,10 @@
 
 # comparisons & api calls:
 from core.services.sync.compare_tables import compare_tables
-from core.services.sync.get_data_from_api import get_data_from_api
+# from core.services.sync.get_data_from_api import get_data_from_api
+
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse
 
 # ORM for metadata:
 from core.models.manage.session_management import get_session
@@ -147,11 +150,13 @@ def coordinate_cache_sync(
         - 'filtered_metadata': the metadata from the server after removing keys not in the ORM
     """
     
-    api_result = get_data_from_api("cachedsync", client_observer, connection_observer)
+    full_request_url = f"{Constants.SP_API_BASE_URL}/cachedsync"
+    api_result = get_data_from_api(full_request_url, client_observer, connection_observer)
 
     # we trust the 'get_data_from_api' function to give us a dictionary:
-    if not api_result.get("success"):
+    if not api_result.valid:
-        error_msg = api_result.get("error", "Unknown API error")
+        print(f"Sync service says its not a valid reply")
+        error_msg = api_result.message
         logger.error(f"API sync failed: {error_msg}")
         return {
             "success": False,
@@ -161,7 +166,11 @@ def coordinate_cache_sync(
         }
 
     # Extract the data from the client's own trusted function,
-    new_data = api_result.get("data")
+    new_data = api_result.data
+    print(f"inside sync_service, new_data is {new_data}")
+
+    if new_data:
+        logger.debug(f"We got an API response with some data, let's check if it's a valid format...")
 
     # Validate the API's payload, (we don't trust the API's structure)
     if not isinstance(new_data, dict):

core/services/using_tickets/send_unblinded.py

@@ -3,8 +3,11 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
+
 # services & helpers
-from core.services.networking.send_data_to_server import send_data_to_server
+from core.services.networking.api_requests.step1_get_or_post import send_data_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
 from core.services.networking.make_url import make_url
 from core.services.helpers.get_which_billing_key import get_which_billing_key
 # utils
@@ -20,7 +23,7 @@ def send_unblinded_ticket_to_server(
     which_ticket: int,
     which_location: str,
     connection_observer: ConnectionObserver,
-) -> dict:
+) -> dict|ApiResponse:
 
     try:
         ticket_data = get_raw_string(which_ticket, "unblinded_final_ticket")
@@ -46,9 +49,9 @@ def send_unblinded_ticket_to_server(
         # send it:
         which_endpoint = "validate"
         url = make_url(which_endpoint)
-        reply = send_data_to_server(payload, url, connection_observer)
+        reply_object = send_data_to_server(payload, url, connection_observer)
 
-        return reply
+        return reply_object
 
     except Exception as e:
         human_readable_error_msg = f"The send_unblinded_ticket_to_server function's try-except block failed for ticket {which_ticket}.  Returning False..."

core/services/using_tickets/use_ticket_orchestrator.py

@@ -5,6 +5,8 @@ if TYPE_CHECKING:
     from essentials.observers.ConnectionObserver import ConnectionObserver
 ##############
 from core.services.using_tickets.send_unblinded import send_unblinded_ticket_to_server
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+
 from core.utils.basic_operations.write_or_read_from_json import (
     update_value_in_json_with_two_values,
 )
@@ -21,25 +23,33 @@ def use_ticket_orchestrator(
     which_ticket: int,
     which_location: str,
     connection_observer: ConnectionObserver,
-) -> dict:
+) -> dict | ApiResponse:
 
     # prep the ticket tracker:
     billing_folder = Constants.HV_TICKETING_CONFIG_HOME
     ticket_tracker_path = f"{billing_folder}/ticket_tracker.json"
 
     # send to server:
-    reply = send_unblinded_ticket_to_server(
+    api_reply_object = send_unblinded_ticket_to_server(
         which_ticket, which_location, connection_observer
     )
 
-    logger.debug(f"reply is {reply}")
+    if not api_reply_object.valid:
+        return api_reply_object
 
-    if "valid" not in reply:
+    raw_data_dict = api_reply_object.data
+
+    logger.debug(f"raw_data_dict is {raw_data_dict}")
+
+    if not isinstance(raw_data_dict, dict):
         return {"valid": False, "message": "invalid_format"}
-    is_it_valid = reply.get("valid")
 
-    if "billing_code" in reply:
+    if "valid" not in raw_data_dict:
-        billing_code = reply.get("billing_code", None)
+        return {"valid": False, "message": "invalid_format"}
+    is_it_valid = raw_data_dict.get("valid")
+
+    if "billing_code" in raw_data_dict:
+        billing_code = raw_data_dict.get("billing_code", None)
 
     """
     the reason this 'billing_code' check is a seperate function (and before the validity test),
@@ -63,10 +73,10 @@ def use_ticket_orchestrator(
             )
         return {"valid": True, "billing_code": billing_code}
 
-    if "message" not in reply:
+    if "message" not in raw_data_dict:
         return {"valid": False, "message": "invalid_format"}
 
-    message = reply.get("message")
+    message = raw_data_dict.get("message")
 
     if message == "already_used":
         # update the ticket tracker to reflect that it's already used with that billing code:
@@ -84,7 +94,7 @@ def use_ticket_orchestrator(
                 subscription_update, which_ticket, billing_code, billing_folder
             )
 
-    return reply
+    return raw_data_dict
 
 
 def make_sure_sub_saved(

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sp-hydra-veil-core"
-version = "2.3.8"
+version = "2.3.9"
 authors = [
     { name = "Simplified Privacy" },
 ]