From 1722905e62320fdcfd77700257a799aa411f0327 Mon Sep 17 00:00:00 2001 From: SimplifiedPrivacy Date: Mon, 13 Jul 2026 15:50:04 -0400 Subject: [PATCH] Added Support for Retries of Tor DNS failures with a custom HTTPX DNS resolver using DoH Quad9 via Tor, to then go direct. And setup ticket and sync modules to use this new error solver. --- core/controllers/ClientController.py | 3 - .../tickets/TicketPayController.py | 32 +- .../tickets/TicketSyncController.py | 23 +- .../api_requests/ApiResponseModel.py | 5 + .../api_requests/step3_classify_http.py | 2 - .../api_requests/step4_error_classifier.py | 5 +- .../api_requests/step5_solve_api_problems.py | 60 +++ .../subtools/custom_httpx_dns_resolver.py | 46 +++ .../api_requests/subtools/direct_dns_tools.py | 343 ++++++++++++++++++ .../api_requests/subtools/extract_domain.py | 33 +- .../save_and_send_intitial_billing.py | 12 +- .../prepare_tickets/send_blind_commitments.py | 18 +- .../services/sync/orm_methods/sync_one_orm.py | 24 +- core/services/sync/sync_service.py | 34 +- core/services/using_tickets/send_unblinded.py | 16 +- .../using_tickets/use_ticket_orchestrator.py | 8 +- core/test_dns.py | 42 --- pyproject.toml | 3 + 18 files changed, 614 insertions(+), 95 deletions(-) create mode 100644 core/services/networking/api_requests/step5_solve_api_problems.py create mode 100644 core/services/networking/api_requests/subtools/custom_httpx_dns_resolver.py create mode 100644 core/services/networking/api_requests/subtools/direct_dns_tools.py delete mode 100644 core/test_dns.py diff --git a/core/controllers/ClientController.py b/core/controllers/ClientController.py index 7e1ea66..759fc4a 100644 --- a/core/controllers/ClientController.py +++ b/core/controllers/ClientController.py @@ -55,14 +55,11 @@ class ClientController: @staticmethod def sync(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None): - if client_observer is not None: client_observer.notify('synchronizing', "Fetching list of new data ..") result = coordinate_cache_sync(client_observer, connection_observer) - # logger.info(f"We got a Result from the API of {result}") - # Outright Error: if not result["success"]: error_msg = result["error"] diff --git a/core/controllers/tickets/TicketPayController.py b/core/controllers/tickets/TicketPayController.py index db1ecd3..05423aa 100644 --- a/core/controllers/tickets/TicketPayController.py +++ b/core/controllers/tickets/TicketPayController.py @@ -9,8 +9,8 @@ from core.models.invoice.TicketInvoice import TicketInvoice from core.services.prepare_tickets.get_pub_key import get_pub_key from core.observers.BaseObserver import BaseObserver from core.services.payment_phase.save_and_send_intitial_billing import save_and_send_intitial_billing -# from core.services.payment_phase.check_if_paid import _check_if_paid from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems from core.services.prepare_tickets.ticket_tracker import does_ticket_tracker_exist @@ -186,15 +186,23 @@ def check_if_paid( # literally send: api_reply_object = send_data_to_server(payload, url, connection_observer) - # return the payload with GUI/CLI to interpret results: - if api_reply_object.valid: - reply_dict = api_reply_object.data - logger.debug(f"inside ticketpay controller the reply is {reply_dict}") - return reply_dict + if not api_reply_object.valid: + api_reply_object = solve_api_problems( + api_reply_object=api_reply_object, + get_or_post="post", + url=url, + payload=payload, + connection_observer=connection_observer, + client_observer=None + ) + if not api_reply_object.valid: + # 2nd try, return the error of why we have no payload: + error_msg = f"Connection/API Error: {api_reply_object.message}" + logger.error(f"[TICKET PayController] 2nd Post Request inside ticketpay controller had a {error_msg}") + return {"valid": False, "message": error_msg} + + # return the payload with GUI/CLI to interpret results: + reply_dict = api_reply_object.data + logger.debug(f"[TICKET PayController] We have a valid reply from the API inside ticketpay controller of {reply_dict}") + return reply_dict - # return the error of why we have no payload: - else: - error_msg = api_reply_object.message - error_msg = f"Connection/API Error: {error_msg}" - logger.error(f"inside ticketpay controller there was an a {error_msg}") - return {"valid": False, "message": error_msg} diff --git a/core/controllers/tickets/TicketSyncController.py b/core/controllers/tickets/TicketSyncController.py index 186d53d..43879c1 100644 --- a/core/controllers/tickets/TicketSyncController.py +++ b/core/controllers/tickets/TicketSyncController.py @@ -9,6 +9,7 @@ from core.Constants import Constants from core.observers.BaseObserver import BaseObserver from core.services.networking.api_requests.step1_get_or_post import get_data_from_api from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems from core.errors.logger import logger @@ -31,12 +32,24 @@ def sync_ticket_prices( url = f"{base_url}/sync" try: - sync_results = get_data_from_api(url, None, connection_observer) + api_result = get_data_from_api(url, None, connection_observer) - if sync_results.valid: - return {"valid": True, "data": sync_results.data} - else: - return {"valid": False, "error_code": "sync_failed"} + if not api_result.valid: + error_msg = api_result.message + logger.error(f"[TICKET SYNC Controller] There's an issue with the sync of endpoint {url} the API Reply: {error_msg}") + api_result = solve_api_problems( + api_reply_object=api_result, + get_or_post="get", + url=url, + payload=None, + connection_observer=connection_observer, + client_observer=None + ) + # 2nd try: + if not api_result.valid: + return {"valid": False, "error_code": "sync_failed"} + + return {"valid": True, "data": api_result.data} except: return {"valid": False, "error_code": "sync_failed"} diff --git a/core/services/networking/api_requests/ApiResponseModel.py b/core/services/networking/api_requests/ApiResponseModel.py index a94c45a..9d0e879 100644 --- a/core/services/networking/api_requests/ApiResponseModel.py +++ b/core/services/networking/api_requests/ApiResponseModel.py @@ -6,12 +6,15 @@ class ErrorType(Enum): """Classified error categories.""" SUCCESS = "success" TOR_NOT_WORKING = "tor_not_working" + TOR_DNS_BLOCKED = "tor_dns_blocked" DNS_RESOLUTION = "dns_resolution" + QUAD9_DNS_RESOLUTION = "quad9_dns_resolution" NO_INTERNET = "no_internet" NETWORK_ERROR = "network_error" INVALID_INPUT = "invalid_input" RATE_LIMITED = "rate_limited" SERVER_ERROR = "server_error" + DEVELOPER_ERROR = "developer_error" UNKNOWN = "unknown" @dataclass @@ -25,6 +28,7 @@ class ApiResponse: retry_later: bool = False tor: bool = False ip_address: str = None + ask_clearweb: bool = None def to_dict(self) -> dict: """Convert to dict for backwards compatibility.""" @@ -36,6 +40,7 @@ class ApiResponse: "retry_now": self.retry_now, "retry_later": self.retry_later, "tor": self.tor, + "ask_clearweb": self.ask_clearweb } def get(self, key: str, default=None): diff --git a/core/services/networking/api_requests/step3_classify_http.py b/core/services/networking/api_requests/step3_classify_http.py index 82b1ae5..4cbc8e6 100644 --- a/core/services/networking/api_requests/step3_classify_http.py +++ b/core/services/networking/api_requests/step3_classify_http.py @@ -13,8 +13,6 @@ def classify_http_response(response) -> ApiResponse: """ import requests - print("We're inside classify http") - if not isinstance(response, requests.Response): return ApiResponse( valid=False, diff --git a/core/services/networking/api_requests/step4_error_classifier.py b/core/services/networking/api_requests/step4_error_classifier.py index dfb2f6d..a5f2797 100644 --- a/core/services/networking/api_requests/step4_error_classifier.py +++ b/core/services/networking/api_requests/step4_error_classifier.py @@ -25,7 +25,10 @@ def classify_request_error( client_observer: Optional[ClientObserver] = None, ) -> ApiResponse: """ - Classify all error requests, Tor or Not. + Classify all error requests, + Tor Connection. + DNS + Internet Connection """ domain_only = extract_domain(url) diff --git a/core/services/networking/api_requests/step5_solve_api_problems.py b/core/services/networking/api_requests/step5_solve_api_problems.py new file mode 100644 index 0000000..a10b063 --- /dev/null +++ b/core/services/networking/api_requests/step5_solve_api_problems.py @@ -0,0 +1,60 @@ +from core.errors.logger import logger +from core.Constants import Constants +from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from essentials.observers.ConnectionObserver import ConnectionObserver +from essentials.services.ConnectionService import ConnectionService +from core.observers.ClientObserver import ClientObserver + +# tools to solve: +from core.services.networking.api_requests.subtools.direct_dns_tools import get_DNS_then_use_it + +# generic +from typing import Optional + + +def solve_api_problems( + api_reply_object: ApiResponse, + get_or_post: str, + url: str, + payload: str = None, + connection_observer: Optional[ConnectionObserver] = None, + client_observer: Optional[ClientObserver] = None, +) -> ApiResponse: + + if api_reply_object.valid: + logger.debug(f"[API SOLVER] The API call worked, so there's no reason to try to solve it.") + return api_reply_object + + reason_for_error = api_reply_object.error_type + + cant_be_solved = [ErrorType.QUAD9_DNS_RESOLUTION, ErrorType.UNKNOWN, ErrorType.NO_INTERNET] + + if reason_for_error in cant_be_solved: + logger.debug(f"[API SOLVER] This can't be solved if the reason is {reason_for_error}") + return api_reply_object + + elif reason_for_error == ErrorType.DEVELOPER_ERROR: + logger.error(f"[API SOLVER] The developer made an error {api_reply_object.message}") + return api_reply_object + + elif reason_for_error == ErrorType.DNS_RESOLUTION: + logger.debug(f"[API SOLVER] We're solving a DNS resolution error {api_reply_object.message}") + if api_reply_object.tor: + dns_fix = get_DNS_then_use_it(get_or_post, url, payload, connection_observer, client_observer) + return dns_fix + else: + logger.error(f"[API SOLVER] We have NOT yet setup clearweb DNS solutions") + return api_reply_object + + elif reason_for_error == ErrorType.TOR_NOT_WORKING: + api_reply_object.ask_clearweb = True + return api_reply_object + + elif reason_for_error == ErrorType.NETWORK_ERROR: + if api_reply_object.tor: + api_reply_object.ask_clearweb = True + return api_reply_object + + else: + logger.error(f"[API SOLVER] Unknown reason {reason_for_error} with message: {api_reply_object.message}") + return api_reply_object diff --git a/core/services/networking/api_requests/subtools/custom_httpx_dns_resolver.py b/core/services/networking/api_requests/subtools/custom_httpx_dns_resolver.py new file mode 100644 index 0000000..2f34f6f --- /dev/null +++ b/core/services/networking/api_requests/subtools/custom_httpx_dns_resolver.py @@ -0,0 +1,46 @@ +import json +import httpx +import ssl +import certifi +from unittest.mock import patch +import httpcore + +def create_httpx_client_with_custom_dns(hostname_to_ip_map: dict, custom_proxy: str): + """ + Purpose: + Creates an httpx.Client that connects to IPs while verifying certificates + against the original hostnames (via SNI). + """ + + ssl_context = ssl.create_default_context(cafile=certifi.where()) + ssl_context.check_hostname = True + ssl_context.verify_mode = ssl.CERT_REQUIRED + + # Store the original wrap_socket method + original_wrap_socket = ssl.SSLContext.wrap_socket + + def patched_wrap_socket(self, sock, *args, **kwargs): + # Inject server_hostname for IPs in our map + for hostname, ip in hostname_to_ip_map.items(): + try: + peer_addr = sock.getpeername() + if peer_addr[0] == ip: + kwargs['server_hostname'] = hostname + break + except OSError: + pass + return original_wrap_socket(self, sock, *args, **kwargs) + + # Apply the patch PERMANENTLY + ssl.SSLContext.wrap_socket = patched_wrap_socket + + # Build mounts + transport = httpx.HTTPTransport(verify=ssl_context, http2=True) + mounts = {} + for hostname, ip in hostname_to_ip_map.items(): + mounts[f"https://{hostname}"] = transport + mounts[f"https://{ip}"] = transport + + # Create and return client + client = httpx.Client(mounts=mounts, http2=True, proxy=custom_proxy) + return client diff --git a/core/services/networking/api_requests/subtools/direct_dns_tools.py b/core/services/networking/api_requests/subtools/direct_dns_tools.py new file mode 100644 index 0000000..719fb78 --- /dev/null +++ b/core/services/networking/api_requests/subtools/direct_dns_tools.py @@ -0,0 +1,343 @@ +from core.errors.logger import logger +from core.Constants import Constants +from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.services.networking.api_requests.subtools.custom_httpx_dns_resolver import create_httpx_client_with_custom_dns +from core.services.networking.api_requests.subtools.extract_domain import extract_domain, swap_domain_for_ip + +# tor +from essentials.modules.TorModule import TorModule +from essentials.observers.ConnectionObserver import ConnectionObserver +from essentials.services.ConnectionService import ConnectionService +from core.observers.ClientObserver import ClientObserver + +# pip install pip install httpx[http2] +# pip install httpx[socks] +import httpx +import dns.message +import dns.name +import dns.rdatatype +import base64 + +import json +from typing import Optional + + +def get_DNS_then_use_it( + get_or_post: str, + full_url: str, + payload: str = None, + connection_observer: Optional[ConnectionObserver] = None, + client_observer: Optional[ClientObserver] = None, +) -> ApiResponse: + + if not full_url: + return ApiResponse( + valid=False, + error_type=ErrorType.DEVELOPER_ERROR, + tor=True, + message="Dev Error! Invalid Inputs to Function" + ) + + # ================= Setup Tor =============== + if client_observer: + status_update = "Evading Tor DNS block..." + client_observer.notify('synchronizing', status_update) + + port_number = None + tor_module = None + client = None + + try: + port_number = ConnectionService.get_random_available_port_number() + tor_module = TorModule(Constants.HV_TOR_STATE_HOME) + tor_module.create_session(port_number, connection_observer) + + tor_proxy = f"socks5h://127.0.0.1:{port_number}" + + logger.debug(f"[DNS TOOLS] Tor Proxy Setup on Port {port_number}") + + # ================= Do Quad9 Query =============== + domain = extract_domain(full_url) + logger.debug(f"[DNS TOOLS] We'll be doing a DNS lookup to the domain {domain}..") + ip_address = quad9_proxy_dns_lookup(domain, tor_proxy) + + if not ip_address: + return ApiResponse( + valid=False, + error_type=ErrorType.QUAD9_DNS_RESOLUTION, + tor=True, + message=f"Cannot resolve {domain} via Tor or Quad9" + ) + + logger.debug(f"[DNS TOOLS] We got an IP address from Quad9 for {domain} of: {ip_address}") + + # ================= Use Custom DNS =============== + client = create_httpx_client_with_custom_dns({domain: ip_address}, tor_proxy) + url_with_ip = swap_domain_for_ip(domain, full_url, ip_address) + print(f"url_with_ip is {url_with_ip}") + + print(f"we are doing a {get_or_post} request") + + if get_or_post == "get": + response = client.get(url_with_ip, headers={"Host": domain}) + elif get_or_post == "post": + if payload: + response = client.post(url_with_ip, json=payload, headers={"Host": domain}) + else: + return ApiResponse( + valid=False, + error_type=ErrorType.DEVELOPER_ERROR, + ip_address=ip_address, + tor=True, + message=f"Dev Error! No Payload for POST request" + ) + else: + return ApiResponse( + valid=False, + error_type=ErrorType.DEVELOPER_ERROR, + ip_address=ip_address, + tor=True, + message=f"Dev Error! Invalid Input of get_or_post: {get_or_post}" + ) + + # print(response.json()) + + if not response: + error_msg = f"Could not connect to {domain} via {ip_address}" + return ApiResponse( + valid=False, + error_type=ErrorType.NETWORK_ERROR, + ip_address=ip_address, + tor=True, + message=error_msg + ) + + response_status_code = response.status_code + + if 200 <= response_status_code < 300: + try: + data = response.json() + return ApiResponse( + valid=True, + error_type=ErrorType.TOR_DNS_BLOCKED, + ip_address=ip_address, + tor=True, + data=data + ) + except ValueError: + print("We could not get the value from it.") + return ApiResponse( + valid=True, + error_type=ErrorType.SERVER_ERROR, + ip_address=ip_address, + tor=True, + data=response.text + ) + else: + error_msg = f"Server replied with {response_status_code} at {ip_address}" + return ApiResponse( + valid=False, + error_type=ErrorType.SERVER_ERROR, + ip_address=ip_address, + tor=True, + message=error_msg + ) + + except Exception as e: + error_msg = f"Tor failed: {type(e).__name__}: {e}" + logger.error(f"[DNS] {error_msg}") + return ApiResponse( + valid=False, + error_type=ErrorType.NETWORK_ERROR, + tor=True, + message=error_msg + ) + + finally: + if client: + client.close() + + if tor_module and port_number: + try: + tor_module.destroy_session(port_number) + except Exception as e: + logger.warning(f"[DNS] Error destroying Tor session: {e}") + + + + +# This version is passed a "generic" proxy with external Tor: +def quad9_proxy_dns_lookup( + domain: str, + custom_proxy: str, + timeout: int = 10, +) -> str: + + logger.debug("Doing a Proxy Quad9 DNS lookup") + + # ================= Use Proxy for Quad9 =============== + + qname = dns.name.from_text(domain) + query = dns.message.make_query(qname, dns.rdatatype.A) + wire_format = query.to_wire() + encoded = base64.urlsafe_b64encode(wire_format).decode().rstrip('=') + + # Use httpx with HTTP/2 + with httpx.Client(http2=True, proxy=custom_proxy, timeout=timeout) as client: + response = client.get( + "https://dns.quad9.net/dns-query", + params={"dns": encoded}, + headers={"Accept": "application/dns-message"} + ) + + logger.debug(f"[QUAD9-DNS] Status: {response.status_code}") + logger.debug(f"[QUAD9-DNS] Headers: {response.headers}") + + if response.status_code == 200: + # ================= Filter Results =============== + result = dns.message.from_wire(response.content) + logger.debug(f"[QUAD9-DNS] Quad9 Returned {result}") + + ip_address = None + for rrset in result.answer: + for rr in rrset: + if rr.rdtype == dns.rdatatype.A: + ip_address = rr.address # Get just the IP address + break + if ip_address: + break + + return ip_address + else: + logger.error(f"Quad9's Invalid Response Body: {response.text[:500]}") + if client_observer: + status_update = f"Quad9 Failed to Resolve {domain}" + client_observer.notify('synchronizing', status_update) + return False + + + +# This version does Tor & the lookup same time. +def quad9_tor_dns_lookup( + domain: str, + connection_observer: Optional[ConnectionObserver] = None, + client_observer: Optional[ClientObserver] = None, + timeout: int = 10, +) -> str: + + import httpx + import dns.message + import dns.name + import dns.rdatatype + import base64 + + logger.debug("Doing a Tor Quad9 DNS lookup") + + # ================= Setup Tor =============== + if client_observer: + status_update = "Initializing Tor connection..." + client_observer.notify('synchronizing', status_update) + + port_number = None + tor_module = None + + try: + port_number = ConnectionService.get_random_available_port_number() + tor_module = TorModule(Constants.HV_TOR_STATE_HOME) + tor_module.create_session(port_number, connection_observer) + + tor_proxy = f"socks5h://127.0.0.1:{port_number}" + + logger.debug(f"[QUAD9-DNS] Tor Proxy Setup on Port {port_number}") + + # ================= Use Proxy for Quad9 =============== + + qname = dns.name.from_text(domain) + query = dns.message.make_query(qname, dns.rdatatype.A) + wire_format = query.to_wire() + encoded = base64.urlsafe_b64encode(wire_format).decode().rstrip('=') + + # Use httpx with HTTP/2 + with httpx.Client(http2=True, proxy=tor_proxy, timeout=timeout) as client: + response = client.get( + "https://dns.quad9.net/dns-query", + params={"dns": encoded}, + headers={"Accept": "application/dns-message"} + ) + + logger.debug(f"[QUAD9-DNS] Status: {response.status_code}") + logger.debug(f"[QUAD9-DNS] Headers: {response.headers}") + + if response.status_code == 200: + # ================= Filter Results =============== + result = dns.message.from_wire(response.content) + logger.debug(f"[QUAD9-DNS] Quad9 Returned {result}") + + ip_address = None + for rrset in result.answer: + for rr in rrset: + if rr.rdtype == dns.rdatatype.A: + ip_address = rr.address # Get just the IP address + break + if ip_address: + break + + return ip_address + else: + logger.error(f"Quad9's Invalid Response Body: {response.text[:500]}") + if client_observer: + status_update = f"Quad9 Failed to Resolve {domain}" + client_observer.notify('synchronizing', status_update) + return False + + except Exception as e: + logger.debug(f"[QUAD9-DNS] Tor Quad9 GET failed: {type(e).__name__}: {e}") + return None + + finally: + if tor_module and port_number: + try: + tor_module.destroy_session(port_number) + except Exception as e: + logger.warning(f"[QUAD9-DNS] Error destroying Tor session: {e}") + + return result + + + +def quad9_clearweb_dns_lookup(domain: str) -> str: + import httpx + import dns.message + import dns.name + import dns.rdatatype + import base64 + + logger.debug("Doing a Clearweb Quad9 DNS lookup") + + qname = dns.name.from_text(domain) + query = dns.message.make_query(qname, dns.rdatatype.A) + wire_format = query.to_wire() + encoded = base64.urlsafe_b64encode(wire_format).decode().rstrip('=') + + try: + # Use httpx with HTTP/2 + with httpx.Client(http2=True) as client: + response = client.get( + "https://dns.quad9.net/dns-query", + params={"dns": encoded}, + headers={"Accept": "application/dns-message"} + ) + + logger.debug(f"Status: {response.status_code}") + logger.debug(f"Headers: {response.headers}") + + if response.status_code == 200: + result = dns.message.from_wire(response.content) + logger.debug(f"Quad9 Returned {result}") + return result + else: + logger.error(f"Quad9 Body: {response.text[:500]}") + return False + except: + logger.error("Regular Quad9 request failed.") + return None \ No newline at end of file diff --git a/core/services/networking/api_requests/subtools/extract_domain.py b/core/services/networking/api_requests/subtools/extract_domain.py index 24e23d3..c7db57b 100644 --- a/core/services/networking/api_requests/subtools/extract_domain.py +++ b/core/services/networking/api_requests/subtools/extract_domain.py @@ -1,10 +1,12 @@ -""" -Extract domain from URL, stripping protocol and path. -If exact TLD match isn't found, tries to return the best guess. -""" - +# used in the 2nd function, not the first, +import re def extract_domain(url: str) -> str: + """ + Purpose: + Extract domain from URL, stripping protocol and path. + If exact TLD match isn't found, tries to return the best guess. + """ common_domains = [".com", ".net", ".org", ".is"] # Remove protocol @@ -29,3 +31,24 @@ def extract_domain(url: str) -> str: # Fallback: return everything before the first slash (best guess) return url.split("/")[0] + + +def swap_domain_for_ip(raw_domain: str, full_url: str, ip_address: str): + """ + Purpose: + Switch the real domain for the raw IP address, + But keep the rest of the domain intact. + + Rank: + Helper + + Raises Errors? + False + """ + # Escape the domain in case it has regex special characters (like dots) + escaped_domain = re.escape(raw_domain) + + # Replace only the first instance of the domain with the IP + url_with_ip = re.sub(escaped_domain, ip_address, full_url, count=1) + + return url_with_ip diff --git a/core/services/payment_phase/save_and_send_intitial_billing.py b/core/services/payment_phase/save_and_send_intitial_billing.py index a5230f0..2ccde47 100644 --- a/core/services/payment_phase/save_and_send_intitial_billing.py +++ b/core/services/payment_phase/save_and_send_intitial_billing.py @@ -9,6 +9,7 @@ from core.models.invoice.TicketInvoice import TicketInvoice from core.services.networking.api_requests.step1_get_or_post import send_data_to_server # from core.services.networking.send_data_to_server import send_data_to_server from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems from core.services.networking.make_url import make_url @@ -49,7 +50,16 @@ def save_and_send_intitial_billing( api_reply_object = send_data_to_server(payload, url, connection_observer) if not api_reply_object.valid: - return api_reply_object + api_reply_object = solve_api_problems( + api_reply_object=api_reply_object, + get_or_post="post", + url=url, + payload=payload, + connection_observer=connection_observer, + client_observer=None + ) + if not api_reply_object.valid: + return api_reply_object reply_dict_data = api_reply_object.data diff --git a/core/services/prepare_tickets/send_blind_commitments.py b/core/services/prepare_tickets/send_blind_commitments.py index aabfbd1..30e9e3a 100644 --- a/core/services/prepare_tickets/send_blind_commitments.py +++ b/core/services/prepare_tickets/send_blind_commitments.py @@ -7,6 +7,7 @@ if TYPE_CHECKING: # services from core.services.networking.api_requests.step1_get_or_post import send_data_to_server from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems from core.services.networking.make_url import make_url from core.services.helpers.get_which_billing_key import get_which_billing_key @@ -50,11 +51,20 @@ def send_blind_commitments( # did API call work? if not api_reply_object.valid: - error_msg = api_reply_object.message - final_error_msg = f"API/Connection Error {error_msg}" + final_error_msg = f"[SEND BLIND COMMITMENTS] First Post request failed {api_reply_object.message}" logger.error(final_error_msg) - ticket_observer.notify("error", subject=final_error_msg) - return None + api_reply_object = solve_api_problems( + api_reply_object=api_reply_object, + get_or_post="post", + url=url, + payload=payload, + connection_observer=connection_observer, + client_observer=None + ) + if not api_reply_object.valid: + logger.error(f"[SEND BLIND COMMITMENTS] Second Post request failed. {api_reply_object.message}") + ticket_observer.notify("error", subject=api_reply_object.message) + return None # assuming it worked, extract the data: reply = api_reply_object.data diff --git a/core/services/sync/orm_methods/sync_one_orm.py b/core/services/sync/orm_methods/sync_one_orm.py index ed20b3e..8af5912 100644 --- a/core/services/sync/orm_methods/sync_one_orm.py +++ b/core/services/sync/orm_methods/sync_one_orm.py @@ -1,6 +1,8 @@ # from core.services.sync.get_data_from_api import get_data_from_api from core.services.networking.api_requests.step1_get_or_post import get_data_from_api from core.services.networking.api_requests.ApiResponseModel import ApiResponse +from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems + from core.models.manage.insert import insert_into_model from core.models.manage.denormalize import denormalize @@ -22,15 +24,25 @@ def sync_one_orm_model( ) -> dict: full_request_url = f"{Constants.SP_API_BASE_URL}/{which_endpoint}" - api_result = get_data_from_api(full_request_url, ClientObserver, ConnectionObserver) + api_result = get_data_from_api(full_request_url, client_observer, connection_observer) if not api_result.valid: error_msg = api_result.message - logger.error(f"API sync failed for {which_endpoint} with: {error_msg}") - return { - "success": False, - "error": error_msg - } + logger.error(f"[ONE ORM SYNC] There's an issue with the sync of endpoint {which_endpoint} the API Reply: {error_msg}") + api_result = solve_api_problems( + api_reply_object=api_result, + get_or_post="get", + url=full_request_url, + payload=None, + connection_observer=connection_observer, + client_observer=client_observer + ) + # 2nd try: + if not api_result.valid: + return { + "success": False, + "error": api_result.message + } new_data = api_result.data diff --git a/core/services/sync/sync_service.py b/core/services/sync/sync_service.py index 30c4851..44ba559 100644 --- a/core/services/sync/sync_service.py +++ b/core/services/sync/sync_service.py @@ -4,7 +4,8 @@ from core.services.sync.compare_tables import compare_tables # from core.services.sync.get_data_from_api import get_data_from_api from core.services.networking.api_requests.step1_get_or_post import get_data_from_api -from core.services.networking.api_requests.ApiResponseModel import ApiResponse +from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems # ORM for metadata: from core.models.manage.session_management import get_session @@ -153,21 +154,32 @@ def coordinate_cache_sync( full_request_url = f"{Constants.SP_API_BASE_URL}/cachedsync" api_result = get_data_from_api(full_request_url, client_observer, connection_observer) + # ================== SOLVE API CALL PROBLEMS ================== # we trust the 'get_data_from_api' function to give us a dictionary: if not api_result.valid: - print(f"Sync service says its not a valid reply") error_msg = api_result.message - logger.error(f"API sync failed: {error_msg}") - return { - "success": False, - "changed_tables": [], - "new_model_types": [], - "error": error_msg, - } + logger.error(f"[SYNC SERVICE] There's an issue with the API Reply: {error_msg}") + api_result = solve_api_problems( + api_reply_object=api_result, + get_or_post="get", + url=full_request_url, + payload=None, + connection_observer=connection_observer, + client_observer=client_observer + ) + # 2nd try: + if not api_result.valid: + return { + "success": False, + "changed_tables": [], + "new_model_types": [], + "error": api_result.message, + } - # Extract the data from the client's own trusted function, + # ================== VALIDATE DATA ================== + + # Extract the data from our own trusted function without type checking, new_data = api_result.data - print(f"inside sync_service, new_data is {new_data}") if new_data: logger.debug(f"We got an API response with some data, let's check if it's a valid format...") diff --git a/core/services/using_tickets/send_unblinded.py b/core/services/using_tickets/send_unblinded.py index d4abf9e..aa8181e 100644 --- a/core/services/using_tickets/send_unblinded.py +++ b/core/services/using_tickets/send_unblinded.py @@ -7,6 +7,7 @@ if TYPE_CHECKING: # services & helpers from core.services.networking.api_requests.step1_get_or_post import send_data_to_server from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems from core.services.networking.make_url import make_url from core.services.helpers.get_which_billing_key import get_which_billing_key @@ -49,9 +50,20 @@ def send_unblinded_ticket_to_server( # send it: which_endpoint = "validate" url = make_url(which_endpoint) - reply_object = send_data_to_server(payload, url, connection_observer) + api_reply_object = send_data_to_server(payload, different_url, connection_observer) - return reply_object + # This is if the API failed, not if it's not a valid ticket: + if not api_reply_object.valid: + api_reply_object = solve_api_problems( + api_reply_object=api_reply_object, + get_or_post="post", + url=url, + payload=payload, + connection_observer=connection_observer, + client_observer=None + ) + + return api_reply_object except Exception as e: human_readable_error_msg = f"The send_unblinded_ticket_to_server function's try-except block failed for ticket {which_ticket}. Returning False..." diff --git a/core/services/using_tickets/use_ticket_orchestrator.py b/core/services/using_tickets/use_ticket_orchestrator.py index 036981d..9fdf014 100644 --- a/core/services/using_tickets/use_ticket_orchestrator.py +++ b/core/services/using_tickets/use_ticket_orchestrator.py @@ -39,12 +39,14 @@ def use_ticket_orchestrator( raw_data_dict = api_reply_object.data - logger.debug(f"raw_data_dict is {raw_data_dict}") + # logger.debug(f"[USE TICKET ORCHESTRATOR] We have received a raw dictionary of {raw_data_dict}") if not isinstance(raw_data_dict, dict): + logger.debug(f"[USE TICKET ORCHESTRATOR] Invalid Data format returned from the API server.") return {"valid": False, "message": "invalid_format"} if "valid" not in raw_data_dict: + logger.debug(f"[USE TICKET ORCHESTRATOR] Invalid Data format returned from the API server.") return {"valid": False, "message": "invalid_format"} is_it_valid = raw_data_dict.get("valid") @@ -57,6 +59,8 @@ def use_ticket_orchestrator( """ if is_it_valid: + logger.debug(f"[USE TICKET ORCHESTRATOR] The API server returned a valid billing code.") + # update the ticket tracker: status_update = update_value_in_json_with_two_values( ticket_tracker_path, which_ticket, "status", "used" @@ -79,6 +83,8 @@ def use_ticket_orchestrator( message = raw_data_dict.get("message") if message == "already_used": + logger.debug(f"[USE TICKET ORCHESTRATOR] This billing code has already been used, but it is still potentially still valid.") + # update the ticket tracker to reflect that it's already used with that billing code: status_update = update_value_in_json_with_two_values( ticket_tracker_path, which_ticket, "status", "used" diff --git a/core/test_dns.py b/core/test_dns.py deleted file mode 100644 index 782f653..0000000 --- a/core/test_dns.py +++ /dev/null @@ -1,42 +0,0 @@ -print("hi") -import httpx -import ssl -import certifi - -def load_custom_dns_resolver(): - """Create a client with custom transports for Tor DNS bypass.""" - - def create_client_with_custom_dns(hostname_to_ip_map: dict): - """ - hostname_to_ip_map: {"example.com": "203.0.113.45", "other.com": "203.0.113.46"} - """ - mounts = {} - - for hostname, ip in hostname_to_ip_map.items(): - ctx = ssl.create_default_context(cafile=certifi.where()) - ctx.check_hostname = True - ctx.verify_mode = ssl.CERT_REQUIRED - - # transport = httpx.HTTPTransport(ssl_context=ctx, http2=True) - transport = httpx.HTTPTransport(ssl_context=context) - - # Mount for both hostname and IP patterns - mounts[f"https://{hostname}"] = transport - mounts[f"https://{ip}"] = transport - - # client = httpx.Client(mounts=mounts, http2=True) - client = httpx.Client(mounts=mounts) - return client - - return create_client_with_custom_dns - -# Usage -create_client = load_custom_dns_resolver() -client = create_client({"api.hydraveil.net": "185.165.169.91"}) - -# Connect to IP, but Host header and cert verification use original hostname -response = client.get("https://185.165.169.91/api/v1/cachedsync", headers={"Host": "api.hydraveil.net"}) - - -print(response) -print(response.json) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index dfd862b..b986b35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,9 @@ dependencies = [ "typing-inspection==0.4.2", "typing_extensions==4.15.0", "urllib3==2.6.3", + "httpx==0.28.1", + "dnspython==2.8.0", + "socksio==1.0.0", ] [project.urls]