Integrated connect.singleendpoint() with the start of bulk sync getting metadata sync. Also improved observer feedback

core/services/networking/httpx/connect.py

@@ -34,7 +34,6 @@ def single_endpoint(method: str, url: str, observer: ConnectionObserver, payload
     Purpose:
         Try a request to a single endpoint using either an existing client, or creating a new one, then error handling.
     """
-    
     connection_type = ConfigurationController.get_connection_enum()
 
     client = httpx_client.get_http_session()
@@ -43,8 +42,10 @@ def single_endpoint(method: str, url: str, observer: ConnectionObserver, payload
     # NO CLIENT
     ########################################################
     if client is None:
+        observer.notify('message', "Testing Connection..")
         made_client = _make_client(connection_type, observer)
         if not made_client and connection_type == ConnectionChoice.TOR:
+            observer.notify('message', "Tor Bootstrap..")
             return bootstrap_and_try_again(
                 method=method,
                 url=url,
@@ -165,11 +166,14 @@ def bootstrap_and_try_again(method: str, url: str, observer: ConnectionObserver,
     # BOOTSTRAP SUCCESS
     _port_used = bootstrap_results.port
 
+    observer.notify('message', "Testing Tor..")
     made_client = httpx_client.init_tor_session(_port_used)
     if not made_client:
         return bootstrap_results
+    observer.notify('message', "Tor Confirmed")
 
     client = httpx_client.get_http_session()
+    observer.notify('message', "Making Request..")
     return make_request(
         method=method,
         url=url,

core/services/networking/tor_tools/monitor_bootstrap.py

@@ -53,7 +53,7 @@ class BootstrapState:
 
 def monitor_bootstrap_progress(
     process: subprocess.Popen,
-    observer: ConnectionObserver,
+    connection_observer: ConnectionObserver,
     timeout_seconds: int = 60
 ) -> tuple[bool, Optional[str]]:
     """
@@ -151,7 +151,7 @@ def monitor_bootstrap_progress(
             match = progress_pattern.search(line)
             if match:
                 progress_pct = int(match.group(1))
-                result = _evaluate_progress(progress_pct, state, observer)
+                result = _evaluate_progress(progress_pct, state, connection_observer)
                 
                 # None = keep going; tuple = exit condition reached
                 if result is None:
@@ -178,7 +178,7 @@ def monitor_bootstrap_progress(
 def _evaluate_progress(
     progress: int,
     state: BootstrapState,
-    observer: ConnectionObserver
+    connection_observer: ConnectionObserver
 ) -> Optional[tuple[bool, Optional[str]]]:
     """
     Purpose:
@@ -192,26 +192,34 @@ def _evaluate_progress(
     # Record first time we see any pattern
     if state.first_pattern_time is None:
         state.first_pattern_time = time.time()
-        logger.error(f"Tor Bootstrap {progress}%")
+        logger.info(f"Start of Tor Bootstrap {progress}%")
         return None
 
     # Success?
     if progress == 100:  # note: this 100% check has to be before the progress increase, or it will reset before hitting it,
         logger.info("Tor Bootstrap 100%. Returning..")
+        try:
+            connection_observer.notify("tor_bootstrapped")
+        except:
+            logger.error("Can't notify the UI that Tor Bootstrap finished.")
         return True, None
 
     # Progress increased?
     if state.last_progress_value is not None and progress > state.last_progress_value:
         state.last_progress_value = progress
         state.last_progress_time = time.time()
-        print(f"Tor Bootstrap {progress}%")
+        progress_string = f"Tor Bootstrap {progress}%"
+        connection_observer.notify("tor_bootstrap_progressing", progress_string)
         return None
     
     # First percentage?
     if state.last_progress_value is None:
         state.last_progress_value = progress
         state.last_progress_time = time.time()
-        logger.error(f"Tor Bootstrap {progress}%")
+        error_msg = f"Stuck on Tor Bootstrap {progress}%"
+        logger.error(error_msg)
+        if connection_observer:
+            connection_observer.notify("error", error_msg)
         return None
     
     # Stalled?
@@ -219,7 +227,10 @@ def _evaluate_progress(
         assert state.last_progress_time is not None
         stall_duration = time.time() - state.last_progress_time
         if stall_duration > TIMEOUT_WINDOW:
-            logger.error(f"Bootstrap stalled at {progress}% for {stall_duration:.1f}s")
+            error_msg = f"Bootstrap stalled at {progress}% for {stall_duration:.1f}s"
+            logger.error(error_msg)
+            if connection_observer:
+                connection_observer.notify("error", error_msg)
             return False, "\n".join(state.stderr_buffer)
         return None
         

core/services/sync/sync_service.py

@@ -4,9 +4,9 @@ from core.services.networking.httpx import connect
 
 
 from core.services.sync.compare_tables import compare_tables
-# from core.services.sync.get_data_from_api import get_data_from_api
 
-from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+# from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+from core.services.networking.httpx import connect
 from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
 from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
 
@@ -155,7 +155,13 @@ def coordinate_cache_sync(
     """
     
     full_request_url = f"{Constants.SP_API_BASE_URL}/cachedsync"
-    api_result = get_data_from_api(full_request_url, client_observer, connection_observer)
+    # api_result = get_data_from_api(full_request_url, client_observer, connection_observer)
+    api_result = connect.single_endpoint(
+        method="get",
+        url=full_request_url,
+        observer=connection_observer,
+        payload=None
+    )
 
     # ================== SOLVE API CALL PROBLEMS  ==================
     # we trust the 'get_data_from_api' function to give us a dictionary:
@@ -189,7 +195,8 @@ def coordinate_cache_sync(
 
     # Validate the API's payload, (we don't trust the API's structure)
     if not isinstance(new_data, dict):
-        error_msg = "API returned invalid metadata: 'data' is not a dict"
+        error_msg = f"API returned invalid metadata of {type(new_data)}.  When 'new_data' has to be in dictionary format. The raw data is {new_data}"
+        client_observer.notify('synchronizing', f'Server replied with invalid metadata of type {type(new_data)}')
         logger.error(error_msg)
         return {
             "success": False,