Sync Streamline

.gitignore

@@ -1,3 +1,4 @@
+.dev
 prototype_client.py
 .idea
 .venv

core/.metadata.md

@@ -0,0 +1 @@
+this is june 13 version

core/Constants.py

@@ -10,8 +10,7 @@ class Constants:
         "TICKET_API_BASE_URL", "https://ticket.hydraveil.net"
     )
 
-    SP_API_BASE_URL: Final[str] = os.environ.get('SP_API_BASE_URL', 'https://dev-us-002.simplifiedprivacy.net/api/v1')
+    SP_API_BASE_URL: Final[str] = os.environ.get('SP_API_BASE_URL', 'https://api.hydraveil.net/api/v1')
-    # SP_API_BASE_URL: Final[str] = os.environ.get('SP_API_BASE_URL', 'https://api.hydraveil.net/api/v1')
     PING_URL: Final[str] = os.environ.get('PING_URL', 'https://api.hydraveil.net/api/v1/health')
 
     CONNECTION_RETRY_INTERVAL: Final[int] = int(os.environ.get('CONNECTION_RETRY_INTERVAL', '5'))

core/controllers/ClientController.py

@@ -1,5 +1,7 @@
+# new sync refactor:
 from core.services.sync.sync_service import coordinate_cache_sync
-
+from core.errors.logger import logger
+# prior versions:
 from core.Constants import Constants
 from core.Errors import UnknownClientPathError, UnknownClientVersionError, CommandNotFoundError
 from core.controllers.ApplicationController import ApplicationController
@@ -47,37 +49,39 @@ class ClientController:
     def sync(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None):
 
         from core.models.manage.session_management import init_session, close_session
-        init_session()
-        print("I sync'ed")
 
         # Use Cached Method:
-        changed_tables = coordinate_cache_sync(ClientObserver, ConnectionObserver) 
+        init_session()
+        result = coordinate_cache_sync(ClientObserver, ConnectionObserver)
 
-        # SPOOFING
+        logger.info(f"We got a Result from the API of {result}")
-        # changed_tables = ['version', 'applications', 'application_versions', 'client_version', 'operators', 'locations', 'subscriptions']
 
-        print(f"changed_tables: {changed_tables}")       
+        close_session()
 
-        # this should be same or any errors:
+        # Outright Error:
-        if isinstance(changed_tables, dict):
+        if not result["success"]:
-            # check result is "same"?
+            error_msg = result["error"]
-            # or an errro
+            if client_observer is not None:
+                client_observer.notify('synchronizing', f'Error! {error_msg}')
+                return
+
+        # Same:
+        changed_tables = result["changed_tables"]
+        if not changed_tables:
             if client_observer is not None:
                 client_observer.notify('synchronized')
+                return
 
-        # now it's likely to have results:
+        # We only make it past this point if there's New Data
-        if isinstance(changed_tables, list):
-            quantity_of_changes = len(changed_tables)
 
-            print(f"quantity_of_changes is {quantity_of_changes}")
+        # Fetch and update those models...
-
+        try:
-            # sanity check:
+            from core.controllers.ConnectionController import ConnectionController
-            if quantity_of_changes >= 0:
+            ConnectionController.with_preferred_connection(task=ClientController.__sync, changed_tables=changed_tables, client_observer=client_observer, connection_observer=connection_observer)
-                from core.controllers.ConnectionController import ConnectionController
+        except:
-                ConnectionController.with_preferred_connection(task=ClientController.__sync, changed_tables=changed_tables, client_observer=client_observer, connection_observer=connection_observer)
+            # sync failed here,
-            else:
+            if client_observer is not None:
-                if client_observer is not None:
+                client_observer.notify('synchronizing', 'Error! Sync Failed.')
-                    client_observer.notify('synchronized')
 
     @staticmethod
     def update(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None):
@@ -93,49 +97,58 @@ class ClientController:
 
         return path
 
-    # def cached_sync(client_observer: Optional[ClientObserver] = None, proxies: Optional[dict] = None):
 
     @staticmethod
     def __sync(changed_tables: list, client_observer: Optional[ClientObserver] = None, proxies: Optional[dict] = None):
 
-        # SPOOFING
-        # changed_tables = ['version', 'applications', 'application_versions', 'client_version', 'operators', 'locations', 'subscriptions']
-
         if client_observer is not None:
             client_observer.notify('synchronizing')
 
         if "applications" in changed_tables:
-            print("sync applications")
+            logger.info("Sync applications..")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Browser List')
             # noinspection PyProtectedMember
             ApplicationController._sync(proxies=proxies)
 
         if "application_versions" in changed_tables:
-            print("sync application_versions")
+            logger.info("Sync Application Versions..")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Browser Version List')
             # noinspection PyProtectedMember
             ApplicationVersionController._sync(proxies=proxies)
 
         if "client_version" in changed_tables:
-            print("sync client_version")
+            logger.info("Sync of client version")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Client Version List')
             # noinspection PyProtectedMember
             ClientVersionController._sync(proxies=proxies)
 
         if "operators" in changed_tables:
-            print("sync operators")
+            logger.info("Sync of Operators")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Operators List')
             # noinspection PyProtectedMember
             OperatorController._sync(proxies=proxies)
 
         if "locations" in changed_tables:
-            print("sync locations")
+            logger.info("Sync of Locations")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Locations List')
             # noinspection PyProtectedMember
             LocationController._sync(proxies=proxies)
 
         if "subscriptions" in changed_tables:
-            print("sync subscriptions")
+            logger.info("Sync of Subscriptions")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Subscription List')
             # noinspection PyProtectedMember
             SubscriptionPlanController._sync(proxies=proxies)
 
         ConfigurationController.update_last_synced_at()
 
+        logger.info("Entire Sync Completed Successfully")
         if client_observer is not None:
             client_observer.notify('synchronized')
 

core/models/manage/get_from_model.py

@@ -4,7 +4,7 @@ from sqlalchemy.exc import SQLAlchemyError
 
 def get_from_model(model_class: Type, **filters) -> List[Dict[str, Any]]:
     """
-    Retrieve data from a model.
+    Retrieve data from a model.  Keeping this generic for reuse
     
     Args:
         model_class: The ORM model class

core/models/manage/insert.py

@@ -1,14 +1,15 @@
+from core.errors.logger import logger
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy import create_engine
 from typing import Type, Dict, Any
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
-from core.models.python_models.Base import Base
+from core.models.orm_models.Base import Base
-from core.models.python_models.CachedSync import CachedSync
+from core.models.orm_models.CachedSync import CachedSync
 from core.models.manage.session_management import get_session
 
 def insert_into_model(model_class: Type, all_data: dict | list, override=False) -> bool:
     """
-    Generic ORM insert for any model.
+    Generic ORM insert for any model.  Keeping it generic for reuse.
     
     Args:
         model_class: The ORM model class
@@ -19,15 +20,16 @@ def insert_into_model(model_class: Type, all_data: dict | list, override=False)
         True if successful
     """
 
-    print(f"all_data is {all_data}")
+    logger.info(f"All the public data preparing to be inserted is {all_data}")
 
     with get_session() as session:
         # Step 1: Wipe the table if override=True
         if override:
+            logger.info(f"First, WIPING the pre-existing data for {model_class.__name__}. Are you sure you intended to completely delete the old data?")
             session.query(model_class).delete()
             session.commit()
 
-        print(f"Starting insert for {model_class.__name__}")
+        logger.info(f"Starting insert for {model_class.__name__}")
 
         # Normalize to list for uniform handling
         data_list = all_data if isinstance(all_data, list) else [all_data]
@@ -38,6 +40,7 @@ def insert_into_model(model_class: Type, all_data: dict | list, override=False)
                 session.add(instance)
             
             session.commit()
+            logger.info(f"Completed! SQL Insertion Successful following the {model_class.__name__} Model’s Type Rules")
             return True
 
         except TypeError as e:
@@ -56,83 +59,3 @@ def insert_into_model(model_class: Type, all_data: dict | list, override=False)
             print(f"Generic exception caught: {e}")
             session.rollback()
             raise e
-
-
-
-# def insert_into_model(model_class: Type, all_data: list, override=False) -> Any:
-#     """
-#     Generic ORM insert for any model. But not used for the legacy codebase yet.
-    
-#     Args:
-#         model_class: The ORM model class
-#         json_data: Dictionary with data matching the model
-    
-#     Returns:
-#         The created instance
-#     """
-
-#     print(f"all_data is {all_data}")
-
-#     with get_session() as session:
-#         # Step 1: Wipe the table if override=True
-#         if override:
-#             session.query(model_class).delete()
-#             session.commit()
-
-#         print("we made it past getting the session")
-
-#         print(f"Starting insert for {model_class.__name__}")
-
-#         type_of_data = type(all_data)
-#         print(f"all_data type is: {type_of_data}")
-
-#         if isinstance(all_data, dict):
-#             try:
-#                 instance = model_class(**all_data)
-#                 session.add(instance)
-#                 session.commit()
-#             except TypeError as e:
-#                 print(f"TypeError caught: {e}")
-#                 session.rollback()
-#                 raise ValueError(f"Invalid fields for {model_class.__name__}: {e}")
-#             except IntegrityError as e:
-#                 print(f"IntegrityError caught: {e.orig}")
-#                 session.rollback()
-#                 raise ValueError(f"Constraint violation: {e.orig}")
-#             except SQLAlchemyError as e:
-#                 print(f"SQLAlchemyError caught: {e}")
-#                 session.rollback()
-#                 raise
-#             except Exception as e:
-#                 print(f"Generic exception caught: {e}")
-#                 session.rollback()
-#                 raise e
-#             return True
-
-#         elif isinstance(all_data, list):
-#             for each_json in all_data:
-#                 try:
-#                     instance = model_class(**each_json)
-#                     session.add(instance)
-#                     session.commit()
-#                 except TypeError as e:
-#                     print(f"TypeError caught: {e}")
-#                     session.rollback()
-#                     raise ValueError(f"Invalid fields for {model_class.__name__}: {e}")
-#                 except IntegrityError as e:
-#                     print(f"IntegrityError caught: {e.orig}")
-#                     session.rollback()
-#                     raise ValueError(f"Constraint violation: {e.orig}")
-#                 except SQLAlchemyError as e:
-#                     print(f"SQLAlchemyError caught: {e}")
-#                     session.rollback()
-#                     raise
-#                 except Exception as e:
-#                     print(f"Generic exception caught: {e}")
-#                     session.rollback()
-#                     raise e
-#             return True
-
-#         else:
-#             print("invalid data type")
-#             return False

core/models/manage/manae_session.py

@@ -1,33 +0,0 @@
-# db.py
-from core.models.python_models.Base import Base
-from sqlalchemy import create_engine
-from sqlalchemy.orm import scoped_session, sessionmaker
-from sqlalchemy.exc import SQLAlchemyError
-
-# Create engine
-try:
-    engine = create_engine('sqlite:///data.db', echo=False)
-except Exception as e:
-    print(f"Failed to create engine: {e}")
-    raise
-
-Session = scoped_session(sessionmaker(bind=engine))
-
-def init_session():
-    """Initialize database tables."""
-    try:
-        Base.metadata.create_all(engine)
-    except SQLAlchemyError as e:
-        print(f"Failed to initialize database: {e}")
-        raise
-
-def get_session():
-    """Get the thread-local session."""
-    return Session()
-
-def close_session():
-    """Clean up the thread-local session."""
-    try:
-        Session.remove()
-    except Exception as e:
-        print(f"Error closing session: {e}")

core/models/manage/session_management.py

@@ -1,14 +1,22 @@
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy import create_engine
-from core.models.python_models.Base import Base
+from core.models.orm_models.Base import Base
+from core.Constants import Constants
 
-engine = create_engine("sqlite:///data.db")
+database_path = Constants.HV_STORAGE_DATABASE_PATH
+engine = create_engine(f"sqlite:///{database_path}")
 Session = sessionmaker(bind=engine)
 
 _session = None
 
 def init_session():
-    """Call this once at application startup"""
+    """
+    Called once at application startup using global variables for the session, 
+    which initialize prior to being called with a false flag.
+
+    The reason for this is so the app can load prior to be called,
+    while keeping it a globally accessible variable
+    """
     global _session
     Base.metadata.create_all(engine)
     _session = Session()
@@ -20,7 +28,12 @@ def get_session():
     return _session
 
 def close_session():
-    """Call this when shutting down the application"""
+    """
+    Used prior to shutting down the application to break SQL connections.
+
+    In the current refactor, it may be used prior to shut down, such as sync completion,
+    because a low amount of the total models currently use the new system.
+    """
     global _session
     if _session:
         _session.close()

core/models/manage/setup.py

@@ -1,19 +0,0 @@
-from contextlib import contextmanager
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy import create_engine
-
-
-engine = create_engine("sqlite:///alchemy.db")
-Session = sessionmaker(bind=engine)
-
-Base.metadata.create_all(engine)
-
-@contextmanager
-def get_session():
-    session = Session()
-    try:
-        yield session
-    finally:
-        session.close()
-
-

core/models/orm_models/Base.py

@@ -0,0 +1,9 @@
+# base.py
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()
+
+"""
+This will eventually replace the legacy base model,
+but is still required, even if empty, to setup the declarative base. 
+"""

core/models/orm_models/CachedSync.py

@@ -2,12 +2,18 @@ from sqlalchemy import Integer, String, ForeignKey
 from sqlalchemy.orm import declarative_base, mapped_column, Mapped, relationship
 from typing import Optional
 from sqlalchemy.orm import Mapped
-from core.models.python_models.Base import Base
+from core.models.orm_models.Base import Base
+
+"""
+This is one of the SQLAlchemy Models from the refactor.
+
+That’s why it lacks functions to get or insert data.
+"""
 
 class CachedSync(Base):
     __tablename__ = 'cached_sync'
 
-    # version of the cached sync itself primary key
+    # version of the cached sync itself is the primary key
     version: Mapped[int] = mapped_column(Integer, primary_key=True)
 
     applications: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)
@@ -16,33 +22,3 @@ class CachedSync(Base):
     operators: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)
     locations: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)
     subscriptions: Mapped[Optional[str]] = mapped_column(Integer, nullable=True, default=None)
-
-# to use:
-# Lookup by composite primary key
-# record = session.get(Hello, (b_value))
-
-
-
-# # from core.models.Model import Model
-# from dataclasses import dataclass
-# from typing import Union, Optional
-
-# _table_name: str = 'cached_sync'
-
-# _table_definition: str = """
-#     'version' int UNIQUE,
-#     'applications' int,
-#     'application_versions' int,
-#     'operators' int,
-#     'locations' int,
-#     'subscriptions' int,
-# """
-
-# @dataclass
-# class CachedSync(Model):
-#     version: int
-#     applications: int
-#     application_versions: int
-#     operators: int
-#     locations: int
-#     subscriptions: int

core/models/python_models/Base.py

@@ -1,4 +0,0 @@
-# base.py (or models/base.py)
-from sqlalchemy.orm import declarative_base
-
-Base = declarative_base()

core/services/sync/compare_tables.py

@@ -1,69 +1,70 @@
 
-def compare_tables(new_data: dict, old_table: dict) -> tuple:
+import logging
+from typing import Optional
 
-    type_of_new = type(new_data)
+logger = logging.getLogger(__name__)
-    type_of_old = type(old_table)
-    print(type_of_new)
-    print(type_of_old)
 
-    # ============== FILTER ==================
-    new_is_dictionary = isinstance(new_data, dict)
-    old_is_dictionary = isinstance(old_table, dict)
-    if new_is_dictionary and old_is_dictionary:
-        print("Filter Passed. Both things are dictionaries.")
-    else:
-        return None, None
 
-    # ============== PREPARE ==================
+def compare_tables(new_data: dict, old_table: dict) -> tuple[list[str], list[str]]:
+    """
+    Compare version numbers in new and old metadata dictionaries.
     
-    # what are the tables?
+    Args:
-    new_keys = new_data.keys()
+        new_data: Latest metadata from API (e.g., {"version": 2, "model_one": 2})
-    old_keys = old_table.keys()
+        old_table: Previously cached metadata from database
     
-    print(f"new_keys is {new_keys}")
+    Returns:
-    print(f"old_keys is {old_keys}")
+        Tuple of (changed_tables, new_model_types)
+        - changed_tables: model keys where version increased
+        - new_model_types: model keys that don't exist in old_table (reserved for future use)
+    
+    Raises:
+        TypeError: If either argument is not a dictionary
+        ValueError: If a model version is not an integer
+    """
+    # Validate inputs
+    if not isinstance(new_data, dict) or not isinstance(old_table, dict):
+        raise TypeError("Both new_data and old_table must be dictionaries")
     
     changed_tables = []
-    new_data_types = []
+    new_model_types = []
     
-    # ============== EVALUATE ==================
+    for model_key, new_version in new_data.items():
-
+        # Skip the "version" metadata key itself
-    for new_key in new_keys:
+        if model_key == "version":
-        print(f"This iteration's new_key is {new_key}")
-        each_value = new_data[new_key]
-        print(f"This value is {each_value}")
-
-        # is the VALUE we're evaluating even a number?
-        if not isinstance(each_value, int):
-            print("skipping, it's not an int")
             continue
         
-        # do we even know about this new table category?
+        # Type check: versions must be integers
-        if new_key not in old_keys:
+        if not isinstance(new_version, int):
-            new_data_types.append(new_key)
+            raise ValueError(
+                f"Model '{model_key}' has non-integer version: {new_version!r}. "
+                f"Expected int."
+            )
+        
+        # Identify new model types (not in old data)
+        if model_key not in old_table:
+            new_model_types.append(model_key)
             continue
         
-        # we know about it, let's see if it's new:
+        # Identify changed models (version increased)
-        if each_value > old_table[new_key]:
+        if new_version > old_table[model_key]:
-            changed_tables.append(new_key)
+            changed_tables.append(model_key)
     
-    return changed_tables, new_data_types
+    return changed_tables, new_model_types
 
-# ============== TESTING ==================
+# ============== ISOLATION TESTING ==================
-# ISOLATION TEST:
+# SPOOF ISOLATION TEST:
-"""
+# new_data = {
-new_data = {
+#     "a": 1,
-    "a": 1,
+#     "b": 2,
-    "b": 2,
+#     "c": 4,
-    "c": 4,
+#     "d": 1
-    "d": 1
+# }
-}
+# old_data = {
-old_data = {
+#     "a": 1,
-    "a": 1,
+#     "b": 2,
-    "b": 2,
+#     "c": 3,
-    "c": 3,
+# }
-}
+# changed_tables, new_data_types = compare_tables(new_data, old_data)
-changed_tables, new_data_types = compare_tables(new_data, old_data)
+# print(f"changed tables: {changed_tables}")
-print(f"changed tables: {changed_tables}")
+# print(f"new data types: {new_data_types}")
-print(f"new data types: {new_data_types}")
-"""

core/services/sync/sync_service.py

@@ -12,7 +12,7 @@ from core.services.sync.compare_tables import compare_tables
 from core.models.manage.session_management import get_session
 from core.models.manage.get_from_model import get_from_model
 from core.models.manage.insert import insert_into_model
-from core.models.python_models.CachedSync import CachedSync
+from core.models.orm_models.CachedSync import CachedSync
 
 from core.Constants import Constants
 from core.observers.BaseObserver import BaseObserver
@@ -24,161 +24,288 @@ from core.observers.ConnectionObserver import ConnectionObserver
 # generic
 from sqlalchemy import func
 from typing import Optional
+from typing import Optional
+
+
+def _get_cached_metadata():
+    """Retrieve cached metadata from the database.
+    
+    Returns:
+        tuple: (old_data dict, previous_highest version number)
+               If no cache exists: ({}, 0)
+    
+    Raises:
+        Exception: Any database query errors propagate up.
+    """
+    old_data_as_list = get_from_model(CachedSync)
+    
+    if not old_data_as_list:
+        logger.warning("No cached sync data in database. Treating as first sync.")
+        return {}, 0
+
+    if not isinstance(old_data_as_list, list):
+        logger.critical("Error with get_from_model improperly returning non-lists.")
+        return {}, 0
+    
+    old_data = old_data_as_list[0]
+    previous_highest = old_data.get("version", 0)
+    logger.debug(f"Cached metadata version: {previous_highest}")
+    
+    return old_data, previous_highest
+
+
+def _get_changed_models(new_data, old_data):
+    """Determine which models changed between API and cached versions.
+    
+    Args:
+        new_data: Dict of new metadata from API
+        old_data: Dict of old metadata from cache (empty dict if first sync)
+    
+    Returns:
+        tuple: (changed_tables list, new_model_types list)
+    
+    Raises:
+        TypeError: If inputs have invalid structure
+        ValueError: If version values are invalid
+    """
+    if not old_data:
+        # First sync: no baseline, treat all models as needing fetch
+        changed_tables = [key for key in new_data if key != "version"]
+        new_model_types = []
+        logger.info(f"First sync: fetching all models: {changed_tables}")
+    else:
+        # Subsequent sync: compare against baseline
+        changed_tables, new_model_types = compare_tables(new_data, old_data)
+    
+    return changed_tables, new_model_types
+
+
+def _filter_valid_models(data, invalid_keys):
+    """Remove invalid model types from data dict.
+    
+    Args:
+        data: Dict of metadata
+        invalid_keys: List of keys to exclude
+    
+    Returns:
+        Dict with invalid keys removed
+    """
+    filtered = {}
+    for key, value in data.items():
+        if key not in invalid_keys:
+            filtered[key] = value
+    return filtered
+
+
+def save_data(data: dict) -> bool:
+    """Save metadata to database."""
+    try:
+        logger.debug(f"Inserting metadata into database: {data}")
+        did_it_work = insert_into_model(CachedSync, data, True)
+        
+        if did_it_work:
+            logger.debug("Metadata saved successfully")
+        else:
+            logger.warning("insert_into_model returned False")
+        
+        return did_it_work
+    
+    except Exception as e:
+        logger.error(f"Failed to save metadata to database: {str(e)}", exc_info=True)
+        return False
+
+
+def _log_what_changed(new_model_types, changed_tables):
+    if new_model_types:
+        logger.debug(f"New model types detected (reserved): {new_model_types}")
+
+    if changed_tables:
+        logger.info(f"Models to sync: {changed_tables}")
+    else:
+        logger.info("No models changed between versions.")
+
 
 # Highest Level in this Flow:
 def coordinate_cache_sync(
     client_observer: Optional[ClientObserver] = None,
     connection_observer: Optional[ConnectionObserver] = None
 ) -> dict:
+    """
+    Fetch version metadata from API and return changed/new model keys.
     
-    # ==================== GET & VALIDATE DATA ============================
+    Returns:
+        Dictionary with keys:
+        - 'success' (bool): Whether comparison succeeded
+        - 'changed_tables' (list): Model keys that changed
+        - 'new_model_types' (list): New model keys (reserved for future use)
+        - 'error' (str or None): Error message if success is False
+    """
 
-    # get data from the endpoint:
+    api_result = _get_sync_cache_from_api(ClientObserver, ConnectionObserver)
-    new_data_payload = get_sync_cache_from_api(ClientObserver, ConnectionObserver)
 
-    print("")
+    # we trust the '_get_sync_cache_from_api' function to give us a dictionary:
-    print(f"We have new data: {new_data_payload}")
+    if not api_result.get("success"):
-    print(type(new_data_payload))
+        error_msg = api_result.get("error", "Unknown API error")
-    print("")
+        logger.error(f"API sync failed: {error_msg}")
+        return {
+            "success": False,
+            "changed_tables": [],
+            "new_model_types": [],
+            "error": error_msg,
+        }
 
-    # is is a dictionary?
+    # Extract the data from the client's own trusted function,
-    if not isinstance(new_data_payload, dict):
+    new_data = api_result.get("data")
-        return {"valid": False, "error_msg": "invalid_format_reply"}
 
-    if "data" not in new_data_payload:
+    # Validate the API's payload, (we don't trust the API's structure)
-        return new_data_payload
+    if not isinstance(new_data, dict):
-
+        error_msg = "API returned invalid metadata: 'data' is not a dict"
-    new_data = new_data_payload.get("data", None)
+        logger.error(error_msg)
-
+        return {
-    # SPOOFING:
+            "success": False,
-    # new_data = {
+            "changed_tables": [],
-    #     "version": 1,
+            "new_model_types": [],
-    #     "applications": 1,
+            "error": error_msg,
-    #     "application_versions": 1,
+        }
-    #     "client_version": 1,
-    #     "operators": 1,
-    #     "locations": 1,
-    #     "subscriptions": 1
-    # }
 
     if "version" not in new_data:
-        return new_data # it should have the error in there.
+        error_msg = "API returned invalid metadata: missing 'version' key"
+        logger.error(error_msg)
+        return {
+            "success": False,
+            "changed_tables": [],
+            "new_model_types": [],
+            "error": error_msg,
+        }
 
-    new_highest = new_data.get("version", None)
 
-    print(f"new highest version is {new_highest}")
+    new_highest = new_data.get("version")
+    logger.debug(f"API metadata version: {new_highest}")
 
-    if not new_highest: # version is empty
+    # Fetch old cached version from database
-        return new_data # it should have the error in there.
-
-    # ==================== INITIAL SQL SESSION =====================
-    session = get_session()
-
-    # Get the highest version already saved:
-    previous_highest = session.query(func.max(CachedSync.version)).scalar()
-    print(f"previous_highest is {previous_highest}")
-
-    # if this is the first sync, we have no data to compare it to,
-    if previous_highest is None:
-        print("No previous previous_highest or Previous data. Saving..")
-
-        # so save the data,
-        saved = save_data(new_data)
-        if saved:
-            print("Saved!")
-
-        # then the "changed" keys are all of the new ones,
-        changed_tables_keys = new_data.keys()
-        changed_tables = list(changed_tables_keys)
-        return changed_tables
-
-    # ==================== EVALUATE DATA =====================
-
-    print(f"previous_highest is {previous_highest}")
-    print(f"new_highest is {new_highest}")
-
-    # bail on the rest of this function if it's the same:
-    if previous_highest >= new_highest:
-        return {"result": "same"}
-    else:
-        print("The data is new! We now evaluate...")
-
-    # ==================== COMPARE DATA ============================
-
-    # we are only doing this if we actually have old data to compare it to.
-    query = {"version": previous_highest}
-    old_data_as_list = get_from_model(CachedSync, **query)
-
-    # if it's a list, get it out of there,
-    old_data = old_data_as_list[0]
-
-    print(f"old data is: {old_data}")
-
-    if old_data:
-        print(f"We got the old data successfully. Now comparing..")
-
-        # let's see which tables are NEW:
-        changed_tables, new_data_types = compare_tables(new_data, old_data)
-
-        print("")
-        print(f"changed_tables is {changed_tables}")
-        print(f"new_data_types is {new_data_types}")
-        print("")
-
-        if new_data_types:
-            quantity_of_new_types = len(new_data_types)
-    else:
-        # we can't fetch the OLD data due to database error, so we need to replace it all,
-        print("database error. add logic here for quantity_of_new_types.")
-        changed_tables_keys = new_data.keys()
-        changed_tables = list(changed_tables_keys)
-        quantity_of_new_types = len(changed_tables)
-
-    # ==================== FINALLY =====================
-    return changed_tables
-
-### Add error logging to this:
-def save_data(data: dict) -> bool:
-    print(f"about to insert the NEW data into the database.. {data}")
     try:
-        did_it_work = insert_into_model(CachedSync, data, True)
+        old_data, previous_highest = _get_cached_metadata()
-        print(f"Did it insert? {did_it_work}")
-    except:
-        print("Error! We Failed to Insert the New Data!")
-        did_it_work = False
         
-    return did_it_work
+    except Exception as e:
+        error_msg = f"Database query failed: {str(e)}"
+        logger.error(error_msg, exc_info=True)
+        return {
+            "success": False,
+            "changed_tables": [],
+            "new_model_types": [],
+            "error": error_msg,
+        }
+
+    # Version hasn't changed
+    if new_highest == previous_highest:
+        logger.info(f"Versions match (both {new_highest}). No sync needed.")
+        return {
+            "success": True,
+            "changed_tables": [],
+            "new_model_types": [],
+            "error": None,
+        }
+
+    # Determine which models changed
+    try:
+        changed_tables, new_model_types = _get_changed_models(new_data, old_data)
+        
+    except (TypeError, ValueError) as e:
+        error_msg = f"Invalid metadata: {str(e)}"
+        logger.error(error_msg)
+        return {
+            "success": False,
+            "changed_tables": [],
+            "new_model_types": [],
+            "error": error_msg,
+        }
+
+    # this is not saving it to SQL, it's a temp debug log:
+    _log_what_changed(new_model_types, changed_tables)
+
+    # Filter out new model types for the upcoming save:
+    filtered_data = _filter_valid_models(new_data, new_model_types)
+
+    # Save only currently existing models to SQL:
+    save_successful = save_data(filtered_data)
+    if save_successful:
+        return {
+            "success": True,
+            "changed_tables": changed_tables,
+            "new_model_types": new_model_types,
+            "error": None,
+        }
+    else:
+        return {
+            "success": False,
+            "changed_tables": [],
+            "new_model_types": [],
+            "error": "Failed to save metadata to database",
+        }
 
 
-def get_sync_cache_from_api(
+
+
+def _get_sync_cache_from_api(
     client_observer: Optional[ClientObserver] = None,
     connection_observer: Optional[ConnectionObserver] = None
 ) -> dict:
+    logger.info("Syncing with the API...")
 
-    print("we are syncing with the API..")
+    # Get and validate the base URL
-
-    # get the base url & verify it:
     rejected_list = [None, False, ""]
-    # base_url = Constants.SP_API_BASE_URL
+    base_url = Constants.SP_API_BASE_URL
-    base_url = 'https://dev-us-002.simplifiedprivacy.net/api/v1' # temp spoof
     if base_url in rejected_list:
-        # notification = "Base URL is Empty, so it can't fetch prices.."
+        error_msg = "Invalid base URL from configuration"
-        # client_observer.notify(notification)
+        logger.error(error_msg)
-        return {"valid": False, "error_code": "invalid_url"}
+        return {"success": False, "data": None, "error": error_msg}
 
-    # combine with endpoint:
+    # Construct the full endpoint URL
     url = f"{base_url}/cachedsync"
+    logger.debug(f"API endpoint: {url}")
 
     try:
-        print("getting data from server...")
+        logger.debug("Fetching data from API server...")
         sync_results = get_data_from_server(url, connection_observer)
 
         if sync_results in rejected_list:
-            return {"valid": False, "error_code": "sync_failed"}
+            error_msg = "API returned no data"
+            logger.error(error_msg)
+            return {"success": False, "data": None, "error": error_msg}
 
-        logger.debug(f"Inside the sync controller, sync_results is: {sync_results}")
+        logger.debug(f"Raw API response: {sync_results}")
-    except:
-        return {"valid": False, "error_code": "sync_failed"}
 
-    return sync_results
+        final_result = sync_results.get("data", None)
 
+        if final_result:
+            logger.info("Successfully retrieved sync versions metadata from API")
+            return {"success": True, "data": final_result, "error": None}
+        
+        error_msg = "API response missing 'data' field"
+        logger.error(error_msg)
+        return {"success": False, "data": None, "error": error_msg}
+
+    except Exception as e:
+        error_msg = f"API fetch failed: {str(e)}"
+        logger.error(error_msg)
+        return {"success": False, "data": None, "error": error_msg}
+
+
+
+
+
+
+# ============== ISOLATION TESTING ==================
+
+# SPOOF ISOLATION TEST:
+# def _get_sync_cache_from_api(
+#     final_result = {
+#         "version": 5,
+#         "applications": 3,
+#         "application_versions": 2,
+#         "client_version": 2,
+#         "operators": 2,
+#         "locations": 2,
+#         "subscriptions": 1
+#     }
+#     return {"success": True, "data": final_result, "error": None}