Introduced Database Compatibility checks for old or new versions with different schemas

core/Constants.py

@@ -6,6 +6,8 @@ import os
 @dataclass(frozen=True)
 class Constants:
 
+    DB_VERSION_THIS_APP_WANTS = 1
+
     # ticketing group:
     TICKET_API_BASE_URL: Final[str] = os.environ.get(
         "TICKET_API_BASE_URL", "https://ticket.hydraveil.net"

core/controllers/ProfileController.py

@@ -30,9 +30,7 @@ class ProfileController:
     @staticmethod
     def create(profile: Union[SessionProfile, SystemProfile], profile_observer: ProfileObserver = None):
 
-        print("inside profile controller about to save")
         profile.save()
-        print("inside profile controller finished save")
 
         if profile_observer is not None:
             profile_observer.notify('created', profile)

core/models/BaseProfile.py

@@ -65,26 +65,15 @@ class BaseProfile(ABC):
         return type(self).__name__ == 'SystemProfile'
 
     def save(self: Self):
-        print("save has been called")
-        
         config_dict = self.to_dict()  # Get dict, not JSON string
-        print(f"got past the to_dict, config_dict: {config_dict}")
-
-        print("making a location dict")
-        location_dict = self.location.to_dict()
-        print(f"got a location dict {location_dict}")
+        location_dict = self.location.to_dict() # this is from SQLAlchemy, and not JSON-models, that's why it's separate.
 
         if self.location:
-            print("if self.location is true..")
             config_dict["location"] = location_dict
 
-        print(f"dumping into config now {config_dict}")
-
         config_file_contents = json.dumps(config_dict, indent=4) + '\n'
 
-        print(f"config_file_contents: {config_file_contents}")
-
-        # legacy:
+        # legacy version:
         # config_file_contents = f'{self.to_json(indent=4)}\n'
 
         os.makedirs(self.get_config_path(), exist_ok=True)

core/models/manage/insert.py

@@ -4,7 +4,7 @@ from sqlalchemy import create_engine
 from typing import Type, Dict, Any
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
 from core.models.manage.session_management import get_session
-from core.models.orm_models.Base import Base
+from core.models.orm_models.Base import BaseModel
 
 # all models it knows how to do:
 from core.models.orm_models.CachedSync import CachedSync

core/models/manage/session_management.py

@@ -1,61 +1,120 @@
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy import create_engine, inspect
+from core.errors.logger import logger
 from core.models.orm_models.Base import BaseModel
-# from core.Constants import Constants
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
 import os
 from pathlib import Path
 
+"""
+Note:
+At the bottom it initializes the Session and sets it up outside the function loose.
+"""
+
+# ============================================================================
+# PATH & INITIALIZATION
+# ============================================================================
+
 def get_path():
-    # XDG Base Directory Specification
-    xdg_data_home = os.environ.get("XDG_DATA_HOME")
-    if xdg_data_home:
-        return Path(xdg_data_home) / "my-app"
+    """Returns XDG Base Directory path or ~/.local/share/hydra-veil"""
+    xdg_data = os.getenv("XDG_DATA_HOME")
+    if xdg_data:
+        return Path(xdg_data) / "my-app"
     return Path.home() / ".local" / "share" / "hydra-veil"
 
+# ============================================================================
+# GLOBAL STATE
+# ============================================================================
+
 system_path = get_path()
-database_path = f"{system_path}/storage.db"
-
-# Ensure the database directory exists
-db_dir = Path(database_path).parent
-db_dir.mkdir(parents=True, exist_ok=True)
-
-# Create the engine
-engine = create_engine(f"sqlite:///{database_path}")
-
-# Create all tables
-BaseModel.metadata.create_all(engine)
-
-# Create session
-Session = sessionmaker(bind=engine)
-
+database_path = system_path / "storage.db"
+engine = None
+Session = None
 _session = None
 
-def init_session():
-    """
-    Called once at application startup using global variables for the session, 
-    which initialize prior to being called with a false flag.
+# ============================================================================
+# ENGINE & SESSION MANAGEMENT
+# ============================================================================
 
-    The reason for this is so the app can load prior to be called,
-    while keeping it a globally accessible variable
+def _reinitialize_engine_and_session():
     """
+    Recreate the global engine and Session factory.
+    Call this after deleting the database to sync globals with the new DB file.
+    """
+    global engine, Session
+    
+    system_path.mkdir(parents=True, exist_ok=True)
+    engine = create_engine(f"sqlite:///{database_path}")
+    Session = sessionmaker(bind=engine)
+
+
+def init_session():
+    """Initialize the global _session from the global Session factory."""
     global _session
-    BaseModel.metadata.create_all(engine)
+    if Session is None:
+        raise RuntimeError("Session factory not initialized. Call _reinitialize_engine_and_session() first.")
     _session = Session()
 
+
 def get_session():
-    """Get the persistent session"""
+    """Return the global _session or raise RuntimeError."""
     if _session is None:
         raise RuntimeError("Session not initialized. Call init_session() first.")
     return _session
 
-def close_session():
-    """
-    Used prior to shutting down the application to break SQL connections.
 
-    In the current refactor, it may be used prior to shut down, such as sync completion,
-    because a low amount of the total models currently use the new system.
-    """
+def close_session():
+    """Close and reset the global _session."""
     global _session
-    if _session:
+    if _session is not None:
         _session.close()
-        _session = None
+    _session = None
+
+
+# ============================================================================
+# DATABASE OPERATIONS
+# ============================================================================
+
+def create_ONLY_db_version_table():
+    """Create only the database_version table using the global engine."""
+    if engine is None:
+        raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
+    
+    from core.models.orm_models.DatabaseVersion import database_version
+    database_version.create(engine, checkfirst=True)
+    # DatabaseVersion.__table__.create(engine, checkfirst=True)
+
+
+def create_ALL_tables():
+    """Create all tables from BaseModel.metadata using the global engine."""
+    print("starting create_ALL_tables")
+    if engine is None:
+        raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
+    print("passed engine test")
+
+    from core.models.orm_models.Location import Location
+    from core.models.orm_models.Operator import Operator
+    from core.models.orm_models.CachedSync import CachedSync
+    from core.models.orm_models.EncryptedProxy import EncryptedProxy
+
+    from core.models.SubscriptionPlan import SubscriptionPlan
+    from core.models.session.ApplicationVersion import ApplicationVersion
+    
+
+    try:
+        print("trying this")
+        BaseModel.metadata.create_all(engine, checkfirst=True)
+        print("Returning True for Creating all Tables..")
+        return True
+    except:
+        print("couldn't make all tables")
+        return False
+
+
+# ============================================================================
+# STARTUP: Initialize engine and session on import
+# ============================================================================
+
+_reinitialize_engine_and_session()
+logger.info("[DB MANAGEMENT] Started Engine in Session Management Module..")
+init_session()
+logger.info("[DB MANAGEMENT] Initialized Session")

core/models/manage/version_check.py

@@ -0,0 +1,348 @@
+from core.models.orm_models.DatabaseVersion import database_version
+from core.Constants import Constants
+from core.errors.logger import logger
+
+
+# generic
+from sqlalchemy import exc, text
+from sqlalchemy.orm import Session
+from typing import Optional
+
+
+def get_database_version(session: Session) -> Optional[int]:
+    """
+    Purpose:
+        Retrieve the current database version from the single-row version table.
+
+    Rank:
+        Helper
+
+    Called By:
+        check_database_compatibility
+
+    Features:
+        Handles errors with multiple items, failed loads, and other conditions.
+
+    Returns:
+        int: The database version, or None if retrieval fails.
+    """
+    func_name = "get_database_version"
+    
+    # Validate session
+    if session is None:
+        logger.error(f"{func_name}: Session is None")
+        return None
+    
+    if not isinstance(session, Session):
+        logger.error(f"{func_name}: Invalid session type: {type(session)}")
+        return None
+    
+    try:
+        # Attempt to query the table
+        logger.debug(f"{func_name}: Attempting to query database_version table")
+        result = session.query(database_version).all()
+        
+        # Handle empty table
+        if not result:
+            logger.warning(f"{func_name}: Table is empty, no version row found")
+            return None
+        
+        # Handle multiple rows (shouldn't happen, but log it)
+        if len(result) > 1:
+            logger.warning(
+                f"{func_name}: Expected 1 row, found {len(result)}. "
+                "Table should contain only a single row. Returning first version."
+            )
+        
+        # Extract version from first row
+        version_row = result[0]
+        version = version_row.version
+        
+        # Type validation
+        if not isinstance(version, int):
+            logger.error(
+                f"{func_name}: Version column type mismatch. "
+                f"Expected int, got {type(version).__name__}: {version}"
+            )
+            return None
+        
+        if version < 0:
+            logger.warning(
+                f"{func_name}: Version is negative ({version}). "
+                "This may indicate corrupted data."
+            )
+        
+        logger.info(f"{func_name}: Successfully retrieved version: {version}")
+        return version
+    
+    except exc.NoSuchTableError:
+        logger.error(
+            f"{func_name}: Table 'database_version' does not exist. "
+            "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
+        )
+        return None
+    
+    except exc.OperationalError as e:
+        logger.error(
+            f"{func_name}: Database operational error (connection, locked, or permissions). "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except exc.DatabaseError as e:
+        logger.error(
+            f"{func_name}: General database error. "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except exc.StatementError as e:
+        logger.error(
+            f"{func_name}: SQL statement error. "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except AttributeError as e:
+        logger.error(
+            f"{func_name}: Row does not have 'version' attribute. "
+            f"Verify table schema matches definition. Details: {str(e)}"
+        )
+        return None
+    
+    except TypeError as e:
+        logger.error(
+            f"{func_name}: Type error when accessing version. "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except Exception as e:
+        logger.critical(
+            f"{func_name}: Unexpected exception type {type(e).__name__}. "
+            f"Details: {str(e)}",
+            exc_info=True
+        )
+        return None
+
+
+def insert_new_version(session: Session, new_version: int) -> bool:
+    """
+    Purpose:
+        Insert (or replace) the database version. Clears existing rows and inserts new version.
+
+    Rank:
+        Helper
+
+    Called By:
+        GUI's startup script in __main__
+
+    Args:
+        session: SQLAlchemy session
+        new_version: The version number to insert (must be a positive integer)
+    
+    Returns:
+        bool: True if successful, False otherwise.
+        does NOT raise errors.
+    """
+    func_name = "insert_new_version" # for logging purposes
+    
+    # Validate session
+    if session is None:
+        logger.error(f"{func_name}: Session is None")
+        return False
+    
+    if not isinstance(session, Session):
+        logger.error(f"{func_name}: Invalid session type: {type(session)}")
+        return False
+    
+    # Validate new_version parameter
+    if new_version is None:
+        logger.error(f"{func_name}: new_version is None")
+        return False
+    
+    if not isinstance(new_version, int):
+        logger.error(
+            f"{func_name}: new_version must be an integer, got {type(new_version).__name__}: {new_version}"
+        )
+        return False
+    
+    if new_version < 0:
+        logger.error(
+            f"{func_name}: new_version must be non-negative, got {new_version}"
+        )
+        return False
+    
+    try:
+        logger.debug(f"{func_name}: Starting transaction for version insert/replace")
+        
+        # Check if table exists before proceeding
+        try:
+            session.query(database_version).limit(1).all()
+        except exc.NoSuchTableError:
+            logger.error(
+                f"{func_name}: Table 'database_version' does not exist. "
+                "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
+            )
+            return False
+        
+        # Delete existing rows
+        logger.debug(f"{func_name}: Deleting existing version rows")
+        deleted_count = session.query(database_version).delete()
+        logger.debug(f"{func_name}: Deleted {deleted_count} existing row(s)")
+        
+        if deleted_count > 1:
+            logger.warning(
+                f"{func_name}: Deleted {deleted_count} rows. "
+                "Table should contain only a single row at any time."
+            )
+        
+        # Actually Insert the new version
+        logger.debug(f"{func_name}: Inserting new version {new_version}")
+        insert_stmt = database_version.insert().values(version=new_version)
+        session.execute(insert_stmt)
+        
+        # Commit transaction
+        session.commit()
+        logger.info(f"{func_name}: Successfully inserted version {new_version}")
+        return True
+    
+    except exc.NoSuchTableError:
+        logger.error(
+            f"{func_name}: Table 'database_version' does not exist. "
+            "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
+        )
+        session.rollback()
+        return False
+    
+    except exc.IntegrityError as e:
+        logger.error(
+            f"{func_name}: Integrity constraint violation (e.g., primary key conflict). "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.OperationalError as e:
+        logger.error(
+            f"{func_name}: Database operational error (connection, locked, or permissions). "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.DatabaseError as e:
+        logger.error(
+            f"{func_name}: General database error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.StatementError as e:
+        logger.error(
+            f"{func_name}: SQL statement error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except ValueError as e:
+        logger.error(
+            f"{func_name}: Value error during insert. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except Exception as e:
+        logger.critical(
+            f"{func_name}: Unexpected exception type {type(e).__name__}. "
+            f"Details: {str(e)}",
+            exc_info=True
+        )
+        session.rollback()
+        return False
+
+"""Spits back a string based on the reason and error."""
+def get_custom_message(reason: str, compatability_dict: dict) -> str:
+    """
+    Called By:
+        GUI's startup script in __main__
+    """
+    if reason == "upgrade":
+        custom_error = "You just upgraded HydraVeil, so let's upgrade your database to handle the new data types."
+    elif reason == "old_app":
+        custom_error = "You're using an old version of HydraVeil, with a newer version of the database.  This means the older app would crash trying to handle the newer data."
+    else:
+        error_msg = compatability_dict.get("error", "Unknown reason.")
+        custom_error = f"There was an error with upgrading your database version.  Please tell customer support: {error_msg}."
+
+    return custom_error
+
+# ============================================================================
+# ORCHESTRATOR
+# ============================================================================
+def check_database_compatibility(session: Session) -> dict:
+    """
+    Purpose:
+        Evaluates if the user's existing database is incompatible with the app's version.
+
+    Rank:
+        Orchestrator
+
+    Called By:
+        GUI's startup script in __main__
+
+    Returns:
+        Dictionary of True/False, with reason
+        NO error raises.
+    """
+    db_version_you_have = get_database_version(session)
+   
+    if db_version_you_have is None:
+        print("Nothing there")
+        # First run, corrupted, or empty
+        return {
+            "result": True,
+            "reason": "no_database" # Allow initialization still, since there's nothing to delete.
+        }
+
+    logger.info(f"[DB MANAGEMENT] The Database version this APP WANTS is {Constants.DB_VERSION_THIS_APP_WANTS}")
+    logger.info(f"[DB MANAGEMENT] The Database version this device currently has is {db_version_you_have}")
+
+    if Constants.DB_VERSION_THIS_APP_WANTS == db_version_you_have:
+        logger.info(f"[DB MANAGEMENT] Great, the database versions match exactly.")
+        return {
+            "result": True,
+            "reason": None,
+        }
+
+    # BAD!! Old version,
+    elif Constants.DB_VERSION_THIS_APP_WANTS < db_version_you_have:
+        logger.info(f"[DB MANAGEMENT] The App version is older, this is NOT compatabile")
+        # make them wipe the tables, or switch versions of the app
+        return {
+           "result": False,
+            "reason": "old_app",
+            "old_db_version": db_version_you_have
+        }
+
+    # Upgrade User:
+    elif Constants.DB_VERSION_THIS_APP_WANTS > db_version_you_have:
+        logger.info(f"[DB MANAGEMENT] The user has an upgraded App, and an older database")
+        return {
+            "result": False,
+            "reason": "upgrade",
+            "old_db_version": db_version_you_have
+        }
+
+    # this is an error
+    else:
+        logger.error(f"[DB MANAGEMENT] Unknown Error inside version check module with the King Orchestration")
+        return {
+            "result": False,
+            "reason": "error",
+            "old_db_version": db_version_you_have,
+            "error": "unknown result"
+        }

core/models/orm_models/DatabaseVersion.py

@@ -0,0 +1,11 @@
+from sqlalchemy import Column, Integer, MetaData, Table
+
+# Separate metadata for the compatibility table only
+compat_metadata = MetaData()
+
+database_version = Table(
+    'database_version',
+    compat_metadata,
+    Column('version', Integer, primary_key=True)
+)
+

core/services/sync/get_data_from_api.py

@@ -14,7 +14,7 @@ def get_data_from_api(
     client_observer: Optional[ClientObserver] = None,
     connection_observer: Optional[ConnectionObserver] = None
 ) -> dict:
-    logger.info("Syncing with the API...")
+    logger.info("Syncing with the API in get_data_from_api...")
 
     # Get and validate the base URL
     rejected_list = [None, False, ""]
@@ -44,10 +44,10 @@ def get_data_from_api(
         if final_result:
             logger.info("Successfully retrieved sync versions metadata from API")
             return {"success": True, "data": final_result, "error": None}
-        
-        error_msg = "API response missing 'data' field"
-        logger.error(error_msg)
-        return {"success": False, "data": None, "error": error_msg}
+        else:
+            error_msg = f"API response missing 'data' field at the end of get_data_from_api. This is the response from the other module: {sync_results}"
+            logger.error(error_msg)
+            return {"success": False, "data": None, "error": error_msg}
 
     except Exception as e:
         error_msg = f"API fetch failed: {str(e)}"