From 69834d85d5ff628b7708612b5327704fe00275a0 Mon Sep 17 00:00:00 2001 From: SimplifiedPrivacy Date: Sun, 28 Jun 2026 21:49:19 -0400 Subject: [PATCH] Introduced Database Compatibility checks for old or new versions with different schemas --- core/Constants.py | 2 + core/controllers/ProfileController.py | 2 - core/models/BaseProfile.py | 15 +- core/models/manage/insert.py | 2 +- core/models/manage/session_management.py | 135 ++++++--- core/models/manage/version_check.py | 348 ++++++++++++++++++++++ core/models/orm_models/DatabaseVersion.py | 11 + core/services/sync/get_data_from_api.py | 10 +- 8 files changed, 466 insertions(+), 59 deletions(-) create mode 100644 core/models/manage/version_check.py create mode 100644 core/models/orm_models/DatabaseVersion.py diff --git a/core/Constants.py b/core/Constants.py index e7be2ea..5e8e9f2 100644 --- a/core/Constants.py +++ b/core/Constants.py @@ -6,6 +6,8 @@ import os @dataclass(frozen=True) class Constants: + DB_VERSION_THIS_APP_WANTS = 1 + # ticketing group: TICKET_API_BASE_URL: Final[str] = os.environ.get( "TICKET_API_BASE_URL", "https://ticket.hydraveil.net" diff --git a/core/controllers/ProfileController.py b/core/controllers/ProfileController.py index 5742a74..ffdad9c 100644 --- a/core/controllers/ProfileController.py +++ b/core/controllers/ProfileController.py @@ -30,9 +30,7 @@ class ProfileController: @staticmethod def create(profile: Union[SessionProfile, SystemProfile], profile_observer: ProfileObserver = None): - print("inside profile controller about to save") profile.save() - print("inside profile controller finished save") if profile_observer is not None: profile_observer.notify('created', profile) diff --git a/core/models/BaseProfile.py b/core/models/BaseProfile.py index 54e5c98..3c861af 100644 --- a/core/models/BaseProfile.py +++ b/core/models/BaseProfile.py @@ -65,26 +65,15 @@ class BaseProfile(ABC): return type(self).__name__ == 'SystemProfile' def save(self: Self): - print("save has been called") - config_dict = self.to_dict() # Get dict, not JSON string - print(f"got past the to_dict, config_dict: {config_dict}") - - print("making a location dict") - location_dict = self.location.to_dict() - print(f"got a location dict {location_dict}") + location_dict = self.location.to_dict() # this is from SQLAlchemy, and not JSON-models, that's why it's separate. if self.location: - print("if self.location is true..") config_dict["location"] = location_dict - print(f"dumping into config now {config_dict}") - config_file_contents = json.dumps(config_dict, indent=4) + '\n' - print(f"config_file_contents: {config_file_contents}") - - # legacy: + # legacy version: # config_file_contents = f'{self.to_json(indent=4)}\n' os.makedirs(self.get_config_path(), exist_ok=True) diff --git a/core/models/manage/insert.py b/core/models/manage/insert.py index 9616445..29f20b4 100644 --- a/core/models/manage/insert.py +++ b/core/models/manage/insert.py @@ -4,7 +4,7 @@ from sqlalchemy import create_engine from typing import Type, Dict, Any from sqlalchemy.exc import IntegrityError, SQLAlchemyError from core.models.manage.session_management import get_session -from core.models.orm_models.Base import Base +from core.models.orm_models.Base import BaseModel # all models it knows how to do: from core.models.orm_models.CachedSync import CachedSync diff --git a/core/models/manage/session_management.py b/core/models/manage/session_management.py index 98c4406..90571de 100644 --- a/core/models/manage/session_management.py +++ b/core/models/manage/session_management.py @@ -1,61 +1,120 @@ -from sqlalchemy.orm import sessionmaker -from sqlalchemy import create_engine, inspect +from core.errors.logger import logger from core.models.orm_models.Base import BaseModel -# from core.Constants import Constants +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker import os from pathlib import Path +""" +Note: +At the bottom it initializes the Session and sets it up outside the function loose. +""" + +# ============================================================================ +# PATH & INITIALIZATION +# ============================================================================ + def get_path(): - # XDG Base Directory Specification - xdg_data_home = os.environ.get("XDG_DATA_HOME") - if xdg_data_home: - return Path(xdg_data_home) / "my-app" + """Returns XDG Base Directory path or ~/.local/share/hydra-veil""" + xdg_data = os.getenv("XDG_DATA_HOME") + if xdg_data: + return Path(xdg_data) / "my-app" return Path.home() / ".local" / "share" / "hydra-veil" +# ============================================================================ +# GLOBAL STATE +# ============================================================================ + system_path = get_path() -database_path = f"{system_path}/storage.db" - -# Ensure the database directory exists -db_dir = Path(database_path).parent -db_dir.mkdir(parents=True, exist_ok=True) - -# Create the engine -engine = create_engine(f"sqlite:///{database_path}") - -# Create all tables -BaseModel.metadata.create_all(engine) - -# Create session -Session = sessionmaker(bind=engine) - +database_path = system_path / "storage.db" +engine = None +Session = None _session = None -def init_session(): - """ - Called once at application startup using global variables for the session, - which initialize prior to being called with a false flag. +# ============================================================================ +# ENGINE & SESSION MANAGEMENT +# ============================================================================ - The reason for this is so the app can load prior to be called, - while keeping it a globally accessible variable +def _reinitialize_engine_and_session(): """ + Recreate the global engine and Session factory. + Call this after deleting the database to sync globals with the new DB file. + """ + global engine, Session + + system_path.mkdir(parents=True, exist_ok=True) + engine = create_engine(f"sqlite:///{database_path}") + Session = sessionmaker(bind=engine) + + +def init_session(): + """Initialize the global _session from the global Session factory.""" global _session - BaseModel.metadata.create_all(engine) + if Session is None: + raise RuntimeError("Session factory not initialized. Call _reinitialize_engine_and_session() first.") _session = Session() + def get_session(): - """Get the persistent session""" + """Return the global _session or raise RuntimeError.""" if _session is None: raise RuntimeError("Session not initialized. Call init_session() first.") return _session -def close_session(): - """ - Used prior to shutting down the application to break SQL connections. - In the current refactor, it may be used prior to shut down, such as sync completion, - because a low amount of the total models currently use the new system. - """ +def close_session(): + """Close and reset the global _session.""" global _session - if _session: + if _session is not None: _session.close() - _session = None + _session = None + + +# ============================================================================ +# DATABASE OPERATIONS +# ============================================================================ + +def create_ONLY_db_version_table(): + """Create only the database_version table using the global engine.""" + if engine is None: + raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.") + + from core.models.orm_models.DatabaseVersion import database_version + database_version.create(engine, checkfirst=True) + # DatabaseVersion.__table__.create(engine, checkfirst=True) + + +def create_ALL_tables(): + """Create all tables from BaseModel.metadata using the global engine.""" + print("starting create_ALL_tables") + if engine is None: + raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.") + print("passed engine test") + + from core.models.orm_models.Location import Location + from core.models.orm_models.Operator import Operator + from core.models.orm_models.CachedSync import CachedSync + from core.models.orm_models.EncryptedProxy import EncryptedProxy + + from core.models.SubscriptionPlan import SubscriptionPlan + from core.models.session.ApplicationVersion import ApplicationVersion + + + try: + print("trying this") + BaseModel.metadata.create_all(engine, checkfirst=True) + print("Returning True for Creating all Tables..") + return True + except: + print("couldn't make all tables") + return False + + +# ============================================================================ +# STARTUP: Initialize engine and session on import +# ============================================================================ + +_reinitialize_engine_and_session() +logger.info("[DB MANAGEMENT] Started Engine in Session Management Module..") +init_session() +logger.info("[DB MANAGEMENT] Initialized Session") diff --git a/core/models/manage/version_check.py b/core/models/manage/version_check.py new file mode 100644 index 0000000..c308ee3 --- /dev/null +++ b/core/models/manage/version_check.py @@ -0,0 +1,348 @@ +from core.models.orm_models.DatabaseVersion import database_version +from core.Constants import Constants +from core.errors.logger import logger + + +# generic +from sqlalchemy import exc, text +from sqlalchemy.orm import Session +from typing import Optional + + +def get_database_version(session: Session) -> Optional[int]: + """ + Purpose: + Retrieve the current database version from the single-row version table. + + Rank: + Helper + + Called By: + check_database_compatibility + + Features: + Handles errors with multiple items, failed loads, and other conditions. + + Returns: + int: The database version, or None if retrieval fails. + """ + func_name = "get_database_version" + + # Validate session + if session is None: + logger.error(f"{func_name}: Session is None") + return None + + if not isinstance(session, Session): + logger.error(f"{func_name}: Invalid session type: {type(session)}") + return None + + try: + # Attempt to query the table + logger.debug(f"{func_name}: Attempting to query database_version table") + result = session.query(database_version).all() + + # Handle empty table + if not result: + logger.warning(f"{func_name}: Table is empty, no version row found") + return None + + # Handle multiple rows (shouldn't happen, but log it) + if len(result) > 1: + logger.warning( + f"{func_name}: Expected 1 row, found {len(result)}. " + "Table should contain only a single row. Returning first version." + ) + + # Extract version from first row + version_row = result[0] + version = version_row.version + + # Type validation + if not isinstance(version, int): + logger.error( + f"{func_name}: Version column type mismatch. " + f"Expected int, got {type(version).__name__}: {version}" + ) + return None + + if version < 0: + logger.warning( + f"{func_name}: Version is negative ({version}). " + "This may indicate corrupted data." + ) + + logger.info(f"{func_name}: Successfully retrieved version: {version}") + return version + + except exc.NoSuchTableError: + logger.error( + f"{func_name}: Table 'database_version' does not exist. " + "Ensure the table is created with `database_version.create(engine, checkfirst=True)`" + ) + return None + + except exc.OperationalError as e: + logger.error( + f"{func_name}: Database operational error (connection, locked, or permissions). " + f"Details: {str(e)}" + ) + return None + + except exc.DatabaseError as e: + logger.error( + f"{func_name}: General database error. " + f"Details: {str(e)}" + ) + return None + + except exc.StatementError as e: + logger.error( + f"{func_name}: SQL statement error. " + f"Details: {str(e)}" + ) + return None + + except AttributeError as e: + logger.error( + f"{func_name}: Row does not have 'version' attribute. " + f"Verify table schema matches definition. Details: {str(e)}" + ) + return None + + except TypeError as e: + logger.error( + f"{func_name}: Type error when accessing version. " + f"Details: {str(e)}" + ) + return None + + except Exception as e: + logger.critical( + f"{func_name}: Unexpected exception type {type(e).__name__}. " + f"Details: {str(e)}", + exc_info=True + ) + return None + + +def insert_new_version(session: Session, new_version: int) -> bool: + """ + Purpose: + Insert (or replace) the database version. Clears existing rows and inserts new version. + + Rank: + Helper + + Called By: + GUI's startup script in __main__ + + Args: + session: SQLAlchemy session + new_version: The version number to insert (must be a positive integer) + + Returns: + bool: True if successful, False otherwise. + does NOT raise errors. + """ + func_name = "insert_new_version" # for logging purposes + + # Validate session + if session is None: + logger.error(f"{func_name}: Session is None") + return False + + if not isinstance(session, Session): + logger.error(f"{func_name}: Invalid session type: {type(session)}") + return False + + # Validate new_version parameter + if new_version is None: + logger.error(f"{func_name}: new_version is None") + return False + + if not isinstance(new_version, int): + logger.error( + f"{func_name}: new_version must be an integer, got {type(new_version).__name__}: {new_version}" + ) + return False + + if new_version < 0: + logger.error( + f"{func_name}: new_version must be non-negative, got {new_version}" + ) + return False + + try: + logger.debug(f"{func_name}: Starting transaction for version insert/replace") + + # Check if table exists before proceeding + try: + session.query(database_version).limit(1).all() + except exc.NoSuchTableError: + logger.error( + f"{func_name}: Table 'database_version' does not exist. " + "Ensure the table is created with `database_version.create(engine, checkfirst=True)`" + ) + return False + + # Delete existing rows + logger.debug(f"{func_name}: Deleting existing version rows") + deleted_count = session.query(database_version).delete() + logger.debug(f"{func_name}: Deleted {deleted_count} existing row(s)") + + if deleted_count > 1: + logger.warning( + f"{func_name}: Deleted {deleted_count} rows. " + "Table should contain only a single row at any time." + ) + + # Actually Insert the new version + logger.debug(f"{func_name}: Inserting new version {new_version}") + insert_stmt = database_version.insert().values(version=new_version) + session.execute(insert_stmt) + + # Commit transaction + session.commit() + logger.info(f"{func_name}: Successfully inserted version {new_version}") + return True + + except exc.NoSuchTableError: + logger.error( + f"{func_name}: Table 'database_version' does not exist. " + "Ensure the table is created with `database_version.create(engine, checkfirst=True)`" + ) + session.rollback() + return False + + except exc.IntegrityError as e: + logger.error( + f"{func_name}: Integrity constraint violation (e.g., primary key conflict). " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except exc.OperationalError as e: + logger.error( + f"{func_name}: Database operational error (connection, locked, or permissions). " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except exc.DatabaseError as e: + logger.error( + f"{func_name}: General database error. " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except exc.StatementError as e: + logger.error( + f"{func_name}: SQL statement error. " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except ValueError as e: + logger.error( + f"{func_name}: Value error during insert. " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except Exception as e: + logger.critical( + f"{func_name}: Unexpected exception type {type(e).__name__}. " + f"Details: {str(e)}", + exc_info=True + ) + session.rollback() + return False + +"""Spits back a string based on the reason and error.""" +def get_custom_message(reason: str, compatability_dict: dict) -> str: + """ + Called By: + GUI's startup script in __main__ + """ + if reason == "upgrade": + custom_error = "You just upgraded HydraVeil, so let's upgrade your database to handle the new data types." + elif reason == "old_app": + custom_error = "You're using an old version of HydraVeil, with a newer version of the database. This means the older app would crash trying to handle the newer data." + else: + error_msg = compatability_dict.get("error", "Unknown reason.") + custom_error = f"There was an error with upgrading your database version. Please tell customer support: {error_msg}." + + return custom_error + +# ============================================================================ +# ORCHESTRATOR +# ============================================================================ +def check_database_compatibility(session: Session) -> dict: + """ + Purpose: + Evaluates if the user's existing database is incompatible with the app's version. + + Rank: + Orchestrator + + Called By: + GUI's startup script in __main__ + + Returns: + Dictionary of True/False, with reason + NO error raises. + """ + db_version_you_have = get_database_version(session) + + if db_version_you_have is None: + print("Nothing there") + # First run, corrupted, or empty + return { + "result": True, + "reason": "no_database" # Allow initialization still, since there's nothing to delete. + } + + logger.info(f"[DB MANAGEMENT] The Database version this APP WANTS is {Constants.DB_VERSION_THIS_APP_WANTS}") + logger.info(f"[DB MANAGEMENT] The Database version this device currently has is {db_version_you_have}") + + if Constants.DB_VERSION_THIS_APP_WANTS == db_version_you_have: + logger.info(f"[DB MANAGEMENT] Great, the database versions match exactly.") + return { + "result": True, + "reason": None, + } + + # BAD!! Old version, + elif Constants.DB_VERSION_THIS_APP_WANTS < db_version_you_have: + logger.info(f"[DB MANAGEMENT] The App version is older, this is NOT compatabile") + # make them wipe the tables, or switch versions of the app + return { + "result": False, + "reason": "old_app", + "old_db_version": db_version_you_have + } + + # Upgrade User: + elif Constants.DB_VERSION_THIS_APP_WANTS > db_version_you_have: + logger.info(f"[DB MANAGEMENT] The user has an upgraded App, and an older database") + return { + "result": False, + "reason": "upgrade", + "old_db_version": db_version_you_have + } + + # this is an error + else: + logger.error(f"[DB MANAGEMENT] Unknown Error inside version check module with the King Orchestration") + return { + "result": False, + "reason": "error", + "old_db_version": db_version_you_have, + "error": "unknown result" + } \ No newline at end of file diff --git a/core/models/orm_models/DatabaseVersion.py b/core/models/orm_models/DatabaseVersion.py new file mode 100644 index 0000000..8f85ebc --- /dev/null +++ b/core/models/orm_models/DatabaseVersion.py @@ -0,0 +1,11 @@ +from sqlalchemy import Column, Integer, MetaData, Table + +# Separate metadata for the compatibility table only +compat_metadata = MetaData() + +database_version = Table( + 'database_version', + compat_metadata, + Column('version', Integer, primary_key=True) +) + diff --git a/core/services/sync/get_data_from_api.py b/core/services/sync/get_data_from_api.py index 38f1244..e28ad95 100644 --- a/core/services/sync/get_data_from_api.py +++ b/core/services/sync/get_data_from_api.py @@ -14,7 +14,7 @@ def get_data_from_api( client_observer: Optional[ClientObserver] = None, connection_observer: Optional[ConnectionObserver] = None ) -> dict: - logger.info("Syncing with the API...") + logger.info("Syncing with the API in get_data_from_api...") # Get and validate the base URL rejected_list = [None, False, ""] @@ -44,10 +44,10 @@ def get_data_from_api( if final_result: logger.info("Successfully retrieved sync versions metadata from API") return {"success": True, "data": final_result, "error": None} - - error_msg = "API response missing 'data' field" - logger.error(error_msg) - return {"success": False, "data": None, "error": error_msg} + else: + error_msg = f"API response missing 'data' field at the end of get_data_from_api. This is the response from the other module: {sync_results}" + logger.error(error_msg) + return {"success": False, "data": None, "error": error_msg} except Exception as e: error_msg = f"API fetch failed: {str(e)}"