From c3bdd9c5db07e25e5e3dc893c0749eadab70859d Mon Sep 17 00:00:00 2001 From: SimplifiedPrivacy Date: Tue, 10 Nov 2026 14:39:39 -0500 Subject: [PATCH] If all profiles fail to load, it does a database recovery (migration/wipe) --- core/Constants.py | 2 +- core/controllers/ProfileController.py | 11 +++- core/infrastructure/plumbing.py | 20 ++++++ core/models/Result.py | 3 + core/models/manage/database_recovery.py | 60 ++++++++++++++++++ core/models/manage/migrations.py | 34 ++++------ core/models/orm_models/CachedSync.py | 2 +- core/services/profiles/ProfileRepository.py | 11 +++- core/services/profiles/model_manipulator.py | 62 +++++++++++++++++-- core/services/profiles/profile_connections.py | 6 +- 10 files changed, 175 insertions(+), 36 deletions(-) create mode 100644 core/models/manage/database_recovery.py diff --git a/core/Constants.py b/core/Constants.py index bf98b56..284e6dd 100644 --- a/core/Constants.py +++ b/core/Constants.py @@ -6,7 +6,7 @@ import os @dataclass(frozen=True) class Constants: - DB_VERSION_THIS_APP_WANTS = 3 + DB_VERSION_THIS_APP_WANTS = 4 # Fallback for development (running outside AppImage) fallback_non_appimage = os.path.dirname(os.path.abspath(__file__)) diff --git a/core/controllers/ProfileController.py b/core/controllers/ProfileController.py index 43015fa..844e255 100644 --- a/core/controllers/ProfileController.py +++ b/core/controllers/ProfileController.py @@ -33,7 +33,7 @@ class ProfileController: return ProfileController._repo.get(id) @staticmethod - def get_all(): + def get_all() -> Result: return ProfileController._repo.get_all() @staticmethod @@ -103,8 +103,13 @@ class ProfileController: ProfileController._repo.invalidate(profile_id=profile_id) @staticmethod - def what_type_is_profile(profile_id: int) -> ProfileType: - all_data = ProfileController._repo.get_all() + def what_type_is_profile(profile_id: int) -> ProfileType|None: + profiles_result = ProfileController._repo.get_all() + if not profiles_result.valid: + logger.error(f"Fetch of profiles was invalid. {profiles_result.message}") + return None + + all_data = profiles_result.data given_profile = all_data[profile_id] return given_profile.type diff --git a/core/infrastructure/plumbing.py b/core/infrastructure/plumbing.py index 6e13b6f..529551c 100644 --- a/core/infrastructure/plumbing.py +++ b/core/infrastructure/plumbing.py @@ -66,6 +66,8 @@ class ThreadPlumbing: def _on_complete(self, original_result: Result): """Get a potential function prescription and return it to the main thread.""" + logger.info(original_result) + if not original_result: return @@ -117,10 +119,14 @@ class ThreadPlumbing: #---------------------------- # Back to main thread #---------------------------- + logger.info("bringing back to main thread") self.ui_dispatcher.run_on_main_thread(self._back_on_main_thread, result) def _back_on_main_thread(self, result: Result): """Decide: retry, show popup, or final callback""" + + logger.info(result) + #---------------------------- # OUTRIGHT ERROR STACK #---------------------------- @@ -132,6 +138,20 @@ class ThreadPlumbing: sys.exit() return + #---------------------------- + # END ENTIRE PROGRAM + #---------------------------- + if result.error_type == ResultError.KILL_ENTIRE_APP: + logger.info("Killing the entire app!") + popup = self.error_implementation( + choice=False, + title="Restart the app", + message=result.message, + passed_result=result, + end_app=True + ) + popup.show() + #---------------------------- # NAVIGATE AWAY #---------------------------- diff --git a/core/models/Result.py b/core/models/Result.py index 37f3c8e..3d95929 100644 --- a/core/models/Result.py +++ b/core/models/Result.py @@ -12,6 +12,8 @@ class ResultError(Enum): SUCCESS = "success" NOT_SUPPORTED = "NOT_SUPPORTED" INVALID_INPUT = "invalid_input" + REVERT_VERSION = "revert_version" + DELETE_DB = "delete_db" MISSING_FILE = "missing_file" MISSING_DEPENDENCY = "missing_dependency" MISSING_DATA = "missing_data" @@ -68,6 +70,7 @@ class ResultError(Enum): ATTEMPTS = "attempts" RECONNECT = "reconnect" NAVIGATE_TO = "navigate_to" + KILL_ENTIRE_APP = "kill_entire_app" UNKNOWN = "unknown" @dataclass diff --git a/core/models/manage/database_recovery.py b/core/models/manage/database_recovery.py new file mode 100644 index 0000000..01b57ed --- /dev/null +++ b/core/models/manage/database_recovery.py @@ -0,0 +1,60 @@ +from core.utils.basic_operations.file_tools import delete_file +from core.models.manage.session_management import close_session +from core.models.DatabaseOperation import DatabaseOperation, DBErrorType +from core.models.manage.migrations import migrate_sql +from core.errors.logger import logger +from core.models.Result import Result, ResultError +from core.Constants import Constants + +def wipe_database() -> Result: + """Close the SQLAlchemy session & delete the DB file.""" + logger.info("Closing the session") + close_session() + + logger.info("Deleting the database") + database_path = f"{Constants.HV_DATA_HOME}/storage.db" + logger.info(f"wiping {database_path}") + deleted, error_msg = delete_file(database_path) + + if deleted: + logger.info("Wipe successfull.") + return Result(valid=True) + + logger.error(error_msg) + return Result(valid=False, error_type=ResultError.DATABASE, message=error_msg) + + +def recommend_database_wipe(string_of_problem: str) -> Result: + error_msg = f"{string_of_problem}. We tried a database migration, which did not work. Can we delete the SQL database and restart the app? No profiles will be lost, this is public sync data." + return Result(valid=False, error_type=ResultError.DELETE_DB, message=error_msg) + + +def wipe_and_kill_entire_app() -> Result: + """Wipe the database and close the entire app via plumbing""" + logger.info("Proceeding with the wipe") + wiped = wipe_database() + if wiped.valid: + logger.info("Wipe was valid, returning to close the app") + return Result(valid=False, error_type=ResultError.KILL_ENTIRE_APP, message="It worked. Please restart the app.") + else: + logger.error(wiped.message) + return wiped.wrap("couldn't wipe db") + + +def manage_initial_recovery(string_of_problem: str) -> Result: + """Attempt to migrate, and if it does not work, recommend a course of action.""" + + logger.info("Entering recovery mode. And starting migration") + migration = migrate_sql() + + # worked + if migration.valid: + return Result(valid=True) + + # developer error in python (wipe won't help) + if migration.error_type == DBErrorType.PYTHON_MODEL_STRUCTURE: + error_msg = f"This version of the software has a bug in the database models. Please revert to the earlier version. Model migration gave this error: {migration.message}" + return Result(valid=False, error_type=ResultError.REVERT_VERSION, message=error_msg) + + # proceed with wipe: + return recommend_database_wipe(string_of_problem) \ No newline at end of file diff --git a/core/models/manage/migrations.py b/core/models/manage/migrations.py index 7462f34..b411183 100644 --- a/core/models/manage/migrations.py +++ b/core/models/manage/migrations.py @@ -2,12 +2,7 @@ from core.errors.logger import logger from core.models.DatabaseOperation import DatabaseOperation, DBErrorType from core.Constants import Constants - from core.models.manage.session_management import create_ALL_tables -# from core.models.orm_models.Location import Location -# from core.models.orm_models.Operator import Operator -# from core.models.orm_models.CachedSync import CachedSync -# from core.models.orm_models.EncryptedProxy import EncryptedProxy # generic from pathlib import Path @@ -26,20 +21,12 @@ Situation Used: Metaphor: This replaces something like alembic, by doing it directly - -Confusion: - This may be confusing because we're using an ORM in general, - but doing migrations manually. - -Why: - The reason is because alembic is good for servers, but not clients, - due to a large amount of boilerplate for migrations. """ def get_model_columns(model_class): """ Extract all columns directly from a model class. - Bypasses metadata, caching, and all SQLAlchemy indirection. + Captures server_default to satisfy SQLite NOT NULL requirements. """ try: mapper = inspect(model_class) @@ -49,7 +36,8 @@ def get_model_columns(model_class): columns[column.name] = { 'type': str(column.type), 'nullable': column.nullable, - 'default': column.default, + # server_default is an object; .arg contains the literal SQL string (e.g., '0') + 'server_default': column.server_default.arg if column.server_default else None, 'primary_key': column.primary_key, } @@ -57,7 +45,6 @@ def get_model_columns(model_class): except Exception as e: raise ValueError(f"Failed to inspect {model_class}: {e}") - def migrate_sql() -> DatabaseOperation: from core.models.manage.session_management import engine from core.models.orm_models.Base import BaseModel, Base @@ -75,7 +62,7 @@ def migrate_sql() -> DatabaseOperation: changes_made = [] - # Step 1: Extract column definitions directly from each model + # Step 1: Extract column definitions (including server_default) model_schema = {} for model_class in MODELS: table_name = model_class.__tablename__ @@ -99,7 +86,6 @@ def migrate_sql() -> DatabaseOperation: logger.info(f"[MIGRATION] Table '{table_name}' is MISSING from database") else: db_columns = {col['name']: col for col in inspector.get_columns(table_name)} - for col_name, col_info in expected_columns.items(): if col_name not in db_columns: changes_made.append(col_name) @@ -113,8 +99,8 @@ def migrate_sql() -> DatabaseOperation: try: with engine.begin() as conn: for table_name, expected_columns in model_schema.items(): + # Re-inspect to see what currently exists before adding inspector = inspect(engine) - if table_name in inspector.get_table_names(): db_columns = {col['name'] for col in inspector.get_columns(table_name)} @@ -122,8 +108,15 @@ def migrate_sql() -> DatabaseOperation: if col_name not in db_columns: col_type = col_info['type'] nullable = "NULL" if col_info['nullable'] else "NOT NULL" + + # Fix: Inject server_default so SQLite can handle NOT NULL columns + default_val = col_info.get('server_default') + default_clause = f"DEFAULT {default_val}" if default_val is not None else "" + print(f"[MIGRATION] Adding column '{col_name}' to '{table_name}'") - conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {nullable}")) + # Final SQL: TYPE -> DEFAULT -> NULLABILITY + sql = f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {default_clause} {nullable}" + conn.execute(text(sql)) except (OperationalError, DBAPIError) as e: logger.error(f"[MIGRATION] Failed: {str(e)}") @@ -147,4 +140,3 @@ def migrate_sql() -> DatabaseOperation: valid=True, message=no_migration_needed ) - diff --git a/core/models/orm_models/CachedSync.py b/core/models/orm_models/CachedSync.py index 3a711bf..72f92eb 100644 --- a/core/models/orm_models/CachedSync.py +++ b/core/models/orm_models/CachedSync.py @@ -27,7 +27,7 @@ class CachedSync(Base): subscriptions: Mapped[Optional[int]] = mapped_column(Integer, nullable=False, default=0) # v2 refactor: - plans: Mapped[Optional[int]] = mapped_column(Integer, nullable=False, default=0) + plans: Mapped[Optional[int]] = mapped_column(Integer, nullable=False, server_default="0") diff --git a/core/services/profiles/ProfileRepository.py b/core/services/profiles/ProfileRepository.py index 7d0f756..2f8743e 100644 --- a/core/services/profiles/ProfileRepository.py +++ b/core/services/profiles/ProfileRepository.py @@ -1,4 +1,5 @@ from core.services.profiles import model_manipulator +from core.models.Result import Result from core.errors.logger import logger @@ -61,16 +62,20 @@ class ProfileRepository: return profile - def get_all(self) -> Dict[int, any]: + def get_all(self) -> Result: # -> Dict[int, any]: """On the first run, return all profiles from disk. On future runs, return it from cache.""" # Only run the slow SQL/file operation if the cache is empty if not self._cache: # model_manipulator returns a dict { id: profile } - all_profiles_dict = model_manipulator.get_all_profiles() + profile_result = model_manipulator.get_all_profiles() + if not profile_result.valid: + return profile_result + all_profiles_dict = profile_result.data + logger.info("Got all profiles.") # Sync the cache with whatever is currently on disk self._cache.update(all_profiles_dict) - return self._cache + return Result(valid=True, data=self._cache) diff --git a/core/services/profiles/model_manipulator.py b/core/services/profiles/model_manipulator.py index 227cb8e..54ab4fd 100644 --- a/core/services/profiles/model_manipulator.py +++ b/core/services/profiles/model_manipulator.py @@ -1,3 +1,4 @@ +from core.models.manage.database_recovery import manage_initial_recovery, recommend_database_wipe from core.services.config_tools.manage_sudo_configs import remove_sudo_config from core.models.session.SessionProfile import SessionProfile from core.models.orm_calls.location_calls import LocationCache @@ -137,7 +138,8 @@ def make_subscription_from_config(config_contents: dict) -> Subscription: def create_profile_object_from_disk(profile_id: int) -> Result: config_contents = read_profile_from_disk(profile_id) - logger.info(f"config_contents are {config_contents}") + # logger.info(f"config_contents are {config_contents}") + if not config_contents: return Result(valid=False, error_type=ResultError.MISSING_DATA, message="Missing the config") @@ -215,13 +217,15 @@ def create_profile(profile_id) -> Profile|None: return result.data - -def get_all_profiles() -> dict: +def _get_all_profiles() -> Result: """Find and Create all Profiles from what's available in the folders.""" + all_profiles = {} skipped_profiles = 0 + total_items = 0 for directory_entry in os.listdir(Constants.HV_PROFILE_CONFIG_HOME): + total_items = total_items + 1 try: profile_id = int(directory_entry) except ValueError: @@ -241,9 +245,55 @@ def get_all_profiles() -> dict: all_profiles[profile_id] = profile full_final_dict = dict(sorted(all_profiles.items())) - if skipped_profiles > 0: - logger.error(f"Skipped {skipped_profiles} profiles") - return full_final_dict + + # SUCCESS + if skipped_profiles == 0: + return Result(valid=True, data=full_final_dict) + + logger.error(f"Skipped {skipped_profiles} profiles") + + # TOTAL FAILURE + if skipped_profiles == total_items: + load_error = f"All {total_items} profiles were skipped from loading!" + return Result(valid=False, data=None, error_type=ResultError.MISSING_DATA, message=load_error) + + logger.error(f"Partial profile load failure.") + return Result(valid=False, data=full_final_dict) + + +def get_all_profiles() -> Result: + """Attempt to get all profiles, if it fails, try migration or ask for a database wipe.""" + logger.info("Getting all profiles") + initial_try = _get_all_profiles() + + logger.info(f"Did it work? {initial_try.valid}") + + # worked: + if initial_try.valid: + logger.info("It fully worked for all profiles. Returning data.") + return initial_try + + # partial success, get the data: + if initial_try.data is not None: + logger.info("Some profiles failed, but others worked. Partial success, flipping it to Result True because we do not yet have other means to solve this.") + initial_try.valid = True + return initial_try + + load_error = initial_try.message + logger.error(f"{load_error} This is likely a database issue. First try to migrate") + initial_recovery = manage_initial_recovery(string_of_problem=load_error) + if not initial_recovery.valid: + logger.error(initial_recovery.message) + return initial_recovery.wrap("Couldn't migrate") + + logger.info("Initial steps, such as migration, worked, let's try again.") + attempt_profile_load_again = _get_all_profiles() + + # worked after initial recovery: + if attempt_profile_load_again.valid: + return attempt_profile_load_again + + return recommend_database_wipe(string_of_problem=load_error) def is_actually_different(val1, val2): diff --git a/core/services/profiles/profile_connections.py b/core/services/profiles/profile_connections.py index 0d61208..3278ecc 100644 --- a/core/services/profiles/profile_connections.py +++ b/core/services/profiles/profile_connections.py @@ -265,7 +265,11 @@ def disable_profile( # ============================================================================ if profile.is_system_profile(): - subjects = ProfileController.get_all().values() + profile_result = ProfileController.get_all() + if not profile_result.valid: + return profile_result.wrap("Unable to get all profiles do disconnect.") + + subjects = profile_result.data.values() for subject in subjects: