If all profiles fail to load, it does a database recovery (migration/wipe)

This commit is contained in:
SimplifiedPrivacy 2026-11-10 14:39:39 -05:00
parent fad5f99fb6
commit c3bdd9c5db
10 changed files with 175 additions and 36 deletions

View file

@ -6,7 +6,7 @@ import os
@dataclass(frozen=True) @dataclass(frozen=True)
class Constants: class Constants:
DB_VERSION_THIS_APP_WANTS = 3 DB_VERSION_THIS_APP_WANTS = 4
# Fallback for development (running outside AppImage) # Fallback for development (running outside AppImage)
fallback_non_appimage = os.path.dirname(os.path.abspath(__file__)) fallback_non_appimage = os.path.dirname(os.path.abspath(__file__))

View file

@ -33,7 +33,7 @@ class ProfileController:
return ProfileController._repo.get(id) return ProfileController._repo.get(id)
@staticmethod @staticmethod
def get_all(): def get_all() -> Result:
return ProfileController._repo.get_all() return ProfileController._repo.get_all()
@staticmethod @staticmethod
@ -103,8 +103,13 @@ class ProfileController:
ProfileController._repo.invalidate(profile_id=profile_id) ProfileController._repo.invalidate(profile_id=profile_id)
@staticmethod @staticmethod
def what_type_is_profile(profile_id: int) -> ProfileType: def what_type_is_profile(profile_id: int) -> ProfileType|None:
all_data = ProfileController._repo.get_all() profiles_result = ProfileController._repo.get_all()
if not profiles_result.valid:
logger.error(f"Fetch of profiles was invalid. {profiles_result.message}")
return None
all_data = profiles_result.data
given_profile = all_data[profile_id] given_profile = all_data[profile_id]
return given_profile.type return given_profile.type

View file

@ -66,6 +66,8 @@ class ThreadPlumbing:
def _on_complete(self, original_result: Result): def _on_complete(self, original_result: Result):
"""Get a potential function prescription and return it to the main thread.""" """Get a potential function prescription and return it to the main thread."""
logger.info(original_result)
if not original_result: if not original_result:
return return
@ -117,10 +119,14 @@ class ThreadPlumbing:
#---------------------------- #----------------------------
# Back to main thread # Back to main thread
#---------------------------- #----------------------------
logger.info("bringing back to main thread")
self.ui_dispatcher.run_on_main_thread(self._back_on_main_thread, result) self.ui_dispatcher.run_on_main_thread(self._back_on_main_thread, result)
def _back_on_main_thread(self, result: Result): def _back_on_main_thread(self, result: Result):
"""Decide: retry, show popup, or final callback""" """Decide: retry, show popup, or final callback"""
logger.info(result)
#---------------------------- #----------------------------
# OUTRIGHT ERROR STACK # OUTRIGHT ERROR STACK
#---------------------------- #----------------------------
@ -132,6 +138,20 @@ class ThreadPlumbing:
sys.exit() sys.exit()
return return
#----------------------------
# END ENTIRE PROGRAM
#----------------------------
if result.error_type == ResultError.KILL_ENTIRE_APP:
logger.info("Killing the entire app!")
popup = self.error_implementation(
choice=False,
title="Restart the app",
message=result.message,
passed_result=result,
end_app=True
)
popup.show()
#---------------------------- #----------------------------
# NAVIGATE AWAY # NAVIGATE AWAY
#---------------------------- #----------------------------

View file

@ -12,6 +12,8 @@ class ResultError(Enum):
SUCCESS = "success" SUCCESS = "success"
NOT_SUPPORTED = "NOT_SUPPORTED" NOT_SUPPORTED = "NOT_SUPPORTED"
INVALID_INPUT = "invalid_input" INVALID_INPUT = "invalid_input"
REVERT_VERSION = "revert_version"
DELETE_DB = "delete_db"
MISSING_FILE = "missing_file" MISSING_FILE = "missing_file"
MISSING_DEPENDENCY = "missing_dependency" MISSING_DEPENDENCY = "missing_dependency"
MISSING_DATA = "missing_data" MISSING_DATA = "missing_data"
@ -68,6 +70,7 @@ class ResultError(Enum):
ATTEMPTS = "attempts" ATTEMPTS = "attempts"
RECONNECT = "reconnect" RECONNECT = "reconnect"
NAVIGATE_TO = "navigate_to" NAVIGATE_TO = "navigate_to"
KILL_ENTIRE_APP = "kill_entire_app"
UNKNOWN = "unknown" UNKNOWN = "unknown"
@dataclass @dataclass

View file

@ -0,0 +1,60 @@
from core.utils.basic_operations.file_tools import delete_file
from core.models.manage.session_management import close_session
from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
from core.models.manage.migrations import migrate_sql
from core.errors.logger import logger
from core.models.Result import Result, ResultError
from core.Constants import Constants
def wipe_database() -> Result:
"""Close the SQLAlchemy session & delete the DB file."""
logger.info("Closing the session")
close_session()
logger.info("Deleting the database")
database_path = f"{Constants.HV_DATA_HOME}/storage.db"
logger.info(f"wiping {database_path}")
deleted, error_msg = delete_file(database_path)
if deleted:
logger.info("Wipe successfull.")
return Result(valid=True)
logger.error(error_msg)
return Result(valid=False, error_type=ResultError.DATABASE, message=error_msg)
def recommend_database_wipe(string_of_problem: str) -> Result:
error_msg = f"{string_of_problem}. We tried a database migration, which did not work. Can we delete the SQL database and restart the app? No profiles will be lost, this is public sync data."
return Result(valid=False, error_type=ResultError.DELETE_DB, message=error_msg)
def wipe_and_kill_entire_app() -> Result:
"""Wipe the database and close the entire app via plumbing"""
logger.info("Proceeding with the wipe")
wiped = wipe_database()
if wiped.valid:
logger.info("Wipe was valid, returning to close the app")
return Result(valid=False, error_type=ResultError.KILL_ENTIRE_APP, message="It worked. Please restart the app.")
else:
logger.error(wiped.message)
return wiped.wrap("couldn't wipe db")
def manage_initial_recovery(string_of_problem: str) -> Result:
"""Attempt to migrate, and if it does not work, recommend a course of action."""
logger.info("Entering recovery mode. And starting migration")
migration = migrate_sql()
# worked
if migration.valid:
return Result(valid=True)
# developer error in python (wipe won't help)
if migration.error_type == DBErrorType.PYTHON_MODEL_STRUCTURE:
error_msg = f"This version of the software has a bug in the database models. Please revert to the earlier version. Model migration gave this error: {migration.message}"
return Result(valid=False, error_type=ResultError.REVERT_VERSION, message=error_msg)
# proceed with wipe:
return recommend_database_wipe(string_of_problem)

View file

@ -2,12 +2,7 @@
from core.errors.logger import logger from core.errors.logger import logger
from core.models.DatabaseOperation import DatabaseOperation, DBErrorType from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
from core.Constants import Constants from core.Constants import Constants
from core.models.manage.session_management import create_ALL_tables from core.models.manage.session_management import create_ALL_tables
# from core.models.orm_models.Location import Location
# from core.models.orm_models.Operator import Operator
# from core.models.orm_models.CachedSync import CachedSync
# from core.models.orm_models.EncryptedProxy import EncryptedProxy
# generic # generic
from pathlib import Path from pathlib import Path
@ -26,20 +21,12 @@ Situation Used:
Metaphor: Metaphor:
This replaces something like alembic, by doing it directly This replaces something like alembic, by doing it directly
Confusion:
This may be confusing because we're using an ORM in general,
but doing migrations manually.
Why:
The reason is because alembic is good for servers, but not clients,
due to a large amount of boilerplate for migrations.
""" """
def get_model_columns(model_class): def get_model_columns(model_class):
""" """
Extract all columns directly from a model class. Extract all columns directly from a model class.
Bypasses metadata, caching, and all SQLAlchemy indirection. Captures server_default to satisfy SQLite NOT NULL requirements.
""" """
try: try:
mapper = inspect(model_class) mapper = inspect(model_class)
@ -49,7 +36,8 @@ def get_model_columns(model_class):
columns[column.name] = { columns[column.name] = {
'type': str(column.type), 'type': str(column.type),
'nullable': column.nullable, 'nullable': column.nullable,
'default': column.default, # server_default is an object; .arg contains the literal SQL string (e.g., '0')
'server_default': column.server_default.arg if column.server_default else None,
'primary_key': column.primary_key, 'primary_key': column.primary_key,
} }
@ -57,7 +45,6 @@ def get_model_columns(model_class):
except Exception as e: except Exception as e:
raise ValueError(f"Failed to inspect {model_class}: {e}") raise ValueError(f"Failed to inspect {model_class}: {e}")
def migrate_sql() -> DatabaseOperation: def migrate_sql() -> DatabaseOperation:
from core.models.manage.session_management import engine from core.models.manage.session_management import engine
from core.models.orm_models.Base import BaseModel, Base from core.models.orm_models.Base import BaseModel, Base
@ -75,7 +62,7 @@ def migrate_sql() -> DatabaseOperation:
changes_made = [] changes_made = []
# Step 1: Extract column definitions directly from each model # Step 1: Extract column definitions (including server_default)
model_schema = {} model_schema = {}
for model_class in MODELS: for model_class in MODELS:
table_name = model_class.__tablename__ table_name = model_class.__tablename__
@ -99,7 +86,6 @@ def migrate_sql() -> DatabaseOperation:
logger.info(f"[MIGRATION] Table '{table_name}' is MISSING from database") logger.info(f"[MIGRATION] Table '{table_name}' is MISSING from database")
else: else:
db_columns = {col['name']: col for col in inspector.get_columns(table_name)} db_columns = {col['name']: col for col in inspector.get_columns(table_name)}
for col_name, col_info in expected_columns.items(): for col_name, col_info in expected_columns.items():
if col_name not in db_columns: if col_name not in db_columns:
changes_made.append(col_name) changes_made.append(col_name)
@ -113,8 +99,8 @@ def migrate_sql() -> DatabaseOperation:
try: try:
with engine.begin() as conn: with engine.begin() as conn:
for table_name, expected_columns in model_schema.items(): for table_name, expected_columns in model_schema.items():
# Re-inspect to see what currently exists before adding
inspector = inspect(engine) inspector = inspect(engine)
if table_name in inspector.get_table_names(): if table_name in inspector.get_table_names():
db_columns = {col['name'] for col in inspector.get_columns(table_name)} db_columns = {col['name'] for col in inspector.get_columns(table_name)}
@ -122,8 +108,15 @@ def migrate_sql() -> DatabaseOperation:
if col_name not in db_columns: if col_name not in db_columns:
col_type = col_info['type'] col_type = col_info['type']
nullable = "NULL" if col_info['nullable'] else "NOT NULL" nullable = "NULL" if col_info['nullable'] else "NOT NULL"
# Fix: Inject server_default so SQLite can handle NOT NULL columns
default_val = col_info.get('server_default')
default_clause = f"DEFAULT {default_val}" if default_val is not None else ""
print(f"[MIGRATION] Adding column '{col_name}' to '{table_name}'") print(f"[MIGRATION] Adding column '{col_name}' to '{table_name}'")
conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {nullable}")) # Final SQL: TYPE -> DEFAULT -> NULLABILITY
sql = f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {default_clause} {nullable}"
conn.execute(text(sql))
except (OperationalError, DBAPIError) as e: except (OperationalError, DBAPIError) as e:
logger.error(f"[MIGRATION] Failed: {str(e)}") logger.error(f"[MIGRATION] Failed: {str(e)}")
@ -147,4 +140,3 @@ def migrate_sql() -> DatabaseOperation:
valid=True, valid=True,
message=no_migration_needed message=no_migration_needed
) )

View file

@ -27,7 +27,7 @@ class CachedSync(Base):
subscriptions: Mapped[Optional[int]] = mapped_column(Integer, nullable=False, default=0) subscriptions: Mapped[Optional[int]] = mapped_column(Integer, nullable=False, default=0)
# v2 refactor: # v2 refactor:
plans: Mapped[Optional[int]] = mapped_column(Integer, nullable=False, default=0) plans: Mapped[Optional[int]] = mapped_column(Integer, nullable=False, server_default="0")

View file

@ -1,4 +1,5 @@
from core.services.profiles import model_manipulator from core.services.profiles import model_manipulator
from core.models.Result import Result
from core.errors.logger import logger from core.errors.logger import logger
@ -61,16 +62,20 @@ class ProfileRepository:
return profile return profile
def get_all(self) -> Dict[int, any]: def get_all(self) -> Result: # -> Dict[int, any]:
"""On the first run, return all profiles from disk. On future runs, return it from cache.""" """On the first run, return all profiles from disk. On future runs, return it from cache."""
# Only run the slow SQL/file operation if the cache is empty # Only run the slow SQL/file operation if the cache is empty
if not self._cache: if not self._cache:
# model_manipulator returns a dict { id: profile } # model_manipulator returns a dict { id: profile }
all_profiles_dict = model_manipulator.get_all_profiles() profile_result = model_manipulator.get_all_profiles()
if not profile_result.valid:
return profile_result
all_profiles_dict = profile_result.data
logger.info("Got all profiles.")
# Sync the cache with whatever is currently on disk # Sync the cache with whatever is currently on disk
self._cache.update(all_profiles_dict) self._cache.update(all_profiles_dict)
return self._cache return Result(valid=True, data=self._cache)

View file

@ -1,3 +1,4 @@
from core.models.manage.database_recovery import manage_initial_recovery, recommend_database_wipe
from core.services.config_tools.manage_sudo_configs import remove_sudo_config from core.services.config_tools.manage_sudo_configs import remove_sudo_config
from core.models.session.SessionProfile import SessionProfile from core.models.session.SessionProfile import SessionProfile
from core.models.orm_calls.location_calls import LocationCache from core.models.orm_calls.location_calls import LocationCache
@ -137,7 +138,8 @@ def make_subscription_from_config(config_contents: dict) -> Subscription:
def create_profile_object_from_disk(profile_id: int) -> Result: def create_profile_object_from_disk(profile_id: int) -> Result:
config_contents = read_profile_from_disk(profile_id) config_contents = read_profile_from_disk(profile_id)
logger.info(f"config_contents are {config_contents}") # logger.info(f"config_contents are {config_contents}")
if not config_contents: if not config_contents:
return Result(valid=False, error_type=ResultError.MISSING_DATA, message="Missing the config") return Result(valid=False, error_type=ResultError.MISSING_DATA, message="Missing the config")
@ -215,13 +217,15 @@ def create_profile(profile_id) -> Profile|None:
return result.data return result.data
def _get_all_profiles() -> Result:
def get_all_profiles() -> dict:
"""Find and Create all Profiles from what's available in the folders.""" """Find and Create all Profiles from what's available in the folders."""
all_profiles = {} all_profiles = {}
skipped_profiles = 0 skipped_profiles = 0
total_items = 0
for directory_entry in os.listdir(Constants.HV_PROFILE_CONFIG_HOME): for directory_entry in os.listdir(Constants.HV_PROFILE_CONFIG_HOME):
total_items = total_items + 1
try: try:
profile_id = int(directory_entry) profile_id = int(directory_entry)
except ValueError: except ValueError:
@ -241,9 +245,55 @@ def get_all_profiles() -> dict:
all_profiles[profile_id] = profile all_profiles[profile_id] = profile
full_final_dict = dict(sorted(all_profiles.items())) full_final_dict = dict(sorted(all_profiles.items()))
if skipped_profiles > 0:
logger.error(f"Skipped {skipped_profiles} profiles") # SUCCESS
return full_final_dict if skipped_profiles == 0:
return Result(valid=True, data=full_final_dict)
logger.error(f"Skipped {skipped_profiles} profiles")
# TOTAL FAILURE
if skipped_profiles == total_items:
load_error = f"All {total_items} profiles were skipped from loading!"
return Result(valid=False, data=None, error_type=ResultError.MISSING_DATA, message=load_error)
logger.error(f"Partial profile load failure.")
return Result(valid=False, data=full_final_dict)
def get_all_profiles() -> Result:
"""Attempt to get all profiles, if it fails, try migration or ask for a database wipe."""
logger.info("Getting all profiles")
initial_try = _get_all_profiles()
logger.info(f"Did it work? {initial_try.valid}")
# worked:
if initial_try.valid:
logger.info("It fully worked for all profiles. Returning data.")
return initial_try
# partial success, get the data:
if initial_try.data is not None:
logger.info("Some profiles failed, but others worked. Partial success, flipping it to Result True because we do not yet have other means to solve this.")
initial_try.valid = True
return initial_try
load_error = initial_try.message
logger.error(f"{load_error} This is likely a database issue. First try to migrate")
initial_recovery = manage_initial_recovery(string_of_problem=load_error)
if not initial_recovery.valid:
logger.error(initial_recovery.message)
return initial_recovery.wrap("Couldn't migrate")
logger.info("Initial steps, such as migration, worked, let's try again.")
attempt_profile_load_again = _get_all_profiles()
# worked after initial recovery:
if attempt_profile_load_again.valid:
return attempt_profile_load_again
return recommend_database_wipe(string_of_problem=load_error)
def is_actually_different(val1, val2): def is_actually_different(val1, val2):

View file

@ -265,7 +265,11 @@ def disable_profile(
# ============================================================================ # ============================================================================
if profile.is_system_profile(): if profile.is_system_profile():
subjects = ProfileController.get_all().values() profile_result = ProfileController.get_all()
if not profile_result.valid:
return profile_result.wrap("Unable to get all profiles do disconnect.")
subjects = profile_result.data.values()
for subject in subjects: for subject in subjects: