From 3977a5b69591bc4486444ea051a7a1c24f904c3f Mon Sep 17 00:00:00 2001 From: SimplifiedPrivacy Date: Thu, 16 Jul 2026 15:35:03 -0400 Subject: [PATCH] Migrations Transition Stable for old versions of the app to update the schema --- core/controllers/ClientController.py | 2 - core/controllers/LocationController.py | 27 +++- core/models/BaseProfile.py | 45 ++++-- core/models/DatabaseOperation.py | 5 + core/models/manage/migrations.py | 166 ++++++++++++++--------- core/models/manage/session_management.py | 1 + core/models/manage/wrapper.py | 1 + core/models/orm_calls/location_calls.py | 31 +++++ 8 files changed, 196 insertions(+), 82 deletions(-) create mode 100644 core/models/orm_calls/location_calls.py diff --git a/core/controllers/ClientController.py b/core/controllers/ClientController.py index a0128e7..1a0ee25 100644 --- a/core/controllers/ClientController.py +++ b/core/controllers/ClientController.py @@ -18,8 +18,6 @@ from core.controllers.ApplicationController import ApplicationController from core.controllers.ApplicationVersionController import ApplicationVersionController from core.controllers.ClientVersionController import ClientVersionController from core.controllers.ConfigurationController import ConfigurationController -# from core.controllers.LocationController import LocationController -# from core.controllers.OperatorController import OperatorController from core.controllers.SubscriptionPlanController import SubscriptionPlanController from core.observers.ClientObserver import ClientObserver from core.observers.ConnectionObserver import ConnectionObserver diff --git a/core/controllers/LocationController.py b/core/controllers/LocationController.py index 627d1b7..e18d0db 100644 --- a/core/controllers/LocationController.py +++ b/core/controllers/LocationController.py @@ -1,23 +1,36 @@ from core.models.orm_models.Location import Location from core.models.orm_models.Operator import Operator +from core.errors.logger import logger from core.models.manage.session_management import get_session from typing import Optional from sqlalchemy import select from sqlalchemy.orm import joinedload +from core.models.DatabaseOperation import DatabaseOperation, DBErrorType +from core.models.orm_calls.location_calls import get_profile_location_data class LocationController: @staticmethod def get(country_code: str, city_code: str): - with get_session() as session: - location_object = session.execute( - select(Location) - .where((Location.country_code == country_code) & (Location.code == city_code)) - .options(joinedload(Location.operator)) - ).scalar_one_or_none() + location_object = execute_location_sql(country_code, city_code) + if location_object.valid: + return location_object.data + else: + critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}" + logger.error(critical_error) + print(critical_error) + return None - return location_object + + # with get_session() as session: + # location_object = session.execute( + # select(Location) + # .where((Location.country_code == country_code) & (Location.code == city_code)) + # .options(joinedload(Location.operator)) + # ).scalar_one_or_none() + + # return location_object # legacy: # Location.find(country_code, code) diff --git a/core/models/BaseProfile.py b/core/models/BaseProfile.py index 3c861af..50dbe89 100644 --- a/core/models/BaseProfile.py +++ b/core/models/BaseProfile.py @@ -1,4 +1,6 @@ from core.models.manage.session_management import get_session +from core.errors.logger import logger + from sqlalchemy import select from sqlalchemy.orm import joinedload @@ -6,7 +8,9 @@ from abc import ABC, abstractmethod from core.Constants import Constants from core.Helpers import write_atomically -# from core.models.Location import Location +from core.models.manage.wrapper import safe_db_operation, WrapperRollback +from core.models.DatabaseOperation import DatabaseOperation, DBErrorType + from core.controllers.LocationController import LocationController from core.models.orm_models.Location import Location from core.models.orm_models.Operator import Operator @@ -23,6 +27,26 @@ import os import re import shutil import tempfile +from sqlalchemy.orm import Session + +@safe_db_operation +def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation: + location_object = session.execute( + select(Location) + .where((Location.country_code == country_code) & (Location.code == city_code)) + .options(joinedload(Location.operator)) + ).scalar_one_or_none() + return location_object + + +def get_profile_location_data(country_code: str, city_code: str) -> Location: + # with get_session() as session: + location_object = execute_location_sql(country_code, city_code) + if location_object.valid: + return location_object.data + else: + logger.error(f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}") + return None @dataclass_json @@ -186,28 +210,24 @@ class BaseProfile(ABC): profiles_location = profile['location'] if profile['location'] is not None: + # =========== GET COUNTRY & LOCATION =========== if isinstance(profiles_location, dict): try: country_code = profile['location']['country_code'] city_code = profile['location']['code'] except: - print("ERROR! CANT FIND country code or city") + logger.error(f"CRITICAL ERROR! Can't find country code or city for profile id {profile['id']} inside BaseProfile") return else: - # potentially coming from SQLAlchemy: + # potentially coming from SQLAlchemy ALREADY: country_code = profile.location.country_code city_code = profile.location.code - with get_session() as session: - location_object = session.execute( - select(Location) - .where((Location.country_code == country_code) & (Location.code == city_code)) - .options(joinedload(Location.operator)) - ).scalar_one_or_none() + # =========== GET DATA USING THAT COUNTRY & LOCATION =========== + location_object = get_profile_location_data(country_code, city_code) - - if location_object: - profile['location'] = location_object + if location_object: + profile['location'] = location_object # this needs error handling if there's no location or malconformed config. @@ -241,6 +261,7 @@ class BaseProfile(ABC): return profile + @staticmethod def exists(id: int): return re.match(r'^\d{1,2}$', str(id)) and os.path.isfile(f'{BaseProfile.__get_config_path(id)}/config.json') diff --git a/core/models/DatabaseOperation.py b/core/models/DatabaseOperation.py index 65bb033..4fa368f 100644 --- a/core/models/DatabaseOperation.py +++ b/core/models/DatabaseOperation.py @@ -14,6 +14,11 @@ class DBErrorType(Enum): MISSING_SQL = "missing_sql_file" MISSING_DEPENDENCY = "missing_dependency" INTEGRITY_ERROR = "integrity_error" + PERMISSION_ERROR = "permission_error" + FILESYSTEM_FULL = "filesystem_full" + DATABASE_LOCKED = "database_locked" + CORRUPTED_DATABASE = "corrupted_database" + PYTHON_MODEL_STRUCTURE = "python_model_structure" UNKNOWN = "unknown" @dataclass diff --git a/core/models/manage/migrations.py b/core/models/manage/migrations.py index 76e0a41..1859a34 100644 --- a/core/models/manage/migrations.py +++ b/core/models/manage/migrations.py @@ -1,6 +1,15 @@ # custom from core.errors.logger import logger from core.models.DatabaseOperation import DatabaseOperation, DBErrorType +from core.Constants import Constants + + +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator +from core.models.orm_models.CachedSync import CachedSync +from core.models.orm_models.EncryptedProxy import EncryptedProxy + +MODELS = [Location, Operator, CachedSync, EncryptedProxy] # generic from pathlib import Path @@ -8,6 +17,7 @@ from sqlalchemy import Column, Integer, String, create_engine, inspect, text from sqlalchemy.orm import declarative_base, Session from sqlalchemy.exc import OperationalError, DBAPIError from sqlalchemy import inspect +from sqlalchemy import text """ Purpose: @@ -28,69 +38,103 @@ Why: due to a large amount of boilerplate for migrations. """ +def get_model_columns(model_class): + """ + Extract all columns directly from a model class. + Bypasses metadata, caching, and all SQLAlchemy indirection. + """ + try: + mapper = inspect(model_class) + columns = {} + + for column in mapper.columns: + columns[column.name] = { + 'type': str(column.type), + 'nullable': column.nullable, + 'default': column.default, + 'primary_key': column.primary_key, + } + + return columns + except Exception as e: + raise ValueError(f"Failed to inspect {model_class}: {e}") + + def migrate_sql() -> DatabaseOperation: from core.models.manage.session_management import engine - from core.models.orm_models.Base import Base, BaseModel - - try: - logger.info("[MIGRATION] Starting Migrations..") - inspector = inspect(engine) - existing_tables = inspector.get_table_names() - changes_made = [] - - for table in Base.metadata.tables.values(): - table_name = table.name - - if table_name not in existing_tables: - table.create(engine) - added_table = f"Created table: {table_name}" - changes_made.append(added_table) - logger.info(f"[MIGRATION] {added_table}") - continue - - # Check for MISSING columns (add them) - existing_columns = {col['name'] for col in inspector.get_columns(table_name)} - model_columns = {col.name for col in table.columns} - missing_columns = model_columns - existing_columns - - if missing_columns: - with engine.begin() as conn: - for col_name in missing_columns: - col = table.columns[col_name] - col_type = str(col.type.compile(engine.dialect)) - alter_sql = f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type}" - conn.execute(text(alter_sql)) - - changes_made.append(f"Added columns to {table_name}: {missing_columns}") - - # Check for EXTRA columns (drop them) - extra_columns = existing_columns - model_columns - if extra_columns: - with engine.begin() as conn: - for col_name in extra_columns: - alter_sql = f"ALTER TABLE {table_name} DROP COLUMN {col_name}" - logger.info(f"[MIGRATION] Dropping extra column: {alter_sql}") - conn.execute(text(alter_sql)) - - changes_made.append(f"Dropped extra columns from {table_name}: {extra_columns}") - - if changes_made: - logger.info(f"[MIGRATION] Completed! Schema synced successfully") - return DatabaseOperation( - valid=True, - data=changes_made, - message=f"Schema synced: {'; '.join(changes_made)}" - ) - else: - return DatabaseOperation( - valid=True, - message="Schema already matches models" - ) + from core.models.orm_models.Base import BaseModel, Base + changes_made = [] - except (OperationalError, DBAPIError) as e: - logger.error(f"[MIGRATION] Failed: {str(e)}") + # Step 1: Extract column definitions directly from each model + model_schema = {} + for model_class in MODELS: + table_name = model_class.__tablename__ + try: + model_schema[table_name] = get_model_columns(model_class) + except ValueError as e: + return DatabaseOperation( + valid=False, + error_type=DBErrorType.PYTHON_MODEL_STRUCTURE, + message=e + ) + logger.info(f"[MIGRATION] Model '{table_name}': {list(model_schema[table_name].keys())}") + + # Step 2: Compare to database schema + inspector = inspect(engine) + db_tables = set(inspector.get_table_names()) + + for table_name, expected_columns in model_schema.items(): + if table_name not in db_tables: + changes_made.append(table_name) + logger.info(f"[MIGRATION] Table '{table_name}' is MISSING from database") + else: + db_columns = {col['name']: col for col in inspector.get_columns(table_name)} + + for col_name, col_info in expected_columns.items(): + if col_name not in db_columns: + changes_made.append(col_name) + logger.info(f"[MIGRATION] Column '{col_name}' is MISSING from table '{table_name}'") + + # Step 3: Perform the actual migration + if changes_made: + logger.info("[MIGRATION] Changes detected. Running schema update...") + Base.metadata.create_all(engine) + + try: + with engine.begin() as conn: + for table_name, expected_columns in model_schema.items(): + inspector = inspect(engine) + + if table_name in inspector.get_table_names(): + db_columns = {col['name'] for col in inspector.get_columns(table_name)} + + for col_name, col_info in expected_columns.items(): + if col_name not in db_columns: + col_type = col_info['type'] + nullable = "NULL" if col_info['nullable'] else "NOT NULL" + print(f"[MIGRATION] Adding column '{col_name}' to '{table_name}'") + conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {nullable}")) + + except (OperationalError, DBAPIError) as e: + logger.error(f"[MIGRATION] Failed: {str(e)}") + return DatabaseOperation( + valid=False, + error_type=DBErrorType.UNKNOWN, + message=f"Schema sync failed: {str(e)}" + ) + + logger.info(f"[MIGRATION] Completed! Schema synced successfully with these changes: {changes_made}") return DatabaseOperation( - valid=False, - error_type=DBErrorType.UNKNOWN, - message=f"Schema sync failed: {str(e)}" + valid=True, + data=changes_made, + message=f"Synced these tables and columns: {'; '.join(changes_made)}" ) + + else: + no_migration_needed = "Schema already matches models. No migration needed." + logger.info(f"[MIGRATION] {no_migration_needed}") + return DatabaseOperation( + valid=True, + message=no_migration_needed + ) + diff --git a/core/models/manage/session_management.py b/core/models/manage/session_management.py index f0c111a..286b126 100644 --- a/core/models/manage/session_management.py +++ b/core/models/manage/session_management.py @@ -1,6 +1,7 @@ from core.errors.logger import logger from core.models.orm_models.Base import BaseModel from core.models.manage.version_check import insert_new_version +from core.Constants import Constants from sqlalchemy import create_engine, inspect diff --git a/core/models/manage/wrapper.py b/core/models/manage/wrapper.py index 5bb00fd..0b9611a 100644 --- a/core/models/manage/wrapper.py +++ b/core/models/manage/wrapper.py @@ -3,6 +3,7 @@ from core.errors.logger import logger from core.models.DatabaseOperation import DatabaseOperation, DBErrorType from core.models.manage.migrations import migrate_sql from core.models.manage.session_management import get_session +from core.Constants import Constants # generic from sqlalchemy.orm import Session diff --git a/core/models/orm_calls/location_calls.py b/core/models/orm_calls/location_calls.py new file mode 100644 index 0000000..3db2dea --- /dev/null +++ b/core/models/orm_calls/location_calls.py @@ -0,0 +1,31 @@ +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator +from core.models.manage.wrapper import safe_db_operation, WrapperRollback +from core.models.DatabaseOperation import DatabaseOperation, DBErrorType +from core.errors.logger import logger + +# generic +from sqlalchemy import select +from sqlalchemy.orm import joinedload +from sqlalchemy.orm import Session + +@safe_db_operation +def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation: + location_object = session.execute( + select(Location) + .where((Location.country_code == country_code) & (Location.code == city_code)) + .options(joinedload(Location.operator)) + ).scalar_one_or_none() + return location_object + + +def get_profile_location_data(country_code: str, city_code: str) -> Location: + location_object = execute_location_sql(country_code, city_code) + if location_object.valid: + return location_object.data + else: + critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}" + logger.error(critical_error) + print(critical_error) + return None +