Migrations Transition Stable for old versions of the app to update the schema

core/controllers/ClientController.py

@@ -18,8 +18,6 @@ from core.controllers.ApplicationController import ApplicationController
 from core.controllers.ApplicationVersionController import ApplicationVersionController
 from core.controllers.ClientVersionController import ClientVersionController
 from core.controllers.ConfigurationController import ConfigurationController
-# from core.controllers.LocationController import LocationController
-# from core.controllers.OperatorController import OperatorController
 from core.controllers.SubscriptionPlanController import SubscriptionPlanController
 from core.observers.ClientObserver import ClientObserver
 from core.observers.ConnectionObserver import ConnectionObserver

core/controllers/LocationController.py

@@ -1,23 +1,36 @@
 from core.models.orm_models.Location import Location
 from core.models.orm_models.Operator import Operator
+from core.errors.logger import logger
 
 from core.models.manage.session_management import get_session
 from typing import Optional
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.orm_calls.location_calls import get_profile_location_data
 
 class LocationController:
 
     @staticmethod
     def get(country_code: str, city_code: str):
-        with get_session() as session:
+        location_object = execute_location_sql(country_code, city_code)
-            location_object = session.execute(
+        if location_object.valid:
-                select(Location)
+            return location_object.data
-                .where((Location.country_code == country_code) & (Location.code == city_code))
+        else:
-                .options(joinedload(Location.operator))
+            critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}"
-            ).scalar_one_or_none()
+            logger.error(critical_error)
+            print(critical_error)
+            return None
 
-        return location_object
+
+        # with get_session() as session:
+        #     location_object = session.execute(
+        #         select(Location)
+        #         .where((Location.country_code == country_code) & (Location.code == city_code))
+        #         .options(joinedload(Location.operator))
+        #     ).scalar_one_or_none()
+
+        # return location_object
 
     # legacy:
     # Location.find(country_code, code)

core/models/BaseProfile.py

@@ -1,4 +1,6 @@
 from core.models.manage.session_management import get_session
+from core.errors.logger import logger
+
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 
@@ -6,7 +8,9 @@ from abc import ABC, abstractmethod
 from core.Constants import Constants
 from core.Helpers import write_atomically
 
-# from core.models.Location import Location
+from core.models.manage.wrapper import safe_db_operation, WrapperRollback
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+
 from core.controllers.LocationController import LocationController
 from core.models.orm_models.Location import Location
 from core.models.orm_models.Operator import Operator
@@ -23,6 +27,26 @@ import os
 import re
 import shutil
 import tempfile
+from sqlalchemy.orm import Session
+
+@safe_db_operation
+def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
+    location_object = session.execute(
+        select(Location)
+        .where((Location.country_code == country_code) & (Location.code == city_code))
+        .options(joinedload(Location.operator))
+    ).scalar_one_or_none()
+    return location_object
+
+
+def get_profile_location_data(country_code: str, city_code: str) -> Location:
+    # with get_session() as session:
+    location_object = execute_location_sql(country_code, city_code)
+    if location_object.valid:
+        return location_object.data
+    else:
+        logger.error(f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}")
+        return None
 
 
 @dataclass_json
@@ -186,28 +210,24 @@ class BaseProfile(ABC):
         profiles_location = profile['location']
 
         if profile['location'] is not None:
+            # =========== GET COUNTRY & LOCATION ===========
             if isinstance(profiles_location, dict):
                 try:
                     country_code = profile['location']['country_code']
                     city_code = profile['location']['code']
                 except:
-                    print("ERROR! CANT FIND country code or city")
+                    logger.error(f"CRITICAL ERROR! Can't find country code or city for profile id {profile['id']} inside BaseProfile")
                     return
             else:
-                # potentially coming from SQLAlchemy:
+                # potentially coming from SQLAlchemy ALREADY:
                 country_code = profile.location.country_code
                 city_code = profile.location.code
 
-            with get_session() as session:
+            # =========== GET DATA USING THAT COUNTRY & LOCATION ===========
-                location_object = session.execute(
+            location_object = get_profile_location_data(country_code, city_code)
-                    select(Location)
-                    .where((Location.country_code == country_code) & (Location.code == city_code))
-                    .options(joinedload(Location.operator))
-                ).scalar_one_or_none()
 
-
+            if location_object:
-                if location_object:
+                profile['location'] = location_object
-                    profile['location'] = location_object
 
             # this needs error handling if there's no location or malconformed config.
 
@@ -241,6 +261,7 @@ class BaseProfile(ABC):
 
         return profile
 
+
     @staticmethod
     def exists(id: int):
         return re.match(r'^\d{1,2}$', str(id)) and os.path.isfile(f'{BaseProfile.__get_config_path(id)}/config.json')

core/models/DatabaseOperation.py

@@ -14,6 +14,11 @@ class DBErrorType(Enum):
     MISSING_SQL = "missing_sql_file"
     MISSING_DEPENDENCY = "missing_dependency"
     INTEGRITY_ERROR = "integrity_error"
+    PERMISSION_ERROR = "permission_error"
+    FILESYSTEM_FULL = "filesystem_full"
+    DATABASE_LOCKED = "database_locked"
+    CORRUPTED_DATABASE = "corrupted_database"
+    PYTHON_MODEL_STRUCTURE = "python_model_structure"
     UNKNOWN = "unknown"
 
 @dataclass

core/models/manage/migrations.py

@@ -1,6 +1,15 @@
 # custom
 from core.errors.logger import logger
 from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.Constants import Constants
+
+
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.models.orm_models.CachedSync import CachedSync
+from core.models.orm_models.EncryptedProxy import EncryptedProxy
+
+MODELS = [Location, Operator, CachedSync, EncryptedProxy]
 
 # generic
 from pathlib import Path
@@ -8,6 +17,7 @@ from sqlalchemy import Column, Integer, String, create_engine, inspect, text
 from sqlalchemy.orm import declarative_base, Session
 from sqlalchemy.exc import OperationalError, DBAPIError
 from sqlalchemy import inspect
+from sqlalchemy import text
 
 """
 Purpose:
@@ -28,69 +38,103 @@ Why:
     due to a large amount of boilerplate for migrations.
 """
 
+def get_model_columns(model_class):
+    """
+    Extract all columns directly from a model class.
+    Bypasses metadata, caching, and all SQLAlchemy indirection.
+    """
+    try:
+        mapper = inspect(model_class)
+        columns = {}
+        
+        for column in mapper.columns:
+            columns[column.name] = {
+                'type': str(column.type),
+                'nullable': column.nullable,
+                'default': column.default,
+                'primary_key': column.primary_key,
+            }
+        
+        return columns
+    except Exception as e:
+        raise ValueError(f"Failed to inspect {model_class}: {e}")
+
+
 def migrate_sql() -> DatabaseOperation:
     from core.models.manage.session_management import engine
-    from core.models.orm_models.Base import Base, BaseModel
+    from core.models.orm_models.Base import BaseModel, Base
-
+    changes_made = []
-    try:
-        logger.info("[MIGRATION] Starting Migrations..")
-        inspector = inspect(engine)
-        existing_tables = inspector.get_table_names()
-        changes_made = []
-        
-        for table in Base.metadata.tables.values():
-            table_name = table.name
-
-            if table_name not in existing_tables:
-                table.create(engine)
-                added_table = f"Created table: {table_name}"
-                changes_made.append(added_table)
-                logger.info(f"[MIGRATION] {added_table}")
-                continue
-            
-            # Check for MISSING columns (add them)
-            existing_columns = {col['name'] for col in inspector.get_columns(table_name)}
-            model_columns = {col.name for col in table.columns}
-            missing_columns = model_columns - existing_columns
-            
-            if missing_columns:
-                with engine.begin() as conn:
-                    for col_name in missing_columns:
-                        col = table.columns[col_name]
-                        col_type = str(col.type.compile(engine.dialect))
-                        alter_sql = f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type}"
-                        conn.execute(text(alter_sql))
-                
-                changes_made.append(f"Added columns to {table_name}: {missing_columns}")
-            
-            # Check for EXTRA columns (drop them)
-            extra_columns = existing_columns - model_columns
-            if extra_columns:
-                with engine.begin() as conn:
-                    for col_name in extra_columns:
-                        alter_sql = f"ALTER TABLE {table_name} DROP COLUMN {col_name}"
-                        logger.info(f"[MIGRATION] Dropping extra column: {alter_sql}")
-                        conn.execute(text(alter_sql))
-                
-                changes_made.append(f"Dropped extra columns from {table_name}: {extra_columns}")
-        
-        if changes_made:
-            logger.info(f"[MIGRATION] Completed! Schema synced successfully")
-            return DatabaseOperation(
-                valid=True,
-                data=changes_made,
-                message=f"Schema synced: {'; '.join(changes_made)}"
-            )
-        else:
-            return DatabaseOperation(
-                valid=True,
-                message="Schema already matches models"
-            )
     
-    except (OperationalError, DBAPIError) as e:
+    # Step 1: Extract column definitions directly from each model
-        logger.error(f"[MIGRATION] Failed: {str(e)}")
+    model_schema = {}
+    for model_class in MODELS:
+        table_name = model_class.__tablename__
+        try:
+            model_schema[table_name] = get_model_columns(model_class)
+        except ValueError as e:
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.PYTHON_MODEL_STRUCTURE,
+                message=e
+            )
+        logger.info(f"[MIGRATION] Model '{table_name}': {list(model_schema[table_name].keys())}")
+    
+    # Step 2: Compare to database schema
+    inspector = inspect(engine)
+    db_tables = set(inspector.get_table_names())
+        
+    for table_name, expected_columns in model_schema.items():
+        if table_name not in db_tables:
+            changes_made.append(table_name)
+            logger.info(f"[MIGRATION] Table '{table_name}' is MISSING from database")
+        else:
+            db_columns = {col['name']: col for col in inspector.get_columns(table_name)}
+            
+            for col_name, col_info in expected_columns.items():
+                if col_name not in db_columns:
+                    changes_made.append(col_name)
+                    logger.info(f"[MIGRATION] Column '{col_name}' is MISSING from table '{table_name}'")
+    
+    # Step 3: Perform the actual migration
+    if changes_made:
+        logger.info("[MIGRATION] Changes detected. Running schema update...")
+        Base.metadata.create_all(engine)
+        
+        try:
+            with engine.begin() as conn:
+                for table_name, expected_columns in model_schema.items():
+                    inspector = inspect(engine)
+                    
+                    if table_name in inspector.get_table_names():
+                        db_columns = {col['name'] for col in inspector.get_columns(table_name)}
+                        
+                        for col_name, col_info in expected_columns.items():
+                            if col_name not in db_columns:
+                                col_type = col_info['type']
+                                nullable = "NULL" if col_info['nullable'] else "NOT NULL"
+                                print(f"[MIGRATION] Adding column '{col_name}' to '{table_name}'")
+                                conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {nullable}"))
+
+        except (OperationalError, DBAPIError) as e:
+            logger.error(f"[MIGRATION] Failed: {str(e)}")
+            return DatabaseOperation(
+                valid=False,
+                error_type=DBErrorType.UNKNOWN,
+                message=f"Schema sync failed: {str(e)}"
+            )
+
+        logger.info(f"[MIGRATION] Completed! Schema synced successfully with these changes: {changes_made}")
         return DatabaseOperation(
-            valid=False,
+            valid=True,
-            error_type=DBErrorType.UNKNOWN,
+            data=changes_made,
-            message=f"Schema sync failed: {str(e)}"
+            message=f"Synced these tables and columns: {'; '.join(changes_made)}"
         )
+
+    else:
+        no_migration_needed = "Schema already matches models.  No migration needed."
+        logger.info(f"[MIGRATION] {no_migration_needed}")
+        return DatabaseOperation(
+            valid=True,
+            message=no_migration_needed
+        )
+

core/models/manage/session_management.py

@@ -1,6 +1,7 @@
 from core.errors.logger import logger
 from core.models.orm_models.Base import BaseModel
 from core.models.manage.version_check import insert_new_version
+from core.Constants import Constants
 
 
 from sqlalchemy import create_engine, inspect

core/models/manage/wrapper.py

@@ -3,6 +3,7 @@ from core.errors.logger import logger
 from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
 from core.models.manage.migrations import migrate_sql
 from core.models.manage.session_management import get_session
+from core.Constants import Constants
 
 # generic
 from sqlalchemy.orm import Session

core/models/orm_calls/location_calls.py

@@ -0,0 +1,31 @@
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.models.manage.wrapper import safe_db_operation, WrapperRollback
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.errors.logger import logger
+
+# generic
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import Session
+
+@safe_db_operation
+def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
+    location_object = session.execute(
+        select(Location)
+        .where((Location.country_code == country_code) & (Location.code == city_code))
+        .options(joinedload(Location.operator))
+    ).scalar_one_or_none()
+    return location_object
+
+
+def get_profile_location_data(country_code: str, city_code: str) -> Location:
+    location_object = execute_location_sql(country_code, city_code)
+    if location_object.valid:
+        return location_object.data
+    else:
+        critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}"
+        logger.error(critical_error)
+        print(critical_error)
+        return None
+