Support/sp-hydra-veil-core
Support/sp-hydra-veil-core
2
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Migrations Transition Stable for old versions of the app to update the schema

Browse source
This commit is contained in:
SimplifiedPrivacy 2026-07-16 15:35:03 -04:00
parent a085586448
commit 3977a5b695
8 changed files with 196 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/controllers/ClientController.py
View file

@ -18,8 +18,6 @@ from core.controllers.ApplicationController import ApplicationController
from core.controllers.ApplicationVersionController import ApplicationVersionController
from core.controllers.ClientVersionController import ClientVersionController
from core.controllers.ConfigurationController import ConfigurationController
# from core.controllers.LocationController import LocationController
# from core.controllers.OperatorController import OperatorController
from core.controllers.SubscriptionPlanController import SubscriptionPlanController
from core.observers.ClientObserver import ClientObserver
from core.observers.ConnectionObserver import ConnectionObserver

27
core/controllers/LocationController.py
View file

@ -1,23 +1,36 @@
from core.models.orm_models.Location import Location
from core.models.orm_models.Operator import Operator
from core.errors.logger import logger
from core.models.manage.session_management import get_session
from typing import Optional
from sqlalchemy import select
from sqlalchemy.orm import joinedload
from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
from core.models.orm_calls.location_calls import get_profile_location_data
class LocationController:
@staticmethod
def get(country_code: str, city_code: str):
with get_session() as session:
location_object = session.execute(
select(Location)
.where((Location.country_code == country_code) & (Location.code == city_code))
.options(joinedload(Location.operator))
).scalar_one_or_none()
location_object = execute_location_sql(country_code, city_code)
if location_object.valid:
return location_object.data
else:
critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}"
logger.error(critical_error)
print(critical_error)
return None
return location_object
# with get_session() as session:
# location_object = session.execute(
# select(Location)
# .where((Location.country_code == country_code) & (Location.code == city_code))
# .options(joinedload(Location.operator))
# ).scalar_one_or_none()
# return location_object
# legacy:
# Location.find(country_code, code)

45
core/models/BaseProfile.py
View file

@ -1,4 +1,6 @@
from core.models.manage.session_management import get_session
from core.errors.logger import logger
from sqlalchemy import select
from sqlalchemy.orm import joinedload
@ -6,7 +8,9 @@ from abc import ABC, abstractmethod
from core.Constants import Constants
from core.Helpers import write_atomically
# from core.models.Location import Location
from core.models.manage.wrapper import safe_db_operation, WrapperRollback
from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
from core.controllers.LocationController import LocationController
from core.models.orm_models.Location import Location
from core.models.orm_models.Operator import Operator
@ -23,6 +27,26 @@ import os
import re
import shutil
import tempfile
from sqlalchemy.orm import Session
@safe_db_operation
def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
location_object = session.execute(
select(Location)
.where((Location.country_code == country_code) & (Location.code == city_code))
.options(joinedload(Location.operator))
).scalar_one_or_none()
return location_object
def get_profile_location_data(country_code: str, city_code: str) -> Location:
# with get_session() as session:
location_object = execute_location_sql(country_code, city_code)
if location_object.valid:
return location_object.data
else:
logger.error(f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}")
return None
@dataclass_json
@ -186,28 +210,24 @@ class BaseProfile(ABC):
profiles_location = profile['location']
if profile['location'] is not None:
# =========== GET COUNTRY & LOCATION ===========
if isinstance(profiles_location, dict):
try:
country_code = profile['location']['country_code']
city_code = profile['location']['code']
except:
print("ERROR! CANT FIND country code or city")
logger.error(f"CRITICAL ERROR! Can't find country code or city for profile id {profile['id']} inside BaseProfile")
return
else:
# potentially coming from SQLAlchemy:
# potentially coming from SQLAlchemy ALREADY:
country_code = profile.location.country_code
city_code = profile.location.code
with get_session() as session:
location_object = session.execute(
select(Location)
.where((Location.country_code == country_code) & (Location.code == city_code))
.options(joinedload(Location.operator))
).scalar_one_or_none()
# =========== GET DATA USING THAT COUNTRY & LOCATION ===========
location_object = get_profile_location_data(country_code, city_code)
if location_object:
profile['location'] = location_object
if location_object:
profile['location'] = location_object
# this needs error handling if there's no location or malconformed config.
@ -241,6 +261,7 @@ class BaseProfile(ABC):
return profile
@staticmethod
def exists(id: int):
return re.match(r'^\d{1,2}$', str(id)) and os.path.isfile(f'{BaseProfile.__get_config_path(id)}/config.json')

5
core/models/DatabaseOperation.py
View file

@ -14,6 +14,11 @@ class DBErrorType(Enum):
MISSING_SQL = "missing_sql_file"
MISSING_DEPENDENCY = "missing_dependency"
INTEGRITY_ERROR = "integrity_error"
PERMISSION_ERROR = "permission_error"
FILESYSTEM_FULL = "filesystem_full"
DATABASE_LOCKED = "database_locked"
CORRUPTED_DATABASE = "corrupted_database"
PYTHON_MODEL_STRUCTURE = "python_model_structure"
UNKNOWN = "unknown"
@dataclass

154
core/models/manage/migrations.py
View file

@ -1,6 +1,15 @@
# custom
from core.errors.logger import logger
from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
from core.Constants import Constants
from core.models.orm_models.Location import Location
from core.models.orm_models.Operator import Operator
from core.models.orm_models.CachedSync import CachedSync
from core.models.orm_models.EncryptedProxy import EncryptedProxy
MODELS = [Location, Operator, CachedSync, EncryptedProxy]
# generic
from pathlib import Path
@ -8,6 +17,7 @@ from sqlalchemy import Column, Integer, String, create_engine, inspect, text
from sqlalchemy.orm import declarative_base, Session
from sqlalchemy.exc import OperationalError, DBAPIError
from sqlalchemy import inspect
from sqlalchemy import text
"""
Purpose:
@ -28,69 +38,103 @@ Why:
due to a large amount of boilerplate for migrations.
"""
def get_model_columns(model_class):
"""
Extract all columns directly from a model class.
Bypasses metadata, caching, and all SQLAlchemy indirection.
"""
try:
mapper = inspect(model_class)
columns = {}
for column in mapper.columns:
columns[column.name] = {
'type': str(column.type),
'nullable': column.nullable,
'default': column.default,
'primary_key': column.primary_key,
}
return columns
except Exception as e:
raise ValueError(f"Failed to inspect {model_class}: {e}")
def migrate_sql() -> DatabaseOperation:
from core.models.manage.session_management import engine
from core.models.orm_models.Base import Base, BaseModel
from core.models.orm_models.Base import BaseModel, Base
changes_made = []
try:
logger.info("[MIGRATION] Starting Migrations..")
inspector = inspect(engine)
existing_tables = inspector.get_table_names()
changes_made = []
for table in Base.metadata.tables.values():
table_name = table.name
if table_name not in existing_tables:
table.create(engine)
added_table = f"Created table: {table_name}"
changes_made.append(added_table)
logger.info(f"[MIGRATION] {added_table}")
continue
# Check for MISSING columns (add them)
existing_columns = {col['name'] for col in inspector.get_columns(table_name)}
model_columns = {col.name for col in table.columns}
missing_columns = model_columns - existing_columns
if missing_columns:
with engine.begin() as conn:
for col_name in missing_columns:
col = table.columns[col_name]
col_type = str(col.type.compile(engine.dialect))
alter_sql = f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type}"
conn.execute(text(alter_sql))
changes_made.append(f"Added columns to {table_name}: {missing_columns}")
# Check for EXTRA columns (drop them)
extra_columns = existing_columns - model_columns
if extra_columns:
with engine.begin() as conn:
for col_name in extra_columns:
alter_sql = f"ALTER TABLE {table_name} DROP COLUMN {col_name}"
logger.info(f"[MIGRATION] Dropping extra column: {alter_sql}")
conn.execute(text(alter_sql))
changes_made.append(f"Dropped extra columns from {table_name}: {extra_columns}")
if changes_made:
logger.info(f"[MIGRATION] Completed! Schema synced successfully")
# Step 1: Extract column definitions directly from each model
model_schema = {}
for model_class in MODELS:
table_name = model_class.__tablename__
try:
model_schema[table_name] = get_model_columns(model_class)
except ValueError as e:
return DatabaseOperation(
valid=True,
data=changes_made,
message=f"Schema synced: {'; '.join(changes_made)}"
valid=False,
error_type=DBErrorType.PYTHON_MODEL_STRUCTURE,
message=e
)
logger.info(f"[MIGRATION] Model '{table_name}': {list(model_schema[table_name].keys())}")
# Step 2: Compare to database schema
inspector = inspect(engine)
db_tables = set(inspector.get_table_names())
for table_name, expected_columns in model_schema.items():
if table_name not in db_tables:
changes_made.append(table_name)
logger.info(f"[MIGRATION] Table '{table_name}' is MISSING from database")
else:
db_columns = {col['name']: col for col in inspector.get_columns(table_name)}
for col_name, col_info in expected_columns.items():
if col_name not in db_columns:
changes_made.append(col_name)
logger.info(f"[MIGRATION] Column '{col_name}' is MISSING from table '{table_name}'")
# Step 3: Perform the actual migration
if changes_made:
logger.info("[MIGRATION] Changes detected. Running schema update...")
Base.metadata.create_all(engine)
try:
with engine.begin() as conn:
for table_name, expected_columns in model_schema.items():
inspector = inspect(engine)
if table_name in inspector.get_table_names():
db_columns = {col['name'] for col in inspector.get_columns(table_name)}
for col_name, col_info in expected_columns.items():
if col_name not in db_columns:
col_type = col_info['type']
nullable = "NULL" if col_info['nullable'] else "NOT NULL"
print(f"[MIGRATION] Adding column '{col_name}' to '{table_name}'")
conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_type} {nullable}"))
except (OperationalError, DBAPIError) as e:
logger.error(f"[MIGRATION] Failed: {str(e)}")
return DatabaseOperation(
valid=True,
message="Schema already matches models"
valid=False,
error_type=DBErrorType.UNKNOWN,
message=f"Schema sync failed: {str(e)}"
)
except (OperationalError, DBAPIError) as e:
logger.error(f"[MIGRATION] Failed: {str(e)}")
logger.info(f"[MIGRATION] Completed! Schema synced successfully with these changes: {changes_made}")
return DatabaseOperation(
valid=False,
error_type=DBErrorType.UNKNOWN,
message=f"Schema sync failed: {str(e)}"
valid=True,
data=changes_made,
message=f"Synced these tables and columns: {'; '.join(changes_made)}"
)
else:
no_migration_needed = "Schema already matches models. No migration needed."
logger.info(f"[MIGRATION] {no_migration_needed}")
return DatabaseOperation(
valid=True,
message=no_migration_needed
)

1
core/models/manage/session_management.py
View file

@ -1,6 +1,7 @@
from core.errors.logger import logger
from core.models.orm_models.Base import BaseModel
from core.models.manage.version_check import insert_new_version
from core.Constants import Constants
from sqlalchemy import create_engine, inspect

1
core/models/manage/wrapper.py
View file

@ -3,6 +3,7 @@ from core.errors.logger import logger
from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
from core.models.manage.migrations import migrate_sql
from core.models.manage.session_management import get_session
from core.Constants import Constants
# generic
from sqlalchemy.orm import Session

31
core/models/orm_calls/location_calls.py Normal file
View file

@ -0,0 +1,31 @@
from core.models.orm_models.Location import Location
from core.models.orm_models.Operator import Operator
from core.models.manage.wrapper import safe_db_operation, WrapperRollback
from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
from core.errors.logger import logger
# generic
from sqlalchemy import select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import Session
@safe_db_operation
def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
location_object = session.execute(
select(Location)
.where((Location.country_code == country_code) & (Location.code == city_code))
.options(joinedload(Location.operator))
).scalar_one_or_none()
return location_object
def get_profile_location_data(country_code: str, city_code: str) -> Location:
location_object = execute_location_sql(country_code, city_code)
if location_object.valid:
return location_object.data
else:
critical_error = f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}"
logger.error(critical_error)
print(critical_error)
return None