From 8d7d038bae5dbca746f0e63011859412ae4f7b9b Mon Sep 17 00:00:00 2001 From: SimplifiedPrivacy Date: Tue, 17 Nov 2026 21:34:54 -0500 Subject: [PATCH] Introduced Full SQL State Refresh post-sync if the SQL model tables updated. This means if sync updates location or app tables, the entire profile repo and all gui calls upon it, get a refresh and recreate --- core/controllers/ProfileController.py | 53 +++++++++- core/controllers/SyncController.py | 11 ++- core/infrastructure/plumbing.py | 2 - .../global_state_controller.py | 24 ++++- core/models/BaseProfile.py | 7 +- core/models/manage/lifecycle_tools.py | 97 ++++++++++++++++++- core/models/orm_calls/location_calls.py | 7 +- core/models/orm_models/Base.py | 19 ++++ core/services/profiles/ProfileRepository.py | 9 +- core/services/profiles/model_manipulator.py | 7 +- 10 files changed, 217 insertions(+), 19 deletions(-) diff --git a/core/controllers/ProfileController.py b/core/controllers/ProfileController.py index 844e255..a238774 100644 --- a/core/controllers/ProfileController.py +++ b/core/controllers/ProfileController.py @@ -1,3 +1,4 @@ +from core.models.manage.lifecycle_tools import serialize_dataclass_with_models from core.services.profiles import model_manipulator from core.services.profiles.create_objects import create_new_profile_object from core.services.profiles.ProfileRepository import ProfileRepository @@ -22,6 +23,7 @@ import base64 import re import time +PROFILE_TABLES_WITH_ORM = ["locations", "operators", "application_versions"] class ProfileController: # Initialize the singleton repository @@ -36,6 +38,56 @@ class ProfileController: def get_all() -> Result: return ProfileController._repo.get_all() + @staticmethod + def get_all_serialized() -> Result: + got_all = ProfileController._repo.get_all() + + if not got_all.valid: + return got_all.wrap("failed to get from the repo") + + initial_results = got_all.data + converted_results = {} + logger.info(f"initial_results type {type(initial_results)}") + + for key, each_result in initial_results.items(): + if not each_result: + logger.info(f"Skipping profile {key}") + try: + converted_dict = serialize_dataclass_with_models(each_result) + converted_results[key] = converted_dict + except Exception as e: + error_msg = str(e) + logger.error(error_msg) + + logger.info("finally..") + return Result(valid=True, data=converted_results) + + + @staticmethod + def get_ONE_serialized(profile_id: int) -> Result: + logger.info("Fetching from the repo") + got_one = ProfileController._repo.get(profile_id) + + logger.info(f"Got data, converting it. of type {type(got_one)}") + converted_dict = serialize_dataclass_with_models(got_one) + logger.info(f"converted_dict is {converted_dict}") + + return Result(valid=True, data=converted_dict) + + @staticmethod + def clear_cache_and_get_fresh_data() -> Result: + """Clear the Repo's Cache and gets new fresh data.""" + logger.info("Clearing Repo cache") + result = ProfileController._repo.refresh_cache() + + # update ui: + if result.valid: + logger.info("Updating notifications of change") + Notifications.emit(EventType.PROFILES_CHANGED) + + return result + + @staticmethod def _save_and_sync(profile, event_type: str, observer: ProfileObserver = None): """Internal helper to handle the actual persistence and cache synchronization.""" @@ -86,7 +138,6 @@ class ProfileController: # update ui: Notifications.emit(EventType.PROFILES_CHANGED) - @staticmethod def delete(profile: Profile, profile_observer: ProfileObserver = None): """Remove the profile from both disk and cache""" diff --git a/core/controllers/SyncController.py b/core/controllers/SyncController.py index ba8e295..ee6571c 100644 --- a/core/controllers/SyncController.py +++ b/core/controllers/SyncController.py @@ -33,6 +33,7 @@ from core.models.orm_models.ApplicationVersion import ApplicationVersion # generic import sys import json +from typing import Optional ORM_TABLES = { "locations": Location, @@ -74,11 +75,11 @@ def get_orm_model(key) -> bool: return ORM_TABLES[key] -def _sync_success(): +def _sync_success(which_models_sync: Optional[dict]): notification = "Finshed sync." Notifications.emit(EventType.STATUS_UPDATE, notification) Notifications.emit(EventType.SYNC_COMPLETE, notification) - global_state_marks_sync_completed() + global_state_marks_sync_completed(which_models_sync) return Result(valid=True, message=notification) @@ -123,7 +124,7 @@ def new_sync(client_observer: ClientObserver, connection_observer: ConnectionObs notification = "Sync Complete. (already up to date)" Notifications.emit(EventType.STATUS_UPDATE, notification) Notifications.emit(EventType.SYNC_COMPLETE, notification) - global_state_marks_sync_completed() + global_state_marks_sync_completed(which_models_sync=None) return Result(valid=True) # We only make it past this point if there's New Data @@ -198,12 +199,12 @@ def new_sync(client_observer: ClientObserver, connection_observer: ConnectionObs # forced_sync being true, forces this to none: if filtered_metadata is None: - return _sync_success() + return _sync_success(which_models_sync=all_results) # otherwise evaluate saving of metadata: save_successful = save_metadata(filtered_metadata) if save_successful: - return _sync_success() + return _sync_success(which_models_sync=all_results) else: error_msg = "Finshed sync, but had issues with the saving of metadata for next time." logger.error(error_msg) diff --git a/core/infrastructure/plumbing.py b/core/infrastructure/plumbing.py index e4566e9..f58d0b3 100644 --- a/core/infrastructure/plumbing.py +++ b/core/infrastructure/plumbing.py @@ -123,8 +123,6 @@ class ThreadPlumbing: def _back_on_main_thread(self, result: Result): """Decide: retry, show popup, or final callback""" - logger.info(result) - #---------------------------- # OUTRIGHT ERROR STACK #---------------------------- diff --git a/core/infrastructure/state_management/global_state_controller.py b/core/infrastructure/state_management/global_state_controller.py index 80c7788..5a67b94 100644 --- a/core/infrastructure/state_management/global_state_controller.py +++ b/core/infrastructure/state_management/global_state_controller.py @@ -1,4 +1,9 @@ +from core.models.orm_calls.location_calls import LocationCache +from core.controllers.ProfileController import ProfileController, PROFILE_TABLES_WITH_ORM from core.infrastructure.global_state import ProfileStateRepository +from core.errors.logger import logger + +from typing import Optional def is_systemwide_on() -> bool: """Returns a boolean if systemwide is connected. True is Yes, False is No""" @@ -61,7 +66,24 @@ def is_connected(profile_id: int) -> bool: def is_synced() -> bool: return ProfileStateRepository().global_state.synced -def global_state_marks_sync_completed(): +def global_state_marks_sync_completed(which_models_sync: Optional[dict]): + """Update global state that the sync is completed. And see if cache must be cleared.""" state_repo = ProfileStateRepository() state_repo.sync_done() + if not which_models_sync: + logger.info("Clear of cache not needed.") + return + + needs_refresh = False + + for each_table in PROFILE_TABLES_WITH_ORM: + if each_table in which_models_sync: + needs_refresh = True + break + + if needs_refresh: + logger.info("Clear Location cache") + LocationCache().clear() + logger.info("Clear Profile repo cache") + _ = ProfileController.clear_cache_and_get_fresh_data() \ No newline at end of file diff --git a/core/models/BaseProfile.py b/core/models/BaseProfile.py index 7e35754..310656f 100644 --- a/core/models/BaseProfile.py +++ b/core/models/BaseProfile.py @@ -29,9 +29,10 @@ PROTOCOL_CHOICES = { @dataclass_json @dataclass class BaseProfile(ABC): - id: int = field( - metadata=config(exclude=Exclude.ALWAYS) - ) + id: int + # id: int = field( + # metadata=config(exclude=Exclude.ALWAYS) + # ) name: str subscription: Optional[Subscription] type: ProfileType diff --git a/core/models/manage/lifecycle_tools.py b/core/models/manage/lifecycle_tools.py index 34e6b56..ab9f62e 100644 --- a/core/models/manage/lifecycle_tools.py +++ b/core/models/manage/lifecycle_tools.py @@ -1,9 +1,100 @@ - - +from dataclasses import asdict from sqlalchemy import inspect +from sqlalchemy.orm.exc import UnmappedInstanceError +from sqlalchemy import inspect +from sqlalchemy.exc import NoInspectionAvailable +from core.models.orm_models.Base import BaseModel +from types import SimpleNamespace def is_model_stale(obj): - """"checks the internal state of an object, to evaluate if its detatched or expired."""" + """"checks the internal state of an object, to evaluate if its detatched or expired.""" ins = inspect(obj) return ins.detached or ins.expired +def is_sqlalchemy_model_via_inspect(obj) -> bool: + """Evaluates if an object is an SQLAlchemy model using inspection.""" + try: + inspect(obj) + return True + except (UnmappedInstanceError, NoInspectionAvailable): + return False + +def is_sqlalchemy_model(obj): + """Evaluates if an object is an SQLAlchemy model using the BaseModel inheritance.""" + return isinstance(obj, BaseModel) + +def deep_serialize(data, visited=None): + """ + Recursively wanders through lists and dicts to find + SQLAlchemy models and call their .to_dict() method. + """ + if visited is None: + visited = set() + + # Case 1: It's a SQLAlchemy model + if is_sqlalchemy_model(data): + obj_id = id(data) + if obj_id in visited: + # Return None or a primary key to break circular references + # Returning None prevents infinite loops in bidirectional relationships + return None + + visited.add(obj_id) + # Recurse into the dictionary returned by to_dict() to handle nested models + return deep_serialize(data.to_dict(), visited) + + # Case 2: It's a list (iterate through items) + if isinstance(data, list): + return [deep_serialize(item, visited) for item in data] + + # Case 3: It's a dictionary (iterate through values) + if isinstance(data, dict): + return {k: deep_serialize(v, visited) for k, v in data.items()} + + # Case 4: It's a primitive (int, str, etc.) + return data + +def serialize_dataclass_with_models(dc_obj): + """ + Converts a @dataclass_json object containing SQLAlchemy models + into a pure Python dictionary. + """ + raw_dict = asdict(dc_obj) + return deep_serialize(raw_dict) + + + +######################################################################## +### NAMESPACE VERSION +######################################################################## + +def deep_serialize_to_namespace(data): + """ + Recursively converts SQLAlchemy models and dicts into + SimpleNamespace objects. + """ + # 1. Handle SQLAlchemy models + if is_sqlalchemy_model(data): + # Convert model to dict, then process that dict recursively + return deep_serialize_to_namespace(data.convert_to_dict()) + + # 2. Handle Lists + if isinstance(data, list): + return [deep_serialize_to_namespace(item) for item in data] + + # 3. Handle Dictionaries + if isinstance(data, dict): + return SimpleNamespace(**{k: deep_serialize_to_namespace(v) for k, v in data.items()}) + + # 4. Handle Primitives + return data + +def serialize_dataclass_to_namespace(dc_obj): + """ + Converts a dataclass containing SQLAlchemy models + into a nested SimpleNamespace structure. + """ + raw_dict = asdict(dc_obj) + + return deep_serialize_to_namespace(raw_dict) + diff --git a/core/models/orm_calls/location_calls.py b/core/models/orm_calls/location_calls.py index ca5425b..eeee877 100644 --- a/core/models/orm_calls/location_calls.py +++ b/core/models/orm_calls/location_calls.py @@ -13,6 +13,7 @@ import random @safe_db_operation def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation: + print("getting by country code") location_object = session.execute( select(Location) .where((Location.country_code == country_code) & (Location.code == city_code)) @@ -34,7 +35,8 @@ def get_profile_location_data(country_code: str, city_code: str) -> Location: @safe_db_operation def execute_all_locations(session: Session) -> DatabaseOperation: - return session.query(Location).all() + print("getting all") + return session.query(Location).options(joinedload(Location.operator)).all() def get_random_location(filter_out_list: Optional[list] = None) -> Location: @@ -62,10 +64,13 @@ def get_random_location(filter_out_list: Optional[list] = None) -> Location: @safe_db_operation def execute_location_by_id(location_id: int, session: Session) -> DatabaseOperation: + print("getting by id") location_object = session.execute( select(Location) .where((Location.id == location_id)) + .options(joinedload(Location.operator)) ).scalar_one_or_none() + print(location_object) return location_object diff --git a/core/models/orm_models/Base.py b/core/models/orm_models/Base.py index 9cd85a6..b842d00 100644 --- a/core/models/orm_models/Base.py +++ b/core/models/orm_models/Base.py @@ -17,3 +17,22 @@ class BaseModel(Base): mapper = inspect(self.__class__) fields = ', '.join(f'{col.name}={getattr(self, col.name)}' for col in mapper.columns) return f'{self.__class__.__name__}({fields})' + + def to_dict(self): + """Convert to dict with any foreign keys""" + # 1. Get the mapper for this instance + mapper = inspect(self).mapper + + # 2. Get all column values + # We use .key instead of .name because .key is the Python attribute name + data = {column.key: getattr(self, column.key) for column in mapper.column_attrs} + + # 3. Get all relationship values + relationships = {rel.key: getattr(self, rel.key) for rel in mapper.relationships} + + # Merge them together + return {**data, **relationships} + + + def to_dict_without_foreign_keys(self): + return {column.name: getattr(self, column.name) for column in self.__table__.columns} diff --git a/core/services/profiles/ProfileRepository.py b/core/services/profiles/ProfileRepository.py index 2f8743e..c323057 100644 --- a/core/services/profiles/ProfileRepository.py +++ b/core/services/profiles/ProfileRepository.py @@ -67,6 +67,7 @@ class ProfileRepository: # Only run the slow SQL/file operation if the cache is empty if not self._cache: + logger.info("Hydrating cache with model's fresh data..") # model_manipulator returns a dict { id: profile } profile_result = model_manipulator.get_all_profiles() @@ -77,5 +78,11 @@ class ProfileRepository: # Sync the cache with whatever is currently on disk self._cache.update(all_profiles_dict) - + return Result(valid=True, data=self._cache) + + def refresh_cache(self) -> Result: + """Clears the cache, fetches from disk the model objects again, and hydrates the data in cache.""" + logger.info("Clearing cache") + self._cache = {} + return self.get_all() \ No newline at end of file diff --git a/core/services/profiles/model_manipulator.py b/core/services/profiles/model_manipulator.py index 54ab4fd..6ad9c35 100644 --- a/core/services/profiles/model_manipulator.py +++ b/core/services/profiles/model_manipulator.py @@ -379,16 +379,19 @@ def ensure_dict_format(profile: Profile, string_name: str, potential_sql_type) - return potential_data else: - error_msg = "Invalid format for application version." + error_msg = "Invalid format for this model." logger.error(error_msg) return None -# dict_format = ensure_dict_format(profile=profile, string_name=, potential_sql_type) +# dict_format = ensure_dict_format(profile=profile, string_name="location", potential_sql_type) # if not dict_format: # return Result(valid=False, error_type=ResultError.INVALID_INPUT, message=error_msg) +def fully_serialize_profile(profile: Profile) -> dict: + return profile.to_dict() + def save_to_disk(profile: Profile) -> Result: """