Introduced Full SQL State Refresh post-sync if the SQL model tables updated. This means if sync updates location or app tables, the entire profile repo and all gui calls upon it, get a refresh and recreate

This commit is contained in:
SimplifiedPrivacy 2026-11-17 21:34:54 -05:00
parent 53dc71bf01
commit 8d7d038bae
10 changed files with 217 additions and 19 deletions

View file

@ -1,3 +1,4 @@
from core.models.manage.lifecycle_tools import serialize_dataclass_with_models
from core.services.profiles import model_manipulator
from core.services.profiles.create_objects import create_new_profile_object
from core.services.profiles.ProfileRepository import ProfileRepository
@ -22,6 +23,7 @@ import base64
import re
import time
PROFILE_TABLES_WITH_ORM = ["locations", "operators", "application_versions"]
class ProfileController:
# Initialize the singleton repository
@ -36,6 +38,56 @@ class ProfileController:
def get_all() -> Result:
return ProfileController._repo.get_all()
@staticmethod
def get_all_serialized() -> Result:
got_all = ProfileController._repo.get_all()
if not got_all.valid:
return got_all.wrap("failed to get from the repo")
initial_results = got_all.data
converted_results = {}
logger.info(f"initial_results type {type(initial_results)}")
for key, each_result in initial_results.items():
if not each_result:
logger.info(f"Skipping profile {key}")
try:
converted_dict = serialize_dataclass_with_models(each_result)
converted_results[key] = converted_dict
except Exception as e:
error_msg = str(e)
logger.error(error_msg)
logger.info("finally..")
return Result(valid=True, data=converted_results)
@staticmethod
def get_ONE_serialized(profile_id: int) -> Result:
logger.info("Fetching from the repo")
got_one = ProfileController._repo.get(profile_id)
logger.info(f"Got data, converting it. of type {type(got_one)}")
converted_dict = serialize_dataclass_with_models(got_one)
logger.info(f"converted_dict is {converted_dict}")
return Result(valid=True, data=converted_dict)
@staticmethod
def clear_cache_and_get_fresh_data() -> Result:
"""Clear the Repo's Cache and gets new fresh data."""
logger.info("Clearing Repo cache")
result = ProfileController._repo.refresh_cache()
# update ui:
if result.valid:
logger.info("Updating notifications of change")
Notifications.emit(EventType.PROFILES_CHANGED)
return result
@staticmethod
def _save_and_sync(profile, event_type: str, observer: ProfileObserver = None):
"""Internal helper to handle the actual persistence and cache synchronization."""
@ -86,7 +138,6 @@ class ProfileController:
# update ui:
Notifications.emit(EventType.PROFILES_CHANGED)
@staticmethod
def delete(profile: Profile, profile_observer: ProfileObserver = None):
"""Remove the profile from both disk and cache"""

View file

@ -33,6 +33,7 @@ from core.models.orm_models.ApplicationVersion import ApplicationVersion
# generic
import sys
import json
from typing import Optional
ORM_TABLES = {
"locations": Location,
@ -74,11 +75,11 @@ def get_orm_model(key) -> bool:
return ORM_TABLES[key]
def _sync_success():
def _sync_success(which_models_sync: Optional[dict]):
notification = "Finshed sync."
Notifications.emit(EventType.STATUS_UPDATE, notification)
Notifications.emit(EventType.SYNC_COMPLETE, notification)
global_state_marks_sync_completed()
global_state_marks_sync_completed(which_models_sync)
return Result(valid=True, message=notification)
@ -123,7 +124,7 @@ def new_sync(client_observer: ClientObserver, connection_observer: ConnectionObs
notification = "Sync Complete. (already up to date)"
Notifications.emit(EventType.STATUS_UPDATE, notification)
Notifications.emit(EventType.SYNC_COMPLETE, notification)
global_state_marks_sync_completed()
global_state_marks_sync_completed(which_models_sync=None)
return Result(valid=True)
# We only make it past this point if there's New Data
@ -198,12 +199,12 @@ def new_sync(client_observer: ClientObserver, connection_observer: ConnectionObs
# forced_sync being true, forces this to none:
if filtered_metadata is None:
return _sync_success()
return _sync_success(which_models_sync=all_results)
# otherwise evaluate saving of metadata:
save_successful = save_metadata(filtered_metadata)
if save_successful:
return _sync_success()
return _sync_success(which_models_sync=all_results)
else:
error_msg = "Finshed sync, but had issues with the saving of metadata for next time."
logger.error(error_msg)

View file

@ -123,8 +123,6 @@ class ThreadPlumbing:
def _back_on_main_thread(self, result: Result):
"""Decide: retry, show popup, or final callback"""
logger.info(result)
#----------------------------
# OUTRIGHT ERROR STACK
#----------------------------

View file

@ -1,4 +1,9 @@
from core.models.orm_calls.location_calls import LocationCache
from core.controllers.ProfileController import ProfileController, PROFILE_TABLES_WITH_ORM
from core.infrastructure.global_state import ProfileStateRepository
from core.errors.logger import logger
from typing import Optional
def is_systemwide_on() -> bool:
"""Returns a boolean if systemwide is connected. True is Yes, False is No"""
@ -61,7 +66,24 @@ def is_connected(profile_id: int) -> bool:
def is_synced() -> bool:
return ProfileStateRepository().global_state.synced
def global_state_marks_sync_completed():
def global_state_marks_sync_completed(which_models_sync: Optional[dict]):
"""Update global state that the sync is completed. And see if cache must be cleared."""
state_repo = ProfileStateRepository()
state_repo.sync_done()
if not which_models_sync:
logger.info("Clear of cache not needed.")
return
needs_refresh = False
for each_table in PROFILE_TABLES_WITH_ORM:
if each_table in which_models_sync:
needs_refresh = True
break
if needs_refresh:
logger.info("Clear Location cache")
LocationCache().clear()
logger.info("Clear Profile repo cache")
_ = ProfileController.clear_cache_and_get_fresh_data()

View file

@ -29,9 +29,10 @@ PROTOCOL_CHOICES = {
@dataclass_json
@dataclass
class BaseProfile(ABC):
id: int = field(
metadata=config(exclude=Exclude.ALWAYS)
)
id: int
# id: int = field(
# metadata=config(exclude=Exclude.ALWAYS)
# )
name: str
subscription: Optional[Subscription]
type: ProfileType

View file

@ -1,9 +1,100 @@
from dataclasses import asdict
from sqlalchemy import inspect
from sqlalchemy.orm.exc import UnmappedInstanceError
from sqlalchemy import inspect
from sqlalchemy.exc import NoInspectionAvailable
from core.models.orm_models.Base import BaseModel
from types import SimpleNamespace
def is_model_stale(obj):
""""checks the internal state of an object, to evaluate if its detatched or expired.""""
""""checks the internal state of an object, to evaluate if its detatched or expired."""
ins = inspect(obj)
return ins.detached or ins.expired
def is_sqlalchemy_model_via_inspect(obj) -> bool:
"""Evaluates if an object is an SQLAlchemy model using inspection."""
try:
inspect(obj)
return True
except (UnmappedInstanceError, NoInspectionAvailable):
return False
def is_sqlalchemy_model(obj):
"""Evaluates if an object is an SQLAlchemy model using the BaseModel inheritance."""
return isinstance(obj, BaseModel)
def deep_serialize(data, visited=None):
"""
Recursively wanders through lists and dicts to find
SQLAlchemy models and call their .to_dict() method.
"""
if visited is None:
visited = set()
# Case 1: It's a SQLAlchemy model
if is_sqlalchemy_model(data):
obj_id = id(data)
if obj_id in visited:
# Return None or a primary key to break circular references
# Returning None prevents infinite loops in bidirectional relationships
return None
visited.add(obj_id)
# Recurse into the dictionary returned by to_dict() to handle nested models
return deep_serialize(data.to_dict(), visited)
# Case 2: It's a list (iterate through items)
if isinstance(data, list):
return [deep_serialize(item, visited) for item in data]
# Case 3: It's a dictionary (iterate through values)
if isinstance(data, dict):
return {k: deep_serialize(v, visited) for k, v in data.items()}
# Case 4: It's a primitive (int, str, etc.)
return data
def serialize_dataclass_with_models(dc_obj):
"""
Converts a @dataclass_json object containing SQLAlchemy models
into a pure Python dictionary.
"""
raw_dict = asdict(dc_obj)
return deep_serialize(raw_dict)
########################################################################
### NAMESPACE VERSION
########################################################################
def deep_serialize_to_namespace(data):
"""
Recursively converts SQLAlchemy models and dicts into
SimpleNamespace objects.
"""
# 1. Handle SQLAlchemy models
if is_sqlalchemy_model(data):
# Convert model to dict, then process that dict recursively
return deep_serialize_to_namespace(data.convert_to_dict())
# 2. Handle Lists
if isinstance(data, list):
return [deep_serialize_to_namespace(item) for item in data]
# 3. Handle Dictionaries
if isinstance(data, dict):
return SimpleNamespace(**{k: deep_serialize_to_namespace(v) for k, v in data.items()})
# 4. Handle Primitives
return data
def serialize_dataclass_to_namespace(dc_obj):
"""
Converts a dataclass containing SQLAlchemy models
into a nested SimpleNamespace structure.
"""
raw_dict = asdict(dc_obj)
return deep_serialize_to_namespace(raw_dict)

View file

@ -13,6 +13,7 @@ import random
@safe_db_operation
def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
print("getting by country code")
location_object = session.execute(
select(Location)
.where((Location.country_code == country_code) & (Location.code == city_code))
@ -34,7 +35,8 @@ def get_profile_location_data(country_code: str, city_code: str) -> Location:
@safe_db_operation
def execute_all_locations(session: Session) -> DatabaseOperation:
return session.query(Location).all()
print("getting all")
return session.query(Location).options(joinedload(Location.operator)).all()
def get_random_location(filter_out_list: Optional[list] = None) -> Location:
@ -62,10 +64,13 @@ def get_random_location(filter_out_list: Optional[list] = None) -> Location:
@safe_db_operation
def execute_location_by_id(location_id: int, session: Session) -> DatabaseOperation:
print("getting by id")
location_object = session.execute(
select(Location)
.where((Location.id == location_id))
.options(joinedload(Location.operator))
).scalar_one_or_none()
print(location_object)
return location_object

View file

@ -17,3 +17,22 @@ class BaseModel(Base):
mapper = inspect(self.__class__)
fields = ', '.join(f'{col.name}={getattr(self, col.name)}' for col in mapper.columns)
return f'{self.__class__.__name__}({fields})'
def to_dict(self):
"""Convert to dict with any foreign keys"""
# 1. Get the mapper for this instance
mapper = inspect(self).mapper
# 2. Get all column values
# We use .key instead of .name because .key is the Python attribute name
data = {column.key: getattr(self, column.key) for column in mapper.column_attrs}
# 3. Get all relationship values
relationships = {rel.key: getattr(self, rel.key) for rel in mapper.relationships}
# Merge them together
return {**data, **relationships}
def to_dict_without_foreign_keys(self):
return {column.name: getattr(self, column.name) for column in self.__table__.columns}

View file

@ -67,6 +67,7 @@ class ProfileRepository:
# Only run the slow SQL/file operation if the cache is empty
if not self._cache:
logger.info("Hydrating cache with model's fresh data..")
# model_manipulator returns a dict { id: profile }
profile_result = model_manipulator.get_all_profiles()
@ -79,3 +80,9 @@ class ProfileRepository:
self._cache.update(all_profiles_dict)
return Result(valid=True, data=self._cache)
def refresh_cache(self) -> Result:
"""Clears the cache, fetches from disk the model objects again, and hydrates the data in cache."""
logger.info("Clearing cache")
self._cache = {}
return self.get_all()

View file

@ -379,16 +379,19 @@ def ensure_dict_format(profile: Profile, string_name: str, potential_sql_type) -
return potential_data
else:
error_msg = "Invalid format for application version."
error_msg = "Invalid format for this model."
logger.error(error_msg)
return None
# dict_format = ensure_dict_format(profile=profile, string_name=, potential_sql_type)
# dict_format = ensure_dict_format(profile=profile, string_name="location", potential_sql_type)
# if not dict_format:
# return Result(valid=False, error_type=ResultError.INVALID_INPUT, message=error_msg)
def fully_serialize_profile(profile: Profile) -> dict:
return profile.to_dict()
def save_to_disk(profile: Profile) -> Result:
"""