diff --git a/assets/yaml_mappings/locations.yaml b/assets/yaml_mappings/locations.yaml index 762af31..97ee248 100644 --- a/assets/yaml_mappings/locations.yaml +++ b/assets/yaml_mappings/locations.yaml @@ -33,9 +33,14 @@ fields: path: ['is_wireguard_capable'] required: true + - name: is_hysteria2_capable + path: ['is_hysteria2_capable'] - # required: true + - name: is_vless_capable + path: ['is_vless_capable'] + +# original version: # locations.append(( # location['country']['code'], # location['code'], diff --git a/core/Constants.py b/core/Constants.py index a6e755a..de0656e 100644 --- a/core/Constants.py +++ b/core/Constants.py @@ -6,6 +6,9 @@ import os @dataclass(frozen=True) class Constants: + DB_VERSION_THIS_APP_WANTS = 1 + + # ticketing group: TICKET_API_BASE_URL: Final[str] = os.environ.get( "TICKET_API_BASE_URL", "https://ticket.hydraveil.net" ) @@ -51,7 +54,7 @@ class Constants: HV_SESSION_STATE_HOME: Final[str] = f'{HV_STATE_HOME}/sessions' HV_TOR_STATE_HOME: Final[str] = f'{HV_STATE_HOME}/tor' -# ── sing-box ────────────────────────────────────────────────────────────── + # ── sing-box ────────────────────────────────────────────────────────────── SINGBOX_WRAPPER: Final[str] = os.environ.get( 'SINGBOX_WRAPPER', '/usr/local/bin/hydraveil-singbox' ) diff --git a/core/controllers/ClientController.py b/core/controllers/ClientController.py index bc77c23..42ad26d 100644 --- a/core/controllers/ClientController.py +++ b/core/controllers/ClientController.py @@ -1,7 +1,14 @@ # new sync refactor: from core.models.manage.session_management import init_session, close_session from core.services.sync.sync_service import coordinate_cache_sync, save_metadata +from core.services.sync.orm_methods.sync_one_orm import sync_one_orm_model from core.errors.logger import logger + +# ORM models that can be sync'ed: +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator + + # prior versions: from core.Constants import Constants from core.Errors import UnknownClientPathError, UnknownClientVersionError, CommandNotFoundError @@ -9,8 +16,8 @@ from core.controllers.ApplicationController import ApplicationController from core.controllers.ApplicationVersionController import ApplicationVersionController from core.controllers.ClientVersionController import ClientVersionController from core.controllers.ConfigurationController import ConfigurationController -from core.controllers.LocationController import LocationController -from core.controllers.OperatorController import OperatorController +# from core.controllers.LocationController import LocationController +# from core.controllers.OperatorController import OperatorController from core.controllers.SubscriptionPlanController import SubscriptionPlanController from core.observers.ClientObserver import ClientObserver from core.observers.ConnectionObserver import ConnectionObserver @@ -52,12 +59,9 @@ class ClientController: if client_observer is not None: client_observer.notify('synchronizing', "Fetching list of new data ..") - # Use Cached Method: - init_session() - result = coordinate_cache_sync(ClientObserver, ConnectionObserver) - logger.info(f"We got a Result from the API of {result}") + # logger.info(f"We got a Result from the API of {result}") # Outright Error: if not result["success"]: @@ -78,8 +82,30 @@ class ClientController: # flag for after the save, data_was_saved = False + # Fetch and update the real data (no longer metadata)... + + # =================== ORM BASED MODELS ================== + """ + Note: for the new ORM based models, + it does the Tor/system check in the API call itself. + """ + + if "locations" in changed_tables: + logger.info("Sync of Locations") + if client_observer is not None: + client_observer.notify('synchronizing', 'Fetching Locations List..') + + final_result = sync_one_orm_model(Location, "locations") + + if "operators" in changed_tables: + logger.info("Sync of Operators") + if client_observer is not None: + client_observer.notify('synchronizing', 'Fetching Operators List..') + + final_result_two = sync_one_orm_model(Operator, "operators") + + # =================== MANUAL-SQL BASED MODELS ================== try: - # Fetch and update the real data (no longer metadata)... from core.controllers.ConnectionController import ConnectionController ConnectionController.with_preferred_connection(task=ClientController.__sync, changed_tables=changed_tables, client_observer=client_observer, connection_observer=connection_observer) @@ -98,9 +124,6 @@ class ClientController: filtered_metadata = result["filtered_metadata"] # from the top of the function save_successful = save_metadata(filtered_metadata) # the "save_data" function is inside sync_service - # Regardless of the outcome, - close_session() - if client_observer is None: logger.error("Error: No client_observer to update the UI, the final part of the sync function skipped") return # can't update their UI @@ -151,20 +174,6 @@ class ClientController: # noinspection PyProtectedMember ClientVersionController._sync(proxies=proxies) - if "operators" in changed_tables: - logger.info("Sync of Operators") - if client_observer is not None: - client_observer.notify('synchronizing', 'Fetching Operators List..') - # noinspection PyProtectedMember - OperatorController._sync(proxies=proxies) - - if "locations" in changed_tables: - logger.info("Sync of Locations") - if client_observer is not None: - client_observer.notify('synchronizing', 'Fetching Locations List..') - # noinspection PyProtectedMember - LocationController._sync(proxies=proxies) - if "subscriptions" in changed_tables: logger.info("Sync of Subscriptions") if client_observer is not None: diff --git a/core/controllers/LocationController.py b/core/controllers/LocationController.py index c9d84b2..627d1b7 100644 --- a/core/controllers/LocationController.py +++ b/core/controllers/LocationController.py @@ -1,22 +1,49 @@ -from core.models.Location import Location -from core.services.WebServiceApiService import WebServiceApiService -from typing import Optional +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator +from core.models.manage.session_management import get_session +from typing import Optional +from sqlalchemy import select +from sqlalchemy.orm import joinedload class LocationController: @staticmethod - def get(country_code: str, code: str): - return Location.find(country_code, code) + def get(country_code: str, city_code: str): + with get_session() as session: + location_object = session.execute( + select(Location) + .where((Location.country_code == country_code) & (Location.code == city_code)) + .options(joinedload(Location.operator)) + ).scalar_one_or_none() + + return location_object + + # legacy: + # Location.find(country_code, code) + + @staticmethod def get_all(): - return Location.all() + with get_session() as session: + all_records = session.execute( + select(Location) + .options(joinedload(Location.operator)) + ).scalars().all() + return all_records - @staticmethod - def _sync(proxies: Optional[dict] = None): + # legacy: + # return Location.all() - locations = WebServiceApiService.get_locations(proxies) - Location.truncate() - Location.save_many(locations) +# Deprecated legacy sync, + +# from core.services.WebServiceApiService import WebServiceApiService + # @staticmethod + # def _sync(proxies: Optional[dict] = None): + + # locations = WebServiceApiService.get_locations(proxies) + + # Location.truncate() + # Location.save_many(locations) diff --git a/core/controllers/OperatorController.py b/core/controllers/OperatorController.py deleted file mode 100644 index 9d54c4a..0000000 --- a/core/controllers/OperatorController.py +++ /dev/null @@ -1,22 +0,0 @@ -from core.models.Operator import Operator -from core.services.WebServiceApiService import WebServiceApiService -from typing import Optional - - -class OperatorController: - - @staticmethod - def get(id: int): - return Operator.find_by_id(id) - - @staticmethod - def get_all(): - return Operator.all() - - @staticmethod - def _sync(proxies: Optional[dict] = None): - - operators = WebServiceApiService.get_operators(proxies) - - Operator.truncate() - Operator.save_many(operators) diff --git a/core/models/BaseProfile.py b/core/models/BaseProfile.py index 1add011..75cd58d 100644 --- a/core/models/BaseProfile.py +++ b/core/models/BaseProfile.py @@ -1,11 +1,21 @@ +from core.models.manage.session_management import get_session +from sqlalchemy import select +from sqlalchemy.orm import joinedload + from abc import ABC, abstractmethod from core.Constants import Constants from core.Helpers import write_atomically -from core.models.Location import Location + +# from core.models.Location import Location +from core.controllers.LocationController import LocationController +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator + from core.models.Subscription import Subscription from core.models.session.ApplicationVersion import ApplicationVersion from dataclasses import dataclass, field, asdict from dataclasses_json import config, Exclude, dataclass_json + from pathlib import Path from typing import Optional, Self import json @@ -23,7 +33,10 @@ class BaseProfile(ABC): ) name: str subscription: Optional[Subscription] - location: Optional[Location] + location: Optional[Location] = field(metadata=config(exclude=Exclude.ALWAYS)) # SQLAlchemy object + + # legacy version included it to be serialized, but now it's an SQLAlchemy object. + # location: Optional[Location] @abstractmethod def get_wireguard_configuration_path(self): @@ -52,8 +65,15 @@ class BaseProfile(ABC): return type(self).__name__ == 'SystemProfile' def save(self: Self): + config_dict = self.to_dict() # Get dict, not JSON string + location_dict = self.location.to_dict() if self.location else None if self.location else None # this is from SQLAlchemy, and not JSON-models, that's why it's separate. - config_file_contents = f'{self.to_json(indent=4)}\n' + config_dict["location"] = location_dict + + config_file_contents = json.dumps(config_dict, indent=4) + '\n' + + # legacy version: + # config_file_contents = f'{self.to_json(indent=4)}\n' os.makedirs(self.get_config_path(), exist_ok=True) os.makedirs(self.get_data_path(), exist_ok=True) @@ -146,6 +166,7 @@ class BaseProfile(ABC): return list([key for key, value in asdict(self).items() if value != asdict(reference).get(key)]) + @staticmethod def find_by_id(id: int): @@ -161,16 +182,43 @@ class BaseProfile(ABC): profile['id'] = id + profiles_location = profile['location'] + if profile['location'] is not None: + if isinstance(profiles_location, dict): + try: + country_code = profile['location']['country_code'] + city_code = profile['location']['code'] + except: + print("ERROR! CANT FIND country code or city") + return + else: + # potentially coming from SQLAlchemy: + country_code = profile.location.country_code + city_code = profile.location.code - location = Location.find(profile['location']['country_code'] or None, profile['location']['code'] or None) + with get_session() as session: + location_object = session.execute( + select(Location) + .where((Location.country_code == country_code) & (Location.code == city_code)) + .options(joinedload(Location.operator)) + ).scalar_one_or_none() - if location is not None: - if profile['location'].get('time_zone') is not None: - location.time_zone = profile['location']['time_zone'] + if location_object: + profile['location'] = location_object - profile['location'] = location + # this needs error handling if there's no location or malconformed config. + + + # legacy phased out since SQLAlchemy handles the time_zone. + + # if location is not None: + + # if profile['location'].get('time_zone') is not None: + # location.time_zone = profile['location']['time_zone'] + + # profile['location'] = location if 'application_version' in profile: diff --git a/core/models/Location.py b/core/models/Location.py deleted file mode 100644 index 62c963b..0000000 --- a/core/models/Location.py +++ /dev/null @@ -1,115 +0,0 @@ -from core.models.Model import Model -from core.models.Operator import Operator -from dataclasses import dataclass, field -from dataclasses_json import config, Exclude -from typing import Optional - -_table_name: str = 'locations' - -_table_definition: str = """ - 'id' int UNIQUE, - 'country_code' varchar, - 'country_name' varchar, - 'code' varchar, - 'name' varchar, - 'time_zone' varchar, - 'operator_id' int, - 'provider_name' varchar, - 'is_proxy_capable' bool, - 'is_wireguard_capable' bool, - UNIQUE(code, country_code) -""" - - -@dataclass -class Location(Model): - country_code: str - code: str - id: Optional[int] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - country_name: Optional[str] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - name: Optional[str] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - time_zone: Optional[str] = None - operator_id: Optional[int] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - provider_name: Optional[str] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - is_proxy_capable: Optional[bool] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - is_wireguard_capable: Optional[bool] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - operator: Optional[Operator] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - available: Optional[bool] = field( - default=False, - metadata=config(exclude=Exclude.ALWAYS) - ) - - def __post_init__(self): - self.operator = Operator.find_by_id(self.operator_id) - self.available = self.exists(self.country_code, self.code) - - if isinstance(self.is_proxy_capable, int): - self.is_proxy_capable = bool(self.is_proxy_capable) - - if isinstance(self.is_wireguard_capable, int): - self.is_wireguard_capable = bool(self.is_wireguard_capable) - - def is_available(self): - return self.exists(self.country_code, self.code) - - @staticmethod - def find_by_id(id: int): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_one('SELECT * FROM locations WHERE id = ? LIMIT 1', Location.factory, [id]) - - @staticmethod - def find(country_code: str, code: str): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_one('SELECT * FROM locations WHERE country_code = ? AND code = ? LIMIT 1', Location.factory, [country_code, code]) - - @staticmethod - def exists(country_code: str, code: str): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_exists('SELECT * FROM locations WHERE country_code = ? AND code = ?', [country_code, code]) - - @staticmethod - def all(): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_all('SELECT * FROM locations', Location.factory) - - @staticmethod - def truncate(): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True) - - @staticmethod - def save_many(locations): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - Model._insert_many('INSERT INTO locations VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', Location.tuple_factory, locations) - - @staticmethod - def factory(cursor, row): - local_fields = [column[0] for column in cursor.description] - return Location(**{key: value for key, value in zip(local_fields, row)}) - - @staticmethod - def tuple_factory(location): - return location.id, location.country_code, location.country_name, location.code, location.name, location.time_zone, location.operator_id, location.provider_name, location.is_proxy_capable, location.is_wireguard_capable diff --git a/core/models/Operator.py b/core/models/Operator.py deleted file mode 100644 index 8cb21e7..0000000 --- a/core/models/Operator.py +++ /dev/null @@ -1,64 +0,0 @@ -from core.models.Model import Model -from dataclasses import dataclass - -_table_name: str = 'operators' - -_table_definition: str = """ - 'id' int UNIQUE, - 'name' varchar, - 'type' varchar, - 'public_key' varchar, - 'nostr_public_key' varchar, - 'nostr_profile_reference' varchar, - 'nostr_attestation_event_reference' varchar -""" - - -@dataclass -class Operator(Model): - id: int - name: str - type: str - public_key: str - nostr_public_key: str - nostr_profile_reference: str - nostr_attestation_event_reference: str - - def is_external(self) -> bool: - return self.type == 'external' - - def is_internal(self) -> bool: - return self.type == 'internal' - - @staticmethod - def find_by_id(id: int): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_one('SELECT * FROM operators WHERE id = ? LIMIT 1', Operator.factory, [id]) - - @staticmethod - def exists(id: int): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_exists('SELECT * FROM operators WHERE id = ?', [id]) - - @staticmethod - def all(): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_all('SELECT * FROM operators', Operator.factory) - - @staticmethod - def truncate(): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True) - - @staticmethod - def save_many(operators): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - Model._insert_many('INSERT INTO operators VALUES(?, ?, ?, ?, ?, ?, ?)', Operator.tuple_factory, operators) - - @staticmethod - def factory(cursor, row): - local_fields = [column[0] for column in cursor.description] - return Operator(**{key: value for key, value in zip(local_fields, row)}) - - @staticmethod - def tuple_factory(operator): - return operator.id, operator.name, operator.type, operator.public_key, operator.nostr_public_key, operator.nostr_profile_reference, operator.nostr_attestation_event_reference \ No newline at end of file diff --git a/core/models/manage/denormalize.py b/core/models/manage/denormalize.py new file mode 100644 index 0000000..795ac9c --- /dev/null +++ b/core/models/manage/denormalize.py @@ -0,0 +1,75 @@ +import yaml + +""" +This function extracts the value from a nested dictionary, +By looping through each layer. And comparing the current dictionary's key to pre-made YAML key mappings. +""" +def extract_from_nested(obj, yaml_keys, default=None): + # loop through each key from the YAML mapping: + for each_yaml_key in yaml_keys: + + # Check if each layer down of the 'object' is a dictionary with that key, + if isinstance(obj, dict) and each_yaml_key in obj: + # then extract the value for that key, and replace the object placeholder itself with it. + obj = obj[each_yaml_key] + + # this has the effect of going deeper into the nesting on the next round of the loop, + + # But if it's not a dictionary or doesn't have the key, + else: + # then we hit a dead end, + return default + + # We finished looping through all the keys of valid dictionaries, and extracted the values, + return obj + +""" +This function removes unnecessary nesting, +by comparing it to a pre-made YAML mapping +""" +def denormalize(data, which_yaml_file): + final_results = [] + + # prep the mapping config: + mapping_file = f"assets/yaml_mappings/{which_yaml_file}" + with open(mapping_file) as f: + mapping = yaml.safe_load(f) + + # Extract out the 'data' variable if it exists, otherwise just use the data directly + extracted_data = data.get('data', data) + + # go through the data + for each_item in extracted_data: + + # setup temp flags/containers + each_denormalized_piece = {} + valid = True + + # go through mapping: + for field in mapping['fields']: + """ + Extract the values based on the YAML mapping + field['path'] - Gets a Python list of the keys from the YAML mapping + extract_from_nested - This function checks it's actually a dictionary and extracts it + """ + value = extract_from_nested(each_item, field['path']) + + # Store the extracted value. If it wasn't there, then it's None + each_denormalized_piece[field['name']] = value + + # if it's required & not there, ditch it, + if field.get('required') and value is None: + print(f"It's required and value is none, skipping {value}") + valid = False + break + + # Only add items where all required fields were present + if valid: + final_results.append(each_denormalized_piece) + + # Return all successfully denormalized items + return final_results + + +# Isolated testing: +# items = denormalize(response.json(), 'example.yaml') diff --git a/core/models/manage/insert.py b/core/models/manage/insert.py index 8b3520e..29f20b4 100644 --- a/core/models/manage/insert.py +++ b/core/models/manage/insert.py @@ -3,9 +3,14 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy import create_engine from typing import Type, Dict, Any from sqlalchemy.exc import IntegrityError, SQLAlchemyError -from core.models.orm_models.Base import Base -from core.models.orm_models.CachedSync import CachedSync from core.models.manage.session_management import get_session +from core.models.orm_models.Base import BaseModel + +# all models it knows how to do: +from core.models.orm_models.CachedSync import CachedSync +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator +from core.models.orm_models.EncryptedProxy import EncryptedProxy def insert_into_model(model_class: Type, all_data: dict | list, override=False) -> bool: """ diff --git a/core/models/manage/session_management.py b/core/models/manage/session_management.py index 9f9d226..90571de 100644 --- a/core/models/manage/session_management.py +++ b/core/models/manage/session_management.py @@ -1,40 +1,120 @@ -from sqlalchemy.orm import sessionmaker +from core.errors.logger import logger +from core.models.orm_models.Base import BaseModel from sqlalchemy import create_engine -from core.models.orm_models.Base import Base -from core.Constants import Constants +from sqlalchemy.orm import sessionmaker +import os +from pathlib import Path -database_path = Constants.HV_STORAGE_DATABASE_PATH -engine = create_engine(f"sqlite:///{database_path}") -Session = sessionmaker(bind=engine) +""" +Note: +At the bottom it initializes the Session and sets it up outside the function loose. +""" +# ============================================================================ +# PATH & INITIALIZATION +# ============================================================================ + +def get_path(): + """Returns XDG Base Directory path or ~/.local/share/hydra-veil""" + xdg_data = os.getenv("XDG_DATA_HOME") + if xdg_data: + return Path(xdg_data) / "my-app" + return Path.home() / ".local" / "share" / "hydra-veil" + +# ============================================================================ +# GLOBAL STATE +# ============================================================================ + +system_path = get_path() +database_path = system_path / "storage.db" +engine = None +Session = None _session = None -def init_session(): - """ - Called once at application startup using global variables for the session, - which initialize prior to being called with a false flag. +# ============================================================================ +# ENGINE & SESSION MANAGEMENT +# ============================================================================ - The reason for this is so the app can load prior to be called, - while keeping it a globally accessible variable +def _reinitialize_engine_and_session(): """ + Recreate the global engine and Session factory. + Call this after deleting the database to sync globals with the new DB file. + """ + global engine, Session + + system_path.mkdir(parents=True, exist_ok=True) + engine = create_engine(f"sqlite:///{database_path}") + Session = sessionmaker(bind=engine) + + +def init_session(): + """Initialize the global _session from the global Session factory.""" global _session - Base.metadata.create_all(engine) + if Session is None: + raise RuntimeError("Session factory not initialized. Call _reinitialize_engine_and_session() first.") _session = Session() + def get_session(): - """Get the persistent session""" + """Return the global _session or raise RuntimeError.""" if _session is None: raise RuntimeError("Session not initialized. Call init_session() first.") return _session -def close_session(): - """ - Used prior to shutting down the application to break SQL connections. - In the current refactor, it may be used prior to shut down, such as sync completion, - because a low amount of the total models currently use the new system. - """ +def close_session(): + """Close and reset the global _session.""" global _session - if _session: + if _session is not None: _session.close() - _session = None + _session = None + + +# ============================================================================ +# DATABASE OPERATIONS +# ============================================================================ + +def create_ONLY_db_version_table(): + """Create only the database_version table using the global engine.""" + if engine is None: + raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.") + + from core.models.orm_models.DatabaseVersion import database_version + database_version.create(engine, checkfirst=True) + # DatabaseVersion.__table__.create(engine, checkfirst=True) + + +def create_ALL_tables(): + """Create all tables from BaseModel.metadata using the global engine.""" + print("starting create_ALL_tables") + if engine is None: + raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.") + print("passed engine test") + + from core.models.orm_models.Location import Location + from core.models.orm_models.Operator import Operator + from core.models.orm_models.CachedSync import CachedSync + from core.models.orm_models.EncryptedProxy import EncryptedProxy + + from core.models.SubscriptionPlan import SubscriptionPlan + from core.models.session.ApplicationVersion import ApplicationVersion + + + try: + print("trying this") + BaseModel.metadata.create_all(engine, checkfirst=True) + print("Returning True for Creating all Tables..") + return True + except: + print("couldn't make all tables") + return False + + +# ============================================================================ +# STARTUP: Initialize engine and session on import +# ============================================================================ + +_reinitialize_engine_and_session() +logger.info("[DB MANAGEMENT] Started Engine in Session Management Module..") +init_session() +logger.info("[DB MANAGEMENT] Initialized Session") diff --git a/core/models/manage/version_check.py b/core/models/manage/version_check.py new file mode 100644 index 0000000..c308ee3 --- /dev/null +++ b/core/models/manage/version_check.py @@ -0,0 +1,348 @@ +from core.models.orm_models.DatabaseVersion import database_version +from core.Constants import Constants +from core.errors.logger import logger + + +# generic +from sqlalchemy import exc, text +from sqlalchemy.orm import Session +from typing import Optional + + +def get_database_version(session: Session) -> Optional[int]: + """ + Purpose: + Retrieve the current database version from the single-row version table. + + Rank: + Helper + + Called By: + check_database_compatibility + + Features: + Handles errors with multiple items, failed loads, and other conditions. + + Returns: + int: The database version, or None if retrieval fails. + """ + func_name = "get_database_version" + + # Validate session + if session is None: + logger.error(f"{func_name}: Session is None") + return None + + if not isinstance(session, Session): + logger.error(f"{func_name}: Invalid session type: {type(session)}") + return None + + try: + # Attempt to query the table + logger.debug(f"{func_name}: Attempting to query database_version table") + result = session.query(database_version).all() + + # Handle empty table + if not result: + logger.warning(f"{func_name}: Table is empty, no version row found") + return None + + # Handle multiple rows (shouldn't happen, but log it) + if len(result) > 1: + logger.warning( + f"{func_name}: Expected 1 row, found {len(result)}. " + "Table should contain only a single row. Returning first version." + ) + + # Extract version from first row + version_row = result[0] + version = version_row.version + + # Type validation + if not isinstance(version, int): + logger.error( + f"{func_name}: Version column type mismatch. " + f"Expected int, got {type(version).__name__}: {version}" + ) + return None + + if version < 0: + logger.warning( + f"{func_name}: Version is negative ({version}). " + "This may indicate corrupted data." + ) + + logger.info(f"{func_name}: Successfully retrieved version: {version}") + return version + + except exc.NoSuchTableError: + logger.error( + f"{func_name}: Table 'database_version' does not exist. " + "Ensure the table is created with `database_version.create(engine, checkfirst=True)`" + ) + return None + + except exc.OperationalError as e: + logger.error( + f"{func_name}: Database operational error (connection, locked, or permissions). " + f"Details: {str(e)}" + ) + return None + + except exc.DatabaseError as e: + logger.error( + f"{func_name}: General database error. " + f"Details: {str(e)}" + ) + return None + + except exc.StatementError as e: + logger.error( + f"{func_name}: SQL statement error. " + f"Details: {str(e)}" + ) + return None + + except AttributeError as e: + logger.error( + f"{func_name}: Row does not have 'version' attribute. " + f"Verify table schema matches definition. Details: {str(e)}" + ) + return None + + except TypeError as e: + logger.error( + f"{func_name}: Type error when accessing version. " + f"Details: {str(e)}" + ) + return None + + except Exception as e: + logger.critical( + f"{func_name}: Unexpected exception type {type(e).__name__}. " + f"Details: {str(e)}", + exc_info=True + ) + return None + + +def insert_new_version(session: Session, new_version: int) -> bool: + """ + Purpose: + Insert (or replace) the database version. Clears existing rows and inserts new version. + + Rank: + Helper + + Called By: + GUI's startup script in __main__ + + Args: + session: SQLAlchemy session + new_version: The version number to insert (must be a positive integer) + + Returns: + bool: True if successful, False otherwise. + does NOT raise errors. + """ + func_name = "insert_new_version" # for logging purposes + + # Validate session + if session is None: + logger.error(f"{func_name}: Session is None") + return False + + if not isinstance(session, Session): + logger.error(f"{func_name}: Invalid session type: {type(session)}") + return False + + # Validate new_version parameter + if new_version is None: + logger.error(f"{func_name}: new_version is None") + return False + + if not isinstance(new_version, int): + logger.error( + f"{func_name}: new_version must be an integer, got {type(new_version).__name__}: {new_version}" + ) + return False + + if new_version < 0: + logger.error( + f"{func_name}: new_version must be non-negative, got {new_version}" + ) + return False + + try: + logger.debug(f"{func_name}: Starting transaction for version insert/replace") + + # Check if table exists before proceeding + try: + session.query(database_version).limit(1).all() + except exc.NoSuchTableError: + logger.error( + f"{func_name}: Table 'database_version' does not exist. " + "Ensure the table is created with `database_version.create(engine, checkfirst=True)`" + ) + return False + + # Delete existing rows + logger.debug(f"{func_name}: Deleting existing version rows") + deleted_count = session.query(database_version).delete() + logger.debug(f"{func_name}: Deleted {deleted_count} existing row(s)") + + if deleted_count > 1: + logger.warning( + f"{func_name}: Deleted {deleted_count} rows. " + "Table should contain only a single row at any time." + ) + + # Actually Insert the new version + logger.debug(f"{func_name}: Inserting new version {new_version}") + insert_stmt = database_version.insert().values(version=new_version) + session.execute(insert_stmt) + + # Commit transaction + session.commit() + logger.info(f"{func_name}: Successfully inserted version {new_version}") + return True + + except exc.NoSuchTableError: + logger.error( + f"{func_name}: Table 'database_version' does not exist. " + "Ensure the table is created with `database_version.create(engine, checkfirst=True)`" + ) + session.rollback() + return False + + except exc.IntegrityError as e: + logger.error( + f"{func_name}: Integrity constraint violation (e.g., primary key conflict). " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except exc.OperationalError as e: + logger.error( + f"{func_name}: Database operational error (connection, locked, or permissions). " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except exc.DatabaseError as e: + logger.error( + f"{func_name}: General database error. " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except exc.StatementError as e: + logger.error( + f"{func_name}: SQL statement error. " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except ValueError as e: + logger.error( + f"{func_name}: Value error during insert. " + f"Details: {str(e)}" + ) + session.rollback() + return False + + except Exception as e: + logger.critical( + f"{func_name}: Unexpected exception type {type(e).__name__}. " + f"Details: {str(e)}", + exc_info=True + ) + session.rollback() + return False + +"""Spits back a string based on the reason and error.""" +def get_custom_message(reason: str, compatability_dict: dict) -> str: + """ + Called By: + GUI's startup script in __main__ + """ + if reason == "upgrade": + custom_error = "You just upgraded HydraVeil, so let's upgrade your database to handle the new data types." + elif reason == "old_app": + custom_error = "You're using an old version of HydraVeil, with a newer version of the database. This means the older app would crash trying to handle the newer data." + else: + error_msg = compatability_dict.get("error", "Unknown reason.") + custom_error = f"There was an error with upgrading your database version. Please tell customer support: {error_msg}." + + return custom_error + +# ============================================================================ +# ORCHESTRATOR +# ============================================================================ +def check_database_compatibility(session: Session) -> dict: + """ + Purpose: + Evaluates if the user's existing database is incompatible with the app's version. + + Rank: + Orchestrator + + Called By: + GUI's startup script in __main__ + + Returns: + Dictionary of True/False, with reason + NO error raises. + """ + db_version_you_have = get_database_version(session) + + if db_version_you_have is None: + print("Nothing there") + # First run, corrupted, or empty + return { + "result": True, + "reason": "no_database" # Allow initialization still, since there's nothing to delete. + } + + logger.info(f"[DB MANAGEMENT] The Database version this APP WANTS is {Constants.DB_VERSION_THIS_APP_WANTS}") + logger.info(f"[DB MANAGEMENT] The Database version this device currently has is {db_version_you_have}") + + if Constants.DB_VERSION_THIS_APP_WANTS == db_version_you_have: + logger.info(f"[DB MANAGEMENT] Great, the database versions match exactly.") + return { + "result": True, + "reason": None, + } + + # BAD!! Old version, + elif Constants.DB_VERSION_THIS_APP_WANTS < db_version_you_have: + logger.info(f"[DB MANAGEMENT] The App version is older, this is NOT compatabile") + # make them wipe the tables, or switch versions of the app + return { + "result": False, + "reason": "old_app", + "old_db_version": db_version_you_have + } + + # Upgrade User: + elif Constants.DB_VERSION_THIS_APP_WANTS > db_version_you_have: + logger.info(f"[DB MANAGEMENT] The user has an upgraded App, and an older database") + return { + "result": False, + "reason": "upgrade", + "old_db_version": db_version_you_have + } + + # this is an error + else: + logger.error(f"[DB MANAGEMENT] Unknown Error inside version check module with the King Orchestration") + return { + "result": False, + "reason": "error", + "old_db_version": db_version_you_have, + "error": "unknown result" + } \ No newline at end of file diff --git a/core/models/orm_models/Base.py b/core/models/orm_models/Base.py index f2c489f..9cd85a6 100644 --- a/core/models/orm_models/Base.py +++ b/core/models/orm_models/Base.py @@ -1,9 +1,19 @@ # base.py from sqlalchemy.orm import declarative_base +from sqlalchemy import inspect Base = declarative_base() """ -This will eventually replace the legacy base model, -but is still required, even if empty, to setup the declarative base. +This mapper exists so the children classes +have the ability to print human-readable strings +as dictionaries for the UI. """ + +class BaseModel(Base): + __abstract__ = True + + def __repr__(self): + mapper = inspect(self.__class__) + fields = ', '.join(f'{col.name}={getattr(self, col.name)}' for col in mapper.columns) + return f'{self.__class__.__name__}({fields})' diff --git a/core/models/orm_models/DatabaseVersion.py b/core/models/orm_models/DatabaseVersion.py new file mode 100644 index 0000000..8f85ebc --- /dev/null +++ b/core/models/orm_models/DatabaseVersion.py @@ -0,0 +1,11 @@ +from sqlalchemy import Column, Integer, MetaData, Table + +# Separate metadata for the compatibility table only +compat_metadata = MetaData() + +database_version = Table( + 'database_version', + compat_metadata, + Column('version', Integer, primary_key=True) +) + diff --git a/core/models/orm_models/EncryptedProxy.py b/core/models/orm_models/EncryptedProxy.py new file mode 100644 index 0000000..1851573 --- /dev/null +++ b/core/models/orm_models/EncryptedProxy.py @@ -0,0 +1,40 @@ +from sqlalchemy import Integer, String, ForeignKey +from sqlalchemy.orm import declarative_base, mapped_column +from typing import Optional +from sqlalchemy.orm import Mapped +from core.models.orm_models.Base import Base +from core.models.orm_models.Operator import Operator +from core.models.orm_models.Location import Location + +class EncryptedProxy(Base): + __tablename__ = 'encryptedproxies' + + id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False) + protocol_type: Mapped[str] = mapped_column(String, nullable=False) # changed from name "type" + + username: Mapped[str] = mapped_column(String, nullable=True, default=None) + password: Mapped[str] = mapped_column(String, nullable=False) + + links: Mapped[str] = mapped_column(String, nullable=False) + subscription_url: Mapped[str] = mapped_column(String, nullable=False) + + # Foreign key column + operator_id: Mapped[Optional[int]] = mapped_column( + Integer, + ForeignKey("operators.id"), + nullable=True, + default=None + ) + + # Foreign key column + location_id: Mapped[Optional[int]] = mapped_column( + Integer, + ForeignKey("locations.id"), + nullable=True, + default=None + ) + + operator_domain: Mapped[str] = mapped_column(String, nullable=True, default=None) + operator_hysteria2_host: Mapped[str] = mapped_column(String, nullable=True, default=None) + operator_vless_host: Mapped[str] = mapped_column(String, nullable=True, default=None) + server_ip: Mapped[str] = mapped_column(String, nullable=True, default=None) # pre-resolved IP — avoids DNS leak at connect time diff --git a/core/models/orm_models/Location.py b/core/models/orm_models/Location.py new file mode 100644 index 0000000..85621fb --- /dev/null +++ b/core/models/orm_models/Location.py @@ -0,0 +1,73 @@ + +from sqlalchemy import Integer, String, ForeignKey +from sqlalchemy.orm import declarative_base, mapped_column, Mapped, relationship +from typing import Optional +from sqlalchemy.orm import Mapped +from core.models.orm_models.Base import BaseModel +from core.models.orm_models.Operator import Operator + + +class Location(BaseModel): + __tablename__ = 'locations' + + # model primary key, but only used for business lookups + id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True) + + # BUSINESS logic primary keys: + country_code: Mapped[str] = mapped_column(String, nullable=False) + code: Mapped[str] = mapped_column(String, unique=True, nullable=False) + + # country name: + country_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None) + + # this is CITY name: + name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None) + + time_zone: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None) + + # Foreign key column + operator_id: Mapped[Optional[int]] = mapped_column( + Integer, + ForeignKey("operators.id"), + nullable=True, + default=None + ) + + # Relationship with selectin eager loading + operator: Mapped[Optional["Operator"]] = relationship( + "Operator", + lazy="selectin" + ) + + provider_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None) + available: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + is_proxy_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + is_wireguard_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + is_hysteria2_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + is_vless_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + + def to_dict(self): + return { + "country_code": self.country_code, + "code": self.code, + "time_zone": self.time_zone + } + + +# to use: +# Lookup by location/city codes +# record = session.get(Location, (country_code, code)) + +# example data: +""" +id: 7 +country_code: is +country_name: Iceland +code: 1 +name: Capital Region +time_zone: Atlantic/Reykjavik +operator_id: 6 +provider_name: FlokiNET +is_proxy_capable: 0 +is_wireguard_capable: 1 +""" diff --git a/core/models/orm_models/Operator.py b/core/models/orm_models/Operator.py new file mode 100644 index 0000000..c2d3f19 --- /dev/null +++ b/core/models/orm_models/Operator.py @@ -0,0 +1,18 @@ +from sqlalchemy import Integer, String +from sqlalchemy.orm import declarative_base, mapped_column +from typing import Optional +from sqlalchemy.orm import Mapped +from core.models.orm_models.Base import BaseModel + +class Operator(BaseModel): + __tablename__ = 'operators' + + id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False) + name: Mapped[str] = mapped_column(String, nullable=False) + public_key: Mapped[str] = mapped_column(String, nullable=True, default=None) + nostr_public_key: Mapped[str] = mapped_column(String, nullable=False) + nostr_profile_reference: Mapped[str] = mapped_column(String, nullable=False) + nostr_attestation_event_reference: Mapped[str] = mapped_column(String, nullable=False) + + # legacy: + # operator: Mapped[str] = mapped_column(String, nullable=False) diff --git a/core/services/WebServiceApiService.py b/core/services/WebServiceApiService.py index 266e2e3..70a0b2d 100644 --- a/core/services/WebServiceApiService.py +++ b/core/services/WebServiceApiService.py @@ -1,7 +1,7 @@ from core.Constants import Constants from core.models.ClientVersion import ClientVersion -from core.models.Location import Location -from core.models.Operator import Operator +# from core.models.Location import Location # migrated to ORM +# from core.models.Operator import Operator # migrated to ORM from core.models.OperatorProxySession import OperatorProxySession from core.models.Subscription import Subscription from core.models.SubscriptionPlan import SubscriptionPlan @@ -231,4 +231,4 @@ class WebServiceApiService: if response.status_code == status_codes.OK: return response.json() else: - return None + return None \ No newline at end of file diff --git a/core/services/sync/get_metadata_from_api.py b/core/services/sync/get_data_from_api.py similarity index 84% rename from core/services/sync/get_metadata_from_api.py rename to core/services/sync/get_data_from_api.py index a9944b8..e28ad95 100644 --- a/core/services/sync/get_metadata_from_api.py +++ b/core/services/sync/get_data_from_api.py @@ -9,11 +9,12 @@ from typing import Optional # this can be spoofed with the isolation testing comment below -def get_metadata_from_api( +def get_data_from_api( + endpoint: str, client_observer: Optional[ClientObserver] = None, connection_observer: Optional[ConnectionObserver] = None ) -> dict: - logger.info("Syncing with the API...") + logger.info("Syncing with the API in get_data_from_api...") # Get and validate the base URL rejected_list = [None, False, ""] @@ -24,7 +25,7 @@ def get_metadata_from_api( return {"success": False, "data": None, "error": error_msg} # Construct the full endpoint URL - url = f"{base_url}/cachedsync" + url = f"{base_url}/{endpoint}" logger.debug(f"API endpoint: {url}") try: @@ -43,10 +44,10 @@ def get_metadata_from_api( if final_result: logger.info("Successfully retrieved sync versions metadata from API") return {"success": True, "data": final_result, "error": None} - - error_msg = "API response missing 'data' field" - logger.error(error_msg) - return {"success": False, "data": None, "error": error_msg} + else: + error_msg = f"API response missing 'data' field at the end of get_data_from_api. This is the response from the other module: {sync_results}" + logger.error(error_msg) + return {"success": False, "data": None, "error": error_msg} except Exception as e: error_msg = f"API fetch failed: {str(e)}" diff --git a/core/services/sync/orm_methods/sync_one_orm.py b/core/services/sync/orm_methods/sync_one_orm.py new file mode 100644 index 0000000..5e87f5c --- /dev/null +++ b/core/services/sync/orm_methods/sync_one_orm.py @@ -0,0 +1,60 @@ +from core.services.sync.get_data_from_api import get_data_from_api +from core.models.manage.insert import insert_into_model +from core.models.manage.denormalize import denormalize +from core.models.orm_models.Base import Base +from core.observers.ClientObserver import ClientObserver +from core.observers.ConnectionObserver import ConnectionObserver +from core.errors.logger import logger + +from typing import Optional +import json + +def sync_one_orm_model( + which_model: Base, + which_endpoint: str, + client_observer: Optional[ClientObserver] = None, + connection_observer: Optional[ConnectionObserver] = None +) -> dict: + + api_result = get_data_from_api(which_endpoint, ClientObserver, ConnectionObserver) + + if not api_result.get("success"): + error_msg = api_result.get("error", "Unknown API error") + logger.error(f"API sync failed for {which_endpoint} with: {error_msg}") + return { + "success": False, + "error": error_msg + } + + new_data = api_result.get("data") + + # prep data (denormalize) + yaml_filename = f"{which_endpoint}.yaml" + denormalized_data = denormalize(new_data, yaml_filename) + + if denormalized_data: + # Debug Pretty print with indentation + # print(json.dumps(denormalized_data, indent=2)) + + try: + did_it_work = insert_into_model(which_model, denormalized_data, True) + if did_it_work: + return { + "success": True + } + else: + return { + "success": False, + "error": f"We got the data from the endpoint {which_endpoint}, but failed to insert into the model. That raw data is {denormalized_data}" + } + except: + return { + "success": False, + "error": f"The except triggered on a sync_one_orm_model's failure to insert into the model with the endpoint {which_endpoint}. We got the raw data of {denormalized_data}" + } + + else: + return { + "success": False, + "error": f"We had issues with denormalizing the data from the server. That data was {new_data}." + } diff --git a/core/services/sync/sync_service.py b/core/services/sync/sync_service.py index db40fb2..540f3b2 100644 --- a/core/services/sync/sync_service.py +++ b/core/services/sync/sync_service.py @@ -1,7 +1,7 @@ # comparisons & api calls: from core.services.sync.compare_tables import compare_tables -from core.services.sync.get_metadata_from_api import get_metadata_from_api +from core.services.sync.get_data_from_api import get_data_from_api # ORM for metadata: from core.models.manage.session_management import get_session @@ -147,9 +147,9 @@ def coordinate_cache_sync( - 'filtered_metadata': the metadata from the server after removing keys not in the ORM """ - api_result = get_metadata_from_api(ClientObserver, ConnectionObserver) + api_result = get_data_from_api("cachedsync", ClientObserver, ConnectionObserver) - # we trust the 'get_metadata_from_api' function to give us a dictionary: + # we trust the 'get_data_from_api' function to give us a dictionary: if not api_result.get("success"): error_msg = api_result.get("error", "Unknown API error") logger.error(f"API sync failed: {error_msg}")