From 2aa077a400e54c8ffe337dd487141cf2a7be4903 Mon Sep 17 00:00:00 2001 From: SimplifiedPrivacy Date: Sat, 27 Jun 2026 17:10:28 -0400 Subject: [PATCH] Big Transition: Location & Operator turned into SQLAlchemy models. This forces a change across baseprofile, Locationcontroller, and even GUIs type handling --- assets/yaml_mappings/locations.yaml | 7 +- core/Constants.py | 1 + core/controllers/ClientController.py | 57 +++++---- core/controllers/LocationController.py | 49 ++++++-- core/controllers/OperatorController.py | 22 ---- core/controllers/ProfileController.py | 2 + core/models/BaseProfile.py | 76 ++++++++++-- core/models/Location.py | 115 ------------------ core/models/Operator.py | 56 --------- core/models/manage/denormalize.py | 75 ++++++++++++ core/models/manage/insert.py | 9 +- core/models/manage/session_management.py | 31 ++++- core/models/orm_models/Base.py | 14 ++- core/models/orm_models/EncryptedProxy.py | 40 ++++++ core/models/orm_models/Location.py | 73 +++++++++++ core/models/orm_models/Operator.py | 18 +++ core/services/WebServiceApiService.py | 4 +- ...adata_from_api.py => get_data_from_api.py} | 5 +- .../services/sync/orm_methods/sync_one_orm.py | 60 +++++++++ core/services/sync/sync_service.py | 6 +- 20 files changed, 467 insertions(+), 253 deletions(-) delete mode 100644 core/controllers/OperatorController.py delete mode 100644 core/models/Location.py delete mode 100644 core/models/Operator.py create mode 100644 core/models/manage/denormalize.py create mode 100644 core/models/orm_models/EncryptedProxy.py create mode 100644 core/models/orm_models/Location.py create mode 100644 core/models/orm_models/Operator.py rename core/services/sync/{get_metadata_from_api.py => get_data_from_api.py} (97%) create mode 100644 core/services/sync/orm_methods/sync_one_orm.py diff --git a/assets/yaml_mappings/locations.yaml b/assets/yaml_mappings/locations.yaml index 762af31..97ee248 100644 --- a/assets/yaml_mappings/locations.yaml +++ b/assets/yaml_mappings/locations.yaml @@ -33,9 +33,14 @@ fields: path: ['is_wireguard_capable'] required: true + - name: is_hysteria2_capable + path: ['is_hysteria2_capable'] - # required: true + - name: is_vless_capable + path: ['is_vless_capable'] + +# original version: # locations.append(( # location['country']['code'], # location['code'], diff --git a/core/Constants.py b/core/Constants.py index 26d2815..e7be2ea 100644 --- a/core/Constants.py +++ b/core/Constants.py @@ -5,6 +5,7 @@ import os @dataclass(frozen=True) class Constants: + # ticketing group: TICKET_API_BASE_URL: Final[str] = os.environ.get( "TICKET_API_BASE_URL", "https://ticket.hydraveil.net" diff --git a/core/controllers/ClientController.py b/core/controllers/ClientController.py index bc77c23..42ad26d 100644 --- a/core/controllers/ClientController.py +++ b/core/controllers/ClientController.py @@ -1,7 +1,14 @@ # new sync refactor: from core.models.manage.session_management import init_session, close_session from core.services.sync.sync_service import coordinate_cache_sync, save_metadata +from core.services.sync.orm_methods.sync_one_orm import sync_one_orm_model from core.errors.logger import logger + +# ORM models that can be sync'ed: +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator + + # prior versions: from core.Constants import Constants from core.Errors import UnknownClientPathError, UnknownClientVersionError, CommandNotFoundError @@ -9,8 +16,8 @@ from core.controllers.ApplicationController import ApplicationController from core.controllers.ApplicationVersionController import ApplicationVersionController from core.controllers.ClientVersionController import ClientVersionController from core.controllers.ConfigurationController import ConfigurationController -from core.controllers.LocationController import LocationController -from core.controllers.OperatorController import OperatorController +# from core.controllers.LocationController import LocationController +# from core.controllers.OperatorController import OperatorController from core.controllers.SubscriptionPlanController import SubscriptionPlanController from core.observers.ClientObserver import ClientObserver from core.observers.ConnectionObserver import ConnectionObserver @@ -52,12 +59,9 @@ class ClientController: if client_observer is not None: client_observer.notify('synchronizing', "Fetching list of new data ..") - # Use Cached Method: - init_session() - result = coordinate_cache_sync(ClientObserver, ConnectionObserver) - logger.info(f"We got a Result from the API of {result}") + # logger.info(f"We got a Result from the API of {result}") # Outright Error: if not result["success"]: @@ -78,8 +82,30 @@ class ClientController: # flag for after the save, data_was_saved = False + # Fetch and update the real data (no longer metadata)... + + # =================== ORM BASED MODELS ================== + """ + Note: for the new ORM based models, + it does the Tor/system check in the API call itself. + """ + + if "locations" in changed_tables: + logger.info("Sync of Locations") + if client_observer is not None: + client_observer.notify('synchronizing', 'Fetching Locations List..') + + final_result = sync_one_orm_model(Location, "locations") + + if "operators" in changed_tables: + logger.info("Sync of Operators") + if client_observer is not None: + client_observer.notify('synchronizing', 'Fetching Operators List..') + + final_result_two = sync_one_orm_model(Operator, "operators") + + # =================== MANUAL-SQL BASED MODELS ================== try: - # Fetch and update the real data (no longer metadata)... from core.controllers.ConnectionController import ConnectionController ConnectionController.with_preferred_connection(task=ClientController.__sync, changed_tables=changed_tables, client_observer=client_observer, connection_observer=connection_observer) @@ -98,9 +124,6 @@ class ClientController: filtered_metadata = result["filtered_metadata"] # from the top of the function save_successful = save_metadata(filtered_metadata) # the "save_data" function is inside sync_service - # Regardless of the outcome, - close_session() - if client_observer is None: logger.error("Error: No client_observer to update the UI, the final part of the sync function skipped") return # can't update their UI @@ -151,20 +174,6 @@ class ClientController: # noinspection PyProtectedMember ClientVersionController._sync(proxies=proxies) - if "operators" in changed_tables: - logger.info("Sync of Operators") - if client_observer is not None: - client_observer.notify('synchronizing', 'Fetching Operators List..') - # noinspection PyProtectedMember - OperatorController._sync(proxies=proxies) - - if "locations" in changed_tables: - logger.info("Sync of Locations") - if client_observer is not None: - client_observer.notify('synchronizing', 'Fetching Locations List..') - # noinspection PyProtectedMember - LocationController._sync(proxies=proxies) - if "subscriptions" in changed_tables: logger.info("Sync of Subscriptions") if client_observer is not None: diff --git a/core/controllers/LocationController.py b/core/controllers/LocationController.py index c9d84b2..627d1b7 100644 --- a/core/controllers/LocationController.py +++ b/core/controllers/LocationController.py @@ -1,22 +1,49 @@ -from core.models.Location import Location -from core.services.WebServiceApiService import WebServiceApiService -from typing import Optional +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator +from core.models.manage.session_management import get_session +from typing import Optional +from sqlalchemy import select +from sqlalchemy.orm import joinedload class LocationController: @staticmethod - def get(country_code: str, code: str): - return Location.find(country_code, code) + def get(country_code: str, city_code: str): + with get_session() as session: + location_object = session.execute( + select(Location) + .where((Location.country_code == country_code) & (Location.code == city_code)) + .options(joinedload(Location.operator)) + ).scalar_one_or_none() + + return location_object + + # legacy: + # Location.find(country_code, code) + + @staticmethod def get_all(): - return Location.all() + with get_session() as session: + all_records = session.execute( + select(Location) + .options(joinedload(Location.operator)) + ).scalars().all() + return all_records - @staticmethod - def _sync(proxies: Optional[dict] = None): + # legacy: + # return Location.all() - locations = WebServiceApiService.get_locations(proxies) - Location.truncate() - Location.save_many(locations) +# Deprecated legacy sync, + +# from core.services.WebServiceApiService import WebServiceApiService + # @staticmethod + # def _sync(proxies: Optional[dict] = None): + + # locations = WebServiceApiService.get_locations(proxies) + + # Location.truncate() + # Location.save_many(locations) diff --git a/core/controllers/OperatorController.py b/core/controllers/OperatorController.py deleted file mode 100644 index 9d54c4a..0000000 --- a/core/controllers/OperatorController.py +++ /dev/null @@ -1,22 +0,0 @@ -from core.models.Operator import Operator -from core.services.WebServiceApiService import WebServiceApiService -from typing import Optional - - -class OperatorController: - - @staticmethod - def get(id: int): - return Operator.find_by_id(id) - - @staticmethod - def get_all(): - return Operator.all() - - @staticmethod - def _sync(proxies: Optional[dict] = None): - - operators = WebServiceApiService.get_operators(proxies) - - Operator.truncate() - Operator.save_many(operators) diff --git a/core/controllers/ProfileController.py b/core/controllers/ProfileController.py index ffdad9c..5742a74 100644 --- a/core/controllers/ProfileController.py +++ b/core/controllers/ProfileController.py @@ -30,7 +30,9 @@ class ProfileController: @staticmethod def create(profile: Union[SessionProfile, SystemProfile], profile_observer: ProfileObserver = None): + print("inside profile controller about to save") profile.save() + print("inside profile controller finished save") if profile_observer is not None: profile_observer.notify('created', profile) diff --git a/core/models/BaseProfile.py b/core/models/BaseProfile.py index 1add011..54e5c98 100644 --- a/core/models/BaseProfile.py +++ b/core/models/BaseProfile.py @@ -1,11 +1,21 @@ +from core.models.manage.session_management import get_session +from sqlalchemy import select +from sqlalchemy.orm import joinedload + from abc import ABC, abstractmethod from core.Constants import Constants from core.Helpers import write_atomically -from core.models.Location import Location + +# from core.models.Location import Location +from core.controllers.LocationController import LocationController +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator + from core.models.Subscription import Subscription from core.models.session.ApplicationVersion import ApplicationVersion from dataclasses import dataclass, field, asdict from dataclasses_json import config, Exclude, dataclass_json + from pathlib import Path from typing import Optional, Self import json @@ -23,7 +33,10 @@ class BaseProfile(ABC): ) name: str subscription: Optional[Subscription] - location: Optional[Location] + location: Optional[Location] = field(metadata=config(exclude=Exclude.ALWAYS)) # SQLAlchemy object + + # legacy version included it to be serialized, but now it's an SQLAlchemy object. + # location: Optional[Location] @abstractmethod def get_wireguard_configuration_path(self): @@ -52,8 +65,27 @@ class BaseProfile(ABC): return type(self).__name__ == 'SystemProfile' def save(self: Self): + print("save has been called") + + config_dict = self.to_dict() # Get dict, not JSON string + print(f"got past the to_dict, config_dict: {config_dict}") - config_file_contents = f'{self.to_json(indent=4)}\n' + print("making a location dict") + location_dict = self.location.to_dict() + print(f"got a location dict {location_dict}") + + if self.location: + print("if self.location is true..") + config_dict["location"] = location_dict + + print(f"dumping into config now {config_dict}") + + config_file_contents = json.dumps(config_dict, indent=4) + '\n' + + print(f"config_file_contents: {config_file_contents}") + + # legacy: + # config_file_contents = f'{self.to_json(indent=4)}\n' os.makedirs(self.get_config_path(), exist_ok=True) os.makedirs(self.get_data_path(), exist_ok=True) @@ -146,6 +178,7 @@ class BaseProfile(ABC): return list([key for key, value in asdict(self).items() if value != asdict(reference).get(key)]) + @staticmethod def find_by_id(id: int): @@ -161,16 +194,43 @@ class BaseProfile(ABC): profile['id'] = id + profiles_location = profile['location'] + if profile['location'] is not None: + if isinstance(profiles_location, dict): + try: + country_code = profile['location']['country_code'] + city_code = profile['location']['code'] + except: + print("ERROR! CANT FIND country code or city") + return + else: + # potentially coming from SQLAlchemy: + country_code = profile.location.country_code + city_code = profile.location.code - location = Location.find(profile['location']['country_code'] or None, profile['location']['code'] or None) + with get_session() as session: + location_object = session.execute( + select(Location) + .where((Location.country_code == country_code) & (Location.code == city_code)) + .options(joinedload(Location.operator)) + ).scalar_one_or_none() - if location is not None: - if profile['location'].get('time_zone') is not None: - location.time_zone = profile['location']['time_zone'] + if location_object: + profile['location'] = location_object - profile['location'] = location + # this needs error handling if there's no location or malconformed config. + + + # legacy phased out since SQLAlchemy handles the time_zone. + + # if location is not None: + + # if profile['location'].get('time_zone') is not None: + # location.time_zone = profile['location']['time_zone'] + + # profile['location'] = location if 'application_version' in profile: diff --git a/core/models/Location.py b/core/models/Location.py deleted file mode 100644 index 62c963b..0000000 --- a/core/models/Location.py +++ /dev/null @@ -1,115 +0,0 @@ -from core.models.Model import Model -from core.models.Operator import Operator -from dataclasses import dataclass, field -from dataclasses_json import config, Exclude -from typing import Optional - -_table_name: str = 'locations' - -_table_definition: str = """ - 'id' int UNIQUE, - 'country_code' varchar, - 'country_name' varchar, - 'code' varchar, - 'name' varchar, - 'time_zone' varchar, - 'operator_id' int, - 'provider_name' varchar, - 'is_proxy_capable' bool, - 'is_wireguard_capable' bool, - UNIQUE(code, country_code) -""" - - -@dataclass -class Location(Model): - country_code: str - code: str - id: Optional[int] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - country_name: Optional[str] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - name: Optional[str] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - time_zone: Optional[str] = None - operator_id: Optional[int] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - provider_name: Optional[str] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - is_proxy_capable: Optional[bool] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - is_wireguard_capable: Optional[bool] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - operator: Optional[Operator] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - available: Optional[bool] = field( - default=False, - metadata=config(exclude=Exclude.ALWAYS) - ) - - def __post_init__(self): - self.operator = Operator.find_by_id(self.operator_id) - self.available = self.exists(self.country_code, self.code) - - if isinstance(self.is_proxy_capable, int): - self.is_proxy_capable = bool(self.is_proxy_capable) - - if isinstance(self.is_wireguard_capable, int): - self.is_wireguard_capable = bool(self.is_wireguard_capable) - - def is_available(self): - return self.exists(self.country_code, self.code) - - @staticmethod - def find_by_id(id: int): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_one('SELECT * FROM locations WHERE id = ? LIMIT 1', Location.factory, [id]) - - @staticmethod - def find(country_code: str, code: str): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_one('SELECT * FROM locations WHERE country_code = ? AND code = ? LIMIT 1', Location.factory, [country_code, code]) - - @staticmethod - def exists(country_code: str, code: str): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_exists('SELECT * FROM locations WHERE country_code = ? AND code = ?', [country_code, code]) - - @staticmethod - def all(): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_all('SELECT * FROM locations', Location.factory) - - @staticmethod - def truncate(): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True) - - @staticmethod - def save_many(locations): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - Model._insert_many('INSERT INTO locations VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', Location.tuple_factory, locations) - - @staticmethod - def factory(cursor, row): - local_fields = [column[0] for column in cursor.description] - return Location(**{key: value for key, value in zip(local_fields, row)}) - - @staticmethod - def tuple_factory(location): - return location.id, location.country_code, location.country_name, location.code, location.name, location.time_zone, location.operator_id, location.provider_name, location.is_proxy_capable, location.is_wireguard_capable diff --git a/core/models/Operator.py b/core/models/Operator.py deleted file mode 100644 index 8ad7856..0000000 --- a/core/models/Operator.py +++ /dev/null @@ -1,56 +0,0 @@ -from core.models.Model import Model -from dataclasses import dataclass - -_table_name: str = 'operators' - -_table_definition: str = """ - 'id' int UNIQUE, - 'name' varchar, - 'public_key' varchar, - 'nostr_public_key' varchar, - 'nostr_profile_reference' varchar, - 'nostr_attestation_event_reference' varchar -""" - - -@dataclass -class Operator(Model): - id: int - name: str - public_key: str - nostr_public_key: str - nostr_profile_reference: str - nostr_attestation_event_reference: str - - @staticmethod - def find_by_id(id: int): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_one('SELECT * FROM operators WHERE id = ? LIMIT 1', Operator.factory, [id]) - - @staticmethod - def exists(id: int): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_exists('SELECT * FROM operators WHERE id = ?', [id]) - - @staticmethod - def all(): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_all('SELECT * FROM operators', Operator.factory) - - @staticmethod - def truncate(): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True) - - @staticmethod - def save_many(operators): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - Model._insert_many('INSERT INTO operators VALUES(?, ?, ?, ?, ?, ?)', Operator.tuple_factory, operators) - - @staticmethod - def factory(cursor, row): - local_fields = [column[0] for column in cursor.description] - return Operator(**{key: value for key, value in zip(local_fields, row)}) - - @staticmethod - def tuple_factory(operator): - return operator.id, operator.name, operator.public_key, operator.nostr_public_key, operator.nostr_profile_reference, operator.nostr_attestation_event_reference diff --git a/core/models/manage/denormalize.py b/core/models/manage/denormalize.py new file mode 100644 index 0000000..795ac9c --- /dev/null +++ b/core/models/manage/denormalize.py @@ -0,0 +1,75 @@ +import yaml + +""" +This function extracts the value from a nested dictionary, +By looping through each layer. And comparing the current dictionary's key to pre-made YAML key mappings. +""" +def extract_from_nested(obj, yaml_keys, default=None): + # loop through each key from the YAML mapping: + for each_yaml_key in yaml_keys: + + # Check if each layer down of the 'object' is a dictionary with that key, + if isinstance(obj, dict) and each_yaml_key in obj: + # then extract the value for that key, and replace the object placeholder itself with it. + obj = obj[each_yaml_key] + + # this has the effect of going deeper into the nesting on the next round of the loop, + + # But if it's not a dictionary or doesn't have the key, + else: + # then we hit a dead end, + return default + + # We finished looping through all the keys of valid dictionaries, and extracted the values, + return obj + +""" +This function removes unnecessary nesting, +by comparing it to a pre-made YAML mapping +""" +def denormalize(data, which_yaml_file): + final_results = [] + + # prep the mapping config: + mapping_file = f"assets/yaml_mappings/{which_yaml_file}" + with open(mapping_file) as f: + mapping = yaml.safe_load(f) + + # Extract out the 'data' variable if it exists, otherwise just use the data directly + extracted_data = data.get('data', data) + + # go through the data + for each_item in extracted_data: + + # setup temp flags/containers + each_denormalized_piece = {} + valid = True + + # go through mapping: + for field in mapping['fields']: + """ + Extract the values based on the YAML mapping + field['path'] - Gets a Python list of the keys from the YAML mapping + extract_from_nested - This function checks it's actually a dictionary and extracts it + """ + value = extract_from_nested(each_item, field['path']) + + # Store the extracted value. If it wasn't there, then it's None + each_denormalized_piece[field['name']] = value + + # if it's required & not there, ditch it, + if field.get('required') and value is None: + print(f"It's required and value is none, skipping {value}") + valid = False + break + + # Only add items where all required fields were present + if valid: + final_results.append(each_denormalized_piece) + + # Return all successfully denormalized items + return final_results + + +# Isolated testing: +# items = denormalize(response.json(), 'example.yaml') diff --git a/core/models/manage/insert.py b/core/models/manage/insert.py index 8b3520e..9616445 100644 --- a/core/models/manage/insert.py +++ b/core/models/manage/insert.py @@ -3,9 +3,14 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy import create_engine from typing import Type, Dict, Any from sqlalchemy.exc import IntegrityError, SQLAlchemyError -from core.models.orm_models.Base import Base -from core.models.orm_models.CachedSync import CachedSync from core.models.manage.session_management import get_session +from core.models.orm_models.Base import Base + +# all models it knows how to do: +from core.models.orm_models.CachedSync import CachedSync +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator +from core.models.orm_models.EncryptedProxy import EncryptedProxy def insert_into_model(model_class: Type, all_data: dict | list, override=False) -> bool: """ diff --git a/core/models/manage/session_management.py b/core/models/manage/session_management.py index 9f9d226..98c4406 100644 --- a/core/models/manage/session_management.py +++ b/core/models/manage/session_management.py @@ -1,10 +1,31 @@ from sqlalchemy.orm import sessionmaker -from sqlalchemy import create_engine -from core.models.orm_models.Base import Base -from core.Constants import Constants +from sqlalchemy import create_engine, inspect +from core.models.orm_models.Base import BaseModel +# from core.Constants import Constants +import os +from pathlib import Path -database_path = Constants.HV_STORAGE_DATABASE_PATH +def get_path(): + # XDG Base Directory Specification + xdg_data_home = os.environ.get("XDG_DATA_HOME") + if xdg_data_home: + return Path(xdg_data_home) / "my-app" + return Path.home() / ".local" / "share" / "hydra-veil" + +system_path = get_path() +database_path = f"{system_path}/storage.db" + +# Ensure the database directory exists +db_dir = Path(database_path).parent +db_dir.mkdir(parents=True, exist_ok=True) + +# Create the engine engine = create_engine(f"sqlite:///{database_path}") + +# Create all tables +BaseModel.metadata.create_all(engine) + +# Create session Session = sessionmaker(bind=engine) _session = None @@ -18,7 +39,7 @@ def init_session(): while keeping it a globally accessible variable """ global _session - Base.metadata.create_all(engine) + BaseModel.metadata.create_all(engine) _session = Session() def get_session(): diff --git a/core/models/orm_models/Base.py b/core/models/orm_models/Base.py index f2c489f..9cd85a6 100644 --- a/core/models/orm_models/Base.py +++ b/core/models/orm_models/Base.py @@ -1,9 +1,19 @@ # base.py from sqlalchemy.orm import declarative_base +from sqlalchemy import inspect Base = declarative_base() """ -This will eventually replace the legacy base model, -but is still required, even if empty, to setup the declarative base. +This mapper exists so the children classes +have the ability to print human-readable strings +as dictionaries for the UI. """ + +class BaseModel(Base): + __abstract__ = True + + def __repr__(self): + mapper = inspect(self.__class__) + fields = ', '.join(f'{col.name}={getattr(self, col.name)}' for col in mapper.columns) + return f'{self.__class__.__name__}({fields})' diff --git a/core/models/orm_models/EncryptedProxy.py b/core/models/orm_models/EncryptedProxy.py new file mode 100644 index 0000000..1851573 --- /dev/null +++ b/core/models/orm_models/EncryptedProxy.py @@ -0,0 +1,40 @@ +from sqlalchemy import Integer, String, ForeignKey +from sqlalchemy.orm import declarative_base, mapped_column +from typing import Optional +from sqlalchemy.orm import Mapped +from core.models.orm_models.Base import Base +from core.models.orm_models.Operator import Operator +from core.models.orm_models.Location import Location + +class EncryptedProxy(Base): + __tablename__ = 'encryptedproxies' + + id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False) + protocol_type: Mapped[str] = mapped_column(String, nullable=False) # changed from name "type" + + username: Mapped[str] = mapped_column(String, nullable=True, default=None) + password: Mapped[str] = mapped_column(String, nullable=False) + + links: Mapped[str] = mapped_column(String, nullable=False) + subscription_url: Mapped[str] = mapped_column(String, nullable=False) + + # Foreign key column + operator_id: Mapped[Optional[int]] = mapped_column( + Integer, + ForeignKey("operators.id"), + nullable=True, + default=None + ) + + # Foreign key column + location_id: Mapped[Optional[int]] = mapped_column( + Integer, + ForeignKey("locations.id"), + nullable=True, + default=None + ) + + operator_domain: Mapped[str] = mapped_column(String, nullable=True, default=None) + operator_hysteria2_host: Mapped[str] = mapped_column(String, nullable=True, default=None) + operator_vless_host: Mapped[str] = mapped_column(String, nullable=True, default=None) + server_ip: Mapped[str] = mapped_column(String, nullable=True, default=None) # pre-resolved IP — avoids DNS leak at connect time diff --git a/core/models/orm_models/Location.py b/core/models/orm_models/Location.py new file mode 100644 index 0000000..85621fb --- /dev/null +++ b/core/models/orm_models/Location.py @@ -0,0 +1,73 @@ + +from sqlalchemy import Integer, String, ForeignKey +from sqlalchemy.orm import declarative_base, mapped_column, Mapped, relationship +from typing import Optional +from sqlalchemy.orm import Mapped +from core.models.orm_models.Base import BaseModel +from core.models.orm_models.Operator import Operator + + +class Location(BaseModel): + __tablename__ = 'locations' + + # model primary key, but only used for business lookups + id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True) + + # BUSINESS logic primary keys: + country_code: Mapped[str] = mapped_column(String, nullable=False) + code: Mapped[str] = mapped_column(String, unique=True, nullable=False) + + # country name: + country_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None) + + # this is CITY name: + name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None) + + time_zone: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None) + + # Foreign key column + operator_id: Mapped[Optional[int]] = mapped_column( + Integer, + ForeignKey("operators.id"), + nullable=True, + default=None + ) + + # Relationship with selectin eager loading + operator: Mapped[Optional["Operator"]] = relationship( + "Operator", + lazy="selectin" + ) + + provider_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None) + available: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + is_proxy_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + is_wireguard_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + is_hysteria2_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + is_vless_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None) + + def to_dict(self): + return { + "country_code": self.country_code, + "code": self.code, + "time_zone": self.time_zone + } + + +# to use: +# Lookup by location/city codes +# record = session.get(Location, (country_code, code)) + +# example data: +""" +id: 7 +country_code: is +country_name: Iceland +code: 1 +name: Capital Region +time_zone: Atlantic/Reykjavik +operator_id: 6 +provider_name: FlokiNET +is_proxy_capable: 0 +is_wireguard_capable: 1 +""" diff --git a/core/models/orm_models/Operator.py b/core/models/orm_models/Operator.py new file mode 100644 index 0000000..c2d3f19 --- /dev/null +++ b/core/models/orm_models/Operator.py @@ -0,0 +1,18 @@ +from sqlalchemy import Integer, String +from sqlalchemy.orm import declarative_base, mapped_column +from typing import Optional +from sqlalchemy.orm import Mapped +from core.models.orm_models.Base import BaseModel + +class Operator(BaseModel): + __tablename__ = 'operators' + + id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False) + name: Mapped[str] = mapped_column(String, nullable=False) + public_key: Mapped[str] = mapped_column(String, nullable=True, default=None) + nostr_public_key: Mapped[str] = mapped_column(String, nullable=False) + nostr_profile_reference: Mapped[str] = mapped_column(String, nullable=False) + nostr_attestation_event_reference: Mapped[str] = mapped_column(String, nullable=False) + + # legacy: + # operator: Mapped[str] = mapped_column(String, nullable=False) diff --git a/core/services/WebServiceApiService.py b/core/services/WebServiceApiService.py index 55ea9ad..553f6c3 100644 --- a/core/services/WebServiceApiService.py +++ b/core/services/WebServiceApiService.py @@ -1,7 +1,7 @@ from core.Constants import Constants from core.models.ClientVersion import ClientVersion -from core.models.Location import Location -from core.models.Operator import Operator +# from core.models.Location import Location +# from core.models.Operator import Operator from core.models.Subscription import Subscription from core.models.SubscriptionPlan import SubscriptionPlan from core.models.invoice.Invoice import Invoice diff --git a/core/services/sync/get_metadata_from_api.py b/core/services/sync/get_data_from_api.py similarity index 97% rename from core/services/sync/get_metadata_from_api.py rename to core/services/sync/get_data_from_api.py index a9944b8..38f1244 100644 --- a/core/services/sync/get_metadata_from_api.py +++ b/core/services/sync/get_data_from_api.py @@ -9,7 +9,8 @@ from typing import Optional # this can be spoofed with the isolation testing comment below -def get_metadata_from_api( +def get_data_from_api( + endpoint: str, client_observer: Optional[ClientObserver] = None, connection_observer: Optional[ConnectionObserver] = None ) -> dict: @@ -24,7 +25,7 @@ def get_metadata_from_api( return {"success": False, "data": None, "error": error_msg} # Construct the full endpoint URL - url = f"{base_url}/cachedsync" + url = f"{base_url}/{endpoint}" logger.debug(f"API endpoint: {url}") try: diff --git a/core/services/sync/orm_methods/sync_one_orm.py b/core/services/sync/orm_methods/sync_one_orm.py new file mode 100644 index 0000000..5e87f5c --- /dev/null +++ b/core/services/sync/orm_methods/sync_one_orm.py @@ -0,0 +1,60 @@ +from core.services.sync.get_data_from_api import get_data_from_api +from core.models.manage.insert import insert_into_model +from core.models.manage.denormalize import denormalize +from core.models.orm_models.Base import Base +from core.observers.ClientObserver import ClientObserver +from core.observers.ConnectionObserver import ConnectionObserver +from core.errors.logger import logger + +from typing import Optional +import json + +def sync_one_orm_model( + which_model: Base, + which_endpoint: str, + client_observer: Optional[ClientObserver] = None, + connection_observer: Optional[ConnectionObserver] = None +) -> dict: + + api_result = get_data_from_api(which_endpoint, ClientObserver, ConnectionObserver) + + if not api_result.get("success"): + error_msg = api_result.get("error", "Unknown API error") + logger.error(f"API sync failed for {which_endpoint} with: {error_msg}") + return { + "success": False, + "error": error_msg + } + + new_data = api_result.get("data") + + # prep data (denormalize) + yaml_filename = f"{which_endpoint}.yaml" + denormalized_data = denormalize(new_data, yaml_filename) + + if denormalized_data: + # Debug Pretty print with indentation + # print(json.dumps(denormalized_data, indent=2)) + + try: + did_it_work = insert_into_model(which_model, denormalized_data, True) + if did_it_work: + return { + "success": True + } + else: + return { + "success": False, + "error": f"We got the data from the endpoint {which_endpoint}, but failed to insert into the model. That raw data is {denormalized_data}" + } + except: + return { + "success": False, + "error": f"The except triggered on a sync_one_orm_model's failure to insert into the model with the endpoint {which_endpoint}. We got the raw data of {denormalized_data}" + } + + else: + return { + "success": False, + "error": f"We had issues with denormalizing the data from the server. That data was {new_data}." + } diff --git a/core/services/sync/sync_service.py b/core/services/sync/sync_service.py index db40fb2..540f3b2 100644 --- a/core/services/sync/sync_service.py +++ b/core/services/sync/sync_service.py @@ -1,7 +1,7 @@ # comparisons & api calls: from core.services.sync.compare_tables import compare_tables -from core.services.sync.get_metadata_from_api import get_metadata_from_api +from core.services.sync.get_data_from_api import get_data_from_api # ORM for metadata: from core.models.manage.session_management import get_session @@ -147,9 +147,9 @@ def coordinate_cache_sync( - 'filtered_metadata': the metadata from the server after removing keys not in the ORM """ - api_result = get_metadata_from_api(ClientObserver, ConnectionObserver) + api_result = get_data_from_api("cachedsync", ClientObserver, ConnectionObserver) - # we trust the 'get_metadata_from_api' function to give us a dictionary: + # we trust the 'get_data_from_api' function to give us a dictionary: if not api_result.get("success"): error_msg = api_result.get("error", "Unknown API error") logger.error(f"API sync failed: {error_msg}")