Big Transition: Location & Operator turned into SQLAlchemy models. This forces a change across baseprofile, Locationcontroller, and even GUIs type handling

assets/yaml_mappings/locations.yaml

@@ -33,9 +33,14 @@ fields:
     path: ['is_wireguard_capable']
     required: true
 
+  - name: is_hysteria2_capable
+    path: ['is_hysteria2_capable']
 
-  #     required: true
+  - name: is_vless_capable
+    path: ['is_vless_capable']
 
+
+# original version:
   # locations.append((
   # location['country']['code'],
   # location['code'],

core/Constants.py

@@ -5,6 +5,7 @@ import os
 
 @dataclass(frozen=True)
 class Constants:
+
     # ticketing group:
     TICKET_API_BASE_URL: Final[str] = os.environ.get(
         "TICKET_API_BASE_URL", "https://ticket.hydraveil.net"

core/controllers/ClientController.py

@@ -1,7 +1,14 @@
 # new sync refactor:
 from core.models.manage.session_management import init_session, close_session
 from core.services.sync.sync_service import coordinate_cache_sync, save_metadata
+from core.services.sync.orm_methods.sync_one_orm import sync_one_orm_model
 from core.errors.logger import logger
+
+# ORM models that can be sync'ed:
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+
+
 # prior versions:
 from core.Constants import Constants
 from core.Errors import UnknownClientPathError, UnknownClientVersionError, CommandNotFoundError
@@ -9,8 +16,8 @@ from core.controllers.ApplicationController import ApplicationController
 from core.controllers.ApplicationVersionController import ApplicationVersionController
 from core.controllers.ClientVersionController import ClientVersionController
 from core.controllers.ConfigurationController import ConfigurationController
-from core.controllers.LocationController import LocationController
-from core.controllers.OperatorController import OperatorController
+# from core.controllers.LocationController import LocationController
+# from core.controllers.OperatorController import OperatorController
 from core.controllers.SubscriptionPlanController import SubscriptionPlanController
 from core.observers.ClientObserver import ClientObserver
 from core.observers.ConnectionObserver import ConnectionObserver
@@ -52,12 +59,9 @@ class ClientController:
         if client_observer is not None:
             client_observer.notify('synchronizing', "Fetching list of new data ..")
 
-        # Use Cached Method:
-        init_session()
-
         result = coordinate_cache_sync(ClientObserver, ConnectionObserver)
 
-        logger.info(f"We got a Result from the API of {result}")
+        # logger.info(f"We got a Result from the API of {result}")
 
         # Outright Error:
         if not result["success"]:
@@ -78,8 +82,30 @@ class ClientController:
         # flag for after the save,
         data_was_saved = False
 
+        # Fetch and update the real data (no longer metadata)...
+
+        # =================== ORM BASED MODELS ==================
+        """
+        Note: for the new ORM based models,
+        it does the Tor/system check in the API call itself.
+        """
+
+        if "locations" in changed_tables:
+            logger.info("Sync of Locations")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Locations List..')
+
+            final_result = sync_one_orm_model(Location, "locations")
+
+        if "operators" in changed_tables:
+            logger.info("Sync of Operators")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Operators List..')
+
+            final_result_two = sync_one_orm_model(Operator, "operators")
+
+        # =================== MANUAL-SQL BASED MODELS ==================
         try:
-            # Fetch and update the real data (no longer metadata)...
             from core.controllers.ConnectionController import ConnectionController
             ConnectionController.with_preferred_connection(task=ClientController.__sync, changed_tables=changed_tables, client_observer=client_observer, connection_observer=connection_observer)
 
@@ -98,9 +124,6 @@ class ClientController:
                 filtered_metadata = result["filtered_metadata"] # from the top of the function
                 save_successful = save_metadata(filtered_metadata) # the "save_data" function is inside sync_service
 
-                # Regardless of the outcome,
-                close_session()
-
                 if client_observer is None:
                     logger.error("Error: No client_observer to update the UI, the final part of the sync function skipped")
                     return # can't update their UI
@@ -151,20 +174,6 @@ class ClientController:
             # noinspection PyProtectedMember
             ClientVersionController._sync(proxies=proxies)
 
-        if "operators" in changed_tables:
-            logger.info("Sync of Operators")
-            if client_observer is not None:
-                client_observer.notify('synchronizing', 'Fetching Operators List..')
-            # noinspection PyProtectedMember
-            OperatorController._sync(proxies=proxies)
-
-        if "locations" in changed_tables:
-            logger.info("Sync of Locations")
-            if client_observer is not None:
-                client_observer.notify('synchronizing', 'Fetching Locations List..')
-            # noinspection PyProtectedMember
-            LocationController._sync(proxies=proxies)
-
         if "subscriptions" in changed_tables:
             logger.info("Sync of Subscriptions")
             if client_observer is not None:

core/controllers/LocationController.py

@@ -1,22 +1,49 @@
-from core.models.Location import Location
-from core.services.WebServiceApiService import WebServiceApiService
-from typing import Optional
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
 
+from core.models.manage.session_management import get_session
+from typing import Optional
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
 
 class LocationController:
 
     @staticmethod
-    def get(country_code: str, code: str):
-        return Location.find(country_code, code)
+    def get(country_code: str, city_code: str):
+        with get_session() as session:
+            location_object = session.execute(
+                select(Location)
+                .where((Location.country_code == country_code) & (Location.code == city_code))
+                .options(joinedload(Location.operator))
+            ).scalar_one_or_none()
+
+        return location_object
+
+    # legacy:
+    # Location.find(country_code, code)
+
+            
 
     @staticmethod
     def get_all():
-        return Location.all()
+        with get_session() as session:
+            all_records = session.execute(
+                select(Location)
+                .options(joinedload(Location.operator))
+            ).scalars().all()
+            return all_records
 
-    @staticmethod
-    def _sync(proxies: Optional[dict] = None):
+    # legacy:
+    # return Location.all()
 
-        locations = WebServiceApiService.get_locations(proxies)
 
-        Location.truncate()
-        Location.save_many(locations)
+# Deprecated legacy sync,
+
+# from core.services.WebServiceApiService import WebServiceApiService
+    # @staticmethod
+    # def _sync(proxies: Optional[dict] = None):
+
+    #     locations = WebServiceApiService.get_locations(proxies)
+
+    #     Location.truncate()
+    #     Location.save_many(locations)

core/controllers/OperatorController.py

@@ -1,22 +0,0 @@
-from core.models.Operator import Operator
-from core.services.WebServiceApiService import WebServiceApiService
-from typing import Optional
-
-
-class OperatorController:
-
-    @staticmethod
-    def get(id: int):
-        return Operator.find_by_id(id)
-
-    @staticmethod
-    def get_all():
-        return Operator.all()
-
-    @staticmethod
-    def _sync(proxies: Optional[dict] = None):
-
-        operators = WebServiceApiService.get_operators(proxies)
-
-        Operator.truncate()
-        Operator.save_many(operators)

core/controllers/ProfileController.py

@@ -30,7 +30,9 @@ class ProfileController:
     @staticmethod
     def create(profile: Union[SessionProfile, SystemProfile], profile_observer: ProfileObserver = None):
 
+        print("inside profile controller about to save")
         profile.save()
+        print("inside profile controller finished save")
 
         if profile_observer is not None:
             profile_observer.notify('created', profile)

core/models/BaseProfile.py

@@ -1,11 +1,21 @@
+from core.models.manage.session_management import get_session
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
+
 from abc import ABC, abstractmethod
 from core.Constants import Constants
 from core.Helpers import write_atomically
-from core.models.Location import Location
+
+# from core.models.Location import Location
+from core.controllers.LocationController import LocationController
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+
 from core.models.Subscription import Subscription
 from core.models.session.ApplicationVersion import ApplicationVersion
 from dataclasses import dataclass, field, asdict
 from dataclasses_json import config, Exclude, dataclass_json
+
 from pathlib import Path
 from typing import Optional, Self
 import json
@@ -23,7 +33,10 @@ class BaseProfile(ABC):
     )
     name: str
     subscription: Optional[Subscription]
-    location: Optional[Location]
+    location: Optional[Location] = field(metadata=config(exclude=Exclude.ALWAYS))  # SQLAlchemy object
+
+    # legacy version included it to be serialized, but now it's an SQLAlchemy object.
+    # location: Optional[Location]
 
     @abstractmethod
     def get_wireguard_configuration_path(self):
@@ -52,8 +65,27 @@ class BaseProfile(ABC):
         return type(self).__name__ == 'SystemProfile'
 
     def save(self: Self):
+        print("save has been called")
+        
+        config_dict = self.to_dict()  # Get dict, not JSON string
+        print(f"got past the to_dict, config_dict: {config_dict}")
 
-        config_file_contents = f'{self.to_json(indent=4)}\n'
+        print("making a location dict")
+        location_dict = self.location.to_dict()
+        print(f"got a location dict {location_dict}")
+
+        if self.location:
+            print("if self.location is true..")
+            config_dict["location"] = location_dict
+
+        print(f"dumping into config now {config_dict}")
+
+        config_file_contents = json.dumps(config_dict, indent=4) + '\n'
+
+        print(f"config_file_contents: {config_file_contents}")
+
+        # legacy:
+        # config_file_contents = f'{self.to_json(indent=4)}\n'
 
         os.makedirs(self.get_config_path(), exist_ok=True)
         os.makedirs(self.get_data_path(), exist_ok=True)
@@ -146,6 +178,7 @@ class BaseProfile(ABC):
 
         return list([key for key, value in asdict(self).items() if value != asdict(reference).get(key)])
 
+
     @staticmethod
     def find_by_id(id: int):
 
@@ -161,16 +194,43 @@ class BaseProfile(ABC):
 
         profile['id'] = id
 
+        profiles_location = profile['location']
+
         if profile['location'] is not None:
+            if isinstance(profiles_location, dict):
+                try:
+                    country_code = profile['location']['country_code']
+                    city_code = profile['location']['code']
+                except:
+                    print("ERROR! CANT FIND country code or city")
+                    return
+            else:
+                # potentially coming from SQLAlchemy:
+                country_code = profile.location.country_code
+                city_code = profile.location.code
 
-            location = Location.find(profile['location']['country_code'] or None, profile['location']['code'] or None)
+            with get_session() as session:
+                location_object = session.execute(
+                    select(Location)
+                    .where((Location.country_code == country_code) & (Location.code == city_code))
+                    .options(joinedload(Location.operator))
+                ).scalar_one_or_none()
 
-            if location is not None:
 
-                if profile['location'].get('time_zone') is not None:
-                    location.time_zone = profile['location']['time_zone']
+                if location_object:
+                    profile['location'] = location_object
 
-                profile['location'] = location
+            # this needs error handling if there's no location or malconformed config.
+
+
+            # legacy phased out since SQLAlchemy handles the time_zone.
+
+            # if location is not None:
+
+            #     if profile['location'].get('time_zone') is not None:
+            #         location.time_zone = profile['location']['time_zone']
+
+            #     profile['location'] = location
 
         if 'application_version' in profile:
 

core/models/Location.py

@@ -1,115 +0,0 @@
-from core.models.Model import Model
-from core.models.Operator import Operator
-from dataclasses import dataclass, field
-from dataclasses_json import config, Exclude
-from typing import Optional
-
-_table_name: str = 'locations'
-
-_table_definition: str = """
-    'id' int UNIQUE,
-    'country_code' varchar,
-    'country_name' varchar,
-    'code' varchar,
-    'name' varchar,
-    'time_zone' varchar,
-    'operator_id' int,
-    'provider_name' varchar,
-    'is_proxy_capable' bool,
-    'is_wireguard_capable' bool,
-    UNIQUE(code, country_code)
-"""
-
-
-@dataclass
-class Location(Model):
-    country_code: str
-    code: str
-    id: Optional[int] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    country_name: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    name: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    time_zone: Optional[str] = None
-    operator_id: Optional[int] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    provider_name: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    is_proxy_capable: Optional[bool] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    is_wireguard_capable: Optional[bool] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    operator: Optional[Operator] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    available: Optional[bool] = field(
-        default=False,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-
-    def __post_init__(self):
-        self.operator = Operator.find_by_id(self.operator_id)
-        self.available = self.exists(self.country_code, self.code)
-
-        if isinstance(self.is_proxy_capable, int):
-            self.is_proxy_capable = bool(self.is_proxy_capable)
-
-        if isinstance(self.is_wireguard_capable, int):
-            self.is_wireguard_capable = bool(self.is_wireguard_capable)
-
-    def is_available(self):
-        return self.exists(self.country_code, self.code)
-
-    @staticmethod
-    def find_by_id(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM locations WHERE id = ? LIMIT 1', Location.factory, [id])
-
-    @staticmethod
-    def find(country_code: str, code: str):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM locations WHERE country_code = ? AND code = ? LIMIT 1', Location.factory, [country_code, code])
-
-    @staticmethod
-    def exists(country_code: str, code: str):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_exists('SELECT * FROM locations WHERE country_code = ? AND code = ?', [country_code, code])
-
-    @staticmethod
-    def all():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_all('SELECT * FROM locations', Location.factory)
-
-    @staticmethod
-    def truncate():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True)
-
-    @staticmethod
-    def save_many(locations):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        Model._insert_many('INSERT INTO locations VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', Location.tuple_factory, locations)
-
-    @staticmethod
-    def factory(cursor, row):
-        local_fields = [column[0] for column in cursor.description]
-        return Location(**{key: value for key, value in zip(local_fields, row)})
-
-    @staticmethod
-    def tuple_factory(location):
-        return location.id, location.country_code, location.country_name, location.code, location.name, location.time_zone, location.operator_id, location.provider_name, location.is_proxy_capable, location.is_wireguard_capable

core/models/Operator.py

@@ -1,56 +0,0 @@
-from core.models.Model import Model
-from dataclasses import dataclass
-
-_table_name: str = 'operators'
-
-_table_definition: str = """
-    'id' int UNIQUE,
-    'name' varchar,
-    'public_key' varchar,
-    'nostr_public_key' varchar,
-    'nostr_profile_reference' varchar,
-    'nostr_attestation_event_reference' varchar
-"""
-
-
-@dataclass
-class Operator(Model):
-    id: int
-    name: str
-    public_key: str
-    nostr_public_key: str
-    nostr_profile_reference: str
-    nostr_attestation_event_reference: str
-
-    @staticmethod
-    def find_by_id(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM operators WHERE id = ? LIMIT 1', Operator.factory, [id])
-
-    @staticmethod
-    def exists(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_exists('SELECT * FROM operators WHERE id = ?', [id])
-
-    @staticmethod
-    def all():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_all('SELECT * FROM operators', Operator.factory)
-
-    @staticmethod
-    def truncate():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True)
-
-    @staticmethod
-    def save_many(operators):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        Model._insert_many('INSERT INTO operators VALUES(?, ?, ?, ?, ?, ?)', Operator.tuple_factory, operators)
-
-    @staticmethod
-    def factory(cursor, row):
-        local_fields = [column[0] for column in cursor.description]
-        return Operator(**{key: value for key, value in zip(local_fields, row)})
-
-    @staticmethod
-    def tuple_factory(operator):
-        return operator.id, operator.name, operator.public_key, operator.nostr_public_key, operator.nostr_profile_reference, operator.nostr_attestation_event_reference

core/models/manage/denormalize.py

@@ -0,0 +1,75 @@
+import yaml
+
+"""
+This function extracts the value from a nested dictionary,
+By looping through each layer. And comparing the current dictionary's key to pre-made YAML key mappings.
+"""
+def extract_from_nested(obj, yaml_keys, default=None):
+    # loop through each key from the YAML mapping:
+    for each_yaml_key in yaml_keys:
+
+        # Check if each layer down of the 'object' is a dictionary with that key,
+        if isinstance(obj, dict) and each_yaml_key in obj:
+            # then extract the value for that key, and replace the object placeholder itself with it. 
+            obj = obj[each_yaml_key]
+
+            # this has the effect of going deeper into the nesting on the next round of the loop,
+
+        # But if it's not a dictionary or doesn't have the key,
+        else:
+            # then we hit a dead end,
+            return default
+
+    # We finished looping through all the keys of valid dictionaries, and extracted the values,
+    return obj
+
+"""
+This function removes unnecessary nesting,
+by comparing it to a pre-made YAML mapping
+"""
+def denormalize(data, which_yaml_file):
+    final_results = []
+
+    # prep the mapping config:
+    mapping_file = f"assets/yaml_mappings/{which_yaml_file}"
+    with open(mapping_file) as f:
+        mapping = yaml.safe_load(f)
+
+    # Extract out the 'data' variable if it exists, otherwise just use the data directly
+    extracted_data = data.get('data', data)
+
+    # go through the data
+    for each_item in extracted_data:
+
+        # setup temp flags/containers
+        each_denormalized_piece = {}
+        valid = True
+
+        # go through mapping:
+        for field in mapping['fields']:
+            """
+            Extract the values based on the YAML mapping
+            field['path'] - Gets a Python list of the keys from the YAML mapping
+            extract_from_nested - This function checks it's actually a dictionary and extracts it
+            """
+            value = extract_from_nested(each_item, field['path'])
+ 
+            # Store the extracted value. If it wasn't there, then it's None
+            each_denormalized_piece[field['name']] = value
+
+            # if it's required & not there, ditch it,
+            if field.get('required') and value is None:
+                print(f"It's required and value is none, skipping {value}")
+                valid = False
+                break
+
+        # Only add items where all required fields were present
+        if valid:
+            final_results.append(each_denormalized_piece)
+
+    # Return all successfully denormalized items
+    return final_results
+
+
+# Isolated testing:
+# items = denormalize(response.json(), 'example.yaml')

core/models/manage/insert.py

@@ -3,9 +3,14 @@ from sqlalchemy.orm import sessionmaker
 from sqlalchemy import create_engine
 from typing import Type, Dict, Any
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
-from core.models.orm_models.Base import Base
-from core.models.orm_models.CachedSync import CachedSync
 from core.models.manage.session_management import get_session
+from core.models.orm_models.Base import Base
+
+# all models it knows how to do:
+from core.models.orm_models.CachedSync import CachedSync
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.models.orm_models.EncryptedProxy import EncryptedProxy
 
 def insert_into_model(model_class: Type, all_data: dict | list, override=False) -> bool:
     """

core/models/manage/session_management.py

@@ -1,10 +1,31 @@
 from sqlalchemy.orm import sessionmaker
-from sqlalchemy import create_engine
-from core.models.orm_models.Base import Base
-from core.Constants import Constants
+from sqlalchemy import create_engine, inspect
+from core.models.orm_models.Base import BaseModel
+# from core.Constants import Constants
+import os
+from pathlib import Path
 
-database_path = Constants.HV_STORAGE_DATABASE_PATH
+def get_path():
+    # XDG Base Directory Specification
+    xdg_data_home = os.environ.get("XDG_DATA_HOME")
+    if xdg_data_home:
+        return Path(xdg_data_home) / "my-app"
+    return Path.home() / ".local" / "share" / "hydra-veil"
+
+system_path = get_path()
+database_path = f"{system_path}/storage.db"
+
+# Ensure the database directory exists
+db_dir = Path(database_path).parent
+db_dir.mkdir(parents=True, exist_ok=True)
+
+# Create the engine
 engine = create_engine(f"sqlite:///{database_path}")
+
+# Create all tables
+BaseModel.metadata.create_all(engine)
+
+# Create session
 Session = sessionmaker(bind=engine)
 
 _session = None
@@ -18,7 +39,7 @@ def init_session():
     while keeping it a globally accessible variable
     """
     global _session
-    Base.metadata.create_all(engine)
+    BaseModel.metadata.create_all(engine)
     _session = Session()
 
 def get_session():

core/models/orm_models/Base.py

@@ -1,9 +1,19 @@
 # base.py
 from sqlalchemy.orm import declarative_base
+from sqlalchemy import inspect
 
 Base = declarative_base()
 
 """
-This will eventually replace the legacy base model,
-but is still required, even if empty, to setup the declarative base. 
+This mapper exists so the children classes
+have the ability to print human-readable strings
+as dictionaries for the UI.
 """
+
+class BaseModel(Base):
+    __abstract__ = True
+    
+    def __repr__(self):
+        mapper = inspect(self.__class__)
+        fields = ', '.join(f'{col.name}={getattr(self, col.name)}' for col in mapper.columns)
+        return f'{self.__class__.__name__}({fields})'

core/models/orm_models/EncryptedProxy.py

@@ -0,0 +1,40 @@
+from sqlalchemy import Integer, String, ForeignKey
+from sqlalchemy.orm import declarative_base, mapped_column
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import Base
+from core.models.orm_models.Operator import Operator
+from core.models.orm_models.Location import Location
+
+class EncryptedProxy(Base):
+    __tablename__ = 'encryptedproxies'
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+    protocol_type: Mapped[str] = mapped_column(String, nullable=False) # changed from name "type"
+
+    username: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    password: Mapped[str] = mapped_column(String, nullable=False)
+
+    links: Mapped[str] = mapped_column(String, nullable=False)
+    subscription_url: Mapped[str] = mapped_column(String, nullable=False)
+
+    # Foreign key column
+    operator_id: Mapped[Optional[int]] = mapped_column(
+        Integer, 
+        ForeignKey("operators.id"), 
+        nullable=True, 
+        default=None
+    )
+
+    # Foreign key column
+    location_id: Mapped[Optional[int]] = mapped_column(
+        Integer, 
+        ForeignKey("locations.id"), 
+        nullable=True, 
+        default=None
+    )
+
+    operator_domain: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    operator_hysteria2_host: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    operator_vless_host: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    server_ip: Mapped[str] = mapped_column(String, nullable=True, default=None) # pre-resolved IP — avoids DNS leak at connect time

core/models/orm_models/Location.py

@@ -0,0 +1,73 @@
+
+from sqlalchemy import Integer, String, ForeignKey
+from sqlalchemy.orm import declarative_base, mapped_column, Mapped, relationship
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import BaseModel
+from core.models.orm_models.Operator import Operator
+
+
+class Location(BaseModel):
+    __tablename__ = 'locations'
+
+    # model primary key, but only used for business lookups
+    id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
+
+    # BUSINESS logic primary keys:
+    country_code: Mapped[str] = mapped_column(String, nullable=False)
+    code: Mapped[str] = mapped_column(String, unique=True, nullable=False)
+
+    # country name:
+    country_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+
+    # this is CITY name:
+    name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+
+    time_zone: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+    
+    # Foreign key column
+    operator_id: Mapped[Optional[int]] = mapped_column(
+        Integer, 
+        ForeignKey("operators.id"), 
+        nullable=True, 
+        default=None
+    )
+    
+    # Relationship with selectin eager loading
+    operator: Mapped[Optional["Operator"]] = relationship(
+        "Operator", 
+        lazy="selectin"
+    )
+
+    provider_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+    available: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_proxy_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_wireguard_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_hysteria2_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_vless_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    
+    def to_dict(self):
+        return {
+            "country_code": self.country_code,
+            "code": self.code,
+            "time_zone": self.time_zone
+            }
+
+
+# to use:
+# Lookup by location/city codes
+# record = session.get(Location, (country_code, code))
+
+# example data:
+"""
+id: 7
+country_code: is
+country_name: Iceland
+code: 1
+name: Capital Region
+time_zone: Atlantic/Reykjavik
+operator_id: 6
+provider_name: FlokiNET
+is_proxy_capable: 0
+is_wireguard_capable: 1
+"""

core/models/orm_models/Operator.py

@@ -0,0 +1,18 @@
+from sqlalchemy import Integer, String
+from sqlalchemy.orm import declarative_base, mapped_column
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import BaseModel
+
+class Operator(BaseModel):
+    __tablename__ = 'operators'
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+    name: Mapped[str] = mapped_column(String, nullable=False)
+    public_key: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    nostr_public_key: Mapped[str] = mapped_column(String, nullable=False)
+    nostr_profile_reference: Mapped[str] = mapped_column(String, nullable=False)
+    nostr_attestation_event_reference: Mapped[str] = mapped_column(String, nullable=False)
+
+    # legacy:
+    # operator: Mapped[str] = mapped_column(String, nullable=False)

core/services/WebServiceApiService.py

@@ -1,7 +1,7 @@
 from core.Constants import Constants
 from core.models.ClientVersion import ClientVersion
-from core.models.Location import Location
-from core.models.Operator import Operator
+# from core.models.Location import Location
+# from core.models.Operator import Operator
 from core.models.Subscription import Subscription
 from core.models.SubscriptionPlan import SubscriptionPlan
 from core.models.invoice.Invoice import Invoice

core/services/sync/get_data_from_api.py

@@ -9,7 +9,8 @@ from typing import Optional
 
 # this can be spoofed with the isolation testing comment below
 
-def get_metadata_from_api(
+def get_data_from_api(
+    endpoint: str,
     client_observer: Optional[ClientObserver] = None,
     connection_observer: Optional[ConnectionObserver] = None
 ) -> dict:
@@ -24,7 +25,7 @@ def get_metadata_from_api(
         return {"success": False, "data": None, "error": error_msg}
 
     # Construct the full endpoint URL
-    url = f"{base_url}/cachedsync"
+    url = f"{base_url}/{endpoint}"
     logger.debug(f"API endpoint: {url}")
 
     try:

core/services/sync/orm_methods/sync_one_orm.py

@@ -0,0 +1,60 @@
+from core.services.sync.get_data_from_api import get_data_from_api
+from core.models.manage.insert import insert_into_model
+from core.models.manage.denormalize import denormalize
+from core.models.orm_models.Base import Base
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+from core.errors.logger import logger
+
+from typing import Optional
+import json
+
+def sync_one_orm_model(
+    which_model: Base,
+    which_endpoint: str,
+    client_observer: Optional[ClientObserver] = None,
+    connection_observer: Optional[ConnectionObserver] = None
+) -> dict:
+
+    api_result = get_data_from_api(which_endpoint, ClientObserver, ConnectionObserver)
+
+    if not api_result.get("success"):
+        error_msg = api_result.get("error", "Unknown API error")
+        logger.error(f"API sync failed for {which_endpoint} with: {error_msg}")
+        return {
+            "success": False,
+            "error": error_msg
+        }
+
+    new_data = api_result.get("data")
+
+    # prep data (denormalize)
+    yaml_filename = f"{which_endpoint}.yaml"
+    denormalized_data = denormalize(new_data, yaml_filename)
+
+    if denormalized_data:
+        # Debug Pretty print with indentation
+        # print(json.dumps(denormalized_data, indent=2))
+
+        try:
+            did_it_work = insert_into_model(which_model, denormalized_data, True)
+            if did_it_work:
+                return {
+                    "success": True
+                }
+            else:
+                return {
+                    "success": False,
+                    "error": f"We got the data from the endpoint {which_endpoint}, but failed to insert into the model.  That raw data is {denormalized_data}"
+                }
+        except:
+            return {
+                "success": False,
+                "error": f"The except triggered on a sync_one_orm_model's failure to insert into the model with the endpoint {which_endpoint}.  We got the raw data of {denormalized_data}"
+            }
+
+    else:
+        return {
+            "success": False,
+            "error": f"We had issues with denormalizing the data from the server.  That data was {new_data}."
+        }

core/services/sync/sync_service.py

@@ -1,7 +1,7 @@
 
 # comparisons & api calls:
 from core.services.sync.compare_tables import compare_tables
-from core.services.sync.get_metadata_from_api import get_metadata_from_api
+from core.services.sync.get_data_from_api import get_data_from_api
 
 # ORM for metadata:
 from core.models.manage.session_management import get_session
@@ -147,9 +147,9 @@ def coordinate_cache_sync(
         - 'filtered_metadata': the metadata from the server after removing keys not in the ORM
     """
 
-    api_result = get_metadata_from_api(ClientObserver, ConnectionObserver)
+    api_result = get_data_from_api("cachedsync", ClientObserver, ConnectionObserver)
 
-    # we trust the 'get_metadata_from_api' function to give us a dictionary:
+    # we trust the 'get_data_from_api' function to give us a dictionary:
     if not api_result.get("success"):
         error_msg = api_result.get("error", "Unknown API error")
         logger.error(f"API sync failed: {error_msg}")