Update CLI to use new ORM and initialize database for sync

assets/yaml_mappings/locations.yaml

@@ -33,9 +33,14 @@ fields:
     path: ['is_wireguard_capable']
     required: true
 
+  - name: is_hysteria2_capable
+    path: ['is_hysteria2_capable']
 
-  #     required: true
+  - name: is_vless_capable
+    path: ['is_vless_capable']
 
+
+# original version:
   # locations.append((
   # location['country']['code'],
   # location['code'],

core/Constants.py

@@ -6,6 +6,9 @@ import os
 @dataclass(frozen=True)
 class Constants:
 
+    DB_VERSION_THIS_APP_WANTS = 1
+
+    # ticketing group:
     TICKET_API_BASE_URL:        Final[str] = os.environ.get(
         "TICKET_API_BASE_URL", "https://ticket.hydraveil.net"
     )

core/controllers/ClientController.py

@@ -1,7 +1,14 @@
 # new sync refactor:
 from core.models.manage.session_management import init_session, close_session
 from core.services.sync.sync_service import coordinate_cache_sync, save_metadata
+from core.services.sync.orm_methods.sync_one_orm import sync_one_orm_model
 from core.errors.logger import logger
+
+# ORM models that can be sync'ed:
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+
+
 # prior versions:
 from core.Constants import Constants
 from core.Errors import UnknownClientPathError, UnknownClientVersionError, CommandNotFoundError
@@ -9,8 +16,8 @@ from core.controllers.ApplicationController import ApplicationController
 from core.controllers.ApplicationVersionController import ApplicationVersionController
 from core.controllers.ClientVersionController import ClientVersionController
 from core.controllers.ConfigurationController import ConfigurationController
-from core.controllers.LocationController import LocationController
+# from core.controllers.LocationController import LocationController
-from core.controllers.OperatorController import OperatorController
+# from core.controllers.OperatorController import OperatorController
 from core.controllers.SubscriptionPlanController import SubscriptionPlanController
 from core.observers.ClientObserver import ClientObserver
 from core.observers.ConnectionObserver import ConnectionObserver
@@ -52,12 +59,9 @@ class ClientController:
         if client_observer is not None:
             client_observer.notify('synchronizing', "Fetching list of new data ..")
 
-        # Use Cached Method:
-        init_session()
-
         result = coordinate_cache_sync(ClientObserver, ConnectionObserver)
 
-        logger.info(f"We got a Result from the API of {result}")
+        # logger.info(f"We got a Result from the API of {result}")
 
         # Outright Error:
         if not result["success"]:
@@ -78,8 +82,30 @@ class ClientController:
         # flag for after the save,
         data_was_saved = False
 
-        try:
         # Fetch and update the real data (no longer metadata)...
+
+        # =================== ORM BASED MODELS ==================
+        """
+        Note: for the new ORM based models,
+        it does the Tor/system check in the API call itself.
+        """
+
+        if "locations" in changed_tables:
+            logger.info("Sync of Locations")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Locations List..')
+
+            final_result = sync_one_orm_model(Location, "locations")
+
+        if "operators" in changed_tables:
+            logger.info("Sync of Operators")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Operators List..')
+
+            final_result_two = sync_one_orm_model(Operator, "operators")
+
+        # =================== MANUAL-SQL BASED MODELS ==================
+        try:
             from core.controllers.ConnectionController import ConnectionController
             ConnectionController.with_preferred_connection(task=ClientController.__sync, changed_tables=changed_tables, client_observer=client_observer, connection_observer=connection_observer)
 
@@ -98,9 +124,6 @@ class ClientController:
                 filtered_metadata = result["filtered_metadata"] # from the top of the function
                 save_successful = save_metadata(filtered_metadata) # the "save_data" function is inside sync_service
 
-                # Regardless of the outcome,
-                close_session()
-
                 if client_observer is None:
                     logger.error("Error: No client_observer to update the UI, the final part of the sync function skipped")
                     return # can't update their UI
@@ -151,20 +174,6 @@ class ClientController:
             # noinspection PyProtectedMember
             ClientVersionController._sync(proxies=proxies)
 
-        if "operators" in changed_tables:
-            logger.info("Sync of Operators")
-            if client_observer is not None:
-                client_observer.notify('synchronizing', 'Fetching Operators List..')
-            # noinspection PyProtectedMember
-            OperatorController._sync(proxies=proxies)
-
-        if "locations" in changed_tables:
-            logger.info("Sync of Locations")
-            if client_observer is not None:
-                client_observer.notify('synchronizing', 'Fetching Locations List..')
-            # noinspection PyProtectedMember
-            LocationController._sync(proxies=proxies)
-
         if "subscriptions" in changed_tables:
             logger.info("Sync of Subscriptions")
             if client_observer is not None:

core/controllers/LocationController.py

@@ -1,22 +1,49 @@
-from core.models.Location import Location
+from core.models.orm_models.Location import Location
-from core.services.WebServiceApiService import WebServiceApiService
+from core.models.orm_models.Operator import Operator
-from typing import Optional
 
+from core.models.manage.session_management import get_session
+from typing import Optional
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
 
 class LocationController:
 
     @staticmethod
-    def get(country_code: str, code: str):
+    def get(country_code: str, city_code: str):
-        return Location.find(country_code, code)
+        with get_session() as session:
+            location_object = session.execute(
+                select(Location)
+                .where((Location.country_code == country_code) & (Location.code == city_code))
+                .options(joinedload(Location.operator))
+            ).scalar_one_or_none()
+
+        return location_object
+
+    # legacy:
+    # Location.find(country_code, code)
+
+            
 
     @staticmethod
     def get_all():
-        return Location.all()
+        with get_session() as session:
+            all_records = session.execute(
+                select(Location)
+                .options(joinedload(Location.operator))
+            ).scalars().all()
+            return all_records
 
-    @staticmethod
+    # legacy:
-    def _sync(proxies: Optional[dict] = None):
+    # return Location.all()
 
-        locations = WebServiceApiService.get_locations(proxies)
 
-        Location.truncate()
+# Deprecated legacy sync,
-        Location.save_many(locations)
+
+# from core.services.WebServiceApiService import WebServiceApiService
+    # @staticmethod
+    # def _sync(proxies: Optional[dict] = None):
+
+    #     locations = WebServiceApiService.get_locations(proxies)
+
+    #     Location.truncate()
+    #     Location.save_many(locations)

core/controllers/OperatorController.py

@@ -1,22 +0,0 @@
-from core.models.Operator import Operator
-from core.services.WebServiceApiService import WebServiceApiService
-from typing import Optional
-
-
-class OperatorController:
-
-    @staticmethod
-    def get(id: int):
-        return Operator.find_by_id(id)
-
-    @staticmethod
-    def get_all():
-        return Operator.all()
-
-    @staticmethod
-    def _sync(proxies: Optional[dict] = None):
-
-        operators = WebServiceApiService.get_operators(proxies)
-
-        Operator.truncate()
-        Operator.save_many(operators)

core/models/BaseProfile.py

@@ -1,11 +1,21 @@
+from core.models.manage.session_management import get_session
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
+
 from abc import ABC, abstractmethod
 from core.Constants import Constants
 from core.Helpers import write_atomically
-from core.models.Location import Location
+
+# from core.models.Location import Location
+from core.controllers.LocationController import LocationController
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+
 from core.models.Subscription import Subscription
 from core.models.session.ApplicationVersion import ApplicationVersion
 from dataclasses import dataclass, field, asdict
 from dataclasses_json import config, Exclude, dataclass_json
+
 from pathlib import Path
 from typing import Optional, Self
 import json
@@ -23,7 +33,10 @@ class BaseProfile(ABC):
     )
     name: str
     subscription: Optional[Subscription]
-    location: Optional[Location]
+    location: Optional[Location] = field(metadata=config(exclude=Exclude.ALWAYS))  # SQLAlchemy object
+
+    # legacy version included it to be serialized, but now it's an SQLAlchemy object.
+    # location: Optional[Location]
 
     @abstractmethod
     def get_wireguard_configuration_path(self):
@@ -52,8 +65,15 @@ class BaseProfile(ABC):
         return type(self).__name__ == 'SystemProfile'
 
     def save(self: Self):
+        config_dict = self.to_dict()  # Get dict, not JSON string
+        location_dict = self.location.to_dict() if self.location else None if self.location else None # this is from SQLAlchemy, and not JSON-models, that's why it's separate.
 
-        config_file_contents = f'{self.to_json(indent=4)}\n'
+        config_dict["location"] = location_dict
+
+        config_file_contents = json.dumps(config_dict, indent=4) + '\n'
+
+        # legacy version:
+        # config_file_contents = f'{self.to_json(indent=4)}\n'
 
         os.makedirs(self.get_config_path(), exist_ok=True)
         os.makedirs(self.get_data_path(), exist_ok=True)
@@ -146,6 +166,7 @@ class BaseProfile(ABC):
 
         return list([key for key, value in asdict(self).items() if value != asdict(reference).get(key)])
 
+
     @staticmethod
     def find_by_id(id: int):
 
@@ -161,16 +182,43 @@ class BaseProfile(ABC):
 
         profile['id'] = id
 
+        profiles_location = profile['location']
+
         if profile['location'] is not None:
+            if isinstance(profiles_location, dict):
+                try:
+                    country_code = profile['location']['country_code']
+                    city_code = profile['location']['code']
+                except:
+                    print("ERROR! CANT FIND country code or city")
+                    return
+            else:
+                # potentially coming from SQLAlchemy:
+                country_code = profile.location.country_code
+                city_code = profile.location.code
 
-            location = Location.find(profile['location']['country_code'] or None, profile['location']['code'] or None)
+            with get_session() as session:
+                location_object = session.execute(
+                    select(Location)
+                    .where((Location.country_code == country_code) & (Location.code == city_code))
+                    .options(joinedload(Location.operator))
+                ).scalar_one_or_none()
 
-            if location is not None:
 
-                if profile['location'].get('time_zone') is not None:
+                if location_object:
-                    location.time_zone = profile['location']['time_zone']
+                    profile['location'] = location_object
 
-                profile['location'] = location
+            # this needs error handling if there's no location or malconformed config.
+
+
+            # legacy phased out since SQLAlchemy handles the time_zone.
+
+            # if location is not None:
+
+            #     if profile['location'].get('time_zone') is not None:
+            #         location.time_zone = profile['location']['time_zone']
+
+            #     profile['location'] = location
 
         if 'application_version' in profile:
 

core/models/Location.py

@@ -1,115 +0,0 @@
-from core.models.Model import Model
-from core.models.Operator import Operator
-from dataclasses import dataclass, field
-from dataclasses_json import config, Exclude
-from typing import Optional
-
-_table_name: str = 'locations'
-
-_table_definition: str = """
-    'id' int UNIQUE,
-    'country_code' varchar,
-    'country_name' varchar,
-    'code' varchar,
-    'name' varchar,
-    'time_zone' varchar,
-    'operator_id' int,
-    'provider_name' varchar,
-    'is_proxy_capable' bool,
-    'is_wireguard_capable' bool,
-    UNIQUE(code, country_code)
-"""
-
-
-@dataclass
-class Location(Model):
-    country_code: str
-    code: str
-    id: Optional[int] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    country_name: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    name: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    time_zone: Optional[str] = None
-    operator_id: Optional[int] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    provider_name: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    is_proxy_capable: Optional[bool] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    is_wireguard_capable: Optional[bool] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    operator: Optional[Operator] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    available: Optional[bool] = field(
-        default=False,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-
-    def __post_init__(self):
-        self.operator = Operator.find_by_id(self.operator_id)
-        self.available = self.exists(self.country_code, self.code)
-
-        if isinstance(self.is_proxy_capable, int):
-            self.is_proxy_capable = bool(self.is_proxy_capable)
-
-        if isinstance(self.is_wireguard_capable, int):
-            self.is_wireguard_capable = bool(self.is_wireguard_capable)
-
-    def is_available(self):
-        return self.exists(self.country_code, self.code)
-
-    @staticmethod
-    def find_by_id(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM locations WHERE id = ? LIMIT 1', Location.factory, [id])
-
-    @staticmethod
-    def find(country_code: str, code: str):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM locations WHERE country_code = ? AND code = ? LIMIT 1', Location.factory, [country_code, code])
-
-    @staticmethod
-    def exists(country_code: str, code: str):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_exists('SELECT * FROM locations WHERE country_code = ? AND code = ?', [country_code, code])
-
-    @staticmethod
-    def all():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_all('SELECT * FROM locations', Location.factory)
-
-    @staticmethod
-    def truncate():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True)
-
-    @staticmethod
-    def save_many(locations):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        Model._insert_many('INSERT INTO locations VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', Location.tuple_factory, locations)
-
-    @staticmethod
-    def factory(cursor, row):
-        local_fields = [column[0] for column in cursor.description]
-        return Location(**{key: value for key, value in zip(local_fields, row)})
-
-    @staticmethod
-    def tuple_factory(location):
-        return location.id, location.country_code, location.country_name, location.code, location.name, location.time_zone, location.operator_id, location.provider_name, location.is_proxy_capable, location.is_wireguard_capable

core/models/Operator.py

@@ -1,64 +0,0 @@
-from core.models.Model import Model
-from dataclasses import dataclass
-
-_table_name: str = 'operators'
-
-_table_definition: str = """
-    'id' int UNIQUE,
-    'name' varchar,
-    'type' varchar,
-    'public_key' varchar,
-    'nostr_public_key' varchar,
-    'nostr_profile_reference' varchar,
-    'nostr_attestation_event_reference' varchar
-"""
-
-
-@dataclass
-class Operator(Model):
-    id: int
-    name: str
-    type: str
-    public_key: str
-    nostr_public_key: str
-    nostr_profile_reference: str
-    nostr_attestation_event_reference: str
-
-    def is_external(self) -> bool:
-        return self.type == 'external'
-
-    def is_internal(self) -> bool:
-        return self.type == 'internal'
-
-    @staticmethod
-    def find_by_id(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM operators WHERE id = ? LIMIT 1', Operator.factory, [id])
-
-    @staticmethod
-    def exists(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_exists('SELECT * FROM operators WHERE id = ?', [id])
-
-    @staticmethod
-    def all():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_all('SELECT * FROM operators', Operator.factory)
-
-    @staticmethod
-    def truncate():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True)
-
-    @staticmethod
-    def save_many(operators):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        Model._insert_many('INSERT INTO operators VALUES(?, ?, ?, ?, ?, ?, ?)', Operator.tuple_factory, operators)
-
-    @staticmethod
-    def factory(cursor, row):
-        local_fields = [column[0] for column in cursor.description]
-        return Operator(**{key: value for key, value in zip(local_fields, row)})
-
-    @staticmethod
-    def tuple_factory(operator):
-        return operator.id, operator.name, operator.type, operator.public_key, operator.nostr_public_key, operator.nostr_profile_reference, operator.nostr_attestation_event_reference

core/models/manage/denormalize.py

@@ -0,0 +1,75 @@
+import yaml
+
+"""
+This function extracts the value from a nested dictionary,
+By looping through each layer. And comparing the current dictionary's key to pre-made YAML key mappings.
+"""
+def extract_from_nested(obj, yaml_keys, default=None):
+    # loop through each key from the YAML mapping:
+    for each_yaml_key in yaml_keys:
+
+        # Check if each layer down of the 'object' is a dictionary with that key,
+        if isinstance(obj, dict) and each_yaml_key in obj:
+            # then extract the value for that key, and replace the object placeholder itself with it. 
+            obj = obj[each_yaml_key]
+
+            # this has the effect of going deeper into the nesting on the next round of the loop,
+
+        # But if it's not a dictionary or doesn't have the key,
+        else:
+            # then we hit a dead end,
+            return default
+
+    # We finished looping through all the keys of valid dictionaries, and extracted the values,
+    return obj
+
+"""
+This function removes unnecessary nesting,
+by comparing it to a pre-made YAML mapping
+"""
+def denormalize(data, which_yaml_file):
+    final_results = []
+
+    # prep the mapping config:
+    mapping_file = f"assets/yaml_mappings/{which_yaml_file}"
+    with open(mapping_file) as f:
+        mapping = yaml.safe_load(f)
+
+    # Extract out the 'data' variable if it exists, otherwise just use the data directly
+    extracted_data = data.get('data', data)
+
+    # go through the data
+    for each_item in extracted_data:
+
+        # setup temp flags/containers
+        each_denormalized_piece = {}
+        valid = True
+
+        # go through mapping:
+        for field in mapping['fields']:
+            """
+            Extract the values based on the YAML mapping
+            field['path'] - Gets a Python list of the keys from the YAML mapping
+            extract_from_nested - This function checks it's actually a dictionary and extracts it
+            """
+            value = extract_from_nested(each_item, field['path'])
+ 
+            # Store the extracted value. If it wasn't there, then it's None
+            each_denormalized_piece[field['name']] = value
+
+            # if it's required & not there, ditch it,
+            if field.get('required') and value is None:
+                print(f"It's required and value is none, skipping {value}")
+                valid = False
+                break
+
+        # Only add items where all required fields were present
+        if valid:
+            final_results.append(each_denormalized_piece)
+
+    # Return all successfully denormalized items
+    return final_results
+
+
+# Isolated testing:
+# items = denormalize(response.json(), 'example.yaml')

core/models/manage/insert.py

@@ -3,9 +3,14 @@ from sqlalchemy.orm import sessionmaker
 from sqlalchemy import create_engine
 from typing import Type, Dict, Any
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
-from core.models.orm_models.Base import Base
-from core.models.orm_models.CachedSync import CachedSync
 from core.models.manage.session_management import get_session
+from core.models.orm_models.Base import BaseModel
+
+# all models it knows how to do:
+from core.models.orm_models.CachedSync import CachedSync
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.models.orm_models.EncryptedProxy import EncryptedProxy
 
 def insert_into_model(model_class: Type, all_data: dict | list, override=False) -> bool:
     """

core/models/manage/session_management.py

@@ -1,40 +1,120 @@
-from sqlalchemy.orm import sessionmaker
+from core.errors.logger import logger
+from core.models.orm_models.Base import BaseModel
 from sqlalchemy import create_engine
-from core.models.orm_models.Base import Base
+from sqlalchemy.orm import sessionmaker
-from core.Constants import Constants
+import os
+from pathlib import Path
 
-database_path = Constants.HV_STORAGE_DATABASE_PATH
+"""
+Note:
+At the bottom it initializes the Session and sets it up outside the function loose.
+"""
+
+# ============================================================================
+# PATH & INITIALIZATION
+# ============================================================================
+
+def get_path():
+    """Returns XDG Base Directory path or ~/.local/share/hydra-veil"""
+    xdg_data = os.getenv("XDG_DATA_HOME")
+    if xdg_data:
+        return Path(xdg_data) / "my-app"
+    return Path.home() / ".local" / "share" / "hydra-veil"
+
+# ============================================================================
+# GLOBAL STATE
+# ============================================================================
+
+system_path = get_path()
+database_path = system_path / "storage.db"
+engine = None
+Session = None
+_session = None
+
+# ============================================================================
+# ENGINE & SESSION MANAGEMENT
+# ============================================================================
+
+def _reinitialize_engine_and_session():
+    """
+    Recreate the global engine and Session factory.
+    Call this after deleting the database to sync globals with the new DB file.
+    """
+    global engine, Session
+    
+    system_path.mkdir(parents=True, exist_ok=True)
     engine = create_engine(f"sqlite:///{database_path}")
     Session = sessionmaker(bind=engine)
 
-_session = None
 
 def init_session():
-    """
+    """Initialize the global _session from the global Session factory."""
-    Called once at application startup using global variables for the session, 
-    which initialize prior to being called with a false flag.
-
-    The reason for this is so the app can load prior to be called,
-    while keeping it a globally accessible variable
-    """
     global _session
-    Base.metadata.create_all(engine)
+    if Session is None:
+        raise RuntimeError("Session factory not initialized. Call _reinitialize_engine_and_session() first.")
     _session = Session()
 
+
 def get_session():
-    """Get the persistent session"""
+    """Return the global _session or raise RuntimeError."""
     if _session is None:
         raise RuntimeError("Session not initialized. Call init_session() first.")
     return _session
 
-def close_session():
-    """
-    Used prior to shutting down the application to break SQL connections.
 
-    In the current refactor, it may be used prior to shut down, such as sync completion,
+def close_session():
-    because a low amount of the total models currently use the new system.
+    """Close and reset the global _session."""
-    """
     global _session
-    if _session:
+    if _session is not None:
         _session.close()
     _session = None
+
+
+# ============================================================================
+# DATABASE OPERATIONS
+# ============================================================================
+
+def create_ONLY_db_version_table():
+    """Create only the database_version table using the global engine."""
+    if engine is None:
+        raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
+    
+    from core.models.orm_models.DatabaseVersion import database_version
+    database_version.create(engine, checkfirst=True)
+    # DatabaseVersion.__table__.create(engine, checkfirst=True)
+
+
+def create_ALL_tables():
+    """Create all tables from BaseModel.metadata using the global engine."""
+    print("starting create_ALL_tables")
+    if engine is None:
+        raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
+    print("passed engine test")
+
+    from core.models.orm_models.Location import Location
+    from core.models.orm_models.Operator import Operator
+    from core.models.orm_models.CachedSync import CachedSync
+    from core.models.orm_models.EncryptedProxy import EncryptedProxy
+
+    from core.models.SubscriptionPlan import SubscriptionPlan
+    from core.models.session.ApplicationVersion import ApplicationVersion
+    
+
+    try:
+        print("trying this")
+        BaseModel.metadata.create_all(engine, checkfirst=True)
+        print("Returning True for Creating all Tables..")
+        return True
+    except:
+        print("couldn't make all tables")
+        return False
+
+
+# ============================================================================
+# STARTUP: Initialize engine and session on import
+# ============================================================================
+
+_reinitialize_engine_and_session()
+logger.info("[DB MANAGEMENT] Started Engine in Session Management Module..")
+init_session()
+logger.info("[DB MANAGEMENT] Initialized Session")

core/models/manage/version_check.py

@@ -0,0 +1,348 @@
+from core.models.orm_models.DatabaseVersion import database_version
+from core.Constants import Constants
+from core.errors.logger import logger
+
+
+# generic
+from sqlalchemy import exc, text
+from sqlalchemy.orm import Session
+from typing import Optional
+
+
+def get_database_version(session: Session) -> Optional[int]:
+    """
+    Purpose:
+        Retrieve the current database version from the single-row version table.
+
+    Rank:
+        Helper
+
+    Called By:
+        check_database_compatibility
+
+    Features:
+        Handles errors with multiple items, failed loads, and other conditions.
+
+    Returns:
+        int: The database version, or None if retrieval fails.
+    """
+    func_name = "get_database_version"
+    
+    # Validate session
+    if session is None:
+        logger.error(f"{func_name}: Session is None")
+        return None
+    
+    if not isinstance(session, Session):
+        logger.error(f"{func_name}: Invalid session type: {type(session)}")
+        return None
+    
+    try:
+        # Attempt to query the table
+        logger.debug(f"{func_name}: Attempting to query database_version table")
+        result = session.query(database_version).all()
+        
+        # Handle empty table
+        if not result:
+            logger.warning(f"{func_name}: Table is empty, no version row found")
+            return None
+        
+        # Handle multiple rows (shouldn't happen, but log it)
+        if len(result) > 1:
+            logger.warning(
+                f"{func_name}: Expected 1 row, found {len(result)}. "
+                "Table should contain only a single row. Returning first version."
+            )
+        
+        # Extract version from first row
+        version_row = result[0]
+        version = version_row.version
+        
+        # Type validation
+        if not isinstance(version, int):
+            logger.error(
+                f"{func_name}: Version column type mismatch. "
+                f"Expected int, got {type(version).__name__}: {version}"
+            )
+            return None
+        
+        if version < 0:
+            logger.warning(
+                f"{func_name}: Version is negative ({version}). "
+                "This may indicate corrupted data."
+            )
+        
+        logger.info(f"{func_name}: Successfully retrieved version: {version}")
+        return version
+    
+    except exc.NoSuchTableError:
+        logger.error(
+            f"{func_name}: Table 'database_version' does not exist. "
+            "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
+        )
+        return None
+    
+    except exc.OperationalError as e:
+        logger.error(
+            f"{func_name}: Database operational error (connection, locked, or permissions). "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except exc.DatabaseError as e:
+        logger.error(
+            f"{func_name}: General database error. "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except exc.StatementError as e:
+        logger.error(
+            f"{func_name}: SQL statement error. "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except AttributeError as e:
+        logger.error(
+            f"{func_name}: Row does not have 'version' attribute. "
+            f"Verify table schema matches definition. Details: {str(e)}"
+        )
+        return None
+    
+    except TypeError as e:
+        logger.error(
+            f"{func_name}: Type error when accessing version. "
+            f"Details: {str(e)}"
+        )
+        return None
+    
+    except Exception as e:
+        logger.critical(
+            f"{func_name}: Unexpected exception type {type(e).__name__}. "
+            f"Details: {str(e)}",
+            exc_info=True
+        )
+        return None
+
+
+def insert_new_version(session: Session, new_version: int) -> bool:
+    """
+    Purpose:
+        Insert (or replace) the database version. Clears existing rows and inserts new version.
+
+    Rank:
+        Helper
+
+    Called By:
+        GUI's startup script in __main__
+
+    Args:
+        session: SQLAlchemy session
+        new_version: The version number to insert (must be a positive integer)
+    
+    Returns:
+        bool: True if successful, False otherwise.
+        does NOT raise errors.
+    """
+    func_name = "insert_new_version" # for logging purposes
+    
+    # Validate session
+    if session is None:
+        logger.error(f"{func_name}: Session is None")
+        return False
+    
+    if not isinstance(session, Session):
+        logger.error(f"{func_name}: Invalid session type: {type(session)}")
+        return False
+    
+    # Validate new_version parameter
+    if new_version is None:
+        logger.error(f"{func_name}: new_version is None")
+        return False
+    
+    if not isinstance(new_version, int):
+        logger.error(
+            f"{func_name}: new_version must be an integer, got {type(new_version).__name__}: {new_version}"
+        )
+        return False
+    
+    if new_version < 0:
+        logger.error(
+            f"{func_name}: new_version must be non-negative, got {new_version}"
+        )
+        return False
+    
+    try:
+        logger.debug(f"{func_name}: Starting transaction for version insert/replace")
+        
+        # Check if table exists before proceeding
+        try:
+            session.query(database_version).limit(1).all()
+        except exc.NoSuchTableError:
+            logger.error(
+                f"{func_name}: Table 'database_version' does not exist. "
+                "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
+            )
+            return False
+        
+        # Delete existing rows
+        logger.debug(f"{func_name}: Deleting existing version rows")
+        deleted_count = session.query(database_version).delete()
+        logger.debug(f"{func_name}: Deleted {deleted_count} existing row(s)")
+        
+        if deleted_count > 1:
+            logger.warning(
+                f"{func_name}: Deleted {deleted_count} rows. "
+                "Table should contain only a single row at any time."
+            )
+        
+        # Actually Insert the new version
+        logger.debug(f"{func_name}: Inserting new version {new_version}")
+        insert_stmt = database_version.insert().values(version=new_version)
+        session.execute(insert_stmt)
+        
+        # Commit transaction
+        session.commit()
+        logger.info(f"{func_name}: Successfully inserted version {new_version}")
+        return True
+    
+    except exc.NoSuchTableError:
+        logger.error(
+            f"{func_name}: Table 'database_version' does not exist. "
+            "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
+        )
+        session.rollback()
+        return False
+    
+    except exc.IntegrityError as e:
+        logger.error(
+            f"{func_name}: Integrity constraint violation (e.g., primary key conflict). "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.OperationalError as e:
+        logger.error(
+            f"{func_name}: Database operational error (connection, locked, or permissions). "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.DatabaseError as e:
+        logger.error(
+            f"{func_name}: General database error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except exc.StatementError as e:
+        logger.error(
+            f"{func_name}: SQL statement error. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except ValueError as e:
+        logger.error(
+            f"{func_name}: Value error during insert. "
+            f"Details: {str(e)}"
+        )
+        session.rollback()
+        return False
+    
+    except Exception as e:
+        logger.critical(
+            f"{func_name}: Unexpected exception type {type(e).__name__}. "
+            f"Details: {str(e)}",
+            exc_info=True
+        )
+        session.rollback()
+        return False
+
+"""Spits back a string based on the reason and error."""
+def get_custom_message(reason: str, compatability_dict: dict) -> str:
+    """
+    Called By:
+        GUI's startup script in __main__
+    """
+    if reason == "upgrade":
+        custom_error = "You just upgraded HydraVeil, so let's upgrade your database to handle the new data types."
+    elif reason == "old_app":
+        custom_error = "You're using an old version of HydraVeil, with a newer version of the database.  This means the older app would crash trying to handle the newer data."
+    else:
+        error_msg = compatability_dict.get("error", "Unknown reason.")
+        custom_error = f"There was an error with upgrading your database version.  Please tell customer support: {error_msg}."
+
+    return custom_error
+
+# ============================================================================
+# ORCHESTRATOR
+# ============================================================================
+def check_database_compatibility(session: Session) -> dict:
+    """
+    Purpose:
+        Evaluates if the user's existing database is incompatible with the app's version.
+
+    Rank:
+        Orchestrator
+
+    Called By:
+        GUI's startup script in __main__
+
+    Returns:
+        Dictionary of True/False, with reason
+        NO error raises.
+    """
+    db_version_you_have = get_database_version(session)
+   
+    if db_version_you_have is None:
+        print("Nothing there")
+        # First run, corrupted, or empty
+        return {
+            "result": True,
+            "reason": "no_database" # Allow initialization still, since there's nothing to delete.
+        }
+
+    logger.info(f"[DB MANAGEMENT] The Database version this APP WANTS is {Constants.DB_VERSION_THIS_APP_WANTS}")
+    logger.info(f"[DB MANAGEMENT] The Database version this device currently has is {db_version_you_have}")
+
+    if Constants.DB_VERSION_THIS_APP_WANTS == db_version_you_have:
+        logger.info(f"[DB MANAGEMENT] Great, the database versions match exactly.")
+        return {
+            "result": True,
+            "reason": None,
+        }
+
+    # BAD!! Old version,
+    elif Constants.DB_VERSION_THIS_APP_WANTS < db_version_you_have:
+        logger.info(f"[DB MANAGEMENT] The App version is older, this is NOT compatabile")
+        # make them wipe the tables, or switch versions of the app
+        return {
+           "result": False,
+            "reason": "old_app",
+            "old_db_version": db_version_you_have
+        }
+
+    # Upgrade User:
+    elif Constants.DB_VERSION_THIS_APP_WANTS > db_version_you_have:
+        logger.info(f"[DB MANAGEMENT] The user has an upgraded App, and an older database")
+        return {
+            "result": False,
+            "reason": "upgrade",
+            "old_db_version": db_version_you_have
+        }
+
+    # this is an error
+    else:
+        logger.error(f"[DB MANAGEMENT] Unknown Error inside version check module with the King Orchestration")
+        return {
+            "result": False,
+            "reason": "error",
+            "old_db_version": db_version_you_have,
+            "error": "unknown result"
+        }

core/models/orm_models/Base.py

@@ -1,9 +1,19 @@
 # base.py
 from sqlalchemy.orm import declarative_base
+from sqlalchemy import inspect
 
 Base = declarative_base()
 
 """
-This will eventually replace the legacy base model,
+This mapper exists so the children classes
-but is still required, even if empty, to setup the declarative base. 
+have the ability to print human-readable strings
+as dictionaries for the UI.
 """
+
+class BaseModel(Base):
+    __abstract__ = True
+    
+    def __repr__(self):
+        mapper = inspect(self.__class__)
+        fields = ', '.join(f'{col.name}={getattr(self, col.name)}' for col in mapper.columns)
+        return f'{self.__class__.__name__}({fields})'

core/models/orm_models/DatabaseVersion.py

@@ -0,0 +1,11 @@
+from sqlalchemy import Column, Integer, MetaData, Table
+
+# Separate metadata for the compatibility table only
+compat_metadata = MetaData()
+
+database_version = Table(
+    'database_version',
+    compat_metadata,
+    Column('version', Integer, primary_key=True)
+)
+

core/models/orm_models/EncryptedProxy.py

@@ -0,0 +1,40 @@
+from sqlalchemy import Integer, String, ForeignKey
+from sqlalchemy.orm import declarative_base, mapped_column
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import Base
+from core.models.orm_models.Operator import Operator
+from core.models.orm_models.Location import Location
+
+class EncryptedProxy(Base):
+    __tablename__ = 'encryptedproxies'
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+    protocol_type: Mapped[str] = mapped_column(String, nullable=False) # changed from name "type"
+
+    username: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    password: Mapped[str] = mapped_column(String, nullable=False)
+
+    links: Mapped[str] = mapped_column(String, nullable=False)
+    subscription_url: Mapped[str] = mapped_column(String, nullable=False)
+
+    # Foreign key column
+    operator_id: Mapped[Optional[int]] = mapped_column(
+        Integer, 
+        ForeignKey("operators.id"), 
+        nullable=True, 
+        default=None
+    )
+
+    # Foreign key column
+    location_id: Mapped[Optional[int]] = mapped_column(
+        Integer, 
+        ForeignKey("locations.id"), 
+        nullable=True, 
+        default=None
+    )
+
+    operator_domain: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    operator_hysteria2_host: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    operator_vless_host: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    server_ip: Mapped[str] = mapped_column(String, nullable=True, default=None) # pre-resolved IP — avoids DNS leak at connect time

core/models/orm_models/Location.py

@@ -0,0 +1,73 @@
+
+from sqlalchemy import Integer, String, ForeignKey
+from sqlalchemy.orm import declarative_base, mapped_column, Mapped, relationship
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import BaseModel
+from core.models.orm_models.Operator import Operator
+
+
+class Location(BaseModel):
+    __tablename__ = 'locations'
+
+    # model primary key, but only used for business lookups
+    id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
+
+    # BUSINESS logic primary keys:
+    country_code: Mapped[str] = mapped_column(String, nullable=False)
+    code: Mapped[str] = mapped_column(String, unique=True, nullable=False)
+
+    # country name:
+    country_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+
+    # this is CITY name:
+    name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+
+    time_zone: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+    
+    # Foreign key column
+    operator_id: Mapped[Optional[int]] = mapped_column(
+        Integer, 
+        ForeignKey("operators.id"), 
+        nullable=True, 
+        default=None
+    )
+    
+    # Relationship with selectin eager loading
+    operator: Mapped[Optional["Operator"]] = relationship(
+        "Operator", 
+        lazy="selectin"
+    )
+
+    provider_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
+    available: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_proxy_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_wireguard_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_hysteria2_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    is_vless_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
+    
+    def to_dict(self):
+        return {
+            "country_code": self.country_code,
+            "code": self.code,
+            "time_zone": self.time_zone
+            }
+
+
+# to use:
+# Lookup by location/city codes
+# record = session.get(Location, (country_code, code))
+
+# example data:
+"""
+id: 7
+country_code: is
+country_name: Iceland
+code: 1
+name: Capital Region
+time_zone: Atlantic/Reykjavik
+operator_id: 6
+provider_name: FlokiNET
+is_proxy_capable: 0
+is_wireguard_capable: 1
+"""

core/models/orm_models/Operator.py

@@ -0,0 +1,18 @@
+from sqlalchemy import Integer, String
+from sqlalchemy.orm import declarative_base, mapped_column
+from typing import Optional
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import BaseModel
+
+class Operator(BaseModel):
+    __tablename__ = 'operators'
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+    name: Mapped[str] = mapped_column(String, nullable=False)
+    public_key: Mapped[str] = mapped_column(String, nullable=True, default=None)
+    nostr_public_key: Mapped[str] = mapped_column(String, nullable=False)
+    nostr_profile_reference: Mapped[str] = mapped_column(String, nullable=False)
+    nostr_attestation_event_reference: Mapped[str] = mapped_column(String, nullable=False)
+
+    # legacy:
+    # operator: Mapped[str] = mapped_column(String, nullable=False)

core/services/WebServiceApiService.py

@@ -1,7 +1,7 @@
 from core.Constants import Constants
 from core.models.ClientVersion import ClientVersion
-from core.models.Location import Location
+# from core.models.Location import Location  # migrated to ORM
-from core.models.Operator import Operator
+# from core.models.Operator import Operator  # migrated to ORM
 from core.models.OperatorProxySession import OperatorProxySession
 from core.models.Subscription import Subscription
 from core.models.SubscriptionPlan import SubscriptionPlan

core/services/sync/get_data_from_api.py

@@ -9,11 +9,12 @@ from typing import Optional
 
 # this can be spoofed with the isolation testing comment below
 
-def get_metadata_from_api(
+def get_data_from_api(
+    endpoint: str,
     client_observer: Optional[ClientObserver] = None,
     connection_observer: Optional[ConnectionObserver] = None
 ) -> dict:
-    logger.info("Syncing with the API...")
+    logger.info("Syncing with the API in get_data_from_api...")
 
     # Get and validate the base URL
     rejected_list = [None, False, ""]
@@ -24,7 +25,7 @@ def get_metadata_from_api(
         return {"success": False, "data": None, "error": error_msg}
 
     # Construct the full endpoint URL
-    url = f"{base_url}/cachedsync"
+    url = f"{base_url}/{endpoint}"
     logger.debug(f"API endpoint: {url}")
 
     try:
@@ -43,8 +44,8 @@ def get_metadata_from_api(
         if final_result:
             logger.info("Successfully retrieved sync versions metadata from API")
             return {"success": True, "data": final_result, "error": None}
-        
+        else:
-        error_msg = "API response missing 'data' field"
+            error_msg = f"API response missing 'data' field at the end of get_data_from_api. This is the response from the other module: {sync_results}"
             logger.error(error_msg)
             return {"success": False, "data": None, "error": error_msg}
 

core/services/sync/orm_methods/sync_one_orm.py

@@ -0,0 +1,60 @@
+from core.services.sync.get_data_from_api import get_data_from_api
+from core.models.manage.insert import insert_into_model
+from core.models.manage.denormalize import denormalize
+from core.models.orm_models.Base import Base
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+from core.errors.logger import logger
+
+from typing import Optional
+import json
+
+def sync_one_orm_model(
+    which_model: Base,
+    which_endpoint: str,
+    client_observer: Optional[ClientObserver] = None,
+    connection_observer: Optional[ConnectionObserver] = None
+) -> dict:
+
+    api_result = get_data_from_api(which_endpoint, ClientObserver, ConnectionObserver)
+
+    if not api_result.get("success"):
+        error_msg = api_result.get("error", "Unknown API error")
+        logger.error(f"API sync failed for {which_endpoint} with: {error_msg}")
+        return {
+            "success": False,
+            "error": error_msg
+        }
+
+    new_data = api_result.get("data")
+
+    # prep data (denormalize)
+    yaml_filename = f"{which_endpoint}.yaml"
+    denormalized_data = denormalize(new_data, yaml_filename)
+
+    if denormalized_data:
+        # Debug Pretty print with indentation
+        # print(json.dumps(denormalized_data, indent=2))
+
+        try:
+            did_it_work = insert_into_model(which_model, denormalized_data, True)
+            if did_it_work:
+                return {
+                    "success": True
+                }
+            else:
+                return {
+                    "success": False,
+                    "error": f"We got the data from the endpoint {which_endpoint}, but failed to insert into the model.  That raw data is {denormalized_data}"
+                }
+        except:
+            return {
+                "success": False,
+                "error": f"The except triggered on a sync_one_orm_model's failure to insert into the model with the endpoint {which_endpoint}.  We got the raw data of {denormalized_data}"
+            }
+
+    else:
+        return {
+            "success": False,
+            "error": f"We had issues with denormalizing the data from the server.  That data was {new_data}."
+        }

core/services/sync/sync_service.py

@@ -1,7 +1,7 @@
 
 # comparisons & api calls:
 from core.services.sync.compare_tables import compare_tables
-from core.services.sync.get_metadata_from_api import get_metadata_from_api
+from core.services.sync.get_data_from_api import get_data_from_api
 
 # ORM for metadata:
 from core.models.manage.session_management import get_session
@@ -147,9 +147,9 @@ def coordinate_cache_sync(
         - 'filtered_metadata': the metadata from the server after removing keys not in the ORM
     """
 
-    api_result = get_metadata_from_api(ClientObserver, ConnectionObserver)
+    api_result = get_data_from_api("cachedsync", ClientObserver, ConnectionObserver)
 
-    # we trust the 'get_metadata_from_api' function to give us a dictionary:
+    # we trust the 'get_data_from_api' function to give us a dictionary:
     if not api_result.get("success"):
         error_msg = api_result.get("error", "Unknown API error")
         logger.error(f"API sync failed: {error_msg}")