assets/yaml_mappings/locations.yaml

@@ -33,14 +33,9 @@ fields:
     path: ['is_wireguard_capable']
     required: true
 
-  - name: is_hysteria2_capable
-    path: ['is_hysteria2_capable']
 
-  - name: is_vless_capable
-    path: ['is_vless_capable']
+  #     required: true
 
-
-# original version:
   # locations.append((
   # location['country']['code'],
   # location['code'],

core/Constants.py

@@ -6,9 +6,6 @@ import os
 @dataclass(frozen=True)
 class Constants:
 
-    DB_VERSION_THIS_APP_WANTS = 1
-
-    # ticketing group:
     TICKET_API_BASE_URL:        Final[str] = os.environ.get(
         "TICKET_API_BASE_URL", "https://ticket.hydraveil.net"
     )

core/controllers/ClientController.py

@@ -1,14 +1,7 @@
 # new sync refactor:
 from core.models.manage.session_management import init_session, close_session
 from core.services.sync.sync_service import coordinate_cache_sync, save_metadata
-from core.services.sync.orm_methods.sync_one_orm import sync_one_orm_model
 from core.errors.logger import logger
-
-# ORM models that can be sync'ed:
-from core.models.orm_models.Location import Location
-from core.models.orm_models.Operator import Operator
-
-
 # prior versions:
 from core.Constants import Constants
 from core.Errors import UnknownClientPathError, UnknownClientVersionError, CommandNotFoundError
@@ -16,8 +9,8 @@ from core.controllers.ApplicationController import ApplicationController
 from core.controllers.ApplicationVersionController import ApplicationVersionController
 from core.controllers.ClientVersionController import ClientVersionController
 from core.controllers.ConfigurationController import ConfigurationController
-# from core.controllers.LocationController import LocationController
-# from core.controllers.OperatorController import OperatorController
+from core.controllers.LocationController import LocationController
+from core.controllers.OperatorController import OperatorController
 from core.controllers.SubscriptionPlanController import SubscriptionPlanController
 from core.observers.ClientObserver import ClientObserver
 from core.observers.ConnectionObserver import ConnectionObserver
@@ -59,9 +52,12 @@ class ClientController:
         if client_observer is not None:
             client_observer.notify('synchronizing', "Fetching list of new data ..")
 
+        # Use Cached Method:
+        init_session()
+
         result = coordinate_cache_sync(ClientObserver, ConnectionObserver)
 
-        # logger.info(f"We got a Result from the API of {result}")
+        logger.info(f"We got a Result from the API of {result}")
 
         # Outright Error:
         if not result["success"]:
@@ -82,30 +78,8 @@ class ClientController:
         # flag for after the save,
         data_was_saved = False
 
-        # Fetch and update the real data (no longer metadata)...
-
-        # =================== ORM BASED MODELS ==================
-        """
-        Note: for the new ORM based models,
-        it does the Tor/system check in the API call itself.
-        """
-
-        if "locations" in changed_tables:
-            logger.info("Sync of Locations")
-            if client_observer is not None:
-                client_observer.notify('synchronizing', 'Fetching Locations List..')
-
-            final_result = sync_one_orm_model(Location, "locations")
-
-        if "operators" in changed_tables:
-            logger.info("Sync of Operators")
-            if client_observer is not None:
-                client_observer.notify('synchronizing', 'Fetching Operators List..')
-
-            final_result_two = sync_one_orm_model(Operator, "operators")
-
-        # =================== MANUAL-SQL BASED MODELS ==================
         try:
+            # Fetch and update the real data (no longer metadata)...
             from core.controllers.ConnectionController import ConnectionController
             ConnectionController.with_preferred_connection(task=ClientController.__sync, changed_tables=changed_tables, client_observer=client_observer, connection_observer=connection_observer)
 
@@ -124,6 +98,9 @@ class ClientController:
                 filtered_metadata = result["filtered_metadata"] # from the top of the function
                 save_successful = save_metadata(filtered_metadata) # the "save_data" function is inside sync_service
 
+                # Regardless of the outcome,
+                close_session()
+
                 if client_observer is None:
                     logger.error("Error: No client_observer to update the UI, the final part of the sync function skipped")
                     return # can't update their UI
@@ -174,6 +151,20 @@ class ClientController:
             # noinspection PyProtectedMember
             ClientVersionController._sync(proxies=proxies)
 
+        if "operators" in changed_tables:
+            logger.info("Sync of Operators")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Operators List..')
+            # noinspection PyProtectedMember
+            OperatorController._sync(proxies=proxies)
+
+        if "locations" in changed_tables:
+            logger.info("Sync of Locations")
+            if client_observer is not None:
+                client_observer.notify('synchronizing', 'Fetching Locations List..')
+            # noinspection PyProtectedMember
+            LocationController._sync(proxies=proxies)
+
         if "subscriptions" in changed_tables:
             logger.info("Sync of Subscriptions")
             if client_observer is not None:

core/controllers/LocationController.py

@@ -1,49 +1,22 @@
-from core.models.orm_models.Location import Location
-from core.models.orm_models.Operator import Operator
-
-from core.models.manage.session_management import get_session
+from core.models.Location import Location
+from core.services.WebServiceApiService import WebServiceApiService
 from typing import Optional
-from sqlalchemy import select
-from sqlalchemy.orm import joinedload
+
 
 class LocationController:
 
     @staticmethod
-    def get(country_code: str, city_code: str):
-        with get_session() as session:
-            location_object = session.execute(
-                select(Location)
-                .where((Location.country_code == country_code) & (Location.code == city_code))
-                .options(joinedload(Location.operator))
-            ).scalar_one_or_none()
-
-        return location_object
-
-    # legacy:
-    # Location.find(country_code, code)
-
-            
+    def get(country_code: str, code: str):
+        return Location.find(country_code, code)
 
     @staticmethod
     def get_all():
-        with get_session() as session:
-            all_records = session.execute(
-                select(Location)
-                .options(joinedload(Location.operator))
-            ).scalars().all()
-            return all_records
+        return Location.all()
 
-    # legacy:
-    # return Location.all()
+    @staticmethod
+    def _sync(proxies: Optional[dict] = None):
 
+        locations = WebServiceApiService.get_locations(proxies)
 
-# Deprecated legacy sync,
-
-# from core.services.WebServiceApiService import WebServiceApiService
-    # @staticmethod
-    # def _sync(proxies: Optional[dict] = None):
-
-    #     locations = WebServiceApiService.get_locations(proxies)
-
-    #     Location.truncate()
-    #     Location.save_many(locations)
+        Location.truncate()
+        Location.save_many(locations)

core/controllers/OperatorController.py

@@ -0,0 +1,22 @@
+from core.models.Operator import Operator
+from core.services.WebServiceApiService import WebServiceApiService
+from typing import Optional
+
+
+class OperatorController:
+
+    @staticmethod
+    def get(id: int):
+        return Operator.find_by_id(id)
+
+    @staticmethod
+    def get_all():
+        return Operator.all()
+
+    @staticmethod
+    def _sync(proxies: Optional[dict] = None):
+
+        operators = WebServiceApiService.get_operators(proxies)
+
+        Operator.truncate()
+        Operator.save_many(operators)

core/models/BaseProfile.py

@@ -1,21 +1,11 @@
-from core.models.manage.session_management import get_session
-from sqlalchemy import select
-from sqlalchemy.orm import joinedload
-
 from abc import ABC, abstractmethod
 from core.Constants import Constants
 from core.Helpers import write_atomically
-
-# from core.models.Location import Location
-from core.controllers.LocationController import LocationController
-from core.models.orm_models.Location import Location
-from core.models.orm_models.Operator import Operator
-
+from core.models.Location import Location
 from core.models.Subscription import Subscription
 from core.models.session.ApplicationVersion import ApplicationVersion
 from dataclasses import dataclass, field, asdict
 from dataclasses_json import config, Exclude, dataclass_json
-
 from pathlib import Path
 from typing import Optional, Self
 import json
@@ -33,10 +23,7 @@ class BaseProfile(ABC):
     )
     name: str
     subscription: Optional[Subscription]
-    location: Optional[Location] = field(metadata=config(exclude=Exclude.ALWAYS))  # SQLAlchemy object
-
-    # legacy version included it to be serialized, but now it's an SQLAlchemy object.
-    # location: Optional[Location]
+    location: Optional[Location]
 
     @abstractmethod
     def get_wireguard_configuration_path(self):
@@ -65,15 +52,8 @@ class BaseProfile(ABC):
         return type(self).__name__ == 'SystemProfile'
 
     def save(self: Self):
-        config_dict = self.to_dict()  # Get dict, not JSON string
-        location_dict = self.location.to_dict() if self.location else None if self.location else None # this is from SQLAlchemy, and not JSON-models, that's why it's separate.
 
-        config_dict["location"] = location_dict
-
-        config_file_contents = json.dumps(config_dict, indent=4) + '\n'
-
-        # legacy version:
-        # config_file_contents = f'{self.to_json(indent=4)}\n'
+        config_file_contents = f'{self.to_json(indent=4)}\n'
 
         os.makedirs(self.get_config_path(), exist_ok=True)
         os.makedirs(self.get_data_path(), exist_ok=True)
@@ -166,7 +146,6 @@ class BaseProfile(ABC):
 
         return list([key for key, value in asdict(self).items() if value != asdict(reference).get(key)])
 
-
     @staticmethod
     def find_by_id(id: int):
 
@@ -182,43 +161,16 @@ class BaseProfile(ABC):
 
         profile['id'] = id
 
-        profiles_location = profile['location']
-
         if profile['location'] is not None:
-            if isinstance(profiles_location, dict):
-                try:
-                    country_code = profile['location']['country_code']
-                    city_code = profile['location']['code']
-                except:
-                    print("ERROR! CANT FIND country code or city")
-                    return
-            else:
-                # potentially coming from SQLAlchemy:
-                country_code = profile.location.country_code
-                city_code = profile.location.code
 
-            with get_session() as session:
-                location_object = session.execute(
-                    select(Location)
-                    .where((Location.country_code == country_code) & (Location.code == city_code))
-                    .options(joinedload(Location.operator))
-                ).scalar_one_or_none()
+            location = Location.find(profile['location']['country_code'] or None, profile['location']['code'] or None)
 
+            if location is not None:
 
-                if location_object:
-                    profile['location'] = location_object
+                if profile['location'].get('time_zone') is not None:
+                    location.time_zone = profile['location']['time_zone']
 
-            # this needs error handling if there's no location or malconformed config.
-
-
-            # legacy phased out since SQLAlchemy handles the time_zone.
-
-            # if location is not None:
-
-            #     if profile['location'].get('time_zone') is not None:
-            #         location.time_zone = profile['location']['time_zone']
-
-            #     profile['location'] = location
+                profile['location'] = location
 
         if 'application_version' in profile:
 

core/models/Location.py

@@ -0,0 +1,115 @@
+from core.models.Model import Model
+from core.models.Operator import Operator
+from dataclasses import dataclass, field
+from dataclasses_json import config, Exclude
+from typing import Optional
+
+_table_name: str = 'locations'
+
+_table_definition: str = """
+    'id' int UNIQUE,
+    'country_code' varchar,
+    'country_name' varchar,
+    'code' varchar,
+    'name' varchar,
+    'time_zone' varchar,
+    'operator_id' int,
+    'provider_name' varchar,
+    'is_proxy_capable' bool,
+    'is_wireguard_capable' bool,
+    UNIQUE(code, country_code)
+"""
+
+
+@dataclass
+class Location(Model):
+    country_code: str
+    code: str
+    id: Optional[int] = field(
+        default=None,
+        metadata=config(exclude=Exclude.ALWAYS)
+    )
+    country_name: Optional[str] = field(
+        default=None,
+        metadata=config(exclude=Exclude.ALWAYS)
+    )
+    name: Optional[str] = field(
+        default=None,
+        metadata=config(exclude=Exclude.ALWAYS)
+    )
+    time_zone: Optional[str] = None
+    operator_id: Optional[int] = field(
+        default=None,
+        metadata=config(exclude=Exclude.ALWAYS)
+    )
+    provider_name: Optional[str] = field(
+        default=None,
+        metadata=config(exclude=Exclude.ALWAYS)
+    )
+    is_proxy_capable: Optional[bool] = field(
+        default=None,
+        metadata=config(exclude=Exclude.ALWAYS)
+    )
+    is_wireguard_capable: Optional[bool] = field(
+        default=None,
+        metadata=config(exclude=Exclude.ALWAYS)
+    )
+    operator: Optional[Operator] = field(
+        default=None,
+        metadata=config(exclude=Exclude.ALWAYS)
+    )
+    available: Optional[bool] = field(
+        default=False,
+        metadata=config(exclude=Exclude.ALWAYS)
+    )
+
+    def __post_init__(self):
+        self.operator = Operator.find_by_id(self.operator_id)
+        self.available = self.exists(self.country_code, self.code)
+
+        if isinstance(self.is_proxy_capable, int):
+            self.is_proxy_capable = bool(self.is_proxy_capable)
+
+        if isinstance(self.is_wireguard_capable, int):
+            self.is_wireguard_capable = bool(self.is_wireguard_capable)
+
+    def is_available(self):
+        return self.exists(self.country_code, self.code)
+
+    @staticmethod
+    def find_by_id(id: int):
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
+        return Model._query_one('SELECT * FROM locations WHERE id = ? LIMIT 1', Location.factory, [id])
+
+    @staticmethod
+    def find(country_code: str, code: str):
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
+        return Model._query_one('SELECT * FROM locations WHERE country_code = ? AND code = ? LIMIT 1', Location.factory, [country_code, code])
+
+    @staticmethod
+    def exists(country_code: str, code: str):
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
+        return Model._query_exists('SELECT * FROM locations WHERE country_code = ? AND code = ?', [country_code, code])
+
+    @staticmethod
+    def all():
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
+        return Model._query_all('SELECT * FROM locations', Location.factory)
+
+    @staticmethod
+    def truncate():
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True)
+
+    @staticmethod
+    def save_many(locations):
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
+        Model._insert_many('INSERT INTO locations VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', Location.tuple_factory, locations)
+
+    @staticmethod
+    def factory(cursor, row):
+        local_fields = [column[0] for column in cursor.description]
+        return Location(**{key: value for key, value in zip(local_fields, row)})
+
+    @staticmethod
+    def tuple_factory(location):
+        return location.id, location.country_code, location.country_name, location.code, location.name, location.time_zone, location.operator_id, location.provider_name, location.is_proxy_capable, location.is_wireguard_capable

core/models/Operator.py

@@ -0,0 +1,64 @@
+from core.models.Model import Model
+from dataclasses import dataclass
+
+_table_name: str = 'operators'
+
+_table_definition: str = """
+    'id' int UNIQUE,
+    'name' varchar,
+    'type' varchar,
+    'public_key' varchar,
+    'nostr_public_key' varchar,
+    'nostr_profile_reference' varchar,
+    'nostr_attestation_event_reference' varchar
+"""
+
+
+@dataclass
+class Operator(Model):
+    id: int
+    name: str
+    type: str
+    public_key: str
+    nostr_public_key: str
+    nostr_profile_reference: str
+    nostr_attestation_event_reference: str
+
+    def is_external(self) -> bool:
+        return self.type == 'external'
+
+    def is_internal(self) -> bool:
+        return self.type == 'internal'
+
+    @staticmethod
+    def find_by_id(id: int):
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
+        return Model._query_one('SELECT * FROM operators WHERE id = ? LIMIT 1', Operator.factory, [id])
+
+    @staticmethod
+    def exists(id: int):
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
+        return Model._query_exists('SELECT * FROM operators WHERE id = ?', [id])
+
+    @staticmethod
+    def all():
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
+        return Model._query_all('SELECT * FROM operators', Operator.factory)
+
+    @staticmethod
+    def truncate():
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True)
+
+    @staticmethod
+    def save_many(operators):
+        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
+        Model._insert_many('INSERT INTO operators VALUES(?, ?, ?, ?, ?, ?, ?)', Operator.tuple_factory, operators)
+
+    @staticmethod
+    def factory(cursor, row):
+        local_fields = [column[0] for column in cursor.description]
+        return Operator(**{key: value for key, value in zip(local_fields, row)})
+
+    @staticmethod
+    def tuple_factory(operator):
+        return operator.id, operator.name, operator.type, operator.public_key, operator.nostr_public_key, operator.nostr_profile_reference, operator.nostr_attestation_event_reference

core/models/manage/denormalize.py

@@ -1,75 +0,0 @@
-import yaml
-
-"""
-This function extracts the value from a nested dictionary,
-By looping through each layer. And comparing the current dictionary's key to pre-made YAML key mappings.
-"""
-def extract_from_nested(obj, yaml_keys, default=None):
-    # loop through each key from the YAML mapping:
-    for each_yaml_key in yaml_keys:
-
-        # Check if each layer down of the 'object' is a dictionary with that key,
-        if isinstance(obj, dict) and each_yaml_key in obj:
-            # then extract the value for that key, and replace the object placeholder itself with it. 
-            obj = obj[each_yaml_key]
-
-            # this has the effect of going deeper into the nesting on the next round of the loop,
-
-        # But if it's not a dictionary or doesn't have the key,
-        else:
-            # then we hit a dead end,
-            return default
-
-    # We finished looping through all the keys of valid dictionaries, and extracted the values,
-    return obj
-
-"""
-This function removes unnecessary nesting,
-by comparing it to a pre-made YAML mapping
-"""
-def denormalize(data, which_yaml_file):
-    final_results = []
-
-    # prep the mapping config:
-    mapping_file = f"assets/yaml_mappings/{which_yaml_file}"
-    with open(mapping_file) as f:
-        mapping = yaml.safe_load(f)
-
-    # Extract out the 'data' variable if it exists, otherwise just use the data directly
-    extracted_data = data.get('data', data)
-
-    # go through the data
-    for each_item in extracted_data:
-
-        # setup temp flags/containers
-        each_denormalized_piece = {}
-        valid = True
-
-        # go through mapping:
-        for field in mapping['fields']:
-            """
-            Extract the values based on the YAML mapping
-            field['path'] - Gets a Python list of the keys from the YAML mapping
-            extract_from_nested - This function checks it's actually a dictionary and extracts it
-            """
-            value = extract_from_nested(each_item, field['path'])
- 
-            # Store the extracted value. If it wasn't there, then it's None
-            each_denormalized_piece[field['name']] = value
-
-            # if it's required & not there, ditch it,
-            if field.get('required') and value is None:
-                print(f"It's required and value is none, skipping {value}")
-                valid = False
-                break
-
-        # Only add items where all required fields were present
-        if valid:
-            final_results.append(each_denormalized_piece)
-
-    # Return all successfully denormalized items
-    return final_results
-
-
-# Isolated testing:
-# items = denormalize(response.json(), 'example.yaml')

core/models/manage/insert.py

@@ -3,14 +3,9 @@ from sqlalchemy.orm import sessionmaker
 from sqlalchemy import create_engine
 from typing import Type, Dict, Any
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
-from core.models.manage.session_management import get_session
-from core.models.orm_models.Base import BaseModel
-
-# all models it knows how to do:
+from core.models.orm_models.Base import Base
 from core.models.orm_models.CachedSync import CachedSync
-from core.models.orm_models.Location import Location
-from core.models.orm_models.Operator import Operator
-from core.models.orm_models.EncryptedProxy import EncryptedProxy
+from core.models.manage.session_management import get_session
 
 def insert_into_model(model_class: Type, all_data: dict | list, override=False) -> bool:
     """

core/models/manage/session_management.py

@@ -1,120 +1,40 @@
-from core.errors.logger import logger
-from core.models.orm_models.Base import BaseModel
-from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
-import os
-from pathlib import Path
+from sqlalchemy import create_engine
+from core.models.orm_models.Base import Base
+from core.Constants import Constants
 
-"""
-Note:
-At the bottom it initializes the Session and sets it up outside the function loose.
-"""
-
-# ============================================================================
-# PATH & INITIALIZATION
-# ============================================================================
-
-def get_path():
-    """Returns XDG Base Directory path or ~/.local/share/hydra-veil"""
-    xdg_data = os.getenv("XDG_DATA_HOME")
-    if xdg_data:
-        return Path(xdg_data) / "my-app"
-    return Path.home() / ".local" / "share" / "hydra-veil"
-
-# ============================================================================
-# GLOBAL STATE
-# ============================================================================
-
-system_path = get_path()
-database_path = system_path / "storage.db"
-engine = None
-Session = None
-_session = None
-
-# ============================================================================
-# ENGINE & SESSION MANAGEMENT
-# ============================================================================
-
-def _reinitialize_engine_and_session():
-    """
-    Recreate the global engine and Session factory.
-    Call this after deleting the database to sync globals with the new DB file.
-    """
-    global engine, Session
-    
-    system_path.mkdir(parents=True, exist_ok=True)
+database_path = Constants.HV_STORAGE_DATABASE_PATH
 engine = create_engine(f"sqlite:///{database_path}")
 Session = sessionmaker(bind=engine)
 
+_session = None
 
 def init_session():
-    """Initialize the global _session from the global Session factory."""
+    """
+    Called once at application startup using global variables for the session, 
+    which initialize prior to being called with a false flag.
+
+    The reason for this is so the app can load prior to be called,
+    while keeping it a globally accessible variable
+    """
     global _session
-    if Session is None:
-        raise RuntimeError("Session factory not initialized. Call _reinitialize_engine_and_session() first.")
+    Base.metadata.create_all(engine)
     _session = Session()
 
-
 def get_session():
-    """Return the global _session or raise RuntimeError."""
+    """Get the persistent session"""
     if _session is None:
         raise RuntimeError("Session not initialized. Call init_session() first.")
     return _session
 
-
 def close_session():
-    """Close and reset the global _session."""
+    """
+    Used prior to shutting down the application to break SQL connections.
+
+    In the current refactor, it may be used prior to shut down, such as sync completion,
+    because a low amount of the total models currently use the new system.
+    """
     global _session
-    if _session is not None:
+    if _session:
         _session.close()
         _session = None
-
-
-# ============================================================================
-# DATABASE OPERATIONS
-# ============================================================================
-
-def create_ONLY_db_version_table():
-    """Create only the database_version table using the global engine."""
-    if engine is None:
-        raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
-    
-    from core.models.orm_models.DatabaseVersion import database_version
-    database_version.create(engine, checkfirst=True)
-    # DatabaseVersion.__table__.create(engine, checkfirst=True)
-
-
-def create_ALL_tables():
-    """Create all tables from BaseModel.metadata using the global engine."""
-    print("starting create_ALL_tables")
-    if engine is None:
-        raise RuntimeError("Engine not initialized. Call _reinitialize_engine_and_session() first.")
-    print("passed engine test")
-
-    from core.models.orm_models.Location import Location
-    from core.models.orm_models.Operator import Operator
-    from core.models.orm_models.CachedSync import CachedSync
-    from core.models.orm_models.EncryptedProxy import EncryptedProxy
-
-    from core.models.SubscriptionPlan import SubscriptionPlan
-    from core.models.session.ApplicationVersion import ApplicationVersion
-    
-
-    try:
-        print("trying this")
-        BaseModel.metadata.create_all(engine, checkfirst=True)
-        print("Returning True for Creating all Tables..")
-        return True
-    except:
-        print("couldn't make all tables")
-        return False
-
-
-# ============================================================================
-# STARTUP: Initialize engine and session on import
-# ============================================================================
-
-_reinitialize_engine_and_session()
-logger.info("[DB MANAGEMENT] Started Engine in Session Management Module..")
-init_session()
-logger.info("[DB MANAGEMENT] Initialized Session")

core/models/manage/version_check.py

@@ -1,348 +0,0 @@
-from core.models.orm_models.DatabaseVersion import database_version
-from core.Constants import Constants
-from core.errors.logger import logger
-
-
-# generic
-from sqlalchemy import exc, text
-from sqlalchemy.orm import Session
-from typing import Optional
-
-
-def get_database_version(session: Session) -> Optional[int]:
-    """
-    Purpose:
-        Retrieve the current database version from the single-row version table.
-
-    Rank:
-        Helper
-
-    Called By:
-        check_database_compatibility
-
-    Features:
-        Handles errors with multiple items, failed loads, and other conditions.
-
-    Returns:
-        int: The database version, or None if retrieval fails.
-    """
-    func_name = "get_database_version"
-    
-    # Validate session
-    if session is None:
-        logger.error(f"{func_name}: Session is None")
-        return None
-    
-    if not isinstance(session, Session):
-        logger.error(f"{func_name}: Invalid session type: {type(session)}")
-        return None
-    
-    try:
-        # Attempt to query the table
-        logger.debug(f"{func_name}: Attempting to query database_version table")
-        result = session.query(database_version).all()
-        
-        # Handle empty table
-        if not result:
-            logger.warning(f"{func_name}: Table is empty, no version row found")
-            return None
-        
-        # Handle multiple rows (shouldn't happen, but log it)
-        if len(result) > 1:
-            logger.warning(
-                f"{func_name}: Expected 1 row, found {len(result)}. "
-                "Table should contain only a single row. Returning first version."
-            )
-        
-        # Extract version from first row
-        version_row = result[0]
-        version = version_row.version
-        
-        # Type validation
-        if not isinstance(version, int):
-            logger.error(
-                f"{func_name}: Version column type mismatch. "
-                f"Expected int, got {type(version).__name__}: {version}"
-            )
-            return None
-        
-        if version < 0:
-            logger.warning(
-                f"{func_name}: Version is negative ({version}). "
-                "This may indicate corrupted data."
-            )
-        
-        logger.info(f"{func_name}: Successfully retrieved version: {version}")
-        return version
-    
-    except exc.NoSuchTableError:
-        logger.error(
-            f"{func_name}: Table 'database_version' does not exist. "
-            "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
-        )
-        return None
-    
-    except exc.OperationalError as e:
-        logger.error(
-            f"{func_name}: Database operational error (connection, locked, or permissions). "
-            f"Details: {str(e)}"
-        )
-        return None
-    
-    except exc.DatabaseError as e:
-        logger.error(
-            f"{func_name}: General database error. "
-            f"Details: {str(e)}"
-        )
-        return None
-    
-    except exc.StatementError as e:
-        logger.error(
-            f"{func_name}: SQL statement error. "
-            f"Details: {str(e)}"
-        )
-        return None
-    
-    except AttributeError as e:
-        logger.error(
-            f"{func_name}: Row does not have 'version' attribute. "
-            f"Verify table schema matches definition. Details: {str(e)}"
-        )
-        return None
-    
-    except TypeError as e:
-        logger.error(
-            f"{func_name}: Type error when accessing version. "
-            f"Details: {str(e)}"
-        )
-        return None
-    
-    except Exception as e:
-        logger.critical(
-            f"{func_name}: Unexpected exception type {type(e).__name__}. "
-            f"Details: {str(e)}",
-            exc_info=True
-        )
-        return None
-
-
-def insert_new_version(session: Session, new_version: int) -> bool:
-    """
-    Purpose:
-        Insert (or replace) the database version. Clears existing rows and inserts new version.
-
-    Rank:
-        Helper
-
-    Called By:
-        GUI's startup script in __main__
-
-    Args:
-        session: SQLAlchemy session
-        new_version: The version number to insert (must be a positive integer)
-    
-    Returns:
-        bool: True if successful, False otherwise.
-        does NOT raise errors.
-    """
-    func_name = "insert_new_version" # for logging purposes
-    
-    # Validate session
-    if session is None:
-        logger.error(f"{func_name}: Session is None")
-        return False
-    
-    if not isinstance(session, Session):
-        logger.error(f"{func_name}: Invalid session type: {type(session)}")
-        return False
-    
-    # Validate new_version parameter
-    if new_version is None:
-        logger.error(f"{func_name}: new_version is None")
-        return False
-    
-    if not isinstance(new_version, int):
-        logger.error(
-            f"{func_name}: new_version must be an integer, got {type(new_version).__name__}: {new_version}"
-        )
-        return False
-    
-    if new_version < 0:
-        logger.error(
-            f"{func_name}: new_version must be non-negative, got {new_version}"
-        )
-        return False
-    
-    try:
-        logger.debug(f"{func_name}: Starting transaction for version insert/replace")
-        
-        # Check if table exists before proceeding
-        try:
-            session.query(database_version).limit(1).all()
-        except exc.NoSuchTableError:
-            logger.error(
-                f"{func_name}: Table 'database_version' does not exist. "
-                "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
-            )
-            return False
-        
-        # Delete existing rows
-        logger.debug(f"{func_name}: Deleting existing version rows")
-        deleted_count = session.query(database_version).delete()
-        logger.debug(f"{func_name}: Deleted {deleted_count} existing row(s)")
-        
-        if deleted_count > 1:
-            logger.warning(
-                f"{func_name}: Deleted {deleted_count} rows. "
-                "Table should contain only a single row at any time."
-            )
-        
-        # Actually Insert the new version
-        logger.debug(f"{func_name}: Inserting new version {new_version}")
-        insert_stmt = database_version.insert().values(version=new_version)
-        session.execute(insert_stmt)
-        
-        # Commit transaction
-        session.commit()
-        logger.info(f"{func_name}: Successfully inserted version {new_version}")
-        return True
-    
-    except exc.NoSuchTableError:
-        logger.error(
-            f"{func_name}: Table 'database_version' does not exist. "
-            "Ensure the table is created with `database_version.create(engine, checkfirst=True)`"
-        )
-        session.rollback()
-        return False
-    
-    except exc.IntegrityError as e:
-        logger.error(
-            f"{func_name}: Integrity constraint violation (e.g., primary key conflict). "
-            f"Details: {str(e)}"
-        )
-        session.rollback()
-        return False
-    
-    except exc.OperationalError as e:
-        logger.error(
-            f"{func_name}: Database operational error (connection, locked, or permissions). "
-            f"Details: {str(e)}"
-        )
-        session.rollback()
-        return False
-    
-    except exc.DatabaseError as e:
-        logger.error(
-            f"{func_name}: General database error. "
-            f"Details: {str(e)}"
-        )
-        session.rollback()
-        return False
-    
-    except exc.StatementError as e:
-        logger.error(
-            f"{func_name}: SQL statement error. "
-            f"Details: {str(e)}"
-        )
-        session.rollback()
-        return False
-    
-    except ValueError as e:
-        logger.error(
-            f"{func_name}: Value error during insert. "
-            f"Details: {str(e)}"
-        )
-        session.rollback()
-        return False
-    
-    except Exception as e:
-        logger.critical(
-            f"{func_name}: Unexpected exception type {type(e).__name__}. "
-            f"Details: {str(e)}",
-            exc_info=True
-        )
-        session.rollback()
-        return False
-
-"""Spits back a string based on the reason and error."""
-def get_custom_message(reason: str, compatability_dict: dict) -> str:
-    """
-    Called By:
-        GUI's startup script in __main__
-    """
-    if reason == "upgrade":
-        custom_error = "You just upgraded HydraVeil, so let's upgrade your database to handle the new data types."
-    elif reason == "old_app":
-        custom_error = "You're using an old version of HydraVeil, with a newer version of the database.  This means the older app would crash trying to handle the newer data."
-    else:
-        error_msg = compatability_dict.get("error", "Unknown reason.")
-        custom_error = f"There was an error with upgrading your database version.  Please tell customer support: {error_msg}."
-
-    return custom_error
-
-# ============================================================================
-# ORCHESTRATOR
-# ============================================================================
-def check_database_compatibility(session: Session) -> dict:
-    """
-    Purpose:
-        Evaluates if the user's existing database is incompatible with the app's version.
-
-    Rank:
-        Orchestrator
-
-    Called By:
-        GUI's startup script in __main__
-
-    Returns:
-        Dictionary of True/False, with reason
-        NO error raises.
-    """
-    db_version_you_have = get_database_version(session)
-   
-    if db_version_you_have is None:
-        print("Nothing there")
-        # First run, corrupted, or empty
-        return {
-            "result": True,
-            "reason": "no_database" # Allow initialization still, since there's nothing to delete.
-        }
-
-    logger.info(f"[DB MANAGEMENT] The Database version this APP WANTS is {Constants.DB_VERSION_THIS_APP_WANTS}")
-    logger.info(f"[DB MANAGEMENT] The Database version this device currently has is {db_version_you_have}")
-
-    if Constants.DB_VERSION_THIS_APP_WANTS == db_version_you_have:
-        logger.info(f"[DB MANAGEMENT] Great, the database versions match exactly.")
-        return {
-            "result": True,
-            "reason": None,
-        }
-
-    # BAD!! Old version,
-    elif Constants.DB_VERSION_THIS_APP_WANTS < db_version_you_have:
-        logger.info(f"[DB MANAGEMENT] The App version is older, this is NOT compatabile")
-        # make them wipe the tables, or switch versions of the app
-        return {
-           "result": False,
-            "reason": "old_app",
-            "old_db_version": db_version_you_have
-        }
-
-    # Upgrade User:
-    elif Constants.DB_VERSION_THIS_APP_WANTS > db_version_you_have:
-        logger.info(f"[DB MANAGEMENT] The user has an upgraded App, and an older database")
-        return {
-            "result": False,
-            "reason": "upgrade",
-            "old_db_version": db_version_you_have
-        }
-
-    # this is an error
-    else:
-        logger.error(f"[DB MANAGEMENT] Unknown Error inside version check module with the King Orchestration")
-        return {
-            "result": False,
-            "reason": "error",
-            "old_db_version": db_version_you_have,
-            "error": "unknown result"
-        }

core/models/orm_models/Base.py

@@ -1,19 +1,9 @@
 # base.py
 from sqlalchemy.orm import declarative_base
-from sqlalchemy import inspect
 
 Base = declarative_base()
 
 """
-This mapper exists so the children classes
-have the ability to print human-readable strings
-as dictionaries for the UI.
+This will eventually replace the legacy base model,
+but is still required, even if empty, to setup the declarative base. 
 """
-
-class BaseModel(Base):
-    __abstract__ = True
-    
-    def __repr__(self):
-        mapper = inspect(self.__class__)
-        fields = ', '.join(f'{col.name}={getattr(self, col.name)}' for col in mapper.columns)
-        return f'{self.__class__.__name__}({fields})'

core/models/orm_models/DatabaseVersion.py

@@ -1,11 +0,0 @@
-from sqlalchemy import Column, Integer, MetaData, Table
-
-# Separate metadata for the compatibility table only
-compat_metadata = MetaData()
-
-database_version = Table(
-    'database_version',
-    compat_metadata,
-    Column('version', Integer, primary_key=True)
-)
-

core/models/orm_models/EncryptedProxy.py

@@ -1,40 +0,0 @@
-from sqlalchemy import Integer, String, ForeignKey
-from sqlalchemy.orm import declarative_base, mapped_column
-from typing import Optional
-from sqlalchemy.orm import Mapped
-from core.models.orm_models.Base import Base
-from core.models.orm_models.Operator import Operator
-from core.models.orm_models.Location import Location
-
-class EncryptedProxy(Base):
-    __tablename__ = 'encryptedproxies'
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
-    protocol_type: Mapped[str] = mapped_column(String, nullable=False) # changed from name "type"
-
-    username: Mapped[str] = mapped_column(String, nullable=True, default=None)
-    password: Mapped[str] = mapped_column(String, nullable=False)
-
-    links: Mapped[str] = mapped_column(String, nullable=False)
-    subscription_url: Mapped[str] = mapped_column(String, nullable=False)
-
-    # Foreign key column
-    operator_id: Mapped[Optional[int]] = mapped_column(
-        Integer, 
-        ForeignKey("operators.id"), 
-        nullable=True, 
-        default=None
-    )
-
-    # Foreign key column
-    location_id: Mapped[Optional[int]] = mapped_column(
-        Integer, 
-        ForeignKey("locations.id"), 
-        nullable=True, 
-        default=None
-    )
-
-    operator_domain: Mapped[str] = mapped_column(String, nullable=True, default=None)
-    operator_hysteria2_host: Mapped[str] = mapped_column(String, nullable=True, default=None)
-    operator_vless_host: Mapped[str] = mapped_column(String, nullable=True, default=None)
-    server_ip: Mapped[str] = mapped_column(String, nullable=True, default=None) # pre-resolved IP — avoids DNS leak at connect time

core/models/orm_models/Location.py

@@ -1,73 +0,0 @@
-
-from sqlalchemy import Integer, String, ForeignKey
-from sqlalchemy.orm import declarative_base, mapped_column, Mapped, relationship
-from typing import Optional
-from sqlalchemy.orm import Mapped
-from core.models.orm_models.Base import BaseModel
-from core.models.orm_models.Operator import Operator
-
-
-class Location(BaseModel):
-    __tablename__ = 'locations'
-
-    # model primary key, but only used for business lookups
-    id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
-
-    # BUSINESS logic primary keys:
-    country_code: Mapped[str] = mapped_column(String, nullable=False)
-    code: Mapped[str] = mapped_column(String, unique=True, nullable=False)
-
-    # country name:
-    country_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
-
-    # this is CITY name:
-    name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
-
-    time_zone: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
-    
-    # Foreign key column
-    operator_id: Mapped[Optional[int]] = mapped_column(
-        Integer, 
-        ForeignKey("operators.id"), 
-        nullable=True, 
-        default=None
-    )
-    
-    # Relationship with selectin eager loading
-    operator: Mapped[Optional["Operator"]] = relationship(
-        "Operator", 
-        lazy="selectin"
-    )
-
-    provider_name: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
-    available: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
-    is_proxy_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
-    is_wireguard_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
-    is_hysteria2_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
-    is_vless_capable: Mapped[Optional[bool]] = mapped_column(String, nullable=True, default=None)
-    
-    def to_dict(self):
-        return {
-            "country_code": self.country_code,
-            "code": self.code,
-            "time_zone": self.time_zone
-            }
-
-
-# to use:
-# Lookup by location/city codes
-# record = session.get(Location, (country_code, code))
-
-# example data:
-"""
-id: 7
-country_code: is
-country_name: Iceland
-code: 1
-name: Capital Region
-time_zone: Atlantic/Reykjavik
-operator_id: 6
-provider_name: FlokiNET
-is_proxy_capable: 0
-is_wireguard_capable: 1
-"""

core/models/orm_models/Operator.py

@@ -1,18 +0,0 @@
-from sqlalchemy import Integer, String
-from sqlalchemy.orm import declarative_base, mapped_column
-from typing import Optional
-from sqlalchemy.orm import Mapped
-from core.models.orm_models.Base import BaseModel
-
-class Operator(BaseModel):
-    __tablename__ = 'operators'
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
-    name: Mapped[str] = mapped_column(String, nullable=False)
-    public_key: Mapped[str] = mapped_column(String, nullable=True, default=None)
-    nostr_public_key: Mapped[str] = mapped_column(String, nullable=False)
-    nostr_profile_reference: Mapped[str] = mapped_column(String, nullable=False)
-    nostr_attestation_event_reference: Mapped[str] = mapped_column(String, nullable=False)
-
-    # legacy:
-    # operator: Mapped[str] = mapped_column(String, nullable=False)

core/services/WebServiceApiService.py

@@ -1,7 +1,7 @@
 from core.Constants import Constants
 from core.models.ClientVersion import ClientVersion
-# from core.models.Location import Location  # migrated to ORM
-# from core.models.Operator import Operator  # migrated to ORM
+from core.models.Location import Location
+from core.models.Operator import Operator
 from core.models.OperatorProxySession import OperatorProxySession
 from core.models.Subscription import Subscription
 from core.models.SubscriptionPlan import SubscriptionPlan

core/services/sync/get_metadata_from_api.py

@@ -9,12 +9,11 @@ from typing import Optional
 
 # this can be spoofed with the isolation testing comment below
 
-def get_data_from_api(
-    endpoint: str,
+def get_metadata_from_api(
     client_observer: Optional[ClientObserver] = None,
     connection_observer: Optional[ConnectionObserver] = None
 ) -> dict:
-    logger.info("Syncing with the API in get_data_from_api...")
+    logger.info("Syncing with the API...")
 
     # Get and validate the base URL
     rejected_list = [None, False, ""]
@@ -25,7 +24,7 @@ def get_data_from_api(
         return {"success": False, "data": None, "error": error_msg}
 
     # Construct the full endpoint URL
-    url = f"{base_url}/{endpoint}"
+    url = f"{base_url}/cachedsync"
     logger.debug(f"API endpoint: {url}")
 
     try:
@@ -44,8 +43,8 @@ def get_data_from_api(
         if final_result:
             logger.info("Successfully retrieved sync versions metadata from API")
             return {"success": True, "data": final_result, "error": None}
-        else:
-            error_msg = f"API response missing 'data' field at the end of get_data_from_api. This is the response from the other module: {sync_results}"
+        
+        error_msg = "API response missing 'data' field"
         logger.error(error_msg)
         return {"success": False, "data": None, "error": error_msg}
 

core/services/sync/orm_methods/sync_one_orm.py

@@ -1,60 +0,0 @@
-from core.services.sync.get_data_from_api import get_data_from_api
-from core.models.manage.insert import insert_into_model
-from core.models.manage.denormalize import denormalize
-from core.models.orm_models.Base import Base
-from core.observers.ClientObserver import ClientObserver
-from core.observers.ConnectionObserver import ConnectionObserver
-from core.errors.logger import logger
-
-from typing import Optional
-import json
-
-def sync_one_orm_model(
-    which_model: Base,
-    which_endpoint: str,
-    client_observer: Optional[ClientObserver] = None,
-    connection_observer: Optional[ConnectionObserver] = None
-) -> dict:
-
-    api_result = get_data_from_api(which_endpoint, ClientObserver, ConnectionObserver)
-
-    if not api_result.get("success"):
-        error_msg = api_result.get("error", "Unknown API error")
-        logger.error(f"API sync failed for {which_endpoint} with: {error_msg}")
-        return {
-            "success": False,
-            "error": error_msg
-        }
-
-    new_data = api_result.get("data")
-
-    # prep data (denormalize)
-    yaml_filename = f"{which_endpoint}.yaml"
-    denormalized_data = denormalize(new_data, yaml_filename)
-
-    if denormalized_data:
-        # Debug Pretty print with indentation
-        # print(json.dumps(denormalized_data, indent=2))
-
-        try:
-            did_it_work = insert_into_model(which_model, denormalized_data, True)
-            if did_it_work:
-                return {
-                    "success": True
-                }
-            else:
-                return {
-                    "success": False,
-                    "error": f"We got the data from the endpoint {which_endpoint}, but failed to insert into the model.  That raw data is {denormalized_data}"
-                }
-        except:
-            return {
-                "success": False,
-                "error": f"The except triggered on a sync_one_orm_model's failure to insert into the model with the endpoint {which_endpoint}.  We got the raw data of {denormalized_data}"
-            }
-
-    else:
-        return {
-            "success": False,
-            "error": f"We had issues with denormalizing the data from the server.  That data was {new_data}."
-        }

core/services/sync/sync_service.py

@@ -1,7 +1,7 @@
 
 # comparisons & api calls:
 from core.services.sync.compare_tables import compare_tables
-from core.services.sync.get_data_from_api import get_data_from_api
+from core.services.sync.get_metadata_from_api import get_metadata_from_api
 
 # ORM for metadata:
 from core.models.manage.session_management import get_session
@@ -147,9 +147,9 @@ def coordinate_cache_sync(
         - 'filtered_metadata': the metadata from the server after removing keys not in the ORM
     """
 
-    api_result = get_data_from_api("cachedsync", ClientObserver, ConnectionObserver)
+    api_result = get_metadata_from_api(ClientObserver, ConnectionObserver)
 
-    # we trust the 'get_data_from_api' function to give us a dictionary:
+    # we trust the 'get_metadata_from_api' function to give us a dictionary:
     if not api_result.get("success"):
         error_msg = api_result.get("error", "Unknown API error")
         logger.error(f"API sync failed: {error_msg}")