Sync from the main app streamlines into the Connect module, to do async bulk sync. Additionally ApplicationVersion was transitioned from manual SQL to the ORM, for easier compatibility with the newer sync system. The side effect functions of this have been tested to be stable.

assets/yaml_mappings/application_versions.yaml

@@ -0,0 +1,24 @@
+fields:
+  - name: id
+    path: ['id']
+    required: true
+
+  - name: application_code
+    path: ['application', 'code']
+    required: true
+
+  - name: version_number
+    path: ['version_number']
+    required: true
+
+  - name: format_revision
+    path: ['format_revision']
+
+  - name: download_path
+    path: ['download_path']
+
+  - name: released_at
+    path: ['released_at']
+
+  - name: file_hash
+    path: ['file_hash']

change_log.md

@@ -1,6 +1,12 @@
 # Major Change Log: 
 
 
+
+# Sync Now Integrated & ApplicationVersion Model Transition  
+### August 4, 2026  
+Sync from the main app streamlines into the Connect module, to do async bulk sync.  Additionally ApplicationVersion was transitioned from manual SQL to the ORM, for easier compatibility with the newer sync system.  The side effect functions of this have been tested to be stable.  
+</br>
+
 # Connect Module & Async  
 ### August 3, 2026  
 Introduced Connect Module, which manages HTTPx Clients, solves network/DNS issues, and coordinates Tor Bootstraps.  Introduced async & single endpoint full workflows.  This version is stable and tested for DNS & Tor problems.  Also modified Configuration to be Pydantic, instead of @dataclass_json, and changed the connection type to enums.  Further, there's a new enum function to get the new enum types, but the legacy function exists for backwards compatability.  

core/assets/yaml_mappings/application_versions.yaml

@@ -0,0 +1,24 @@
+fields:
+  - name: id
+    path: ['id']
+    required: true
+
+  - name: application_code
+    path: ['application', 'code']
+    required: true
+
+  - name: version_number
+    path: ['version_number']
+    required: true
+
+  - name: format_revision
+    path: ['format_revision']
+
+  - name: download_path
+    path: ['download_path']
+
+  - name: released_at
+    path: ['released_at']
+
+  - name: file_hash
+    path: ['file_hash']

core/controllers/ApplicationController.py

@@ -2,7 +2,9 @@ from core.Constants import Constants
 from core.Errors import CommandNotFoundError
 from core.controllers.SessionStateController import SessionStateController
 from core.models.session.Application import Application
-from core.models.session.ApplicationVersion import ApplicationVersion
+# from core.models.session.ApplicationVersion import ApplicationVersion
+from core.models.orm_models.ApplicationVersion import ApplicationVersion
+
 from core.models.session.SessionProfile import SessionProfile
 from core.models.session.SessionState import SessionState
 from core.observers.ProfileObserver import ProfileObserver

core/controllers/ApplicationVersionController.py

@@ -1,31 +1,59 @@
 from core.Errors import FileIntegrityError, UnsupportedApplicationVersionError, ApplicationAlreadyInstalledError
 from core.controllers.ApplicationController import ApplicationController
 from core.models.session.Application import Application
-from core.models.session.ApplicationVersion import ApplicationVersion
+# from core.models.session.ApplicationVersion import ApplicationVersion
+from core.models.orm_models.ApplicationVersion import ApplicationVersion
+from core.models.manage.wrapper import safe_db_operation, WrapperRollback
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.BaseProfile import get_application_version
+
 from core.observers.ApplicationVersionObserver import ApplicationVersionObserver
 from core.observers.ConnectionObserver import ConnectionObserver
 from core.services.WebServiceApiService import WebServiceApiService
+from core.errors.logger import logger
+
 from io import BytesIO
 from typing import Optional
 import hashlib
 import shutil
 import tarfile
+from sqlalchemy.orm import Session
+from sqlalchemy import select
 
+@safe_db_operation
+def execute_get_all(application: Optional[Application] = None, session: Session = None) -> DatabaseOperation:
+    query = select(ApplicationVersion)
+    
+    if application is not None:
+        query = query.where(ApplicationVersion.application_code == application.code)
+    
+    result = session.execute(query).scalars().all()
+    return DatabaseOperation(valid=True, data=result)
 
 class ApplicationVersionController:
 
     @staticmethod
     def get(application_code: str, version_number: str):
-        return ApplicationVersion.find(application_code, version_number)
+        return get_application_version(application_code, version_number)
 
     @staticmethod
     def get_all(application: Optional[Application] = None):
-        return ApplicationVersion.all(application)
+        database_object = execute_get_all()
+        if database_object.valid:
+            return database_object.data
+        else:
+            logger.error(f"[Application Version Controller] Got invalid SQL Query which could not be solved by the wrapper, with error message {database_object.message} and type {database_object.error_type}")
+            return None
+
+    # legacy:
+    # @staticmethod
+    # def get_all(application: Optional[Application] = None):
+    #     return ApplicationVersion.all(application)
 
     @staticmethod
     def install(application_version: ApplicationVersion, reinstall: bool = False, application_version_observer: Optional[ApplicationVersionObserver] = None, connection_observer: Optional[ConnectionObserver] = None):
 
-        if not application_version.is_supported():
+        if not application_version.is_supported:
             raise UnsupportedApplicationVersionError('The application version in question is not supported.')
 
         if reinstall:

core/controllers/ClientController.py

@@ -60,7 +60,7 @@ class ClientController:
         return not ClientVersionController.is_latest(version)
 
     @staticmethod
-    def sync(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None):
+    def legacy_sync(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None):
         if client_observer is not None:
             client_observer.notify('synchronizing', "Fetching list of new data ..")
 

core/controllers/SyncController.py

@@ -0,0 +1,224 @@
+
+from core.models.manage.session_management import init_session, close_session
+from core.services.sync.sync_service import coordinate_cache_sync, save_metadata
+from core.services.sync.insert_for_orm import insert_one_orm_model
+from core.services.networking.httpx import connect
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.Result import Result, ResultError
+
+
+from core.errors.logger import logger
+
+# ORM models that can be sync'ed:
+from core.models.orm_models.Location import Location
+from core.models.orm_models.Operator import Operator
+from core.Constants import Constants
+
+from core.services.sync import legacy_insert
+
+from core.controllers.ApplicationController import ApplicationController
+from core.controllers.ApplicationVersionController import ApplicationVersionController
+from core.controllers.ClientVersionController import ClientVersionController
+from core.controllers.ConfigurationController import ConfigurationController
+from core.controllers.SubscriptionPlanController import SubscriptionPlanController
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+from core.models.orm_models.ApplicationVersion import ApplicationVersion
+
+
+import sys
+import json
+
+ORM_TABLES = {
+    "locations": Location,
+    "operators": Operator,
+    "application_versions": ApplicationVersion
+}
+
+LEGACY_SQL_FUNCT_DICT = {
+    "applications": legacy_insert.for_applications,
+    "client_version": legacy_insert.for_client_version,
+    "subscriptions": legacy_insert.for_subscriptions,
+}
+
+
+APP_CODES = {
+    "firefox": 1,
+    "chromium": 2,
+    "brave": 3,
+    "librewolf": 5
+}
+
+def call_legacy_insert_function(key: str, new_data: dict):
+    func = LEGACY_SQL_FUNCT_DICT.get(key)
+    if func is None:
+        return DatabaseOperation(valid=False, error_type=DBErrorType.UNKNOWN_MODEL)
+    if key in APP_CODES:
+        code = APP_CODES[key]
+        return func(new_data, code)
+    else:
+        return func(new_data)
+
+
+def get_orm_model(key) -> bool:
+    if key not in ORM_TABLES:
+        return False
+    return ORM_TABLES[key]
+
+
+
+def new_sync(client_observer: ClientObserver, connection_observer: ConnectionObserver) -> Result:
+    client_observer.notify('synchronizing', "Fetching list of new data ..")
+
+    ####################################
+    # METADATA. Should we even sync?
+    ####################################
+    metadata_result = coordinate_cache_sync(client_observer, connection_observer)
+
+    # Outright Error:
+    if not metadata_result["success"]:
+        error_msg = metadata_result["error"]
+        client_observer.notify('synchronizing', f'Error! {error_msg}')
+        return Result(valid=False)
+
+    # Nothing changed if the 'changed_tables' variable does NOT exist
+    changed_tables = metadata_result["changed_tables"]
+    if not changed_tables:
+        client_observer.notify('synchronized')
+        return Result(valid=True)
+
+    # We only make it past this point if there's New Data
+
+    ####################################
+    # API CALLS: GET NEW DATA IN BULK
+    ####################################
+    results = connect.bulk_async(
+        wanted_list=changed_tables,
+        observer=connection_observer
+    )
+
+    if not results.valid:
+        error_msg = f"Sync failed!  {results.error_type}"
+        logger.error(f"{error_msg} {results.message}")
+        client_observer.notify('synchronizing', f'Error! {error_msg}')
+        return Result(valid=False)
+
+    quantity_of_entries = len(results.data)
+    logger.info(f"We have valid API call results.  There are {quantity_of_entries} entries")
+
+    ####################################
+    # LOOP INSERT INTO DATABASE
+    ####################################
+    client_observer.notify('synchronizing', f'Inserting into Database..')
+    skipped = []
+    FIRST_APP_VERSION_LOOP_ITERATION = True
+
+    for key, each_api_result in results.data.items():
+        # logger.info(f"We are inserting {key}")
+        if not each_api_result.valid:
+            logger.info(f"Skipping invalid api response for {key}")
+            skipped.append(key)
+            continue
+
+        logger.info(f"Inserting valid api data for {key} into the Database, and FIRST_APP_VERSION_LOOP_ITERATION is {FIRST_APP_VERSION_LOOP_ITERATION}")
+        each_insert = insert_data(
+            key=key,
+            each_api_calls_data=each_api_result.data,
+            client_observer=client_observer,
+            FIRST_APP_VERSION_LOOP_ITERATION=FIRST_APP_VERSION_LOOP_ITERATION,
+        )
+        logger.info(f"Exited the insert data function with a result of {each_insert.valid}")
+
+        if each_insert.which_table == "application_versions":
+            FIRST_APP_VERSION_LOOP_ITERATION = False
+
+        if each_insert.valid:
+            logger.info(f"Success with insert of {key}")
+            client_observer.notify('synchronizing', f'Inserted {key}')
+            continue
+        else:
+            skipped.append(key)
+            logger.error(f"Database error with inserting {key}, because {each_insert.error_type}")
+            client_observer.notify('synchronizing', f'Failed inserting {key} because {each_insert.message}')
+
+        logger.info("This only gets triggered in errors, but Moving on to the next item..")
+
+    ####################################
+    # FINAL EVALUATION
+    ####################################
+    total_skipped = len(skipped)
+    logger.info(f"We exited the loop, doing the final evaluation.  And a total of {total_skipped} were skipped.")
+    ConfigurationController.update_last_synced_at()
+
+    filtered_metadata = metadata_result["filtered_metadata"] # from the top of the function
+
+    if total_skipped == 0:
+        client_observer.notify('synchronized', "Fetch & Save Complete!")
+        save_successful = save_metadata(filtered_metadata) # the "save_data" function is inside sync_service
+        return Result(valid=True, message="Finshed sync.")
+    elif total_skipped < quantity_of_entries:
+        error_msg = f"Partial Success. {total_skipped} skipped."
+        client_observer.notify('synchronized', error_msg)
+        return Result(valid=True, data=skipped, message=error_msg)
+    else:
+        error_msg = f"Sync Failed. All {total_skipped} entries were skipped!"
+        client_observer.notify('synchronized', error_msg)
+        return Result(valid=False, data=skipped, message="Complete Failure, all data failed to insert.")
+
+
+def insert_data(
+    key: str,
+    each_api_calls_data: dict,
+    client_observer: ClientObserver,
+    FIRST_APP_VERSION_LOOP_ITERATION: bool
+) -> DatabaseOperation:
+
+    ####################################
+    # NEW ORM SYSTEM
+    ####################################
+    new_orm_model = get_orm_model(key)
+
+    if new_orm_model:
+        db_result = insert_one_orm_model(
+            which_key=key,
+            which_model=new_orm_model,
+            new_data=each_api_calls_data,
+            override=True
+        )
+        return db_result
+
+    # application versions
+    if key in APP_CODES:
+        logger.info(f"{key} counts as a new ORM but uses 'application_versions' and FIRST_APP_VERSION_LOOP_ITERATION is {FIRST_APP_VERSION_LOOP_ITERATION}")
+        new_orm_model = get_orm_model("application_versions")
+
+        db_result = insert_one_orm_model(
+            which_key="application_versions",
+            which_model=new_orm_model,
+            new_data=each_api_calls_data,
+            override=FIRST_APP_VERSION_LOOP_ITERATION,
+        )
+        db_result.which_table = "application_versions"
+        return db_result
+
+    ####################################
+    # LEGACY MANUAL SQL
+    ####################################
+    return call_legacy_insert_function(key=key, new_data=each_api_calls_data)
+
+    ####################################
+    # UNKNOWN MODEL
+    ####################################
+    return DatabaseOperation(valid=False, error_type=DBErrorType.UNKNOWN_MODEL)
+
+
+
+# Legacy version:
+# "firefox": legacy_insert.for_application_versions,
+# "chromium": legacy_insert.for_application_versions,
+# "brave": legacy_insert.for_application_versions,
+# "librewolf": legacy_insert.for_application_versions,
+
+# ISOLATION TESTS:
+# changed_tables = ["locations", "operators", "applications", "client_version", "subscriptions", "applications", "firefox", "chromium", "brave", "librewolf"]

core/controllers/profile_state/update_profile.py

@@ -11,7 +11,10 @@ from core.models.system.SystemProfile import SystemProfile
 # ORM Models
 from core.models.orm_models.Location import Location
 from core.models.orm_models.Operator import Operator
+
+# ORM Calls
 from core.models.orm_calls.location_calls import get_profile_location_data
+from core.models.BaseProfile import get_application_version
 
 SYSTEMWIDE_CHOICES = ['wireguard', 'hysteria2', 'vless']
 SESSION_CHOICES = ['wireguard', 'tor', 'proxy']
@@ -72,8 +75,17 @@ def update_profile(
     # ============= BROWSER =============
     elif key == 'browser':
         browser_type, browser_version = new_value.split(':', 1)
-        profile.application_version.application_code = browser_type
-        profile.application_version.version_number = browser_version
+
+        application_version = get_application_version(   # SQLAlchemy
+            application_code=browser_type,
+            version_number=browser_version
+        )
+        # SQLAlchemy foreign key assignment — fills in id, timezone, operator, etc.
+        profile.application_version = application_version
+
+        # legacy:
+        # profile.application_version.application_code = browser_type
+        # profile.application_version.version_number = browser_version
 
     elif key == 'protocol':
         # ============= SYSTEMWIDE =============

core/models/BaseProfile.py

@@ -20,7 +20,9 @@ from core.models.orm_models.Location import Location
 from core.models.orm_models.Operator import Operator
 
 from core.models.Subscription import Subscription
-from core.models.session.ApplicationVersion import ApplicationVersion
+# from core.models.session.ApplicationVersion import ApplicationVersion
+from core.models.orm_models.ApplicationVersion import ApplicationVersion
+
 from dataclasses import dataclass, field, asdict
 from dataclasses_json import config, Exclude, dataclass_json
 
@@ -34,6 +36,10 @@ import tempfile
 from sqlalchemy.orm import Session
 from enum import Enum
 
+
+from core.models.manage.session_management import init_session, get_session
+
+
 @safe_db_operation
 def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation:
     location_object = session.execute(
@@ -43,6 +49,23 @@ def execute_location_sql(country_code: str, city_code: str, session: Session) ->
     ).scalar_one_or_none()
     return location_object
 
+@safe_db_operation
+def execute_application_sql(application_code: str, version_number: str, session: Session) -> DatabaseOperation:
+    """Query for a specific application version"""
+    data = session.query(ApplicationVersion).filter(
+        ApplicationVersion.application_code == application_code,
+        ApplicationVersion.version_number == version_number
+    ).first()
+    return DatabaseOperation(valid=True, data=data)
+
+
+def get_application_version(application_code: str, version_number: str) -> Optional[ApplicationVersion]:
+    database_object = execute_application_sql(application_code, version_number)
+    if database_object.valid:
+        return database_object.data
+    else:
+        return None
+
 
 def get_profile_location_data(country_code: str, city_code: str) -> Location:
     # with get_session() as session:
@@ -52,7 +75,7 @@ def get_profile_location_data(country_code: str, city_code: str) -> Location:
         location_obj = api_reply_object.data
         return location_obj
     else:
-        logger.error(f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}")
+        logger.error(f"[get_profile_location_data] Got invalid SQL Query")
         return None
 
 
@@ -101,7 +124,9 @@ class BaseProfile(ABC):
         return type(self).__name__ == 'SystemProfile'
 
 
-    def save(self: Self):
+    def save(self: Self, app_version_dict: dict = None):
+        print("We are able to trigger save on the parent")
+
         # === SERIALIZATION ===
         config_dict = self.to_dict()
 
@@ -110,6 +135,10 @@ class BaseProfile(ABC):
         if self.location:
             config_dict["location"] = location_dict
 
+        # === APPLICATION ===
+        if app_version_dict:
+            config_dict["application_version"] = app_version_dict
+
         # === FILE I/O ===
         config_file_contents = json.dumps(config_dict, indent=4) + '\n'
 
@@ -251,13 +280,26 @@ class BaseProfile(ABC):
         # =========== SESSION ===========
         if 'application_version' in profile:
             profile['type'] = ProfileType.SESSION
-
             if profile['application_version'] is not None:
-                application_version = ApplicationVersion.find(profile['application_version']['application_code'] or None, profile['application_version']['version_number'] or None)
+                application_version = get_application_version(
+                    profile['application_version']['application_code'] or None, 
+                    profile['application_version']['version_number'] or None
+                )
                 
                 if application_version is not None:
                     profile['application_version'] = application_version
 
+                if application_version is None:
+                    profile['application_version'] = None
+
+            # legacy
+            # if profile['application_version'] is not None:
+            #     application_version = ApplicationVersion.find(profile['application_version']['application_code'] or None, profile['application_version']['version_number'] or None)
+
+            #     if application_version is not None:
+            #         profile['application_version'] = application_version
+
+
             from core.models.session.SessionProfile import SessionProfile
             # noinspection PyUnresolvedReferences
             profile = SessionProfile.from_dict(profile)

core/models/DatabaseOperation.py

@@ -7,6 +7,7 @@ from typing import Optional, Any
 class DBErrorType(Enum):
     """Classified error categories."""
     SUCCESS = "success"
+    WRONG_DATA_FORMAT = "wrong_data_format"
     NEED_MIGRATION = "need_migration"
     MIGRATION_FAILED = "migration_failed"
     OLD_CLIENT_NEW_API = "old_client_new_api"
@@ -19,6 +20,7 @@ class DBErrorType(Enum):
     DATABASE_LOCKED = "database_locked"
     CORRUPTED_DATABASE = "corrupted_database"
     PYTHON_MODEL_STRUCTURE = "python_model_structure"
+    UNKNOWN_MODEL = "unknown_model"
     UNKNOWN = "unknown"
 
 @dataclass

core/models/manage/insert.py

@@ -10,7 +10,7 @@ from core.models.manage.session_management import get_session
 from core.models.orm_models.Base import BaseModel
 
 from core.models.manage.wrapper import safe_db_operation, WrapperRollback
-from core.models.DatabaseOperation import DatabaseOperation
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
 
 # all models it knows how to do:
 from core.models.orm_models.CachedSync import CachedSync
@@ -32,8 +32,6 @@ def insert_into_model(model_class: Type, all_data: dict | list, override=False)
     Returns:
         DatabaseOperation object with true/false
     """
-    logger.info(f"All the public data preparing to be inserted is {all_data}")
-
     # Normalize to list for uniform handling
     data_list = all_data if isinstance(all_data, list) else [all_data]
 
@@ -43,7 +41,7 @@ def insert_into_model(model_class: Type, all_data: dict | list, override=False)
 
 @safe_db_operation
 def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session, override=False) -> DatabaseOperation:
-    """Insert items. Doesn't manage session."""
+    """Insert items. Doesn't manage session.  It's managed by the wrapper safe_db_operation"""
 
     if override:
         logger.info(f"First, WIPING the pre-existing data for {model_class.__name__}. Are you sure you intended to completely delete the old data?")
@@ -56,6 +54,7 @@ def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session,
             instance = model_class(**each_json)
             session.add(instance)
         session.commit()  # Commits both delete + inserts atomically
+        logger.info("Finished wrapped insert!")
         return DatabaseOperation(valid=True)
 
     except TypeError as e:

core/models/manage/session_management.py

@@ -9,6 +9,8 @@ from sqlalchemy.orm import sessionmaker
 import os
 from pathlib import Path
 
+import sys
+
 """
 Note:
 At the bottom it initializes the Session and sets it up outside the function loose.
@@ -143,11 +145,12 @@ def create_ALL_tables():
     from core.models.orm_models.CachedSync import CachedSync
     from core.models.orm_models.EncryptedProxy import EncryptedProxy
     from core.models.SubscriptionPlan import SubscriptionPlan
-    from core.models.session.ApplicationVersion import ApplicationVersion
-    
+    # from core.models.session.ApplicationVersion import ApplicationVersion
+    from core.models.orm_models.ApplicationVersion import ApplicationVersion
     try:
         BaseModel.metadata.create_all(engine, checkfirst=True)
         logger.info("[DB MANAGEMENT] All Tables have been successfully created.")
+        print("created all tables")
         return True
     except:
         logger.error("[DB MANAGEMENT] Fatal Error with creating all tables in the create_ALL_tables function of session management.")

core/models/orm_models/ApplicationVersion.py

@@ -0,0 +1,79 @@
+from sqlalchemy.orm import Mapped
+from core.models.orm_models.Base import BaseModel
+from core.Constants import Constants
+
+from functools import cached_property
+from sqlalchemy import Column, Integer, String, Boolean, DateTime, UniqueConstraint, TypeDecorator
+from sqlalchemy.orm import declarative_base
+from datetime import datetime
+from typing import Optional
+import os
+
+class ISODateTime(TypeDecorator):
+    impl = String
+    cache_ok = True
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            return value.isoformat() if isinstance(value, datetime) else value
+        return value
+
+    def process_result_value(self, value, dialect):
+        if value is not None:
+            return datetime.fromisoformat(value.replace('Z', '+00:00'))
+        return value
+
+
+class ApplicationVersion(BaseModel):
+    __tablename__ = 'application_versions'
+
+    id = Column(Integer, primary_key=True)
+    application_code = Column(String, unique=False, nullable=False)
+    version_number = Column(String, unique=False, nullable=False)
+    format_revision = Column(Integer, nullable=True)
+    download_path = Column(String, unique=True, nullable=True)
+    # released_at = Column(DateTime, nullable=True)
+    released_at = Column(ISODateTime, nullable=True)
+    file_hash = Column(String, nullable=True)
+    
+    __table_args__ = (
+        UniqueConstraint('application_code', 'version_number', name='uq_app_version'),
+    )
+
+
+    @property
+    def supported(self) -> bool:
+        """Computed property: supported if format_revision is 2"""
+        return self.format_revision == 2
+
+    @property
+    def is_supported(self) -> bool:
+        return self.format_revision == 2
+
+
+    @cached_property
+    def installed(self) -> bool:
+        return self.is_installed()
+
+
+    def convert_to_dict(self) -> dict:
+        return {
+            "application_code": self.application_code,
+            "version_number": self.version_number
+        }
+
+    def get_installation_path(self):
+        return f'{Constants.HV_APPLICATION_DATA_HOME}/{self.application_code}/{self.version_number}'
+
+    def is_installed(self):
+        return os.path.isdir(self.get_installation_path()) and len(os.listdir(self.get_installation_path())) > 0
+
+    def get_installed_file_hash(self):
+
+        try:
+            return open(f'{self.get_installation_path()}/.sha3-512').readline().strip()
+        except FileNotFoundError:
+            return None
+
+    def is_fresh(self):
+        return self.is_installed() and (not self.is_supported or self.file_hash == self.get_installed_file_hash())

core/models/session/ApplicationVersion.py

@@ -1,134 +0,0 @@
-from core.Constants import Constants
-from core.models.Model import Model
-from core.models.session.Application import Application
-from dataclasses import dataclass, field
-from dataclasses_json import config, Exclude
-from datetime import datetime
-from dateutil.parser import isoparse
-from marshmallow import fields
-from typing import Optional
-import os
-
-_table_name: str = 'application_versions'
-
-_table_definition: str = """
-    'id' int UNIQUE,
-    'application_code' varchar,
-    'version_number' varchar,
-    'format_revision' int,
-    'download_path' varchar UNIQUE,
-    'released_at' varchar,
-    'file_hash' varchar,
-    UNIQUE(application_code, version_number)
-"""
-
-
-@dataclass
-class ApplicationVersion(Model):
-    application_code: str
-    version_number: str
-    format_revision: Optional[int] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    id: Optional[int] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    download_path: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    released_at: Optional[datetime] = field(
-        default=None,
-        metadata=config(
-            encoder=datetime.isoformat,
-            decoder=datetime.fromisoformat,
-            mm_field=fields.DateTime(format='iso'),
-            exclude=Exclude.ALWAYS
-        )
-    )
-    file_hash: Optional[str] = field(
-        default=None,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    installed: Optional[bool] = field(
-        default=False,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-    supported: Optional[bool] = field(
-        default=False,
-        metadata=config(exclude=Exclude.ALWAYS)
-    )
-
-    def __post_init__(self):
-        self.installed = self.is_installed()
-        self.supported = self.is_supported()
-
-    def get_installation_path(self):
-        return f'{Constants.HV_APPLICATION_DATA_HOME}/{self.application_code}/{self.version_number}'
-
-    def is_installed(self):
-        return os.path.isdir(self.get_installation_path()) and len(os.listdir(self.get_installation_path())) > 0
-
-    def is_supported(self):
-        return self.exists(self.application_code, self.version_number) and self.format_revision == 2
-
-    def get_installed_file_hash(self):
-
-        try:
-            return open(f'{self.get_installation_path()}/.sha3-512').readline().strip()
-        except FileNotFoundError:
-            return None
-
-    def is_fresh(self):
-        return self.is_installed() and (not self.is_supported() or self.file_hash == self.get_installed_file_hash())
-
-    @staticmethod
-    def find_by_id(id: int):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM application_versions WHERE id = ? LIMIT 1', ApplicationVersion.factory, [id])
-
-    @staticmethod
-    def find(application_code: str, version_number: str):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_one('SELECT * FROM application_versions WHERE application_code = ? AND version_number = ? LIMIT 1', ApplicationVersion.factory, [application_code, version_number])
-
-    @staticmethod
-    def all(application: Optional[Application] = None):
-
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-
-        if application is None:
-            return Model._query_all('SELECT * FROM application_versions', ApplicationVersion.factory)
-
-        else:
-            return Model._query_all('SELECT * FROM application_versions WHERE application_code = ?', ApplicationVersion.factory, [application.code])
-
-    @staticmethod
-    def exists(application_code: str, version_number: str):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        return Model._query_exists('SELECT * FROM application_versions WHERE application_code = ? AND version_number = ?', [application_code, version_number])
-
-    @staticmethod
-    def truncate():
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True)
-
-    @staticmethod
-    def save_many(application_versions):
-        Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition)
-        Model._insert_many('INSERT INTO application_versions VALUES(?, ?, ?, ?, ?, ?, ?)', ApplicationVersion.tuple_factory, application_versions)
-
-    @staticmethod
-    def factory(cursor, row):
-
-        database_fields = [column[0] for column in cursor.description]
-
-        application_version = ApplicationVersion(**{key: value for key, value in zip(database_fields, row)})
-        application_version.released_at = isoparse(str(application_version.released_at))
-
-        return application_version
-
-    @staticmethod
-    def tuple_factory(application_version):
-        return application_version.id, application_version.application_code, application_version.version_number, application_version.format_revision, application_version.download_path, application_version.released_at, application_version.file_hash

core/models/session/SessionProfile.py

@@ -1,7 +1,11 @@
 from core.Constants import Constants
 from core.Errors import UnknownTimeZoneError
 from core.models.BaseProfile import BaseProfile
-from core.models.session.ApplicationVersion import ApplicationVersion
+# from core.models.session.ApplicationVersion import ApplicationVersion
+
+from core.models.orm_models.ApplicationVersion import ApplicationVersion
+
+
 from core.models.session.ProxyConfiguration import ProxyConfiguration
 from core.models.session.SessionConnection import SessionConnection
 from dataclasses import dataclass
@@ -22,6 +26,7 @@ class SessionProfile(BaseProfile):
         return self.connection is not None
 
     def save(self):
+        print("We are able to trigger save on the child session")
 
         if 'application_version' in self._get_dirty_keys():
 
@@ -35,7 +40,11 @@ class SessionProfile(BaseProfile):
             self.__delete_proxy_configuration()
             self.__delete_wireguard_configuration()
 
-        super().save()
+        # === APPLICATION ===
+        app_version_dict = self.application_version.convert_to_dict()
+        print(f"session child got {app_version_dict} as dict..")
+
+        super().save(app_version_dict=app_version_dict)
 
     def attach_proxy_configuration(self, proxy_configuration):
 

core/services/WebServiceApiService.py

@@ -7,7 +7,9 @@ from core.models.SubscriptionPlan import SubscriptionPlan
 from core.models.invoice.Invoice import Invoice
 from core.models.invoice.PaymentMethod import PaymentMethod
 from core.models.session.Application import Application
-from core.models.session.ApplicationVersion import ApplicationVersion
+# from core.models.session.ApplicationVersion import ApplicationVersion
+from core.models.orm_models.ApplicationVersion import ApplicationVersion
+
 from core.models.session.ProxyConfiguration import ProxyConfiguration
 from typing import Optional
 import re

core/services/helpers/manage_assets.py

@@ -9,13 +9,16 @@ from pathlib import Path
 import shutil
 import os
 
-def assets_folder_setup():
 initial_appimage_assets = f"{Constants.APPDIR_HOME}/assets"
 current_assets_folder = f"{Constants.HV_DATA_HOME}/assets"
+
+def assets_folder_setup():
     if os.path.exists(current_assets_folder):
         return True
     return copy_folders(initial_appimage_assets, current_assets_folder)
 
+def updated_assets_folder_changes():
+    return copy_folders(initial_appimage_assets, current_assets_folder)
 
 def sudo_assets_folder_setup() -> bool:
     current_assets_folder = f"{Constants.HOME}/Downloads/hydraveil_sudo_scripts"

core/services/networking/httpx/async_batch_requests.py

@@ -13,7 +13,7 @@ async def create_async_clearweb_client() -> httpx.AsyncClient:
     client = httpx.AsyncClient(http2=True, timeout=30)
     return client
 
-async def create_async_tor_client(port: int) -> httpx.AsyncClient:
+async def create_async_tor_client(port: int, already_checked: bool = False) -> httpx.AsyncClient:
 
     logger.info(f"Creating async client on port {port}")
 
@@ -21,8 +21,9 @@ async def create_async_tor_client(port: int) -> httpx.AsyncClient:
     transport = AsyncProxyTransport.from_url(f"socks5://127.0.0.1:{port}")
     client = httpx.AsyncClient(transport=transport, http2=True, timeout=30)
 
-    # for spoofing errors if we don't want to run more checks:
-    # return client
+    if already_checked:
+        return client
+
     try:
         response = await make_async_request(
             method="get",
@@ -50,11 +51,11 @@ async def create_async_tor_client(port: int) -> httpx.AsyncClient:
         return None
 
 
-async def _async_parallel(desired_endpoints: dict, port: int|None = None) -> dict:
+async def _async_parallel(desired_endpoints: dict, port: int|None = None, already_checked: bool = False) -> dict:
 
     # Tor
     if port is not None:
-        client = await create_async_tor_client(port)
+        client = await create_async_tor_client(port, already_checked)
         if client is None:
             return ApiResponse(valid=False, error_type=ErrorType.TOR_NOT_WORKING)
 
@@ -86,9 +87,9 @@ async def _async_parallel(desired_endpoints: dict, port: int|None = None) -> dic
         await client.aclose()
 
 
-def async_parallel(desired_endpoints: dict, port: int|None = None) -> dict:
+def async_parallel(desired_endpoints: dict, port: int|None = None, already_checked: bool = False) -> dict:
     logger.info(f"We got port of {port}")
-    return asyncio.run(_async_parallel(desired_endpoints, port))
+    return asyncio.run(_async_parallel(desired_endpoints, port, already_checked))
     
 
 

core/services/networking/httpx/connect.py

@@ -125,6 +125,7 @@ def _make_client(connection_type: str, observer) -> httpx.Client | ApiResponse:
 
     if _port_used is None:
         _port_used = Constants.DEFAULT_TOR_PORT
+        logger.info(f"Set to default port of {Constants.DEFAULT_TOR_PORT}")
 
     # Tor:
 
@@ -199,10 +200,13 @@ def bulk_async(wanted_list: list, observer: ConnectionObserver) -> ApiResponse:
     connection_type = ConfigurationController.get_connection_enum()
     desired_endpoints = get_endpoints(wanted_list=wanted_list, dns_resolver=False)
 
+    logger.info(f"We are doing a sync via {connection_type} and the desired_endpoints of {desired_endpoints}")
+
     ########################################################
     # CLEARWEB "regular" INTERNET
     ########################################################
     if connection_type == ConnectionChoice.SYSTEM:
+        logger.info("We are entering a clearweb async bulk..")
         return async_parallel(desired_endpoints=desired_endpoints, port=None)
 
 
@@ -211,17 +215,20 @@ def bulk_async(wanted_list: list, observer: ConnectionObserver) -> ApiResponse:
     ########################################################
     if _port_used is None:
         _port_used = Constants.DEFAULT_TOR_PORT
+        already_checked = False
+    else:
+        already_checked = True
 
     # check if that (default) port is even listening:
     listening = ports.is_port_in_use(_port_used)
 
     if listening:
-        results = async_parallel(desired_endpoints=desired_endpoints, port=_port_used)
+        results = async_parallel(desired_endpoints=desired_endpoints, port=_port_used, already_checked=already_checked)
     else:
         new_port = just_only_bootstrap_port(port_used=_port_used, observer=observer)
         if new_port:
             _port_used = new_port # Set this as the port for next time
-            results = async_parallel(desired_endpoints=desired_endpoints, port=new_port)
+            results = async_parallel(desired_endpoints=desired_endpoints, port=new_port, already_checked = False) # Since it's a new port, already_checked is False
         else:
             return ApiResponse(valid=False, error_type=ErrorType.TOR_NOT_WORKING)
 
@@ -267,7 +274,7 @@ def _evaluate_results(results: ApiResponse, observer: ConnectionObserver) -> Api
     # Categorize each result
     ########################################################
     for key, reply in all_data.items():
-        logger.info(f"DEBUG: Processing failure key={key}.  with the reply={reply}, reply.error_type={reply.error_type if hasattr(reply, 'error_type') else 'NO ATTR'}")
+        logger.info(f"DEBUG: Processing {key}")
         if reply.valid:
             working_results[key] = reply
             continue
@@ -304,7 +311,7 @@ def _evaluate_results(results: ApiResponse, observer: ConnectionObserver) -> Api
         tor_endpoints = get_endpoints(wanted_list=tor_problem_list, dns_resolver=False)
         new_port = just_only_bootstrap_port(port_tried=_port_used, observer=observer)
         if new_port:
-            return async_parallel(desired_endpoints=tor_endpoints, port=new_port)
+            return async_parallel(desired_endpoints=tor_endpoints, port=new_port, already_checked = False) # new port means already_checked = False
         else:
             # Preserve working results—caller sees what *did* work
             return ApiResponse(valid=False, data=working_results, error_type=ErrorType.TOR_NOT_WORKING)
@@ -313,6 +320,7 @@ def _evaluate_results(results: ApiResponse, observer: ConnectionObserver) -> Api
     # IF IT MADE IT HERE THEN: Either it worked, or we can't solve it.
     ########################################################
     # Send working results — Caller can't solve the failure if we can't.
+    logger.info(f"Returning working results")
     return ApiResponse(valid=True, data=working_results)
 
 

core/services/networking/httpx/endpoints.py

@@ -6,11 +6,16 @@ def switch_endpoint_domain(domain: str) -> dict:
     return {
         "locations": f"https://{domain}/api/v1/locations",
         "operators": f"https://{domain}/api/v1/operators",
-        "client": f"https://{domain}/api/v1/platforms/linux-x86_64/appimage/client-versions",
-        "sub_plans": f"https://{domain}/api/v1/subscription-plans",
+        "client_version": f"https://{domain}/api/v1/platforms/linux-x86_64/appimage/client-versions",
+        "subscriptions": f"https://{domain}/api/v1/subscription-plans",
         "applications": f"https://{domain}/api/v1/platforms/linux-x86_64/applications",
+        "firefox": f"https://{domain}/api/v1/platforms/linux-x86_64/applications/firefox/application-versions",
+        "chromium": f"https://{domain}/api/v1/platforms/linux-x86_64/applications/chromium/application-versions",
+        "brave": f"https://{domain}/api/v1/platforms/linux-x86_64/applications/brave/application-versions",
+        "librewolf": f"https://{domain}/api/v1/platforms/linux-x86_64/applications/librewolf/application-versions"
     }
 
+
 def get_endpoints(wanted_list: list, dns_resolver: bool = False) -> dict:
 
     if dns_resolver:

core/services/sync/insert_for_orm.py

@@ -0,0 +1,67 @@
+# API
+from core.services.networking.api_requests.step1_get_or_post import get_data_from_api
+from core.services.networking.api_requests.ApiResponseModel import ApiResponse
+from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems
+
+# Database
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.models.manage.insert import insert_into_model
+from core.models.manage.denormalize import denormalize
+from core.models.orm_models.Base import Base
+
+# observers & loggers
+from core.observers.ClientObserver import ClientObserver
+from core.observers.ConnectionObserver import ConnectionObserver
+from core.errors.logger import logger
+
+# basic utils
+from core.utils.basic_operations.get_parent_directory import get_parent_directory
+from core.Constants import Constants
+from core.services.helpers.manage_assets import updated_assets_folder_changes
+
+from typing import Optional
+import json
+from pathlib import Path
+
+def insert_one_orm_model(
+    which_key: str,
+    which_model: Base,
+    new_data: dict,
+    override: bool = True
+) -> DatabaseOperation:
+
+    # new_data = api_result.data
+
+    # prep data (denormalize)
+    current_assets_folder = f"{Constants.HV_DATA_HOME}/assets"
+    yaml_filename = f"{which_key}.yaml"
+    full_yaml_path = f"{current_assets_folder}/yaml_mappings/{yaml_filename}"
+    yaml_file = Path(full_yaml_path)
+
+    if yaml_file.is_file():
+        try:
+            denormalized_data = denormalize(new_data, str(full_yaml_path))
+            return insert_into_model(which_model, denormalized_data, override)
+        except Exception as e:
+            error_msg = f"We had issues with denormalizing the data from the server: {str(e)}  That data was {new_data}."
+            logger.error(error_msg)
+            return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg)
+
+    else:
+        logger.info(f"We did not find the yaml asset at {yaml_file}, let's try to update it..")
+        if updated_assets_folder_changes():
+            try:
+                denormalized_data = denormalize(new_data, str(full_yaml_path))
+                return insert_into_model(which_model, denormalized_data, override)
+            except Exception as e:
+                error_msg = f"We had issues with denormalizing the data from the server: {str(e)}  That data was {new_data}."
+                logger.error(error_msg)
+                return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg)
+
+        # otherwise:
+        logger.info(f"Skipping denormalization for {which_key} because there was no assets folder yaml at path {yaml_file}")
+        extracted_data = new_data.get('data', new_data)
+        return insert_into_model(which_model, extracted_data, override)
+
+
+

core/services/sync/legacy_insert.py

@@ -0,0 +1,87 @@
+
+from core.models.session.Application import Application
+from core.models.ClientVersion import ClientVersion
+from core.models.SubscriptionPlan import SubscriptionPlan
+# from core.models.session.ApplicationVersion import ApplicationVersion
+from core.models.orm_models.ApplicationVersion import ApplicationVersion
+
+from core.models.DatabaseOperation import DatabaseOperation, DBErrorType
+from core.errors.logger import logger
+
+
+def for_applications(api_response: dict) -> DatabaseOperation:
+    logger.info("Triggered for_applications")
+    try:
+        applications_list = []
+
+        for application in api_response['data']:
+            logger.info(f"doing an application {application['name']}")
+            applications_list.append(Application(application['code'], application['name'], application['id']))
+
+        Application.truncate()
+        Application.save_many(applications_list)
+        logger.info("Inserted applications!!!")
+        return DatabaseOperation(valid=True)
+    except Exception as e:
+        logger.info(f"Error with legacy applications insert: {str(e)}")
+        return DatabaseOperation(valid=False, error_type= DBErrorType.UNKNOWN, message=str(e))
+
+
+def for_client_version(api_response: dict) -> DatabaseOperation:
+    client_versions = []
+    try:
+        for client_version in api_response['data']:
+            client_versions.append(ClientVersion(client_version['version_number'], client_version['released_at'], client_version['id'], client_version['download_path']))
+
+        ClientVersion.truncate()
+        ClientVersion.save_many(client_versions)
+
+        return DatabaseOperation(valid=True)
+    except Exception as e:
+        logger.info(f"Error with legacy client_version insert: {str(e)}")
+        return DatabaseOperation(valid=False, error_type= DBErrorType.UNKNOWN, message=str(e))
+
+
+def for_subscriptions(api_response: dict) -> DatabaseOperation:
+    subscription_plans = []
+    try:
+        for subscription_plan in api_response['data']:
+            subscription_plans.append(SubscriptionPlan(subscription_plan['id'], subscription_plan['code'], subscription_plan['wireguard_session_limit'], subscription_plan['duration'], subscription_plan['price'], subscription_plan['features_proxy'], subscription_plan['features_wireguard']))
+
+        SubscriptionPlan.truncate()
+        SubscriptionPlan.save_many(subscription_plans)
+
+        return DatabaseOperation(valid=True)
+    except Exception as e:
+        logger.info(f"Error with legacy for_subscriptions insert: {str(e)}")
+        return DatabaseOperation(valid=False, error_type= DBErrorType.UNKNOWN, message=str(e))
+
+
+# APP_CODES = {
+#     "firefox": 1,
+#     "chromium": 2,
+#     "brave": 3,
+#     "librewolf": 5
+# }
+
+def for_application_versions(api_response: dict, code: str) -> DatabaseOperation:
+    try:
+        application_versions = []
+        for application_version in api_response['data']:
+            # print(f"application_version['format_revision'] is {application_version['format_revision']}")
+            app = ApplicationVersion(code, 
+            application_version['version_number'],
+            application_version['format_revision'],
+            application_version['id'],
+            application_version['download_path'],
+            application_version['released_at'],
+            application_version['file_hash'])
+            application_versions.append(app)
+
+        ApplicationVersion.truncate()
+        ApplicationVersion.save_many(application_versions)
+
+        return DatabaseOperation(valid=True)
+    except Exception as e:
+        logger.info(f"Error with legacy for_application_versions insert: {str(e)}")
+        return DatabaseOperation(valid=False, error_type= DBErrorType.UNKNOWN, message=str(e))

core/services/sync/orm_methods/sync_one_orm.py

@@ -9,7 +9,7 @@ from core.models.manage.insert import insert_into_model
 from core.models.manage.denormalize import denormalize
 from core.models.orm_models.Base import Base
 
-# observers & loiggers
+# observers & loggers
 from core.observers.ClientObserver import ClientObserver
 from core.observers.ConnectionObserver import ConnectionObserver
 from core.errors.logger import logger

core/services/sync/sync_service.py

@@ -1,5 +1,8 @@
 
 # comparisons & api calls:
+from core.services.networking.httpx import connect
+
+
 from core.services.sync.compare_tables import compare_tables
 # from core.services.sync.get_data_from_api import get_data_from_api