diff --git a/assets/yaml_mappings/application_versions.yaml b/assets/yaml_mappings/application_versions.yaml new file mode 100644 index 0000000..9bbeb6b --- /dev/null +++ b/assets/yaml_mappings/application_versions.yaml @@ -0,0 +1,24 @@ +fields: + - name: id + path: ['id'] + required: true + + - name: application_code + path: ['application', 'code'] + required: true + + - name: version_number + path: ['version_number'] + required: true + + - name: format_revision + path: ['format_revision'] + + - name: download_path + path: ['download_path'] + + - name: released_at + path: ['released_at'] + + - name: file_hash + path: ['file_hash'] diff --git a/change_log.md b/change_log.md index 6c4c067..88c12a1 100644 --- a/change_log.md +++ b/change_log.md @@ -1,6 +1,12 @@ # Major Change Log: + +# Sync Now Integrated & ApplicationVersion Model Transition +### August 4, 2026 +Sync from the main app streamlines into the Connect module, to do async bulk sync. Additionally ApplicationVersion was transitioned from manual SQL to the ORM, for easier compatibility with the newer sync system. The side effect functions of this have been tested to be stable. +
+ # Connect Module & Async ### August 3, 2026 Introduced Connect Module, which manages HTTPx Clients, solves network/DNS issues, and coordinates Tor Bootstraps. Introduced async & single endpoint full workflows. This version is stable and tested for DNS & Tor problems. Also modified Configuration to be Pydantic, instead of @dataclass_json, and changed the connection type to enums. Further, there's a new enum function to get the new enum types, but the legacy function exists for backwards compatability. diff --git a/core/assets/yaml_mappings/application_versions.yaml b/core/assets/yaml_mappings/application_versions.yaml new file mode 100644 index 0000000..9bbeb6b --- /dev/null +++ b/core/assets/yaml_mappings/application_versions.yaml @@ -0,0 +1,24 @@ +fields: + - name: id + path: ['id'] + required: true + + - name: application_code + path: ['application', 'code'] + required: true + + - name: version_number + path: ['version_number'] + required: true + + - name: format_revision + path: ['format_revision'] + + - name: download_path + path: ['download_path'] + + - name: released_at + path: ['released_at'] + + - name: file_hash + path: ['file_hash'] diff --git a/core/controllers/ApplicationController.py b/core/controllers/ApplicationController.py index ce9d165..06300c5 100644 --- a/core/controllers/ApplicationController.py +++ b/core/controllers/ApplicationController.py @@ -2,7 +2,9 @@ from core.Constants import Constants from core.Errors import CommandNotFoundError from core.controllers.SessionStateController import SessionStateController from core.models.session.Application import Application -from core.models.session.ApplicationVersion import ApplicationVersion +# from core.models.session.ApplicationVersion import ApplicationVersion +from core.models.orm_models.ApplicationVersion import ApplicationVersion + from core.models.session.SessionProfile import SessionProfile from core.models.session.SessionState import SessionState from core.observers.ProfileObserver import ProfileObserver diff --git a/core/controllers/ApplicationVersionController.py b/core/controllers/ApplicationVersionController.py index 0193342..95d1a4a 100644 --- a/core/controllers/ApplicationVersionController.py +++ b/core/controllers/ApplicationVersionController.py @@ -1,31 +1,59 @@ from core.Errors import FileIntegrityError, UnsupportedApplicationVersionError, ApplicationAlreadyInstalledError from core.controllers.ApplicationController import ApplicationController from core.models.session.Application import Application -from core.models.session.ApplicationVersion import ApplicationVersion +# from core.models.session.ApplicationVersion import ApplicationVersion +from core.models.orm_models.ApplicationVersion import ApplicationVersion +from core.models.manage.wrapper import safe_db_operation, WrapperRollback +from core.models.DatabaseOperation import DatabaseOperation, DBErrorType +from core.models.BaseProfile import get_application_version + from core.observers.ApplicationVersionObserver import ApplicationVersionObserver from core.observers.ConnectionObserver import ConnectionObserver from core.services.WebServiceApiService import WebServiceApiService +from core.errors.logger import logger + from io import BytesIO from typing import Optional import hashlib import shutil import tarfile +from sqlalchemy.orm import Session +from sqlalchemy import select +@safe_db_operation +def execute_get_all(application: Optional[Application] = None, session: Session = None) -> DatabaseOperation: + query = select(ApplicationVersion) + + if application is not None: + query = query.where(ApplicationVersion.application_code == application.code) + + result = session.execute(query).scalars().all() + return DatabaseOperation(valid=True, data=result) class ApplicationVersionController: @staticmethod def get(application_code: str, version_number: str): - return ApplicationVersion.find(application_code, version_number) + return get_application_version(application_code, version_number) @staticmethod def get_all(application: Optional[Application] = None): - return ApplicationVersion.all(application) + database_object = execute_get_all() + if database_object.valid: + return database_object.data + else: + logger.error(f"[Application Version Controller] Got invalid SQL Query which could not be solved by the wrapper, with error message {database_object.message} and type {database_object.error_type}") + return None + + # legacy: + # @staticmethod + # def get_all(application: Optional[Application] = None): + # return ApplicationVersion.all(application) @staticmethod def install(application_version: ApplicationVersion, reinstall: bool = False, application_version_observer: Optional[ApplicationVersionObserver] = None, connection_observer: Optional[ConnectionObserver] = None): - if not application_version.is_supported(): + if not application_version.is_supported: raise UnsupportedApplicationVersionError('The application version in question is not supported.') if reinstall: diff --git a/core/controllers/ClientController.py b/core/controllers/ClientController.py index 1a0ee25..5e8289d 100644 --- a/core/controllers/ClientController.py +++ b/core/controllers/ClientController.py @@ -60,7 +60,7 @@ class ClientController: return not ClientVersionController.is_latest(version) @staticmethod - def sync(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None): + def legacy_sync(client_observer: ClientObserver = None, connection_observer: ConnectionObserver = None): if client_observer is not None: client_observer.notify('synchronizing', "Fetching list of new data ..") diff --git a/core/controllers/SyncController.py b/core/controllers/SyncController.py new file mode 100644 index 0000000..7d2e241 --- /dev/null +++ b/core/controllers/SyncController.py @@ -0,0 +1,224 @@ + +from core.models.manage.session_management import init_session, close_session +from core.services.sync.sync_service import coordinate_cache_sync, save_metadata +from core.services.sync.insert_for_orm import insert_one_orm_model +from core.services.networking.httpx import connect +from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.models.DatabaseOperation import DatabaseOperation, DBErrorType +from core.models.Result import Result, ResultError + + +from core.errors.logger import logger + +# ORM models that can be sync'ed: +from core.models.orm_models.Location import Location +from core.models.orm_models.Operator import Operator +from core.Constants import Constants + +from core.services.sync import legacy_insert + +from core.controllers.ApplicationController import ApplicationController +from core.controllers.ApplicationVersionController import ApplicationVersionController +from core.controllers.ClientVersionController import ClientVersionController +from core.controllers.ConfigurationController import ConfigurationController +from core.controllers.SubscriptionPlanController import SubscriptionPlanController +from core.observers.ClientObserver import ClientObserver +from core.observers.ConnectionObserver import ConnectionObserver +from core.models.orm_models.ApplicationVersion import ApplicationVersion + + +import sys +import json + +ORM_TABLES = { + "locations": Location, + "operators": Operator, + "application_versions": ApplicationVersion +} + +LEGACY_SQL_FUNCT_DICT = { + "applications": legacy_insert.for_applications, + "client_version": legacy_insert.for_client_version, + "subscriptions": legacy_insert.for_subscriptions, +} + + +APP_CODES = { + "firefox": 1, + "chromium": 2, + "brave": 3, + "librewolf": 5 +} + +def call_legacy_insert_function(key: str, new_data: dict): + func = LEGACY_SQL_FUNCT_DICT.get(key) + if func is None: + return DatabaseOperation(valid=False, error_type=DBErrorType.UNKNOWN_MODEL) + if key in APP_CODES: + code = APP_CODES[key] + return func(new_data, code) + else: + return func(new_data) + + +def get_orm_model(key) -> bool: + if key not in ORM_TABLES: + return False + return ORM_TABLES[key] + + + +def new_sync(client_observer: ClientObserver, connection_observer: ConnectionObserver) -> Result: + client_observer.notify('synchronizing', "Fetching list of new data ..") + + #################################### + # METADATA. Should we even sync? + #################################### + metadata_result = coordinate_cache_sync(client_observer, connection_observer) + + # Outright Error: + if not metadata_result["success"]: + error_msg = metadata_result["error"] + client_observer.notify('synchronizing', f'Error! {error_msg}') + return Result(valid=False) + + # Nothing changed if the 'changed_tables' variable does NOT exist + changed_tables = metadata_result["changed_tables"] + if not changed_tables: + client_observer.notify('synchronized') + return Result(valid=True) + + # We only make it past this point if there's New Data + + #################################### + # API CALLS: GET NEW DATA IN BULK + #################################### + results = connect.bulk_async( + wanted_list=changed_tables, + observer=connection_observer + ) + + if not results.valid: + error_msg = f"Sync failed! {results.error_type}" + logger.error(f"{error_msg} {results.message}") + client_observer.notify('synchronizing', f'Error! {error_msg}') + return Result(valid=False) + + quantity_of_entries = len(results.data) + logger.info(f"We have valid API call results. There are {quantity_of_entries} entries") + + #################################### + # LOOP INSERT INTO DATABASE + #################################### + client_observer.notify('synchronizing', f'Inserting into Database..') + skipped = [] + FIRST_APP_VERSION_LOOP_ITERATION = True + + for key, each_api_result in results.data.items(): + # logger.info(f"We are inserting {key}") + if not each_api_result.valid: + logger.info(f"Skipping invalid api response for {key}") + skipped.append(key) + continue + + logger.info(f"Inserting valid api data for {key} into the Database, and FIRST_APP_VERSION_LOOP_ITERATION is {FIRST_APP_VERSION_LOOP_ITERATION}") + each_insert = insert_data( + key=key, + each_api_calls_data=each_api_result.data, + client_observer=client_observer, + FIRST_APP_VERSION_LOOP_ITERATION=FIRST_APP_VERSION_LOOP_ITERATION, + ) + logger.info(f"Exited the insert data function with a result of {each_insert.valid}") + + if each_insert.which_table == "application_versions": + FIRST_APP_VERSION_LOOP_ITERATION = False + + if each_insert.valid: + logger.info(f"Success with insert of {key}") + client_observer.notify('synchronizing', f'Inserted {key}') + continue + else: + skipped.append(key) + logger.error(f"Database error with inserting {key}, because {each_insert.error_type}") + client_observer.notify('synchronizing', f'Failed inserting {key} because {each_insert.message}') + + logger.info("This only gets triggered in errors, but Moving on to the next item..") + + #################################### + # FINAL EVALUATION + #################################### + total_skipped = len(skipped) + logger.info(f"We exited the loop, doing the final evaluation. And a total of {total_skipped} were skipped.") + ConfigurationController.update_last_synced_at() + + filtered_metadata = metadata_result["filtered_metadata"] # from the top of the function + + if total_skipped == 0: + client_observer.notify('synchronized', "Fetch & Save Complete!") + save_successful = save_metadata(filtered_metadata) # the "save_data" function is inside sync_service + return Result(valid=True, message="Finshed sync.") + elif total_skipped < quantity_of_entries: + error_msg = f"Partial Success. {total_skipped} skipped." + client_observer.notify('synchronized', error_msg) + return Result(valid=True, data=skipped, message=error_msg) + else: + error_msg = f"Sync Failed. All {total_skipped} entries were skipped!" + client_observer.notify('synchronized', error_msg) + return Result(valid=False, data=skipped, message="Complete Failure, all data failed to insert.") + + +def insert_data( + key: str, + each_api_calls_data: dict, + client_observer: ClientObserver, + FIRST_APP_VERSION_LOOP_ITERATION: bool +) -> DatabaseOperation: + + #################################### + # NEW ORM SYSTEM + #################################### + new_orm_model = get_orm_model(key) + + if new_orm_model: + db_result = insert_one_orm_model( + which_key=key, + which_model=new_orm_model, + new_data=each_api_calls_data, + override=True + ) + return db_result + + # application versions + if key in APP_CODES: + logger.info(f"{key} counts as a new ORM but uses 'application_versions' and FIRST_APP_VERSION_LOOP_ITERATION is {FIRST_APP_VERSION_LOOP_ITERATION}") + new_orm_model = get_orm_model("application_versions") + + db_result = insert_one_orm_model( + which_key="application_versions", + which_model=new_orm_model, + new_data=each_api_calls_data, + override=FIRST_APP_VERSION_LOOP_ITERATION, + ) + db_result.which_table = "application_versions" + return db_result + + #################################### + # LEGACY MANUAL SQL + #################################### + return call_legacy_insert_function(key=key, new_data=each_api_calls_data) + + #################################### + # UNKNOWN MODEL + #################################### + return DatabaseOperation(valid=False, error_type=DBErrorType.UNKNOWN_MODEL) + + + +# Legacy version: +# "firefox": legacy_insert.for_application_versions, +# "chromium": legacy_insert.for_application_versions, +# "brave": legacy_insert.for_application_versions, +# "librewolf": legacy_insert.for_application_versions, + +# ISOLATION TESTS: +# changed_tables = ["locations", "operators", "applications", "client_version", "subscriptions", "applications", "firefox", "chromium", "brave", "librewolf"] diff --git a/core/controllers/profile_state/update_profile.py b/core/controllers/profile_state/update_profile.py index 847c1d5..3c3c0ad 100644 --- a/core/controllers/profile_state/update_profile.py +++ b/core/controllers/profile_state/update_profile.py @@ -11,7 +11,10 @@ from core.models.system.SystemProfile import SystemProfile # ORM Models from core.models.orm_models.Location import Location from core.models.orm_models.Operator import Operator + +# ORM Calls from core.models.orm_calls.location_calls import get_profile_location_data +from core.models.BaseProfile import get_application_version SYSTEMWIDE_CHOICES = ['wireguard', 'hysteria2', 'vless'] SESSION_CHOICES = ['wireguard', 'tor', 'proxy'] @@ -72,8 +75,17 @@ def update_profile( # ============= BROWSER ============= elif key == 'browser': browser_type, browser_version = new_value.split(':', 1) - profile.application_version.application_code = browser_type - profile.application_version.version_number = browser_version + + application_version = get_application_version( # SQLAlchemy + application_code=browser_type, + version_number=browser_version + ) + # SQLAlchemy foreign key assignment — fills in id, timezone, operator, etc. + profile.application_version = application_version + + # legacy: + # profile.application_version.application_code = browser_type + # profile.application_version.version_number = browser_version elif key == 'protocol': # ============= SYSTEMWIDE ============= diff --git a/core/models/BaseProfile.py b/core/models/BaseProfile.py index 7f560cb..85afaa8 100644 --- a/core/models/BaseProfile.py +++ b/core/models/BaseProfile.py @@ -20,7 +20,9 @@ from core.models.orm_models.Location import Location from core.models.orm_models.Operator import Operator from core.models.Subscription import Subscription -from core.models.session.ApplicationVersion import ApplicationVersion +# from core.models.session.ApplicationVersion import ApplicationVersion +from core.models.orm_models.ApplicationVersion import ApplicationVersion + from dataclasses import dataclass, field, asdict from dataclasses_json import config, Exclude, dataclass_json @@ -34,6 +36,10 @@ import tempfile from sqlalchemy.orm import Session from enum import Enum + +from core.models.manage.session_management import init_session, get_session + + @safe_db_operation def execute_location_sql(country_code: str, city_code: str, session: Session) -> DatabaseOperation: location_object = session.execute( @@ -43,6 +49,23 @@ def execute_location_sql(country_code: str, city_code: str, session: Session) -> ).scalar_one_or_none() return location_object +@safe_db_operation +def execute_application_sql(application_code: str, version_number: str, session: Session) -> DatabaseOperation: + """Query for a specific application version""" + data = session.query(ApplicationVersion).filter( + ApplicationVersion.application_code == application_code, + ApplicationVersion.version_number == version_number + ).first() + return DatabaseOperation(valid=True, data=data) + + +def get_application_version(application_code: str, version_number: str) -> Optional[ApplicationVersion]: + database_object = execute_application_sql(application_code, version_number) + if database_object.valid: + return database_object.data + else: + return None + def get_profile_location_data(country_code: str, city_code: str) -> Location: # with get_session() as session: @@ -52,7 +75,7 @@ def get_profile_location_data(country_code: str, city_code: str) -> Location: location_obj = api_reply_object.data return location_obj else: - logger.error(f"[BaseProfile] Got invalid SQL Query which could not be solved by the wrapper, with error message {location_object.message} and type {location_object.error_type}") + logger.error(f"[get_profile_location_data] Got invalid SQL Query") return None @@ -101,7 +124,9 @@ class BaseProfile(ABC): return type(self).__name__ == 'SystemProfile' - def save(self: Self): + def save(self: Self, app_version_dict: dict = None): + print("We are able to trigger save on the parent") + # === SERIALIZATION === config_dict = self.to_dict() @@ -110,6 +135,10 @@ class BaseProfile(ABC): if self.location: config_dict["location"] = location_dict + # === APPLICATION === + if app_version_dict: + config_dict["application_version"] = app_version_dict + # === FILE I/O === config_file_contents = json.dumps(config_dict, indent=4) + '\n' @@ -251,13 +280,26 @@ class BaseProfile(ABC): # =========== SESSION =========== if 'application_version' in profile: profile['type'] = ProfileType.SESSION - if profile['application_version'] is not None: - application_version = ApplicationVersion.find(profile['application_version']['application_code'] or None, profile['application_version']['version_number'] or None) - + application_version = get_application_version( + profile['application_version']['application_code'] or None, + profile['application_version']['version_number'] or None + ) + if application_version is not None: profile['application_version'] = application_version + if application_version is None: + profile['application_version'] = None + + # legacy + # if profile['application_version'] is not None: + # application_version = ApplicationVersion.find(profile['application_version']['application_code'] or None, profile['application_version']['version_number'] or None) + + # if application_version is not None: + # profile['application_version'] = application_version + + from core.models.session.SessionProfile import SessionProfile # noinspection PyUnresolvedReferences profile = SessionProfile.from_dict(profile) diff --git a/core/models/DatabaseOperation.py b/core/models/DatabaseOperation.py index 4fa368f..ffa6114 100644 --- a/core/models/DatabaseOperation.py +++ b/core/models/DatabaseOperation.py @@ -7,6 +7,7 @@ from typing import Optional, Any class DBErrorType(Enum): """Classified error categories.""" SUCCESS = "success" + WRONG_DATA_FORMAT = "wrong_data_format" NEED_MIGRATION = "need_migration" MIGRATION_FAILED = "migration_failed" OLD_CLIENT_NEW_API = "old_client_new_api" @@ -19,6 +20,7 @@ class DBErrorType(Enum): DATABASE_LOCKED = "database_locked" CORRUPTED_DATABASE = "corrupted_database" PYTHON_MODEL_STRUCTURE = "python_model_structure" + UNKNOWN_MODEL = "unknown_model" UNKNOWN = "unknown" @dataclass diff --git a/core/models/manage/insert.py b/core/models/manage/insert.py index 84b81eb..6f10790 100644 --- a/core/models/manage/insert.py +++ b/core/models/manage/insert.py @@ -10,7 +10,7 @@ from core.models.manage.session_management import get_session from core.models.orm_models.Base import BaseModel from core.models.manage.wrapper import safe_db_operation, WrapperRollback -from core.models.DatabaseOperation import DatabaseOperation +from core.models.DatabaseOperation import DatabaseOperation, DBErrorType # all models it knows how to do: from core.models.orm_models.CachedSync import CachedSync @@ -32,8 +32,6 @@ def insert_into_model(model_class: Type, all_data: dict | list, override=False) Returns: DatabaseOperation object with true/false """ - logger.info(f"All the public data preparing to be inserted is {all_data}") - # Normalize to list for uniform handling data_list = all_data if isinstance(all_data, list) else [all_data] @@ -43,7 +41,7 @@ def insert_into_model(model_class: Type, all_data: dict | list, override=False) @safe_db_operation def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session, override=False) -> DatabaseOperation: - """Insert items. Doesn't manage session.""" + """Insert items. Doesn't manage session. It's managed by the wrapper safe_db_operation""" if override: logger.info(f"First, WIPING the pre-existing data for {model_class.__name__}. Are you sure you intended to completely delete the old data?") @@ -56,6 +54,7 @@ def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session, instance = model_class(**each_json) session.add(instance) session.commit() # Commits both delete + inserts atomically + logger.info("Finished wrapped insert!") return DatabaseOperation(valid=True) except TypeError as e: diff --git a/core/models/manage/session_management.py b/core/models/manage/session_management.py index 286b126..40c0aa6 100644 --- a/core/models/manage/session_management.py +++ b/core/models/manage/session_management.py @@ -9,6 +9,8 @@ from sqlalchemy.orm import sessionmaker import os from pathlib import Path +import sys + """ Note: At the bottom it initializes the Session and sets it up outside the function loose. @@ -143,11 +145,12 @@ def create_ALL_tables(): from core.models.orm_models.CachedSync import CachedSync from core.models.orm_models.EncryptedProxy import EncryptedProxy from core.models.SubscriptionPlan import SubscriptionPlan - from core.models.session.ApplicationVersion import ApplicationVersion - + # from core.models.session.ApplicationVersion import ApplicationVersion + from core.models.orm_models.ApplicationVersion import ApplicationVersion try: BaseModel.metadata.create_all(engine, checkfirst=True) logger.info("[DB MANAGEMENT] All Tables have been successfully created.") + print("created all tables") return True except: logger.error("[DB MANAGEMENT] Fatal Error with creating all tables in the create_ALL_tables function of session management.") diff --git a/core/models/orm_models/ApplicationVersion.py b/core/models/orm_models/ApplicationVersion.py new file mode 100644 index 0000000..7d36c18 --- /dev/null +++ b/core/models/orm_models/ApplicationVersion.py @@ -0,0 +1,79 @@ +from sqlalchemy.orm import Mapped +from core.models.orm_models.Base import BaseModel +from core.Constants import Constants + +from functools import cached_property +from sqlalchemy import Column, Integer, String, Boolean, DateTime, UniqueConstraint, TypeDecorator +from sqlalchemy.orm import declarative_base +from datetime import datetime +from typing import Optional +import os + +class ISODateTime(TypeDecorator): + impl = String + cache_ok = True + + def process_bind_param(self, value, dialect): + if value is not None: + return value.isoformat() if isinstance(value, datetime) else value + return value + + def process_result_value(self, value, dialect): + if value is not None: + return datetime.fromisoformat(value.replace('Z', '+00:00')) + return value + + +class ApplicationVersion(BaseModel): + __tablename__ = 'application_versions' + + id = Column(Integer, primary_key=True) + application_code = Column(String, unique=False, nullable=False) + version_number = Column(String, unique=False, nullable=False) + format_revision = Column(Integer, nullable=True) + download_path = Column(String, unique=True, nullable=True) + # released_at = Column(DateTime, nullable=True) + released_at = Column(ISODateTime, nullable=True) + file_hash = Column(String, nullable=True) + + __table_args__ = ( + UniqueConstraint('application_code', 'version_number', name='uq_app_version'), + ) + + + @property + def supported(self) -> bool: + """Computed property: supported if format_revision is 2""" + return self.format_revision == 2 + + @property + def is_supported(self) -> bool: + return self.format_revision == 2 + + + @cached_property + def installed(self) -> bool: + return self.is_installed() + + + def convert_to_dict(self) -> dict: + return { + "application_code": self.application_code, + "version_number": self.version_number + } + + def get_installation_path(self): + return f'{Constants.HV_APPLICATION_DATA_HOME}/{self.application_code}/{self.version_number}' + + def is_installed(self): + return os.path.isdir(self.get_installation_path()) and len(os.listdir(self.get_installation_path())) > 0 + + def get_installed_file_hash(self): + + try: + return open(f'{self.get_installation_path()}/.sha3-512').readline().strip() + except FileNotFoundError: + return None + + def is_fresh(self): + return self.is_installed() and (not self.is_supported or self.file_hash == self.get_installed_file_hash()) diff --git a/core/models/session/ApplicationVersion.py b/core/models/session/ApplicationVersion.py deleted file mode 100644 index 39afe27..0000000 --- a/core/models/session/ApplicationVersion.py +++ /dev/null @@ -1,134 +0,0 @@ -from core.Constants import Constants -from core.models.Model import Model -from core.models.session.Application import Application -from dataclasses import dataclass, field -from dataclasses_json import config, Exclude -from datetime import datetime -from dateutil.parser import isoparse -from marshmallow import fields -from typing import Optional -import os - -_table_name: str = 'application_versions' - -_table_definition: str = """ - 'id' int UNIQUE, - 'application_code' varchar, - 'version_number' varchar, - 'format_revision' int, - 'download_path' varchar UNIQUE, - 'released_at' varchar, - 'file_hash' varchar, - UNIQUE(application_code, version_number) -""" - - -@dataclass -class ApplicationVersion(Model): - application_code: str - version_number: str - format_revision: Optional[int] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - id: Optional[int] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - download_path: Optional[str] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - released_at: Optional[datetime] = field( - default=None, - metadata=config( - encoder=datetime.isoformat, - decoder=datetime.fromisoformat, - mm_field=fields.DateTime(format='iso'), - exclude=Exclude.ALWAYS - ) - ) - file_hash: Optional[str] = field( - default=None, - metadata=config(exclude=Exclude.ALWAYS) - ) - installed: Optional[bool] = field( - default=False, - metadata=config(exclude=Exclude.ALWAYS) - ) - supported: Optional[bool] = field( - default=False, - metadata=config(exclude=Exclude.ALWAYS) - ) - - def __post_init__(self): - self.installed = self.is_installed() - self.supported = self.is_supported() - - def get_installation_path(self): - return f'{Constants.HV_APPLICATION_DATA_HOME}/{self.application_code}/{self.version_number}' - - def is_installed(self): - return os.path.isdir(self.get_installation_path()) and len(os.listdir(self.get_installation_path())) > 0 - - def is_supported(self): - return self.exists(self.application_code, self.version_number) and self.format_revision == 2 - - def get_installed_file_hash(self): - - try: - return open(f'{self.get_installation_path()}/.sha3-512').readline().strip() - except FileNotFoundError: - return None - - def is_fresh(self): - return self.is_installed() and (not self.is_supported() or self.file_hash == self.get_installed_file_hash()) - - @staticmethod - def find_by_id(id: int): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_one('SELECT * FROM application_versions WHERE id = ? LIMIT 1', ApplicationVersion.factory, [id]) - - @staticmethod - def find(application_code: str, version_number: str): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_one('SELECT * FROM application_versions WHERE application_code = ? AND version_number = ? LIMIT 1', ApplicationVersion.factory, [application_code, version_number]) - - @staticmethod - def all(application: Optional[Application] = None): - - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - - if application is None: - return Model._query_all('SELECT * FROM application_versions', ApplicationVersion.factory) - - else: - return Model._query_all('SELECT * FROM application_versions WHERE application_code = ?', ApplicationVersion.factory, [application.code]) - - @staticmethod - def exists(application_code: str, version_number: str): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - return Model._query_exists('SELECT * FROM application_versions WHERE application_code = ? AND version_number = ?', [application_code, version_number]) - - @staticmethod - def truncate(): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition, drop_existing=True) - - @staticmethod - def save_many(application_versions): - Model._create_table_if_not_exists(table_name=_table_name, table_definition=_table_definition) - Model._insert_many('INSERT INTO application_versions VALUES(?, ?, ?, ?, ?, ?, ?)', ApplicationVersion.tuple_factory, application_versions) - - @staticmethod - def factory(cursor, row): - - database_fields = [column[0] for column in cursor.description] - - application_version = ApplicationVersion(**{key: value for key, value in zip(database_fields, row)}) - application_version.released_at = isoparse(str(application_version.released_at)) - - return application_version - - @staticmethod - def tuple_factory(application_version): - return application_version.id, application_version.application_code, application_version.version_number, application_version.format_revision, application_version.download_path, application_version.released_at, application_version.file_hash diff --git a/core/models/session/SessionProfile.py b/core/models/session/SessionProfile.py index d5e784c..e712a9d 100644 --- a/core/models/session/SessionProfile.py +++ b/core/models/session/SessionProfile.py @@ -1,7 +1,11 @@ from core.Constants import Constants from core.Errors import UnknownTimeZoneError from core.models.BaseProfile import BaseProfile -from core.models.session.ApplicationVersion import ApplicationVersion +# from core.models.session.ApplicationVersion import ApplicationVersion + +from core.models.orm_models.ApplicationVersion import ApplicationVersion + + from core.models.session.ProxyConfiguration import ProxyConfiguration from core.models.session.SessionConnection import SessionConnection from dataclasses import dataclass @@ -22,6 +26,7 @@ class SessionProfile(BaseProfile): return self.connection is not None def save(self): + print("We are able to trigger save on the child session") if 'application_version' in self._get_dirty_keys(): @@ -35,7 +40,11 @@ class SessionProfile(BaseProfile): self.__delete_proxy_configuration() self.__delete_wireguard_configuration() - super().save() + # === APPLICATION === + app_version_dict = self.application_version.convert_to_dict() + print(f"session child got {app_version_dict} as dict..") + + super().save(app_version_dict=app_version_dict) def attach_proxy_configuration(self, proxy_configuration): diff --git a/core/services/WebServiceApiService.py b/core/services/WebServiceApiService.py index 553f6c3..3cfd0a0 100644 --- a/core/services/WebServiceApiService.py +++ b/core/services/WebServiceApiService.py @@ -7,7 +7,9 @@ from core.models.SubscriptionPlan import SubscriptionPlan from core.models.invoice.Invoice import Invoice from core.models.invoice.PaymentMethod import PaymentMethod from core.models.session.Application import Application -from core.models.session.ApplicationVersion import ApplicationVersion +# from core.models.session.ApplicationVersion import ApplicationVersion +from core.models.orm_models.ApplicationVersion import ApplicationVersion + from core.models.session.ProxyConfiguration import ProxyConfiguration from typing import Optional import re diff --git a/core/services/helpers/manage_assets.py b/core/services/helpers/manage_assets.py index 6a90d18..0c6f092 100644 --- a/core/services/helpers/manage_assets.py +++ b/core/services/helpers/manage_assets.py @@ -9,13 +9,16 @@ from pathlib import Path import shutil import os +initial_appimage_assets = f"{Constants.APPDIR_HOME}/assets" +current_assets_folder = f"{Constants.HV_DATA_HOME}/assets" + def assets_folder_setup(): - initial_appimage_assets = f"{Constants.APPDIR_HOME}/assets" - current_assets_folder = f"{Constants.HV_DATA_HOME}/assets" if os.path.exists(current_assets_folder): return True return copy_folders(initial_appimage_assets, current_assets_folder) +def updated_assets_folder_changes(): + return copy_folders(initial_appimage_assets, current_assets_folder) def sudo_assets_folder_setup() -> bool: current_assets_folder = f"{Constants.HOME}/Downloads/hydraveil_sudo_scripts" diff --git a/core/services/networking/httpx/async_batch_requests.py b/core/services/networking/httpx/async_batch_requests.py index 958b17b..5c48c56 100644 --- a/core/services/networking/httpx/async_batch_requests.py +++ b/core/services/networking/httpx/async_batch_requests.py @@ -13,7 +13,7 @@ async def create_async_clearweb_client() -> httpx.AsyncClient: client = httpx.AsyncClient(http2=True, timeout=30) return client -async def create_async_tor_client(port: int) -> httpx.AsyncClient: +async def create_async_tor_client(port: int, already_checked: bool = False) -> httpx.AsyncClient: logger.info(f"Creating async client on port {port}") @@ -21,8 +21,9 @@ async def create_async_tor_client(port: int) -> httpx.AsyncClient: transport = AsyncProxyTransport.from_url(f"socks5://127.0.0.1:{port}") client = httpx.AsyncClient(transport=transport, http2=True, timeout=30) - # for spoofing errors if we don't want to run more checks: - # return client + if already_checked: + return client + try: response = await make_async_request( method="get", @@ -50,11 +51,11 @@ async def create_async_tor_client(port: int) -> httpx.AsyncClient: return None -async def _async_parallel(desired_endpoints: dict, port: int|None = None) -> dict: +async def _async_parallel(desired_endpoints: dict, port: int|None = None, already_checked: bool = False) -> dict: # Tor if port is not None: - client = await create_async_tor_client(port) + client = await create_async_tor_client(port, already_checked) if client is None: return ApiResponse(valid=False, error_type=ErrorType.TOR_NOT_WORKING) @@ -86,9 +87,9 @@ async def _async_parallel(desired_endpoints: dict, port: int|None = None) -> dic await client.aclose() -def async_parallel(desired_endpoints: dict, port: int|None = None) -> dict: +def async_parallel(desired_endpoints: dict, port: int|None = None, already_checked: bool = False) -> dict: logger.info(f"We got port of {port}") - return asyncio.run(_async_parallel(desired_endpoints, port)) + return asyncio.run(_async_parallel(desired_endpoints, port, already_checked)) diff --git a/core/services/networking/httpx/connect.py b/core/services/networking/httpx/connect.py index 86d167b..b9507d9 100644 --- a/core/services/networking/httpx/connect.py +++ b/core/services/networking/httpx/connect.py @@ -125,6 +125,7 @@ def _make_client(connection_type: str, observer) -> httpx.Client | ApiResponse: if _port_used is None: _port_used = Constants.DEFAULT_TOR_PORT + logger.info(f"Set to default port of {Constants.DEFAULT_TOR_PORT}") # Tor: @@ -199,10 +200,13 @@ def bulk_async(wanted_list: list, observer: ConnectionObserver) -> ApiResponse: connection_type = ConfigurationController.get_connection_enum() desired_endpoints = get_endpoints(wanted_list=wanted_list, dns_resolver=False) + logger.info(f"We are doing a sync via {connection_type} and the desired_endpoints of {desired_endpoints}") + ######################################################## # CLEARWEB "regular" INTERNET ######################################################## if connection_type == ConnectionChoice.SYSTEM: + logger.info("We are entering a clearweb async bulk..") return async_parallel(desired_endpoints=desired_endpoints, port=None) @@ -211,17 +215,20 @@ def bulk_async(wanted_list: list, observer: ConnectionObserver) -> ApiResponse: ######################################################## if _port_used is None: _port_used = Constants.DEFAULT_TOR_PORT + already_checked = False + else: + already_checked = True # check if that (default) port is even listening: listening = ports.is_port_in_use(_port_used) if listening: - results = async_parallel(desired_endpoints=desired_endpoints, port=_port_used) + results = async_parallel(desired_endpoints=desired_endpoints, port=_port_used, already_checked=already_checked) else: new_port = just_only_bootstrap_port(port_used=_port_used, observer=observer) if new_port: _port_used = new_port # Set this as the port for next time - results = async_parallel(desired_endpoints=desired_endpoints, port=new_port) + results = async_parallel(desired_endpoints=desired_endpoints, port=new_port, already_checked = False) # Since it's a new port, already_checked is False else: return ApiResponse(valid=False, error_type=ErrorType.TOR_NOT_WORKING) @@ -267,7 +274,7 @@ def _evaluate_results(results: ApiResponse, observer: ConnectionObserver) -> Api # Categorize each result ######################################################## for key, reply in all_data.items(): - logger.info(f"DEBUG: Processing failure key={key}. with the reply={reply}, reply.error_type={reply.error_type if hasattr(reply, 'error_type') else 'NO ATTR'}") + logger.info(f"DEBUG: Processing {key}") if reply.valid: working_results[key] = reply continue @@ -304,7 +311,7 @@ def _evaluate_results(results: ApiResponse, observer: ConnectionObserver) -> Api tor_endpoints = get_endpoints(wanted_list=tor_problem_list, dns_resolver=False) new_port = just_only_bootstrap_port(port_tried=_port_used, observer=observer) if new_port: - return async_parallel(desired_endpoints=tor_endpoints, port=new_port) + return async_parallel(desired_endpoints=tor_endpoints, port=new_port, already_checked = False) # new port means already_checked = False else: # Preserve working results—caller sees what *did* work return ApiResponse(valid=False, data=working_results, error_type=ErrorType.TOR_NOT_WORKING) @@ -313,6 +320,7 @@ def _evaluate_results(results: ApiResponse, observer: ConnectionObserver) -> Api # IF IT MADE IT HERE THEN: Either it worked, or we can't solve it. ######################################################## # Send working results — Caller can't solve the failure if we can't. + logger.info(f"Returning working results") return ApiResponse(valid=True, data=working_results) diff --git a/core/services/networking/httpx/endpoints.py b/core/services/networking/httpx/endpoints.py index 90ea35f..27f2ad2 100644 --- a/core/services/networking/httpx/endpoints.py +++ b/core/services/networking/httpx/endpoints.py @@ -6,11 +6,16 @@ def switch_endpoint_domain(domain: str) -> dict: return { "locations": f"https://{domain}/api/v1/locations", "operators": f"https://{domain}/api/v1/operators", - "client": f"https://{domain}/api/v1/platforms/linux-x86_64/appimage/client-versions", - "sub_plans": f"https://{domain}/api/v1/subscription-plans", + "client_version": f"https://{domain}/api/v1/platforms/linux-x86_64/appimage/client-versions", + "subscriptions": f"https://{domain}/api/v1/subscription-plans", "applications": f"https://{domain}/api/v1/platforms/linux-x86_64/applications", + "firefox": f"https://{domain}/api/v1/platforms/linux-x86_64/applications/firefox/application-versions", + "chromium": f"https://{domain}/api/v1/platforms/linux-x86_64/applications/chromium/application-versions", + "brave": f"https://{domain}/api/v1/platforms/linux-x86_64/applications/brave/application-versions", + "librewolf": f"https://{domain}/api/v1/platforms/linux-x86_64/applications/librewolf/application-versions" } + def get_endpoints(wanted_list: list, dns_resolver: bool = False) -> dict: if dns_resolver: diff --git a/core/services/sync/insert_for_orm.py b/core/services/sync/insert_for_orm.py new file mode 100644 index 0000000..6e8eab1 --- /dev/null +++ b/core/services/sync/insert_for_orm.py @@ -0,0 +1,67 @@ +# API +from core.services.networking.api_requests.step1_get_or_post import get_data_from_api +from core.services.networking.api_requests.ApiResponseModel import ApiResponse +from core.services.networking.api_requests.step5_solve_api_problems import solve_api_problems + +# Database +from core.models.DatabaseOperation import DatabaseOperation, DBErrorType +from core.models.manage.insert import insert_into_model +from core.models.manage.denormalize import denormalize +from core.models.orm_models.Base import Base + +# observers & loggers +from core.observers.ClientObserver import ClientObserver +from core.observers.ConnectionObserver import ConnectionObserver +from core.errors.logger import logger + +# basic utils +from core.utils.basic_operations.get_parent_directory import get_parent_directory +from core.Constants import Constants +from core.services.helpers.manage_assets import updated_assets_folder_changes + +from typing import Optional +import json +from pathlib import Path + +def insert_one_orm_model( + which_key: str, + which_model: Base, + new_data: dict, + override: bool = True +) -> DatabaseOperation: + + # new_data = api_result.data + + # prep data (denormalize) + current_assets_folder = f"{Constants.HV_DATA_HOME}/assets" + yaml_filename = f"{which_key}.yaml" + full_yaml_path = f"{current_assets_folder}/yaml_mappings/{yaml_filename}" + yaml_file = Path(full_yaml_path) + + if yaml_file.is_file(): + try: + denormalized_data = denormalize(new_data, str(full_yaml_path)) + return insert_into_model(which_model, denormalized_data, override) + except Exception as e: + error_msg = f"We had issues with denormalizing the data from the server: {str(e)} That data was {new_data}." + logger.error(error_msg) + return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg) + + else: + logger.info(f"We did not find the yaml asset at {yaml_file}, let's try to update it..") + if updated_assets_folder_changes(): + try: + denormalized_data = denormalize(new_data, str(full_yaml_path)) + return insert_into_model(which_model, denormalized_data, override) + except Exception as e: + error_msg = f"We had issues with denormalizing the data from the server: {str(e)} That data was {new_data}." + logger.error(error_msg) + return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg) + + # otherwise: + logger.info(f"Skipping denormalization for {which_key} because there was no assets folder yaml at path {yaml_file}") + extracted_data = new_data.get('data', new_data) + return insert_into_model(which_model, extracted_data, override) + + + diff --git a/core/services/sync/legacy_insert.py b/core/services/sync/legacy_insert.py new file mode 100644 index 0000000..4fd673b --- /dev/null +++ b/core/services/sync/legacy_insert.py @@ -0,0 +1,87 @@ + +from core.models.session.Application import Application +from core.models.ClientVersion import ClientVersion +from core.models.SubscriptionPlan import SubscriptionPlan +# from core.models.session.ApplicationVersion import ApplicationVersion +from core.models.orm_models.ApplicationVersion import ApplicationVersion + +from core.models.DatabaseOperation import DatabaseOperation, DBErrorType +from core.errors.logger import logger + + +def for_applications(api_response: dict) -> DatabaseOperation: + logger.info("Triggered for_applications") + try: + applications_list = [] + + for application in api_response['data']: + logger.info(f"doing an application {application['name']}") + applications_list.append(Application(application['code'], application['name'], application['id'])) + + Application.truncate() + Application.save_many(applications_list) + logger.info("Inserted applications!!!") + return DatabaseOperation(valid=True) + except Exception as e: + logger.info(f"Error with legacy applications insert: {str(e)}") + return DatabaseOperation(valid=False, error_type= DBErrorType.UNKNOWN, message=str(e)) + + +def for_client_version(api_response: dict) -> DatabaseOperation: + client_versions = [] + try: + for client_version in api_response['data']: + client_versions.append(ClientVersion(client_version['version_number'], client_version['released_at'], client_version['id'], client_version['download_path'])) + + ClientVersion.truncate() + ClientVersion.save_many(client_versions) + + return DatabaseOperation(valid=True) + except Exception as e: + logger.info(f"Error with legacy client_version insert: {str(e)}") + return DatabaseOperation(valid=False, error_type= DBErrorType.UNKNOWN, message=str(e)) + + +def for_subscriptions(api_response: dict) -> DatabaseOperation: + subscription_plans = [] + try: + for subscription_plan in api_response['data']: + subscription_plans.append(SubscriptionPlan(subscription_plan['id'], subscription_plan['code'], subscription_plan['wireguard_session_limit'], subscription_plan['duration'], subscription_plan['price'], subscription_plan['features_proxy'], subscription_plan['features_wireguard'])) + + SubscriptionPlan.truncate() + SubscriptionPlan.save_many(subscription_plans) + + return DatabaseOperation(valid=True) + except Exception as e: + logger.info(f"Error with legacy for_subscriptions insert: {str(e)}") + return DatabaseOperation(valid=False, error_type= DBErrorType.UNKNOWN, message=str(e)) + + +# APP_CODES = { +# "firefox": 1, +# "chromium": 2, +# "brave": 3, +# "librewolf": 5 +# } + +def for_application_versions(api_response: dict, code: str) -> DatabaseOperation: + try: + application_versions = [] + for application_version in api_response['data']: + # print(f"application_version['format_revision'] is {application_version['format_revision']}") + app = ApplicationVersion(code, + application_version['version_number'], + application_version['format_revision'], + application_version['id'], + application_version['download_path'], + application_version['released_at'], + application_version['file_hash']) + application_versions.append(app) + + ApplicationVersion.truncate() + ApplicationVersion.save_many(application_versions) + + return DatabaseOperation(valid=True) + except Exception as e: + logger.info(f"Error with legacy for_application_versions insert: {str(e)}") + return DatabaseOperation(valid=False, error_type= DBErrorType.UNKNOWN, message=str(e)) diff --git a/core/services/sync/orm_methods/sync_one_orm.py b/core/services/sync/orm_methods/sync_one_orm.py index ff55b59..600cb8b 100644 --- a/core/services/sync/orm_methods/sync_one_orm.py +++ b/core/services/sync/orm_methods/sync_one_orm.py @@ -9,7 +9,7 @@ from core.models.manage.insert import insert_into_model from core.models.manage.denormalize import denormalize from core.models.orm_models.Base import Base -# observers & loiggers +# observers & loggers from core.observers.ClientObserver import ClientObserver from core.observers.ConnectionObserver import ConnectionObserver from core.errors.logger import logger diff --git a/core/services/sync/sync_service.py b/core/services/sync/sync_service.py index 44ba559..eb77e3c 100644 --- a/core/services/sync/sync_service.py +++ b/core/services/sync/sync_service.py @@ -1,5 +1,8 @@ # comparisons & api calls: +from core.services.networking.httpx import connect + + from core.services.sync.compare_tables import compare_tables # from core.services.sync.get_data_from_api import get_data_from_api