diff --git a/change_log.md b/change_log.md index 4677c84..5176d9f 100644 --- a/change_log.md +++ b/change_log.md @@ -1,5 +1,10 @@ # Major Change Log: +# Reduce Redundancy +### Aug 13, 2026 +Isolated download file modules to their own module, then had both the install application versions (browsers), and install dependencies both use that same module. +
+ # Singbox Setup ### Aug 12, 2026 Prepared Singbox setup modules, which includes installation, download, move to sudo folder, and sudo setup scripts. Added a `Dependency` model, endpoint, and the ability to sync that model. (Related note: Server-side prepared the endpoint, and stocked with real data.) And also the sync service modules were adjusted to handle new data types more smoothly, before they had errors. As part of that sync flow change, the CachedSync metadata model was transitioned to ints instead of strings, with default 0 values. This should in theory migrate all clients without further changes needed. diff --git a/core/controllers/ApplicationVersionController.py b/core/controllers/ApplicationVersionController.py index db6fbfc..565a6ac 100644 --- a/core/controllers/ApplicationVersionController.py +++ b/core/controllers/ApplicationVersionController.py @@ -1,6 +1,9 @@ from core.services.networking.httpx import httpx_client from core.services.networking.httpx import connect from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.models.Result import Result, ResultError +from core.services.helpers.download_file import download_file_and_verify + from core.controllers.ConfigurationController import ConfigurationController from core.models.Configuration import Configuration, ConnectionChoice @@ -18,7 +21,7 @@ from core.observers.ConnectionObserver import ConnectionObserver from core.services.WebServiceApiService import WebServiceApiService from core.errors.logger import logger -import httpx +# import httpx from io import BytesIO from typing import Optional import hashlib @@ -87,54 +90,27 @@ class ApplicationVersionController: def __install(application_version: ApplicationVersion, application_version_observer: Optional[ApplicationVersionObserver] = None, connection_observer: Optional[ConnectionObserver] = None): target_app_name = application_version.application_code.capitalize() target_app_version = application_version.version_number + download_path = application_version.download_path + target_file_hash = application_version.file_hash if application_version_observer is not None: application_version_observer.notify('downloading', f"Downloading {target_app_name} {target_app_version}. Connecting..") - # legacy: - # application_version_observer.notify('downloading', application_version) + + download_result = download_file_and_verify( + target_app_name=target_app_name, + download_path=download_path, + target_file_hash=target_file_hash, + application_version_observer=application_version_observer, + target_app_version=target_app_version, + connection_observer=connection_observer + ) ################################################ - # SETUP HTTP CLIENT + # IT WORKED - SAVE IT ################################################ - client = httpx_client.get_http_session() - logger.info(f"client type is {type(client)}") - if client is None: - client = _get_httpx_client(target_app_name=target_app_name, connection_observer=connection_observer) - - ################################################ - # GET THE DATA - ################################################ - download_path = application_version.download_path - logger.info(f"download_path is {download_path}") - with client.stream('GET', download_path) as response: - logger.info("doing the stream...") - if response.status_code == 200: - response_size = int(response.headers.get('Content-Length', 0)) - response_buffer = BytesIO() - - block_size = 1024 - bytes_written = 0 - for data in response.iter_bytes(block_size): - - bytes_written += len(data) - response_buffer.write(data) - progress = (bytes_written / response_size) * 100 if response_size > 0 else 0 - - if application_version_observer is not None: - application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}% v: {target_app_version}") - else: - raise ConnectionError('The application version could not be downloaded.') - - application_version_observer.notify('downloaded', f"Downloaded {target_app_name} {target_app_version}") - response_buffer.seek(0) - - ################################################ - # VERIFY THE HASH - ################################################ - file_hash = ApplicationVersionController.__calculate_file_hash(response_buffer) - - if file_hash != application_version.file_hash: - raise FileIntegrityError('Application version file integrity could not be verified.') + if download_result.valid: + response_buffer = download_result.data + file_hash = download_result.message with tarfile.open(fileobj=response_buffer, mode = 'r:gz') as tar_file: tar_file.extractall(application_version.get_installation_path()) @@ -142,43 +118,93 @@ class ApplicationVersionController: with open(f'{application_version.get_installation_path()}/.sha3-512', 'w') as hash_file: hash_file.write(f'{file_hash}\n') - @staticmethod - def __calculate_file_hash(file): - - hasher = hashlib.sha3_512() - buffer = file.read(65536) - - while len(buffer) > 0: - - hasher.update(buffer) - buffer = file.read(65536) - - file.seek(0) - - return hasher.hexdigest() - -# This function is a temporary transition for the connect module to get better public APIs -def _get_httpx_client(target_app_name: str, connection_observer: Optional[ConnectionObserver]) -> httpx.Client: - connection_type = ConfigurationController.get_connection_enum() - client = connect.make_client(connection_type, connection_observer) - logger.info(f"client type is {type(client)}") - - if isinstance(client, ApiResponse): - if not client.valid: - raise ConnectionError(f'Could not connect, to download {target_app_name}.') - - if isinstance(client, bool): - if not client: - raise ConnectionError(f'Could not connect, to download {target_app_name}.') + ################################################ + # FAILED + ################################################ else: - logger.info("It's a boolean, getting the client now.") - client = httpx_client.get_http_session() - logger.info(f"client type is {type(client)}") - return client - + if download_result.error_type == ResultError.CONNECTION: + raise ConnectionError(f'Could not connect, to download {target_app_name}.') + elif download_result.error_type == ResultError.INVALID_INPUT: + raise FileIntegrityError('Application version file integrity could not be verified.') + else: + return None # legacy: + +# @staticmethod +# def __calculate_file_hash(file): + +# hasher = hashlib.sha3_512() +# buffer = file.read(65536) + +# while len(buffer) > 0: + +# hasher.update(buffer) +# buffer = file.read(65536) + +# file.seek(0) + +# return hasher.hexdigest() + + +# def _get_httpx_client(target_app_name: str, connection_observer: Optional[ConnectionObserver]) -> httpx.Client: +# connection_type = ConfigurationController.get_connection_enum() +# did_it_work = connect.make_client(connection_type, connection_observer) # always gets boolean + +# if not did_it_work: +# raise ConnectionError(f'Could not connect, to download {target_app_name}.') +# else: +# client = httpx_client.get_http_session() +# logger.info(f"client type is {type(client)}") +# return client + # @staticmethod # def get_all(application: Optional[Application] = None): # return ApplicationVersion.all(application) + +# legacy: +# application_version_observer.notify('downloading', application_version) + +################################################ +# SETUP HTTP CLIENT +################################################ +# client = httpx_client.get_http_session() +# logger.info(f"client type is {type(client)}") +# if client is None: +# client = _get_httpx_client(target_app_name=target_app_name, connection_observer=connection_observer) + +# ################################################ +# # GET THE DATA +# ################################################ +# download_path = application_version.download_path +# logger.info(f"download_path is {download_path}") +# with client.stream('GET', download_path) as response: +# logger.info("doing the stream...") +# if response.status_code == 200: +# response_size = int(response.headers.get('Content-Length', 0)) +# response_buffer = BytesIO() + +# block_size = 1024 +# bytes_written = 0 +# for data in response.iter_bytes(block_size): + +# bytes_written += len(data) +# response_buffer.write(data) +# progress = (bytes_written / response_size) * 100 if response_size > 0 else 0 + +# if application_version_observer is not None: +# application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}% v: {target_app_version}") +# else: +# raise ConnectionError('The application version could not be downloaded.') + +# application_version_observer.notify('downloaded', f"Downloaded {target_app_name} {target_app_version}") +# response_buffer.seek(0) + +################################################ +# VERIFY THE HASH +################################################ +# file_hash = ApplicationVersionController.__calculate_file_hash(response_buffer) + +# if file_hash != application_version.file_hash: +# raise FileIntegrityError('Application version file integrity could not be verified.') diff --git a/core/services/helpers/download_file.py b/core/services/helpers/download_file.py new file mode 100644 index 0000000..8d62a9d --- /dev/null +++ b/core/services/helpers/download_file.py @@ -0,0 +1,101 @@ +from core.services.networking.httpx import httpx_client +from core.services.networking.httpx import connect +from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType +from core.models.Result import Result, ResultError +from core.errors.logger import logger +from core.controllers.ConfigurationController import ConfigurationController +from core.models.Configuration import Configuration, ConnectionChoice +from core.observers.ApplicationVersionObserver import ApplicationVersionObserver +from core.observers.ConnectionObserver import ConnectionObserver + +import httpx +from io import BytesIO +from typing import Optional +import hashlib + + +def download_file_and_verify( + target_app_name: str, + download_path: str, + target_file_hash: str, + application_version_observer: Optional[ApplicationVersionObserver] = None, + target_app_version: Optional[str] = None, + connection_observer: Optional[ConnectionObserver] = None +) -> Result: + """ + Download/stream a file, and return the BytesIO buffer + """ + ################################################ + # SETUP HTTP CLIENT + ################################################ + client = httpx_client.get_http_session() + if client is None: + client = _get_httpx_client(target_app_name=target_app_name, connection_observer=connection_observer) + + ################################################ + # GET THE DATA + ################################################ + with client.stream('GET', download_path) as response: + if response.status_code == 200: + response_size = int(response.headers.get('Content-Length', 0)) + response_buffer = BytesIO() + + block_size = 1024 + bytes_written = 0 + for data in response.iter_bytes(block_size): + + bytes_written += len(data) + response_buffer.write(data) + progress = (bytes_written / response_size) * 100 if response_size > 0 else 0 + + if application_version_observer is not None: + if target_app_version: + application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}% v: {target_app_version}") + else: + application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}%") + else: + error_msg = f"Could not download {target_app_name} because of a Connection Error." + logger.error(error_msg) + return Result(valid=False, error_type=ResultError.CONNECTION, message=error_msg) + + if application_version_observer is not None: + application_version_observer.notify('downloaded', f"Downloaded {target_app_name}") + response_buffer.seek(0) + + ################################################ + # VERIFY THE HASH + ################################################ + real_file_hash = __calculate_file_hash(response_buffer) + + if real_file_hash != target_file_hash: + error_msg = f'Application version file integrity could not be verified. We are targeting {target_file_hash}, but got {real_file_hash}' + logger.error(error_msg) + return Result(valid=False, error_type=ResultError.INVALID_INPUT, message=error_msg) + else: + return Result(valid=True, data=response_buffer, message=real_file_hash) + + +def _get_httpx_client(target_app_name: str, connection_observer: Optional[ConnectionObserver]) -> httpx.Client: + connection_type = ConfigurationController.get_connection_enum() + did_it_work = connect.make_client(connection_type, connection_observer) # always gets boolean + + if not did_it_work: + raise ConnectionError(f'Could not connect, to download {target_app_name}.') + else: + client = httpx_client.get_http_session() + logger.info(f"client type is {type(client)}") + return client + +def __calculate_file_hash(file): + + hasher = hashlib.sha3_512() + buffer = file.read(65536) + + while len(buffer) > 0: + + hasher.update(buffer) + buffer = file.read(65536) + + file.seek(0) + + return hasher.hexdigest() diff --git a/core/services/helpers/install_dependencies.py b/core/services/helpers/install_dependencies.py index dd980ff..989fe21 100644 --- a/core/services/helpers/install_dependencies.py +++ b/core/services/helpers/install_dependencies.py @@ -1,4 +1,5 @@ from core.services.networking.httpx.httpx_client import get_http_session, init_session +from core.services.helpers.download_file import download_file_and_verify from core.utils.basic_operations.folder_tools import validate_folder_structure from core.utils.basic_operations.does_file_exist import does_file_exist from core.models.Result import Result, ResultError @@ -262,90 +263,111 @@ def download_and_verify( application_version_observer.notify('downloading', "singbox") ################################################ - # SETUP HTTP CLIENT + # GET & VERIFY ################################################ - init_session() - client = get_http_session() - if client is None: - init_session - client = get_http_session() + download_result = download_file_and_verify( + target_app_name=target_app_name, + download_path=download_path, + target_file_hash=target_file_hash, + application_version_observer=application_version_observer, + target_app_version=target_version + ) + if not download_result.valid: + return download_result ################################################ - # GET THE DATA + # SAVE IT IN CORRECT STRUCTURE ################################################ - with client.stream('GET', download_path) as response: - if response.status_code == 200: - response_size = int(response.headers.get('Content-Length', 0)) - response_buffer = BytesIO() + response_buffer = download_result.data + temp_dir = f"{target_folder}/temp_dir" + final_target_folder = f"{target_folder}/{target_version}" - block_size = 1024 - bytes_written = 0 - for data in response.iter_bytes(block_size): + # Create the temp folder (if it doesn't exist) + os.makedirs(temp_dir, exist_ok=True) - bytes_written += len(data) - response_buffer.write(data) - progress = (bytes_written / response_size) * 100 if response_size > 0 else 0 + # Save buffer to a temp directory: + with tarfile.open(fileobj=response_buffer, mode = 'r:gz') as tar_file: + tar_file.extractall(temp_dir) - if application_version_observer is not None: - application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}%") - else: - error_msg = f"Could not download {target_app_name} because of a Connection Error." - logger.error(error_msg) - return Result(valid=False, error_type=ResultError.CONNECTION, message=error_msg) + # make sure it has the file in the temp, and move it to the correct structure, + target_file_is_in_payload = validate_folder_structure( + temp_dir=temp_dir, + target_folder=final_target_folder, + target_file=target_app_name + ) - if application_version_observer is not None: - application_version_observer.notify('downloaded', f"Downloaded {target_app_name}") - response_buffer.seek(0) - - ################################################ - # VERIFY THE HASH - ################################################ - real_file_hash = __calculate_file_hash(response_buffer) - - if real_file_hash != target_file_hash: - error_msg = f'Application version file integrity could not be verified. We are targeting {target_file_hash}, but got {real_file_hash}' - logger.error(error_msg) - return Result(valid=False, error_type=ResultError.INVALID_INPUT, message=error_msg) - - ################################################ - # SAVE IT IN CORRECT STRUCTURE - ################################################ - temp_dir = f"{target_folder}/temp_dir" - final_target_folder = f"{target_folder}/{target_version}" - - # Create the temp folder (if it doesn't exist) - os.makedirs(temp_dir, exist_ok=True) - - # Save buffer to a temp directory: - with tarfile.open(fileobj=response_buffer, mode = 'r:gz') as tar_file: - tar_file.extractall(temp_dir) - - # make sure it has the file in the temp, and move it to the correct structure, - target_file_is_in_payload = validate_folder_structure( - temp_dir=temp_dir, - target_folder=final_target_folder, - target_file=target_app_name - ) - - if target_file_is_in_payload: - return Result(valid=True) - else: - return Result(valid=False, error_type=ResultError.MISSING_FILE) + if target_file_is_in_payload: + return Result(valid=True) + else: + return Result(valid=False, error_type=ResultError.MISSING_FILE) -# Repeat function outside ApplicationController, -# because we are moving away from fake static object structure. -def __calculate_file_hash(file): +# Legacy: - hasher = hashlib.sha3_512() - buffer = file.read(65536) +# def __calculate_file_hash(file): - while len(buffer) > 0: +# hasher = hashlib.sha3_512() +# buffer = file.read(65536) - hasher.update(buffer) - buffer = file.read(65536) +# while len(buffer) > 0: - file.seek(0) +# hasher.update(buffer) +# buffer = file.read(65536) + +# file.seek(0) + +# return hasher.hexdigest() + + + +# ################################################ +# # SETUP HTTP CLIENT +# ################################################ +# init_session() +# client = get_http_session() +# if client is None: +# init_session +# client = get_http_session() + + +################################################ +# GET THE DATA +################################################ + + + +# with client.stream('GET', download_path) as response: +# if response.status_code == 200: +# response_size = int(response.headers.get('Content-Length', 0)) +# response_buffer = BytesIO() + +# block_size = 1024 +# bytes_written = 0 +# for data in response.iter_bytes(block_size): + +# bytes_written += len(data) +# response_buffer.write(data) +# progress = (bytes_written / response_size) * 100 if response_size > 0 else 0 + +# if application_version_observer is not None: +# application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}%") +# else: +# error_msg = f"Could not download {target_app_name} because of a Connection Error." +# logger.error(error_msg) +# return Result(valid=False, error_type=ResultError.CONNECTION, message=error_msg) + +# if application_version_observer is not None: +# application_version_observer.notify('downloaded', f"Downloaded {target_app_name}") +# response_buffer.seek(0) + +# ################################################ +# # VERIFY THE HASH +# ################################################ +# real_file_hash = __calculate_file_hash(response_buffer) + +# if real_file_hash != target_file_hash: +# error_msg = f'Application version file integrity could not be verified. We are targeting {target_file_hash}, but got {real_file_hash}' +# logger.error(error_msg) +# return Result(valid=False, error_type=ResultError.INVALID_INPUT, message=error_msg) - return hasher.hexdigest()