Compare commits
5 commits
951802fe5c
...
1695e0314e
| Author | SHA1 | Date | |
|---|---|---|---|
| 1695e0314e | |||
| 73c74d339f | |||
| 1d8c508c52 | |||
| d9bee758a8 | |||
| b5e72ae7f6 |
7 changed files with 353 additions and 170 deletions
|
|
@ -1,5 +1,10 @@
|
||||||
# Major Change Log:
|
# Major Change Log:
|
||||||
|
|
||||||
|
# Reduce Redundancy
|
||||||
|
### Aug 13, 2026
|
||||||
|
Isolated download file modules to their own module, then had both the install application versions (browsers), and install dependencies both use that same module.
|
||||||
|
<br/>
|
||||||
|
|
||||||
# Singbox Setup
|
# Singbox Setup
|
||||||
### Aug 12, 2026
|
### Aug 12, 2026
|
||||||
Prepared Singbox setup modules, which includes installation, download, move to sudo folder, and sudo setup scripts. Added a `Dependency` model, endpoint, and the ability to sync that model. (Related note: Server-side prepared the endpoint, and stocked with real data.) And also the sync service modules were adjusted to handle new data types more smoothly, before they had errors. As part of that sync flow change, the CachedSync metadata model was transitioned to ints instead of strings, with default 0 values. This should in theory migrate all clients without further changes needed.
|
Prepared Singbox setup modules, which includes installation, download, move to sudo folder, and sudo setup scripts. Added a `Dependency` model, endpoint, and the ability to sync that model. (Related note: Server-side prepared the endpoint, and stocked with real data.) And also the sync service modules were adjusted to handle new data types more smoothly, before they had errors. As part of that sync flow change, the CachedSync metadata model was transitioned to ints instead of strings, with default 0 values. This should in theory migrate all clients without further changes needed.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,9 @@
|
||||||
from core.services.networking.httpx import httpx_client
|
from core.services.networking.httpx import httpx_client
|
||||||
from core.services.networking.httpx import connect
|
from core.services.networking.httpx import connect
|
||||||
from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
|
from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
|
||||||
|
from core.models.Result import Result, ResultError
|
||||||
|
from core.services.helpers.download_file import download_file_and_verify
|
||||||
|
|
||||||
from core.controllers.ConfigurationController import ConfigurationController
|
from core.controllers.ConfigurationController import ConfigurationController
|
||||||
from core.models.Configuration import Configuration, ConnectionChoice
|
from core.models.Configuration import Configuration, ConnectionChoice
|
||||||
|
|
||||||
|
|
@ -18,7 +21,7 @@ from core.observers.ConnectionObserver import ConnectionObserver
|
||||||
from core.services.WebServiceApiService import WebServiceApiService
|
from core.services.WebServiceApiService import WebServiceApiService
|
||||||
from core.errors.logger import logger
|
from core.errors.logger import logger
|
||||||
|
|
||||||
import httpx
|
# import httpx
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
import hashlib
|
import hashlib
|
||||||
|
|
@ -87,54 +90,27 @@ class ApplicationVersionController:
|
||||||
def __install(application_version: ApplicationVersion, application_version_observer: Optional[ApplicationVersionObserver] = None, connection_observer: Optional[ConnectionObserver] = None):
|
def __install(application_version: ApplicationVersion, application_version_observer: Optional[ApplicationVersionObserver] = None, connection_observer: Optional[ConnectionObserver] = None):
|
||||||
target_app_name = application_version.application_code.capitalize()
|
target_app_name = application_version.application_code.capitalize()
|
||||||
target_app_version = application_version.version_number
|
target_app_version = application_version.version_number
|
||||||
|
download_path = application_version.download_path
|
||||||
|
target_file_hash = application_version.file_hash
|
||||||
|
|
||||||
if application_version_observer is not None:
|
if application_version_observer is not None:
|
||||||
application_version_observer.notify('downloading', f"Downloading {target_app_name} {target_app_version}. Connecting..")
|
application_version_observer.notify('downloading', f"Downloading {target_app_name} {target_app_version}. Connecting..")
|
||||||
# legacy:
|
|
||||||
# application_version_observer.notify('downloading', application_version)
|
download_result = download_file_and_verify(
|
||||||
|
target_app_name=target_app_name,
|
||||||
|
download_path=download_path,
|
||||||
|
target_file_hash=target_file_hash,
|
||||||
|
application_version_observer=application_version_observer,
|
||||||
|
target_app_version=target_app_version,
|
||||||
|
connection_observer=connection_observer
|
||||||
|
)
|
||||||
|
|
||||||
################################################
|
################################################
|
||||||
# SETUP HTTP CLIENT
|
# IT WORKED - SAVE IT
|
||||||
################################################
|
################################################
|
||||||
client = httpx_client.get_http_session()
|
if download_result.valid:
|
||||||
logger.info(f"client type is {type(client)}")
|
response_buffer = download_result.data
|
||||||
if client is None:
|
file_hash = download_result.message
|
||||||
client = _get_httpx_client(target_app_name=target_app_name, connection_observer=connection_observer)
|
|
||||||
|
|
||||||
################################################
|
|
||||||
# GET THE DATA
|
|
||||||
################################################
|
|
||||||
download_path = application_version.download_path
|
|
||||||
logger.info(f"download_path is {download_path}")
|
|
||||||
with client.stream('GET', download_path) as response:
|
|
||||||
logger.info("doing the stream...")
|
|
||||||
if response.status_code == 200:
|
|
||||||
response_size = int(response.headers.get('Content-Length', 0))
|
|
||||||
response_buffer = BytesIO()
|
|
||||||
|
|
||||||
block_size = 1024
|
|
||||||
bytes_written = 0
|
|
||||||
for data in response.iter_bytes(block_size):
|
|
||||||
|
|
||||||
bytes_written += len(data)
|
|
||||||
response_buffer.write(data)
|
|
||||||
progress = (bytes_written / response_size) * 100 if response_size > 0 else 0
|
|
||||||
|
|
||||||
if application_version_observer is not None:
|
|
||||||
application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}% v: {target_app_version}")
|
|
||||||
else:
|
|
||||||
raise ConnectionError('The application version could not be downloaded.')
|
|
||||||
|
|
||||||
application_version_observer.notify('downloaded', f"Downloaded {target_app_name} {target_app_version}")
|
|
||||||
response_buffer.seek(0)
|
|
||||||
|
|
||||||
################################################
|
|
||||||
# VERIFY THE HASH
|
|
||||||
################################################
|
|
||||||
file_hash = ApplicationVersionController.__calculate_file_hash(response_buffer)
|
|
||||||
|
|
||||||
if file_hash != application_version.file_hash:
|
|
||||||
raise FileIntegrityError('Application version file integrity could not be verified.')
|
|
||||||
|
|
||||||
with tarfile.open(fileobj=response_buffer, mode = 'r:gz') as tar_file:
|
with tarfile.open(fileobj=response_buffer, mode = 'r:gz') as tar_file:
|
||||||
tar_file.extractall(application_version.get_installation_path())
|
tar_file.extractall(application_version.get_installation_path())
|
||||||
|
|
@ -142,43 +118,93 @@ class ApplicationVersionController:
|
||||||
with open(f'{application_version.get_installation_path()}/.sha3-512', 'w') as hash_file:
|
with open(f'{application_version.get_installation_path()}/.sha3-512', 'w') as hash_file:
|
||||||
hash_file.write(f'{file_hash}\n')
|
hash_file.write(f'{file_hash}\n')
|
||||||
|
|
||||||
@staticmethod
|
################################################
|
||||||
def __calculate_file_hash(file):
|
# FAILED
|
||||||
|
################################################
|
||||||
hasher = hashlib.sha3_512()
|
|
||||||
buffer = file.read(65536)
|
|
||||||
|
|
||||||
while len(buffer) > 0:
|
|
||||||
|
|
||||||
hasher.update(buffer)
|
|
||||||
buffer = file.read(65536)
|
|
||||||
|
|
||||||
file.seek(0)
|
|
||||||
|
|
||||||
return hasher.hexdigest()
|
|
||||||
|
|
||||||
# This function is a temporary transition for the connect module to get better public APIs
|
|
||||||
def _get_httpx_client(target_app_name: str, connection_observer: Optional[ConnectionObserver]) -> httpx.Client:
|
|
||||||
connection_type = ConfigurationController.get_connection_enum()
|
|
||||||
client = connect.make_client(connection_type, connection_observer)
|
|
||||||
logger.info(f"client type is {type(client)}")
|
|
||||||
|
|
||||||
if isinstance(client, ApiResponse):
|
|
||||||
if not client.valid:
|
|
||||||
raise ConnectionError(f'Could not connect, to download {target_app_name}.')
|
|
||||||
|
|
||||||
if isinstance(client, bool):
|
|
||||||
if not client:
|
|
||||||
raise ConnectionError(f'Could not connect, to download {target_app_name}.')
|
|
||||||
else:
|
else:
|
||||||
logger.info("It's a boolean, getting the client now.")
|
if download_result.error_type == ResultError.CONNECTION:
|
||||||
client = httpx_client.get_http_session()
|
raise ConnectionError(f'Could not connect, to download {target_app_name}.')
|
||||||
logger.info(f"client type is {type(client)}")
|
elif download_result.error_type == ResultError.INVALID_INPUT:
|
||||||
return client
|
raise FileIntegrityError('Application version file integrity could not be verified.')
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
# legacy:
|
# legacy:
|
||||||
|
|
||||||
|
# @staticmethod
|
||||||
|
# def __calculate_file_hash(file):
|
||||||
|
|
||||||
|
# hasher = hashlib.sha3_512()
|
||||||
|
# buffer = file.read(65536)
|
||||||
|
|
||||||
|
# while len(buffer) > 0:
|
||||||
|
|
||||||
|
# hasher.update(buffer)
|
||||||
|
# buffer = file.read(65536)
|
||||||
|
|
||||||
|
# file.seek(0)
|
||||||
|
|
||||||
|
# return hasher.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
# def _get_httpx_client(target_app_name: str, connection_observer: Optional[ConnectionObserver]) -> httpx.Client:
|
||||||
|
# connection_type = ConfigurationController.get_connection_enum()
|
||||||
|
# did_it_work = connect.make_client(connection_type, connection_observer) # always gets boolean
|
||||||
|
|
||||||
|
# if not did_it_work:
|
||||||
|
# raise ConnectionError(f'Could not connect, to download {target_app_name}.')
|
||||||
|
# else:
|
||||||
|
# client = httpx_client.get_http_session()
|
||||||
|
# logger.info(f"client type is {type(client)}")
|
||||||
|
# return client
|
||||||
|
|
||||||
# @staticmethod
|
# @staticmethod
|
||||||
# def get_all(application: Optional[Application] = None):
|
# def get_all(application: Optional[Application] = None):
|
||||||
# return ApplicationVersion.all(application)
|
# return ApplicationVersion.all(application)
|
||||||
|
|
||||||
|
# legacy:
|
||||||
|
# application_version_observer.notify('downloading', application_version)
|
||||||
|
|
||||||
|
################################################
|
||||||
|
# SETUP HTTP CLIENT
|
||||||
|
################################################
|
||||||
|
# client = httpx_client.get_http_session()
|
||||||
|
# logger.info(f"client type is {type(client)}")
|
||||||
|
# if client is None:
|
||||||
|
# client = _get_httpx_client(target_app_name=target_app_name, connection_observer=connection_observer)
|
||||||
|
|
||||||
|
# ################################################
|
||||||
|
# # GET THE DATA
|
||||||
|
# ################################################
|
||||||
|
# download_path = application_version.download_path
|
||||||
|
# logger.info(f"download_path is {download_path}")
|
||||||
|
# with client.stream('GET', download_path) as response:
|
||||||
|
# logger.info("doing the stream...")
|
||||||
|
# if response.status_code == 200:
|
||||||
|
# response_size = int(response.headers.get('Content-Length', 0))
|
||||||
|
# response_buffer = BytesIO()
|
||||||
|
|
||||||
|
# block_size = 1024
|
||||||
|
# bytes_written = 0
|
||||||
|
# for data in response.iter_bytes(block_size):
|
||||||
|
|
||||||
|
# bytes_written += len(data)
|
||||||
|
# response_buffer.write(data)
|
||||||
|
# progress = (bytes_written / response_size) * 100 if response_size > 0 else 0
|
||||||
|
|
||||||
|
# if application_version_observer is not None:
|
||||||
|
# application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}% v: {target_app_version}")
|
||||||
|
# else:
|
||||||
|
# raise ConnectionError('The application version could not be downloaded.')
|
||||||
|
|
||||||
|
# application_version_observer.notify('downloaded', f"Downloaded {target_app_name} {target_app_version}")
|
||||||
|
# response_buffer.seek(0)
|
||||||
|
|
||||||
|
################################################
|
||||||
|
# VERIFY THE HASH
|
||||||
|
################################################
|
||||||
|
# file_hash = ApplicationVersionController.__calculate_file_hash(response_buffer)
|
||||||
|
|
||||||
|
# if file_hash != application_version.file_hash:
|
||||||
|
# raise FileIntegrityError('Application version file integrity could not be verified.')
|
||||||
|
|
|
||||||
107
core/services/helpers/download_file.py
Normal file
107
core/services/helpers/download_file.py
Normal file
|
|
@ -0,0 +1,107 @@
|
||||||
|
from core.services.networking.httpx import httpx_client
|
||||||
|
from core.services.networking.httpx import connect
|
||||||
|
from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
|
||||||
|
from core.models.Result import Result, ResultError
|
||||||
|
from core.errors.logger import logger
|
||||||
|
from core.controllers.ConfigurationController import ConfigurationController
|
||||||
|
from core.models.Configuration import Configuration, ConnectionChoice
|
||||||
|
from core.observers.ApplicationVersionObserver import ApplicationVersionObserver
|
||||||
|
from core.observers.ConnectionObserver import ConnectionObserver
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from io import BytesIO
|
||||||
|
from typing import Optional
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
|
||||||
|
def download_file_and_verify(
|
||||||
|
target_app_name: str,
|
||||||
|
download_path: str,
|
||||||
|
target_file_hash: str,
|
||||||
|
application_version_observer: Optional[ApplicationVersionObserver] = None,
|
||||||
|
target_app_version: Optional[str] = None,
|
||||||
|
connection_observer: Optional[ConnectionObserver] = None
|
||||||
|
) -> Result:
|
||||||
|
"""
|
||||||
|
Download/stream a file, and return the BytesIO buffer
|
||||||
|
"""
|
||||||
|
################################################
|
||||||
|
# SETUP HTTP CLIENT
|
||||||
|
################################################
|
||||||
|
client = httpx_client.get_http_session()
|
||||||
|
if client is None:
|
||||||
|
client = _get_httpx_client(target_app_name=target_app_name, connection_observer=connection_observer)
|
||||||
|
|
||||||
|
################################################
|
||||||
|
# GET THE DATA
|
||||||
|
################################################
|
||||||
|
with client.stream('GET', download_path) as response:
|
||||||
|
if response.status_code == 200:
|
||||||
|
response_size = int(response.headers.get('Content-Length', 0))
|
||||||
|
response_buffer = BytesIO()
|
||||||
|
|
||||||
|
block_size = 1024
|
||||||
|
bytes_written = 0
|
||||||
|
for data in response.iter_bytes(block_size):
|
||||||
|
|
||||||
|
bytes_written += len(data)
|
||||||
|
response_buffer.write(data)
|
||||||
|
progress = (bytes_written / response_size) * 100 if response_size > 0 else 0
|
||||||
|
|
||||||
|
if application_version_observer is not None:
|
||||||
|
if target_app_version:
|
||||||
|
application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}% v: {target_app_version}")
|
||||||
|
else:
|
||||||
|
application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}%")
|
||||||
|
else:
|
||||||
|
error_msg = f"Could not download {target_app_name} because of a Connection Error."
|
||||||
|
logger.error(error_msg)
|
||||||
|
return Result(valid=False, error_type=ResultError.CONNECTION, message=error_msg)
|
||||||
|
|
||||||
|
if application_version_observer is not None:
|
||||||
|
application_version_observer.notify('downloaded', f"Downloaded {target_app_name}")
|
||||||
|
response_buffer.seek(0)
|
||||||
|
|
||||||
|
################################################
|
||||||
|
# VERIFY THE HASH
|
||||||
|
################################################
|
||||||
|
real_file_hash = __calculate_file_hash(response_buffer)
|
||||||
|
|
||||||
|
if real_file_hash != target_file_hash:
|
||||||
|
error_msg = f'Application version file integrity could not be verified. We are targeting {target_file_hash}, but got {real_file_hash}'
|
||||||
|
logger.error(error_msg)
|
||||||
|
return Result(valid=False, error_type=ResultError.INVALID_INPUT, message=error_msg)
|
||||||
|
else:
|
||||||
|
return Result(valid=True, data=response_buffer, message=real_file_hash)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_httpx_client(target_app_name: str, connection_observer: Optional[ConnectionObserver]) -> httpx.Client:
|
||||||
|
connection_type = ConfigurationController.get_connection_enum()
|
||||||
|
made_client = connect.make_client(connection_type, connection_observer) # always gets boolean
|
||||||
|
|
||||||
|
if not made_client:
|
||||||
|
if connection_type == ConnectionChoice.SYSTEM:
|
||||||
|
raise ConnectionError(f'Could not connect, to download {target_app_name}.')
|
||||||
|
else: # Tor:
|
||||||
|
if connection_observer:
|
||||||
|
connection_observer.notify('message', "Tor Bootstrap..")
|
||||||
|
bootstrap_results = connect.coordinate_bootstrap(connection_observer)
|
||||||
|
if not bootstrap_results.valid:
|
||||||
|
raise ConnectionError(f'Could not connect, to download {target_app_name}.')
|
||||||
|
|
||||||
|
client = httpx_client.get_http_session()
|
||||||
|
return client
|
||||||
|
|
||||||
|
def __calculate_file_hash(file):
|
||||||
|
|
||||||
|
hasher = hashlib.sha3_512()
|
||||||
|
buffer = file.read(65536)
|
||||||
|
|
||||||
|
while len(buffer) > 0:
|
||||||
|
|
||||||
|
hasher.update(buffer)
|
||||||
|
buffer = file.read(65536)
|
||||||
|
|
||||||
|
file.seek(0)
|
||||||
|
|
||||||
|
return hasher.hexdigest()
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
from core.services.networking.httpx.httpx_client import get_http_session, init_session
|
from core.services.networking.httpx.httpx_client import get_http_session, init_session
|
||||||
|
from core.services.helpers.download_file import download_file_and_verify
|
||||||
from core.utils.basic_operations.folder_tools import validate_folder_structure
|
from core.utils.basic_operations.folder_tools import validate_folder_structure
|
||||||
from core.utils.basic_operations.does_file_exist import does_file_exist
|
from core.utils.basic_operations.does_file_exist import does_file_exist
|
||||||
from core.models.Result import Result, ResultError
|
from core.models.Result import Result, ResultError
|
||||||
|
|
@ -10,6 +11,7 @@ from core.models.orm_models.Dependency import Dependency
|
||||||
from core.models.orm_calls.dependency_calls import get_dependency_version
|
from core.models.orm_calls.dependency_calls import get_dependency_version
|
||||||
from core.controllers.ConfigurationController import ConfigurationController
|
from core.controllers.ConfigurationController import ConfigurationController
|
||||||
from core.utils.basic_operations.compare_versions import version_update_required
|
from core.utils.basic_operations.compare_versions import version_update_required
|
||||||
|
from core.observers.ConnectionObserver import ConnectionObserver
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
@ -20,7 +22,10 @@ import os
|
||||||
|
|
||||||
SUDO_SINGBOX_LOCATION = f"{Constants.SUDO_TARGET_FOLDER}/sing-box"
|
SUDO_SINGBOX_LOCATION = f"{Constants.SUDO_TARGET_FOLDER}/sing-box"
|
||||||
|
|
||||||
def setup_singbox_binary(application_version_observer: Optional[ApplicationVersionObserver]) -> Result:
|
def setup_singbox_binary(
|
||||||
|
application_version_observer: Optional[ApplicationVersionObserver],
|
||||||
|
connection_observer: Optional[ConnectionObserver]
|
||||||
|
) -> Result:
|
||||||
"""
|
"""
|
||||||
Rank:
|
Rank:
|
||||||
Module's Main Orchestrator
|
Module's Main Orchestrator
|
||||||
|
|
@ -93,7 +98,8 @@ def setup_singbox_binary(application_version_observer: Optional[ApplicationVersi
|
||||||
target_file_hash=target_file_hash,
|
target_file_hash=target_file_hash,
|
||||||
target_app_name="sing-box",
|
target_app_name="sing-box",
|
||||||
target_version=target_version,
|
target_version=target_version,
|
||||||
application_version_observer=application_version_observer
|
application_version_observer=application_version_observer,
|
||||||
|
connection_observer=connection_observer
|
||||||
)
|
)
|
||||||
|
|
||||||
if not file_result.valid:
|
if not file_result.valid:
|
||||||
|
|
@ -249,6 +255,7 @@ def download_and_verify(
|
||||||
target_app_name: str,
|
target_app_name: str,
|
||||||
target_version: str,
|
target_version: str,
|
||||||
application_version_observer: Optional[ApplicationVersionObserver] = None,
|
application_version_observer: Optional[ApplicationVersionObserver] = None,
|
||||||
|
connection_observer: Optional[ConnectionObserver] = None
|
||||||
) -> Result:
|
) -> Result:
|
||||||
"""
|
"""
|
||||||
Purpose:
|
Purpose:
|
||||||
|
|
@ -262,90 +269,112 @@ def download_and_verify(
|
||||||
application_version_observer.notify('downloading', "singbox")
|
application_version_observer.notify('downloading', "singbox")
|
||||||
|
|
||||||
################################################
|
################################################
|
||||||
# SETUP HTTP CLIENT
|
# GET & VERIFY
|
||||||
################################################
|
################################################
|
||||||
init_session()
|
download_result = download_file_and_verify(
|
||||||
client = get_http_session()
|
target_app_name=target_app_name,
|
||||||
if client is None:
|
download_path=download_path,
|
||||||
init_session
|
target_file_hash=target_file_hash,
|
||||||
client = get_http_session()
|
application_version_observer=application_version_observer,
|
||||||
|
target_app_version=target_version,
|
||||||
|
connection_observer=connection_observer
|
||||||
|
)
|
||||||
|
|
||||||
|
if not download_result.valid:
|
||||||
|
return download_result
|
||||||
|
|
||||||
################################################
|
################################################
|
||||||
# GET THE DATA
|
# SAVE IT IN CORRECT STRUCTURE
|
||||||
################################################
|
################################################
|
||||||
with client.stream('GET', download_path) as response:
|
response_buffer = download_result.data
|
||||||
if response.status_code == 200:
|
temp_dir = f"{target_folder}/temp_dir"
|
||||||
response_size = int(response.headers.get('Content-Length', 0))
|
final_target_folder = f"{target_folder}/{target_version}"
|
||||||
response_buffer = BytesIO()
|
|
||||||
|
|
||||||
block_size = 1024
|
# Create the temp folder (if it doesn't exist)
|
||||||
bytes_written = 0
|
os.makedirs(temp_dir, exist_ok=True)
|
||||||
for data in response.iter_bytes(block_size):
|
|
||||||
|
|
||||||
bytes_written += len(data)
|
# Save buffer to a temp directory:
|
||||||
response_buffer.write(data)
|
with tarfile.open(fileobj=response_buffer, mode = 'r:gz') as tar_file:
|
||||||
progress = (bytes_written / response_size) * 100 if response_size > 0 else 0
|
tar_file.extractall(temp_dir)
|
||||||
|
|
||||||
if application_version_observer is not None:
|
# make sure it has the file in the temp, and move it to the correct structure,
|
||||||
application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}%")
|
target_file_is_in_payload = validate_folder_structure(
|
||||||
else:
|
temp_dir=temp_dir,
|
||||||
error_msg = f"Could not download {target_app_name} because of a Connection Error."
|
target_folder=final_target_folder,
|
||||||
logger.error(error_msg)
|
target_file=target_app_name
|
||||||
return Result(valid=False, error_type=ResultError.CONNECTION, message=error_msg)
|
)
|
||||||
|
|
||||||
if application_version_observer is not None:
|
if target_file_is_in_payload:
|
||||||
application_version_observer.notify('downloaded', f"Downloaded {target_app_name}")
|
return Result(valid=True)
|
||||||
response_buffer.seek(0)
|
else:
|
||||||
|
return Result(valid=False, error_type=ResultError.MISSING_FILE)
|
||||||
################################################
|
|
||||||
# VERIFY THE HASH
|
|
||||||
################################################
|
|
||||||
real_file_hash = __calculate_file_hash(response_buffer)
|
|
||||||
|
|
||||||
if real_file_hash != target_file_hash:
|
|
||||||
error_msg = f'Application version file integrity could not be verified. We are targeting {target_file_hash}, but got {real_file_hash}'
|
|
||||||
logger.error(error_msg)
|
|
||||||
return Result(valid=False, error_type=ResultError.INVALID_INPUT, message=error_msg)
|
|
||||||
|
|
||||||
################################################
|
|
||||||
# SAVE IT IN CORRECT STRUCTURE
|
|
||||||
################################################
|
|
||||||
temp_dir = f"{target_folder}/temp_dir"
|
|
||||||
final_target_folder = f"{target_folder}/{target_version}"
|
|
||||||
|
|
||||||
# Create the temp folder (if it doesn't exist)
|
|
||||||
os.makedirs(temp_dir, exist_ok=True)
|
|
||||||
|
|
||||||
# Save buffer to a temp directory:
|
|
||||||
with tarfile.open(fileobj=response_buffer, mode = 'r:gz') as tar_file:
|
|
||||||
tar_file.extractall(temp_dir)
|
|
||||||
|
|
||||||
# make sure it has the file in the temp, and move it to the correct structure,
|
|
||||||
target_file_is_in_payload = validate_folder_structure(
|
|
||||||
temp_dir=temp_dir,
|
|
||||||
target_folder=final_target_folder,
|
|
||||||
target_file=target_app_name
|
|
||||||
)
|
|
||||||
|
|
||||||
if target_file_is_in_payload:
|
|
||||||
return Result(valid=True)
|
|
||||||
else:
|
|
||||||
return Result(valid=False, error_type=ResultError.MISSING_FILE)
|
|
||||||
|
|
||||||
|
|
||||||
# Repeat function outside ApplicationController,
|
# Legacy:
|
||||||
# because we are moving away from fake static object structure.
|
|
||||||
def __calculate_file_hash(file):
|
|
||||||
|
|
||||||
hasher = hashlib.sha3_512()
|
# def __calculate_file_hash(file):
|
||||||
buffer = file.read(65536)
|
|
||||||
|
|
||||||
while len(buffer) > 0:
|
# hasher = hashlib.sha3_512()
|
||||||
|
# buffer = file.read(65536)
|
||||||
|
|
||||||
hasher.update(buffer)
|
# while len(buffer) > 0:
|
||||||
buffer = file.read(65536)
|
|
||||||
|
|
||||||
file.seek(0)
|
# hasher.update(buffer)
|
||||||
|
# buffer = file.read(65536)
|
||||||
|
|
||||||
|
# file.seek(0)
|
||||||
|
|
||||||
|
# return hasher.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ################################################
|
||||||
|
# # SETUP HTTP CLIENT
|
||||||
|
# ################################################
|
||||||
|
# init_session()
|
||||||
|
# client = get_http_session()
|
||||||
|
# if client is None:
|
||||||
|
# init_session
|
||||||
|
# client = get_http_session()
|
||||||
|
|
||||||
|
|
||||||
|
################################################
|
||||||
|
# GET THE DATA
|
||||||
|
################################################
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# with client.stream('GET', download_path) as response:
|
||||||
|
# if response.status_code == 200:
|
||||||
|
# response_size = int(response.headers.get('Content-Length', 0))
|
||||||
|
# response_buffer = BytesIO()
|
||||||
|
|
||||||
|
# block_size = 1024
|
||||||
|
# bytes_written = 0
|
||||||
|
# for data in response.iter_bytes(block_size):
|
||||||
|
|
||||||
|
# bytes_written += len(data)
|
||||||
|
# response_buffer.write(data)
|
||||||
|
# progress = (bytes_written / response_size) * 100 if response_size > 0 else 0
|
||||||
|
|
||||||
|
# if application_version_observer is not None:
|
||||||
|
# application_version_observer.notify('download_progressing', f"Downloading {target_app_name} {progress:.2f}%")
|
||||||
|
# else:
|
||||||
|
# error_msg = f"Could not download {target_app_name} because of a Connection Error."
|
||||||
|
# logger.error(error_msg)
|
||||||
|
# return Result(valid=False, error_type=ResultError.CONNECTION, message=error_msg)
|
||||||
|
|
||||||
|
# if application_version_observer is not None:
|
||||||
|
# application_version_observer.notify('downloaded', f"Downloaded {target_app_name}")
|
||||||
|
# response_buffer.seek(0)
|
||||||
|
|
||||||
|
# ################################################
|
||||||
|
# # VERIFY THE HASH
|
||||||
|
# ################################################
|
||||||
|
# real_file_hash = __calculate_file_hash(response_buffer)
|
||||||
|
|
||||||
|
# if real_file_hash != target_file_hash:
|
||||||
|
# error_msg = f'Application version file integrity could not be verified. We are targeting {target_file_hash}, but got {real_file_hash}'
|
||||||
|
# logger.error(error_msg)
|
||||||
|
# return Result(valid=False, error_type=ResultError.INVALID_INPUT, message=error_msg)
|
||||||
|
|
||||||
return hasher.hexdigest()
|
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ from core.services.networking.tor_tools.tor_orchestrator import establish_tor_co
|
||||||
from core.services.networking.tor_tools.tor_dns import setup_SINGLE_use_resolver
|
from core.services.networking.tor_tools.tor_dns import setup_SINGLE_use_resolver
|
||||||
|
|
||||||
from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
|
from core.services.networking.api_requests.ApiResponseModel import ApiResponse, ErrorType
|
||||||
|
from core.models.Result import Result, ResultError
|
||||||
from core.services.networking.httpx.async_batch_requests import async_parallel
|
from core.services.networking.httpx.async_batch_requests import async_parallel
|
||||||
from core.services.networking.tor_tools.pre_bootstrap import get_bootstrap_port
|
from core.services.networking.tor_tools.pre_bootstrap import get_bootstrap_port
|
||||||
from core.services.networking.api_requests.subtools.get_connection_type import get_connection_type
|
from core.services.networking.api_requests.subtools.get_connection_type import get_connection_type
|
||||||
|
|
@ -44,7 +45,7 @@ def single_endpoint(method: str, url: str, observer: ConnectionObserver, payload
|
||||||
########################################################
|
########################################################
|
||||||
if client is None:
|
if client is None:
|
||||||
observer.notify('message', "Testing Connection..")
|
observer.notify('message', "Testing Connection..")
|
||||||
made_client = make_client(connection_type, observer)
|
made_client = make_client(connection_type, observer) # makes boolean
|
||||||
if not made_client and connection_type == ConnectionChoice.TOR:
|
if not made_client and connection_type == ConnectionChoice.TOR:
|
||||||
observer.notify('message', "Tor Bootstrap..")
|
observer.notify('message', "Tor Bootstrap..")
|
||||||
return bootstrap_and_try_again(
|
return bootstrap_and_try_again(
|
||||||
|
|
@ -114,13 +115,16 @@ def single_endpoint(method: str, url: str, observer: ConnectionObserver, payload
|
||||||
return initial_result
|
return initial_result
|
||||||
|
|
||||||
|
|
||||||
def make_client(connection_type: str, observer: Optional[ConnectionObserver] = None) -> httpx.Client | ApiResponse:
|
def make_client(connection_type: str, observer: Optional[ConnectionObserver] = None) -> bool:
|
||||||
"""
|
"""
|
||||||
Rank:
|
Rank:
|
||||||
Coordinator
|
Coordinator
|
||||||
|
|
||||||
Purpose:
|
Purpose:
|
||||||
Create an HTTPx Client for either kind of transport
|
Create an HTTPx Client for either kind of transport
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Boolean of result
|
||||||
"""
|
"""
|
||||||
global _port_used
|
global _port_used
|
||||||
|
|
||||||
|
|
@ -128,20 +132,21 @@ def make_client(connection_type: str, observer: Optional[ConnectionObserver] = N
|
||||||
if connection_type == ConnectionChoice.SYSTEM:
|
if connection_type == ConnectionChoice.SYSTEM:
|
||||||
return httpx_client.init_session() # this is not the client, its a boolean
|
return httpx_client.init_session() # this is not the client, its a boolean
|
||||||
|
|
||||||
|
|
||||||
|
# Tor:
|
||||||
if _port_used is None:
|
if _port_used is None:
|
||||||
_port_used = Constants.DEFAULT_TOR_PORT
|
_port_used = Constants.DEFAULT_TOR_PORT
|
||||||
logger.info(f"Set to default port of {Constants.DEFAULT_TOR_PORT}")
|
logger.info(f"Set to default port of {Constants.DEFAULT_TOR_PORT}")
|
||||||
|
|
||||||
# Tor:
|
|
||||||
|
|
||||||
# check if port is even listening:
|
# check if port is even listening:
|
||||||
listening = ports.is_port_in_use(_port_used)
|
listening = ports.is_port_in_use(_port_used)
|
||||||
if listening:
|
if listening:
|
||||||
client = httpx_client.init_tor_session(_port_used)
|
boolean_if_worked = httpx_client.init_tor_session(_port_used)
|
||||||
if client:
|
if boolean_if_worked:
|
||||||
return client
|
return True
|
||||||
else:
|
else:
|
||||||
logger.error(f"Could NOT create a Tor HTTPx client on port {_port_used}.. Bootstrapping..")
|
logger.error(f"Could NOT create a Tor HTTPx client on port {_port_used}.. Bootstrapping..")
|
||||||
|
return False
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -160,6 +165,24 @@ def bootstrap_and_try_again(method: str, url: str, observer: ConnectionObserver,
|
||||||
Called by:
|
Called by:
|
||||||
single_endpoint
|
single_endpoint
|
||||||
"""
|
"""
|
||||||
|
bootstrap_results = coordinate_bootstrap(observer)
|
||||||
|
if not bootstrap_results.valid:
|
||||||
|
return bootstrap_results
|
||||||
|
|
||||||
|
observer.notify('message', "Tor Confirmed")
|
||||||
|
|
||||||
|
client = httpx_client.get_http_session()
|
||||||
|
observer.notify('message', "Making Request..")
|
||||||
|
return make_request(
|
||||||
|
method=method,
|
||||||
|
url=url,
|
||||||
|
client=client,
|
||||||
|
payload=payload,
|
||||||
|
billing_code=billing_code
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def coordinate_bootstrap(observer: ConnectionObserver) -> ApiResponse:
|
||||||
global _port_used
|
global _port_used
|
||||||
|
|
||||||
if _port_used is None:
|
if _port_used is None:
|
||||||
|
|
@ -174,22 +197,13 @@ def bootstrap_and_try_again(method: str, url: str, observer: ConnectionObserver,
|
||||||
# BOOTSTRAP SUCCESS
|
# BOOTSTRAP SUCCESS
|
||||||
_port_used = bootstrap_results.port
|
_port_used = bootstrap_results.port
|
||||||
|
|
||||||
observer.notify('message', "Testing Tor..")
|
if observer:
|
||||||
|
observer.notify('message', "Testing Tor..")
|
||||||
made_client = httpx_client.init_tor_session(_port_used)
|
made_client = httpx_client.init_tor_session(_port_used)
|
||||||
if not made_client:
|
if not made_client:
|
||||||
return bootstrap_results
|
return bootstrap_results
|
||||||
observer.notify('message', "Tor Confirmed")
|
else:
|
||||||
|
return ApiResponse(valid=True)
|
||||||
client = httpx_client.get_http_session()
|
|
||||||
observer.notify('message', "Making Request..")
|
|
||||||
return make_request(
|
|
||||||
method=method,
|
|
||||||
url=url,
|
|
||||||
client=client,
|
|
||||||
payload=payload,
|
|
||||||
billing_code=billing_code
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def bulk_async(wanted_list: list, observer: ConnectionObserver, client_observer: ClientObserver) -> ApiResponse:
|
def bulk_async(wanted_list: list, observer: ConnectionObserver, client_observer: ClientObserver) -> ApiResponse:
|
||||||
|
|
|
||||||
|
|
@ -33,4 +33,4 @@ def version_update_required(new_version: str, version_installed: str) -> bool:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# step 4) if equal, go into the last digit
|
# step 4) if equal, go into the last digit
|
||||||
return new_patch > installed_patch
|
return new_patch >= installed_patch
|
||||||
|
|
|
||||||
|
|
@ -50,6 +50,8 @@ def validate_folder_structure(temp_dir: str, target_folder: str, target_file: st
|
||||||
if found_file:
|
if found_file:
|
||||||
if found_file.parent != target_folder:
|
if found_file.parent != target_folder:
|
||||||
shutil.move(str(found_file), str(target_file_path))
|
shutil.move(str(found_file), str(target_file_path))
|
||||||
|
# finally, remove temp directory,
|
||||||
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
return False
|
return False
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue