diff --git a/core/models/manage/insert.py b/core/models/manage/insert.py index 8b877a0..5e06023 100644 --- a/core/models/manage/insert.py +++ b/core/models/manage/insert.py @@ -68,7 +68,7 @@ def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session, # Log if we dropped anything if all_dropped_fields: - logger.warning(f"Filtered unknown fields before insert: {all_dropped_fields}") + logger.info(f"Filtered unknown fields before insert: {all_dropped_fields}") try: for filtered_json in filtered_data_list: diff --git a/core/services/helpers/load_yaml.py b/core/services/helpers/load_yaml.py deleted file mode 100644 index e345cb7..0000000 --- a/core/services/helpers/load_yaml.py +++ /dev/null @@ -1,11 +0,0 @@ -from importlib import resources -import yaml -import os - -def load_yaml(which_yaml: str): - text_content = resources.files('core.assets').joinpath(which_yaml).read_text() - return yaml.safe_load(text_content) - -def is_running_in_appimage(): - """Check if running inside an AppImage""" - return bool(os.environ.get('APPIMAGE')) diff --git a/core/services/sync/insert_for_orm.py b/core/services/sync/insert_for_orm.py index 8ee2ba0..281a2ab 100644 --- a/core/services/sync/insert_for_orm.py +++ b/core/services/sync/insert_for_orm.py @@ -17,6 +17,7 @@ from core.errors.logger import logger from core.utils.basic_operations.get_parent_directory import get_parent_directory from core.Constants import Constants from core.services.helpers.manage_assets import updated_assets_folder_changes +from core.utils import yaml_tools from typing import Optional import json @@ -37,36 +38,29 @@ def insert_one_orm_model( full_yaml_path = f"{current_assets_folder}/yaml_mappings/{yaml_filename}" yaml_file = Path(full_yaml_path) - if yaml_file.is_file(): - try: - denormalized_data = denormalize(new_data, str(full_yaml_path)) - return insert_into_model(which_model, denormalized_data, override) - except Exception as e: - error_msg = f"We had issues with denormalizing the data from the server: {str(e)} That data was {new_data}." - logger.error(error_msg) - return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg) + # NO YAML YET + if not yaml_file.is_file(): + logger.info(f"We did not find the yaml asset at {yaml_file}, let's try get it via a copy of pre-existing bundle..") + data = yaml_tools.load_yaml(yaml_filename) + if data is False or not yaml_tools.save_yaml(data=data, path=full_yaml_path): + logger.info(f"Skipping denormalization for {which_key} because there was no assets folder yaml") - else: - logger.info(f"We did not find the yaml asset at {yaml_file}, let's try to update it..") - if updated_assets_folder_changes(): - try: - denormalized_data = denormalize(new_data, str(full_yaml_path)) - return insert_into_model(which_model, denormalized_data, override) - except Exception as e: - error_msg = f"We had issues with denormalizing the data from the server: {str(e)} That data was {new_data}." - logger.error(error_msg) - return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg) + # The old version of the API wrapped stuff in 'data', while the new endpoints don't: + if isinstance(new_data, dict): + extracted_data = new_data.get('data', new_data) + else: + extracted_data = new_data - # otherwise: - logger.info(f"Skipping denormalization for {which_key} because there was no assets folder yaml at path {yaml_file}") - - # The old version of the API wrapped stuff in 'data', while the new endpoints don't: - if isinstance(new_data, dict): - extracted_data = new_data.get('data', new_data) - else: - extracted_data = new_data - - return insert_into_model(which_model, extracted_data, override) + return insert_into_model(which_model, extracted_data, override) + # YES THERE IS YAML + logger.info("We have a YAML file! Begin denormalize..") + try: + denormalized_data = denormalize(new_data, str(full_yaml_path)) + return insert_into_model(which_model, denormalized_data, override) + except Exception as e: + error_msg = f"We had issues with denormalizing the data from the server: {str(e)} That data was {new_data}." + logger.error(error_msg) + return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg) diff --git a/core/utils/yaml_tools.py b/core/utils/yaml_tools.py new file mode 100644 index 0000000..f4ea9a4 --- /dev/null +++ b/core/utils/yaml_tools.py @@ -0,0 +1,61 @@ +from importlib import resources +import yaml +import os +from typing import Any +from pathlib import Path +from core.errors.logger import logger + +PRE_PACKAGED_YAML_FOLDER = 'core.assets.yaml_mappings' + + +def load_yaml(which_yaml: str) -> Any | bool: + try: + # Load the YAML file + text_content = resources.files(PRE_PACKAGED_YAML_FOLDER).joinpath(which_yaml).read_text() + + # Check if content is empty or blank + if not text_content or not text_content.strip(): + logger.error(f"YAML file {which_yaml} is empty or blank") + return False + + # Parse and return the YAML + return yaml.safe_load(text_content) + + except FileNotFoundError: + logger.error(f"YAML file not found: {which_yaml}") + return False + except yaml.YAMLError as e: + logger.error(f"Failed to parse YAML file {which_yaml}: {e}") + return False + except Exception as e: + logger.error(f"Unexpected error loading YAML {which_yaml}: {e}") + return False + +def save_yaml(data: Any, path: str) -> bool: + try: + # Convert to Path object for cleaner handling + file_path = Path(path) + + # Create parent directories if they don't exist + file_path.parent.mkdir(parents=True, exist_ok=True) + + # Write the YAML file + with open(file_path, 'w') as file: + yaml.safe_dump(data, file, default_flow_style=False) + + return True + + except TypeError as e: + logger.error(f"Data is not YAML-serializable: {e}") + return False + except IOError as e: + logger.error(f"Failed to write YAML file to {path}: {e}") + return False + except Exception as e: + logger.error(f"Unexpected error saving YAML to {path}: {e}") + return False + + +def is_running_in_appimage(): + """Check if running inside an AppImage""" + return bool(os.environ.get('APPIMAGE'))