Fallback for AppImage to preloaded YAMLs via pathlib

core/models/manage/insert.py

@@ -68,7 +68,7 @@ def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session,
     
     # Log if we dropped anything
     if all_dropped_fields:
-        logger.warning(f"Filtered unknown fields before insert: {all_dropped_fields}")
+        logger.info(f"Filtered unknown fields before insert: {all_dropped_fields}")
 
     try:
         for filtered_json in filtered_data_list:

core/services/helpers/load_yaml.py

@@ -1,11 +0,0 @@
-from importlib import resources
-import yaml
-import os
-
-def load_yaml(which_yaml: str):
-    text_content = resources.files('core.assets').joinpath(which_yaml).read_text()
-    return yaml.safe_load(text_content)
-
-def is_running_in_appimage():
-    """Check if running inside an AppImage"""
-    return bool(os.environ.get('APPIMAGE'))

core/services/sync/insert_for_orm.py

@@ -17,6 +17,7 @@ from core.errors.logger import logger
 from core.utils.basic_operations.get_parent_directory import get_parent_directory
 from core.Constants import Constants
 from core.services.helpers.manage_assets import updated_assets_folder_changes
+from core.utils import yaml_tools
 
 from typing import Optional
 import json
@@ -37,36 +38,29 @@ def insert_one_orm_model(
     full_yaml_path = f"{current_assets_folder}/yaml_mappings/{yaml_filename}"
     yaml_file = Path(full_yaml_path)
 
-    if yaml_file.is_file():
-        try:
-            denormalized_data = denormalize(new_data, str(full_yaml_path))
-            return insert_into_model(which_model, denormalized_data, override)
-        except Exception as e:
-            error_msg = f"We had issues with denormalizing the data from the server: {str(e)}  That data was {new_data}."
-            logger.error(error_msg)
-            return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg)
+    # NO YAML YET
+    if not yaml_file.is_file():
+        logger.info(f"We did not find the yaml asset at {yaml_file}, let's try get it via a copy of pre-existing bundle..")
+        data = yaml_tools.load_yaml(yaml_filename)
+        if data is False or not yaml_tools.save_yaml(data=data, path=full_yaml_path):
+            logger.info(f"Skipping denormalization for {which_key} because there was no assets folder yaml")
 
-    else:
-        logger.info(f"We did not find the yaml asset at {yaml_file}, let's try to update it..")
-        if updated_assets_folder_changes():
-            try:
-                denormalized_data = denormalize(new_data, str(full_yaml_path))
-                return insert_into_model(which_model, denormalized_data, override)
-            except Exception as e:
-                error_msg = f"We had issues with denormalizing the data from the server: {str(e)}  That data was {new_data}."
-                logger.error(error_msg)
-                return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg)
+            # The old version of the API wrapped stuff in 'data', while the new endpoints don't:
+            if isinstance(new_data, dict):
+                extracted_data = new_data.get('data', new_data)
+            else:
+                extracted_data = new_data
 
-        # otherwise:
-        logger.info(f"Skipping denormalization for {which_key} because there was no assets folder yaml at path {yaml_file}")
-
-        # The old version of the API wrapped stuff in 'data', while the new endpoints don't:
-        if isinstance(new_data, dict):
-            extracted_data = new_data.get('data', new_data)
-        else:
-            extracted_data = new_data
-
-        return insert_into_model(which_model, extracted_data, override)
+            return insert_into_model(which_model, extracted_data, override)
 
+    # YES THERE IS YAML
+    logger.info("We have a YAML file! Begin denormalize..")
+    try:
+        denormalized_data = denormalize(new_data, str(full_yaml_path))
+        return insert_into_model(which_model, denormalized_data, override)
+    except Exception as e:
+        error_msg = f"We had issues with denormalizing the data from the server: {str(e)}  That data was {new_data}."
+        logger.error(error_msg)
+        return DatabaseOperation(valid=False, error_type=DBErrorType.WRONG_DATA_FORMAT, message=error_msg)
 
 

core/utils/yaml_tools.py

@@ -0,0 +1,61 @@
+from importlib import resources
+import yaml
+import os
+from typing import Any
+from pathlib import Path
+from core.errors.logger import logger
+
+PRE_PACKAGED_YAML_FOLDER = 'core.assets.yaml_mappings'
+
+
+def load_yaml(which_yaml: str) -> Any | bool:
+    try:
+        # Load the YAML file
+        text_content = resources.files(PRE_PACKAGED_YAML_FOLDER).joinpath(which_yaml).read_text()
+        
+        # Check if content is empty or blank
+        if not text_content or not text_content.strip():
+            logger.error(f"YAML file {which_yaml} is empty or blank")
+            return False
+        
+        # Parse and return the YAML
+        return yaml.safe_load(text_content)
+    
+    except FileNotFoundError:
+        logger.error(f"YAML file not found: {which_yaml}")
+        return False
+    except yaml.YAMLError as e:
+        logger.error(f"Failed to parse YAML file {which_yaml}: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"Unexpected error loading YAML {which_yaml}: {e}")
+        return False
+
+def save_yaml(data: Any, path: str) -> bool:
+    try:
+        # Convert to Path object for cleaner handling
+        file_path = Path(path)
+        
+        # Create parent directories if they don't exist
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        # Write the YAML file
+        with open(file_path, 'w') as file:
+            yaml.safe_dump(data, file, default_flow_style=False)
+        
+        return True
+    
+    except TypeError as e:
+        logger.error(f"Data is not YAML-serializable: {e}")
+        return False
+    except IOError as e:
+        logger.error(f"Failed to write YAML file to {path}: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"Unexpected error saving YAML to {path}: {e}")
+        return False
+
+
+def is_running_in_appimage():
+    """Check if running inside an AppImage"""
+    return bool(os.environ.get('APPIMAGE'))