diff --git a/core/models/manage/insert.py b/core/models/manage/insert.py index ebdbedf..89d599e 100644 --- a/core/models/manage/insert.py +++ b/core/models/manage/insert.py @@ -43,7 +43,7 @@ def insert_into_model(model_class: Type, all_data: dict | list, override=False) @safe_db_operation def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session, override=False) -> DatabaseOperation: - """Insert items. Doesn't manage session. It's managed by the wrapper safe_db_operation""" + """Insert items. Doesn't manage session. It's managed by the wrapper safe_db_operation""" if override: logger.info(f"First, WIPING the pre-existing data for {model_class.__name__}. Are you sure you intended to completely delete the old data?") @@ -51,53 +51,46 @@ def _wrapped_insert(model_class: Type, data_list: dict | list, session: Session, logger.info(f"Starting insert for {model_class.__name__}") + # Get valid field names upfront + valid_fields = {col.name for col in model_class.__table__.columns} + + # Filter data proactively + filtered_data_list = [] + all_dropped_fields = set() + + for each_json in data_list: + filtered_json = {k: v for k, v in each_json.items() if k in valid_fields} + filtered_data_list.append(filtered_json) + + # Track dropped fields + dropped = set(each_json.keys()) - valid_fields + all_dropped_fields.update(dropped) + + # Log if we dropped anything + if all_dropped_fields: + logger.warning(f"Filtered unknown fields before insert: {all_dropped_fields}") + try: - for each_json in data_list: - instance = model_class(**each_json) + for filtered_json in filtered_data_list: + instance = model_class(**filtered_json) session.add(instance) - session.commit() # Commits both delete + inserts atomically + session.commit() + logger.info("Finished wrapped insert!") - return DatabaseOperation(valid=True) + return DatabaseOperation( + valid=True, + tried_filtered=bool(all_dropped_fields), + message=f"Inserted successfully" + (f" after filtering {all_dropped_fields}" if all_dropped_fields else "") + ) - except TypeError as e: - """ - This is triggered when an Old client was sent unknown fields from the API in the JSON, - And it's handled inside the wrapped function because it needs access to the variables used. - """ + except Exception as e: + # Unexpected error during insert session.rollback() - - valid_fields = {col.name for col in model_class.__table__.columns} - filtered_data_list = [ - {k: v for k, v in each_json.items() if k in valid_fields} - for each_json in data_list - ] - - dropped_fields = set().union(*(set(d.keys()) - valid_fields for d in data_list)) - if dropped_fields: - logger.warning(f"Old Client Dropped unknown fields: {dropped_fields}") - - try: - for filtered_json in filtered_data_list: - instance = model_class(**filtered_json) - session.add(instance) - session.commit() - - # the fix worked, - return DatabaseOperation( - valid=True, - tried_filtered=True, - message=f"Inserted after filtering {dropped_fields}" - ) - - except Exception as retry_error: - # Signal wrapper to rollback - raise WrapperRollback(DatabaseOperation( - valid=False, - error_type=DBErrorType.OLD_CLIENT_NEW_API, - tried_filtered=True, - message=str(f"Wrapped Insert Failed on Retry of a TypeError: {retry_error}") - )) - + raise WrapperRollback(DatabaseOperation( + valid=False, + error_type=DBErrorType.UNEXPECTED_DB_ERROR, + message=f"Wrapped Insert Failed: {str(e)}" + )) # =================== Isolation Test ====================== """