fix yaml handling

Author: mzuenni

bin/upgrade.py

@@ -55,8 +55,8 @@ def upgrade_generators_yaml(problem_path: Path, bar: ProgressBar) -> None:
     generators_yaml = problem_path / "generators" / "generators.yaml"
     if not generators_yaml.is_file():
         return
-    data = read_yaml(generators_yaml)
-    if data is None or not isinstance(data, dict):
+    yaml_data = read_yaml(generators_yaml)
+    if yaml_data is None or not isinstance(yaml_data, dict):
         return
 
     changed = False
@@ -67,17 +67,19 @@ def upgrade_generators_yaml(problem_path: Path, bar: ProgressBar) -> None:
         ("invalid_outputs", "invalid_output"),
         ("valid_outputs", "valid_output"),
     ]
-    for old_name, new_name in rename:
-        if old_name in data:
-            if new_name in data:
-                bar.error(
-                    f"can't rename 'data.{old_name}', 'data.{new_name}' already exists in generators.yaml",
-                    resume=True,
-                )
-                continue
-            bar.log(f"renaming 'data.{old_name}' to 'data.{new_name}' in generators.yaml")
-            ryaml_replace(data, old_name, new_name)
-            changed = True
+    if "data" in yaml_data and isinstance(yaml_data["data"], dict):
+        data = yaml_data["data"]
+        for old_name, new_name in rename:
+            if old_name in data:
+                if new_name in data:
+                    bar.error(
+                        f"can't rename 'data.{old_name}', 'data.{new_name}' already exists in generators.yaml",
+                        resume=True,
+                    )
+                    continue
+                bar.log(f"renaming 'data.{old_name}' to 'data.{new_name}' in generators.yaml")
+                ryaml_replace(data, old_name, new_name)
+                changed = True
 
     def upgrade_generated_testdata_yaml(data: dict[str, Any], path: str) -> bool:
         changed = False
@@ -111,10 +113,10 @@ def upgrade_generated_testdata_yaml(data: dict[str, Any], path: str) -> bool:
                         )
         return changed
 
-    changed |= upgrade_generated_testdata_yaml(data, "")
+    changed |= upgrade_generated_testdata_yaml(yaml_data, "")
 
     if changed:
-        write_yaml(data, generators_yaml)
+        write_yaml(yaml_data, generators_yaml)
 
 
 def upgrade_statement(problem_path: Path, bar: ProgressBar) -> None:

bin/util.py

@@ -729,7 +729,7 @@ def ryaml_filter(data: Any, remove: str) -> Any:
 
         curr = data
         prev_key = list(data.keys())[remove_index - 1]
-        while isinstance(curr[prev_key], list | dict):
+        while isinstance(curr[prev_key], list | dict) and len(curr[prev_key]):
             # Try to remove the comment from the last element in the preceding list/dict
             curr = curr[prev_key]
             if isinstance(curr, list):
@@ -741,7 +741,7 @@ def ryaml_filter(data: Any, remove: str) -> Any:
             # Move the comment that belongs to the removed key (which comes _after_ the removed key)
             # to the preceding key
             curr.ca.items[prev_key] = data.ca.items.pop(remove)
-        elif prev_key in data.ca.items:
+        elif prev_key in curr.ca.items:
             # If the removed key does not have a comment,
             # the comment after the previous key should be removed
             curr.ca.items.pop(prev_key)
@@ -755,7 +755,9 @@ def ryaml_replace(data: Any, old_key: str, new_key: str, new_value: Any = None)
         if new_value is None:
             new_value = data[old_key]
         data.insert(list(data.keys()).index(old_key), new_key, new_value)
-        ryaml_filter(data, old_key)
+        data.pop(old_key)
+        if old_key in data.ca.items:
+            data.ca.items[new_key] = data.ca.items.pop(old_key)
 
 
 # Only allow one thread to write at the same time. Else, e.g., generating test cases in parallel goes wrong.