Output validator (#435)

* use output_validator

* upgraded tests

* updated skel

* fix symlink

* create output_validators dir

* changed class constants

* fix source_dir(s)

* copy symlinks

* export legacy

* [export] Fix typo in comment

---------

Co-authored-by: Maarten Sijm <9739541+mpsijm@users.noreply.github.com>

Author: mzuenni

bin/export.py

@@ -92,6 +92,10 @@ def build_samples_zip(problems, output, statement_language):
 def build_problem_zip(problem: Problem, output: Path):
     """Make DOMjudge/Kattis ZIP file for specified problem."""
 
+    if not has_ryaml:
+        error("zip needs the ruamel.yaml python3 library. Install python[3]-ruamel.yaml.")
+        return
+
     # Add problem PDF for only one language to the zip file (note that Kattis export does not include PDF)
     statement_language = None if config.args.kattis else force_single_language([problem])
 
@@ -121,7 +125,7 @@ def build_problem_zip(problem: Problem, output: Path):
         ]
 
     if problem.custom_output:
-        files.append(("output_validators/**/*", True))
+        files.append(("output_validator/**/*", True))
 
     if config.args.kattis:
         files.append(("input_validators/**/*", True))
@@ -170,33 +174,64 @@ def add_file(path, source):
                             out = f2.relative_to(problem.path)
                             add_file(out, f2)
 
-    # DOMjudge does not support 'type' in problem.yaml nor 'output_validator_args' in testdata.yaml yet.
-    # TODO: Remove this once it does.
-    if not config.args.kattis:
-        yaml_path = export_dir / "problem.yaml"
-        yaml_data = [yaml_path.read_text(), "\nvalidation:"]
-        if problem.custom_output:
-            yaml_data.append(" custom")
-            if problem.interactive:
-                yaml_data.append(" interactive")
-            if problem.multi_pass:
-                yaml_data.append(" multi-pass")
-        else:
-            yaml_data.append(" default")
-        yaml_data.append("\n")
-
-        validator_flags = " ".join(
-            problem.get_testdata_yaml(
-                problem.path / "data",
-                "output_validator_args",
-                PrintBar("Getting validator_flags for legacy DOMjudge export"),
-            )
+    # DOMjudge and Kattis do not support 2023-07-draft yet.
+    # TODO: Remove once they do.
+    from ruamel.yaml.comments import CommentedMap
+
+    yaml_path = export_dir / "problem.yaml"
+    yaml_data = read_yaml(yaml_path)
+    # drop format version -> legacy
+    if "problem_format_version" in yaml_data:
+        ryaml_filter(yaml_data, "problem_format_version")
+    # type -> validation
+    if "type" in yaml_data:
+        ryaml_filter(yaml_data, "type")
+    validation = []
+    if problem.custom_output:
+        validation.append("custom")
+        if problem.interactive:
+            validation.append("interactive")
+        if problem.multi_pass:
+            validation.append("multi-pass")
+    else:
+        validation.append("default")
+    yaml_data["validation"] = " ".join(validation)
+    # credits -> author
+    if "credits" in yaml_data:
+        ryaml_filter(yaml_data, "credits")
+        if problem.settings.credits.authors:
+            yaml_data["author"] = ", ".join(p.name for p in problem.settings.credits.authors)
+    # change source:
+    if problem.settings.source:
+        if len(problem.settings.source) > 1:
+            util.warn(f"Found multiple sources, using '{problem.settings.source[0].name}'.")
+        yaml_data["source"] = problem.settings.source[0].name
+        yaml_data["source_url"] = problem.settings.source[0].url
+    # limits.time_multipliers -> time_multiplier / time_safety_margin
+    if "limits" not in yaml_data or not yaml_data["limits"]:
+        yaml_data["limits"] = CommentedMap()
+    limits = yaml_data["limits"]
+    if "time_multipliers" in limits:
+        ryaml_filter(limits, "time_multipliers")
+    limits["time_multiplier"] = problem.limits.ac_to_time_limit
+    limits["time_safety_margin"] = problem.limits.time_limit_to_tle
+    # drop explicit timelimit for kattis:
+    if "time_limit" in limits:
+        # keep this for kattis even when "time_limit" is supported
+        ryaml_filter(limits, "time_limit")
+    # validator_flags
+    validator_flags = " ".join(
+        problem.get_testdata_yaml(
+            problem.path / "data",
+            "output_validator_args",
+            PrintBar("Getting validator_flags for legacy export"),
         )
-        if validator_flags:
-            yaml_data.append(f"validator_flags: {validator_flags}\n")
-
-        yaml_path.unlink()
-        yaml_path.write_text("".join(yaml_data))
+    )
+    if validator_flags:
+        yaml_data["validator_flags"] = validator_flags
+    # write legacy style yaml
+    yaml_path.unlink()
+    write_yaml(yaml_data, yaml_path)
 
     # DOMjudge does not support 'limits.time_limit' in problem.yaml yet.
     # TODO: Remove this once it does.
@@ -224,7 +259,7 @@ def add_file(path, source):
     if problem.settings.constants:
         constants_supported = [
             "data/**/testdata.yaml",
-            "output_validators/**/*",
+            "output_validator/**/*",
             "input_validators/**/*",
             # "statement/*", uses \constants
             # "submissions/*/**/*", removed support?
@@ -242,6 +277,13 @@ def add_file(path, source):
                     f.unlink()
                     f.write_text(text)
 
+    # TODO: Remove this if we know others use the output_validator dir
+    if (export_dir / "output_validator").exists():
+        (export_dir / "output_validators").mkdir(parents=True)
+        (export_dir / "output_validator").rename(
+            export_dir / "output_validators" / "output_validator"
+        )
+
     # TODO: Remove this if we know others import the statement folder
     if (export_dir / "statement").exists():
         (export_dir / "statement").rename(export_dir / "problem_statement")
@@ -286,6 +328,10 @@ def add_file(path, source):
 # solutions*.{lang}.pdf
 # Output is <outfile>
 def build_contest_zip(problems, zipfiles, outfile, statement_language):
+    if not has_ryaml:
+        error("zip needs the ruamel.yaml python3 library. Install python[3]-ruamel.yaml.")
+        return
+
     print(f"writing ZIP file {outfile}", file=sys.stderr)
 
     if not config.args.kattis:  # Kattis does not use problems.yaml.

bin/generate.py

@@ -729,10 +729,7 @@ def validate_ans(t, problem: Problem, testcase: Testcase, meta_yaml: dict, bar:
                     f".ans file is {size / 1024 / 1024:.3f}MiB, which is close to output limit (set limits.output to at least {(2 * size + 1024 * 1024 - 1) // 1024 // 1024}MiB in problem.yaml)"
                 )
 
-            answer_validator_hashes = {
-                **testcase.validator_hashes(validate.AnswerValidator, bar),
-                **testcase.validator_hashes(validate.OutputValidator, bar),
-            }
+            answer_validator_hashes = {**testcase.validator_hashes(validate.AnswerValidator, bar)}
             if all(h in meta_yaml["answer_validator_hashes"] for h in answer_validator_hashes):
                 return True
 

bin/interactive.py

@@ -32,7 +32,7 @@ def run_interactive_testcase(
     bar: Optional[ProgressBar] = None,
 ):
     output_validators = run.problem.validators(validate.OutputValidator)
-    if len(output_validators) != 1:
+    if not output_validators:
         return None
     output_validator = output_validators[0]
 

bin/problem.py

@@ -224,7 +224,7 @@ def __init__(
         self.interactive: bool = "interactive" in mode
         self.multi_pass: bool = "multi-pass" in mode
         self.custom_output: bool = (
-            self.interactive or self.multi_pass or (problem.path / "output_validators").exists()
+            self.interactive or self.multi_pass or (problem.path / "output_validator").is_dir()
         )
 
         self.name: dict[str, str] = parse_setting(yaml_data, "name", {"en": ""})
@@ -801,8 +801,6 @@ def validators(
                         warn("No input validators found.")
                     case validate.AnswerValidator, 0:
                         warn("No answer validators found")
-                    case validate.OutputValidator, l if l != 1:
-                        error(f"Found {len(validators)} output validators, expected exactly one.")
 
         build_ok = all(v.ok for v in validators)
 
@@ -817,16 +815,16 @@ def _validators(
         if key in problem._validators_cache:
             return problem._validators_cache[key]
 
-        assert hasattr(cls, "source_dirs")
-        # TODO #424: We should not support multiple output validators inside output_validator/.
-        paths = [p for source_dir in cls.source_dirs for p in glob(problem.path / source_dir, "*")]
-
-        # Handle default output validation
         if cls == validate.OutputValidator:
-            if not paths:
-                if problem.custom_output:
-                    fatal("Problem validation type requires output_validators/")
+            if problem.custom_output:
+                paths = [problem.path / validate.OutputValidator.source_dir]
+            else:
                 paths = [config.TOOLS_ROOT / "support" / "default_output_validator.cpp"]
+        else:
+            assert hasattr(cls, "source_dirs")
+            paths = [
+                p for source_dir in cls.source_dirs for p in glob(problem.path / source_dir, "*")
+            ]
 
         # TODO: Instead of checking file contents, maybe specify this in generators.yaml?
         def has_constraints_checking(f):
@@ -878,13 +876,8 @@ def prepare_run(problem):
         if not testcases:
             return False
 
-        if problem.interactive or problem.multi_pass:
-            validators = problem.validators(validate.OutputValidator)
-            if not validators:
-                return False
-
         # Pre build the output validator to prevent nested ProgressBars.
-        if problem.validators(validate.OutputValidator) is False:
+        if not problem.validators(validate.OutputValidator):
             return False
 
         submissions = problem.submissions()

bin/run.py

@@ -217,12 +217,12 @@ def _prepare_nextpass(self, nextpass):
 
     def _validate_output(self, bar):
         output_validators = self.problem.validators(validate.OutputValidator)
-        if len(output_validators) != 1:
+        if not output_validators:
             return None
-        validator = output_validators[0]
-
-        return validator.run(
-            self.testcase, self, args=self.testcase.testdata_yaml_validator_args(validator, bar)
+        return output_validators[0].run(
+            self.testcase,
+            self,
+            args=self.testcase.testdata_yaml_validator_args(output_validators[0], bar),
         )
 
 
@@ -522,10 +522,8 @@ def test(self):
 
         testcases = self.problem.testcases(needans=False)
 
-        if self.problem.interactive:
-            output_validators = self.problem.validators(validate.OutputValidator)
-            if output_validators is False:
-                return
+        if not self.problem.validators(validate.OutputValidator):
+            return
 
         for testcase in testcases:
             header = ProgressBar.action("Running " + str(self.name), testcase.name)
@@ -589,10 +587,8 @@ def test(self):
 
     # Run the submission using stdin as input.
     def test_interactive(self):
-        if self.problem.interactive:
-            output_validators = self.problem.validators(validate.OutputValidator)
-            if output_validators is False:
-                return
+        if not self.problem.validators(validate.OutputValidator):
+            return
 
         bar = ProgressBar("Running " + str(self.name), max_len=1, count=1)
         bar.start()

bin/skel.py

@@ -243,7 +243,7 @@ def new_problem():
         variables,
         exist_ok=True,
         preserve_symlinks=preserve_symlinks,
-        skip=[skeldir / "output_validators"] if not custom_output else None,
+        skip=[skeldir / "output_validator"] if not custom_output else None,
     )
 
     # Warn about missing problem statement skeletons for non-en languages

bin/stats.py

@@ -53,7 +53,7 @@ def problem_stats(problems):
         ("sol", str(latex.PdfType.SOLUTION.path("*")), 1),
         ("  val: I", ["input_validators/*", "input_format_validators/*"]),
         ("A", ["answer_validators/*"]),
-        ("O", ["output_validators/*"]),
+        ("O", ["output_validator/"]),
         (
             "  sample",
             [lambda s: {x.stem for x in s if x.parts[2] == "sample"}],

bin/testcase.py

@@ -150,7 +150,7 @@ def validator_hashes(self, cls: type["validate.AnyValidator"], bar):
         indicating which validators will be run for this testcase.
         """
         assert cls in [validate.InputValidator, validate.AnswerValidator, validate.OutputValidator]
-        validators = self.problem.validators(cls) or []
+        validators = self.problem.validators(cls)
 
         d = dict()