diff --git a/cms/grading/steps/trusted.py b/cms/grading/steps/trusted.py index 41abc25044..b346c41108 100644 --- a/cms/grading/steps/trusted.py +++ b/cms/grading/steps/trusted.py @@ -182,7 +182,7 @@ def trusted_step(sandbox, commands): def checker_step(sandbox, checker_digest, input_digest, correct_output_digest, - output_filename): + output_filename, extra_args=None): """Run the explicit checker given by the admins sandbox (Sandbox): the sandbox to run the checker in; should already @@ -196,6 +196,7 @@ def checker_step(sandbox, checker_digest, input_digest, correct_output_digest, as "correct_output.txt". output_filename (str): inner filename of the user output (already in the sandbox). + extra_args ([str]|None): extra arguments to pass to the checker. return (bool, float|None, [str]|None): success (true if the checker was able to check the solution successfully), outcome and text (both None @@ -228,7 +229,7 @@ def checker_step(sandbox, checker_digest, input_digest, correct_output_digest, command = ["./%s" % CHECKER_FILENAME, CHECKER_INPUT_FILENAME, CHECKER_CORRECT_OUTPUT_FILENAME, - output_filename] + output_filename] + (extra_args if extra_args is not None else []) box_success, success, unused_stats = trusted_step(sandbox, [command]) if not box_success or not success: logger.error("Sandbox failed during checker step. " diff --git a/cms/grading/tasktypes/BatchAndOutput.py b/cms/grading/tasktypes/BatchAndOutput.py new file mode 100644 index 0000000000..d8cb4c45fa --- /dev/null +++ b/cms/grading/tasktypes/BatchAndOutput.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 + +# Contest Management System - http://cms-dev.github.io/ +# Copyright © 2010-2015 Giovanni Mascellani +# Copyright © 2010-2018 Stefano Maggiolo +# Copyright © 2010-2012 Matteo Boscariol +# Copyright © 2012-2014 Luca Wehrstedt +# Copyright © 2017 Myungwoo Chun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import os + +from cms.db import Executable +from cms.grading.ParameterTypes import ParameterTypeCollection, \ + ParameterTypeChoice, ParameterTypeString +from cms.grading.languagemanager import LANGUAGES, get_language +from cms.grading.steps import compilation_step, evaluation_step, \ + human_evaluation_message +from . import TaskType, \ + check_executables_number, check_files_number, check_manager_present, \ + create_sandbox, delete_sandbox, eval_output, is_manager_for_compilation + + +logger = logging.getLogger(__name__) + + +# Dummy function to mark translatable string. +def N_(message): + return message + + +class BatchAndOutput(TaskType): + """Task type class for a task that is a combination of Batch and + OutputOnly. + + Parameters needs to be a list of three elements. + + The first element is 'grader' or 'alone': in the first + case, the source file is to be compiled with a provided piece of + software ('grader'); in the other by itself. + + The second element is a 2-tuple of the input file name and output file + name. The input file may be '' to denote stdin, and similarly the + output filename may be '' to denote stdout. + + The third element is 'diff' or 'comparator' and says whether the + output is compared with a simple diff algorithm or using a + comparator. + + The fourth element specifies testcases that *must* be provided as + output-only. If a testcase is not in this list, and is not provided + explicitly, then the provided source file will be used to compute + the output. + + Note: the first element is used only in the compilation step; the + others only in the evaluation step. + + A comparator can read argv[1], argv[2], argv[3], and argv[4] (respectively, + input, correct output, user output and 'outputonly' if the testcase was an + outputonly-only testcase, 'batch' otherwise) and should write the outcome + to stdout and the text to stderr. + + """ + # Codename of the checker, if it is used. + CHECKER_CODENAME = "checker" + # Basename of the grader, used in the manager filename and as the main + # class in languages that require us to specify it. + GRADER_BASENAME = "grader" + # Default input and output filenames when not provided as parameters. + DEFAULT_INPUT_FILENAME = "input.txt" + DEFAULT_OUTPUT_FILENAME = "output.txt" + + # Constants used in the parameter definition. + OUTPUT_EVAL_DIFF = "diff" + OUTPUT_EVAL_CHECKER = "comparator" + COMPILATION_ALONE = "alone" + COMPILATION_GRADER = "grader" + + # Template for the filename of the output files provided by the user; %s + # represent the testcase codename. + USER_OUTPUT_FILENAME_TEMPLATE = "output_%s.txt" + + # Other constants to specify the task type behaviour and parameters. + ALLOW_PARTIAL_SUBMISSION = True + + _COMPILATION = ParameterTypeChoice( + "Compilation", + "compilation", + "", + {COMPILATION_ALONE: "Submissions are self-sufficient", + COMPILATION_GRADER: "Submissions are compiled with a grader"}) + + _USE_FILE = ParameterTypeCollection( + "I/O (blank for stdin/stdout)", + "io", + "", + [ + ParameterTypeString("Input file", "inputfile", ""), + ParameterTypeString("Output file", "outputfile", ""), + ]) + + _EVALUATION = ParameterTypeChoice( + "Output evaluation", + "output_eval", + "", + {OUTPUT_EVAL_DIFF: "Outputs compared with white diff", + OUTPUT_EVAL_CHECKER: "Outputs are compared by a comparator"}) + + _OUTPUT_ONLY_TESTCASES = ParameterTypeString( + "Comma-separated list of output only testcases", + "output_only_testcases", + "") + + ACCEPTED_PARAMETERS = [_COMPILATION, _USE_FILE, + _EVALUATION, _OUTPUT_ONLY_TESTCASES] + + @property + def name(self): + """See TaskType.name.""" + # TODO add some details if a grader/comparator is used, etc... + return "BatchAndOutput" + + def __init__(self, parameters): + super().__init__(parameters) + + # Data in the parameters. + self.compilation = self.parameters[0] + self.input_filename, self.output_filename = self.parameters[1] + self.output_eval = self.parameters[2] + self.output_only_testcases = set(self.parameters[3].split(',')) + + # Actual input and output are the files used to store input and + # where the output is checked, regardless of using redirects or not. + self._actual_input = self.input_filename + self._actual_output = self.output_filename + if len(self.input_filename) == 0: + self._actual_input = self.DEFAULT_INPUT_FILENAME + if len(self.output_filename) == 0: + self._actual_output = self.DEFAULT_OUTPUT_FILENAME + + def get_compilation_commands(self, submission_format): + """See TaskType.get_compilation_commands.""" + codenames_to_compile = [] + if self._uses_grader(): + codenames_to_compile.append(self.GRADER_BASENAME + ".%l") + codenames_to_compile.extend( + [x for x in submission_format if x.endswith('.%l')]) + res = dict() + for language in LANGUAGES: + source_ext = language.source_extension + executable_filename = self._executable_filename(submission_format, + language) + res[language.name] = language.get_compilation_commands( + [codename.replace(".%l", source_ext) + for codename in codenames_to_compile], + executable_filename) + return res + + def get_user_managers(self): + """See TaskType.get_user_managers.""" + # In case the task uses a grader, we let the user provide their own + # grader (which is usually a simplified grader provided by the admins). + if self._uses_grader(): + return [self.GRADER_BASENAME + ".%l"] + else: + return [] + + def get_auto_managers(self): + """See TaskType.get_auto_managers.""" + return [] + + def _uses_grader(self): + return self.compilation == self.COMPILATION_GRADER + + def _uses_checker(self): + return self.output_eval == self.OUTPUT_EVAL_CHECKER + + @staticmethod + def _executable_filename(codenames, language): + """Return the chosen executable name computed from the codenames. + + codenames ([str]): submission format or codename of submitted files, + may contain %l. + language (Language): the programming language of the submission. + + return (str): a deterministic executable name. + + """ + name = "_".join(sorted(codename.replace(".%l", "") + for codename in codenames if codename.endswith('.%l'))) + return name + language.executable_extension + + @staticmethod + def _get_user_output_filename(job): + return BatchAndOutput.USER_OUTPUT_FILENAME_TEMPLATE % \ + job.operation.testcase_codename + + def compile(self, job, file_cacher): + """See TaskType.compile.""" + language = get_language(job.language) + source_ext = language.source_extension + + num_source_files = 0 + for (codename, _) in job.files.items(): + if codename.endswith(".%l"): + num_source_files += 1 + + if num_source_files == 0: + # This submission did not have any source files, skip compilation + job.success = True + job.compilation_success = True + job.text = [N_("No compilation needed")] + job.plus = {} + return + + # Create the list of filenames to be passed to the compiler. If we use + # a grader, it needs to be in first position in the command line, and + # we check that it exists. + filenames_to_compile = [] + filenames_and_digests_to_get = {} + # The grader, that must have been provided (copy and add to + # compilation). + if self._uses_grader(): + grader_filename = self.GRADER_BASENAME + source_ext + if not check_manager_present(job, grader_filename): + return + filenames_to_compile.append(grader_filename) + filenames_and_digests_to_get[grader_filename] = \ + job.managers[grader_filename].digest + # User's submitted file(s) (copy and add to compilation). + for codename, file_ in job.files.items(): + if not codename.endswith(".%l"): + continue + filename = codename.replace(".%l", source_ext) + filenames_to_compile.append(filename) + filenames_and_digests_to_get[filename] = file_.digest + # Any other useful manager (just copy). + for filename, manager in job.managers.items(): + if is_manager_for_compilation(filename, language): + filenames_and_digests_to_get[filename] = manager.digest + + # Prepare the compilation command. + executable_filename = self._executable_filename(job.files.keys(), + language) + commands = language.get_compilation_commands( + filenames_to_compile, executable_filename) + + # Create the sandbox. + sandbox = create_sandbox(file_cacher, name="compile") + job.sandboxes.append(sandbox.get_root_path()) + + # Copy required files in the sandbox (includes the grader if present). + for filename, digest in filenames_and_digests_to_get.items(): + sandbox.create_file_from_storage(filename, digest) + + # Run the compilation. + box_success, compilation_success, text, stats = \ + compilation_step(sandbox, commands) + + # Retrieve the compiled executables. + job.success = box_success + job.compilation_success = compilation_success + job.text = text + job.plus = stats + if box_success and compilation_success: + digest = sandbox.get_file_to_storage( + executable_filename, + "Executable %s for %s" % (executable_filename, job.info)) + job.executables[executable_filename] = \ + Executable(executable_filename, digest) + + # Cleanup. + delete_sandbox(sandbox, job.success, job.keep_sandbox) + + def evaluate(self, job, file_cacher): + """See TaskType.evaluate.""" + + user_output_filename = self._get_user_output_filename(job) + + output_file_params = None + sandbox = None + outcome = None + box_success = True + text = "" + stats = {} + + if user_output_filename in job.files: + output_file_params = { + 'user_output_digest': job.files[user_output_filename].digest} + elif job.operation.testcase_codename in self.output_only_testcases: + pass + elif job.executables: + # Prepare the execution + executable_filename = next(iter(job.executables.keys())) + language = get_language(job.language) + main = self.GRADER_BASENAME if self._uses_grader() \ + else os.path.splitext(executable_filename)[0] + commands = language.get_evaluation_commands( + executable_filename, main=main) + executables_to_get = { + executable_filename: job.executables[executable_filename].digest + } + files_to_get = { + self._actual_input: job.input + } + + # Check which redirect we need to perform, and in case we don't + # manage the output via redirect, the submission needs to be able + # to write on it. + files_allowing_write = [] + stdin_redirect = None + stdout_redirect = None + if len(self.input_filename) == 0: + stdin_redirect = self._actual_input + if len(self.output_filename) == 0: + stdout_redirect = self._actual_output + else: + files_allowing_write.append(self._actual_output) + + # Create the sandbox + sandbox = create_sandbox(file_cacher, name="evaluate") + job.sandboxes.append(sandbox.get_root_path()) + + # Put the required files into the sandbox + for filename, digest in executables_to_get.items(): + sandbox.create_file_from_storage( + filename, digest, executable=True) + for filename, digest in files_to_get.items(): + sandbox.create_file_from_storage(filename, digest) + + # Actually performs the execution + box_success, evaluation_success, stats = evaluation_step( + sandbox, + commands, + job.time_limit, + job.memory_limit, + writable_files=files_allowing_write, + stdin_redirect=stdin_redirect, + stdout_redirect=stdout_redirect, + multiprocess=job.multithreaded_sandbox) + + outcome = None + text = None + + # Error in the sandbox: nothing to do! + if not box_success: + pass + + # Contestant's error: the marks won't be good + elif not evaluation_success: + outcome = 0.0 + text = human_evaluation_message(stats) + if job.get_output: + job.user_output = None + + # Otherwise, advance to checking the solution + else: + + # Check that the output file was created + if not sandbox.file_exists(self._actual_output): + outcome = 0.0 + text = [N_("Evaluation didn't produce file %s"), + self._actual_output] + if job.get_output: + job.user_output = None + + else: + # If asked so, put the output file into the storage. + if job.get_output: + job.user_output = sandbox.get_file_to_storage( + self._actual_output, + "Output file in job %s" % job.info, + trunc_len=100 * 1024) + + # If just asked to execute, fill text and set dummy outcome. + if job.only_execution: + outcome = 0.0 + text = [N_("Execution completed successfully")] + + # Otherwise prepare to evaluate the output file. + else: + output_file_params = { + 'user_output_path': sandbox.relative_path( + self._actual_output), + 'user_output_filename': self.output_filename} + + if output_file_params is None and outcome is None: + job.success = True + job.outcome = "0.0" + job.text = [N_("File not submitted")] + job.plus = {} + return + + if output_file_params is not None: + if job.operation.testcase_codename in self.output_only_testcases: + extra_args = ['outputonly'] + else: + extra_args = ['batch'] + box_success, outcome, text = eval_output( + file_cacher, job, + self.CHECKER_CODENAME + if self._uses_checker() else None, + **output_file_params, extra_args=extra_args) + + # Fill in the job with the results. + job.success = box_success + job.outcome = str(outcome) if outcome is not None else None + job.text = text + job.plus = stats + + if sandbox is not None: + delete_sandbox(sandbox, job.success, job.keep_sandbox) diff --git a/cms/grading/tasktypes/util.py b/cms/grading/tasktypes/util.py index a616a3cf75..087e95e07c 100644 --- a/cms/grading/tasktypes/util.py +++ b/cms/grading/tasktypes/util.py @@ -204,7 +204,7 @@ def check_manager_present(job, codename): def eval_output(file_cacher, job, checker_codename, user_output_path=None, user_output_digest=None, - user_output_filename=""): + user_output_filename="", extra_args=None): """Evaluate ("check") a user output using a white diff or a checker. file_cacher (FileCacher): file cacher to use to get files. @@ -217,6 +217,7 @@ def eval_output(file_cacher, job, checker_codename, using the path (exactly one must be non-None). user_output_filename (str): the filename the user was expected to write to, or empty if stdout (used to return an error to the user). + extra_args ([str]|None): additional arguments to pass to the checker return (bool, float|None, [str]|None): success (true if the checker was able to check the solution successfully), outcome and text (both None @@ -256,7 +257,7 @@ def eval_output(file_cacher, job, checker_codename, if checker_codename in job.managers else None success, outcome, text = checker_step( sandbox, checker_digest, job.input, job.output, - EVAL_USER_OUTPUT_FILENAME) + EVAL_USER_OUTPUT_FILENAME, extra_args) delete_sandbox(sandbox, success, job.keep_sandbox) return success, outcome, text diff --git a/setup.py b/setup.py index 200c843de1..646858b2e1 100755 --- a/setup.py +++ b/setup.py @@ -174,6 +174,7 @@ def run(self): ], "cms.grading.tasktypes": [ "Batch=cms.grading.tasktypes.Batch:Batch", + "BatchAndOutput=cms.grading.tasktypes.BatchAndOutput:BatchAndOutput", "Communication=cms.grading.tasktypes.Communication:Communication", "OutputOnly=cms.grading.tasktypes.OutputOnly:OutputOnly", "TwoSteps=cms.grading.tasktypes.TwoSteps:TwoSteps",