Updates for only selecting a single prompt type.

evaluate_genericify_cpp.py

@@ -3,6 +3,8 @@
 import collections
 import os
 
+from typing import Literal
+
 import concurrent.futures
 import datasets
 import evaluate
@@ -119,7 +121,14 @@ class EvaluateGenericifyCpp(evaluate.Metric):
             reference_urls=["http://path.to.reference.url/new_module"],
         )
 
-    def _compute(self, *, predictions, references, k=[1, 10, 100]):
+    def _compute(
+        self,
+        *,
+        predictions,
+        references,
+        cpp_type: Literal["base", "sfinae", "concepts"],
+        k=[1, 10, 100],
+    ):
         """Returns the scores"""
         num_workers = 4
 
@@ -148,6 +157,7 @@ class EvaluateGenericifyCpp(evaluate.Metric):
                     args = (
                         candidate,
                         reference,
+                        cpp_type,
                         task_id,
                         completion_id[task_id],
                     )
@@ -163,16 +173,26 @@ class EvaluateGenericifyCpp(evaluate.Metric):
 
         totals = collections.defaultdict(list)
         corrects = collections.defaultdict(list)
-        for result in results.values():
-            result.sort()
-            for pt in [
+
+        keys = {
+            "base": [
                 "base_run_passed",
                 "base_run_compiled",
+            ],
+            "sfinae": [
                 "sfinae_run_passed",
                 "sfinae_run_compiled",
+                "sfinae_constrain_passed",
+            ],
+            "concepts": [
                 "concepts_run_passed",
                 "concepts_run_compiled",
-            ]:
+                "concepts_constrain_passed",
+            ],
+        }[cpp_type]
+        for result in results.values():
+            result.sort()
+            for pt in keys:
                 passed = [r[1][pt] for r in result]
                 totals[pt].append(len(passed))
                 corrects[pt].append(sum(passed))

execute.py

@@ -5,7 +5,7 @@ import subprocess
 import tempfile
 
 
-def check_correctness(candidate, reference, task_id, completion_id):
+def check_correctness(candidate, reference, cpp_type, task_id, completion_id):
     """
     Evaluates the functional correctness of a completion by running the test
     suite provided in the problem.
@@ -15,75 +15,88 @@ def check_correctness(candidate, reference, task_id, completion_id):
     """
 
     manager = multiprocessing.Manager()
-    base_run_result = manager.list()
-    process_case(
-        unsafe_execute_cpp,
-        candidate["base"],
-        reference["tests"],
-        base_run_result,
-        "c++17",
-    )
-    sfinae_run_result = manager.list()
-    process_case(
-        unsafe_execute_cpp,
-        candidate["sfinae"],
-        reference["tests"],
-        sfinae_run_result,
-        "c++17",
-    )
-    concepts_run_result = manager.list()
-    process_case(
-        unsafe_execute_cpp,
-        candidate["concepts"],
-        reference["tests"],
-        concepts_run_result,
-        "c++20",
-    )
-
-    sfinae_constrain_result = manager.list()
-    process_case(
-        invalid_compile_cpp,
-        candidate["sfinae"],
-        reference["invalids"],
-        sfinae_constrain_result,
-        "c++17",
-    )
-    concepts_constrain_result = manager.list()
-    process_case(
-        invalid_compile_cpp,
-        candidate["concepts"],
-        reference["invalids"],
-        concepts_constrain_result,
-        "c++20",
-    )
 
-    return dict(
+    result = dict(
         task_id=task_id,
         completion_id=completion_id,
-        base_run_passed=base_run_result[0] == "passed",
-        base_run_compiled=(
-            base_run_result[0] == "passed"
-            or base_run_result[0].startswith("failed: runtime error:")
-        ),
-        base_run_result=base_run_result[0],
-        sfinae_run_passed=sfinae_run_result[0] == "passed",
-        sfinae_run_compiled=(
-            sfinae_run_result[0] == "passed"
-            or sfinae_run_result[0].startswith("failed: runtime error:")
-        ),
-        sfinae_run_result=sfinae_run_result[0],
-        concepts_run_passed=concepts_run_result[0] == "passed",
-        concepts_run_compiled=(
-            concepts_run_result[0] == "passed"
-            or concepts_run_result[0].startswith("failed: runtime error:")
-        ),
-        concepts_run_result=concepts_run_result[0],
-        sfinae_constrain_passed=sfinae_constrain_result[0] == "passed",
-        sfinae_constrain_result=sfinae_constrain_result[0],
-        concepts_constrain_passed=concepts_constrain_result[0] == "passed",
-        concepts_constrain_result=concepts_constrain_result[0],
     )
 
+    if cpp_type == "base":
+        base_run_result = manager.list()
+        process_case(
+            unsafe_execute_cpp,
+            candidate,
+            reference["tests"],
+            base_run_result,
+            "c++17",
+        )
+        result |= dict(
+            base_run_passed=base_run_result[0] == "passed",
+            base_run_compiled=(
+                base_run_result[0] == "passed"
+                or base_run_result[0].startswith("failed: runtime error:")
+            ),
+            base_run_result=base_run_result[0],
+        )
+    elif cpp_type == "sfinae":
+        sfinae_run_result = manager.list()
+        process_case(
+            unsafe_execute_cpp,
+            candidate,
+            reference["tests"],
+            sfinae_run_result,
+            "c++17",
+        )
+        sfinae_constrain_result = manager.list()
+        process_case(
+            invalid_compile_cpp,
+            candidate,
+            reference["invalids"],
+            sfinae_constrain_result,
+            "c++17",
+        )
+        result |= dict(
+            sfinae_run_passed=sfinae_run_result[0] == "passed",
+            sfinae_run_compiled=(
+                sfinae_run_result[0] == "passed"
+                or sfinae_run_result[0].startswith("failed: runtime error:")
+            ),
+            sfinae_run_result=sfinae_run_result[0],
+            sfinae_constrain_passed=sfinae_constrain_result[0] == "passed",
+            sfinae_constrain_result=sfinae_constrain_result[0],
+        )
+    elif cpp_type == "concepts":
+        concepts_run_result = manager.list()
+        process_case(
+            unsafe_execute_cpp,
+            candidate,
+            reference["tests"],
+            concepts_run_result,
+            "c++20",
+        )
+        concepts_constrain_result = manager.list()
+        process_case(
+            invalid_compile_cpp,
+            candidate,
+            reference["invalids"],
+            concepts_constrain_result,
+            "c++20",
+        )
+        result |= dict(
+            concepts_run_passed=concepts_run_result[0] == "passed",
+            concepts_run_compiled=(
+                concepts_run_result[0] == "passed"
+                or concepts_run_result[0].startswith("failed: runtime error:")
+            ),
+            concepts_run_result=concepts_run_result[0],
+            concepts_constrain_passed=concepts_constrain_result[0] == "passed",
+            concepts_constrain_result=concepts_constrain_result[0],
+        )
+    else:
+        raise ValueError(f"Unknown cpp_type: {cpp_type}")
+
+    return result
+
 
 def process_case(target, candidate, reference, result, cppstd):
     timeout = 60