Evaluation time travel: build sandbox on the fly (#2491)

2026-01-10 23:38:08 -05:00 · 2024-06-20 19:22:02 -07:00
parent 6dd2491944
commit feabc97aba
15 changed files with 20 additions and 28 deletions
--- a/evaluation/EDA/scripts/run_infer.sh
+++ b/evaluation/EDA/scripts/run_infer.sh
@@ -57,5 +57,3 @@ fi
 # Run the command
 echo $COMMAND
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/agent_bench/scripts/run_infer.sh
+++ b/evaluation/agent_bench/scripts/run_infer.sh
@@ -36,5 +36,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/biocoder/scripts/run_infer.sh
+++ b/evaluation/biocoder/scripts/run_infer.sh
@@ -39,5 +39,3 @@ fi
 # Run the command
 echo $COMMAND
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/bird/scripts/run_infer.sh
+++ b/evaluation/bird/scripts/run_infer.sh
@@ -36,5 +36,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/gaia/scripts/run_infer.sh
+++ b/evaluation/gaia/scripts/run_infer.sh
@@ -47,5 +47,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/gorilla/scripts/run_infer.sh
+++ b/evaluation/gorilla/scripts/run_infer.sh
@@ -45,5 +45,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/gpqa/scripts/run_infer.sh
+++ b/evaluation/gpqa/scripts/run_infer.sh
@@ -44,5 +44,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/humanevalfix/scripts/run_infer.sh
+++ b/evaluation/humanevalfix/scripts/run_infer.sh
@@ -74,5 +74,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/logic_reasoning/scripts/run_infer.sh
+++ b/evaluation/logic_reasoning/scripts/run_infer.sh
@@ -40,5 +40,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/miniwob/scripts/run_infer.sh
+++ b/evaluation/miniwob/scripts/run_infer.sh
@@ -46,5 +46,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/mint/scripts/run_infer.sh
+++ b/evaluation/mint/scripts/run_infer.sh
@@ -42,5 +42,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/ml_bench/scripts/run_infer.sh
+++ b/evaluation/ml_bench/scripts/run_infer.sh
@@ -46,5 +46,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/toolqa/scripts/run_infer.sh
+++ b/evaluation/toolqa/scripts/run_infer.sh
@@ -61,5 +61,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch
--- a/evaluation/utils/version_control.sh
+++ b/evaluation/utils/version_control.sh
@@ -1,8 +1,16 @@
 checkout_eval_branch() {
    if [ -z "$COMMIT_HASH" ]; then
        echo "Commit hash not specified, use current git commit"
+        build_sandbox
        return 0
    fi
+
+    if git diff --quiet $COMMIT_HASH HEAD; then
+        echo "The given hash is equivalent to the current HEAD"
+        build_sandbox
+        return 0
+    fi
+
    echo "Start to checkout opendevin version to $COMMIT_HASH, but keep current evaluation harness"
    if ! git diff-index --quiet HEAD --; then
        echo "There are uncommitted changes, please stash or commit them first"
@@ -15,8 +23,20 @@ checkout_eval_branch() {
        echo "Failed to check out to $COMMIT_HASH"
        exit 1
    fi
+
    echo "Revert changes in evaluation folder"
    git checkout $current_branch -- evaluation
+
+    # Trap the EXIT signal to checkout original branch
+    trap checkout_original_branch EXIT
+
+    build_sandbox
+}
+
+build_sandbox() {
+    echo "Build sandbox locally"
+    docker build -t eval-sandbox -f containers/sandbox/Dockerfile /tmp
+    export SANDBOX_CONTAINER_IMAGE="eval-sandbox"
 }

 checkout_original_branch() {
--- a/evaluation/webarena/scripts/run_infer.sh
+++ b/evaluation/webarena/scripts/run_infer.sh
@@ -44,5 +44,3 @@ fi

 # Run the command
 eval $COMMAND
-
-checkout_original_branch