mirror of
https://github.com/All-Hands-AI/OpenHands.git
synced 2026-01-10 15:28:14 -05:00
4db4a84e2e
* simply jupyter execution via heredoc * make sure /tmp always exists * add integration test for jupyter exec
87 lines
2.6 KiB
Bash
Executable File
87 lines
2.6 KiB
Bash
Executable File
#!/bin/bash
|
|
set -eo pipefail
|
|
|
|
WORKSPACE_MOUNT_PATH=$(pwd)/_test_workspace
|
|
WORKSPACE_BASE=$(pwd)/_test_workspace
|
|
SANDBOX_TYPE="ssh"
|
|
MAX_ITERATIONS=10
|
|
|
|
# FIXME: SWEAgent hangs, so it goes last
|
|
agents=("MonologueAgent" "CodeActAgent" "PlannerAgent" "SWEAgent")
|
|
# only enable iteration reminder for CodeActAgent in tests
|
|
remind_iterations_config=(false true false false)
|
|
tasks=(
|
|
"Fix typos in bad.txt."
|
|
"Write a shell script 'hello.sh' that prints 'hello'."
|
|
"Use Jupyter IPython to write a text file containing 'hello world' to '/workspace/test.txt'."
|
|
)
|
|
test_names=(
|
|
"test_edits"
|
|
"test_write_simple_script"
|
|
"test_ipython"
|
|
)
|
|
|
|
num_of_tests=${#tasks[@]}
|
|
num_of_agents=${#agents[@]}
|
|
|
|
rm -rf logs
|
|
rm -rf _test_workspace
|
|
for ((i = 0; i < num_of_tests; i++)); do
|
|
task=${tasks[i]}
|
|
test_name=${test_names[i]}
|
|
for ((j = 0; j < num_of_agents; j++)); do
|
|
agent=${agents[j]}
|
|
remind_iterations=${remind_iterations_config[j]}
|
|
|
|
if [ "$test_name" = "test_ipython" ]; then
|
|
if [ "$agent" != "CodeActAgent" ]; then
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
echo -e "\n\n\n\n========Running $test_name for $agent========\n\n\n\n"
|
|
rm -rf $WORKSPACE_BASE
|
|
mkdir $WORKSPACE_BASE
|
|
|
|
if [ "$TEST_ONLY" = true ]; then
|
|
set -e
|
|
else
|
|
# Temporarily disable 'exit on error'
|
|
set +e
|
|
fi
|
|
SANDBOX_TYPE=$SANDBOX_TYPE WORKSPACE_BASE=$WORKSPACE_BASE \
|
|
MAX_ITERATIONS=$MAX_ITERATIONS REMIND_ITERATIONS=$remind_iterations \
|
|
WORKSPACE_MOUNT_PATH=$WORKSPACE_MOUNT_PATH AGENT=$agent \
|
|
poetry run pytest -s ./tests/integration/test_agent.py::$test_name
|
|
TEST_STATUS=$?
|
|
# Re-enable 'exit on error'
|
|
set -e
|
|
|
|
if [[ $TEST_STATUS -ne 0 ]]; then
|
|
echo -e "\n\n\n\n========$test_name failed, regenerating test data for $agent========\n\n\n\n"
|
|
# trick: let's not clean up $WORKSPACE_BASE folder, which might contain the
|
|
# artifacts we need that are auto-gerated by the test
|
|
sleep 1
|
|
rm -rf logs
|
|
rm -rf tests/integration/mock/$agent/$test_name/*
|
|
echo -e "/exit\n" | SANDBOX_TYPE=$SANDBOX_TYPE WORKSPACE_BASE=$WORKSPACE_BASE \
|
|
DEBUG=true REMIND_ITERATIONS=$remind_iterations \
|
|
WORKSPACE_MOUNT_PATH=$WORKSPACE_MOUNT_PATH AGENT=$agent \
|
|
poetry run python ./opendevin/core/main.py \
|
|
-i $MAX_ITERATIONS \
|
|
-t "$task Do not ask me for confirmation at any point." \
|
|
-c $agent
|
|
|
|
mkdir -p tests/integration/mock/$agent/$test_name/
|
|
mv logs/llm/**/* tests/integration/mock/$agent/$test_name/
|
|
else
|
|
echo -e "\n\n\n\n========$test_name for $agent PASSED========\n\n\n\n"
|
|
sleep 1
|
|
fi
|
|
done
|
|
done
|
|
|
|
rm -rf logs
|
|
rm -rf _test_workspace
|
|
echo "Done!"
|