github/OpenHands
1
1
Fork 0
mirror of https://github.com/All-Hands-AI/OpenHands.git synced 2026-01-08 22:38:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(runtime): use micromamba instead of mamba and fix build issue (#4154)

Browse Source
This commit is contained in:
Xingyao Wang
2024-10-02 16:23:18 -05:00
committed by GitHub
parent c8a933590a
commit e81c5597d6
11 changed files with 51 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/dummy-agent-test.yml vendored
View File

@@ -45,7 +45,7 @@ jobs:
- name: Run tests
run: |
set -e
poetry run python3 openhands/core/main.py -t "do a flip" -d ./workspace/ -c DummyAgent
SANDBOX_FORCE_REBUILD_RUNTIME=True poetry run python3 openhands/core/main.py -t "do a flip" -d ./workspace/ -c DummyAgent
- name: Check exit code
run: |
if [ $? -ne 0 ]; then

4
.github/workflows/ghcr-build.yml vendored
View File

@@ -293,7 +293,7 @@ jobs:
SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
TEST_IN_CI=true \
RUN_AS_OPENHANDS=false \
poetry run pytest -n 3 -raR --reruns 1 --reruns-delay 3 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
env:
@@ -371,7 +371,7 @@ jobs:
SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
TEST_IN_CI=true \
RUN_AS_OPENHANDS=true \
poetry run pytest -n 3 -raR --reruns 1 --reruns-delay 3 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
env:

2
openhands/core/config/sandbox_config.py
View File

@@ -18,6 +18,7 @@ class SandboxConfig:
enable_auto_lint: Whether to enable auto-lint.
use_host_network: Whether to use the host network.
initialize_plugins: Whether to initialize plugins.
force_rebuild_runtime: Whether to force rebuild the runtime image.
runtime_extra_deps: The extra dependencies to install in the runtime image (typically used for evaluation).
This will be rendered into the end of the Dockerfile that builds the runtime image.
It can contain any valid shell commands (e.g., pip install numpy).
@@ -43,6 +44,7 @@ class SandboxConfig:
)
use_host_network: bool = False
initialize_plugins: bool = True
force_rebuild_runtime: bool = False
runtime_extra_deps: str | None = None
runtime_startup_env_vars: dict[str, str] = field(default_factory=dict)
browsergym_eval_env: str | None = None

4
openhands/runtime/builder/docker.py
View File

@@ -113,8 +113,8 @@ class DockerRuntimeBuilder(RuntimeBuilder):
raise subprocess.CalledProcessError(
return_code,
process.args,
output=None,
stderr=None,
output=process.stdout.read() if process.stdout else None,
stderr=process.stderr.read() if process.stderr else None,
)
except subprocess.CalledProcessError as e:

3
openhands/runtime/client/runtime.py
View File

@@ -167,6 +167,7 @@ class EventStreamRuntime(Runtime):
self.base_container_image,
self.runtime_builder,
extra_deps=self.config.sandbox.runtime_extra_deps,
force_rebuild=self.config.sandbox.force_rebuild_runtime,
)
self.container = self._init_container(
sandbox_workspace_dir=self.config.workspace_mount_path_in_sandbox, # e.g. /workspace
@@ -273,7 +274,7 @@ class EventStreamRuntime(Runtime):
container = self.docker_client.containers.run(
self.runtime_container_image,
command=(
f'/openhands/miniforge3/bin/mamba run --no-capture-output -n base '
f'/openhands/micromamba/bin/micromamba run -n openhands '
f'poetry run '
f'python -u -m openhands.runtime.client.client {self._container_port} '
f'--working-dir "{sandbox_workspace_dir}" '

3
openhands/runtime/plugins/jupyter/__init__.py
View File

@@ -28,7 +28,8 @@ class JupyterPlugin(Plugin):
'cd /openhands/code\n'
'export POETRY_VIRTUALENVS_PATH=/openhands/poetry;\n'
'export PYTHONPATH=/openhands/code:$PYTHONPATH;\n'
'/openhands/miniforge3/bin/mamba run -n base '
'export MAMBA_ROOT_PREFIX=/openhands/micromamba;\n'
'/openhands/micromamba/bin/micromamba run -n openhands '
'poetry run jupyter kernelgateway '
'--KernelGatewayApp.ip=0.0.0.0 '
f'--KernelGatewayApp.port={self.kernel_gateway_port}\n'

5
openhands/runtime/remote/runtime.py
View File

@@ -119,6 +119,7 @@ class RemoteRuntime(Runtime):
self.config.sandbox.base_container_image,
self.runtime_builder,
extra_deps=self.config.sandbox.runtime_extra_deps,
force_rebuild=self.config.sandbox.force_rebuild_runtime,
)
response = send_request(
@@ -144,8 +145,8 @@ class RemoteRuntime(Runtime):
start_request = {
'image': self.container_image,
'command': (
f'/openhands/miniforge3/bin/mamba run --no-capture-output -n base '
'PYTHONUNBUFFERED=1 poetry run '
f'/openhands/micromamba/bin/micromamba run -n openhands '
'poetry run '
f'python -u -m openhands.runtime.client.client {self.port} '
f'--working-dir {self.config.workspace_mount_path_in_sandbox} '
f'{plugin_arg}'

48
openhands/runtime/utils/runtime_templates/Dockerfile.j2
View File

@@ -1,11 +1,13 @@
{% if skip_init %}
FROM {{ base_image }}
{% else %}
# Shared environment variables (regardless of init or not)
ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry
ENV MAMBA_ROOT_PREFIX=/openhands/micromamba
{% if not skip_init %}
# ================================================================
# START: Build Runtime Image from Scratch
# ================================================================
FROM {{ base_image }}
{% if 'ubuntu' in base_image and (base_image.endswith(':latest') or base_image.endswith(':24.04')) %}
{% set LIBGL_MESA = 'libgl1' %}
{% else %}
@@ -14,7 +16,7 @@ FROM {{ base_image }}
# Install necessary packages and clean up in one layer
RUN apt-get update && \
apt-get install -y wget sudo apt-utils {{ LIBGL_MESA }} libasound2-plugins git && \
apt-get install -y wget curl sudo apt-utils {{ LIBGL_MESA }} libasound2-plugins git && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
@@ -26,19 +28,16 @@ RUN mkdir -p /openhands && \
mkdir -p /openhands/logs && \
mkdir -p /openhands/poetry
# Directory containing subdirectories for virtual environment.
ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry
# Install micromamba
RUN mkdir -p /openhands/micromamba/bin && \
/bin/bash -c "PREFIX_LOCATION=/openhands/micromamba BIN_FOLDER=/openhands/micromamba/bin INIT_YES=no CONDA_FORGE_YES=yes $(curl -L https://micro.mamba.pm/install.sh)" && \
/openhands/micromamba/bin/micromamba config remove channels defaults && \
/openhands/micromamba/bin/micromamba config list
RUN if [ ! -d /openhands/miniforge3 ]; then \
wget --progress=bar:force -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \
bash Miniforge3.sh -b -p /openhands/miniforge3 && \
rm Miniforge3.sh && \
chmod -R g+w /openhands/miniforge3 && \
bash -c ". /openhands/miniforge3/etc/profile.d/conda.sh && conda config --set changeps1 False && conda config --append channels conda-forge"; \
fi
# Create the openhands virtual environment and install poetry and python
RUN /openhands/micromamba/bin/micromamba create -n openhands -y && \
/openhands/micromamba/bin/micromamba install -n openhands -c conda-forge poetry python=3.11 -y
# Install Python and Poetry
RUN /openhands/miniforge3/bin/mamba install conda-forge::poetry python=3.11 -y
# ================================================================
# END: Build Runtime Image from Scratch
# ================================================================
@@ -59,27 +58,28 @@ COPY ./code /openhands/code
# virtual environment are used by default.
WORKDIR /openhands/code
RUN \
/openhands/micromamba/bin/micromamba config set changeps1 False && \
# Configure Poetry and create virtual environment
/openhands/miniforge3/bin/mamba run -n base poetry config virtualenvs.path /openhands/poetry && \
/openhands/miniforge3/bin/mamba run -n base poetry env use python3.11 && \
/openhands/micromamba/bin/micromamba run -n openhands poetry config virtualenvs.path /openhands/poetry && \
/openhands/micromamba/bin/micromamba run -n openhands poetry env use python3.11 && \
# Install project dependencies
/openhands/miniforge3/bin/mamba run -n base poetry install --only main,runtime --no-interaction --no-root && \
/openhands/micromamba/bin/micromamba run -n openhands poetry install --only main,runtime --no-interaction --no-root && \
# Update and install additional tools
apt-get update && \
/openhands/miniforge3/bin/mamba run -n base poetry run pip install playwright && \
/openhands/miniforge3/bin/mamba run -n base poetry run playwright install --with-deps chromium && \
/openhands/micromamba/bin/micromamba run -n openhands poetry run pip install playwright && \
/openhands/micromamba/bin/micromamba run -n openhands poetry run playwright install --with-deps chromium && \
# Set environment variables
echo "OH_INTERPRETER_PATH=$(/openhands/miniforge3/bin/mamba run -n base poetry run python -c "import sys; print(sys.executable)")" >> /etc/environment && \
echo "OH_INTERPRETER_PATH=$(/openhands/micromamba/bin/micromamba run -n openhands poetry run python -c "import sys; print(sys.executable)")" >> /etc/environment && \
# Install extra dependencies if specified
{{ extra_deps }} {% if extra_deps %} && {% endif %} \
# Clear caches
/openhands/miniforge3/bin/mamba run -n base poetry cache clear --all . && \
/openhands/micromamba/bin/micromamba run -n openhands poetry cache clear --all . && \
# Set permissions
{% if not skip_init %}chmod -R g+rws /openhands/poetry && {% endif %} \
mkdir -p /openhands/workspace && chmod -R g+rws,o+rw /openhands/workspace && \
# Clean up
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
/openhands/miniforge3/bin/mamba clean --all
/openhands/micromamba/bin/micromamba clean --all
# ================================================================
# END: Copy Project and Install/Update Dependencies
# ================================================================

3
tests/runtime/conftest.py
View File

@@ -208,6 +208,7 @@ def _load_runtime(
base_container_image: str | None = None,
browsergym_eval_env: str | None = None,
use_workspace: bool | None = None,
force_rebuild_runtime: bool = False,
) -> Runtime:
sid = 'rt_' + str(random.randint(100000, 999999))
@@ -217,7 +218,7 @@ def _load_runtime(
config = load_app_config()
config.run_as_openhands = run_as_openhands
config.sandbox.force_rebuild_runtime = force_rebuild_runtime
# Folder where all tests create their own folder
global test_mount_path
if use_workspace:

3
tests/runtime/test_browsing.py
View File

@@ -19,7 +19,7 @@ from openhands.events.observation import (
# Browsing tests
# ============================================================================================================================
PY3_FOR_TESTING = '/openhands/miniforge3/bin/mamba run -n base python3'
PY3_FOR_TESTING = '/openhands/micromamba/bin/micromamba run -n openhands python3'
def test_simple_browse(temp_dir, box_class, run_as_openhands):
@@ -75,6 +75,7 @@ def test_browsergym_eval_env(box_class, temp_dir):
run_as_openhands=False, # need root permission to access file
base_container_image='xingyaoww/od-eval-miniwob:v1.0',
browsergym_eval_env='browsergym/miniwob.choose-list',
force_rebuild_runtime=True,
)
from openhands.runtime.browser.browser_env import (
BROWSER_EVAL_GET_GOAL_ACTION,

24
tests/unit/test_runtime_build.py
View File

@@ -155,16 +155,14 @@ def test_generate_dockerfile_scratch():
)
assert base_image in dockerfile_content
assert 'apt-get update' in dockerfile_content
assert 'apt-get install -y wget sudo apt-utils' in dockerfile_content
assert (
'RUN /openhands/miniforge3/bin/mamba install conda-forge::poetry python=3.11 -y'
in dockerfile_content
)
assert 'apt-get install -y wget curl sudo apt-utils' in dockerfile_content
assert 'poetry' in dockerfile_content and '-c conda-forge' in dockerfile_content
assert 'python=3.11' in dockerfile_content
# Check the update command
assert 'COPY ./code /openhands/code' in dockerfile_content
assert (
'/openhands/miniforge3/bin/mamba run -n base poetry install'
'/openhands/micromamba/bin/micromamba run -n openhands poetry install'
in dockerfile_content
)
@@ -178,17 +176,13 @@ def test_generate_dockerfile_skip_init():
# These commands SHOULD NOT include in the dockerfile if skip_init is True
assert 'RUN apt update && apt install -y wget sudo' not in dockerfile_content
assert (
'RUN /openhands/miniforge3/bin/mamba install conda-forge::poetry python=3.11 -y'
not in dockerfile_content
)
assert '-c conda-forge' not in dockerfile_content
assert 'python=3.11' not in dockerfile_content
assert 'https://micro.mamba.pm/install.sh' not in dockerfile_content
# These update commands SHOULD still in the dockerfile
assert 'COPY ./code /openhands/code' in dockerfile_content
assert (
'/openhands/miniforge3/bin/mamba run -n base poetry install'
in dockerfile_content
)
assert 'poetry install' in dockerfile_content
def test_get_runtime_image_repo_and_tag_eventstream():
@@ -353,7 +347,7 @@ def live_docker_image():
dockerfile_content = f"""
# syntax=docker/dockerfile:1.4
FROM {DEFAULT_BASE_IMAGE} AS base
RUN apt-get update && apt-get install -y wget sudo apt-utils
RUN apt-get update && apt-get install -y wget curl sudo apt-utils
FROM base AS intermediate
RUN mkdir -p /openhands