feat(runtime): use micromamba instead of mamba and fix build issue (#4154)

.github/workflows/dummy-agent-test.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Run tests
         run: |
           set -e
-          poetry run python3 openhands/core/main.py -t "do a flip" -d ./workspace/ -c DummyAgent
+          SANDBOX_FORCE_REBUILD_RUNTIME=True poetry run python3 openhands/core/main.py -t "do a flip" -d ./workspace/ -c DummyAgent
       - name: Check exit code
         run: |
           if [ $? -ne 0 ]; then

.github/workflows/ghcr-build.yml

@@ -293,7 +293,7 @@ jobs:
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=false \
-          poetry run pytest -n 3 -raR --reruns 1 --reruns-delay 3 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
+          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:
@@ -371,7 +371,7 @@ jobs:
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=true \
-          poetry run pytest -n 3 -raR --reruns 1 --reruns-delay 3 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
+          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:

openhands/core/config/sandbox_config.py

@@ -18,6 +18,7 @@ class SandboxConfig:
         enable_auto_lint: Whether to enable auto-lint.
         use_host_network: Whether to use the host network.
         initialize_plugins: Whether to initialize plugins.
+        force_rebuild_runtime: Whether to force rebuild the runtime image.
         runtime_extra_deps: The extra dependencies to install in the runtime image (typically used for evaluation).
             This will be rendered into the end of the Dockerfile that builds the runtime image.
             It can contain any valid shell commands (e.g., pip install numpy).
@@ -43,6 +44,7 @@ class SandboxConfig:
     )
     use_host_network: bool = False
     initialize_plugins: bool = True
+    force_rebuild_runtime: bool = False
     runtime_extra_deps: str | None = None
     runtime_startup_env_vars: dict[str, str] = field(default_factory=dict)
     browsergym_eval_env: str | None = None

openhands/runtime/builder/docker.py

@@ -113,8 +113,8 @@ class DockerRuntimeBuilder(RuntimeBuilder):
                 raise subprocess.CalledProcessError(
                     return_code,
                     process.args,
-                    output=None,
+                    output=process.stdout.read() if process.stdout else None,
-                    stderr=None,
+                    stderr=process.stderr.read() if process.stderr else None,
                 )
 
         except subprocess.CalledProcessError as e:

openhands/runtime/client/runtime.py

@@ -167,6 +167,7 @@ class EventStreamRuntime(Runtime):
                 self.base_container_image,
                 self.runtime_builder,
                 extra_deps=self.config.sandbox.runtime_extra_deps,
+                force_rebuild=self.config.sandbox.force_rebuild_runtime,
             )
         self.container = self._init_container(
             sandbox_workspace_dir=self.config.workspace_mount_path_in_sandbox,  # e.g. /workspace
@@ -273,7 +274,7 @@ class EventStreamRuntime(Runtime):
             container = self.docker_client.containers.run(
                 self.runtime_container_image,
                 command=(
-                    f'/openhands/miniforge3/bin/mamba run --no-capture-output -n base '
+                    f'/openhands/micromamba/bin/micromamba run -n openhands '
                     f'poetry run '
                     f'python -u -m openhands.runtime.client.client {self._container_port} '
                     f'--working-dir "{sandbox_workspace_dir}" '

openhands/runtime/plugins/jupyter/__init__.py

@@ -28,7 +28,8 @@ class JupyterPlugin(Plugin):
                 'cd /openhands/code\n'
                 'export POETRY_VIRTUALENVS_PATH=/openhands/poetry;\n'
                 'export PYTHONPATH=/openhands/code:$PYTHONPATH;\n'
-                '/openhands/miniforge3/bin/mamba run -n base '
+                'export MAMBA_ROOT_PREFIX=/openhands/micromamba;\n'
+                '/openhands/micromamba/bin/micromamba run -n openhands '
                 'poetry run jupyter kernelgateway '
                 '--KernelGatewayApp.ip=0.0.0.0 '
                 f'--KernelGatewayApp.port={self.kernel_gateway_port}\n'

openhands/runtime/remote/runtime.py

@@ -119,6 +119,7 @@ class RemoteRuntime(Runtime):
                 self.config.sandbox.base_container_image,
                 self.runtime_builder,
                 extra_deps=self.config.sandbox.runtime_extra_deps,
+                force_rebuild=self.config.sandbox.force_rebuild_runtime,
             )
 
             response = send_request(
@@ -144,8 +145,8 @@ class RemoteRuntime(Runtime):
         start_request = {
             'image': self.container_image,
             'command': (
-                f'/openhands/miniforge3/bin/mamba run --no-capture-output -n base '
+                f'/openhands/micromamba/bin/micromamba run -n openhands '
-                'PYTHONUNBUFFERED=1 poetry run '
+                'poetry run '
                 f'python -u -m openhands.runtime.client.client {self.port} '
                 f'--working-dir {self.config.workspace_mount_path_in_sandbox} '
                 f'{plugin_arg}'

openhands/runtime/utils/runtime_templates/Dockerfile.j2

@@ -1,11 +1,13 @@
-{% if skip_init %}
 FROM {{ base_image }}
-{% else %}
+
+# Shared environment variables (regardless of init or not)
+ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry
+ENV MAMBA_ROOT_PREFIX=/openhands/micromamba
+
+{% if not skip_init %}
 # ================================================================
 # START: Build Runtime Image from Scratch
 # ================================================================
-FROM {{ base_image }}
-
 {% if 'ubuntu' in base_image and (base_image.endswith(':latest') or base_image.endswith(':24.04')) %}
 {% set LIBGL_MESA = 'libgl1' %}
 {% else %}
@@ -14,7 +16,7 @@ FROM {{ base_image }}
 
 # Install necessary packages and clean up in one layer
 RUN apt-get update && \
-    apt-get install -y wget sudo apt-utils {{ LIBGL_MESA }} libasound2-plugins git && \
+    apt-get install -y wget curl sudo apt-utils {{ LIBGL_MESA }} libasound2-plugins git && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -26,19 +28,16 @@ RUN mkdir -p /openhands && \
     mkdir -p /openhands/logs && \
     mkdir -p /openhands/poetry
 
-# Directory containing subdirectories for virtual environment.
+# Install micromamba
-ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry
+RUN mkdir -p /openhands/micromamba/bin && \
+    /bin/bash -c "PREFIX_LOCATION=/openhands/micromamba BIN_FOLDER=/openhands/micromamba/bin INIT_YES=no CONDA_FORGE_YES=yes $(curl -L https://micro.mamba.pm/install.sh)" && \
+    /openhands/micromamba/bin/micromamba config remove channels defaults && \
+    /openhands/micromamba/bin/micromamba config list
 
-RUN if [ ! -d /openhands/miniforge3 ]; then \
+# Create the openhands virtual environment and install poetry and python
-    wget --progress=bar:force -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \
+RUN /openhands/micromamba/bin/micromamba create -n openhands -y && \
-    bash Miniforge3.sh -b -p /openhands/miniforge3 && \
+    /openhands/micromamba/bin/micromamba install -n openhands -c conda-forge poetry python=3.11 -y
-    rm Miniforge3.sh && \
-    chmod -R g+w /openhands/miniforge3 && \
-    bash -c ". /openhands/miniforge3/etc/profile.d/conda.sh && conda config --set changeps1 False && conda config --append channels conda-forge"; \
-    fi
 
-# Install Python and Poetry
-RUN /openhands/miniforge3/bin/mamba install conda-forge::poetry python=3.11 -y
 # ================================================================
 # END: Build Runtime Image from Scratch
 # ================================================================
@@ -59,27 +58,28 @@ COPY ./code /openhands/code
 # virtual environment are used by default.
 WORKDIR /openhands/code
 RUN \
+    /openhands/micromamba/bin/micromamba config set changeps1 False && \
     # Configure Poetry and create virtual environment
-    /openhands/miniforge3/bin/mamba run -n base poetry config virtualenvs.path /openhands/poetry && \
+    /openhands/micromamba/bin/micromamba run -n openhands poetry config virtualenvs.path /openhands/poetry && \
-    /openhands/miniforge3/bin/mamba run -n base poetry env use python3.11 && \
+    /openhands/micromamba/bin/micromamba run -n openhands poetry env use python3.11 && \
     # Install project dependencies
-    /openhands/miniforge3/bin/mamba run -n base poetry install --only main,runtime --no-interaction --no-root && \
+    /openhands/micromamba/bin/micromamba run -n openhands poetry install --only main,runtime --no-interaction --no-root && \
     # Update and install additional tools
     apt-get update && \
-    /openhands/miniforge3/bin/mamba run -n base poetry run pip install playwright && \
+    /openhands/micromamba/bin/micromamba run -n openhands poetry run pip install playwright && \
-    /openhands/miniforge3/bin/mamba run -n base poetry run playwright install --with-deps chromium && \
+    /openhands/micromamba/bin/micromamba run -n openhands poetry run playwright install --with-deps chromium && \
     # Set environment variables
-    echo "OH_INTERPRETER_PATH=$(/openhands/miniforge3/bin/mamba run -n base poetry run python -c "import sys; print(sys.executable)")" >> /etc/environment && \
+    echo "OH_INTERPRETER_PATH=$(/openhands/micromamba/bin/micromamba run -n openhands poetry run python -c "import sys; print(sys.executable)")" >> /etc/environment && \
     # Install extra dependencies if specified
     {{ extra_deps }} {% if extra_deps %} && {% endif %} \
     # Clear caches
-    /openhands/miniforge3/bin/mamba run -n base poetry cache clear --all . && \
+    /openhands/micromamba/bin/micromamba run -n openhands poetry cache clear --all . && \
     # Set permissions
     {% if not skip_init %}chmod -R g+rws /openhands/poetry && {% endif %} \
     mkdir -p /openhands/workspace && chmod -R g+rws,o+rw /openhands/workspace && \
     # Clean up
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    /openhands/miniforge3/bin/mamba clean --all
+    /openhands/micromamba/bin/micromamba clean --all
 # ================================================================
 # END: Copy Project and Install/Update Dependencies
 # ================================================================

tests/runtime/conftest.py

@@ -208,6 +208,7 @@ def _load_runtime(
     base_container_image: str | None = None,
     browsergym_eval_env: str | None = None,
     use_workspace: bool | None = None,
+    force_rebuild_runtime: bool = False,
 ) -> Runtime:
     sid = 'rt_' + str(random.randint(100000, 999999))
 
@@ -217,7 +218,7 @@ def _load_runtime(
 
     config = load_app_config()
     config.run_as_openhands = run_as_openhands
-
+    config.sandbox.force_rebuild_runtime = force_rebuild_runtime
     # Folder where all tests create their own folder
     global test_mount_path
     if use_workspace:

tests/runtime/test_browsing.py

@@ -19,7 +19,7 @@ from openhands.events.observation import (
 # Browsing tests
 # ============================================================================================================================
 
-PY3_FOR_TESTING = '/openhands/miniforge3/bin/mamba run -n base python3'
+PY3_FOR_TESTING = '/openhands/micromamba/bin/micromamba run -n openhands python3'
 
 
 def test_simple_browse(temp_dir, box_class, run_as_openhands):
@@ -75,6 +75,7 @@ def test_browsergym_eval_env(box_class, temp_dir):
         run_as_openhands=False,  # need root permission to access file
         base_container_image='xingyaoww/od-eval-miniwob:v1.0',
         browsergym_eval_env='browsergym/miniwob.choose-list',
+        force_rebuild_runtime=True,
     )
     from openhands.runtime.browser.browser_env import (
         BROWSER_EVAL_GET_GOAL_ACTION,

tests/unit/test_runtime_build.py

@@ -155,16 +155,14 @@ def test_generate_dockerfile_scratch():
     )
     assert base_image in dockerfile_content
     assert 'apt-get update' in dockerfile_content
-    assert 'apt-get install -y wget sudo apt-utils' in dockerfile_content
+    assert 'apt-get install -y wget curl sudo apt-utils' in dockerfile_content
-    assert (
+    assert 'poetry' in dockerfile_content and '-c conda-forge' in dockerfile_content
-        'RUN /openhands/miniforge3/bin/mamba install conda-forge::poetry python=3.11 -y'
+    assert 'python=3.11' in dockerfile_content
-        in dockerfile_content
-    )
 
     # Check the update command
     assert 'COPY ./code /openhands/code' in dockerfile_content
     assert (
-        '/openhands/miniforge3/bin/mamba run -n base poetry install'
+        '/openhands/micromamba/bin/micromamba run -n openhands poetry install'
         in dockerfile_content
     )
 
@@ -178,17 +176,13 @@ def test_generate_dockerfile_skip_init():
 
     # These commands SHOULD NOT include in the dockerfile if skip_init is True
     assert 'RUN apt update && apt install -y wget sudo' not in dockerfile_content
-    assert (
+    assert '-c conda-forge' not in dockerfile_content
-        'RUN /openhands/miniforge3/bin/mamba install conda-forge::poetry python=3.11 -y'
+    assert 'python=3.11' not in dockerfile_content
-        not in dockerfile_content
+    assert 'https://micro.mamba.pm/install.sh' not in dockerfile_content
-    )
 
     # These update commands SHOULD still in the dockerfile
     assert 'COPY ./code /openhands/code' in dockerfile_content
-    assert (
+    assert 'poetry install' in dockerfile_content
-        '/openhands/miniforge3/bin/mamba run -n base poetry install'
-        in dockerfile_content
-    )
 
 
 def test_get_runtime_image_repo_and_tag_eventstream():
@@ -353,7 +347,7 @@ def live_docker_image():
     dockerfile_content = f"""
     # syntax=docker/dockerfile:1.4
     FROM {DEFAULT_BASE_IMAGE} AS base
-    RUN apt-get update && apt-get install -y wget sudo apt-utils
+    RUN apt-get update && apt-get install -y wget curl sudo apt-utils
 
     FROM base AS intermediate
     RUN mkdir -p /openhands