From affb2123d99d7d15ba64b028ecff5cf118d766af Mon Sep 17 00:00:00 2001 From: Xingyao Wang Date: Mon, 28 Oct 2024 15:52:54 -0500 Subject: [PATCH] feat(runtime): add versioned runtime image (base_name+oh_version) (#4574) --- containers/build.sh | 6 +- containers/runtime/config.sh | 2 +- docs/modules/usage/architecture/runtime.md | 38 ++-- openhands/runtime/builder/docker.py | 6 +- openhands/runtime/impl/modal/modal_runtime.py | 3 +- openhands/runtime/utils/runtime_build.py | 91 ++++++---- .../utils/runtime_templates/Dockerfile.j2 | 57 +++--- tests/unit/test_runtime_build.py | 162 +++++++++++++++--- 8 files changed, 269 insertions(+), 96 deletions(-) diff --git a/containers/build.sh b/containers/build.sh index a05fd266a2..9b6fcfb2e4 100755 --- a/containers/build.sh +++ b/containers/build.sh @@ -98,9 +98,9 @@ if [[ -n "$org_name" ]]; then DOCKER_ORG="$org_name" fi -# If $DOCKER_IMAGE_HASH_TAG is set, add it to the tags -if [[ -n "$DOCKER_IMAGE_HASH_TAG" ]]; then - tags+=("$DOCKER_IMAGE_HASH_TAG") +# If $DOCKER_IMAGE_SOURCE_TAG is set, add it to the tags +if [[ -n "$DOCKER_IMAGE_SOURCE_TAG" ]]; then + tags+=("$DOCKER_IMAGE_SOURCE_TAG") fi # If $DOCKER_IMAGE_TAG is set, add it to the tags if [[ -n "$DOCKER_IMAGE_TAG" ]]; then diff --git a/containers/runtime/config.sh b/containers/runtime/config.sh index 755644db92..99d2eb66cc 100644 --- a/containers/runtime/config.sh +++ b/containers/runtime/config.sh @@ -4,4 +4,4 @@ DOCKER_BASE_DIR="./containers/runtime" DOCKER_IMAGE=runtime # These variables will be appended by the runtime_build.py script # DOCKER_IMAGE_TAG= -# DOCKER_IMAGE_HASH_TAG= +# DOCKER_IMAGE_SOURCE_TAG= diff --git a/docs/modules/usage/architecture/runtime.md b/docs/modules/usage/architecture/runtime.md index 8747f7c590..3aa05056ff 100644 --- a/docs/modules/usage/architecture/runtime.md +++ b/docs/modules/usage/architecture/runtime.md @@ -70,14 +70,22 @@ Check out the [relevant code](https://github.com/All-Hands-AI/OpenHands/blob/mai ### Image Tagging System -OpenHands uses a dual-tagging system for its runtime images to balance reproducibility with flexibility. +OpenHands uses a three-tag system for its runtime images to balance reproducibility with flexibility. Tags may be in one of 2 formats: -- **Generic**: `oh_v{openhands_version}_{16_digit_lock_hash}` (e.g.: `oh_v0.9.9_1234567890abcdef`) -- **Specific**: `oh_v{openhands_version}_{16_digit_lock_hash}_{16_digit_source_hash}` +- **Versioned Tag**: `oh_v{openhands_version}_{base_image}` (e.g.: `oh_v0.9.9_nikolaik_s_python-nodejs_t_python3.12-nodejs22`) +- **Lock Tag**: `oh_v{openhands_version}_{16_digit_lock_hash}` (e.g.: `oh_v0.9.9_1234567890abcdef`) +- **Source Tag**: `oh_v{openhands_version}_{16_digit_lock_hash}_{16_digit_source_hash}` (e.g.: `oh_v0.9.9_1234567890abcdef_1234567890abcdef`) -#### Lock Hash + +#### Source Tag - Most Specific + +This is the first 16 digits of the MD5 of the directory hash for the source directory. This gives a hash +for only the openhands source + + +#### Lock Tag This hash is built from the first 16 digits of the MD5 of: - The name of the base image upon which the image was built (e.g.: `nikolaik/python-nodejs:python3.12-nodejs22`) @@ -86,30 +94,30 @@ This hash is built from the first 16 digits of the MD5 of: This effectively gives a hash for the dependencies of Openhands independent of the source code. -#### Source Hash +#### Versioned Tag - Most Generic -This is the first 16 digits of the MD5 of the directory hash for the source directory. This gives a hash -for only the openhands source +This tag is a concatenation of openhands version and the base image name (transformed to fit in tag standard). #### Build Process When generating an image... -- OpenHands first checks whether an image with the same **Specific** tag exists. If there is such an image, +- **No re-build**: OpenHands first checks whether an image with the same **most specific source tag** exists. If there is such an image, no build is performed - the existing image is used. -- OpenHands next checks whether an image with the **Generic** tag exists. If there is such an image, +- **Fastest re-build**: OpenHands next checks whether an image with the **generic lock tag** exists. If there is such an image, OpenHands builds a new image based upon it, bypassing all installation steps (like `poetry install` and `apt-get`) except a final operation to copy the current source code. The new image is tagged with a - **Specific** tag only. -- If neither a **Specific** nor **Generic** tag exists, a brand new image is built based upon the base - image (Which is a slower operation). This new image is tagged with both the **Generic** and **Specific** - tags. + **source** tag only. +- **Ok-ish re-build**: If neither a **source** nor **lock** tag exists, an image will be built based upon the **versioned** tag image. + In versioned tag image, most dependencies should already been installed hence saving time. +- **Slowest re-build**: If all of the three tags don't exists, a brand new image is built based upon the base + image (Which is a slower operation). This new image is tagged with all the **source**, **lock**, and **versioned** tags. -This dual-tagging approach allows OpenHands to efficiently manage both development and production environments. +This tagging approach allows OpenHands to efficiently manage both development and production environments. 1. Identical source code and Dockerfile always produce the same image (via hash-based tags) 2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images) -3. The generic tag (e.g., `runtime:oh_v0.9.3_1234567890abcdef`) always points to the latest build for a particular base image and OpenHands version combination +3. The **lock** tag (e.g., `runtime:oh_v0.9.3_1234567890abcdef`) always points to the latest build for a particular base image, dependency, and OpenHands version combination ## Runtime Plugin System diff --git a/openhands/runtime/builder/docker.py b/openhands/runtime/builder/docker.py index 5a22302d61..d87cedbd1e 100644 --- a/openhands/runtime/builder/docker.py +++ b/openhands/runtime/builder/docker.py @@ -58,7 +58,7 @@ class DockerRuntimeBuilder(RuntimeBuilder): raise RuntimeError('Docker server version must be >= 18.09 to use BuildKit') target_image_hash_name = tags[0] - target_image_repo, target_image_hash_tag = target_image_hash_name.split(':') + target_image_repo, target_image_source_tag = target_image_hash_name.split(':') target_image_tag = tags[1].split(':')[1] if len(tags) > 1 else None buildx_cmd = [ @@ -160,9 +160,9 @@ class DockerRuntimeBuilder(RuntimeBuilder): ) tags_str = ( - f'{target_image_hash_tag}, {target_image_tag}' + f'{target_image_source_tag}, {target_image_tag}' if target_image_tag - else target_image_hash_tag + else target_image_source_tag ) logger.info( f'Image {target_image_repo} with tags [{tags_str}] built successfully' diff --git a/openhands/runtime/impl/modal/modal_runtime.py b/openhands/runtime/impl/modal/modal_runtime.py index be4603a802..3154567859 100644 --- a/openhands/runtime/impl/modal/modal_runtime.py +++ b/openhands/runtime/impl/modal/modal_runtime.py @@ -18,6 +18,7 @@ from openhands.runtime.impl.eventstream.eventstream_runtime import ( from openhands.runtime.plugins import PluginRequirement from openhands.runtime.utils.command import get_remote_startup_command from openhands.runtime.utils.runtime_build import ( + BuildFromImageType, prep_build_folder, ) from openhands.utils.async_utils import call_sync_from_async @@ -183,7 +184,7 @@ class ModalRuntime(EventStreamRuntime): prep_build_folder( build_folder=Path(build_folder), base_image=base_container_image_id, - build_from_scratch=True, + build_from=BuildFromImageType.SCRATCH, extra_deps=runtime_extra_deps, ) diff --git a/openhands/runtime/utils/runtime_build.py b/openhands/runtime/utils/runtime_build.py index 4fde087107..9ee09f852b 100644 --- a/openhands/runtime/utils/runtime_build.py +++ b/openhands/runtime/utils/runtime_build.py @@ -4,6 +4,7 @@ import os import shutil import string import tempfile +from enum import Enum from pathlib import Path from typing import List @@ -17,20 +18,26 @@ from openhands.core.logger import openhands_logger as logger from openhands.runtime.builder import DockerRuntimeBuilder, RuntimeBuilder +class BuildFromImageType(Enum): + SCRATCH = 'scratch' # Slowest: Build from base image (no dependencies are reused) + VERSIONED = 'versioned' # Medium speed: Reuse the most recent image with the same base image & OH version (a lot of dependencies are already installed) + LOCK = 'lock' # Fastest: Reuse the most recent image with the exact SAME dependencies (lock files) + + def get_runtime_image_repo(): return os.getenv('OH_RUNTIME_RUNTIME_IMAGE_REPO', 'ghcr.io/all-hands-ai/runtime') def _generate_dockerfile( base_image: str, - build_from_scratch: bool = True, + build_from: BuildFromImageType = BuildFromImageType.SCRATCH, extra_deps: str | None = None, ) -> str: """Generate the Dockerfile content for the runtime image based on the base image. Parameters: - base_image (str): The base image provided for the runtime image - - build_from_scratch (boolean): False implies most steps can be skipped (Base image is another openhands instance) + - build_from (BuildFromImageType): The build method for the runtime image. - extra_deps (str): Returns: @@ -45,7 +52,8 @@ def _generate_dockerfile( dockerfile_content = template.render( base_image=base_image, - build_from_scratch=build_from_scratch, + build_from_scratch=build_from == BuildFromImageType.SCRATCH, + build_from_versioned=build_from == BuildFromImageType.VERSIONED, extra_deps=extra_deps if extra_deps is not None else '', ) return dockerfile_content @@ -157,25 +165,36 @@ def build_runtime_image_in_folder( ) -> str: runtime_image_repo, _ = get_runtime_image_repo_and_tag(base_image) lock_tag = f'oh_v{oh_version}_{get_hash_for_lock_files(base_image)}' - hash_tag = f'{lock_tag}_{get_hash_for_source_files()}' - hash_image_name = f'{runtime_image_repo}:{hash_tag}' + versioned_tag = ( + # truncate the base image to 96 characters to fit in the tag max length (128 characters) + f'oh_v{oh_version}_{get_tag_for_versioned_image(base_image)}' + ) + versioned_image_name = f'{runtime_image_repo}:{versioned_tag}' + source_tag = f'{lock_tag}_{get_hash_for_source_files()}' + hash_image_name = f'{runtime_image_repo}:{source_tag}' if force_rebuild: - logger.info(f'Force rebuild: [{runtime_image_repo}:{hash_tag}] from scratch.') - prep_build_folder(build_folder, base_image, True, extra_deps) + logger.info(f'Force rebuild: [{runtime_image_repo}:{source_tag}] from scratch.') + prep_build_folder( + build_folder, + base_image, + build_from=BuildFromImageType.SCRATCH, + extra_deps=extra_deps, + ) if not dry_run: _build_sandbox_image( build_folder, runtime_builder, runtime_image_repo, - hash_tag, + source_tag, lock_tag, + versioned_tag, platform, ) return hash_image_name lock_image_name = f'{runtime_image_repo}:{lock_tag}' - build_from_scratch = True + build_from = BuildFromImageType.SCRATCH # If the exact image already exists, we do not need to build it if runtime_builder.image_exists(hash_image_name, False): @@ -186,21 +205,32 @@ def build_runtime_image_in_folder( # can use it as the base image for the build and just copy source files. This makes the build # much faster. if runtime_builder.image_exists(lock_image_name): - logger.info(f'Build [{hash_image_name}] from [{lock_image_name}]') - build_from_scratch = False + logger.info(f'Build [{hash_image_name}] from lock image [{lock_image_name}]') + build_from = BuildFromImageType.LOCK base_image = lock_image_name + elif runtime_builder.image_exists(versioned_image_name): + logger.info( + f'Build [{hash_image_name}] from versioned image [{versioned_image_name}]' + ) + build_from = BuildFromImageType.VERSIONED + base_image = versioned_image_name else: logger.info(f'Build [{hash_image_name}] from scratch') - prep_build_folder(build_folder, base_image, build_from_scratch, extra_deps) + prep_build_folder(build_folder, base_image, build_from, extra_deps) if not dry_run: _build_sandbox_image( build_folder, runtime_builder, runtime_image_repo, - hash_tag, - lock_tag, - platform, + source_tag=source_tag, + lock_tag=lock_tag, + # Only tag the versioned image if we are building from scratch. + # This avoids too much layers when you lay one image on top of another multiple times + versioned_tag=versioned_tag + if build_from == BuildFromImageType.SCRATCH + else None, + platform=platform, ) return hash_image_name @@ -209,7 +239,7 @@ def build_runtime_image_in_folder( def prep_build_folder( build_folder: Path, base_image: str, - build_from_scratch: bool, + build_from: BuildFromImageType, extra_deps: str | None, ): # Copy the source code to directory. It will end up in build_folder/code @@ -240,7 +270,7 @@ def prep_build_folder( # Create a Dockerfile and write it to build_folder dockerfile_content = _generate_dockerfile( base_image, - build_from_scratch=build_from_scratch, + build_from=build_from, extra_deps=extra_deps, ) with open(Path(build_folder, 'Dockerfile'), 'w') as file: # type: ignore @@ -277,6 +307,10 @@ def get_hash_for_lock_files(base_image: str): return result +def get_tag_for_versioned_image(base_image: str): + return base_image.replace('/', '_s_').replace(':', '_t_').lower()[-96:] + + def get_hash_for_source_files(): openhands_source_dir = Path(openhands.__file__).parent dir_hash = dirhash( @@ -298,20 +332,19 @@ def _build_sandbox_image( build_folder: Path, runtime_builder: RuntimeBuilder, runtime_image_repo: str, - hash_tag: str, + source_tag: str, lock_tag: str, + versioned_tag: str | None, platform: str | None = None, ): """Build and tag the sandbox image. The image will be tagged with all tags that do not yet exist""" - names = [ - name - for name in [ - f'{runtime_image_repo}:{hash_tag}', - f'{runtime_image_repo}:{lock_tag}', - ] - if not runtime_builder.image_exists(name, False) + f'{runtime_image_repo}:{source_tag}', + f'{runtime_image_repo}:{lock_tag}', ] + if versioned_tag is not None: + names.append(f'{runtime_image_repo}:{versioned_tag}') + names = [name for name in names if not runtime_builder.image_exists(name, False)] image_name = runtime_builder.build( path=str(build_folder), tags=names, platform=platform @@ -363,8 +396,8 @@ if __name__ == '__main__': platform=args.platform, ) - _runtime_image_repo, runtime_image_hash_tag = runtime_image_hash_name.split( - ':' + _runtime_image_repo, runtime_image_source_tag = ( + runtime_image_hash_name.split(':') ) # Move contents of temp_dir to build_folder @@ -380,11 +413,11 @@ if __name__ == '__main__': ( f'\n' f'DOCKER_IMAGE_TAG={runtime_image_tag}\n' - f'DOCKER_IMAGE_HASH_TAG={runtime_image_hash_tag}\n' + f'DOCKER_IMAGE_SOURCE_TAG={runtime_image_source_tag}\n' ) ) logger.info( - f'`config.sh` is updated with the image repo[{runtime_image_repo}] and tags [{runtime_image_tag}, {runtime_image_hash_tag}]' + f'`config.sh` is updated with the image repo[{runtime_image_repo}] and tags [{runtime_image_tag}, {runtime_image_source_tag}]' ) logger.info( f'Dockerfile, source code and config.sh are ready in {build_folder}' diff --git a/openhands/runtime/utils/runtime_templates/Dockerfile.j2 b/openhands/runtime/utils/runtime_templates/Dockerfile.j2 index 7cac378a01..3de7f42e5f 100644 --- a/openhands/runtime/utils/runtime_templates/Dockerfile.j2 +++ b/openhands/runtime/utils/runtime_templates/Dockerfile.j2 @@ -4,6 +4,32 @@ FROM {{ base_image }} ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry ENV MAMBA_ROOT_PREFIX=/openhands/micromamba +{% macro install_dependencies() %} +# Install all dependencies +WORKDIR /openhands/code +RUN \ + /openhands/micromamba/bin/micromamba config set changeps1 False && \ + # Configure Poetry and create virtual environment + /openhands/micromamba/bin/micromamba run -n openhands poetry config virtualenvs.path /openhands/poetry && \ + /openhands/micromamba/bin/micromamba run -n openhands poetry env use python3.12 && \ + # Install project dependencies + /openhands/micromamba/bin/micromamba run -n openhands poetry install --only main,runtime --no-interaction --no-root && \ + # Update and install additional tools + apt-get update && \ + /openhands/micromamba/bin/micromamba run -n openhands poetry run pip install playwright && \ + /openhands/micromamba/bin/micromamba run -n openhands poetry run playwright install --with-deps chromium && \ + # Set environment variables + echo "OH_INTERPRETER_PATH=$(/openhands/micromamba/bin/micromamba run -n openhands poetry run python -c "import sys; print(sys.executable)")" >> /etc/environment && \ + # Clear caches + /openhands/micromamba/bin/micromamba run -n openhands poetry cache clear --all . && \ + # Set permissions + chmod -R g+rws /openhands/poetry && \ + mkdir -p /openhands/workspace && chmod -R g+rws,o+rw /openhands/workspace && \ + # Clean up + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + /openhands/micromamba/bin/micromamba clean --all +{% endmacro %} + {% if build_from_scratch %} # ================================================================ # START: Build Runtime Image from Scratch @@ -48,29 +74,7 @@ RUN \ touch /openhands/code/openhands/__init__.py COPY ./code/pyproject.toml ./code/poetry.lock /openhands/code/ -# Install all dependencies -WORKDIR /openhands/code -RUN \ - /openhands/micromamba/bin/micromamba config set changeps1 False && \ - # Configure Poetry and create virtual environment - /openhands/micromamba/bin/micromamba run -n openhands poetry config virtualenvs.path /openhands/poetry && \ - /openhands/micromamba/bin/micromamba run -n openhands poetry env use python3.12 && \ - # Install project dependencies - /openhands/micromamba/bin/micromamba run -n openhands poetry install --only main,runtime --no-interaction --no-root && \ - # Update and install additional tools - apt-get update && \ - /openhands/micromamba/bin/micromamba run -n openhands poetry run pip install playwright && \ - /openhands/micromamba/bin/micromamba run -n openhands poetry run playwright install --with-deps chromium && \ - # Set environment variables - echo "OH_INTERPRETER_PATH=$(/openhands/micromamba/bin/micromamba run -n openhands poetry run python -c "import sys; print(sys.executable)")" >> /etc/environment && \ - # Clear caches - /openhands/micromamba/bin/micromamba run -n openhands poetry cache clear --all . && \ - # Set permissions - chmod -R g+rws /openhands/poetry && \ - mkdir -p /openhands/workspace && chmod -R g+rws,o+rw /openhands/workspace && \ - # Clean up - apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ - /openhands/micromamba/bin/micromamba clean --all +{{ install_dependencies() }} # ================================================================ # END: Build Runtime Image from Scratch @@ -84,5 +88,12 @@ RUN if [ -d /openhands/code/openhands ]; then rm -rf /openhands/code/openhands; COPY ./code/pyproject.toml ./code/poetry.lock /openhands/code/ COPY ./code/openhands /openhands/code/openhands +# ================================================================ +# END: Build from versioned image +# ================================================================ +{% if build_from_versioned %} +{{ install_dependencies() }} +{% endif %} + # Install extra dependencies if specified {% if extra_deps %}RUN {{ extra_deps }} {% endif %} diff --git a/tests/unit/test_runtime_build.py b/tests/unit/test_runtime_build.py index be80d22618..b9d663573e 100644 --- a/tests/unit/test_runtime_build.py +++ b/tests/unit/test_runtime_build.py @@ -16,6 +16,7 @@ from openhands import __version__ as oh_version from openhands.core.logger import openhands_logger as logger from openhands.runtime.builder.docker import DockerRuntimeBuilder from openhands.runtime.utils.runtime_build import ( + BuildFromImageType, _generate_dockerfile, build_runtime_image, get_hash_for_lock_files, @@ -83,7 +84,7 @@ def test_prep_build_folder(temp_dir): prep_build_folder( temp_dir, base_image=DEFAULT_BASE_IMAGE, - build_from_scratch=True, + build_from=BuildFromImageType.SCRATCH, extra_deps=None, ) @@ -130,7 +131,7 @@ def test_generate_dockerfile_build_from_scratch(): base_image = 'debian:11' dockerfile_content = _generate_dockerfile( base_image, - build_from_scratch=True, + build_from=BuildFromImageType.SCRATCH, ) assert base_image in dockerfile_content assert 'apt-get update' in dockerfile_content @@ -146,11 +147,11 @@ def test_generate_dockerfile_build_from_scratch(): ) -def test_generate_dockerfile_build_from_existing(): +def test_generate_dockerfile_build_from_lock(): base_image = 'debian:11' dockerfile_content = _generate_dockerfile( base_image, - build_from_scratch=False, + build_from=BuildFromImageType.LOCK, ) # These commands SHOULD NOT include in the dockerfile if build_from_scratch is False @@ -164,6 +165,24 @@ def test_generate_dockerfile_build_from_existing(): assert 'COPY ./code/openhands /openhands/code/openhands' in dockerfile_content +def test_generate_dockerfile_build_from_versioned(): + base_image = 'debian:11' + dockerfile_content = _generate_dockerfile( + base_image, + build_from=BuildFromImageType.VERSIONED, + ) + + # these commands should not exist when build from versioned + assert 'RUN apt update && apt install -y wget sudo' not in dockerfile_content + assert '-c conda-forge' not in dockerfile_content + assert 'python=3.12' not in dockerfile_content + assert 'https://micro.mamba.pm/install.sh' not in dockerfile_content + + # this SHOULD exist when build from versioned + assert 'poetry install' in dockerfile_content + assert 'COPY ./code/openhands /openhands/code/openhands' in dockerfile_content + + def test_get_runtime_image_repo_and_tag_eventstream(): base_image = 'debian:11' img_repo, img_tag = get_runtime_image_repo_and_tag(base_image) @@ -190,19 +209,22 @@ def test_get_runtime_image_repo_and_tag_eventstream(): def test_build_runtime_image_from_scratch(): base_image = 'debian:11' mock_lock_hash = MagicMock() - mock_lock_hash.return_value = 'mock-lock-hash' + mock_lock_hash.return_value = 'mock-lock-tag' + mock_versioned_tag = MagicMock() + mock_versioned_tag.return_value = 'mock-versioned-tag' mock_source_hash = MagicMock() - mock_source_hash.return_value = 'mock-source-hash' + mock_source_hash.return_value = 'mock-source-tag' mock_runtime_builder = MagicMock() mock_runtime_builder.image_exists.return_value = False mock_runtime_builder.build.return_value = ( - f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash' + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag' ) mock_prep_build_folder = MagicMock() mod = build_runtime_image.__module__ with ( patch(f'{mod}.get_hash_for_lock_files', mock_lock_hash), patch(f'{mod}.get_hash_for_source_files', mock_source_hash), + patch(f'{mod}.get_tag_for_versioned_image', mock_versioned_tag), patch( f'{build_runtime_image.__module__}.prep_build_folder', mock_prep_build_folder, @@ -212,31 +234,40 @@ def test_build_runtime_image_from_scratch(): mock_runtime_builder.build.assert_called_once_with( path=ANY, tags=[ - f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash_mock-source-hash', - f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash', + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag', + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag', + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-versioned-tag', ], platform=None, ) assert ( image_name - == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash_mock-source-hash' + == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag' + ) + mock_prep_build_folder.assert_called_once_with( + ANY, base_image, BuildFromImageType.SCRATCH, None ) - mock_prep_build_folder.assert_called_once_with(ANY, base_image, True, None) def test_build_runtime_image_exact_hash_exist(): base_image = 'debian:11' mock_lock_hash = MagicMock() - mock_lock_hash.return_value = 'mock-lock-hash' + mock_lock_hash.return_value = 'mock-lock-tag' mock_source_hash = MagicMock() - mock_source_hash.return_value = 'mock-source-hash' + mock_source_hash.return_value = 'mock-source-tag' + mock_versioned_tag = MagicMock() + mock_versioned_tag.return_value = 'mock-versioned-tag' mock_runtime_builder = MagicMock() mock_runtime_builder.image_exists.return_value = True + mock_runtime_builder.build.return_value = ( + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag' + ) mock_prep_build_folder = MagicMock() mod = build_runtime_image.__module__ with ( patch(f'{mod}.get_hash_for_lock_files', mock_lock_hash), patch(f'{mod}.get_hash_for_source_files', mock_source_hash), + patch(f'{mod}.get_tag_for_versioned_image', mock_versioned_tag), patch( f'{build_runtime_image.__module__}.prep_build_folder', mock_prep_build_folder, @@ -245,25 +276,45 @@ def test_build_runtime_image_exact_hash_exist(): image_name = build_runtime_image(base_image, mock_runtime_builder) assert ( image_name - == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash_mock-source-hash' + == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag' ) mock_runtime_builder.build.assert_not_called() mock_prep_build_folder.assert_not_called() -def test_build_runtime_image_exact_hash_not_exist(): +def test_build_runtime_image_exact_hash_not_exist_and_lock_exist(): base_image = 'debian:11' mock_lock_hash = MagicMock() - mock_lock_hash.return_value = 'mock-lock-hash' + mock_lock_hash.return_value = 'mock-lock-tag' mock_source_hash = MagicMock() - mock_source_hash.return_value = 'mock-source-hash' + mock_source_hash.return_value = 'mock-source-tag' + mock_versioned_tag = MagicMock() + mock_versioned_tag.return_value = 'mock-versioned-tag' mock_runtime_builder = MagicMock() - mock_runtime_builder.image_exists.side_effect = [False, True, False, True] + + def image_exists_side_effect(image_name, *args): + if 'mock-lock-tag_mock-source-tag' in image_name: + return False + elif 'mock-lock-tag' in image_name: + return True + elif 'mock-versioned-tag' in image_name: + # just to test we should never include versioned tag in a non-from-scratch build + # in real case it should be True when lock exists + return False + else: + raise ValueError(f'Unexpected image name: {image_name}') + + mock_runtime_builder.image_exists.side_effect = image_exists_side_effect + mock_runtime_builder.build.return_value = ( + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag' + ) + mock_prep_build_folder = MagicMock() mod = build_runtime_image.__module__ with ( patch(f'{mod}.get_hash_for_lock_files', mock_lock_hash), patch(f'{mod}.get_hash_for_source_files', mock_source_hash), + patch(f'{mod}.get_tag_for_versioned_image', mock_versioned_tag), patch( f'{build_runtime_image.__module__}.prep_build_folder', mock_prep_build_folder, @@ -272,11 +323,80 @@ def test_build_runtime_image_exact_hash_not_exist(): image_name = build_runtime_image(base_image, mock_runtime_builder) assert ( image_name - == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash_mock-source-hash' + == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag' + ) + mock_runtime_builder.build.assert_called_once_with( + path=ANY, + tags=[ + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag', + # lock tag will NOT be included - since it already exists + # VERSION tag will NOT be included except from scratch + ], + platform=None, ) - mock_runtime_builder.build.assert_called_once() mock_prep_build_folder.assert_called_once_with( - ANY, f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash', False, None + ANY, + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag', + BuildFromImageType.LOCK, + None, + ) + + +def test_build_runtime_image_exact_hash_not_exist_and_lock_not_exist_and_versioned_exist(): + base_image = 'debian:11' + mock_lock_hash = MagicMock() + mock_lock_hash.return_value = 'mock-lock-tag' + mock_source_hash = MagicMock() + mock_source_hash.return_value = 'mock-source-tag' + mock_versioned_tag = MagicMock() + mock_versioned_tag.return_value = 'mock-versioned-tag' + mock_runtime_builder = MagicMock() + + def image_exists_side_effect(image_name, *args): + if 'mock-lock-tag_mock-source-tag' in image_name: + return False + elif 'mock-lock-tag' in image_name: + return False + elif 'mock-versioned-tag' in image_name: + return True + else: + raise ValueError(f'Unexpected image name: {image_name}') + + mock_runtime_builder.image_exists.side_effect = image_exists_side_effect + mock_runtime_builder.build.return_value = ( + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag' + ) + + mock_prep_build_folder = MagicMock() + mod = build_runtime_image.__module__ + with ( + patch(f'{mod}.get_hash_for_lock_files', mock_lock_hash), + patch(f'{mod}.get_hash_for_source_files', mock_source_hash), + patch(f'{mod}.get_tag_for_versioned_image', mock_versioned_tag), + patch( + f'{build_runtime_image.__module__}.prep_build_folder', + mock_prep_build_folder, + ), + ): + image_name = build_runtime_image(base_image, mock_runtime_builder) + assert ( + image_name + == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag' + ) + mock_runtime_builder.build.assert_called_once_with( + path=ANY, + tags=[ + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag', + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag', + # VERSION tag will NOT be included except from scratch + ], + platform=None, + ) + mock_prep_build_folder.assert_called_once_with( + ANY, + f'{get_runtime_image_repo()}:{OH_VERSION}_mock-versioned-tag', + BuildFromImageType.VERSIONED, + None, )