From 33a4a12ae8706d2b36ffb718995e9c84a8864e00 Mon Sep 17 00:00:00 2001 From: Stavros Kois <47820033+stavros-k@users.noreply.github.com> Date: Tue, 25 Feb 2025 10:50:29 +0200 Subject: [PATCH] lib: use pre-built image (#1702) * lib: use pre-built image * lib: use the `tmpfs` key instead of the `volumes` for tmpfs type (#1708) * tmpfs * update hash and add tests --- devbox.json | 2 +- library/2.1.15/volume_mount_types.py | 72 ---------- library/{2.1.15 => 2.1.16}/__init__.py | 0 library/{2.1.15 => 2.1.16}/configs.py | 0 library/{2.1.15 => 2.1.16}/container.py | 13 +- library/{2.1.15 => 2.1.16}/depends.py | 0 library/{2.1.15 => 2.1.16}/deploy.py | 0 library/{2.1.15 => 2.1.16}/deps.py | 0 library/{2.1.15 => 2.1.16}/deps_mariadb.py | 0 library/{2.1.15 => 2.1.16}/deps_perms.py | 0 library/{2.1.15 => 2.1.16}/deps_postgres.py | 131 +----------------- library/{2.1.15 => 2.1.16}/deps_redis.py | 0 library/{2.1.15 => 2.1.16}/device.py | 0 .../{2.1.15 => 2.1.16}/device_cgroup_rules.py | 0 library/{2.1.15 => 2.1.16}/devices.py | 0 library/{2.1.15 => 2.1.16}/dns.py | 0 library/{2.1.15 => 2.1.16}/environment.py | 0 library/{2.1.15 => 2.1.16}/error.py | 0 library/{2.1.15 => 2.1.16}/expose.py | 0 library/{2.1.15 => 2.1.16}/extra_hosts.py | 0 library/{2.1.15 => 2.1.16}/formatter.py | 0 library/{2.1.15 => 2.1.16}/functions.py | 0 library/{2.1.15 => 2.1.16}/healthcheck.py | 0 library/{2.1.15 => 2.1.16}/labels.py | 0 library/{2.1.15 => 2.1.16}/notes.py | 0 library/{2.1.15 => 2.1.16}/portal.py | 0 library/{2.1.15 => 2.1.16}/portals.py | 0 library/{2.1.15 => 2.1.16}/ports.py | 0 library/{2.1.15 => 2.1.16}/render.py | 6 + library/{2.1.15 => 2.1.16}/resources.py | 0 library/{2.1.15 => 2.1.16}/restart.py | 0 library/{2.1.15 => 2.1.16}/security_opts.py | 0 library/{2.1.15 => 2.1.16}/storage.py | 13 +- library/{2.1.15 => 2.1.16}/sysctls.py | 0 library/{2.1.15 => 2.1.16}/tests/__init__.py | 0 .../tests/test_build_image.py | 0 .../{2.1.15 => 2.1.16}/tests/test_configs.py | 0 .../tests/test_container.py | 0 .../{2.1.15 => 2.1.16}/tests/test_depends.py | 0 library/{2.1.15 => 2.1.16}/tests/test_deps.py | 3 +- .../{2.1.15 => 2.1.16}/tests/test_device.py | 0 .../tests/test_device_cgroup_rules.py | 0 library/{2.1.15 => 2.1.16}/tests/test_dns.py | 0 .../tests/test_environment.py | 0 .../{2.1.15 => 2.1.16}/tests/test_expose.py | 0 .../tests/test_extra_hosts.py | 0 .../tests/test_formatter.py | 0 .../tests/test_functions.py | 0 .../tests/test_healthcheck.py | 0 .../{2.1.15 => 2.1.16}/tests/test_labels.py | 0 .../{2.1.15 => 2.1.16}/tests/test_notes.py | 0 .../{2.1.15 => 2.1.16}/tests/test_portal.py | 0 .../{2.1.15 => 2.1.16}/tests/test_ports.py | 0 .../{2.1.15 => 2.1.16}/tests/test_render.py | 0 .../tests/test_resources.py | 0 .../{2.1.15 => 2.1.16}/tests/test_restart.py | 0 .../tests/test_security_opts.py | 0 .../{2.1.15 => 2.1.16}/tests/test_sysctls.py | 0 .../tests/test_validations.py | 0 .../{2.1.15 => 2.1.16}/tests/test_volumes.py | 35 +++-- library/2.1.16/tmpfs.py | 75 ++++++++++ library/{2.1.15 => 2.1.16}/validations.py | 0 library/{2.1.15 => 2.1.16}/volume_mount.py | 9 +- library/2.1.16/volume_mount_types.py | 43 ++++++ library/{2.1.15 => 2.1.16}/volume_sources.py | 0 library/{2.1.15 => 2.1.16}/volume_types.py | 0 library/{2.1.15 => 2.1.16}/volumes.py | 7 +- library/hashes.yaml | 2 +- 68 files changed, 180 insertions(+), 231 deletions(-) delete mode 100644 library/2.1.15/volume_mount_types.py rename library/{2.1.15 => 2.1.16}/__init__.py (100%) rename library/{2.1.15 => 2.1.16}/configs.py (100%) rename library/{2.1.15 => 2.1.16}/container.py (97%) rename library/{2.1.15 => 2.1.16}/depends.py (100%) rename library/{2.1.15 => 2.1.16}/deploy.py (100%) rename library/{2.1.15 => 2.1.16}/deps.py (100%) rename library/{2.1.15 => 2.1.16}/deps_mariadb.py (100%) rename library/{2.1.15 => 2.1.16}/deps_perms.py (100%) rename library/{2.1.15 => 2.1.16}/deps_postgres.py (61%) rename library/{2.1.15 => 2.1.16}/deps_redis.py (100%) rename library/{2.1.15 => 2.1.16}/device.py (100%) rename library/{2.1.15 => 2.1.16}/device_cgroup_rules.py (100%) rename library/{2.1.15 => 2.1.16}/devices.py (100%) rename library/{2.1.15 => 2.1.16}/dns.py (100%) rename library/{2.1.15 => 2.1.16}/environment.py (100%) rename library/{2.1.15 => 2.1.16}/error.py (100%) rename library/{2.1.15 => 2.1.16}/expose.py (100%) rename library/{2.1.15 => 2.1.16}/extra_hosts.py (100%) rename library/{2.1.15 => 2.1.16}/formatter.py (100%) rename library/{2.1.15 => 2.1.16}/functions.py (100%) rename library/{2.1.15 => 2.1.16}/healthcheck.py (100%) rename library/{2.1.15 => 2.1.16}/labels.py (100%) rename library/{2.1.15 => 2.1.16}/notes.py (100%) rename library/{2.1.15 => 2.1.16}/portal.py (100%) rename library/{2.1.15 => 2.1.16}/portals.py (100%) rename library/{2.1.15 => 2.1.16}/ports.py (100%) rename library/{2.1.15 => 2.1.16}/render.py (92%) rename library/{2.1.15 => 2.1.16}/resources.py (100%) rename library/{2.1.15 => 2.1.16}/restart.py (100%) rename library/{2.1.15 => 2.1.16}/security_opts.py (100%) rename library/{2.1.15 => 2.1.16}/storage.py (87%) rename library/{2.1.15 => 2.1.16}/sysctls.py (100%) rename library/{2.1.15 => 2.1.16}/tests/__init__.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_build_image.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_configs.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_container.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_depends.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_deps.py (99%) rename library/{2.1.15 => 2.1.16}/tests/test_device.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_device_cgroup_rules.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_dns.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_environment.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_expose.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_extra_hosts.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_formatter.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_functions.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_healthcheck.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_labels.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_notes.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_portal.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_ports.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_render.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_resources.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_restart.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_security_opts.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_sysctls.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_validations.py (100%) rename library/{2.1.15 => 2.1.16}/tests/test_volumes.py (95%) create mode 100644 library/2.1.16/tmpfs.py rename library/{2.1.15 => 2.1.16}/validations.py (100%) rename library/{2.1.15 => 2.1.16}/volume_mount.py (93%) create mode 100644 library/2.1.16/volume_mount_types.py rename library/{2.1.15 => 2.1.16}/volume_sources.py (100%) rename library/{2.1.15 => 2.1.16}/volume_types.py (100%) rename library/{2.1.15 => 2.1.16}/volumes.py (93%) diff --git a/devbox.json b/devbox.json index defdede931..e1e5489f52 100644 --- a/devbox.json +++ b/devbox.json @@ -13,7 +13,7 @@ "scripts": { "ports": ["python3 ./.github/scripts/port_validation.py"], "lib-test": [ - "pytest library/", + "pytest library/ -vvv", "rm -r library/**/__pycache__", "rm -r library/**/tests/__pycache__" ] diff --git a/library/2.1.15/volume_mount_types.py b/library/2.1.15/volume_mount_types.py deleted file mode 100644 index 00a0ec3a18..0000000000 --- a/library/2.1.15/volume_mount_types.py +++ /dev/null @@ -1,72 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorageTmpfsConfig, IxStorageVolumeConfig, IxStorageBindLikeConfigs - - -try: - from .error import RenderError - from .validations import valid_host_path_propagation, valid_octal_mode_or_raise -except ImportError: - from error import RenderError - from validations import valid_host_path_propagation, valid_octal_mode_or_raise - - -class TmpfsMountType: - def __init__(self, render_instance: "Render", config: "IxStorageTmpfsConfig"): - self._render_instance = render_instance - self.spec = {"tmpfs": {}} - size = config.get("size", None) - mode = config.get("mode", None) - - if size is not None: - if not isinstance(size, int): - raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]") - if not size > 0: - raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]") - # Convert Mebibytes to Bytes - self.spec["tmpfs"]["size"] = size * 1024 * 1024 - - if mode is not None: - mode = valid_octal_mode_or_raise(mode) - self.spec["tmpfs"]["mode"] = int(mode, 8) - - if not self.spec["tmpfs"]: - self.spec.pop("tmpfs") - - def render(self) -> dict: - """Render the tmpfs mount specification.""" - return self.spec - - -class BindMountType: - def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"): - self._render_instance = render_instance - self.spec: dict = {} - - propagation = valid_host_path_propagation(config.get("propagation", "rprivate")) - create_host_path = config.get("create_host_path", False) - - self.spec: dict = { - "bind": { - "create_host_path": create_host_path, - "propagation": propagation, - } - } - - def render(self) -> dict: - """Render the bind mount specification.""" - return self.spec - - -class VolumeMountType: - def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): - self._render_instance = render_instance - self.spec: dict = {} - - self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}} - - def render(self) -> dict: - """Render the volume mount specification.""" - return self.spec diff --git a/library/2.1.15/__init__.py b/library/2.1.16/__init__.py similarity index 100% rename from library/2.1.15/__init__.py rename to library/2.1.16/__init__.py diff --git a/library/2.1.15/configs.py b/library/2.1.16/configs.py similarity index 100% rename from library/2.1.15/configs.py rename to library/2.1.16/configs.py diff --git a/library/2.1.15/container.py b/library/2.1.16/container.py similarity index 97% rename from library/2.1.15/container.py rename to library/2.1.16/container.py index 47dc38eaac..720f0d0c7b 100644 --- a/library/2.1.15/container.py +++ b/library/2.1.16/container.py @@ -20,6 +20,7 @@ try: from .labels import Labels from .ports import Ports from .restart import RestartPolicy + from .tmpfs import Tmpfs from .validations import ( valid_cap_or_raise, valid_ipc_mode_or_raise, @@ -46,6 +47,7 @@ except ImportError: from labels import Labels from ports import Ports from restart import RestartPolicy + from tmpfs import Tmpfs from validations import ( valid_cap_or_raise, valid_ipc_mode_or_raise, @@ -83,7 +85,8 @@ class Container: self._command: list[str] = [] self._grace_period: int | None = None self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) + self._storage: Storage = Storage(self._render_instance, self) + self._tmpfs: Tmpfs = Tmpfs(self._render_instance, self) self._ipc_mode: str | None = None self._device_cgroup_rules: DeviceCGroupRules = DeviceCGroupRules(self._render_instance) self.sysctls: Sysctls = Sysctls(self._render_instance, self) @@ -270,7 +273,10 @@ class Container: self._command = [escape_dollar(str(e)) for e in command] def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) + if config.get("type", "") == "tmpfs": + self._tmpfs.add(mount_path, config) + else: + self._storage.add(mount_path, config) def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): self.add_group(999) @@ -415,4 +421,7 @@ class Container: if self._storage.has_mounts(): result["volumes"] = self._storage.render() + if self._tmpfs.has_tmpfs(): + result["tmpfs"] = self._tmpfs.render() + return result diff --git a/library/2.1.15/depends.py b/library/2.1.16/depends.py similarity index 100% rename from library/2.1.15/depends.py rename to library/2.1.16/depends.py diff --git a/library/2.1.15/deploy.py b/library/2.1.16/deploy.py similarity index 100% rename from library/2.1.15/deploy.py rename to library/2.1.16/deploy.py diff --git a/library/2.1.15/deps.py b/library/2.1.16/deps.py similarity index 100% rename from library/2.1.15/deps.py rename to library/2.1.16/deps.py diff --git a/library/2.1.15/deps_mariadb.py b/library/2.1.16/deps_mariadb.py similarity index 100% rename from library/2.1.15/deps_mariadb.py rename to library/2.1.16/deps_mariadb.py diff --git a/library/2.1.15/deps_perms.py b/library/2.1.16/deps_perms.py similarity index 100% rename from library/2.1.15/deps_perms.py rename to library/2.1.16/deps_perms.py diff --git a/library/2.1.15/deps_postgres.py b/library/2.1.16/deps_postgres.py similarity index 61% rename from library/2.1.15/deps_postgres.py rename to library/2.1.16/deps_postgres.py index c40e900d07..0b25f7a503 100644 --- a/library/2.1.15/deps_postgres.py +++ b/library/2.1.16/deps_postgres.py @@ -70,10 +70,8 @@ class PostgresContainer: # eg we don't want to handle upgrades of pg_vector at the moment if repo == "postgres": target_major_version = self._get_target_version(image) - upg = self._render_instance.add_container(self._upgrade_name, image) - upg.build_image(get_build_manifest()) + upg = self._render_instance.add_container(self._upgrade_name, "postgres_upgrade_image") upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) - upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") upg.restart.set_policy("on-failure", 1) upg.set_user(999, 999) upg.healthcheck.disable() @@ -150,130 +148,3 @@ class PostgresContainer: return addr case _: raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -def get_build_manifest() -> list[str | None]: - return [ - f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", - "WORKDIR /tmp", - ] - - -def get_upgrade_packages(): - return [ - "rsync", - "postgresql-13", - "postgresql-14", - "postgresql-15", - "postgresql-16", - ] - - -def get_upgrade_script(): - return """ -#!/bin/bash -set -euo pipefail - -get_bin_path() { - local version=$1 - echo "/usr/lib/postgresql/$version/bin" -} - -log() { - echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" -} - -check_writable() { - local path=$1 - if [ ! -w "$path" ]; then - log "$path is not writable" - exit 1 - fi -} - -check_writable "$DATA_DIR" - -# Don't do anything if its a fresh install. -if [ ! -f "$DATA_DIR/PG_VERSION" ]; then - log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." - exit 0 -fi - -# Don't do anything if we're already at the target version. -OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") -log "Current version: $OLD_VERSION" -log "Target version: $TARGET_VERSION" -if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then - log "Already at target version $TARGET_VERSION" - exit 0 -fi - -# Fail if we're downgrading. -if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then - log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" - exit 1 -fi - -export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") -if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then - log "File $OLD_PG_BINARY/pg_upgrade does not exist." - exit 1 -fi - -export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") -if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then - log "File $NEW_PG_BINARY/pg_upgrade does not exist." - exit 1 -fi - -export NEW_DATA_DIR="/tmp/new-data-dir" -if [ -d "$NEW_DATA_DIR" ]; then - log "Directory $NEW_DATA_DIR already exists." - exit 1 -fi - -export PGUSER="$POSTGRES_USER" -log "Creating new data dir and initializing..." -PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" - -timestamp=$(date +%Y%m%d%H%M%S) -backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" -log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" -tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" - -log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" -log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" -log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." - -"$NEW_PG_BINARY"/pg_upgrade \ - --old-bindir="$OLD_PG_BINARY" \ - --new-bindir="$NEW_PG_BINARY" \ - --old-datadir="$DATA_DIR" \ - --new-datadir="$NEW_DATA_DIR" \ - --socketdir /var/run/postgresql \ - --check - -log "Compatibility check passed." - -log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." -"$NEW_PG_BINARY"/pg_upgrade \ - --old-bindir="$OLD_PG_BINARY" \ - --new-bindir="$NEW_PG_BINARY" \ - --old-datadir="$DATA_DIR" \ - --new-datadir="$NEW_DATA_DIR" \ - --socketdir /var/run/postgresql - -log "Upgrade complete." - -log "Copying old pg_hba.conf to new pg_hba.conf" -# We need to carry this over otherwise -cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" - -log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." -rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" - -log "Removing $NEW_DATA_DIR." -rm -rf "$NEW_DATA_DIR" - -log "Done." -""" diff --git a/library/2.1.15/deps_redis.py b/library/2.1.16/deps_redis.py similarity index 100% rename from library/2.1.15/deps_redis.py rename to library/2.1.16/deps_redis.py diff --git a/library/2.1.15/device.py b/library/2.1.16/device.py similarity index 100% rename from library/2.1.15/device.py rename to library/2.1.16/device.py diff --git a/library/2.1.15/device_cgroup_rules.py b/library/2.1.16/device_cgroup_rules.py similarity index 100% rename from library/2.1.15/device_cgroup_rules.py rename to library/2.1.16/device_cgroup_rules.py diff --git a/library/2.1.15/devices.py b/library/2.1.16/devices.py similarity index 100% rename from library/2.1.15/devices.py rename to library/2.1.16/devices.py diff --git a/library/2.1.15/dns.py b/library/2.1.16/dns.py similarity index 100% rename from library/2.1.15/dns.py rename to library/2.1.16/dns.py diff --git a/library/2.1.15/environment.py b/library/2.1.16/environment.py similarity index 100% rename from library/2.1.15/environment.py rename to library/2.1.16/environment.py diff --git a/library/2.1.15/error.py b/library/2.1.16/error.py similarity index 100% rename from library/2.1.15/error.py rename to library/2.1.16/error.py diff --git a/library/2.1.15/expose.py b/library/2.1.16/expose.py similarity index 100% rename from library/2.1.15/expose.py rename to library/2.1.16/expose.py diff --git a/library/2.1.15/extra_hosts.py b/library/2.1.16/extra_hosts.py similarity index 100% rename from library/2.1.15/extra_hosts.py rename to library/2.1.16/extra_hosts.py diff --git a/library/2.1.15/formatter.py b/library/2.1.16/formatter.py similarity index 100% rename from library/2.1.15/formatter.py rename to library/2.1.16/formatter.py diff --git a/library/2.1.15/functions.py b/library/2.1.16/functions.py similarity index 100% rename from library/2.1.15/functions.py rename to library/2.1.16/functions.py diff --git a/library/2.1.15/healthcheck.py b/library/2.1.16/healthcheck.py similarity index 100% rename from library/2.1.15/healthcheck.py rename to library/2.1.16/healthcheck.py diff --git a/library/2.1.15/labels.py b/library/2.1.16/labels.py similarity index 100% rename from library/2.1.15/labels.py rename to library/2.1.16/labels.py diff --git a/library/2.1.15/notes.py b/library/2.1.16/notes.py similarity index 100% rename from library/2.1.15/notes.py rename to library/2.1.16/notes.py diff --git a/library/2.1.15/portal.py b/library/2.1.16/portal.py similarity index 100% rename from library/2.1.15/portal.py rename to library/2.1.16/portal.py diff --git a/library/2.1.15/portals.py b/library/2.1.16/portals.py similarity index 100% rename from library/2.1.15/portals.py rename to library/2.1.16/portals.py diff --git a/library/2.1.15/ports.py b/library/2.1.16/ports.py similarity index 100% rename from library/2.1.15/ports.py rename to library/2.1.16/ports.py diff --git a/library/2.1.15/render.py b/library/2.1.16/render.py similarity index 92% rename from library/2.1.15/render.py rename to library/2.1.16/render.py index 9d8fcc28d5..97afb6616e 100644 --- a/library/2.1.15/render.py +++ b/library/2.1.16/render.py @@ -43,6 +43,12 @@ class Render(object): if "python_permissions_image" not in self.values["images"]: self.values["images"]["python_permissions_image"] = {"repository": "python", "tag": "3.13.0-slim-bookworm"} + if "postgres_upgrade_image" not in self.values["images"]: + self.values["images"]["postgres_upgrade_image"] = { + "repository": "ixsystems/postgres-upgrade", + "tag": "1.0.0", + } + def container_names(self): return list(self._containers.keys()) diff --git a/library/2.1.15/resources.py b/library/2.1.16/resources.py similarity index 100% rename from library/2.1.15/resources.py rename to library/2.1.16/resources.py diff --git a/library/2.1.15/restart.py b/library/2.1.16/restart.py similarity index 100% rename from library/2.1.15/restart.py rename to library/2.1.16/restart.py diff --git a/library/2.1.15/security_opts.py b/library/2.1.16/security_opts.py similarity index 100% rename from library/2.1.15/security_opts.py rename to library/2.1.16/security_opts.py diff --git a/library/2.1.15/storage.py b/library/2.1.16/storage.py similarity index 87% rename from library/2.1.15/storage.py rename to library/2.1.16/storage.py index f1650259b3..6f2f69a52d 100644 --- a/library/2.1.15/storage.py +++ b/library/2.1.16/storage.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union if TYPE_CHECKING: + from container import Container from render import Render try: @@ -16,6 +17,8 @@ except ImportError: class IxStorageTmpfsConfig(TypedDict): size: NotRequired[int] mode: NotRequired[str] + uid: NotRequired[int] + gid: NotRequired[int] class AclConfig(TypedDict, total=False): @@ -79,18 +82,24 @@ class IxStorage(TypedDict): class Storage: - def __init__(self, render_instance: "Render"): + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._container_instance = container_instance self._render_instance = render_instance self._volume_mounts: set[VolumeMount] = set() def add(self, mount_path: str, config: "IxStorage"): mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: + if self.is_defined(mount_path): + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + if self._container_instance._tmpfs.is_defined(mount_path): raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") volume_mount = VolumeMount(self._render_instance, mount_path, config) self._volume_mounts.add(volume_mount) + def is_defined(self, mount_path: str): + return mount_path in [m.mount_path for m in self._volume_mounts] + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): mount_path = valid_fs_path_or_raise(mount_path) cfg: "IxStorage" = { diff --git a/library/2.1.15/sysctls.py b/library/2.1.16/sysctls.py similarity index 100% rename from library/2.1.15/sysctls.py rename to library/2.1.16/sysctls.py diff --git a/library/2.1.15/tests/__init__.py b/library/2.1.16/tests/__init__.py similarity index 100% rename from library/2.1.15/tests/__init__.py rename to library/2.1.16/tests/__init__.py diff --git a/library/2.1.15/tests/test_build_image.py b/library/2.1.16/tests/test_build_image.py similarity index 100% rename from library/2.1.15/tests/test_build_image.py rename to library/2.1.16/tests/test_build_image.py diff --git a/library/2.1.15/tests/test_configs.py b/library/2.1.16/tests/test_configs.py similarity index 100% rename from library/2.1.15/tests/test_configs.py rename to library/2.1.16/tests/test_configs.py diff --git a/library/2.1.15/tests/test_container.py b/library/2.1.16/tests/test_container.py similarity index 100% rename from library/2.1.15/tests/test_container.py rename to library/2.1.16/tests/test_container.py diff --git a/library/2.1.15/tests/test_depends.py b/library/2.1.16/tests/test_depends.py similarity index 100% rename from library/2.1.15/tests/test_depends.py rename to library/2.1.16/tests/test_depends.py diff --git a/library/2.1.15/tests/test_deps.py b/library/2.1.16/tests/test_deps.py similarity index 99% rename from library/2.1.15/tests/test_deps.py rename to library/2.1.16/tests/test_deps.py index a1b7f03a60..e8f69357d8 100644 --- a/library/2.1.15/tests/test_deps.py +++ b/library/2.1.16/tests/test_deps.py @@ -472,6 +472,5 @@ def test_postgres_with_upgrade_container(mock_values): assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} assert pgup["restart"] == "on-failure:1" assert pgup["healthcheck"] == {"disable": True} - assert pgup["build"]["dockerfile_inline"] != "" - assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["image"] == "ixsystems/postgres-upgrade:1.0.0" assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/library/2.1.15/tests/test_device.py b/library/2.1.16/tests/test_device.py similarity index 100% rename from library/2.1.15/tests/test_device.py rename to library/2.1.16/tests/test_device.py diff --git a/library/2.1.15/tests/test_device_cgroup_rules.py b/library/2.1.16/tests/test_device_cgroup_rules.py similarity index 100% rename from library/2.1.15/tests/test_device_cgroup_rules.py rename to library/2.1.16/tests/test_device_cgroup_rules.py diff --git a/library/2.1.15/tests/test_dns.py b/library/2.1.16/tests/test_dns.py similarity index 100% rename from library/2.1.15/tests/test_dns.py rename to library/2.1.16/tests/test_dns.py diff --git a/library/2.1.15/tests/test_environment.py b/library/2.1.16/tests/test_environment.py similarity index 100% rename from library/2.1.15/tests/test_environment.py rename to library/2.1.16/tests/test_environment.py diff --git a/library/2.1.15/tests/test_expose.py b/library/2.1.16/tests/test_expose.py similarity index 100% rename from library/2.1.15/tests/test_expose.py rename to library/2.1.16/tests/test_expose.py diff --git a/library/2.1.15/tests/test_extra_hosts.py b/library/2.1.16/tests/test_extra_hosts.py similarity index 100% rename from library/2.1.15/tests/test_extra_hosts.py rename to library/2.1.16/tests/test_extra_hosts.py diff --git a/library/2.1.15/tests/test_formatter.py b/library/2.1.16/tests/test_formatter.py similarity index 100% rename from library/2.1.15/tests/test_formatter.py rename to library/2.1.16/tests/test_formatter.py diff --git a/library/2.1.15/tests/test_functions.py b/library/2.1.16/tests/test_functions.py similarity index 100% rename from library/2.1.15/tests/test_functions.py rename to library/2.1.16/tests/test_functions.py diff --git a/library/2.1.15/tests/test_healthcheck.py b/library/2.1.16/tests/test_healthcheck.py similarity index 100% rename from library/2.1.15/tests/test_healthcheck.py rename to library/2.1.16/tests/test_healthcheck.py diff --git a/library/2.1.15/tests/test_labels.py b/library/2.1.16/tests/test_labels.py similarity index 100% rename from library/2.1.15/tests/test_labels.py rename to library/2.1.16/tests/test_labels.py diff --git a/library/2.1.15/tests/test_notes.py b/library/2.1.16/tests/test_notes.py similarity index 100% rename from library/2.1.15/tests/test_notes.py rename to library/2.1.16/tests/test_notes.py diff --git a/library/2.1.15/tests/test_portal.py b/library/2.1.16/tests/test_portal.py similarity index 100% rename from library/2.1.15/tests/test_portal.py rename to library/2.1.16/tests/test_portal.py diff --git a/library/2.1.15/tests/test_ports.py b/library/2.1.16/tests/test_ports.py similarity index 100% rename from library/2.1.15/tests/test_ports.py rename to library/2.1.16/tests/test_ports.py diff --git a/library/2.1.15/tests/test_render.py b/library/2.1.16/tests/test_render.py similarity index 100% rename from library/2.1.15/tests/test_render.py rename to library/2.1.16/tests/test_render.py diff --git a/library/2.1.15/tests/test_resources.py b/library/2.1.16/tests/test_resources.py similarity index 100% rename from library/2.1.15/tests/test_resources.py rename to library/2.1.16/tests/test_resources.py diff --git a/library/2.1.15/tests/test_restart.py b/library/2.1.16/tests/test_restart.py similarity index 100% rename from library/2.1.15/tests/test_restart.py rename to library/2.1.16/tests/test_restart.py diff --git a/library/2.1.15/tests/test_security_opts.py b/library/2.1.16/tests/test_security_opts.py similarity index 100% rename from library/2.1.15/tests/test_security_opts.py rename to library/2.1.16/tests/test_security_opts.py diff --git a/library/2.1.15/tests/test_sysctls.py b/library/2.1.16/tests/test_sysctls.py similarity index 100% rename from library/2.1.15/tests/test_sysctls.py rename to library/2.1.16/tests/test_sysctls.py diff --git a/library/2.1.15/tests/test_validations.py b/library/2.1.16/tests/test_validations.py similarity index 100% rename from library/2.1.15/tests/test_validations.py rename to library/2.1.16/tests/test_validations.py diff --git a/library/2.1.15/tests/test_volumes.py b/library/2.1.16/tests/test_volumes.py similarity index 95% rename from library/2.1.15/tests/test_volumes.py rename to library/2.1.16/tests/test_volumes.py index 9a98956bdc..e6311afa36 100644 --- a/library/2.1.15/tests/test_volumes.py +++ b/library/2.1.16/tests/test_volumes.py @@ -535,18 +535,37 @@ def test_tmpfs_volume(mock_values): render = Render(mock_values) c1 = render.add_container("test_container", "test_image") c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) + c1.add_storage("/some/path", {"type": "tmpfs"}) + c1.add_storage("/some/other/path", {"type": "tmpfs", "tmpfs_config": {"size": 100}}) + c1.add_storage( + "/some/other/path2", {"type": "tmpfs", "tmpfs_config": {"size": 100, "mode": "0777", "uid": 1000, "gid": 1000}} + ) output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } + assert output["services"]["test_container"]["tmpfs"] == [ + "/some/other/path2:gid=1000,mode=0777,size=104857600,uid=1000", + "/some/other/path:size=104857600", + "/some/path", ] +def test_add_tmpfs_with_existing_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "volume", "volume_config": {"volume_name": "test_volume"}}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs", "tmpfs_config": {"size": 100}}) + + +def test_add_volume_with_existing_tmpfs(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs", "tmpfs_config": {"size": 100}}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "volume", "volume_config": {"volume_name": "test_volume"}}) + + def test_temporary_volume(mock_values): render = Render(mock_values) c1 = render.add_container("test_container", "test_image") diff --git a/library/2.1.16/tmpfs.py b/library/2.1.16/tmpfs.py new file mode 100644 index 0000000000..fd611cd7b0 --- /dev/null +++ b/library/2.1.16/tmpfs.py @@ -0,0 +1,75 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from container import Container + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise, valid_octal_mode_or_raise +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise, valid_octal_mode_or_raise + + +class Tmpfs: + + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._tmpfs: dict = {} + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if self.is_defined(mount_path): + raise RenderError(f"Tmpfs mount path [{mount_path}] already added") + + if self._container_instance.storage.is_defined(mount_path): + raise RenderError(f"Tmpfs mount path [{mount_path}] already used for another volume mount") + + mount_config = config.get("tmpfs_config", {}) + size = mount_config.get("size", None) + mode = mount_config.get("mode", None) + uid = mount_config.get("uid", None) + gid = mount_config.get("gid", None) + + if size is not None: + if not isinstance(size, int): + raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]") + if not size > 0: + raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]") + # Convert Mebibytes to Bytes + size = size * 1024 * 1024 + + if mode is not None: + mode = valid_octal_mode_or_raise(mode) + + if uid is not None and not isinstance(uid, int): + raise RenderError(f"Expected [uid] to be an integer for [tmpfs] type, got [{uid}]") + + if gid is not None and not isinstance(gid, int): + raise RenderError(f"Expected [gid] to be an integer for [tmpfs] type, got [{gid}]") + + self._tmpfs[mount_path] = {} + if size is not None: + self._tmpfs[mount_path]["size"] = str(size) + if mode is not None: + self._tmpfs[mount_path]["mode"] = str(mode) + if uid is not None: + self._tmpfs[mount_path]["uid"] = str(uid) + if gid is not None: + self._tmpfs[mount_path]["gid"] = str(gid) + + def is_defined(self, mount_path: str): + return mount_path in self._tmpfs + + def has_tmpfs(self): + return bool(self._tmpfs) + + def render(self): + result = [] + for mount_path, config in self._tmpfs.items(): + opts = sorted([f"{k}={v}" for k, v in config.items()]) + result.append(f"{mount_path}:{','.join(opts)}" if opts else mount_path) + return sorted(result) diff --git a/library/2.1.15/validations.py b/library/2.1.16/validations.py similarity index 100% rename from library/2.1.15/validations.py rename to library/2.1.16/validations.py diff --git a/library/2.1.15/volume_mount.py b/library/2.1.16/volume_mount.py similarity index 93% rename from library/2.1.15/volume_mount.py rename to library/2.1.16/volume_mount.py index aadd077750..92cb8eb6e6 100644 --- a/library/2.1.15/volume_mount.py +++ b/library/2.1.16/volume_mount.py @@ -7,12 +7,12 @@ if TYPE_CHECKING: try: from .error import RenderError from .formatter import merge_dicts_no_overwrite - from .volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from .volume_mount_types import BindMountType, VolumeMountType from .volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource except ImportError: from error import RenderError from formatter import merge_dicts_no_overwrite - from volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from volume_mount_types import BindMountType, VolumeMountType from volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource @@ -40,11 +40,6 @@ class VolumeMount: raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() source = IxVolumeSource(self._render_instance, mount_config).get() - case "tmpfs": - spec_type = "tmpfs" - mount_config = config.get("tmpfs_config", {}) - mount_type_specific_definition = TmpfsMountType(self._render_instance, mount_config).render() - source = None case "nfs": spec_type = "volume" mount_config = config.get("nfs_config") diff --git a/library/2.1.16/volume_mount_types.py b/library/2.1.16/volume_mount_types.py new file mode 100644 index 0000000000..1bf089789e --- /dev/null +++ b/library/2.1.16/volume_mount_types.py @@ -0,0 +1,43 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageVolumeConfig, IxStorageBindLikeConfigs + + +try: + from .validations import valid_host_path_propagation +except ImportError: + from validations import valid_host_path_propagation + + +class BindMountType: + def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"): + self._render_instance = render_instance + self.spec: dict = {} + + propagation = valid_host_path_propagation(config.get("propagation", "rprivate")) + create_host_path = config.get("create_host_path", False) + + self.spec: dict = { + "bind": { + "create_host_path": create_host_path, + "propagation": propagation, + } + } + + def render(self) -> dict: + """Render the bind mount specification.""" + return self.spec + + +class VolumeMountType: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.spec: dict = {} + + self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}} + + def render(self) -> dict: + """Render the volume mount specification.""" + return self.spec diff --git a/library/2.1.15/volume_sources.py b/library/2.1.16/volume_sources.py similarity index 100% rename from library/2.1.15/volume_sources.py rename to library/2.1.16/volume_sources.py diff --git a/library/2.1.15/volume_types.py b/library/2.1.16/volume_types.py similarity index 100% rename from library/2.1.15/volume_types.py rename to library/2.1.16/volume_types.py diff --git a/library/2.1.15/volumes.py b/library/2.1.16/volumes.py similarity index 93% rename from library/2.1.15/volumes.py rename to library/2.1.16/volumes.py index e6925a402f..1a21d46458 100644 --- a/library/2.1.15/volumes.py +++ b/library/2.1.16/volumes.py @@ -19,12 +19,7 @@ class Volumes: self._render_instance = render_instance self._volumes: dict[str, Volume] = {} - def add_volume( - self, - source: str, - storage_type: str, - config: "IxStorageVolumeLikeConfigs", - ): + def add_volume(self, source: str, storage_type: str, config: "IxStorageVolumeLikeConfigs"): # This method can be called many times from the volume mounts # Only add the volume if it is not already added, but dont raise an error if source == "": diff --git a/library/hashes.yaml b/library/hashes.yaml index 36fb88e158..94efb31917 100644 --- a/library/hashes.yaml +++ b/library/hashes.yaml @@ -1,2 +1,2 @@ 0.0.1: f074617a82a86d2a6cc78a4c8a4296fc9d168e456f12713e50c696557b302133 -2.1.15: e26cffb91766782c787867b379519f7407b1fed8450dce463cefa56340a41d84 +2.1.16: 488c896e0dc7d49e8023e527a0769bfb7cb97a29899fba058fcee81bd323212b