lib: use pre-built image (#1702)

* lib: use pre-built image

* lib: use the `tmpfs` key instead of the `volumes` for tmpfs type (#1708)

* tmpfs

* update hash and add tests

devbox.json

@@ -13,7 +13,7 @@
     "scripts": {
       "ports": ["python3 ./.github/scripts/port_validation.py"],
       "lib-test": [
-        "pytest library/",
+        "pytest library/ -vvv",
         "rm -r library/**/__pycache__",
         "rm -r library/**/tests/__pycache__"
       ]

library/2.1.15/volume_mount_types.py

@@ -1,72 +0,0 @@
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from render import Render
-    from storage import IxStorageTmpfsConfig, IxStorageVolumeConfig, IxStorageBindLikeConfigs
-
-
-try:
-    from .error import RenderError
-    from .validations import valid_host_path_propagation, valid_octal_mode_or_raise
-except ImportError:
-    from error import RenderError
-    from validations import valid_host_path_propagation, valid_octal_mode_or_raise
-
-
-class TmpfsMountType:
-    def __init__(self, render_instance: "Render", config: "IxStorageTmpfsConfig"):
-        self._render_instance = render_instance
-        self.spec = {"tmpfs": {}}
-        size = config.get("size", None)
-        mode = config.get("mode", None)
-
-        if size is not None:
-            if not isinstance(size, int):
-                raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]")
-            if not size > 0:
-                raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]")
-            # Convert Mebibytes to Bytes
-            self.spec["tmpfs"]["size"] = size * 1024 * 1024
-
-        if mode is not None:
-            mode = valid_octal_mode_or_raise(mode)
-            self.spec["tmpfs"]["mode"] = int(mode, 8)
-
-        if not self.spec["tmpfs"]:
-            self.spec.pop("tmpfs")
-
-    def render(self) -> dict:
-        """Render the tmpfs mount specification."""
-        return self.spec
-
-
-class BindMountType:
-    def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"):
-        self._render_instance = render_instance
-        self.spec: dict = {}
-
-        propagation = valid_host_path_propagation(config.get("propagation", "rprivate"))
-        create_host_path = config.get("create_host_path", False)
-
-        self.spec: dict = {
-            "bind": {
-                "create_host_path": create_host_path,
-                "propagation": propagation,
-            }
-        }
-
-    def render(self) -> dict:
-        """Render the bind mount specification."""
-        return self.spec
-
-
-class VolumeMountType:
-    def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"):
-        self._render_instance = render_instance
-        self.spec: dict = {}
-
-        self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}}
-
-    def render(self) -> dict:
-        """Render the volume mount specification."""
-        return self.spec

library/2.1.16/container.py

@@ -20,6 +20,7 @@ try:
     from .labels import Labels
     from .ports import Ports
     from .restart import RestartPolicy
+    from .tmpfs import Tmpfs
     from .validations import (
         valid_cap_or_raise,
         valid_ipc_mode_or_raise,
@@ -46,6 +47,7 @@ except ImportError:
     from labels import Labels
     from ports import Ports
     from restart import RestartPolicy
+    from tmpfs import Tmpfs
     from validations import (
         valid_cap_or_raise,
         valid_ipc_mode_or_raise,
@@ -83,7 +85,8 @@ class Container:
         self._command: list[str] = []
         self._grace_period: int | None = None
         self._shm_size: int | None = None
-        self._storage: Storage = Storage(self._render_instance)
+        self._storage: Storage = Storage(self._render_instance, self)
+        self._tmpfs: Tmpfs = Tmpfs(self._render_instance, self)
         self._ipc_mode: str | None = None
         self._device_cgroup_rules: DeviceCGroupRules = DeviceCGroupRules(self._render_instance)
         self.sysctls: Sysctls = Sysctls(self._render_instance, self)
@@ -270,7 +273,10 @@ class Container:
         self._command = [escape_dollar(str(e)) for e in command]
 
     def add_storage(self, mount_path: str, config: "IxStorage"):
-        self._storage.add(mount_path, config)
+        if config.get("type", "") == "tmpfs":
+            self._tmpfs.add(mount_path, config)
+        else:
+            self._storage.add(mount_path, config)
 
     def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"):
         self.add_group(999)
@@ -415,4 +421,7 @@ class Container:
         if self._storage.has_mounts():
             result["volumes"] = self._storage.render()
 
+        if self._tmpfs.has_tmpfs():
+            result["tmpfs"] = self._tmpfs.render()
+
         return result

library/2.1.16/deps_postgres.py

@@ -70,10 +70,8 @@ class PostgresContainer:
         # eg we don't want to handle upgrades of pg_vector at the moment
         if repo == "postgres":
             target_major_version = self._get_target_version(image)
-            upg = self._render_instance.add_container(self._upgrade_name, image)
-            upg.build_image(get_build_manifest())
+            upg = self._render_instance.add_container(self._upgrade_name, "postgres_upgrade_image")
             upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"])
-            upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755")
             upg.restart.set_policy("on-failure", 1)
             upg.set_user(999, 999)
             upg.healthcheck.disable()
@@ -150,130 +148,3 @@ class PostgresContainer:
                 return addr
             case _:
                 raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]")
-
-
-def get_build_manifest() -> list[str | None]:
-    return [
-        f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}",
-        "WORKDIR /tmp",
-    ]
-
-
-def get_upgrade_packages():
-    return [
-        "rsync",
-        "postgresql-13",
-        "postgresql-14",
-        "postgresql-15",
-        "postgresql-16",
-    ]
-
-
-def get_upgrade_script():
-    return """
-#!/bin/bash
-set -euo pipefail
-
-get_bin_path() {
-  local version=$1
-  echo "/usr/lib/postgresql/$version/bin"
-}
-
-log() {
-  echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1"
-}
-
-check_writable() {
-  local path=$1
-  if [ ! -w "$path" ]; then
-    log "$path is not writable"
-    exit 1
-  fi
-}
-
-check_writable "$DATA_DIR"
-
-# Don't do anything if its a fresh install.
-if [ ! -f "$DATA_DIR/PG_VERSION" ]; then
-  log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install."
-  exit 0
-fi
-
-# Don't do anything if we're already at the target version.
-OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION")
-log "Current version: $OLD_VERSION"
-log "Target version: $TARGET_VERSION"
-if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then
-  log "Already at target version $TARGET_VERSION"
-  exit 0
-fi
-
-# Fail if we're downgrading.
-if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then
-  log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION"
-  exit 1
-fi
-
-export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION")
-if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then
-  log "File $OLD_PG_BINARY/pg_upgrade does not exist."
-  exit 1
-fi
-
-export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION")
-if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then
-  log "File $NEW_PG_BINARY/pg_upgrade does not exist."
-  exit 1
-fi
-
-export NEW_DATA_DIR="/tmp/new-data-dir"
-if [ -d "$NEW_DATA_DIR" ]; then
-  log "Directory $NEW_DATA_DIR already exists."
-  exit 1
-fi
-
-export PGUSER="$POSTGRES_USER"
-log "Creating new data dir and initializing..."
-PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)"
-
-timestamp=$(date +%Y%m%d%H%M%S)
-backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz"
-log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name"
-tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR"
-
-log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]"
-log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]"
-log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..."
-
-"$NEW_PG_BINARY"/pg_upgrade \
-  --old-bindir="$OLD_PG_BINARY" \
-  --new-bindir="$NEW_PG_BINARY" \
-  --old-datadir="$DATA_DIR" \
-  --new-datadir="$NEW_DATA_DIR" \
-  --socketdir /var/run/postgresql \
-  --check
-
-log "Compatibility check passed."
-
-log "Upgrading from $OLD_VERSION to $TARGET_VERSION..."
-"$NEW_PG_BINARY"/pg_upgrade \
-  --old-bindir="$OLD_PG_BINARY" \
-  --new-bindir="$NEW_PG_BINARY" \
-  --old-datadir="$DATA_DIR" \
-  --new-datadir="$NEW_DATA_DIR" \
-  --socketdir /var/run/postgresql
-
-log "Upgrade complete."
-
-log "Copying old pg_hba.conf to new pg_hba.conf"
-# We need to carry this over otherwise
-cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf"
-
-log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)."
-rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/"
-
-log "Removing $NEW_DATA_DIR."
-rm -rf "$NEW_DATA_DIR"
-
-log "Done."
-"""

library/2.1.16/render.py

@@ -43,6 +43,12 @@ class Render(object):
         if "python_permissions_image" not in self.values["images"]:
             self.values["images"]["python_permissions_image"] = {"repository": "python", "tag": "3.13.0-slim-bookworm"}
 
+        if "postgres_upgrade_image" not in self.values["images"]:
+            self.values["images"]["postgres_upgrade_image"] = {
+                "repository": "ixsystems/postgres-upgrade",
+                "tag": "1.0.0",
+            }
+
     def container_names(self):
         return list(self._containers.keys())
 

library/2.1.16/storage.py

@@ -1,6 +1,7 @@
 from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union
 
 if TYPE_CHECKING:
+    from container import Container
     from render import Render
 
 try:
@@ -16,6 +17,8 @@ except ImportError:
 class IxStorageTmpfsConfig(TypedDict):
     size: NotRequired[int]
     mode: NotRequired[str]
+    uid: NotRequired[int]
+    gid: NotRequired[int]
 
 
 class AclConfig(TypedDict, total=False):
@@ -79,18 +82,24 @@ class IxStorage(TypedDict):
 
 
 class Storage:
-    def __init__(self, render_instance: "Render"):
+    def __init__(self, render_instance: "Render", container_instance: "Container"):
+        self._container_instance = container_instance
         self._render_instance = render_instance
         self._volume_mounts: set[VolumeMount] = set()
 
     def add(self, mount_path: str, config: "IxStorage"):
         mount_path = valid_fs_path_or_raise(mount_path)
-        if mount_path in [m.mount_path for m in self._volume_mounts]:
+        if self.is_defined(mount_path):
+            raise RenderError(f"Mount path [{mount_path}] already used for another volume mount")
+        if self._container_instance._tmpfs.is_defined(mount_path):
             raise RenderError(f"Mount path [{mount_path}] already used for another volume mount")
 
         volume_mount = VolumeMount(self._render_instance, mount_path, config)
         self._volume_mounts.add(volume_mount)
 
+    def is_defined(self, mount_path: str):
+        return mount_path in [m.mount_path for m in self._volume_mounts]
+
     def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""):
         mount_path = valid_fs_path_or_raise(mount_path)
         cfg: "IxStorage" = {

library/2.1.16/tests/test_deps.py

@@ -472,6 +472,5 @@ def test_postgres_with_upgrade_container(mock_values):
     assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}}
     assert pgup["restart"] == "on-failure:1"
     assert pgup["healthcheck"] == {"disable": True}
-    assert pgup["build"]["dockerfile_inline"] != ""
-    assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh"
+    assert pgup["image"] == "ixsystems/postgres-upgrade:1.0.0"
     assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"]

library/2.1.16/tests/test_volumes.py

@@ -535,18 +535,37 @@ def test_tmpfs_volume(mock_values):
     render = Render(mock_values)
     c1 = render.add_container("test_container", "test_image")
     c1.healthcheck.disable()
-    vol_config = {"type": "tmpfs"}
-    c1.add_storage("/some/path", vol_config)
+    c1.add_storage("/some/path", {"type": "tmpfs"})
+    c1.add_storage("/some/other/path", {"type": "tmpfs", "tmpfs_config": {"size": 100}})
+    c1.add_storage(
+        "/some/other/path2", {"type": "tmpfs", "tmpfs_config": {"size": 100, "mode": "0777", "uid": 1000, "gid": 1000}}
+    )
     output = render.render()
-    assert output["services"]["test_container"]["volumes"] == [
-        {
-            "type": "tmpfs",
-            "target": "/some/path",
-            "read_only": False,
-        }
+    assert output["services"]["test_container"]["tmpfs"] == [
+        "/some/other/path2:gid=1000,mode=0777,size=104857600,uid=1000",
+        "/some/other/path:size=104857600",
+        "/some/path",
     ]
 
 
+def test_add_tmpfs_with_existing_volume(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    c1.add_storage("/some/path", {"type": "volume", "volume_config": {"volume_name": "test_volume"}})
+    with pytest.raises(Exception):
+        c1.add_storage("/some/path", {"type": "tmpfs", "tmpfs_config": {"size": 100}})
+
+
+def test_add_volume_with_existing_tmpfs(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    c1.add_storage("/some/path", {"type": "tmpfs", "tmpfs_config": {"size": 100}})
+    with pytest.raises(Exception):
+        c1.add_storage("/some/path", {"type": "volume", "volume_config": {"volume_name": "test_volume"}})
+
+
 def test_temporary_volume(mock_values):
     render = Render(mock_values)
     c1 = render.add_container("test_container", "test_image")

library/2.1.16/tmpfs.py

@@ -0,0 +1,75 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from container import Container
+    from render import Render
+    from storage import IxStorage
+
+try:
+    from .error import RenderError
+    from .validations import valid_fs_path_or_raise, valid_octal_mode_or_raise
+except ImportError:
+    from error import RenderError
+    from validations import valid_fs_path_or_raise, valid_octal_mode_or_raise
+
+
+class Tmpfs:
+
+    def __init__(self, render_instance: "Render", container_instance: "Container"):
+        self._render_instance = render_instance
+        self._container_instance = container_instance
+        self._tmpfs: dict = {}
+
+    def add(self, mount_path: str, config: "IxStorage"):
+        mount_path = valid_fs_path_or_raise(mount_path)
+        if self.is_defined(mount_path):
+            raise RenderError(f"Tmpfs mount path [{mount_path}] already added")
+
+        if self._container_instance.storage.is_defined(mount_path):
+            raise RenderError(f"Tmpfs mount path [{mount_path}] already used for another volume mount")
+
+        mount_config = config.get("tmpfs_config", {})
+        size = mount_config.get("size", None)
+        mode = mount_config.get("mode", None)
+        uid = mount_config.get("uid", None)
+        gid = mount_config.get("gid", None)
+
+        if size is not None:
+            if not isinstance(size, int):
+                raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]")
+            if not size > 0:
+                raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]")
+            # Convert Mebibytes to Bytes
+            size = size * 1024 * 1024
+
+        if mode is not None:
+            mode = valid_octal_mode_or_raise(mode)
+
+        if uid is not None and not isinstance(uid, int):
+            raise RenderError(f"Expected [uid] to be an integer for [tmpfs] type, got [{uid}]")
+
+        if gid is not None and not isinstance(gid, int):
+            raise RenderError(f"Expected [gid] to be an integer for [tmpfs] type, got [{gid}]")
+
+        self._tmpfs[mount_path] = {}
+        if size is not None:
+            self._tmpfs[mount_path]["size"] = str(size)
+        if mode is not None:
+            self._tmpfs[mount_path]["mode"] = str(mode)
+        if uid is not None:
+            self._tmpfs[mount_path]["uid"] = str(uid)
+        if gid is not None:
+            self._tmpfs[mount_path]["gid"] = str(gid)
+
+    def is_defined(self, mount_path: str):
+        return mount_path in self._tmpfs
+
+    def has_tmpfs(self):
+        return bool(self._tmpfs)
+
+    def render(self):
+        result = []
+        for mount_path, config in self._tmpfs.items():
+            opts = sorted([f"{k}={v}" for k, v in config.items()])
+            result.append(f"{mount_path}:{','.join(opts)}" if opts else mount_path)
+        return sorted(result)

library/2.1.16/volume_mount.py

@@ -7,12 +7,12 @@ if TYPE_CHECKING:
 try:
     from .error import RenderError
     from .formatter import merge_dicts_no_overwrite
-    from .volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType
+    from .volume_mount_types import BindMountType, VolumeMountType
     from .volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource
 except ImportError:
     from error import RenderError
     from formatter import merge_dicts_no_overwrite
-    from volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType
+    from volume_mount_types import BindMountType, VolumeMountType
     from volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource
 
 
@@ -40,11 +40,6 @@ class VolumeMount:
                     raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.")
                 mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render()
                 source = IxVolumeSource(self._render_instance, mount_config).get()
-            case "tmpfs":
-                spec_type = "tmpfs"
-                mount_config = config.get("tmpfs_config", {})
-                mount_type_specific_definition = TmpfsMountType(self._render_instance, mount_config).render()
-                source = None
             case "nfs":
                 spec_type = "volume"
                 mount_config = config.get("nfs_config")

library/2.1.16/volume_mount_types.py

@@ -0,0 +1,43 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from render import Render
+    from storage import IxStorageVolumeConfig, IxStorageBindLikeConfigs
+
+
+try:
+    from .validations import valid_host_path_propagation
+except ImportError:
+    from validations import valid_host_path_propagation
+
+
+class BindMountType:
+    def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"):
+        self._render_instance = render_instance
+        self.spec: dict = {}
+
+        propagation = valid_host_path_propagation(config.get("propagation", "rprivate"))
+        create_host_path = config.get("create_host_path", False)
+
+        self.spec: dict = {
+            "bind": {
+                "create_host_path": create_host_path,
+                "propagation": propagation,
+            }
+        }
+
+    def render(self) -> dict:
+        """Render the bind mount specification."""
+        return self.spec
+
+
+class VolumeMountType:
+    def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"):
+        self._render_instance = render_instance
+        self.spec: dict = {}
+
+        self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}}
+
+    def render(self) -> dict:
+        """Render the volume mount specification."""
+        return self.spec

library/2.1.16/volumes.py

@@ -19,12 +19,7 @@ class Volumes:
         self._render_instance = render_instance
         self._volumes: dict[str, Volume] = {}
 
-    def add_volume(
-        self,
-        source: str,
-        storage_type: str,
-        config: "IxStorageVolumeLikeConfigs",
-    ):
+    def add_volume(self, source: str, storage_type: str, config: "IxStorageVolumeLikeConfigs"):
         # This method can be called many times from the volume mounts
         # Only add the volume if it is not already added, but dont raise an error
         if source == "":

library/hashes.yaml

@@ -1,2 +1,2 @@
 0.0.1: f074617a82a86d2a6cc78a4c8a4296fc9d168e456f12713e50c696557b302133
-2.1.15: e26cffb91766782c787867b379519f7407b1fed8450dce463cefa56340a41d84
+2.1.16: 488c896e0dc7d49e8023e527a0769bfb7cb97a29899fba058fcee81bd323212b