Add support for postgis/pgvectors db upgrade (#3752)

* set supported repos

* update tests

* dawarich: add support for pg18

* postgis: renovate

* adventurelog: add support for pg 18

* pgvectors

* librechat: add support for postgres 18

* chatwoot: enable support for pg 18

* renovate

* fix test - update lib

* remove suffix

* bump

.github/renovate-config.js

@@ -258,9 +258,14 @@ module.exports = {
     ),
     customVersioning(
       // 17-3.5
-      "^17-(?<major>\\d+)\\.(?<minor>\\d+)$",
+      "^(17|18)-(?<major>\\d+)\\.(?<minor>\\d+)$",
       ["postgis/postgis"]
     ),
+    customVersioning(
+      // 0.8.1-pg18-trixie
+      "^(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)-pg(17|18)(-\\w+)?$",
+      ["pgvector/pgvector"]
+    ),
     customVersioning(
       // 2.4-dev
       "^(?<major>\\d+)\\.(?<minor>\\d+)-dev$",

ix-dev/community/adventurelog/app.yaml

@@ -23,8 +23,8 @@ keywords:
 - location
 - tracker
 - planner
-lib_version: 2.1.65
-lib_version_hash: f92a9ee78c78fc77f86e7d8b545bd4c605c31c599e2c5da59f1615aa516cb8b5
+lib_version: 2.1.66
+lib_version_hash: 986abcd60fd0cc6277c80d55d2798356ddda1bd6348580ca7198b003571707fa
 maintainers:
 - email: dev@truenas.com
   name: truenas
@@ -51,4 +51,4 @@ sources:
 - https://github.com/seanmorley15/AdventureLog
 title: AdventureLog
 train: community
-version: 1.0.8
+version: 1.1.0

ix-dev/community/adventurelog/app_migrations.yaml

@@ -0,0 +1,6 @@
+migrations:
+- file: set_default_postgis_image
+  from:
+    max_version: 1.0.8
+  target:
+    min_version: 1.1.0

ix-dev/community/adventurelog/ix_values.yaml

@@ -8,9 +8,15 @@ images:
   frontend_image:
     repository: ghcr.io/seanmorley15/adventurelog-frontend
     tag: v0.11.0
-  postgis_image:
+  postgis_17_image:
     repository: postgis/postgis
     tag: "17-3.5"
+  postgis_18_image:
+    repository: postgis/postgis
+    tag: "18-3.6"
+  postgres_upgrade_image:
+    repository: ixsystems/postgres-upgrade
+    tag: 1.1.3
 
 consts:
   backend_container_name: backend

ix-dev/community/adventurelog/migrations/set_default_postgis_image

@@ -0,0 +1,20 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import yaml
+
+
+def migrate(values):
+    if not values["adventurelog"].get("postgres_image_selector"):
+        values["adventurelog"]["postgres_image_selector"] = "postgis_17_image"
+    return values
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        exit(1)
+
+    if os.path.exists(sys.argv[1]):
+        with open(sys.argv[1], "r") as f:
+            print(yaml.dump(migrate(yaml.safe_load(f.read()))))

ix-dev/community/adventurelog/questions.yaml

@@ -26,6 +26,24 @@ questions:
     schema:
       type: dict
       attrs:
+        - variable: postgres_image_selector
+          label: Postgres Image (CAUTION)
+          description: |
+            If you are changing this after the postgres directory has been initialized,</br>
+            STOP! and make sure you have a backup of your data.</br>
+            Changing this will trigger an one way database upgrade.</br>
+            You can only select newer versions of postgres.</br>
+            Selecting an older version will refuse to start.</br>
+            If something goes wrong, you will have to restore from backup.
+          schema:
+            type: string
+            default: postgis_18_image
+            required: true
+            enum:
+              - value: postgis_17_image
+                description: Postgres 17
+              - value: postgis_18_image
+                description: Postgres 18
         - variable: db_password
           label: Database Password
           description: The password for AdventureLog.

ix-dev/community/adventurelog/templates/docker-compose.yaml

@@ -7,7 +7,7 @@
 
 {# Postgis #}
 {% set pg_config = {"user": values.consts.db_user, "password": values.adventurelog.db_password, "database": values.consts.db_name, "volume": values.storage.postgres_data} %}
-{% set postgis = tpl.deps.postgres(values.consts.postgis_container_name, "postgis_image", pg_config, perm_container) %}
+{% set postgis = tpl.deps.postgres(values.consts.postgis_container_name, values.adventurelog.postgres_image_selector, pg_config, perm_container) %}
 
 {# Frontend #}
 {% do frontend.set_user(1000, 1000) %}

ix-dev/community/adventurelog/templates/library/base_v2_1_66/deps_postgres.py

@@ -34,6 +34,11 @@ SUPPORTED_REPOS = [
     "pgvector/pgvector",
     "ghcr.io/immich-app/postgres",
 ]
+SUPPORTED_UPGRADE_REPOS = [
+    "postgres",
+    "postgis/postgis",
+    "pgvector/pgvector",
+]
 
 
 def get_major_version(variant: str, tag: str):
@@ -52,15 +57,16 @@ def get_major_version(variant: str, tag: str):
             return x.split("-")[0]
 
     elif variant == "pgvector/pgvector":
-        # 0.8.1-pg17
-        regex = re.compile(r"^\d+\.\d+\.\d+\-pg\d+")
+        # 0.8.1-pg17-trixie
+        regex = re.compile(r"^\d+\.\d+\.\d+\-pg\d+(\-\w+)?")
 
         def oper(x):
-            return x.split("-")[1].lstrip("pg")
+            parts = x.split("-")
+            return parts[1].lstrip("pg")
 
     elif variant == "ghcr.io/immich-app/postgres":
-        # 15-vectorchord0.4.3-pgvectors0.2.0
-        regex = re.compile(r"^\d+\-vectorchord\d+\.\d+\.\d+(\-pgvectors?\d+\.\d+\.\d+)?")
+        # 15-vectorchord0.4.3
+        regex = re.compile(r"^\d+\-vectorchord\d+\.\d+\.\d+")
 
         def oper(x):
             return x.split("-")[0]
@@ -113,7 +119,7 @@ class PostgresContainer:
 
         # eg we don't want to handle upgrades of pg_vector or immich at the moment
         repo = self._get_repo(image)
-        if repo == "postgres":
+        if repo in SUPPORTED_UPGRADE_REPOS:
             self._data_dir = "/var/lib/postgresql"
             target_major_version = self._get_target_version(image)
             # This is the new format upstream Postgres uses/suggests.

ix-dev/community/adventurelog/templates/library/base_v2_1_66/tests/test_deps.py

@@ -537,7 +537,10 @@ def test_add_postgres_with_invalid_tag(mock_values):
 
 
 def test_no_upgrade_container_with_non_postgres_image(mock_values):
-    mock_values["images"]["postgres_image"] = {"repository": "pgvector/pgvector", "tag": "0.8.1-pg17"}
+    mock_values["images"]["postgres_image"] = {
+        "repository": "ghcr.io/immich-app/postgres",
+        "tag": "15-vectorchord0.4.3-pgvectors0.2.0",
+    }
     render = Render(mock_values)
     c1 = render.add_container("test_container", "test_image")
     c1.healthcheck.disable()
@@ -605,7 +608,7 @@ def test_postgres_with_upgrade_container(mock_values):
 
 
 def test_add_mongodb(mock_values):
-    mock_values["images"]["mongodb_image"] = {"repository": "mongodb", "tag": "latest"}
+    mock_values["images"]["mongodb_image"] = {"repository": "mongo", "tag": "latest"}
     render = Render(mock_values)
     c1 = render.add_container("test_container", "test_image")
     c1.healthcheck.disable()
@@ -627,7 +630,7 @@ def test_add_mongodb(mock_values):
     output = render.render()
     assert "devices" not in output["services"]["mongodb_container"]
     assert "reservations" not in output["services"]["mongodb_container"]["deploy"]["resources"]
-    assert output["services"]["mongodb_container"]["image"] == "mongodb:latest"
+    assert output["services"]["mongodb_container"]["image"] == "mongo:latest"
     assert output["services"]["mongodb_container"]["user"] == "568:568"
     assert output["services"]["mongodb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0"
     assert output["services"]["mongodb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M"
@@ -781,7 +784,7 @@ def test_add_meilisearch_unsupported_repo(mock_values):
 
 def test_add_elasticsearch(mock_values):
     mock_values["images"]["elastic_image"] = {
-        "repository": "docker.elastic.co/elasticsearch/elasticsearch",
+        "repository": "elasticsearch",
         "tag": "9.1.2",
     }
     render = Render(mock_values)
@@ -804,7 +807,7 @@ def test_add_elasticsearch(mock_values):
     output = render.render()
     assert "devices" not in output["services"]["elastic_container"]
     assert "reservations" not in output["services"]["elastic_container"]["deploy"]["resources"]
-    assert output["services"]["elastic_container"]["image"] == "docker.elastic.co/elasticsearch/elasticsearch:9.1.2"
+    assert output["services"]["elastic_container"]["image"] == "elasticsearch:9.1.2"
     assert output["services"]["elastic_container"]["user"] == "1000:1000"
     assert output["services"]["elastic_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0"
     assert output["services"]["elastic_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M"

ix-dev/community/adventurelog/templates/test_values/basic-values.yaml

@@ -6,6 +6,7 @@ resources:
 TZ: Etc/UTC
 
 adventurelog:
+  postgres_image_selector: postgis_18_image
   db_password: secret
   backend_url: http://localhost:8080
   frontend_url: http://localhost:8081

ix-dev/community/chatwoot/app.yaml

@@ -12,8 +12,8 @@ icon: https://media.sys.truenas.net/apps/chatwoot/icons/icon.svg
 keywords:
 - support
 - live chat
-lib_version: 2.1.65
-lib_version_hash: f92a9ee78c78fc77f86e7d8b545bd4c605c31c599e2c5da59f1615aa516cb8b5
+lib_version: 2.1.66
+lib_version_hash: 986abcd60fd0cc6277c80d55d2798356ddda1bd6348580ca7198b003571707fa
 maintainers:
 - email: dev@truenas.com
   name: truenas
@@ -41,4 +41,4 @@ sources:
 - https://hub.docker.com/r/chatwoot/chatwoot
 title: Chatwoot
 train: community
-version: 1.0.29
+version: 1.0.30

ix-dev/community/chatwoot/ix_values.yaml

@@ -14,9 +14,12 @@ images:
   postgres_17_image:
     repository: pgvector/pgvector
     tag: 0.8.1-pg17
+  postgres_18_image:
+    repository: pgvector/pgvector
+    tag: 0.8.1-pg18-trixie
   postgres_upgrade_image:
     repository: ixsystems/postgres-upgrade
-    tag: 1.1.2
+    tag: 1.1.3
 
 consts:
   chatwoot_container_name: chatwoot

ix-dev/community/chatwoot/questions.yaml

@@ -50,11 +50,13 @@ questions:
             If something goes wrong, you will have to restore from backup.
           schema:
             type: string
-            default: postgres_17_image
+            default: postgres_18_image
             required: true
             enum:
               - value: postgres_17_image
                 description: Postgres 17
+              - value: postgres_18_image
+                description: Postgres 18
         - variable: db_password
           label: Database Password
           description: The password for Chatwoot.

ix-dev/community/chatwoot/templates/library/base_v2_1_66/deps_postgres.py

@@ -34,6 +34,11 @@ SUPPORTED_REPOS = [
     "pgvector/pgvector",
     "ghcr.io/immich-app/postgres",
 ]
+SUPPORTED_UPGRADE_REPOS = [
+    "postgres",
+    "postgis/postgis",
+    "pgvector/pgvector",
+]
 
 
 def get_major_version(variant: str, tag: str):
@@ -52,15 +57,16 @@ def get_major_version(variant: str, tag: str):
             return x.split("-")[0]
 
     elif variant == "pgvector/pgvector":
-        # 0.8.1-pg17
-        regex = re.compile(r"^\d+\.\d+\.\d+\-pg\d+")
+        # 0.8.1-pg17-trixie
+        regex = re.compile(r"^\d+\.\d+\.\d+\-pg\d+(\-\w+)?")
 
         def oper(x):
-            return x.split("-")[1].lstrip("pg")
+            parts = x.split("-")
+            return parts[1].lstrip("pg")
 
     elif variant == "ghcr.io/immich-app/postgres":
-        # 15-vectorchord0.4.3-pgvectors0.2.0
-        regex = re.compile(r"^\d+\-vectorchord\d+\.\d+\.\d+(\-pgvectors?\d+\.\d+\.\d+)?")
+        # 15-vectorchord0.4.3
+        regex = re.compile(r"^\d+\-vectorchord\d+\.\d+\.\d+")
 
         def oper(x):
             return x.split("-")[0]
@@ -113,7 +119,7 @@ class PostgresContainer:
 
         # eg we don't want to handle upgrades of pg_vector or immich at the moment
         repo = self._get_repo(image)
-        if repo == "postgres":
+        if repo in SUPPORTED_UPGRADE_REPOS:
             self._data_dir = "/var/lib/postgresql"
             target_major_version = self._get_target_version(image)
             # This is the new format upstream Postgres uses/suggests.