opencloud: adds support for tika/fts (#3384)

* lib: add tika dep

* update hash for version 2.1.58 in hashes.yaml

* fix

* opencloud: adds support for tika/fts

* lib

* add missing value

* add migration for removing host network configuration and update related templates

* add container

ix-dev/community/opencloud/app.yaml

@@ -12,8 +12,8 @@ icon: https://media.sys.truenas.net/apps/opencloud/icons/icon.svg
 keywords:
 - file sharing
 - collaboration
-lib_version: 2.1.57
-lib_version_hash: 6659ad369fff2f1df318cb6353bb32bffe7047dc21df4823750d39be7284e605
+lib_version: 2.1.58
+lib_version_hash: 52ea7a7fe35fb3cbf1d27f53df6003f0601881dbdd3313f32b5c3417894c622b
 maintainers:
 - email: dev@truenas.com
   name: truenas
@@ -25,6 +25,11 @@ run_as_context:
   group_name: opencloud
   uid: 568
   user_name: opencloud
+- description: Tika runs as a non-root user.
+  gid: 568
+  group_name: tika
+  uid: 568
+  user_name: tika
 screenshots: []
 sources:
 - https://docs.opencloud.eu/
@@ -32,4 +37,4 @@ sources:
 - https://hub.docker.com/r/opencloudeu/opencloud-rolling
 title: OpenCloud
 train: community
-version: 1.0.10
+version: 1.0.11

ix-dev/community/opencloud/app_migrations.yaml

@@ -0,0 +1,6 @@
+migrations:
+- file: remove_host_net_migration
+  from:
+    max_version: 1.0.10
+  target:
+    min_version: 1.0.11

ix-dev/community/opencloud/ix_values.yaml

@@ -2,6 +2,9 @@ images:
   image:
     repository: opencloudeu/opencloud-rolling
     tag: 3.5.0
+  tika_image:
+    repository: apache/tika
+    tag: "3.2.3.0-full"
   drawio_image:
     repository: opencloudeu/web-extensions
     tag: draw-io-1.0.0
@@ -32,8 +35,11 @@ images:
 
 consts:
   opencloud_container_name: opencloud
+  tika_container_name: tika
   perms_container_name: permissions
 
+  tika_port: 9998
+
   notes_body: |
     Default Credential:
       - Username: `admin`

ix-dev/community/opencloud/migrations/remove_host_net_migration

@@ -0,0 +1,24 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import yaml
+
+
+def migrate(values):
+    if values["network"].get("host_network", False):
+        raise Exception(
+            "Host network is not a valid configuration in Opencloud. Please remove it and retry the upgrade."
+        )
+
+    values["network"].pop("host_network", None)
+    return values
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        exit(1)
+
+    if os.path.exists(sys.argv[1]):
+        with open(sys.argv[1], "r") as f:
+            print(yaml.dump(migrate(yaml.safe_load(f.read()))))

ix-dev/community/opencloud/questions.yaml

@@ -46,6 +46,16 @@ questions:
           schema:
             type: boolean
             default: true
+        - variable: full_text_search
+          label: Full Text Search
+          schema:
+            type: dict
+            attrs:
+              - variable: enabled
+                label: Enabled
+                schema:
+                  type: boolean
+                  default: true
         - variable: smtp
           label: SMTP
           schema:
@@ -263,13 +273,53 @@ questions:
                         required: true
                         $ref:
                           - definitions/node_bind_ip
-        - variable: host_network
-          label: Host Network
-          description: |
-            Bind to the host network. It's recommended to keep this disabled.
+        - variable: metrics_port
+          label: Metrics Port
           schema:
-            type: boolean
-            default: false
+            type: dict
+            attrs:
+              - variable: bind_mode
+                label: Port Bind Mode
+                description: |
+                  The port bind mode.</br>
+                  - Publish: The port will be published on the host for external access.</br>
+                  - Expose: The port will be exposed for inter-container communication.</br>
+                  - None: The port will not be exposed or published.</br>
+                  Note: If the Dockerfile defines an EXPOSE directive,
+                  the port will still be exposed for inter-container communication regardless of this setting.
+                schema:
+                  type: string
+                  default: ""
+                  enum:
+                    - value: "published"
+                      description: Publish port on the host for external access
+                    - value: "exposed"
+                      description: Expose port for inter-container communication
+                    - value: ""
+                      description: None
+              - variable: port_number
+                label: Port Number
+                schema:
+                  type: int
+                  default: 30268
+                  min: 1
+                  max: 65535
+                  required: true
+              - variable: host_ips
+                label: Host IPs
+                description: IPs on the host to bind this port
+                schema:
+                  type: list
+                  show_if: [["bind_mode", "=", "published"]]
+                  default: []
+                  items:
+                    - variable: host_ip
+                      label: Host IP
+                      schema:
+                        type: string
+                        required: true
+                        $ref:
+                          - definitions/node_bind_ip
   - variable: storage
     label: ""
     group: Storage Configuration
@@ -618,6 +668,8 @@ questions:
                         enum:
                           - value: opencloud
                             description: opencloud
+                          - value: tika
+                            description: tika
   - variable: resources
     label: ""
     group: Resources Configuration

ix-dev/community/opencloud/templates/docker-compose.yaml

@@ -46,8 +46,19 @@
 
 {% set extra_services = namespace(x=["notifications"]) %}
 
+{% if values.opencloud.full_text_search.enabled %}
+  {% set tika = tpl.deps.tika(values.consts.tika_container_name, "tika_image", {"port": values.consts.tika_port}) %}
+  {% do c1.depends.add_dependency(values.consts.tika_container_name, "service_healthy") %}
+  {% do c1.environment.add_env("FRONTEND_FULL_TEXT_SEARCH_ENABLED", true) %}
+  {% do c1.environment.add_env("SEARCH_EXTRACTOR_TYPE", "tika") %}
+  {% do c1.environment.add_env("SEARCH_EXTRACTOR_TIKA_TIKA_URL", tika.get_url()) %}
+{% endif %}
+
 {# Networking #}
 {% do c1.environment.add_env("PROXY_HTTP_ADDR", ":%d"|format(values.network.web_port.port_number)) %}
+{% if values.network.metrics_port.bind_mode %}
+  {% do c1.environment.add_env("PROXY_DEBUG_ADDR", ":%d"|format(values.network.metrics_port.port_number)) %}
+{% endif %}
 {% do c1.environment.add_env("OC_URL", values.opencloud.app_url) %}
 {% do c1.environment.add_env("OC_INSECURE", values.opencloud.insecure_access) %}
 

ix-dev/community/opencloud/templates/library/base_v2_1_58/deps.py

@@ -12,6 +12,7 @@ try:
     from .deps_postgres import PostgresContainer, PostgresConfig
     from .deps_redis import RedisContainer, RedisConfig
     from .deps_solr import SolrContainer, SolrConfig
+    from .deps_tika import TikaContainer, TikaConfig
 except ImportError:
     from deps_elastic import ElasticSearchContainer, ElasticConfig
     from deps_mariadb import MariadbContainer, MariadbConfig
@@ -21,6 +22,7 @@ except ImportError:
     from deps_postgres import PostgresContainer, PostgresConfig
     from deps_redis import RedisContainer, RedisConfig
     from deps_solr import SolrContainer, SolrConfig
+    from deps_tika import TikaContainer, TikaConfig
 
 
 class Deps:
@@ -50,3 +52,6 @@ class Deps:
 
     def solr(self, name: str, image: str, config: SolrConfig, perms_instance: PermsContainer):
         return SolrContainer(self._render_instance, name, image, config, perms_instance)
+
+    def tika(self, name: str, image: str, config: TikaConfig):
+        return TikaContainer(self._render_instance, name, image, config)

ix-dev/community/opencloud/templates/library/base_v2_1_58/deps_tika.py

@@ -0,0 +1,63 @@
+from typing import TYPE_CHECKING, TypedDict, NotRequired
+
+
+if TYPE_CHECKING:
+    from render import Render
+
+
+try:
+    from .error import RenderError
+except ImportError:
+    from error import RenderError
+
+
+class TikaConfig(TypedDict):
+    port: NotRequired[int]
+
+
+class TikaContainer:
+    def __init__(self, render_instance: "Render", name: str, image: str, config: TikaConfig):
+        self._render_instance = render_instance
+        self._name = name
+        self._config = config
+
+        c = self._render_instance.add_container(name, image)
+
+        user, group = 568, 568
+        run_as = self._render_instance.values.get("run_as")
+        if run_as:
+            user = run_as["user"] or user  # Avoids running as root
+            group = run_as["group"] or group  # Avoids running as root
+
+        c.set_user(user, group)
+        c.healthcheck.set_test("wget", {"port": self.get_port(), "path": "/tika", "spider": False})
+        c.remove_devices()
+
+        c.set_command(["--port", str(self.get_port())])
+
+        self._get_repo(image, ("apache/tika"))
+
+        # Store container for further configuration
+        # For example: c.depends.add_dependency("other_container", "service_started")
+        self._container = c
+
+    @property
+    def container(self):
+        return self._container
+
+    def _get_repo(self, image, supported_repos):
+        images = self._render_instance.values["images"]
+        if image not in images:
+            raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]")
+        repo = images[image].get("repository")
+        if not repo:
+            raise RenderError("Could not determine repo")
+        if repo not in supported_repos:
+            raise RenderError(f"Unsupported repo [{repo}] for tika. Supported repos: {', '.join(supported_repos)}")
+        return repo
+
+    def get_port(self):
+        return self._config.get("port") or 9998
+
+    def get_url(self):
+        return f"http://{self._name}:{self.get_port()}"

ix-dev/community/opencloud/templates/library/base_v2_1_58/tests/test_deps.py

@@ -952,3 +952,59 @@ def test_add_solr_unsupported_repo(mock_values):
             },
             perms_container,
         )
+
+
+def test_add_tika(mock_values):
+    mock_values["images"]["tika_image"] = {"repository": "apache/tika", "tag": "3.2.3.0-full"}
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    render.deps.tika(
+        "tika_container",
+        "tika_image",
+        {
+            "port": 10999,
+        },
+    )
+    output = render.render()
+    assert "devices" not in output["services"]["tika_container"]
+    assert "reservations" not in output["services"]["tika_container"]["deploy"]["resources"]
+    assert output["services"]["tika_container"]["image"] == "apache/tika:3.2.3.0-full"
+    assert output["services"]["tika_container"]["user"] == "568:568"
+    assert output["services"]["tika_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0"
+    assert output["services"]["tika_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M"
+    assert output["services"]["tika_container"]["healthcheck"] == {
+        "test": [
+            "CMD",
+            "wget",
+            "--quiet",
+            "-O",
+            "/dev/null",
+            "http://127.0.0.1:10999/tika",
+        ],
+        "interval": "30s",
+        "timeout": "5s",
+        "retries": 5,
+        "start_period": "15s",
+        "start_interval": "2s",
+    }
+    assert output["services"]["tika_container"]["environment"] == {
+        "TZ": "Etc/UTC",
+        "UMASK": "002",
+        "UMASK_SET": "002",
+        "NVIDIA_VISIBLE_DEVICES": "void",
+    }
+    assert output["services"]["tika_container"]["command"] == ["--port", "10999"]
+
+
+def test_add_tika_unsupported_repo(mock_values):
+    mock_values["images"]["tika_image"] = {"repository": "unsupported_repo", "tag": "7"}
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    with pytest.raises(Exception):
+        render.deps.tika(
+            "tika_container",
+            "tika_image",
+            {},
+        )

ix-dev/community/opencloud/templates/test_values/basic-values.yaml

@@ -6,6 +6,8 @@ resources:
 opencloud:
   app_url: https://localhost:8080
   insecure_access: true
+  full_text_search:
+    enabled: true
   smtp:
     enabled: false
     sender: null
@@ -32,6 +34,9 @@ network:
   web_port:
     bind_mode: published
     port_number: 8080
+  metrics_port:
+    bind_mode: published
+    port_number: 8081
   certificate_id: null
 
 run_as:

ix-dev/community/opencloud/templates/test_values/https-values.yaml

@@ -6,6 +6,8 @@ resources:
 opencloud:
   app_url: https://10.20.30.6:8080
   insecure_access: true
+  full_text_search:
+    enabled: true
   smtp:
     enabled: false
     sender: null
@@ -32,6 +34,9 @@ network:
   web_port:
     bind_mode: published
     port_number: 8080
+  metrics_port:
+    bind_mode: ""
+    port_number: 8081
   certificate_id: "1"
 
 run_as:
@@ -57,7 +62,6 @@ storage:
       create_host_path: true
   additional_storage: []
 
-
 ix_certificates:
   "1":
     certificate: |