misc: finalized config files

.env.example

@@ -64,4 +64,4 @@ CLIENT_SECRET_GITHUB_LOGIN=
 CLIENT_ID_GITLAB_LOGIN=
 CLIENT_SECRET_GITLAB_LOGIN=
 
-TELEMETRY_EXPORT_URL=
+OTEL_COLLECTOR_OTLP_URL=

backend/package-lock.json

@@ -28,6 +28,7 @@
         "@opentelemetry/api": "^1.8.0",
         "@opentelemetry/auto-instrumentations-node": "^0.46.1",
         "@opentelemetry/exporter-metrics-otlp-proto": "^0.51.1",
+        "@opentelemetry/exporter-prometheus": "^0.51.1",
         "@opentelemetry/instrumentation": "^0.51.1",
         "@opentelemetry/resources": "^1.24.1",
         "@opentelemetry/sdk-metrics": "^1.24.1",
@@ -3484,6 +3485,22 @@
         "@opentelemetry/api": "^1.3.0"
       }
     },
+    "node_modules/@opentelemetry/exporter-prometheus": {
+      "version": "0.51.1",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.51.1.tgz",
+      "integrity": "sha512-c8TrTlLm9JJRIHW6MtFv6ESoZRgXBXD/YrTRYylWiyYBOVbYHo1c5Qaw/j/thXDhkmYOYAn4LAhJZpLl5gBFEQ==",
+      "dependencies": {
+        "@opentelemetry/core": "1.24.1",
+        "@opentelemetry/resources": "1.24.1",
+        "@opentelemetry/sdk-metrics": "1.24.1"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
     "node_modules/@opentelemetry/exporter-trace-otlp-grpc": {
       "version": "0.51.1",
       "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.51.1.tgz",

backend/package.json

@@ -89,6 +89,7 @@
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/auto-instrumentations-node": "^0.46.1",
     "@opentelemetry/exporter-metrics-otlp-proto": "^0.51.1",
+    "@opentelemetry/exporter-prometheus": "^0.51.1",
     "@opentelemetry/instrumentation": "^0.51.1",
     "@opentelemetry/resources": "^1.24.1",
     "@opentelemetry/sdk-metrics": "^1.24.1",

backend/src/lib/config/env.ts

@@ -120,7 +120,7 @@ const envSchema = z
       .optional(),
     INFISICAL_CLOUD: zodStrBool.default("false"),
     MAINTENANCE_MODE: zodStrBool.default("false"),
-    TELEMETRY_EXPORT_URL: zpStr(z.string().optional())
+    OTEL_COLLECTOR_OTLP_URL: zpStr(z.string().optional())
   })
   .transform((data) => ({
     ...data,

backend/src/lib/telemetry/instrumentation.ts

@@ -1,12 +1,13 @@
 import opentelemetry from "@opentelemetry/api";
 import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
 import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
+import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
 import { registerInstrumentations } from "@opentelemetry/instrumentation";
 import { Resource } from "@opentelemetry/resources";
 import { AggregationTemporality, MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
 import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
 
-export const initTelemetry = (exportURL: string) => {
+export const initTelemetry = async ({ otlpURL }: { otlpURL?: string }) => {
   const resource = Resource.default().merge(
     new Resource({
       [SEMRESATTRS_SERVICE_NAME]: "infisical-server",
@@ -14,22 +15,30 @@ export const initTelemetry = (exportURL: string) => {
     })
   );
 
-  const metricExporter = new OTLPMetricExporter({
+  const metricReaders = [];
-    url: `${exportURL}/v1/metrics`,
+  if (otlpURL) {
-    temporalityPreference: AggregationTemporality.DELTA
+    const otlpExporter = new OTLPMetricExporter({
-  });
+      url: `${otlpURL}/v1/metrics`,
+      temporalityPreference: AggregationTemporality.DELTA
+    });
 
-  const metricReader = new PeriodicExportingMetricReader({
+    metricReaders.push(
-    exporter: metricExporter,
+      new PeriodicExportingMetricReader({
-    exportIntervalMillis: 30000
+        exporter: otlpExporter,
-  });
+        exportIntervalMillis: 30000
+      })
+    );
+  } else {
+    const promExporter = new PrometheusExporter();
+    metricReaders.push(promExporter);
+  }
 
-  const myServiceMeterProvider = new MeterProvider({
+  const meterProvider = new MeterProvider({
     resource,
-    readers: [metricReader]
+    readers: metricReaders
   });
 
-  opentelemetry.metrics.setGlobalMeterProvider(myServiceMeterProvider);
+  opentelemetry.metrics.setGlobalMeterProvider(meterProvider);
 
   registerInstrumentations({
     instrumentations: [getNodeAutoInstrumentations()]

backend/src/main.ts

@@ -15,9 +15,7 @@ const run = async () => {
   const logger = await initLogger();
   const appCfg = initEnvConfig(logger);
 
-  if (appCfg.TELEMETRY_EXPORT_URL) {
+  await initTelemetry({ otlpURL: appCfg.OTEL_COLLECTOR_OTLP_URL });
-    initTelemetry(appCfg.TELEMETRY_EXPORT_URL);
-  }
 
   const db = initDbConnection({
     dbConnectionUri: appCfg.DB_CONNECTION_URI,

docker-compose.dev.observable.yml

@@ -0,0 +1,201 @@
+version: "3.9"
+
+services:
+  nginx:
+    container_name: infisical-dev-nginx
+    image: nginx
+    restart: always
+    ports:
+      - 8080:80
+    volumes:
+      - ./nginx/default.dev.conf:/etc/nginx/conf.d/default.conf:ro
+    depends_on:
+      - backend
+      - frontend
+
+  db:
+    image: postgres:14-alpine
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: infisical
+      POSTGRES_USER: infisical
+      POSTGRES_DB: infisical
+
+  redis:
+    image: redis
+    container_name: infisical-dev-redis
+    environment:
+      - ALLOW_EMPTY_PASSWORD=yes
+    ports:
+      - 6379:6379
+    volumes:
+      - redis_data:/data
+
+  redis-commander:
+    container_name: infisical-dev-redis-commander
+    image: rediscommander/redis-commander
+    restart: always
+    depends_on:
+      - redis
+    environment:
+      - REDIS_HOSTS=local:redis:6379
+    ports:
+      - "8085:8081"
+
+  db-test:
+    profiles: ["test"]
+    image: postgres:14-alpine
+    ports:
+      - "5430:5432"
+    environment:
+      POSTGRES_PASSWORD: infisical
+      POSTGRES_USER: infisical
+      POSTGRES_DB: infisical-test
+
+  db-migration:
+    container_name: infisical-db-migration
+    depends_on:
+      - db
+    build:
+      context: ./backend
+      dockerfile: Dockerfile.dev
+    env_file: .env
+    environment:
+      - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable
+    command: npm run migration:latest
+    volumes:
+      - ./backend/src:/app/src
+
+  backend:
+    container_name: infisical-dev-api
+    build:
+      context: ./backend
+      dockerfile: Dockerfile.dev
+    depends_on:
+      db:
+        condition: service_started
+      redis:
+        condition: service_started
+      db-migration:
+        condition: service_completed_successfully
+    env_file:
+      - .env
+    ports:
+      - 4000:4000
+      - 9464:9464 # for OTEL collection of Prometheus metrics
+    environment:
+      - NODE_ENV=development
+      - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable
+      - TELEMETRY_ENABLED=false
+    volumes:
+      - ./backend/src:/app/src
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  frontend:
+    container_name: infisical-dev-frontend
+    restart: unless-stopped
+    depends_on:
+      - backend
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile.dev
+    volumes:
+      - ./frontend/src:/app/src/ # mounted whole src to avoid missing reload on new files
+      - ./frontend/public:/app/public
+    env_file: .env
+    environment:
+      - NEXT_PUBLIC_ENV=development
+      - INFISICAL_TELEMETRY_ENABLED=false
+
+  pgadmin:
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@example.com
+      PGADMIN_DEFAULT_PASSWORD: pass
+    ports:
+      - 5050:80
+    depends_on:
+      - db
+
+  smtp-server:
+    container_name: infisical-dev-smtp-server
+    image: lytrax/mailhog:latest # https://github.com/mailhog/MailHog/issues/353#issuecomment-821137362
+    restart: always
+    logging:
+      driver: "none" # disable saving logs
+    ports:
+      - 1025:1025 # SMTP server
+      - 8025:8025 # Web UI
+
+  openldap: # note: more advanced configuration is available
+    image: osixia/openldap:1.5.0
+    restart: always
+    environment:
+      LDAP_ORGANISATION: Acme
+      LDAP_DOMAIN: acme.com
+      LDAP_ADMIN_PASSWORD: admin
+    ports:
+      - 389:389
+      - 636:636
+    volumes:
+      - ldap_data:/var/lib/ldap
+      - ldap_config:/etc/ldap/slapd.d
+    profiles: [ldap]
+
+  phpldapadmin: # username: cn=admin,dc=acme,dc=com, pass is admin
+    image: osixia/phpldapadmin:latest
+    restart: always
+    environment:
+      - PHPLDAPADMIN_LDAP_HOSTS=openldap
+      - PHPLDAPADMIN_HTTPS=false
+    ports:
+      - 6433:80
+    depends_on:
+      - openldap
+    profiles: [ldap]
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
+    ports:
+      - 1888:1888 # pprof extension
+      - 8888:8888 # Prometheus metrics exposed by the Collector
+      - 8889:8889 # Prometheus exporter metrics
+      - 13133:13133 # health_check extension
+      - 4317:4317 # OTLP gRPC receiver
+      - 4318:4318 # OTLP http receiver
+      - 55679:55679 # zpages extension
+
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    restart: unless-stopped
+    environment:
+      - GF_LOG_LEVEL=debug
+    ports:
+      - "3000:3000"
+    volumes:
+      - "grafana_storage:/var/lib/grafana"
+volumes:
+  postgres-data:
+    driver: local
+  redis_data:
+    driver: local
+  ldap_data:
+  ldap_config:
+  grafana_storage:

docker-compose.dev.yml

@@ -158,38 +158,6 @@ services:
       - openldap
     profiles: [ldap]
 
-  otel-collector:
-    image: otel/opentelemetry-collector-contrib
-    volumes:
-      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
-    ports:
-      - 1888:1888 # pprof extension
-      - 8888:8888 # Prometheus metrics exposed by the Collector
-      - 8889:8889 # Prometheus exporter metrics
-      - 13133:13133 # health_check extension
-      - 4317:4317 # OTLP gRPC receiver
-      - 4318:4318 # OTLP http receiver
-      - 55679:55679 # zpages extension
-
-  prometheus:
-    image: prom/prometheus
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
-    ports:
-      - "9090:9090"
-    command:
-      - "--config.file=/etc/prometheus/prometheus.yml"
-
-  grafana:
-    image: grafana/grafana
-    container_name: grafana
-    restart: unless-stopped
-    environment:
-      - GF_LOG_LEVEL=debug
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana_storage:/var/lib/grafana"
 volumes:
   postgres-data:
     driver: local
@@ -197,4 +165,3 @@ volumes:
     driver: local
   ldap_data:
   ldap_config:
-  grafana_storage:

otel-collector-config.yaml

@@ -3,6 +3,16 @@ receivers:
     protocols:
       http:
         endpoint: 0.0.0.0:4318
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: otel-collector
+          scrape_interval: 30s
+          static_configs:
+            - targets: [backend:9464]
+          metric_relabel_configs:
+            - action: labeldrop
+              regex: "service_instance_id|service_name"
 processors:
   batch:
 
@@ -20,6 +30,6 @@ service:
   extensions: [health_check, pprof, zpages]
   pipelines:
     metrics:
-      receivers: [otlp]
+      receivers: [otlp, prometheus]
       processors: [batch]
       exporters: [prometheus]