Add deployment files

deployment/README.md

@@ -0,0 +1,30 @@
+# Deployment
+
+Deployment files that will be used for the deployments in the experiments.
+
+## light
+[Definition](https://www.notion.so/Chat-Protocol-Benchmarks-1938f96fb65c80d8b22fdf641c5ff003?source=copy_link#1998f96fb65c802095e6fc894dc0b4fe)
+
+A light node, is a status-backend node that runs in “light mode” (edge node in Waku concept).
+Which means it uses Waku Peer Exchange for discovery, light push to send messages,
+filter to receive live messages and store to retrieve historical messages.
+It does not provide services.
+
+
+## relay
+[Definition](https://www.notion.so/Chat-Protocol-Benchmarks-1938f96fb65c80d8b22fdf641c5ff003?source=copy_link#1998f96fb65c8014bf0af42ce2645fa9)
+
+A relay node, is a status-backend node that runs in “relay mode”.
+Which means it uses discv5 and Waku Peer Exchange for discovery (rendezvous wip),
+relay to send and receive live messages, store as a client to retrieved historical messages,
+and provide filter, light push and waku peer exchanges services to other (light/edge) nodes.
+
+
+## service
+[Definition](https://www.notion.so/Chat-Protocol-Benchmarks-1938f96fb65c80d8b22fdf641c5ff003?source=copy_link#1998f96fb65c80c4b972fb560bd0f6f3)
+A service node, is a wakunode2 (nwaku) node that runs service, similar to the Status prod fleet.
+In this context, we can assume it provides Waku peer exchange, light push, filter and store services.
+It uses discv5 for peer discovery. 
+We may get into details to better reproduce fleet setup (boot and store nodes are separate,
+several nwaku store nodes are connected to same posgres db), right now each store node has it's own db.
+

deployment/cleanup.sh

@@ -0,0 +1,12 @@
+KUBECONFIG="$HOME/.kube/sapphire.yaml"
+
+kubectl --kubeconfig $KUBECONFIG delete -f service/status-service-bootstrap.yaml
+kubectl --kubeconfig $KUBECONFIG delete -f service/status-service-node.yaml
+kubectl --kubeconfig $KUBECONFIG delete -f relay/status-backend-relay.yaml
+kubectl --kubeconfig $KUBECONFIG delete -f light/status-backend-light.yaml
+
+#kubectl --kubeconfig $KUBECONFIG delete -f ../base-manifests/anvil-statefulset.yaml
+#kubectl --kubeconfig $KUBECONFIG delete -f ../base-manifests/snt-contract-deployment.yaml
+#kubectl --kubeconfig $KUBECONFIG delete -f ../base-manifests/snt-communities-deployment.yaml
+
+# kubectl --kubeconfig $KUBECONFIG delete -f ../controlbox/controlbox.yaml

deployment/deploy.sh

@@ -0,0 +1,13 @@
+KUBECONFIG="$HOME/.kube/sapphire.yaml"
+
+kubectl --kubeconfig $KUBECONFIG apply -f service/status-service-bootstrap.yaml
+kubectl --kubeconfig $KUBECONFIG rollout status --watch --timeout=30000s statefulset/status-service-bootstrap -n status-go-test
+
+kubectl --kubeconfig $KUBECONFIG apply -f service/status-service-node.yaml
+kubectl --kubeconfig $KUBECONFIG rollout status --watch --timeout=30000s statefulset/status-service-node -n status-go-test
+
+kubectl --kubeconfig $KUBECONFIG apply -f relay/status-backend-relay.yaml
+kubectl --kubeconfig $KUBECONFIG rollout status --watch --timeout=30000s statefulset/status-backend-relay -n status-go-test
+
+kubectl --kubeconfig $KUBECONFIG apply -f light/status-backend-light.yaml
+kubectl --kubeconfig $KUBECONFIG rollout status --watch --timeout=30000s statefulset/status-backend-light -n status-go-test

deployment/light/status-backend-light-service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: status-backend-light
+  namespace: status-go-test
+spec:
+  clusterIP: None
+  selector:
+    app: status-backend-light

deployment/light/status-backend-light.yaml

@@ -0,0 +1,85 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: status-backend-light
+  namespace: status-go-test
+spec:
+  replicas: 1
+  podManagementPolicy: Parallel
+  serviceName: status-backend-light
+  selector:
+    matchLabels:
+      app: status-backend-light
+  template:
+    metadata:
+      labels:
+        app: status-backend-light
+    spec:
+      serviceAccountName: controlbox-sa
+      dnsConfig:
+        searches:
+          - status-backend-relay.status-go-test.svc.cluster.local
+          - status-backend-light.status-go-test.svc.cluster.local
+          - status-service-bootstrap.status-go-test.svc.cluster.local
+          - status-service-node.status-go-test.svc.cluster.local
+      initContainers:
+      - name: getenr
+        image: soutullostatus/getenr:v1.1.0
+        imagePullPolicy: Always
+        volumeMounts:
+          - name: enr-data
+            mountPath: /etc/enr
+        command: ["/bin/sh", "-c"]
+        args:
+          - |
+            /app/getenr.sh 3 status-service-bootstrap.status-go-test /etc/enr/BOOT_ENRS && \
+            /app/getenr.sh 3 status-service-node.status-go-test /etc/enr/STORE_ENRS
+      - name: config-init
+        image: soutullostatus/status-init:v1.0.0
+        imagePullPolicy: Always
+        command:
+          - /bin/bash
+          - -c
+          - |
+            set -a
+            source /etc/enr/BOOT_ENRS
+            source /etc/enr/STORE_ENRS
+            set +a
+            python /init_container.py
+        volumeMounts:
+        - name: enr-data
+          mountPath: /etc/enr
+        - name: relay-config
+          mountPath: /static/configs
+      containers:
+        - name: status-backend
+          image: soutullostatus/status-backend:b22c58bd
+          imagePullPolicy: IfNotPresent
+          args: ["-address", "0.0.0.0:3333"]
+          ports:
+            - containerPort: 3333
+              name: http
+            - containerPort: 30303
+              name: waku
+          volumeMounts:
+            - mountPath: "/static/configs"
+              name: relay-config
+          livenessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  curl -X GET http://localhost:3333/health || exit 1
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 2
+            failureThreshold: 120
+        - name: status-subscriber
+          image: soutullostatus/status-subscriber:v1.0.0
+      volumes:
+        - name: relay-config
+          emptyDir: {}
+        - name: enr-data
+          emptyDir: {}

deployment/relay/status-backend-relay-services.yaml

@@ -0,0 +1,43 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: config-init-sa
+  namespace: status-go-test
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: pod-reader
+  namespace: status-go-test
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pod-reader-binding
+  namespace: status-go-test
+subjects:
+- kind: ServiceAccount
+  name: config-init-sa
+  namespace: status-go-test
+roleRef:
+  kind: ClusterRole
+  name: pod-reader
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: status-backend-relay
+  namespace: status-go-test
+spec:
+  # Headless service to get per-pod DNS names
+  clusterIP: None
+  selector:
+    app: status-backend-relay

deployment/relay/status-backend-relay.yaml

@@ -0,0 +1,85 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: status-backend-relay
+  namespace: status-go-test
+spec:
+  replicas: 2
+  podManagementPolicy: Parallel
+  serviceName: status-backend-relay
+  selector:
+    matchLabels:
+      app: status-backend-relay
+  template:
+    metadata:
+      labels:
+        app: status-backend-relay
+    spec:
+      serviceAccountName: controlbox-sa
+      dnsConfig:
+        searches:
+          - status-backend-relay.status-go-test.svc.cluster.local
+          - status-backend-light.status-go-test.svc.cluster.local
+          - status-service-bootstrap.status-go-test.svc.cluster.local
+          - status-service-node.status-go-test.svc.cluster.local
+      initContainers:
+      - name: getenr
+        image: soutullostatus/getenr:v1.1.0
+        imagePullPolicy: Always
+        volumeMounts:
+          - name: enr-data
+            mountPath: /etc/enr
+        command: ["/bin/sh", "-c"]
+        args:
+          - |
+            /app/getenr.sh 3 status-service-bootstrap.status-go-test /etc/enr/BOOT_ENRS && \
+            /app/getenr.sh 3 status-service-node.status-go-test /etc/enr/STORE_ENRS
+      - name: config-init
+        image: soutullostatus/status-init:v1.0.0
+        imagePullPolicy: Always
+        command:
+          - /bin/bash
+          - -c
+          - |
+            set -a
+            source /etc/enr/BOOT_ENRS
+            source /etc/enr/STORE_ENRS
+            set +a
+            python /init_container.py
+        volumeMounts:
+        - name: enr-data
+          mountPath: /etc/enr
+        - name: relay-config
+          mountPath: /static/configs
+      containers:
+        - name: status-backend
+          image: soutullostatus/status-backend:b22c58bd
+          imagePullPolicy: IfNotPresent
+          args: ["-address", "0.0.0.0:3333"]
+          ports:
+            - containerPort: 3333
+              name: http
+            - containerPort: 30303
+              name: waku
+          volumeMounts:
+            - mountPath: "/static/configs"
+              name: relay-config
+          livenessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  curl -X GET http://localhost:3333/health || exit 1
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 2
+            failureThreshold: 120
+        - name: status-subscriber
+          image: soutullostatus/status-subscriber:v1.0.0
+      volumes:
+        - name: relay-config
+          emptyDir: {}
+        - name: enr-data
+          emptyDir: {}

deployment/service/status-service-bootstrap-service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: status-service-bootstrap
+  namespace: status-go-test
+spec:
+  clusterIP: None
+  selector:
+    app: status-service-bootstrap

deployment/service/status-service-bootstrap.yaml

@@ -0,0 +1,56 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: status-service-bootstrap
+  namespace: status-go-test
+spec:
+  replicas: 1
+  podManagementPolicy: "Parallel"
+  serviceName: status-service-bootstrap
+  selector:
+    matchLabels:
+      app: status-service-bootstrap
+  template:
+    metadata:
+      labels:
+        app: status-service-bootstrap
+    spec:
+      dnsConfig:
+        searches:
+          - status-service-bootstrap.status-go-test.svc.cluster.local
+      containers:
+        - name: waku
+          image: soutullostatus/nwaku-jq-curl:v0.34.0-rc1
+          imagePullPolicy: IfNotPresent
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - "curl -sf http://127.0.0.1:8008/health | grep -q 'OK'"
+            successThreshold: 3
+            initialDelaySeconds: 1
+            periodSeconds: 3
+            failureThreshold: 1
+            timeoutSeconds: 5
+          env:
+            - name: IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "50m"
+            limits:
+              memory: "768Mi"
+              cpu: "400m"
+          ports:
+            - containerPort: 8645
+            - containerPort: 8008
+          command:
+            - sh
+            - -c
+            - |
+              /usr/bin/wakunode --rest=true --rest-address=0.0.0.0 --peer-exchange=true --relay=true --filter=true --lightpush=true --max-connections=500 --discv5-discovery=true --discv5-enr-auto-update=True --log-level=INFO --metrics-server=True --metrics-server-address=0.0.0.0 --nat=extip:$IP --cluster-id=16 --shard=32 --shard=64
+

deployment/service/status-service-node-service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: status-service-node
+  namespace: status-go-test
+spec:
+  clusterIP: None
+  selector:
+    app: status-service-node

deployment/service/status-service-node.yaml

@@ -0,0 +1,104 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: status-service-node
+  namespace: status-go-test
+spec:
+  replicas: 1
+  podManagementPolicy: "Parallel"
+  serviceName: status-service-node
+  selector:
+    matchLabels:
+      app: status-service-node
+  template:
+    metadata:
+      labels:
+        app: status-service-node
+    spec:
+      dnsConfig:
+        searches:
+          - status-service-node.status-go-test.svc.cluster.local
+      volumes:
+        - name: enr-data
+          emptyDir: {}
+        - name: postgres-data
+          emptyDir: {}
+      initContainers:
+        - name: grabenr
+          image: soutullostatus/getenr:v1.1.0
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: enr-data
+              mountPath: /etc/enr
+          command:
+            - /app/getenr.sh
+          args:
+            - "3"
+            - "status-service-bootstrap.status-go-test"
+      containers:
+        - name: postgres
+          image: postgres:15.1-alpine
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: postgres-data
+              mountPath: /var/lib/postgresql/data
+          env:
+            - name: POSTGRES_DB
+              value: wakumessages
+            - name: POSTGRES_USER
+              value: wakuuser
+            - name: POSTGRES_PASSWORD
+              value: wakupassword
+          ports:
+            - containerPort: 5432
+          readinessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - |
+                  pg_isready -U wakuuser -d wakumessages
+            initialDelaySeconds: 5
+            periodSeconds: 2
+            timeoutSeconds: 5
+        - name: waku
+          image: soutullostatus/nwaku-jq-curl:v0.34.0-rc1
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8645
+            - containerPort: 8008
+          volumeMounts:
+            - name: enr-data
+              mountPath: /etc/enr
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - "curl -sf http://127.0.0.1:8008/health | grep -q 'OK'"
+            successThreshold: 5
+            initialDelaySeconds: 5
+            periodSeconds: 1
+            failureThreshold: 2
+            timeoutSeconds: 5
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "150m"
+            limits:
+              memory: "600Mi"
+              cpu: "400m"
+          env:
+            - name: IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POSTGRES_URL
+              value: "postgres://wakuuser:wakupassword@127.0.0.1:5432/wakumessages"
+          command:
+            - sh
+            - -c
+            - |
+              . /etc/enr/ENR
+              echo ENRs are $ENR1 $ENR2 $ENR3
+              nice -n 19 /usr/bin/wakunode --relay=true --peer-exchange=true --lightpush=true --filter=true --store=true --store-message-db-url=${POSTGRES_URL} --max-connections=800 --rest=true --rest-admin=true --rest-address=0.0.0.0 --discv5-discovery=true --discv5-enr-auto-update=True --log-level=INFO --metrics-server=True --metrics-server-address=0.0.0.0 --discv5-bootstrap-node=$ENR1 --discv5-bootstrap-node=$ENR2 --discv5-bootstrap-node=$ENR3 --cluster-id=16 --shard=32 --shard=64