diff --git a/deployment/README.md b/deployment/README.md new file mode 100644 index 0000000..90873fd --- /dev/null +++ b/deployment/README.md @@ -0,0 +1,30 @@ +# Deployment + +Deployment files that will be used for the deployments in the experiments. + +## light +[Definition](https://www.notion.so/Chat-Protocol-Benchmarks-1938f96fb65c80d8b22fdf641c5ff003?source=copy_link#1998f96fb65c802095e6fc894dc0b4fe) + +A light node, is a status-backend node that runs in “light mode” (edge node in Waku concept). +Which means it uses Waku Peer Exchange for discovery, light push to send messages, +filter to receive live messages and store to retrieve historical messages. +It does not provide services. + + +## relay +[Definition](https://www.notion.so/Chat-Protocol-Benchmarks-1938f96fb65c80d8b22fdf641c5ff003?source=copy_link#1998f96fb65c8014bf0af42ce2645fa9) + +A relay node, is a status-backend node that runs in “relay mode”. +Which means it uses discv5 and Waku Peer Exchange for discovery (rendezvous wip), +relay to send and receive live messages, store as a client to retrieved historical messages, +and provide filter, light push and waku peer exchanges services to other (light/edge) nodes. + + +## service +[Definition](https://www.notion.so/Chat-Protocol-Benchmarks-1938f96fb65c80d8b22fdf641c5ff003?source=copy_link#1998f96fb65c80c4b972fb560bd0f6f3) +A service node, is a wakunode2 (nwaku) node that runs service, similar to the Status prod fleet. +In this context, we can assume it provides Waku peer exchange, light push, filter and store services. +It uses discv5 for peer discovery. +We may get into details to better reproduce fleet setup (boot and store nodes are separate, +several nwaku store nodes are connected to same posgres db), right now each store node has it's own db. + diff --git a/deployment/cleanup.sh b/deployment/cleanup.sh new file mode 100644 index 0000000..a000a76 --- /dev/null +++ b/deployment/cleanup.sh @@ -0,0 +1,12 @@ +KUBECONFIG="$HOME/.kube/sapphire.yaml" + +kubectl --kubeconfig $KUBECONFIG delete -f service/status-service-bootstrap.yaml +kubectl --kubeconfig $KUBECONFIG delete -f service/status-service-node.yaml +kubectl --kubeconfig $KUBECONFIG delete -f relay/status-backend-relay.yaml +kubectl --kubeconfig $KUBECONFIG delete -f light/status-backend-light.yaml + +#kubectl --kubeconfig $KUBECONFIG delete -f ../base-manifests/anvil-statefulset.yaml +#kubectl --kubeconfig $KUBECONFIG delete -f ../base-manifests/snt-contract-deployment.yaml +#kubectl --kubeconfig $KUBECONFIG delete -f ../base-manifests/snt-communities-deployment.yaml + +# kubectl --kubeconfig $KUBECONFIG delete -f ../controlbox/controlbox.yaml diff --git a/deployment/deploy.sh b/deployment/deploy.sh new file mode 100644 index 0000000..554a277 --- /dev/null +++ b/deployment/deploy.sh @@ -0,0 +1,13 @@ +KUBECONFIG="$HOME/.kube/sapphire.yaml" + +kubectl --kubeconfig $KUBECONFIG apply -f service/status-service-bootstrap.yaml +kubectl --kubeconfig $KUBECONFIG rollout status --watch --timeout=30000s statefulset/status-service-bootstrap -n status-go-test + +kubectl --kubeconfig $KUBECONFIG apply -f service/status-service-node.yaml +kubectl --kubeconfig $KUBECONFIG rollout status --watch --timeout=30000s statefulset/status-service-node -n status-go-test + +kubectl --kubeconfig $KUBECONFIG apply -f relay/status-backend-relay.yaml +kubectl --kubeconfig $KUBECONFIG rollout status --watch --timeout=30000s statefulset/status-backend-relay -n status-go-test + +kubectl --kubeconfig $KUBECONFIG apply -f light/status-backend-light.yaml +kubectl --kubeconfig $KUBECONFIG rollout status --watch --timeout=30000s statefulset/status-backend-light -n status-go-test diff --git a/deployment/light/status-backend-light-service.yaml b/deployment/light/status-backend-light-service.yaml new file mode 100644 index 0000000..0a15535 --- /dev/null +++ b/deployment/light/status-backend-light-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: status-backend-light + namespace: status-go-test +spec: + clusterIP: None + selector: + app: status-backend-light diff --git a/deployment/light/status-backend-light.yaml b/deployment/light/status-backend-light.yaml new file mode 100644 index 0000000..b328db8 --- /dev/null +++ b/deployment/light/status-backend-light.yaml @@ -0,0 +1,85 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: status-backend-light + namespace: status-go-test +spec: + replicas: 1 + podManagementPolicy: Parallel + serviceName: status-backend-light + selector: + matchLabels: + app: status-backend-light + template: + metadata: + labels: + app: status-backend-light + spec: + serviceAccountName: controlbox-sa + dnsConfig: + searches: + - status-backend-relay.status-go-test.svc.cluster.local + - status-backend-light.status-go-test.svc.cluster.local + - status-service-bootstrap.status-go-test.svc.cluster.local + - status-service-node.status-go-test.svc.cluster.local + initContainers: + - name: getenr + image: soutullostatus/getenr:v1.1.0 + imagePullPolicy: Always + volumeMounts: + - name: enr-data + mountPath: /etc/enr + command: ["/bin/sh", "-c"] + args: + - | + /app/getenr.sh 3 status-service-bootstrap.status-go-test /etc/enr/BOOT_ENRS && \ + /app/getenr.sh 3 status-service-node.status-go-test /etc/enr/STORE_ENRS + - name: config-init + image: soutullostatus/status-init:v1.0.0 + imagePullPolicy: Always + command: + - /bin/bash + - -c + - | + set -a + source /etc/enr/BOOT_ENRS + source /etc/enr/STORE_ENRS + set +a + python /init_container.py + volumeMounts: + - name: enr-data + mountPath: /etc/enr + - name: relay-config + mountPath: /static/configs + containers: + - name: status-backend + image: soutullostatus/status-backend:b22c58bd + imagePullPolicy: IfNotPresent + args: ["-address", "0.0.0.0:3333"] + ports: + - containerPort: 3333 + name: http + - containerPort: 30303 + name: waku + volumeMounts: + - mountPath: "/static/configs" + name: relay-config + livenessProbe: + exec: + command: + - /bin/sh + - -c + - | + curl -X GET http://localhost:3333/health || exit 1 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 120 + - name: status-subscriber + image: soutullostatus/status-subscriber:v1.0.0 + volumes: + - name: relay-config + emptyDir: {} + - name: enr-data + emptyDir: {} diff --git a/deployment/relay/status-backend-relay-services.yaml b/deployment/relay/status-backend-relay-services.yaml new file mode 100644 index 0000000..7c822fc --- /dev/null +++ b/deployment/relay/status-backend-relay-services.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: config-init-sa + namespace: status-go-test + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pod-reader + namespace: status-go-test +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pod-reader-binding + namespace: status-go-test +subjects: +- kind: ServiceAccount + name: config-init-sa + namespace: status-go-test +roleRef: + kind: ClusterRole + name: pod-reader + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Service +metadata: + name: status-backend-relay + namespace: status-go-test +spec: + # Headless service to get per-pod DNS names + clusterIP: None + selector: + app: status-backend-relay \ No newline at end of file diff --git a/deployment/relay/status-backend-relay.yaml b/deployment/relay/status-backend-relay.yaml new file mode 100644 index 0000000..1cd72bb --- /dev/null +++ b/deployment/relay/status-backend-relay.yaml @@ -0,0 +1,85 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: status-backend-relay + namespace: status-go-test +spec: + replicas: 2 + podManagementPolicy: Parallel + serviceName: status-backend-relay + selector: + matchLabels: + app: status-backend-relay + template: + metadata: + labels: + app: status-backend-relay + spec: + serviceAccountName: controlbox-sa + dnsConfig: + searches: + - status-backend-relay.status-go-test.svc.cluster.local + - status-backend-light.status-go-test.svc.cluster.local + - status-service-bootstrap.status-go-test.svc.cluster.local + - status-service-node.status-go-test.svc.cluster.local + initContainers: + - name: getenr + image: soutullostatus/getenr:v1.1.0 + imagePullPolicy: Always + volumeMounts: + - name: enr-data + mountPath: /etc/enr + command: ["/bin/sh", "-c"] + args: + - | + /app/getenr.sh 3 status-service-bootstrap.status-go-test /etc/enr/BOOT_ENRS && \ + /app/getenr.sh 3 status-service-node.status-go-test /etc/enr/STORE_ENRS + - name: config-init + image: soutullostatus/status-init:v1.0.0 + imagePullPolicy: Always + command: + - /bin/bash + - -c + - | + set -a + source /etc/enr/BOOT_ENRS + source /etc/enr/STORE_ENRS + set +a + python /init_container.py + volumeMounts: + - name: enr-data + mountPath: /etc/enr + - name: relay-config + mountPath: /static/configs + containers: + - name: status-backend + image: soutullostatus/status-backend:b22c58bd + imagePullPolicy: IfNotPresent + args: ["-address", "0.0.0.0:3333"] + ports: + - containerPort: 3333 + name: http + - containerPort: 30303 + name: waku + volumeMounts: + - mountPath: "/static/configs" + name: relay-config + livenessProbe: + exec: + command: + - /bin/sh + - -c + - | + curl -X GET http://localhost:3333/health || exit 1 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 120 + - name: status-subscriber + image: soutullostatus/status-subscriber:v1.0.0 + volumes: + - name: relay-config + emptyDir: {} + - name: enr-data + emptyDir: {} diff --git a/deployment/service/status-service-bootstrap-service.yaml b/deployment/service/status-service-bootstrap-service.yaml new file mode 100644 index 0000000..ccbac12 --- /dev/null +++ b/deployment/service/status-service-bootstrap-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: status-service-bootstrap + namespace: status-go-test +spec: + clusterIP: None + selector: + app: status-service-bootstrap diff --git a/deployment/service/status-service-bootstrap.yaml b/deployment/service/status-service-bootstrap.yaml new file mode 100644 index 0000000..ff87138 --- /dev/null +++ b/deployment/service/status-service-bootstrap.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: status-service-bootstrap + namespace: status-go-test +spec: + replicas: 1 + podManagementPolicy: "Parallel" + serviceName: status-service-bootstrap + selector: + matchLabels: + app: status-service-bootstrap + template: + metadata: + labels: + app: status-service-bootstrap + spec: + dnsConfig: + searches: + - status-service-bootstrap.status-go-test.svc.cluster.local + containers: + - name: waku + image: soutullostatus/nwaku-jq-curl:v0.34.0-rc1 + imagePullPolicy: IfNotPresent + readinessProbe: + exec: + command: + - /bin/sh + - -c + - "curl -sf http://127.0.0.1:8008/health | grep -q 'OK'" + successThreshold: 3 + initialDelaySeconds: 1 + periodSeconds: 3 + failureThreshold: 1 + timeoutSeconds: 5 + env: + - name: IP + valueFrom: + fieldRef: + fieldPath: status.podIP + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "768Mi" + cpu: "400m" + ports: + - containerPort: 8645 + - containerPort: 8008 + command: + - sh + - -c + - | + /usr/bin/wakunode --rest=true --rest-address=0.0.0.0 --peer-exchange=true --relay=true --filter=true --lightpush=true --max-connections=500 --discv5-discovery=true --discv5-enr-auto-update=True --log-level=INFO --metrics-server=True --metrics-server-address=0.0.0.0 --nat=extip:$IP --cluster-id=16 --shard=32 --shard=64 + diff --git a/deployment/service/status-service-node-service.yaml b/deployment/service/status-service-node-service.yaml new file mode 100644 index 0000000..6764434 --- /dev/null +++ b/deployment/service/status-service-node-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: status-service-node + namespace: status-go-test +spec: + clusterIP: None + selector: + app: status-service-node diff --git a/deployment/service/status-service-node.yaml b/deployment/service/status-service-node.yaml new file mode 100644 index 0000000..969bf5b --- /dev/null +++ b/deployment/service/status-service-node.yaml @@ -0,0 +1,104 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: status-service-node + namespace: status-go-test +spec: + replicas: 1 + podManagementPolicy: "Parallel" + serviceName: status-service-node + selector: + matchLabels: + app: status-service-node + template: + metadata: + labels: + app: status-service-node + spec: + dnsConfig: + searches: + - status-service-node.status-go-test.svc.cluster.local + volumes: + - name: enr-data + emptyDir: {} + - name: postgres-data + emptyDir: {} + initContainers: + - name: grabenr + image: soutullostatus/getenr:v1.1.0 + imagePullPolicy: Always + volumeMounts: + - name: enr-data + mountPath: /etc/enr + command: + - /app/getenr.sh + args: + - "3" + - "status-service-bootstrap.status-go-test" + containers: + - name: postgres + image: postgres:15.1-alpine + imagePullPolicy: IfNotPresent + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + env: + - name: POSTGRES_DB + value: wakumessages + - name: POSTGRES_USER + value: wakuuser + - name: POSTGRES_PASSWORD + value: wakupassword + ports: + - containerPort: 5432 + readinessProbe: + exec: + command: + - sh + - -c + - | + pg_isready -U wakuuser -d wakumessages + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 5 + - name: waku + image: soutullostatus/nwaku-jq-curl:v0.34.0-rc1 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8645 + - containerPort: 8008 + volumeMounts: + - name: enr-data + mountPath: /etc/enr + readinessProbe: + exec: + command: + - /bin/sh + - -c + - "curl -sf http://127.0.0.1:8008/health | grep -q 'OK'" + successThreshold: 5 + initialDelaySeconds: 5 + periodSeconds: 1 + failureThreshold: 2 + timeoutSeconds: 5 + resources: + requests: + memory: "64Mi" + cpu: "150m" + limits: + memory: "600Mi" + cpu: "400m" + env: + - name: IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POSTGRES_URL + value: "postgres://wakuuser:wakupassword@127.0.0.1:5432/wakumessages" + command: + - sh + - -c + - | + . /etc/enr/ENR + echo ENRs are $ENR1 $ENR2 $ENR3 + nice -n 19 /usr/bin/wakunode --relay=true --peer-exchange=true --lightpush=true --filter=true --store=true --store-message-db-url=${POSTGRES_URL} --max-connections=800 --rest=true --rest-admin=true --rest-address=0.0.0.0 --discv5-discovery=true --discv5-enr-auto-update=True --log-level=INFO --metrics-server=True --metrics-server-address=0.0.0.0 --discv5-bootstrap-node=$ENR1 --discv5-bootstrap-node=$ENR2 --discv5-bootstrap-node=$ENR3 --cluster-id=16 --shard=32 --shard=64 \ No newline at end of file