mirror of
https://github.com/OffchainLabs/prysm.git
synced 2026-01-10 05:47:59 -05:00
Add alert manager config (#2564)
This commit is contained in:
Preston Van Loon
committed by
terence tsao
terence tsao
parent
d1fa88ce4b
commit
78a76e56fb
177
k8s/monitoring/alertmanager-templates.config.yaml
Normal file
177
k8s/monitoring/alertmanager-templates.config.yaml
Normal file
@@ -0,0 +1,177 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: alertmanager-templates
|
||||
namespace: istio-system
|
||||
data:
|
||||
default.tmpl: |
|
||||
{{ define "__alertmanager" }}AlertManager{{ end }}
|
||||
{{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver }}{{ end }}
|
||||
{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
|
||||
{{ define "__description" }}{{ end }}
|
||||
{{ define "__text_alert_list" }}{{ range . }}Labels:
|
||||
{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
|
||||
{{ end }}Annotations:
|
||||
{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}
|
||||
{{ end }}Source: {{ .GeneratorURL }}
|
||||
{{ end }}{{ end }}
|
||||
{{ define "slack.default.title" }}{{ template "__subject" . }}{{ end }}
|
||||
{{ define "slack.default.username" }}{{ template "__alertmanager" . }}{{ end }}
|
||||
{{ define "slack.default.fallback" }}{{ template "slack.default.title" . }} | {{ template "slack.default.titlelink" . }}{{ end }}
|
||||
{{ define "slack.default.pretext" }}{{ end }}
|
||||
{{ define "slack.default.titlelink" }}{{ template "__alertmanagerURL" . }}{{ end }}
|
||||
{{ define "slack.default.iconemoji" }}{{ end }}
|
||||
{{ define "slack.default.iconurl" }}{{ end }}
|
||||
{{ define "slack.default.text" }}{{ end }}
|
||||
{{ define "hipchat.default.from" }}{{ template "__alertmanager" . }}{{ end }}
|
||||
{{ define "hipchat.default.message" }}{{ template "__subject" . }}{{ end }}
|
||||
{{ define "pagerduty.default.description" }}{{ template "__subject" . }}{{ end }}
|
||||
{{ define "pagerduty.default.client" }}{{ template "__alertmanager" . }}{{ end }}
|
||||
{{ define "pagerduty.default.clientURL" }}{{ template "__alertmanagerURL" . }}{{ end }}
|
||||
{{ define "pagerduty.default.instances" }}{{ template "__text_alert_list" . }}{{ end }}
|
||||
{{ define "opsgenie.default.message" }}{{ template "__subject" . }}{{ end }}
|
||||
{{ define "opsgenie.default.description" }}{{ .CommonAnnotations.SortedPairs.Values | join " " }}
|
||||
{{ if gt (len .Alerts.Firing) 0 -}}
|
||||
Alerts Firing:
|
||||
{{ template "__text_alert_list" .Alerts.Firing }}
|
||||
{{- end }}
|
||||
{{ if gt (len .Alerts.Resolved) 0 -}}
|
||||
Alerts Resolved:
|
||||
{{ template "__text_alert_list" .Alerts.Resolved }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ define "opsgenie.default.source" }}{{ template "__alertmanagerURL" . }}{{ end }}
|
||||
{{ define "victorops.default.message" }}{{ template "__subject" . }} | {{ template "__alertmanagerURL" . }}{{ end }}
|
||||
{{ define "victorops.default.from" }}{{ template "__alertmanager" . }}{{ end }}
|
||||
{{ define "email.default.subject" }}{{ template "__subject" . }}{{ end }}
|
||||
{{ define "email.default.html" }}
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<!--
|
||||
Style and HTML derived from https://github.com/mailgun/transactional-email-templates
|
||||
The MIT License (MIT)
|
||||
Copyright (c) 2014 Mailgun
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
-->
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<head style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<meta name="viewport" content="width=device-width" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
|
||||
<title style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{{ template "__subject" . }}</title>
|
||||
</head>
|
||||
<body itemscope="" itemtype="http://schema.org/EmailMessage" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; height: 100%; line-height: 1.6em; width: 100% !important; background-color: #f6f6f6; margin: 0; padding: 0;" bgcolor="#f6f6f6">
|
||||
<table style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6">
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" valign="top"></td>
|
||||
<td width="600" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; width: 100% !important; margin: 0 auto; padding: 0;" valign="top">
|
||||
<div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 0;">
|
||||
<table width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px solid #e9e9e9;" bgcolor="#fff">
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #E6522C; margin: 0; padding: 20px;" align="center" bgcolor="#E6522C" valign="top">
|
||||
{{ .Alerts | len }} alert{{ if gt (len .Alerts) 1 }}s{{ end }} for {{ range .GroupLabels.SortedPairs }}
|
||||
{{ .Name }}={{ .Value }}
|
||||
{{ end }}
|
||||
</td>
|
||||
</tr>
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 10px;" valign="top">
|
||||
<table width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
|
||||
<a href="{{ template "__alertmanagerURL" . }}" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; color: #FFF; text-decoration: none; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize; background-color: #348eda; margin: 0; border-color: #348eda; border-style: solid; border-width: 10px 20px;">View in {{ template "__alertmanager" . }}</a>
|
||||
</td>
|
||||
</tr>
|
||||
{{ if gt (len .Alerts.Firing) 0 }}
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
|
||||
<strong style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">[{{ .Alerts.Firing | len }}] Firing</strong>
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
{{ range .Alerts.Firing }}
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
|
||||
<strong style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">Labels</strong><br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
|
||||
{{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}<br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />{{ end }}
|
||||
{{ if gt (len .Annotations) 0 }}<strong style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">Annotations</strong><br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />{{ end }}
|
||||
{{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}<br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />{{ end }}
|
||||
<a href="{{ .GeneratorURL }}" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; color: #348eda; text-decoration: underline; margin: 0;">Source</a><br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
{{ if gt (len .Alerts.Resolved) 0 }}
|
||||
{{ if gt (len .Alerts.Firing) 0 }}
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
|
||||
<br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
|
||||
<hr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
|
||||
<br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
|
||||
<strong style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">[{{ .Alerts.Resolved | len }}] Resolved</strong>
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
{{ range .Alerts.Resolved }}
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
|
||||
<strong style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">Labels</strong><br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
|
||||
{{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}<br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />{{ end }}
|
||||
{{ if gt (len .Annotations) 0 }}<strong style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">Annotations</strong><br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />{{ end }}
|
||||
{{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}<br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />{{ end }}
|
||||
<a href="{{ .GeneratorURL }}" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; color: #348eda; text-decoration: underline; margin: 0;">Source</a><br style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;">
|
||||
<table width="100%" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; text-align: center; color: #999; margin: 0; padding: 0 0 20px;" align="center" valign="top"><a href="{{ .ExternalURL }}" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 12px; color: #999; text-decoration: underline; margin: 0;">Sent by {{ template "__alertmanager" . }}</a></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div></div>
|
||||
</td>
|
||||
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" valign="top"></td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
{{ end }}
|
||||
{{ define "pushover.default.title" }}{{ template "__subject" . }}{{ end }}
|
||||
{{ define "pushover.default.message" }}{{ .CommonAnnotations.SortedPairs.Values | join " " }}
|
||||
{{ if gt (len .Alerts.Firing) 0 }}
|
||||
Alerts Firing:
|
||||
{{ template "__text_alert_list" .Alerts.Firing }}
|
||||
{{ end }}
|
||||
{{ if gt (len .Alerts.Resolved) 0 }}
|
||||
Alerts Resolved:
|
||||
{{ template "__text_alert_list" .Alerts.Resolved }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
{{ define "pushover.default.url" }}{{ template "__alertmanagerURL" . }}{{ end }}
|
||||
slack.tmpl: |
|
||||
{{ define "slack.devops.text" }}
|
||||
{{range .Alerts}}{{.Annotations.DESCRIPTION}}
|
||||
{{end}}
|
||||
{{ end }}
|
||||
19
k8s/monitoring/alertmanager.config.yaml
Normal file
19
k8s/monitoring/alertmanager.config.yaml
Normal file
@@ -0,0 +1,19 @@
|
||||
kind: ConfigMap
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: alertmanager-config
|
||||
namespace: istio-system
|
||||
data:
|
||||
config.yml: |-
|
||||
global:
|
||||
templates:
|
||||
- '/etc/alertmanager/*.tmpl'
|
||||
route:
|
||||
receiver: discord
|
||||
group_by: ['alertname', 'priority']
|
||||
group_wait: 10s
|
||||
repeat_interval: 30m
|
||||
receivers:
|
||||
- name: discord
|
||||
webhook_configs:
|
||||
- url: http://alertmanager-discord:9094
|
||||
73
k8s/monitoring/alertmanager.deploy.yaml
Normal file
73
k8s/monitoring/alertmanager.deploy.yaml
Normal file
@@ -0,0 +1,73 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: alertmanager
|
||||
namespace: istio-system
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: alertmanager
|
||||
template:
|
||||
metadata:
|
||||
name: alertmanager
|
||||
labels:
|
||||
app: alertmanager
|
||||
spec:
|
||||
priorityClassName: monitoring-priority
|
||||
containers:
|
||||
- name: alertmanager
|
||||
image: prom/alertmanager:latest
|
||||
args:
|
||||
- "--config.file=/etc/alertmanager/config.yml"
|
||||
- "--storage.path=/alertmanager"
|
||||
- "--log.level=debug"
|
||||
ports:
|
||||
- name: alertmanager
|
||||
containerPort: 9093
|
||||
volumeMounts:
|
||||
- name: config-volume
|
||||
mountPath: /etc/alertmanager
|
||||
- name: templates-volume
|
||||
mountPath: /etc/alertmanager-templates
|
||||
- name: alertmanager
|
||||
mountPath: /alertmanager
|
||||
volumes:
|
||||
- name: config-volume
|
||||
configMap:
|
||||
name: alertmanager-config
|
||||
- name: templates-volume
|
||||
configMap:
|
||||
name: alertmanager-templates
|
||||
- name: alertmanager
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: alertmanager-discord
|
||||
namespace: istio-system
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: alertmanager-discord
|
||||
template:
|
||||
metadata:
|
||||
name: alertmanager-discord
|
||||
labels:
|
||||
app: alertmanager-discord
|
||||
spec:
|
||||
priorityClassName: monitoring-priority
|
||||
containers:
|
||||
- name: alertmanager-discord
|
||||
image: gcr.io/prysmaticlabs/alertmanager-discord:latest
|
||||
ports:
|
||||
- name: webhook
|
||||
containerPort: 9094
|
||||
env:
|
||||
- name: DISCORD_WEBHOOK
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: discord-webhook
|
||||
key: url
|
||||
29
k8s/monitoring/alertmanager.service.yaml
Normal file
29
k8s/monitoring/alertmanager.service.yaml
Normal file
@@ -0,0 +1,29 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: alertmanager
|
||||
namespace: istio-system
|
||||
annotations:
|
||||
prometheus.io/scrape: 'true'
|
||||
prometheus.io/path: /
|
||||
prometheus.io/port: '8080'
|
||||
spec:
|
||||
selector:
|
||||
app: alertmanager
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 9093
|
||||
targetPort: 9093
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: alertmanager-discord
|
||||
namespace: istio-system
|
||||
spec:
|
||||
selector:
|
||||
app: alertmanager-discord
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 9094
|
||||
targetPort: 9094
|
||||
30
k8s/monitoring/alerts.yaml
Normal file
30
k8s/monitoring/alerts.yaml
Normal file
@@ -0,0 +1,30 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: prometheus-alerts
|
||||
namespace: istio-system
|
||||
data:
|
||||
prometheus.rules.yml: |-
|
||||
groups:
|
||||
- name: prysm
|
||||
rules:
|
||||
- alert: stalled_chain
|
||||
expr: delta(avg(state_last_slot > scalar(max(state_last_slot) - 100))[10m:30s]) < 50
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: No block slots advanced in 2 minutes
|
||||
- alert: too_long_since_finality
|
||||
expr: max(state_last_slot / 8) - floor(max(state_last_finalized_epoch)) > 10
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: No finality in 10 epochs
|
||||
- alert: high_reorg_rate
|
||||
expr: max(delta(reorg_counter[10m])) > 5
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: Some nodes are seeing more than 5 reorgs in 10 minutes
|
||||
- alert: high_goroutines
|
||||
expr: max_over_time(go_goroutines{component="beacon-chain"}[1m]) > 1000
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: Some nodes are experencing more than 1000 goroutines
|
||||
11
k8s/monitoring/discord-webhook.encrypted_sercret.yaml
Normal file
11
k8s/monitoring/discord-webhook.encrypted_sercret.yaml
Normal file
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
data:
|
||||
url: 3Bh4HrgcSs/QdGDVnvg31lEggQELhDEycmTNAoa5WP7gb7UPd0XC20uPZXftWF37eVX2CF4wWkXn7ge4/E+Ut6Fo3K/nLYQOc5BBGiZQZcgVGnYn5adZ+8SLCXQTTwBaCBhLUmxwB8IQYA9icnJg/ZCgXKjq2pgRnkIEn58Y5kw2lIxlbQ1drFKGha1YCBHCbpJZ+dWHsryGoh7S2d0vDQVBu0AW3L8/cM1O5rIv7OcWkV1QfEY5P4xiAgCtz+uS.yYUhj5ra/rofJcNd.j2DDF/zrMlD0efGqYKhfkg==
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: discord-webhook
|
||||
namespace: istio-system
|
||||
type: Opaque
|
||||
# kubesec:v:3
|
||||
# kubesec:gcp:projects/prysmaticlabs/locations/global/keyRings/prysmatic-k8s-secrets/cryptoKeys/testkey:CiQAaKPz55imKn09+ay5Fipt8Ejsa0fl9RAiDUwIB8QjWJniNfESSQD23sIsO81pJ6gDAzc7733PGECLQ+ftcvluf41iLs5GUBBHrk6ziqmtDmfiTzc9E2YDuXYwAU4EejVjXoIHMESywm9EvetOEt4=
|
||||
# kubesec:mac:JaD9nI5ZuJQNiB9L.5gAZrH4oEyQNTUmUHmSjRQ==
|
||||
Reference in New Issue
Block a user