Fix CI prometheus alertmanager alerts
Besides KubeMemoryOvercommit
and KubeCPUOvercommit
alerts in master CI, which should be fixed in #766 (closed), there are two additional alerts we need to handle:
{
"labels": {
"alertname": "PrometheusOperatorSyncFailed",
"container": "kube-prometheus-stack",
"controller": "alertmanager",
"endpoint": "https",
"instance": "10.42.0.43:10250",
"job": "prometheus-stack-kube-prom-operator",
"namespace": "oas",
"pod": "prometheus-stack-kube-prom-operator-7dcb856d4d-7vf4m",
"service": "prometheus-stack-kube-prom-operator",
"severity": "warning",
"status": "failed"
},
"annotations": {
"description": "Controller alertmanager in oas namespace fails to reconcile 1 objects.",
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusoperatorsyncfailed",
"summary": "Last controller reconciliation failed"
},
"state": "firing",
"activeAt": "2021-04-22T08:26:47.050578157Z",
"value": "1e+00"
},
{
"labels": {
"alertname": "PrometheusNotConnectedToAlertmanagers",
"container": "prometheus",
"endpoint": "web",
"instance": "10.42.0.53:9090",
"job": "prometheus-stack-kube-prom-prometheus",
"namespace": "oas",
"pod": "prometheus-prometheus-stack-kube-prom-prometheus-0",
"service": "prometheus-stack-kube-prom-prometheus",
"severity": "warning"
},
"annotations": {
"description": "Prometheus oas/prometheus-prometheus-stack-kube-prom-prometheus-0 is not connected to any Alertmanagers.",
"summary": "Prometheus is not connected to any Alertmanagers."
},
"state": "firing",
"activeAt": "2021-04-22T08:25:19.77143049Z",
"value": "0e+00"
}