KeyError: 'pod'
Job #19010 failed for b2482a32:
____ test_prometheus_alerts[ansible://200-research-central-logging-options] ____
host = <testinfra.host.Host object at 0x7fc9f8fe6a20>
@pytest.mark.prometheus
def test_prometheus_alerts(host):
def summarize_alerts(alerts):
"""Print a alert summary."""
print('Total alerts: %s' % len(alerts))
print(json.dumps(alerts, indent=2))
print("Starting prometheus test...")
url = 'http://127.0.0.1:30090/api/v1/alerts'
alert_json = json.loads(host.check_output('curl ' + url))
status = alert_json["status"]
alerts = alert_json["data"]["alerts"]
real_alerts = []
ignored_alerts = []
for alert in alerts:
# Filter out the ever firing "Dead mans switch" test alert
> if ignore_alert(alert):
pytest/test_prometheus.py:25:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
alert = {'activeAt': '2019-12-28T16:49:37.751106718Z', 'annotations': {'message': '26% throttling of CPU in namespace oas for ...er', 'namespace': 'oas', 'pod_name': 'ingress-nginx-ingress-controller-d9455899f-rrp2p', ...}, 'state': 'pending', ...}
def ignore_alert(alert):
"""
Returns true if an alert should be ignored. This is when the "severity"
equals "none", or in some application specific cases.
"""
if alert["labels"]["severity"] == "none":
return True
# Ignore `KubeAPILatencyHigh` fom high load during installation
# phase
if alert["labels"]["alertname"] == "KubeAPILatencyHigh":
return True
# Filter out failing Nextcloud installation jobs since a lot of
# them fail until they succeed during installation
> if "nextcloud" in alert["labels"]["pod"]:
E KeyError: 'pod'
pytest/test_prometheus.py:54: KeyError