pytest/test_prometheus.py:60: KeyError
Job #23746 failed for a9cefde9:
=================================== FAILURES ===================================
___________________ test_prometheus_alerts[ansible://master] ___________________
host = <testinfra.host.Host object at 0x7fd1c808dc88>
@pytest.mark.prometheus
def test_prometheus_alerts(host):
"""Test prometheus for firing alerts."""
def summarize_alerts(alerts):
"""Print a alert summary."""
print('Total alerts: %s' % len(alerts))
print(json.dumps(alerts, indent=2))
print("Starting prometheus test...")
prometheus_ip = host.check_output(
'dig +short '
'monitoring-prometheus-oper-prometheus.oas.svc.cluster.local '
'@10.43.0.10')
url = 'http://{0}:9090/api/v1/alerts'.format(prometheus_ip)
alert_json = json.loads(host.check_output('curl ' + url))
status = alert_json["status"]
alerts = alert_json["data"]["alerts"]
real_alerts = []
ignored_alerts = []
for alert in alerts:
# Filter out the ever firing "Dead mans switch" test alert
> if ignore_alert(alert):
pytest/test_prometheus.py:30:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
alert = {'activeAt': '2020-01-14T12:25:37.751106718Z', 'annotations': {'message': '27% throttling of CPU in namespace oas for ...', 'namespace': 'oas', 'pod_name': 'prometheus-monitoring-prometheus-oper-prometheus-0', ...}, 'state': 'pending', ...}
def ignore_alert(alert):
"""
Returns true if an alert should be ignored. This is when the "severity"
equals "none", or in some application specific cases.
"""
if alert["labels"]["severity"] == "none":
return True
# Ignore `KubeAPILatencyHigh` fom high load during installation
# phase
if alert["labels"]["alertname"] == "KubeAPILatencyHigh":
return True
# Filter out failing Nextcloud installation jobs since a lot of
# them fail until they succeed during installation
> if "nextcloud" in alert["labels"]["pod"]:
E KeyError: 'pod'
pytest/test_prometheus.py:60: KeyError
======================= 1 failed, 5 deselected in 10.47s =======================