Merge branch '217-test-if-prometheus-can-reach-all-targets' into 'master'

Resolve "Test if prometheus can reach all targets"

Closes #217

See merge request openappstack/openappstack!45

.gitlab-ci.yml

@@ -121,6 +121,22 @@ certs:
       - openappstack/**/*
       - .gitlab-ci.yml
 
+prometheus-alerts:
+  stage: health-test
+  variables:
+    OAS_DOMAIN: 'ci-${CI_PIPELINE_ID}.ci.openappstack.net'
+  allow_failure: true
+  script:
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
+    - cd test/
+    - py.test -s -m 'prometheus' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
+  only:
+    changes:
+      - ansible/**/*
+      - helmfiles/**/*
+      - test/**/*
+
 behave-nextcloud:
   stage: integration-test
   script:

helmfiles/values/prometheus.yaml.gotmpl

@@ -35,6 +35,8 @@ prometheus:
       runAsUser: 0
       fsGroup: 0
       runAsNonRoot: false
+  service:
+    type: NodePort
 
 grafana:
   adminPassword: "{{ requiredEnv "GRAFANA_ADMIN_PASSWORD" }}"

test/README.md

 # Run testinfra tests
 
-Test host configured in `test/inventory.yml`
+Test host configured in `../clusters/CLUSTERNAME/inventory.yml`
 
-    py.test -v --ansible-inventory=../inventory.yml --hosts='ansible://*'
+    export inventory=../clusters/CLUSTERNAME/inventory.yml
+
+
+    py.test -v --ansible-inventory=${inventory} --hosts='ansible://*'
 
 Specify host manually:
 
     py.test -v --hosts='ssh://root@example.openappstack.net'
 
+Run only tests tagged with `prometheus`:
+
+    py.test -v --ansible-inventory=${inventory} --hosts='ansible://*' -m prometheus
+
+## Cert tests
+
 Run cert test manually using the ansible inventory file:
 
     ADDRESS='example.openappstack.net' py.test -v -m 'certs' \
       --connection=ansible \
-      --ansible-inventory=../ansible/inventory.yml \
+      --ansible-inventory=${inventory} \
       --hosts='ansible://*'
 
 Run cert test manually against a different cluster, not configured in any

test/pytest.ini

@@ -3,3 +3,4 @@
 markers =
     certs: Run tests related to TLS certificates
     testinfra: Run testinfra tests (test OS/package versions etc)
+    prometheus: Test prometheus

test/pytest/test_prometheus.py

+import json
+import pytest
+import requests
+
+@pytest.mark.prometheus
+def test_prometheus_alerts(host):
+
+    print("Starting prometheus test...")
+
+    url = 'http://127.0.0.1:30090/api/v1/alerts'
+    alert_json = json.loads(host.check_output('curl ' + url))
+    status = alert_json["status"]
+    alerts = alert_json["data"]["alerts"]
+
+    # Filter out the ever firing "Dead mans switch"
+    real_alerts = [alert for alert in alerts
+                   if alert["labels"]["severity"] != "none"]
+
+    alert_names = list(map(lambda alert: alert["labels"]["alertname"], real_alerts))
+    count = len(real_alerts)
+
+    assert status == "success", "Failure queriying the prometheus api at" + url
+    assert count == 0, "Firing alerts: {0}".format(str(alert_names))
+
+
+if __name__ == "__main__":
+    test_prometheus_alerts('')