diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdd63d4dac224e1a766afbf4f3f7bccbad31399f..64c1f601f02768763ea69fa7ebe48f5fff2fc87b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,6 +121,22 @@ certs: - openappstack/**/* - .gitlab-ci.yml +prometheus-alerts: + stage: health-test + variables: + OAS_DOMAIN: 'ci-${CI_PIPELINE_ID}.ci.openappstack.net' + allow_failure: true + script: + - eval $(ssh-agent -s) + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null + - cd test/ + - py.test -s -m 'prometheus' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*' + only: + changes: + - ansible/**/* + - helmfiles/**/* + - test/**/* + behave-nextcloud: stage: integration-test script: diff --git a/helmfiles/values/prometheus.yaml.gotmpl b/helmfiles/values/prometheus.yaml.gotmpl index c9ea726fe5bacb7c720021618003dcb8f05fcb89..73d5b724ada847573d418871a5544f36cce276ce 100644 --- a/helmfiles/values/prometheus.yaml.gotmpl +++ b/helmfiles/values/prometheus.yaml.gotmpl @@ -35,6 +35,8 @@ prometheus: runAsUser: 0 fsGroup: 0 runAsNonRoot: false + service: + type: NodePort grafana: adminPassword: "{{ requiredEnv "GRAFANA_ADMIN_PASSWORD" }}" diff --git a/test/README.md b/test/README.md index 5ff076a3f371fb4decbb77fb44d20e889eb3023a..7b4c0d2f6a1bc5fc41b09c217fa23c9678dcadfd 100644 --- a/test/README.md +++ b/test/README.md @@ -1,18 +1,27 @@ # Run testinfra tests -Test host configured in `test/inventory.yml` +Test host configured in `../clusters/CLUSTERNAME/inventory.yml` - py.test -v --ansible-inventory=../inventory.yml --hosts='ansible://*' + export inventory=../clusters/CLUSTERNAME/inventory.yml + + + py.test -v --ansible-inventory=${inventory} --hosts='ansible://*' Specify host manually: py.test -v --hosts='ssh://root@example.openappstack.net' +Run only tests tagged with `prometheus`: + + py.test -v --ansible-inventory=${inventory} --hosts='ansible://*' -m prometheus + +## Cert tests + Run cert test manually using the ansible inventory file: ADDRESS='example.openappstack.net' py.test -v -m 'certs' \ --connection=ansible \ - --ansible-inventory=../ansible/inventory.yml \ + --ansible-inventory=${inventory} \ --hosts='ansible://*' Run cert test manually against a different cluster, not configured in any diff --git a/test/pytest.ini b/test/pytest.ini index 0bf407eab322724e5da48cfc8580ba41305da920..281b543c47df5bbb53f9cb07b703820a3880a5e6 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -3,3 +3,4 @@ markers = certs: Run tests related to TLS certificates testinfra: Run testinfra tests (test OS/package versions etc) + prometheus: Test prometheus diff --git a/test/pytest/test_prometheus.py b/test/pytest/test_prometheus.py new file mode 100755 index 0000000000000000000000000000000000000000..d7a3fcce70941d6aa9d7febe0c1c89ab2c84fd26 --- /dev/null +++ b/test/pytest/test_prometheus.py @@ -0,0 +1,27 @@ +import json +import pytest +import requests + +@pytest.mark.prometheus +def test_prometheus_alerts(host): + + print("Starting prometheus test...") + + url = 'http://127.0.0.1:30090/api/v1/alerts' + alert_json = json.loads(host.check_output('curl ' + url)) + status = alert_json["status"] + alerts = alert_json["data"]["alerts"] + + # Filter out the ever firing "Dead mans switch" + real_alerts = [alert for alert in alerts + if alert["labels"]["severity"] != "none"] + + alert_names = list(map(lambda alert: alert["labels"]["alertname"], real_alerts)) + count = len(real_alerts) + + assert status == "success", "Failure queriying the prometheus api at" + url + assert count == 0, "Firing alerts: {0}".format(str(alert_names)) + + +if __name__ == "__main__": + test_prometheus_alerts('')