diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bdd63d4dac224e1a766afbf4f3f7bccbad31399f..64c1f601f02768763ea69fa7ebe48f5fff2fc87b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,6 +121,22 @@ certs:
       - openappstack/**/*
       - .gitlab-ci.yml
 
+prometheus-alerts:
+  stage: health-test
+  variables:
+    OAS_DOMAIN: 'ci-${CI_PIPELINE_ID}.ci.openappstack.net'
+  allow_failure: true
+  script:
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
+    - cd test/
+    - py.test -s -m 'prometheus' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
+  only:
+    changes:
+      - ansible/**/*
+      - helmfiles/**/*
+      - test/**/*
+
 behave-nextcloud:
   stage: integration-test
   script:
diff --git a/helmfiles/values/prometheus.yaml.gotmpl b/helmfiles/values/prometheus.yaml.gotmpl
index c9ea726fe5bacb7c720021618003dcb8f05fcb89..73d5b724ada847573d418871a5544f36cce276ce 100644
--- a/helmfiles/values/prometheus.yaml.gotmpl
+++ b/helmfiles/values/prometheus.yaml.gotmpl
@@ -35,6 +35,8 @@ prometheus:
       runAsUser: 0
       fsGroup: 0
       runAsNonRoot: false
+  service:
+    type: NodePort
 
 grafana:
   adminPassword: "{{ requiredEnv "GRAFANA_ADMIN_PASSWORD" }}"
diff --git a/test/README.md b/test/README.md
index 5ff076a3f371fb4decbb77fb44d20e889eb3023a..7b4c0d2f6a1bc5fc41b09c217fa23c9678dcadfd 100644
--- a/test/README.md
+++ b/test/README.md
@@ -1,18 +1,27 @@
 # Run testinfra tests
 
-Test host configured in `test/inventory.yml`
+Test host configured in `../clusters/CLUSTERNAME/inventory.yml`
 
-    py.test -v --ansible-inventory=../inventory.yml --hosts='ansible://*'
+    export inventory=../clusters/CLUSTERNAME/inventory.yml
+
+
+    py.test -v --ansible-inventory=${inventory} --hosts='ansible://*'
 
 Specify host manually:
 
     py.test -v --hosts='ssh://root@example.openappstack.net'
 
+Run only tests tagged with `prometheus`:
+
+    py.test -v --ansible-inventory=${inventory} --hosts='ansible://*' -m prometheus
+
+## Cert tests
+
 Run cert test manually using the ansible inventory file:
 
     ADDRESS='example.openappstack.net' py.test -v -m 'certs' \
       --connection=ansible \
-      --ansible-inventory=../ansible/inventory.yml \
+      --ansible-inventory=${inventory} \
       --hosts='ansible://*'
 
 Run cert test manually against a different cluster, not configured in any
diff --git a/test/pytest.ini b/test/pytest.ini
index 0bf407eab322724e5da48cfc8580ba41305da920..281b543c47df5bbb53f9cb07b703820a3880a5e6 100644
--- a/test/pytest.ini
+++ b/test/pytest.ini
@@ -3,3 +3,4 @@
 markers =
     certs: Run tests related to TLS certificates
     testinfra: Run testinfra tests (test OS/package versions etc)
+    prometheus: Test prometheus
diff --git a/test/pytest/test_prometheus.py b/test/pytest/test_prometheus.py
new file mode 100755
index 0000000000000000000000000000000000000000..d7a3fcce70941d6aa9d7febe0c1c89ab2c84fd26
--- /dev/null
+++ b/test/pytest/test_prometheus.py
@@ -0,0 +1,27 @@
+import json
+import pytest
+import requests
+
+@pytest.mark.prometheus
+def test_prometheus_alerts(host):
+
+    print("Starting prometheus test...")
+
+    url = 'http://127.0.0.1:30090/api/v1/alerts'
+    alert_json = json.loads(host.check_output('curl ' + url))
+    status = alert_json["status"]
+    alerts = alert_json["data"]["alerts"]
+
+    # Filter out the ever firing "Dead mans switch"
+    real_alerts = [alert for alert in alerts
+                   if alert["labels"]["severity"] != "none"]
+
+    alert_names = list(map(lambda alert: alert["labels"]["alertname"], real_alerts))
+    count = len(real_alerts)
+
+    assert status == "success", "Failure queriying the prometheus api at" + url
+    assert count == 0, "Firing alerts: {0}".format(str(alert_names))
+
+
+if __name__ == "__main__":
+    test_prometheus_alerts('')