diff --git a/.gitignore b/.gitignore
index 45b29d71b485ca6329f696a6d3833c96f39f902d..54f4866299bd643ed7e1e98d18ebbc3a3f18f7d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ test/rke.log
 /test/secrets/
 /test/inventory.yml
 /test/behave/behave.ini
+/test/behave/rerun_failing.features
 
 # Ignore files created during tests
 /test/behave/**/screenshots/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a3da7c129a76fe8867ba9296a4223fe0c17620f3..ef3194a59114f07af37ebe77d0f7f121103ba9d3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,7 +2,8 @@ stages:
   - build
   - setup-cluster
   - install-apps
-  - test
+  - health-test
+  - integration-test
   - cleanup
 
 image: "${CI_REGISTRY_IMAGE}/bootstrap-ci:${CI_COMMIT_REF_NAME}"
@@ -58,7 +59,7 @@ install:
     expire_in: 1 month
 
 testinfra:
-  stage: test
+  stage: health-test
   script:
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
@@ -66,7 +67,7 @@ testinfra:
     - py.test -v -m 'testinfra' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
 
 certs:
-  stage: test
+  stage: health-test
   variables:
     OAS_DOMAIN: 'ci-${CI_PIPELINE_ID}.ci.openappstack.net'
   allow_failure: true
@@ -77,19 +78,20 @@ certs:
     - py.test -s -m 'certs' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
 
 behave:
-  stage: test
+  stage: integration-test
   script:
     # Debug failing CI caches which happened in the past
     - find . -name behave.ini
     - ls -al test/behave
     - grep -v 'nextcloud.password' test/behave/behave.ini
     - cd test/behave/
-    - behave -D headless=True
+    - behave -D headless=True || behave -D headless=True @rerun_failing.features
   artifacts:
     paths:
     - test/behave/screenshots/
     expire_in: 1 month
     when: on_failure
+  retry: 2
 
 terminate:
   stage: cleanup
diff --git a/test/ci-bootstrap.py b/test/ci-bootstrap.py
index 51b1222a2c7eacbc09c5b6c75f364c5b6a3ea253..367ec79aff4afd2bd7d9f500f5c0e396c01efb97 100755
--- a/test/ci-bootstrap.py
+++ b/test/ci-bootstrap.py
@@ -275,6 +275,8 @@ def write_behave_config(settings=None):
 
     behave_config = configparser.ConfigParser()
     behave_config['behave'] = {'stop': True}
+    behave_config['behave'] = {'format': 'rerun'}
+    behave_config['behave'] = {'outfiles': 'rerun_failing.features'}
     behave_config['behave.userdata'] = {}
     behave_config['behave.userdata']['nextcloud.url'] = \
         'https://files.{}'.format(settings['domain'])