Merge branch 'retry_behave_test' into 'master'

Retry behave job 5 times with greater timeout

Closes #139

See merge request openappstack/bootstrap!90

.gitignore

@@ -20,6 +20,7 @@ test/rke.log
 /test/secrets/
 /test/inventory.yml
 /test/behave/behave.ini
+/test/behave/rerun_failing.features
 
 # Ignore files created during tests
 /test/behave/**/screenshots/

.gitlab-ci.yml

@@ -2,7 +2,8 @@ stages:
   - build
   - setup-cluster
   - install-apps
-  - test
+  - health-test
+  - integration-test
   - cleanup
 
 image: "${CI_REGISTRY_IMAGE}/bootstrap-ci:${CI_COMMIT_REF_NAME}"
@@ -58,7 +59,7 @@ install:
     expire_in: 1 month
 
 testinfra:
-  stage: test
+  stage: health-test
   script:
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
@@ -66,7 +67,7 @@ testinfra:
     - py.test -v -m 'testinfra' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
 
 certs:
-  stage: test
+  stage: health-test
   variables:
     OAS_DOMAIN: 'ci-${CI_PIPELINE_ID}.ci.openappstack.net'
   allow_failure: true
@@ -77,19 +78,20 @@ certs:
     - py.test -s -m 'certs' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
 
 behave:
-  stage: test
+  stage: integration-test
   script:
     # Debug failing CI caches which happened in the past
     - find . -name behave.ini
     - ls -al test/behave
     - grep -v 'nextcloud.password' test/behave/behave.ini
     - cd test/behave/
-    - behave -D headless=True
+    - behave -D headless=True || behave -D headless=True @rerun_failing.features
   artifacts:
     paths:
     - test/behave/screenshots/
     expire_in: 1 month
     when: on_failure
+  retry: 2
 
 terminate:
   stage: cleanup

test/ci-bootstrap.py

@@ -275,6 +275,8 @@ def write_behave_config(settings=None):
 
     behave_config = configparser.ConfigParser()
     behave_config['behave'] = {'stop': True}
+    behave_config['behave'] = {'format': 'rerun'}
+    behave_config['behave'] = {'outfiles': 'rerun_failing.features'}
     behave_config['behave.userdata'] = {}
     behave_config['behave.userdata']['nextcloud.url'] = \
         'https://files.{}'.format(settings['domain'])