diff --git a/test/ci-bootstrap.py b/test/ci-bootstrap.py
index 367ec79aff4afd2bd7d9f500f5c0e396c01efb97..89c8c3a2332091adfd7b4a3dcd14a967c1535ef2 100755
--- a/test/ci-bootstrap.py
+++ b/test/ci-bootstrap.py
@@ -274,9 +274,9 @@ def write_behave_config(settings=None):
         nextcloud_admin_password = yaml.safe_load(stream)
 
     behave_config = configparser.ConfigParser()
-    behave_config['behave'] = {'stop': True}
-    behave_config['behave'] = {'format': 'rerun'}
-    behave_config['behave'] = {'outfiles': 'rerun_failing.features'}
+    behave_config['behave'] = {}
+    behave_config['behave']['format'] = 'rerun'
+    behave_config['behave']['outfiles'] = 'rerun_failing.features'
     behave_config['behave.userdata'] = {}
     behave_config['behave.userdata']['nextcloud.url'] = \
         'https://files.{}'.format(settings['domain'])