diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e4aaa814c2863a36f5cfb3ae0a499731e61193df..ea45b4d9477f3e6ab57bc104f8d4fdfe8e59e29a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,7 +54,7 @@ bootstrap:
   artifacts:
     paths:
     - ansible/rke.log
-    - test/secrets/keycloak_admin_password
+    - test/behave/behave.ini
     expire_in: 1 month
     when: always
 
@@ -71,7 +71,7 @@ behave:
   script:
     # Run behave tests
     - cd test/behave/
-    - behave -D keycloak.admin.url=https://auth.ci-${CI_JOB_ID}.ci.openappstack.net/auth/admin/master/console/ -D keycloak.admin.password=$(cat ../secrets/keycloak_admin_password)
+    - behave
   artifacts:
     paths:
     - test/behave/screenshots/
diff --git a/test/ci-bootstrap.py b/test/ci-bootstrap.py
index 0499df81e75c4dec4c2ad8570dda36ee43e4ca39..cb36d10198668a3f83b2ce444da0762870e47cad 100755
--- a/test/ci-bootstrap.py
+++ b/test/ci-bootstrap.py
@@ -28,6 +28,7 @@ Install requirements:
 """
 
 import argparse
+import configparser
 import cosmos
 import logging
 import os
@@ -197,7 +198,7 @@ if __name__ == "__main__":
     # Bootstrap
     # playbook path here is relative to private_data_dir/project, see
     # https://ansible-runner.readthedocs.io/en/latest/intro.html#inputdir
-    playbook='./bootstrap.yml'
+    playbook = './bootstrap.yml'
     ansible_playbook_cmd = 'ansible-playbook %s' % playbook
 
     log.info('Running %s', ansible_playbook_cmd)
@@ -212,5 +213,19 @@ if __name__ == "__main__":
             traceback.print_exc()
             sys.exit(result.returncode)
 
+    # Write behave config file for later use
+    with open('./secrets/keycloak_admin_password', 'r') as stream:
+        keycloak_admin_password = yaml.load(stream)
+    behave_config = configparser.ConfigParser()
+    behave_config['behave'] = {'stop': True}
+    behave_config['behave.userdata'] = {}
+    behave_config['behave.userdata']['keycloak.admin.url'] = \
+        'https://auth.{}/auth/admin/master/console/'.format(settings['domain'])
+    behave_config['behave.userdata']['keycloak.admin.username'] = 'keycloak'
+    behave_config['behave.userdata']['keycloak.admin.password'] = \
+        keycloak_admin_password
+    with open('./behave/behave.ini', 'w') as configfile:
+        behave_config.write(configfile)
+
     if args.terminate:
         cosmos.terminate_droplet(id)