diff --git a/.gitignore b/.gitignore
index 93ea8ac213d062495152fb6a1ea52f02a1977fb1..8d3754031d39003b6015869e58cbe3ad1520eb9b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,10 @@
 /ansible/secrets/
 ansible/rke.log
 
+# Virtualenvs for the project
+/env
+/venv
+
 # Ignore files created during CI using test/ci-bootstrap.py
 /test/group_vars/
 /test/secrets/
@@ -18,3 +22,4 @@ ansible/rke.log
 
 # Etc
 __pycache__
+*.swp
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a46c63bddca73c1b6887098775233cac50a329d1..cbc618cfb09759db922be2c70e380d7b3207a506 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
 stages:
   - build
-  - deploy
+  - setup-cluster
+  - install-apps
   - test
   - e2e-test
   - cleanup
@@ -24,7 +25,7 @@ ci_test_image:
       - test/requirements.txt
 
 bootstrap:
-  stage: deploy
+  stage: setup-cluster
   image: "${CI_REGISTRY_IMAGE}/bootstrap-ci"
   script:
     # Ensure test/ is not world-writable otherwise ansible-playbook refuses to run, see
@@ -33,18 +34,33 @@ bootstrap:
     - cd test/
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
-    - ANSIBLE_HOST_KEY_CHECKING=False python3 -u ./ci-bootstrap.py --create_droplet
+    - ANSIBLE_HOST_KEY_CHECKING=False python3 -u ./ci-bootstrap.py --create-droplet --create-domain-records --run-ansible --ansible-param skip-tags=helmfile
   cache:
     key: "$CI_PIPELINE_ID"
     paths:
-    - test/behave/behave.ini
     - test/inventory.yml
+    - test/group_vars/all.yml
   artifacts:
     paths:
     - ansible/rke.log
     expire_in: 1 month
     when: always
 
+install:
+  stage: install-apps
+  image: "${CI_REGISTRY_IMAGE}/bootstrap-ci"
+  cache:
+    key: "$CI_PIPELINE_ID"
+    paths:
+    - test/inventory.yml
+    - test/group_vars/all.yml
+    - test/behave/behave.ini
+  script:
+    - cd test/
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
+    - ANSIBLE_HOST_KEY_CHECKING=False python3 -u ./ci-bootstrap.py --use-existing-inventory --run-ansible --ansible-param tags=helmfile --write-behave-config
+
 testinfra:
   stage: test
   image: "${CI_REGISTRY_IMAGE}/bootstrap-ci"
@@ -88,7 +104,13 @@ behave:
 terminate:
   stage: cleanup
   image: "${CI_REGISTRY_IMAGE}/bootstrap-ci"
+  cache:
+    key: "$CI_PIPELINE_ID"
+    paths:
+    - test/inventory.yml
+    policy: pull
   script:
     # Remove droplet after successful tests
     - cd test/
-    - python3 -c "import cosmos; cosmos.terminate_droplets_by_name(\"^ci-${CI_PIPELINE_ID}\$\")"
+    # - python3 -c "import cosmos; cosmos.terminate_droplets_by_name(\"^ci-${CI_PIPELINE_ID}\$\")"
+    - python3 -u ./ci-bootstrap.py --use-existing-inventory --terminate
diff --git a/ansible/group_vars/cluster/settings.yml.example b/ansible/group_vars/cluster/settings.yml.example
index f2133a3e3beca97fef9d5339f216bd2b633c7469..b46ea3744a5cc28983ddceb333ce58854e671b03 100644
--- a/ansible/group_vars/cluster/settings.yml.example
+++ b/ansible/group_vars/cluster/settings.yml.example
@@ -11,10 +11,10 @@ release_name: "test"
 # Keycloak administrator password. If you do not change this value, it gets
 # generated and stored in ./secrets/keycloak_admin_password. You can also choose
 # your own password and fill it in here instead.
-keycloak_password: "{{ lookup('password', './secrets/keycloak_admin_password') }}"
+keycloak_password: "{{ lookup('password', './secrets/keycloak_admin_password chars=ascii_letters') }}"
 # Nextcloud administrator password. Works the same as keycloak password, except
 # it is stored in `secrets/nextcloud_admin_password`.
-nextcloud_password: "{{ lookup('password', './secrets/nextcloud_admin_password') }}"
+nextcloud_password: "{{ lookup('password', './secrets/nextcloud_admin_password chars=ascii_letters') }}"
 # If this is "true" TLS certificates will be requested at the Let's Encrypt
 # staging server. If this is "false", you use Let's Encrypt's production server.
 # Note that LE's production server has stricter rate limits, so set this to
diff --git a/test/ci-bootstrap.py b/test/ci-bootstrap.py
index 858533c774b5f7e5e1fdd1f50d23839688c956b1..c686bf79b22d07f9ae7c2d430261b9eacaa1362f 100755
--- a/test/ci-bootstrap.py
+++ b/test/ci-bootstrap.py
@@ -29,7 +29,6 @@ Install requirements:
 
 import argparse
 import configparser
-import cosmos
 import logging
 import os
 import random
@@ -40,28 +39,14 @@ import sys
 import traceback
 import yaml
 
+import cosmos
 
-def init_logging(log, loglevel):
-    """
-    Configure logging.
-
-    - debug and info go to stdout
-    - warning and above go to stderr
-    """
-    log.setLevel(loglevel)
-    stdout = logging.StreamHandler(sys.stdout)
-    stdout.setLevel(loglevel)
-    stdout.addFilter(lambda record: record.levelno <= logging.INFO)
-
-    stderr = logging.StreamHandler()
-    stderr.setLevel(logging.WARNING)
-
-    log.addHandler(stdout)
-    log.addHandler(stderr)
-
+SETTINGS_FILE = './group_vars/all.yml'
+ANSIBLE_INVENTORY = './inventory.yml'
 
-if __name__ == "__main__":
 
+def main():  # pylint: disable=too-many-statements,too-many-branches
+    """ does everything """
     # Parse command line arguments
     parser = argparse.ArgumentParser(
         description='Run bootstrap script'
@@ -69,17 +54,22 @@ if __name__ == "__main__":
 
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument(
-        '--create_droplet',
+        '--create-droplet',
         action='store_true',
         help='Create droplet automatically')
     group.add_argument(
-        '--droplet_id',
+        '--droplet-id',
         metavar='ID',
         type=int,
         help='ID of droplet to deploy to')
 
+    group.add_argument(
+        '--use-existing-inventory',
+        action='store_true',
+        help='Assumes inventory.yml has already been generated')
+
     parser.add_argument(
-        '--ssh_key_id',
+        '--ssh-key-id',
         metavar='ID',
         type=int,
         default=411,
@@ -95,114 +85,166 @@ if __name__ == "__main__":
         action='store_true',
         help='Be more verbose')
 
+    parser.add_argument(
+        '--ansible-param',
+        metavar=['PARAM[=VALUE]'],
+        action='append',
+        nargs=1,
+        help=('forward ansible parameters to the ansible-playbook call '
+              '(two dashes are prepended to PARAM)'))
+    parser.add_argument(
+        '--run-ansible',
+        action='store_true',
+        help='Runs the ansible bootstrap process')
+
+    parser.add_argument(
+        '--create-domain-records',
+        action='store_true',
+        help='Creates DNS entries for the cluster')
+
+    parser.add_argument(
+        '--write-behave-config',
+        action='store_true',
+        help='Writes a configuration file for behave with cluster information')
+
     args = parser.parse_args()
     verbose = args.verbose
     loglevel = logging.DEBUG if verbose else logging.INFO
-
-    # Setup logging for this script
-    log = logging.getLogger(__name__)
     init_logging(log, loglevel)
 
     # Setup logging for cosmos module
     log_cosmos = logging.getLogger('cosmos')
     init_logging(log_cosmos, loglevel)
 
-    # Start bootstrapping
-    if args.create_droplet:
-        # Create droplet
-
-        # image: 18 = Ubuntu xenial 18.04 x64
-        # ssh_keys
-        # - 411: ci, ed25519
-        # - 407: varac
-
-        if "CI_PIPELINE_ID" in os.environ:
-            instance_id = os.environ['CI_PIPELINE_ID']
+    if not args.use_existing_inventory:
+        # Start bootstrapping
+        if args.create_droplet:
+            # Create droplet
+
+            # image: 18 = Ubuntu xenial 18.04 x64
+            # ssh_keys
+            # - 411: ci, ed25519
+            # - 407: varac
+
+            if "CI_PIPELINE_ID" in os.environ:
+                instance_id = os.environ['CI_PIPELINE_ID']
+            else:
+                # Use random generated ID in case we're not running in gitlab CI
+                # and there's no CI_PIPELINE_ID env var
+                instance_id = ''.join(
+                    random.choice(string.ascii_lowercase + string.digits)
+                    for _ in range(10))
+
+            droplet = cosmos.create_droplet(
+                name='ci-' + instance_id,
+                ssh_key_id=args.ssh_key_id,
+                region='ams1',
+                size=4096,
+                disk=8,
+                image=18)
+            droplet_id = droplet['droplet']['id']
+            log.info('Created droplet id: %s', droplet_id)
+            cosmos.wait_for_state(droplet_id, 'running')
         else:
-            # Use random generated ID in case we're not running in gitlab CI
-            # and there's no CI_PIPELINE_ID env var
-            instance_id = ''.join(
-                random.choice(string.ascii_lowercase + string.digits)
-                for _ in range(10))
-
-        droplet = cosmos.create_droplet(
-            name='ci-' + instance_id,
-            ssh_key_id=args.ssh_key_id,
-            region='ams1',
-            size=4096,
-            disk=8,
-            image=18)
-        id = droplet['droplet']['id']
-        log.info('Created droplet id: %s', id)
-        cosmos.wait_for_state(id, 'running')
+            droplet_id = args.droplet_id
+
+        if verbose:
+            cosmos.list_droplets()
+
+        # Get droplet ip
+        droplet = cosmos.get_droplet(droplet_id)
+        droplet_ip = droplet['networks']['v4'][0]['ip_address']
+        droplet_name = droplet['name']
+
+        # Create inventory
+        with open('../ansible/inventory.yml.example', 'r') as stream:
+            inventory = yaml.safe_load(stream)
+        inventory['all']['hosts'][droplet_name] = inventory['all']['hosts']['oas-dev']
+        del inventory['all']['hosts']['oas-dev']
+
+        inventory['all']['hosts'][droplet_name]['ansible_host'] = droplet_ip
+        inventory['all']['hosts'][droplet_name]['ansible_ssh_extra_args'] = \
+            '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+        inventory['all']['children']['cluster']['hosts'] = droplet_name
+        inventory['all']['children']['master']['hosts'] = droplet_name
+        inventory['all']['children']['worker']['hosts'] = droplet_name
+
+        with open(ANSIBLE_INVENTORY, 'w') as stream:
+            yaml.safe_dump(inventory, stream, default_flow_style=False)
+
+        # Create settings
+        with open('../ansible/group_vars/cluster/settings.yml.example',
+                  'r') as stream:
+            settings = yaml.safe_load(stream)
+
+        settings['ip_address'] = droplet_ip
+        settings['domain'] = droplet_name + '.ci.openappstack.net'
+        settings['admin_email'] = "admin@{0}".format(settings['domain'])
+        settings['acme_staging'] = "true"
+
+        if not os.path.exists('./group_vars'):
+            os.mkdir('./group_vars')
+        with open(SETTINGS_FILE, 'w') as stream:
+            yaml.safe_dump(settings, stream, default_flow_style=False)
+
+        log.debug(yaml.safe_dump(inventory, default_flow_style=False))
+        log.debug(yaml.safe_dump(settings, default_flow_style=False))
+
+        # Wait for ssh
+        cosmos.wait_for_ssh(droplet_ip)
     else:
-        id = args.droplet_id
-
-    if verbose:
-        cosmos.list_droplets()
-
-    # Get droplet ip
-    droplet = cosmos.get_droplet(id)
-    ip = droplet['networks']['v4'][0]['ip_address']
-    name = droplet['name']
-
-    # Create domain records
-    domain_record = cosmos.create_domain_record(
-        domain='openappstack.net', name=name + '.ci', data=ip, record_type='A',
-        update=True)
-    log.info("Domain record: %s", domain_record)
-
-    domain_record = cosmos.create_domain_record(
-        domain='openappstack.net', name='*.' + name + '.ci', data=name + '.ci',
-        record_type='CNAME', update=True)
-    log.info("Domain record: %s", domain_record)
-
-    if verbose:
-        cosmos.list_domain_records('openappstack.net')
-
-    # Wait for ssh
-    cosmos.wait_for_ssh(ip)
-
-    # Create inventory
-    with open('../ansible/inventory.yml.example', 'r') as stream:
-        inventory = yaml.load(stream)
-    inventory['all']['hosts'][name] = inventory['all']['hosts']['oas-dev']
-    del inventory['all']['hosts']['oas-dev']
-
-    inventory['all']['hosts'][name]['ansible_host'] = ip
-    inventory['all']['hosts'][name]['ansible_ssh_extra_args'] = \
-        '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
-    inventory['all']['children']['cluster']['hosts'] = name
-    inventory['all']['children']['master']['hosts'] = name
-    inventory['all']['children']['worker']['hosts'] = name
-
-    with open('./inventory.yml', 'w') as stream:
-        yaml.dump(inventory, stream, default_flow_style=False)
-
-    # Create settings
-    with open('../ansible/group_vars/cluster/settings.yml.example',
-              'r') as stream:
-        settings = yaml.load(stream)
-
-    settings['ip_address'] = ip
-    settings['domain'] = name + '.ci.openappstack.net'
-    settings['admin_email'] = "admin@{0}".format(settings['domain'])
-    settings['acme_staging'] = "true"
-
-    if not os.path.exists('./group_vars'):
-        os.mkdir('./group_vars')
-    with open('./group_vars/all', 'w') as stream:
-        yaml.dump(settings, stream, default_flow_style=False)
-
-    log.debug(yaml.dump(inventory, default_flow_style=False))
-    log.debug(yaml.dump(settings, default_flow_style=False))
-
-    # Bootstrap
+        # Work with the master node from the inventory
+        with open(ANSIBLE_INVENTORY, 'r') as stream:
+            inventory = yaml.safe_load(stream)
+        droplet_name = inventory['all']['children']['master']['hosts']
+        droplet_ip = inventory['all']['hosts'][droplet_name]['ansible_host']
+        log.info("Read data from inventory:\n\tname: %s\n\tip: %s",
+                 droplet_name, droplet_ip)
+
+        # For if write_behave_config is called later:
+        settings = None
+
+    if args.create_domain_records:
+        # Create domain records
+        domain_record = cosmos.create_domain_record(
+            domain='openappstack.net', name=droplet_name + '.ci', data=droplet_ip, record_type='A',
+            update=True)
+        log.info("Domain record: %s", domain_record)
+
+        domain_record = cosmos.create_domain_record(
+            domain='openappstack.net', name='*.' + droplet_name + '.ci', data=droplet_name + '.ci',
+            record_type='CNAME', update=True)
+        log.info("Domain record: %s", domain_record)
+
+        if verbose:
+            cosmos.list_domain_records('openappstack.net')
+
+
+    if args.run_ansible:
+
+        run_ansible('./bootstrap.yml', args.ansible_param)
+
+    if args.write_behave_config:
+        write_behave_config(settings=settings)
+
+    if args.terminate:
+        cosmos.terminate_droplets_by_name(droplet_name)
+
+def run_ansible(playbook, ansible_params):
+    """ Calls `ansible-playbook` directly to run the specified playbook. """
     # playbook path here is relative to private_data_dir/project, see
     # https://ansible-runner.readthedocs.io/en/latest/intro.html#inputdir
-    playbook = './bootstrap.yml'
     ansible_playbook_cmd = 'ansible-playbook %s' % playbook
 
+    if ansible_params:
+        for param in ansible_params:
+            if len(param) > 1:
+                log.warning('More than 1 parameter. Ignoring the rest! Use '
+                            '--ansible-param several times to supply '
+                            'more than 1 parameter')
+            param = param[0]
+            ansible_playbook_cmd += ' --' + param
+
     log.info('Running %s', ansible_playbook_cmd)
 
     result = subprocess.run(shlex.split(ansible_playbook_cmd))
@@ -215,9 +257,13 @@ if __name__ == "__main__":
             traceback.print_exc()
             sys.exit(result.returncode)
 
-    # Write behave config file for later use
+def write_behave_config(settings=None):
+    """ Write behave config file for later use """
+    if settings is None:
+        with open(SETTINGS_FILE) as stream:
+            settings = yaml.safe_load(stream)
     with open('./secrets/keycloak_admin_password', 'r') as stream:
-        keycloak_admin_password = yaml.load(stream)
+        keycloak_admin_password = yaml.safe_load(stream)
     behave_config = configparser.ConfigParser()
     behave_config['behave'] = {'stop': True}
     behave_config['behave.userdata'] = {}
@@ -229,5 +275,26 @@ if __name__ == "__main__":
     with open('./behave/behave.ini', 'w') as configfile:
         behave_config.write(configfile)
 
-    if args.terminate:
-        cosmos.terminate_droplet(id)
+def init_logging(logger, loglevel):
+    """
+    Configure logging.
+
+    - debug and info go to stdout
+    - warning and above go to stderr
+    """
+    logger.setLevel(loglevel)
+    stdout = logging.StreamHandler(sys.stdout)
+    stdout.setLevel(loglevel)
+    stdout.addFilter(lambda record: record.levelno <= logging.INFO)
+
+    stderr = logging.StreamHandler()
+    stderr.setLevel(logging.WARNING)
+
+    logger.addHandler(stdout)
+    logger.addHandler(stderr)
+
+if __name__ == "__main__":
+    # Setup logging for this script
+    log = logging.getLogger(__name__)  # pylint: disable=invalid-name
+
+    main()
diff --git a/test/cosmos.py b/test/cosmos.py
index 8a45d830687b0b6e54d3fcc20df471f87c7b5251..a4449e60c8a6a573adb3d27925d428c75f95eb2b 100755
--- a/test/cosmos.py
+++ b/test/cosmos.py
@@ -19,7 +19,7 @@ def request_api(resource: str, request_type: str = 'GET',
     if 'COSMOS_API_TOKEN' in os.environ:
         api_token = os.environ['COSMOS_API_TOKEN']
     else:
-        raise ValueError('Please export the COSMOS_API_TOKEN'
+        raise ValueError('Please export the COSMOS_API_TOKEN '
                          'environment variable.')
 
     headers = {'Content-Type': 'application/json',