diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4f5102d23efd567316ac16976badabd9cccbc88e..3bd2b3950ef1c9051a6b8782da575dd9650743f0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,12 +37,11 @@ bootstrap:
     - cd test/
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
-    - ANSIBLE_HOST_KEY_CHECKING=False python3 -m openappstack --create-droplet --hostname "ci-$CI_PIPELINE_ID" --ssh-key-id 411 --create-domain-records openappstack.net --domain-record-subdomain "ci-$CI_PIPELINE_ID.ci" --run-ansible --ansible-param skip-tags=helmfile
+    - python3 -m openappstack $CI_PIPELINE_ID create --create-droplet openappstack.net --hostname "ci-$CI_PIPELINE_ID" --ssh-key-id 411 --create-domain-records --subdomain "ci-$CI_PIPELINE_ID.ci" 
+    - python3 -m openappstack $CI_PIPELINE_ID install --ansible-param skip-tags=helmfile
   artifacts:
     paths:
-    - test/cluster_data/rke.log
-    - test/inventory.yml
-    - test/group_vars/all/settings.yml
+    - clusters
     expire_in: 1 month
     when: always
   only:
@@ -62,12 +61,13 @@ install:
     - cd test/
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
-    - python3 -u ./ci-bootstrap.py --use-existing-inventory --run-ansible --ansible-param tags=helmfile --write-behave-config
+    - python3 -m openappstack $CI_PIPELINE_ID install --ansible-param tags=helmfile 
+    - python3 -m openappstack $CI_PIPELINE_ID test --write-behave-config
     # Show versions of installed apps/binaries
     - ansible master -m shell -a 'oas-version-info.sh 2>&1'
   artifacts:
     paths:
-    - test/behave/behave.ini
+    - clusters
     expire_in: 1 month
   only:
     changes:
@@ -112,7 +112,7 @@ behave-nextcloud:
     - ls -al test/behave
     - grep -v 'password' test/behave/behave.ini
     - cd test/behave/
-    - behave -D headless=True -t nextcloud || behave -D headless=True @rerun_failing.features -t nextcloud
+    - python -m openappstack $CI_PIPELINE_ID test --behave --behave-headless --behave-tags nextcloud || python -m openappstack $CI_PIPELINE_ID test --behave --behave-headless --behave-rerun-failing --behave-tags nextcloud
   artifacts:
     paths:
     - test/behave/screenshots/
@@ -133,7 +133,7 @@ behave-grafana:
     - ls -al test/behave
     - grep -v 'password' test/behave/behave.ini
     - cd test/behave/
-    - behave -D headless=True -t grafana || behave -D headless=True @rerun_failing.features -t grafana
+    - python -m openappstack $CI_PIPELINE_ID test --behave --behave-headless --behave-tags grafana || python -m openappstack $CI_PIPELINE_ID test --behave --behave-headless --behave-rerun-failing --behave-tags grafana
   artifacts:
     paths:
     - test/behave/screenshots/
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
index 827f17f5576bde9fb2df95819785946c04c872f0..6a3c2ec672e946f217bb711c439cecea667b5146 100644
--- a/ansible/ansible.cfg
+++ b/ansible/ansible.cfg
@@ -5,3 +5,4 @@ nocows = 1
 stdout_callback = yaml
 strategy_plugins = plugins/mitogen-0.2.8-pre/ansible_mitogen/plugins/strategy
 strategy = mitogen_linear
+host_key_checking = False
diff --git a/ansible/group_vars/all/oas.yml b/ansible/group_vars/all/oas.yml
index b356fa9beb693cfd1ed9523a1a7d33e86e79ae13..3f456b8fc42f8ac3ea2b794abb3c17fd2079cd3f 100644
--- a/ansible/group_vars/all/oas.yml
+++ b/ansible/group_vars/all/oas.yml
@@ -11,10 +11,10 @@ ansible_python_interpreter: "/usr/bin/env python3"
 # Nextcloud administrator password. If you do not change this value, it gets
 # generated and stored in `{{ secret_directory }}/nextcloud_admin_password`.
 # You can also choose your own password and fill it in here instead.
-nextcloud_password: "{{ lookup('password', '{{ secret_directory }}/nextcloud_admin_password chars=ascii_letters') }}"
-nextcloud_mariadb_password: "{{ lookup('password', '{{ secret_directory }}/nextcloud_mariadb_password chars=ascii_letters') }}"
-nextcloud_mariadb_root_password: "{{ lookup('password', '{{ secret_directory }}/nextcloud_mariadb_root_password chars=ascii_letters') }}"
-grafana_admin_password: "{{ lookup('password', '{{ secret_directory }}/grafana_admin_password chars=ascii_letters') }}"
+nextcloud_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/nextcloud_admin_password chars=ascii_letters') }}"
+nextcloud_mariadb_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/nextcloud_mariadb_password chars=ascii_letters') }}"
+nextcloud_mariadb_root_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/nextcloud_mariadb_root_password chars=ascii_letters') }}"
+grafana_admin_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/grafana_admin_password chars=ascii_letters') }}"
 
 # git repo versions
 git_charts_version: 'HEAD'
diff --git a/ansible/group_vars/all/settings.yml.example b/ansible/group_vars/all/settings.yml.example
index 6e84e7b5fa8624a4921a0a04b6ae86194c960b49..882b142674800a04019a63f9af495e19ac94f362 100644
--- a/ansible/group_vars/all/settings.yml.example
+++ b/ansible/group_vars/all/settings.yml.example
@@ -13,7 +13,7 @@ release_name: "test"
 # Important: Don't quote this variable !
 acme_staging: false
 # Directory where we save all the passwords etc.
-secret_directory: "./cluster_data/secrets"
+cluster_dir: "./clusters/example-clusterrname/"
 # Which apps to install from the helmfile.d/ dir
 helmfiles:
   - 00-storage
diff --git a/ansible/roles/setup/tasks/rke.yml b/ansible/roles/setup/tasks/rke.yml
index 2229aee443634be973939402a5612b23f3c21ccf..2d577cc7569ac7971a3056386aed94b499419c17 100644
--- a/ansible/roles/setup/tasks/rke.yml
+++ b/ansible/roles/setup/tasks/rke.yml
@@ -32,8 +32,8 @@
     flat: yes
   loop:
     - src: "{{ data_directory }}/rke/kube_config_cluster.yml"
-      dest: "{{ secret_directory }}/kube_config_cluster.yml"
+      dest: "{{ cluster_dir }}/secrets/kube_config_cluster.yml"
     - src: "{{ log_directory }}/rke.log"
-      dest: cluster_data/rke.log
+      dest: "{{ cluster_dir }}/rke.log"
     - src: "{{ data_directory }}/rke/cluster.yml"
-      dest: cluster_data/rke_cluster.yml
+      dest: "{{ cluster_dir }}/rke_cluster.yml"
diff --git a/openappstack/__main__.py b/openappstack/__main__.py
index 0eaf2dd398ea7110322780ee6d589a36202aecb1..0fc7d177b76a8b07425a49dd04398b66313a942a 100755
--- a/openappstack/__main__.py
+++ b/openappstack/__main__.py
@@ -35,6 +35,7 @@ import argparse
 import logging
 import os
 import sys
+from behave.__main__ import main as behave_main
 from openappstack import name, cluster, cosmos, ansible
 
 
@@ -135,16 +136,31 @@ def main():  # pylint: disable=too-many-statements,too-many-branches,too-many-lo
     test_parser = subparsers.add_parser(
         'test',
         help=("Write test configuration and run tests on your cluster"))
+
     test_parser.set_defaults(func=test)
 
     test_parser.add_argument(
-        '--write-behave-config',
+        '--behave',
+        action='store_true',
+        help='Runs "behave" tests. NOTE: This overrides behave.ini!')
+    test_parser.add_argument(
+        '--behave-rerun-failing',
         action='store_true',
-        help='Writes a configuration file for behave with cluster information')
+        help=('Run behave with @rerun_failing.features'))
+    test_parser.add_argument(
+        '--behave-tags',
+        nargs='+',
+        default=[],
+        help=('Only run behave tests with these tags'))
+    test_parser.add_argument(
+        '--behave-headless',
+        action='store_true',
+        help=('Run behave in headless mode'))
 
     info_parser = subparsers.add_parser(
         'info',
         help=("Show information about a cluster"))
+
     info_parser.set_defaults(func=info)
 
     args = parser.parse_args()
@@ -177,7 +193,11 @@ def create(clus, args):
 
     if args.create_droplet:
         if not args.ssh_key_id:
-            log.error("SSH Key id required when using --create-droplet")
+            log.error("--ssh-key-id required when using --create-droplet")
+            sys.exit(1)
+        if not args.hostname:
+            log.error("--hostname required when using --create-droplet")
+            sys.exit(1)
 
     if args.subdomain:
         domain = "{subdomain}.{domain}".format(
@@ -186,17 +206,17 @@ def create(clus, args):
         domain = args.domain
     clus.domain = domain
     if args.create_droplet:
-        clus.create_droplet(*args)
+        clus.create_droplet(ssh_key_id=args.ssh_key_id, hostname=args.hostname)
         if args.verbose:
             cosmos.list_droplets()
+        # Wait for ssh
+        cosmos.wait_for_ssh(clus.ip_address)
     elif args.droplet_id:
         clus.set_info_by_droplet_id(args.droplet_id)
 
     # Write inventory.yml and settings.yml files
     clus.write_cluster_files()
 
-    # Wait for ssh
-    cosmos.wait_for_ssh(clus.ip_address)
 
     if args.create_domain_records:
         create_domain_records(args.domain, args.subdomain, clus.ip_address)
@@ -214,11 +234,26 @@ def install(clus, args):
 
 
 def test(clus, args):
-    """Parses arguments for the "test" subcommand"""
-    clus.load_data()
-    if args.write_behave_config:
-        clus.write_behave_config()
-
+    """Runs behave or testinfra test. Overwrites behave_path/behave.ini!"""
+    if args.behave:
+        behave_path = os.path.join(os.path.dirname(__file__), '..', 'test',
+                                   'behave')
+        # Run from the behave directory so behave automatically loads all the
+        # necessary files
+        os.chdir(behave_path)
+        clus.load_data()
+        clus.write_behave_config(os.path.join(behave_path, 'behave.ini'))
+        command = []
+        if args.behave_rerun_failing:
+            command.append("@rerun_failing.features")
+        log.info(args.behave_tags)
+        if args.behave_headless:
+            command.append("-D headless=True")
+        for tag in args.behave_tags:
+            log.info(command)
+            command.append("-t {tag}".format(tag=tag))
+        log.info("Running behave command %s", command)
+        behave_main(command)
 
 def create_domain_records(domain, subdomain, droplet_ip):
     """
diff --git a/openappstack/ansible.py b/openappstack/ansible.py
index 77cc20a5239ba247e839bba77d6b70b0011cfb67..413eec87b2b7083146eb7f2445e0627bc53df320 100644
--- a/openappstack/ansible.py
+++ b/openappstack/ansible.py
@@ -79,8 +79,9 @@ def create_inventory(cluster):
 
     inventory['all']['hosts'][cluster.hostname]['ansible_host'] = \
         cluster.ip_address
-    inventory['all']['hosts'][cluster.hostname]['ansible_ssh_extra_args'] = \
-        '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+    # Moved to ansible.cfg (how can we re-enable host key checking?)
+    # inventory['all']['hosts'][cluster.hostname]['ansible_ssh_extra_args'] = \
+    #     '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
     inventory['all']['children']['master']['hosts'] = cluster.hostname
     inventory['all']['children']['worker']['hosts'] = cluster.hostname
 
diff --git a/openappstack/cluster.py b/openappstack/cluster.py
index 17f4d034c52cd53302ffa4fed3c220ff419ee3f0..cf2837fbeb649258ece8f77bfdc7f74849251813 100644
--- a/openappstack/cluster.py
+++ b/openappstack/cluster.py
@@ -90,7 +90,7 @@ class Cluster:
         settings['domain'] = self.domain
         settings['admin_email'] = "admin@{0}".format(self.domain)
         settings['acme_staging'] = True
-        settings['secret_directory'] = self.secret_dir
+        settings['cluster_dir'] = self.cluster_dir
 
         with open(self.settings_file, 'w') as stream:
             yaml.safe_dump(settings, stream, default_flow_style=False)
@@ -122,7 +122,7 @@ class Cluster:
         """Path where all the passwords for cluster admins are saved"""
         return os.path.join(self.cluster_dir, 'secrets')
 
-    def write_behave_config(self):
+    def write_behave_config(self, config_path):
         """Write behave config file for the cluster."""
         secret_directory = self.secret_dir
         with open(os.path.join(
@@ -153,8 +153,8 @@ class Cluster:
         behave_config['behave.userdata']['grafana.password'] = \
             grafana_admin_password
 
-        with open(self.behave_file, 'w') as configfile:
-            behave_config.write(configfile)
+        with open(config_path, 'w') as config_file:
+            behave_config.write(config_file)
 
     def print_info(self):
         """Writes information about the cluster. Useful for debugging"""
@@ -165,12 +165,10 @@ class Cluster:
 
 Configuration:
   - Inventory file: {inventory_file}
-  - Settings file: {settings_file}
-  - Behave file: {behave_file}""").format(
+  - Settings file: {settings_file}""").format(
             name=self.name,
             ip_address=self.ip_address,
             hostname=self.hostname,
             domain=self.domain,
             inventory_file=self.inventory_file,
-            settings_file=self.settings_file,
-            behave_file=self.behave_file))
+            settings_file=self.settings_file))
diff --git a/openappstack/cosmos.py b/openappstack/cosmos.py
index 9655052f4b8a741c73d2877d4a70b5170f558774..fc69c25113f2453a3dbb27589b45f499a8307230 100755
--- a/openappstack/cosmos.py
+++ b/openappstack/cosmos.py
@@ -304,6 +304,8 @@ def wait_for_ssh(ip: str):
     while sock.connect_ex((ip, 22)) != 0:
         sleep(1)
 
+    log.info('SSH became available on ip %s', ip)
+
 
 def wait_for_state(id: int, state):
     """Wait for a droplet to reach a certain state."""