diff --git a/openappstack/__main__.py b/openappstack/__main__.py
index f171548142cc3262f75b0f103d7e75f1f569ac27..44180ec29527e046f2081b8e30bf79e762f46d7c 100755
--- a/openappstack/__main__.py
+++ b/openappstack/__main__.py
@@ -42,7 +42,18 @@ ALL_TESTS = ['behave']
 
 
 def main():  # pylint: disable=too-many-statements,too-many-branches,too-many-locals
-    """Do everything."""
+    """
+    Parse arguments and start program. Depending on the arguments parsed,
+    one of three functions is called:
+
+    - create, responsible for setting up VPSs and their local configuration
+      files
+    - install, responsible for setting up Kubernetes and OpenAppStack on those
+      VPSs
+    - test, responsible for testing if all the setup steps worked.
+
+    Run python3 -m openappstack --help for more information.
+    """
     # Parse command line arguments
     parser = argparse.ArgumentParser(
         prog=name,
@@ -115,9 +126,9 @@ def main():  # pylint: disable=too-many-statements,too-many-branches,too-many-lo
     install_parser.set_defaults(func=install)
 
     parser.add_argument(
-        '--terminate',
+        '--terminate-droplet',
         action='store_true',
-        help=('Shutdown and delete droplet identified by VPS identifier after '
+        help=('Shutdown AND DELETE droplet identified by VPS identifier after '
               'finishing'))
 
     parser.add_argument(
@@ -183,11 +194,12 @@ def main():  # pylint: disable=too-many-statements,too-many-branches,too-many-lo
     if hasattr(args, 'func'):
         args.func(clus, args)
 
-    if args.terminate:
+    if args.terminate_droplet:
         # In case none of the subparser's functions have been called, load data
         clus.load_data()
         cosmos.terminate_droplets_by_name(clus.hostname)
 
+
 def info(clus, _args):
     """Shows cluster information and then exits"""
     clus.load_data()
@@ -256,7 +268,8 @@ def test(clus, args):
         # necessary files
         os.chdir(behave_path)
         clus.load_data()
-        clus.write_behave_config(os.path.join(behave_path, 'behave.ini'))
+        behave_ini = os.path.join(behave_path, 'behave.ini')
+        clus.write_behave_config(behave_ini)
         command = []
         if args.behave_rerun_failing:
             command.append("@rerun_failing.features")
@@ -268,6 +281,10 @@ def test(clus, args):
         log.info("Running behave command %s", command)
         behave_main(command)
 
+        # Remove behave.ini so we don't leave secrets hanging around.
+        os.remove(behave_ini)
+
+
 def create_domain_records(domain, subdomain, droplet_ip):
     """
     Creates domain records at Greenhost in the way OpenAppStack expects it
@@ -284,8 +301,6 @@ def create_domain_records(domain, subdomain, droplet_ip):
     log.info("Domain record: %s", domain_record)
 
 
-
-
 def init_logging(logger, loglevel):
     """
     Configure logging.
diff --git a/openappstack/cluster.py b/openappstack/cluster.py
index e6fd08a75783598d9a7d9aefc2ba8db7f7a63f22..260c2a41efcb89a07592aaba3311e4a4dd10e807 100644
--- a/openappstack/cluster.py
+++ b/openappstack/cluster.py
@@ -5,6 +5,7 @@ import logging
 import os
 import random
 import string
+import sys
 import yaml
 from openappstack import ansible, cosmos
 
@@ -15,7 +16,6 @@ log = logging.getLogger(__name__)  # pylint: disable=invalid-name
 class Cluster:
     """helper class for cluster-related paths, files, etc."""
 
-
     def __init__(self, cluster_name, load_data=False):
         self.name = cluster_name
         self.cluster_dir = os.path.join(CLUSTER_PATH, cluster_name)
@@ -46,7 +46,6 @@ class Cluster:
 
         log.debug("Read data from inventory.yml:\n\thostname: %s", self.hostname)
 
-
     def create_droplet(self, ssh_key_id=0, hostname=None):
         """Uses the Cosmos API to create a droplet with OAS default spec"""
         if hostname is None:
@@ -123,7 +122,19 @@ class Cluster:
         return os.path.join(self.cluster_dir, 'secrets')
 
     def write_behave_config(self, config_path):
-        """Write behave config file for the cluster."""
+        """
+        Write behave config file for the cluster.
+
+        Configuration is written to config_path (e.g.
+        /home/you/openappstack/test/behave.ini)
+
+        If config_path already exists, the program is aborted.
+        """
+        if os.path.isfile(config_path):
+            log.error('%s file already exists, not overwriting '
+                      'file! Remove the file if you want to run behave. '
+                      'Program will exit now', config_path)
+            sys.exit(2)
         secret_directory = self.secret_dir
         with open(os.path.join(
                 secret_directory, 'nextcloud_admin_password'), 'r') as stream: