diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 58dd4ec66b35e7cb137d1118d48923b822c58a4b..b9961957fbb3796b06e8d216e3ee8a2fe4496086 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,9 +15,10 @@ variables:
   # Repeated values, because we're not allowed to use a variable in a variable
   SUBDOMAIN: "${CI_COMMIT_REF_SLUG}.ci"
   DOMAIN: "openappstack.net"
-  ADDRESS: "${CI_COMMIT_REF_SLUG}.ci.openappstack.net"
+  FQDN: "${CI_COMMIT_REF_SLUG}.ci.openappstack.net"
   ANSIBLE_HOST_KEY_CHECKING: "False"
   KANIKO_BUILD_IMAGENAME: "openappstack-ci"
+  CLUSTER_DIR: "/builds/openappstack/openappstack/clusters/${CI_COMMIT_REF_SLUG}"
 
 default:
   image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_COMMIT_REF_SLUG}"
@@ -36,7 +37,7 @@ ci_test_image:
 create-vps:
   stage: create-vps
   script:
-    - echo "hostname $HOSTNAME, subdomain $SUBDOMAIN, domain $DOMAIN, address $ADDRESS";
+    - echo "hostname $HOSTNAME, subdomain $SUBDOMAIN, domain $DOMAIN, FQDN $FQDN";
     - ls clusters/${HOSTNAME} || echo "directory clusters/${HOSTNAME} not found"
     # Creates the VPS only if an old VPS for this branch is not re-usable
     - sh .gitlab/ci_scripts/create_vps.sh
@@ -66,7 +67,7 @@ setup-openappstack:
     # Copy inventory files to ansible folder for use in install-apps step
     - chmod 700 ansible
     - cp clusters/${CI_COMMIT_REF_SLUG}/inventory.yml ansible/
-    - cp clusters/${CI_COMMIT_REF_SLUG}/settings.yml ansible/group_vars/all/
+    - cp clusters/${CI_COMMIT_REF_SLUG}/group_vars/all/settings.yml ansible/group_vars/all/
     # Set up cluster
     - python3 -m openappstack $HOSTNAME install
     # Show versions of installed apps/binaries
@@ -99,7 +100,7 @@ test_helmreleases:
   script:
     - cd ansible/
     - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml"
-    - pytest -v -s -m 'helmreleases' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10
+    - pytest -v -s -m 'helmreleases' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10
   only:
     changes:
       - .gitlab-ci.yml
@@ -113,7 +114,7 @@ testinfra:
   stage: health-test
   script:
     - cd ansible/
-    - pytest -v -m 'testinfra' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*'
+    - pytest -v -m 'testinfra' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
   only:
     changes:
       - .gitlab-ci.yml
@@ -128,7 +129,7 @@ certs:
   allow_failure: true
   script:
     - cd ansible/
-    - pytest -s -m 'certs' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*'
+    - pytest -s -m 'certs' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
   only:
     changes:
       - .gitlab-ci.yml
@@ -140,12 +141,10 @@ certs:
 
 prometheus-alerts:
   stage: health-test
-  variables:
-    OAS_DOMAIN: '${CI_COMMIT_REF_SLUG}.ci.openappstack.net'
   allow_failure: true
   script:
     - cd test/
-    - pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*'
+    - pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
   only:
     changes:
       - .gitlab-ci.yml
@@ -158,9 +157,9 @@ behave-nextcloud:
   stage: integration-test
   script:
     # Wait until flux creates the NextCloud HelmRelease.
-    - ssh root@$ADDRESS '/bin/bash -c "while true; do kubectl get hr -n oas-apps nextcloud; if [ \$? -eq 0 ]; then break; fi; sleep 20; done"'
+    - ssh root@$FQDN '/bin/bash -c "while true; do kubectl get hr -n oas-apps nextcloud; if [ \$? -eq 0 ]; then break; fi; sleep 20; done"'
     # Wait until NextCloud is ready.
-    - ssh root@$ADDRESS '/bin/bash -c "kubectl wait -n oas-apps hr/nextcloud --for condition=Released --timeout=20m"'
+    - ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas-apps hr/nextcloud --for condition=Released --timeout=20m"'
     # Run the behave tests for NextCloud.
     - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags nextcloud || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags nextcloud
   artifacts:
diff --git a/.gitlab/ci_scripts/create_vps.sh b/.gitlab/ci_scripts/create_vps.sh
index 654963ada8c63111186579c718c2279e77d94eca..693afd0db2d4d1c68dd961e4c1d8279144deb0a8 100644
--- a/.gitlab/ci_scripts/create_vps.sh
+++ b/.gitlab/ci_scripts/create_vps.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env sh
 # Check if cluster directory was available from cache
-set -v
+
+set -ve
 
 if [ -d clusters/$HOSTNAME/secrets ]
 then
diff --git a/docs/index.rst b/docs/index.rst
index b10a8df901e4b57f299931d09873b9c12f3a5e17..68d3da6c0769bde964f2a663c48f493c954f2631 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -27,6 +27,7 @@ For more information, go to `the OpenAppStack website`_.
    :caption: Contents:
 
    installation_instructions
+   upgrading
    testing_instructions
    design
    reference
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 9a2600d2139556a1db9ffb7f2a7e04b43d37d234..1dc5d425f1631461570118d7b50a529af8848240 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -6,7 +6,7 @@ Note: `cluster$` indicates that the commands should be run as root on your OAS c
 
 If you encounter problems when you upgrade your cluster, please make sure first
 to include all potential new values of `ansible/group_vars/all/settings.yml.example`
-to your `clusters/YOUR_CLUSTERNAME/settings.yml`, and rerun the installation
+to your `clusters/YOUR_CLUSTERNAME/group_vars/all/settings.yml`, and rerun the installation
 script.
 
 
@@ -20,7 +20,7 @@ debug this:
 
 Did you create your cluster using the `--acme-staging` argument?
 Please check the resulting value of the `acme_staging` key in
-`clusters/YOUR_CLUSTERNAME/settings.yml`. If this is set to `true`, certificates
+`clusters/YOUR_CLUSTERNAME/group_vars/all/settings.yml`. If this is set to `true`, certificates
 are fetched from the [Let's Encrypt staging API](https://letsencrypt.org/docs/staging-environment/),
 which can't be validated by default in your browser.
 
diff --git a/docs/upgrading.md b/docs/upgrading.md
new file mode 100644
index 0000000000000000000000000000000000000000..3c0e7c520f27db99d190758dfa1cf089ee67291e
--- /dev/null
+++ b/docs/upgrading.md
@@ -0,0 +1,18 @@
+# Upgrading OpenAppStack
+
+## From versions earlier than 0.2.3
+
+Upgrading from versions earlier than `0.2.3` requires manual
+intervention.
+
+* Move your local `settings.yml` file to a different location:
+
+    ```
+    cd CLUSTER_DIR
+    mkdir ./group_vars/all
+    mv settings.yml ./group_vars/all/
+    ```
+
+* remove all helm charts (WARNING: Please backup all data before!):
+
+    `helm delete --purge oas-test-cert-manager oas-test-local-storage oas-test-prometheus oas-test-proxy oas-test-files`
diff --git a/openappstack/cluster.py b/openappstack/cluster.py
index 12880f030b24284eb12b98f80b554ec410356ca9..8d1b0ea8b0fc62bec11720e4e835b21731747296 100644
--- a/openappstack/cluster.py
+++ b/openappstack/cluster.py
@@ -152,6 +152,12 @@ class Cluster:
 
         file_contents = yaml.safe_dump(settings, default_flow_style=False)
         log.debug(file_contents)
+
+        # Create CLUSTER_DIR/group_vars/all/ if non-existent
+        vars_dir = os.path.dirname(self.settings_file)
+        if not os.path.exists(vars_dir):
+            os.makedirs(vars_dir)
+
         with open(self.settings_file, 'w') as stream:
             stream.write(file_contents)
             log.info("Created %s", self.settings_file)
@@ -172,7 +178,8 @@ class Cluster:
     @property
     def settings_file(self):
         """Path to the ansible settings.yml for this cluster"""
-        return os.path.join(self.cluster_dir, 'settings.yml')
+        return os.path.join(self.cluster_dir, 'group_vars', 'all',
+                            'settings.yml')
 
     @property
     def behave_file(self):
diff --git a/test/README.md b/test/README.md
index 6f7a9be65f53c166ca6f751578dfb94badc5f291..b91e561c2c81e2f7d4afd4a937fc8876d92940df 100644
--- a/test/README.md
+++ b/test/README.md
@@ -13,36 +13,36 @@ There are two types of tests: "testinfra" tests, and "behave" tests.
 
 ## Run *testinfra* tests
 
-Test host configured in `../clusters/CLUSTERNAME/inventory.yml`
+Export `CLUSTER_DIR` env var with the location of your cluster config directory:
 
-    export INVENTORY=../clusters/CLUSTERNAME/inventory.yml
-    py.test -v --ansible-inventory=${INVENTORY} --hosts='ansible://*'
+    export CLUSTER_DIR="../clusters/CLUSTERNAME"
+
+Run all tests:
+
+    py.test -sv --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
 
 Specify host manually:
 
-    py.test -v --hosts='ssh://root@example.openappstack.net'
+    py.test -sv --hosts='ssh://root@example.openappstack.net'
 
 Run only tests tagged with `prometheus`:
 
-    py.test -v --ansible-inventory=${INVENTORY} --hosts='ansible://*' -m prometheus
+    py.test -sv --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' -m prometheus
 
 ### Cert tests
 
 Run cert test manually using the ansible inventory file:
 
-    ADDRESS='example.openappstack.net' py.test -v -m 'certs' \
-      --connection=ansible \
-      --ansible-inventory=${INVENTORY} \
-      --hosts='ansible://*'
+    py.test -sv --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' -m certs
 
 Run cert test manually against a different cluster, not configured in any
 ansible inventory file, either by using pytest:
 
-    ADDRESS='example.openappstack.net' py.test -v -m 'certs'
+    FQDN='example.openappstack.net' py.test -sv -m 'certs'
 
-or directly (allows better debugging since pytest won't eat stdout):
+or directly:
 
-    ADDRESS='example.openappstack.net' pytest/test_certs.py
+    FQDN='example.openappstack.net' pytest/test_certs.py
 
 
 ## Run *behave* tests
diff --git a/test/pytest/test_certs.py b/test/pytest/test_certs.py
index 472311e1613aca6db185ddb97918beed9e658ee0..294803496c76ed8e64a4f1521976dd34d705cf7d 100755
--- a/test/pytest/test_certs.py
+++ b/test/pytest/test_certs.py
@@ -12,12 +12,12 @@ from OpenSSL import SSL
 
 def add_custom_cert_authorities(ca_file: str,
                                 custom_ca_files: list,
-                                destination_file: str =
+                                dest_file: str =
                                 '/tmp/custom_ca_bundle.crt'):
     """Concatenates existing cert bundle with custom CAs."""
 
-    destination = open(destination_file, 'wb')
-    with open(destination_file, 'wb') as destination,  open(ca_file, 'rb') as ca:
+    destination = open(dest_file, 'wb')
+    with open(dest_file, 'wb') as destination, open(ca_file, 'rb') as ca:
         shutil.copyfileobj(ca, destination)
         for custom_ca_file in custom_ca_files:
             with open(custom_ca_file, 'rb') as custom_ca:
@@ -59,7 +59,7 @@ def print_cert_info(certs: list):
         print('CN: {0} (Issuer: {1})'.format(cn, issuer))
 
 
-def read_certs_from_file(filename:str):
+def read_certs_from_file(filename: str):
     """Read cert from file for debugging/development."""
 
     import OpenSSL.crypto
@@ -97,14 +97,26 @@ def valid_cert(domain: str, ca_file: str = '/tmp/custom_ca_bundle.crt'):
 
 @pytest.mark.certs
 def test_cert_validation(host):
-    domain = os.environ.get("ADDRESS")
-    assert domain, "Please export ADDRESS as environment variable."
+    """Checks for proper cluster certs from exposed services.
+    Check is executed on the local provisioning machine.
+    """
+
+    # Use FQDN env var if set, otherwise use domain var from
+    # settings.yml.
+    domain = os.environ.get("FQDN")
+    if domain:
+        print("Using domain %s from FQDN environment variable." % domain)
+    else:
+        ansible_vars = host.ansible.get_variables()
+        domain = ansible_vars["domain"]
+        print("Using domain %s from ansible settings.yml." % domain)
 
     add_custom_cert_authorities(certifi.where(),
                                 ['pytest/le-staging-bundle.pem'])
 
-    # Check nextcloud cert
     assert valid_cert('files.{0}'.format(domain))
+    assert valid_cert('office.{0}'.format(domain))
+    assert valid_cert('grafana.{0}'.format(domain))
 
 
 if __name__ == "__main__":
diff --git a/test/pytest/test_helmreleases.py b/test/pytest/test_helmreleases.py
index 85483b2af1f8609313f325e6276f02ef7c771d5b..624cf04daa2bd0738c63f94512667424305f8de3 100644
--- a/test/pytest/test_helmreleases.py
+++ b/test/pytest/test_helmreleases.py
@@ -1,6 +1,7 @@
 import pytest
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
+import os
 
 
 def get_release(name, namespace, api):
@@ -26,7 +27,7 @@ def get_release(name, namespace, api):
 
 
 @pytest.mark.helmreleases
-def test_helmreleases():
+def test_helmreleases(host):
     """Checks if all desired HelmReleases installed by weave flux are in
     DEPLOYED state.
     """
@@ -35,7 +36,10 @@ def test_helmreleases():
         'oas-apps': ['nextcloud']
     }
 
-    config.load_kube_config()
+    cluster_dir = os.environ.get("CLUSTER_DIR")
+    kubeconfig = os.path.join(cluster_dir, 'secrets',
+                              'kube_config_cluster.yml')
+    config.load_kube_config(config_file=kubeconfig)
     customObjects = client.CustomObjectsApi()
 
     failed = 0
diff --git a/test/pytest/test_system.py b/test/pytest/test_system.py
index aa0d3ca5bf7c5e1d9f18313f1ef0237ae250ef65..be7192f4be31869bfaef9e814ab75c62410fcf2c 100644
--- a/test/pytest/test_system.py
+++ b/test/pytest/test_system.py
@@ -2,6 +2,6 @@ import pytest
 
 
 @pytest.mark.testinfra
-def test_release_is_bionic(host):
+def test_os_release(host):
     system_info = host.system_info
     assert system_info.release == '10'