diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58dd4ec66b35e7cb137d1118d48923b822c58a4b..b9961957fbb3796b06e8d216e3ee8a2fe4496086 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,9 +15,10 @@ variables: # Repeated values, because we're not allowed to use a variable in a variable SUBDOMAIN: "${CI_COMMIT_REF_SLUG}.ci" DOMAIN: "openappstack.net" - ADDRESS: "${CI_COMMIT_REF_SLUG}.ci.openappstack.net" + FQDN: "${CI_COMMIT_REF_SLUG}.ci.openappstack.net" ANSIBLE_HOST_KEY_CHECKING: "False" KANIKO_BUILD_IMAGENAME: "openappstack-ci" + CLUSTER_DIR: "/builds/openappstack/openappstack/clusters/${CI_COMMIT_REF_SLUG}" default: image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_COMMIT_REF_SLUG}" @@ -36,7 +37,7 @@ ci_test_image: create-vps: stage: create-vps script: - - echo "hostname $HOSTNAME, subdomain $SUBDOMAIN, domain $DOMAIN, address $ADDRESS"; + - echo "hostname $HOSTNAME, subdomain $SUBDOMAIN, domain $DOMAIN, FQDN $FQDN"; - ls clusters/${HOSTNAME} || echo "directory clusters/${HOSTNAME} not found" # Creates the VPS only if an old VPS for this branch is not re-usable - sh .gitlab/ci_scripts/create_vps.sh @@ -66,7 +67,7 @@ setup-openappstack: # Copy inventory files to ansible folder for use in install-apps step - chmod 700 ansible - cp clusters/${CI_COMMIT_REF_SLUG}/inventory.yml ansible/ - - cp clusters/${CI_COMMIT_REF_SLUG}/settings.yml ansible/group_vars/all/ + - cp clusters/${CI_COMMIT_REF_SLUG}/group_vars/all/settings.yml ansible/group_vars/all/ # Set up cluster - python3 -m openappstack $HOSTNAME install # Show versions of installed apps/binaries @@ -99,7 +100,7 @@ test_helmreleases: script: - cd ansible/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml" - - pytest -v -s -m 'helmreleases' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 + - pytest -v -s -m 'helmreleases' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 only: changes: - .gitlab-ci.yml @@ -113,7 +114,7 @@ testinfra: stage: health-test script: - cd ansible/ - - pytest -v -m 'testinfra' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*' + - pytest -v -m 'testinfra' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' only: changes: - .gitlab-ci.yml @@ -128,7 +129,7 @@ certs: allow_failure: true script: - cd ansible/ - - pytest -s -m 'certs' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*' + - pytest -s -m 'certs' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' only: changes: - .gitlab-ci.yml @@ -140,12 +141,10 @@ certs: prometheus-alerts: stage: health-test - variables: - OAS_DOMAIN: '${CI_COMMIT_REF_SLUG}.ci.openappstack.net' allow_failure: true script: - cd test/ - - pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*' + - pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' only: changes: - .gitlab-ci.yml @@ -158,9 +157,9 @@ behave-nextcloud: stage: integration-test script: # Wait until flux creates the NextCloud HelmRelease. - - ssh root@$ADDRESS '/bin/bash -c "while true; do kubectl get hr -n oas-apps nextcloud; if [ \$? -eq 0 ]; then break; fi; sleep 20; done"' + - ssh root@$FQDN '/bin/bash -c "while true; do kubectl get hr -n oas-apps nextcloud; if [ \$? -eq 0 ]; then break; fi; sleep 20; done"' # Wait until NextCloud is ready. - - ssh root@$ADDRESS '/bin/bash -c "kubectl wait -n oas-apps hr/nextcloud --for condition=Released --timeout=20m"' + - ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas-apps hr/nextcloud --for condition=Released --timeout=20m"' # Run the behave tests for NextCloud. - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags nextcloud || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags nextcloud artifacts: diff --git a/.gitlab/ci_scripts/create_vps.sh b/.gitlab/ci_scripts/create_vps.sh index 654963ada8c63111186579c718c2279e77d94eca..693afd0db2d4d1c68dd961e4c1d8279144deb0a8 100644 --- a/.gitlab/ci_scripts/create_vps.sh +++ b/.gitlab/ci_scripts/create_vps.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh # Check if cluster directory was available from cache -set -v + +set -ve if [ -d clusters/$HOSTNAME/secrets ] then diff --git a/docs/index.rst b/docs/index.rst index b10a8df901e4b57f299931d09873b9c12f3a5e17..68d3da6c0769bde964f2a663c48f493c954f2631 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,6 +27,7 @@ For more information, go to `the OpenAppStack website`_. :caption: Contents: installation_instructions + upgrading testing_instructions design reference diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 9a2600d2139556a1db9ffb7f2a7e04b43d37d234..1dc5d425f1631461570118d7b50a529af8848240 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -6,7 +6,7 @@ Note: `cluster$` indicates that the commands should be run as root on your OAS c If you encounter problems when you upgrade your cluster, please make sure first to include all potential new values of `ansible/group_vars/all/settings.yml.example` -to your `clusters/YOUR_CLUSTERNAME/settings.yml`, and rerun the installation +to your `clusters/YOUR_CLUSTERNAME/group_vars/all/settings.yml`, and rerun the installation script. @@ -20,7 +20,7 @@ debug this: Did you create your cluster using the `--acme-staging` argument? Please check the resulting value of the `acme_staging` key in -`clusters/YOUR_CLUSTERNAME/settings.yml`. If this is set to `true`, certificates +`clusters/YOUR_CLUSTERNAME/group_vars/all/settings.yml`. If this is set to `true`, certificates are fetched from the [Let's Encrypt staging API](https://letsencrypt.org/docs/staging-environment/), which can't be validated by default in your browser. diff --git a/docs/upgrading.md b/docs/upgrading.md new file mode 100644 index 0000000000000000000000000000000000000000..3c0e7c520f27db99d190758dfa1cf089ee67291e --- /dev/null +++ b/docs/upgrading.md @@ -0,0 +1,18 @@ +# Upgrading OpenAppStack + +## From versions earlier than 0.2.3 + +Upgrading from versions earlier than `0.2.3` requires manual +intervention. + +* Move your local `settings.yml` file to a different location: + + ``` + cd CLUSTER_DIR + mkdir ./group_vars/all + mv settings.yml ./group_vars/all/ + ``` + +* remove all helm charts (WARNING: Please backup all data before!): + + `helm delete --purge oas-test-cert-manager oas-test-local-storage oas-test-prometheus oas-test-proxy oas-test-files` diff --git a/openappstack/cluster.py b/openappstack/cluster.py index 12880f030b24284eb12b98f80b554ec410356ca9..8d1b0ea8b0fc62bec11720e4e835b21731747296 100644 --- a/openappstack/cluster.py +++ b/openappstack/cluster.py @@ -152,6 +152,12 @@ class Cluster: file_contents = yaml.safe_dump(settings, default_flow_style=False) log.debug(file_contents) + + # Create CLUSTER_DIR/group_vars/all/ if non-existent + vars_dir = os.path.dirname(self.settings_file) + if not os.path.exists(vars_dir): + os.makedirs(vars_dir) + with open(self.settings_file, 'w') as stream: stream.write(file_contents) log.info("Created %s", self.settings_file) @@ -172,7 +178,8 @@ class Cluster: @property def settings_file(self): """Path to the ansible settings.yml for this cluster""" - return os.path.join(self.cluster_dir, 'settings.yml') + return os.path.join(self.cluster_dir, 'group_vars', 'all', + 'settings.yml') @property def behave_file(self): diff --git a/test/README.md b/test/README.md index 6f7a9be65f53c166ca6f751578dfb94badc5f291..b91e561c2c81e2f7d4afd4a937fc8876d92940df 100644 --- a/test/README.md +++ b/test/README.md @@ -13,36 +13,36 @@ There are two types of tests: "testinfra" tests, and "behave" tests. ## Run *testinfra* tests -Test host configured in `../clusters/CLUSTERNAME/inventory.yml` +Export `CLUSTER_DIR` env var with the location of your cluster config directory: - export INVENTORY=../clusters/CLUSTERNAME/inventory.yml - py.test -v --ansible-inventory=${INVENTORY} --hosts='ansible://*' + export CLUSTER_DIR="../clusters/CLUSTERNAME" + +Run all tests: + + py.test -sv --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' Specify host manually: - py.test -v --hosts='ssh://root@example.openappstack.net' + py.test -sv --hosts='ssh://root@example.openappstack.net' Run only tests tagged with `prometheus`: - py.test -v --ansible-inventory=${INVENTORY} --hosts='ansible://*' -m prometheus + py.test -sv --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' -m prometheus ### Cert tests Run cert test manually using the ansible inventory file: - ADDRESS='example.openappstack.net' py.test -v -m 'certs' \ - --connection=ansible \ - --ansible-inventory=${INVENTORY} \ - --hosts='ansible://*' + py.test -sv --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' -m certs Run cert test manually against a different cluster, not configured in any ansible inventory file, either by using pytest: - ADDRESS='example.openappstack.net' py.test -v -m 'certs' + FQDN='example.openappstack.net' py.test -sv -m 'certs' -or directly (allows better debugging since pytest won't eat stdout): +or directly: - ADDRESS='example.openappstack.net' pytest/test_certs.py + FQDN='example.openappstack.net' pytest/test_certs.py ## Run *behave* tests diff --git a/test/pytest/test_certs.py b/test/pytest/test_certs.py index 472311e1613aca6db185ddb97918beed9e658ee0..294803496c76ed8e64a4f1521976dd34d705cf7d 100755 --- a/test/pytest/test_certs.py +++ b/test/pytest/test_certs.py @@ -12,12 +12,12 @@ from OpenSSL import SSL def add_custom_cert_authorities(ca_file: str, custom_ca_files: list, - destination_file: str = + dest_file: str = '/tmp/custom_ca_bundle.crt'): """Concatenates existing cert bundle with custom CAs.""" - destination = open(destination_file, 'wb') - with open(destination_file, 'wb') as destination, open(ca_file, 'rb') as ca: + destination = open(dest_file, 'wb') + with open(dest_file, 'wb') as destination, open(ca_file, 'rb') as ca: shutil.copyfileobj(ca, destination) for custom_ca_file in custom_ca_files: with open(custom_ca_file, 'rb') as custom_ca: @@ -59,7 +59,7 @@ def print_cert_info(certs: list): print('CN: {0} (Issuer: {1})'.format(cn, issuer)) -def read_certs_from_file(filename:str): +def read_certs_from_file(filename: str): """Read cert from file for debugging/development.""" import OpenSSL.crypto @@ -97,14 +97,26 @@ def valid_cert(domain: str, ca_file: str = '/tmp/custom_ca_bundle.crt'): @pytest.mark.certs def test_cert_validation(host): - domain = os.environ.get("ADDRESS") - assert domain, "Please export ADDRESS as environment variable." + """Checks for proper cluster certs from exposed services. + Check is executed on the local provisioning machine. + """ + + # Use FQDN env var if set, otherwise use domain var from + # settings.yml. + domain = os.environ.get("FQDN") + if domain: + print("Using domain %s from FQDN environment variable." % domain) + else: + ansible_vars = host.ansible.get_variables() + domain = ansible_vars["domain"] + print("Using domain %s from ansible settings.yml." % domain) add_custom_cert_authorities(certifi.where(), ['pytest/le-staging-bundle.pem']) - # Check nextcloud cert assert valid_cert('files.{0}'.format(domain)) + assert valid_cert('office.{0}'.format(domain)) + assert valid_cert('grafana.{0}'.format(domain)) if __name__ == "__main__": diff --git a/test/pytest/test_helmreleases.py b/test/pytest/test_helmreleases.py index 85483b2af1f8609313f325e6276f02ef7c771d5b..624cf04daa2bd0738c63f94512667424305f8de3 100644 --- a/test/pytest/test_helmreleases.py +++ b/test/pytest/test_helmreleases.py @@ -1,6 +1,7 @@ import pytest from kubernetes import client, config from kubernetes.client.rest import ApiException +import os def get_release(name, namespace, api): @@ -26,7 +27,7 @@ def get_release(name, namespace, api): @pytest.mark.helmreleases -def test_helmreleases(): +def test_helmreleases(host): """Checks if all desired HelmReleases installed by weave flux are in DEPLOYED state. """ @@ -35,7 +36,10 @@ def test_helmreleases(): 'oas-apps': ['nextcloud'] } - config.load_kube_config() + cluster_dir = os.environ.get("CLUSTER_DIR") + kubeconfig = os.path.join(cluster_dir, 'secrets', + 'kube_config_cluster.yml') + config.load_kube_config(config_file=kubeconfig) customObjects = client.CustomObjectsApi() failed = 0 diff --git a/test/pytest/test_system.py b/test/pytest/test_system.py index aa0d3ca5bf7c5e1d9f18313f1ef0237ae250ef65..be7192f4be31869bfaef9e814ab75c62410fcf2c 100644 --- a/test/pytest/test_system.py +++ b/test/pytest/test_system.py @@ -2,6 +2,6 @@ import pytest @pytest.mark.testinfra -def test_release_is_bionic(host): +def test_os_release(host): system_info = host.system_info assert system_info.release == '10'