diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b85afe08ffa86b60896b8eff6a3e7b14daec01a7..3de8a8207b7d7c4e594c5f0c4f694f86cd25e238 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ default: image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_COMMIT_REF_SLUG}" -ci_test_image: +ci-test-image: stage: build only: changes: @@ -94,12 +94,13 @@ setup-openappstack: - clusters/$HOSTNAME/** key: ${CI_COMMIT_REF_SLUG} -test_helmreleases: +test-helmreleases: stage: wait-for-deployments script: - cd ansible/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml" - pytest -v -s -m 'helmreleases' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 + - pytest -v -s -m 'apps_running' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 only: changes: - .gitlab-ci.yml @@ -155,8 +156,6 @@ prometheus-alerts: behave-nextcloud: stage: integration-test script: - # Wait for Nextcloud and ONLYOFFICE pods to be Ready - - ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas-apps pod -l app.kubernetes.io/instance=nc --for condition=Ready --timeout=20m"' # Run the behave tests for NextCloud. - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags nextcloud || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags nextcloud artifacts: @@ -177,8 +176,6 @@ behave-nextcloud: behave-wordpress: stage: integration-test script: - # Wait for wordpress pod to be Ready - - ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas-apps pod -l release=wordpress --for condition=Ready --timeout=20m"' # Run behave tests against wordpress - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags wordpress || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags wordpress artifacts: @@ -199,8 +196,6 @@ behave-wordpress: behave-rocketchat: stage: integration-test script: - # Wait for Rocket.Chat pod to be Ready - - ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas-apps pod -l app.kubernetes.io/instance=rocketchat --for condition=Ready --timeout=20m"' # Run behave tests against Rocket.Chat - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags rocketchat || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags rocketchat artifacts: @@ -221,8 +216,6 @@ behave-rocketchat: behave-grafana: stage: integration-test script: - # Wait for Grafana pod to be Ready. - - ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas pod -l app=grafana --for condition=Ready --timeout=20m"' # Run behave tests against Grafana - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags grafana || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags grafana artifacts: @@ -239,7 +232,7 @@ behave-grafana: - openappstack/**/* extends: .ssh_setup -terminate_mr_droplet_after_merge: +terminate-mr-droplet-after-merge: stage: cleanup before_script: - echo "We leave MR droplets running even when the pipeline is successful \ @@ -260,7 +253,7 @@ terminate_mr_droplet_after_merge: refs: - master -terminate_old_droplets: +terminate-old-droplets: stage: cleanup script: - echo "Terminate droplets 5 days after creation. Branches that exist longer than 5 days will get a new droplet when CI runs again." diff --git a/test/pytest/test_helmreleases.py b/test/pytest/test_helmreleases.py index 24aebb4261c867280b36bd823febc0ded3bbe436..5b83b8bbaf0bec187426e5f4cfcd34d60c5994c9 100644 --- a/test/pytest/test_helmreleases.py +++ b/test/pytest/test_helmreleases.py @@ -1,10 +1,39 @@ +""" +This module contains tests for: + +- test_helmreleases: if all the helmreleases in EXPECTED_RELEASES are + deployed successfully +- test_apps_running: if all the applications are running without problems + (i.e., condition = Ready for all the related pods) +""" +import os import pytest from kubernetes import client, config from kubernetes.client.rest import ApiException -import os +EXPECTED_RELEASES = { + 'oas': ['cert-manager', 'ingress', 'local-storage', 'monitoring'], + 'oas-apps': ['nextcloud', 'rocketchat', 'wordpress'] +} -def get_release(name, namespace, api): +EXPECTED_APP_LABELS = { + 'nextcloud': { + 'namespace': 'oas-apps', + 'label_selector': 'app.kubernetes.io/instance=nc'}, + 'wordpress': { + 'namespace': 'oas-apps', + 'label_selector': 'release=wordpress'}, + 'rocketchat': { + 'namespace': 'oas-apps', + 'label_selector': 'app.kubernetes.io/instance=rocketchat'}, + 'grafana': { + 'namespace': 'oas', + 'label_selector': 'app=grafana'} +} + + +def get_release_status(name, namespace, api): + """Returns release status for release `name` in `namespace`""" print('Testing %s in namespace %s ...' % (name, namespace), end='') try: release = api.get_namespaced_custom_object( @@ -16,41 +45,78 @@ def get_release(name, namespace, api): ) release_status = release['status']['releaseStatus'] print(release_status) - except ApiException as e: - if e.status == 404: + except ApiException as ex: + if ex.status == 404: release_status = 'not found' else: raise print("**** NOT DEPLOYED, status: %s *****" % release_status) - return(release_status) + return release_status -@pytest.mark.helmreleases -def test_helmreleases(host): - """Checks if all desired HelmReleases installed by weave flux are in - DEPLOYED state. +def check_all_pods_ready(result): + """ + Loop through all the pods in an API result. + If a pod contains a "status" and "condition": + if that condition's type is "Ready" and its status is not true + Then, there is a pod that is not ready, return false. + Otherwise, returns True. """ - expected_releases = { - 'oas': ['cert-manager', 'ingress', 'local-storage', 'monitoring'], - 'oas-apps': ['nextcloud', 'rocketchat', 'wordpress'] - } + ret = True + for pod in result.items: + print("- {}: {}".format(pod.metadata.name, pod.status.phase)) + if pod.status.phase != "Running": + ret = False + return ret + +@pytest.fixture(autouse=True) +def run_around_tests(): + """ + Prepare kube config before running a test + """ cluster_dir = os.environ.get("CLUSTER_DIR") kubeconfig = os.path.join(cluster_dir, 'secrets', 'kube_config_cluster.yml') config.load_kube_config(config_file=kubeconfig) - customObjects = client.CustomObjectsApi() + yield + + +@pytest.mark.helmreleases +def test_helmreleases(host): + """ + Checks if all desired HelmReleases installed by weave flux are in + DEPLOYED state. + """ + custom_objects = client.CustomObjectsApi() failed = 0 print('\n') - for namespace in expected_releases: - for app in expected_releases[namespace]: - app_status = get_release(app, namespace, customObjects) + for namespace in EXPECTED_RELEASES: + for app in EXPECTED_RELEASES[namespace]: + app_status = get_release_status(app, namespace, custom_objects) if app_status != 'DEPLOYED': failed += 1 - assert failed == 0, "Error: %s apps NOT DEPLOYED !" % failed + assert failed == 0, "Error: {} apps not 'DEPLOYED'!".format(failed) -if __name__ == "__main__": - test_helmreleases() +@pytest.mark.apps_running +def test_apps_running(host): + """ + Checks if all the pods related to releases in EXPECTED_RELEASES are + ready + """ + api = client.CoreV1Api() + failed = 0 + print('\n') + for app, info in EXPECTED_APP_LABELS.items(): + print("{}: ".format(app)) + result = api.list_namespaced_pod( + namespace=info['namespace'], + label_selector=info['label_selector'], + ) + if not check_all_pods_ready(result): + failed += 1 + print() + assert failed == 0, "Error: {} apps not 'Running'!".format(failed)