...
 
Commits (6)
......@@ -24,7 +24,7 @@ default:
image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_COMMIT_REF_SLUG}"
ci_test_image:
ci-test-image:
stage: build
only:
changes:
......@@ -94,12 +94,13 @@ setup-openappstack:
- clusters/$HOSTNAME/**
key: ${CI_COMMIT_REF_SLUG}
test_helmreleases:
test-helmreleases:
stage: wait-for-deployments
script:
- cd ansible/
- export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml"
- pytest -v -s -m 'helmreleases' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10
- pytest -v -s -m 'apps_running' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10
only:
changes:
- .gitlab-ci.yml
......@@ -155,8 +156,6 @@ prometheus-alerts:
behave-nextcloud:
stage: integration-test
script:
# Wait for Nextcloud and ONLYOFFICE pods to be Ready
- ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas-apps pod -l app.kubernetes.io/instance=nc --for condition=Ready --timeout=20m"'
# Run the behave tests for NextCloud.
- python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags nextcloud || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags nextcloud
artifacts:
......@@ -177,8 +176,6 @@ behave-nextcloud:
behave-wordpress:
stage: integration-test
script:
# Wait for wordpress pod to be Ready
- ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas-apps pod -l release=wordpress --for condition=Ready --timeout=20m"'
# Run behave tests against wordpress
- python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags wordpress || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags wordpress
artifacts:
......@@ -199,8 +196,6 @@ behave-wordpress:
behave-rocketchat:
stage: integration-test
script:
# Wait for Rocket.Chat pod to be Ready
- ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas-apps pod -l app.kubernetes.io/instance=rocketchat --for condition=Ready --timeout=20m"'
# Run behave tests against Rocket.Chat
- python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags rocketchat || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags rocketchat
artifacts:
......@@ -221,8 +216,6 @@ behave-rocketchat:
behave-grafana:
stage: integration-test
script:
# Wait for Grafana pod to be Ready.
- ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas pod -l app=grafana --for condition=Ready --timeout=20m"'
# Run behave tests against Grafana
- python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags grafana || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags grafana
artifacts:
......@@ -239,7 +232,7 @@ behave-grafana:
- openappstack/**/*
extends: .ssh_setup
terminate_mr_droplet_after_merge:
terminate-mr-droplet-after-merge:
stage: cleanup
before_script:
- echo "We leave MR droplets running even when the pipeline is successful \
......@@ -260,7 +253,7 @@ terminate_mr_droplet_after_merge:
refs:
- master
terminate_old_droplets:
terminate-old-droplets:
stage: cleanup
script:
- echo "Terminate droplets 5 days after creation. Branches that exist longer than 5 days will get a new droplet when CI runs again."
......
"""
This module contains tests for:
- test_helmreleases: if all the helmreleases in EXPECTED_RELEASES are
deployed successfully
- test_apps_running: if all the applications are running without problems
(i.e., condition = Ready for all the related pods)
"""
import os
import pytest
from kubernetes import client, config
from kubernetes.client.rest import ApiException
import os
EXPECTED_RELEASES = {
'oas': [
'cert-manager',
'ingress',
'local-storage',
'monitoring',
'loki-stack',
'single-sign-on'
],
'oas-apps': ['nextcloud', 'rocketchat', 'wordpress']
}
EXPECTED_APP_LABELS = {
'nextcloud': {
'namespace': 'oas-apps',
'label_selector': 'app.kubernetes.io/instance=nc'},
'wordpress': {
'namespace': 'oas-apps',
'label_selector': 'release=wordpress'},
'rocketchat': {
'namespace': 'oas-apps',
'label_selector': 'app.kubernetes.io/instance=rocketchat'},
'grafana': {
'namespace': 'oas',
'label_selector': 'app=grafana'}
}
def get_release(name, namespace, api):
def get_release_status(name, namespace, api):
"""Returns release status for release `name` in `namespace`"""
print('Testing %s in namespace %s ...' % (name, namespace), end='')
try:
release = api.get_namespaced_custom_object(
......@@ -16,41 +52,78 @@ def get_release(name, namespace, api):
)
release_status = release['status']['releaseStatus']
print(release_status)
except ApiException as e:
if e.status == 404:
except ApiException as ex:
if ex.status == 404:
release_status = 'not found'
else:
raise
print("**** NOT DEPLOYED, status: %s *****" % release_status)
return(release_status)
return release_status
@pytest.mark.helmreleases
def test_helmreleases(host):
"""Checks if all desired HelmReleases installed by weave flux are in
DEPLOYED state.
def check_all_pods_running(pods):
"""
Loop through all the pods in an API result.
If a pod does not have an element `status.phase` with the value "Running",
return False. Otherwise, returns True.
:param kubernetes.V1Pod[] pods: list of V1Pod elements to check
"""
expected_releases = {
'oas': ['cert-manager', 'ingress', 'local-storage', 'monitoring'],
'oas-apps': ['nextcloud', 'rocketchat', 'wordpress']
}
ret = True
for pod in pods:
print("- {}: {}".format(pod.metadata.name, pod.status.phase))
if pod.status.phase != "Running":
ret = False
return ret
@pytest.fixture(autouse=True)
def run_around_tests():
"""
Prepare kube config before running a test
"""
cluster_dir = os.environ.get("CLUSTER_DIR")
kubeconfig = os.path.join(cluster_dir, 'secrets',
'kube_config_cluster.yml')
config.load_kube_config(config_file=kubeconfig)
customObjects = client.CustomObjectsApi()
yield
@pytest.mark.helmreleases
def test_helmreleases(host):
"""
Checks if all desired HelmReleases installed by weave flux are in
DEPLOYED state.
"""
custom_objects = client.CustomObjectsApi()
failed = 0
print('\n')
for namespace in expected_releases:
for app in expected_releases[namespace]:
app_status = get_release(app, namespace, customObjects)
for namespace in EXPECTED_RELEASES:
for app in EXPECTED_RELEASES[namespace]:
app_status = get_release_status(app, namespace, custom_objects)
if app_status != 'DEPLOYED':
failed += 1
assert failed == 0, "Error: %s apps NOT DEPLOYED !" % failed
assert failed == 0, "Error: {} apps not 'DEPLOYED'!".format(failed)
if __name__ == "__main__":
test_helmreleases()
@pytest.mark.apps_running
def test_apps_running(host):
"""
Checks if all the pods related to releases in EXPECTED_RELEASES are
ready
"""
api = client.CoreV1Api()
failed = 0
print('\n')
for app, info in EXPECTED_APP_LABELS.items():
print("{}: ".format(app))
result = api.list_namespaced_pod(
namespace=info['namespace'],
label_selector=info['label_selector'],
)
if not check_all_pods_running(result.items):
failed += 1
print()
assert failed == 0, "Error: {} apps not 'Running'!".format(failed)
import re
import json
import pytest
import re
@pytest.mark.prometheus
def test_prometheus_alerts(host):
"""Test prometheus for firing alerts."""
def summarize_alerts(alerts):
"""Print a alert summary."""
print('Total alerts: %s' % len(alerts))
print(json.dumps(alerts, indent=2))
print("Starting prometheus test...")
url = 'http://monitoring-prometheus-oper-prometheus.oas.svc.cluster.local:9090/api/v1/alerts'
alert_json = json.loads(host.check_output('curl ' + url))
status = alert_json["status"]
alerts = alert_json["data"]["alerts"]
real_alerts = []
ignored_alerts = []
for alert in alerts:
# Filter out the ever firing "Dead mans switch" test alert
if ignore_alert(alert):
ignored_alerts.append(alert)
else:
real_alerts.append(alert)
print('\n\n\n========= Ignored ==========')
summarize_alerts(ignored_alerts)
print('\n\n\n========= Firing ==========')
summarize_alerts(real_alerts)
count = len(real_alerts)
assert status == "success", "Failure queriying the prometheus api at" + url
assert count == 0, "Firing alerts: {0}".format(count)
def ignore_alert(alert):
......@@ -92,5 +53,44 @@ def ignore_alert(alert):
return False
@pytest.mark.prometheus
def test_prometheus_alerts(host):
"""Test prometheus for firing alerts."""
def summarize_alerts(alerts):
"""Print a alert summary."""
print('Total alerts: %s' % len(alerts))
print(json.dumps(alerts, indent=2))
print("Starting prometheus test...")
url = 'http://monitoring-prometheus-oper-prometheus.oas.svc.cluster.local:9090/api/v1/alerts'
alert_json = json.loads(host.check_output('curl ' + url))
status = alert_json["status"]
alerts = alert_json["data"]["alerts"]
real_alerts = []
ignored_alerts = []
for alert in alerts:
# Filter out the ever firing "Dead mans switch" test alert
if ignore_alert(alert):
ignored_alerts.append(alert)
else:
real_alerts.append(alert)
print('\n\n\n========= Ignored ==========')
summarize_alerts(ignored_alerts)
print('\n\n\n========= Firing ==========')
summarize_alerts(real_alerts)
count = len(real_alerts)
assert status == "success", "Failure queriying the prometheus api at" + url
assert count == 0, "Firing alerts: {0}".format(count)
if __name__ == "__main__":
test_prometheus_alerts('')