diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d500cc3176a3c5e79190d9a2de35e3ed28532b88..c19bfec4b123167ec700e4d22f0d1cc603f6c592 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,15 +18,13 @@ include: - | echo "Env vars:" echo - echo "HOSTNAME: $HOSTNAME" - echo "IP_ADDRESS: $IP_ADDRESS" + env | grep -E '^(HOSTNAME|CLUSTER_NAME|FQDN|IP_ADDRESS|CLUSTER_DIR|ANSIBLE_HOST_KEY_CHECKING|KANIKO_BUILD_IMAGENAME|SSH_KEY_ID|SHELL|CI_PROJECT_DIR)=' + echo echo "Uptime: $(uptime)" - echo "ANSIBLE_HOST_KEY_CHECKING: $ANSIBLE_HOST_KEY_CHECKING" - echo "KANIKO_BUILD_IMAGENAME: $KANIKO_BUILD_IMAGENAME" echo "KANIKO build image ref: ${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}" - echo "SSH_KEY_ID: $SSH_KEY_ID" - echo "SHELL $SHELL" - echo "CI_PROJECT_DIR: $CI_PROJECT_DIR" + echo + - if [ -f .ci.env ]; then echo "Content of .ci.env:"; cat .ci.env; fi + - if [ -f .cluster.env ]; then echo "Content of .ci.env:"; cat .cluster.env; fi # The dotenv report requires us to report the artifacts in every job that is # required with a `needs:` from another job. @@ -93,22 +91,6 @@ include: - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-monitoring/' - if: '$CI_COMMIT_BRANCH == "main"' -.eventrouter_rules: - extends: - - .monitoring_rules - -.loki_rules: - extends: - - .monitoring_rules - -.promtail_rules: - extends: - - .monitoring_rules - -.kube_prometheus_stack_rules: - extends: - - .monitoring_rules - .nextcloud_rules: rules: - changes: @@ -122,14 +104,6 @@ include: - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-nextcloud/' - if: '$CI_COMMIT_BRANCH == "main"' -.cert_manager_rules: - extends: - - .general_rules - -.local_path_provisioner_rules: - extends: - - .general_rules - .single_sign_on_rules: rules: - changes: @@ -187,14 +161,13 @@ include: stages: - build - - create-vps - - setup-cluster - - configure-cluster - - kustomization - - install-apps - - apps-kustomizations-ready + - install-cluster + - install-stackspin + - base-ready + - configure-stackspin + - optional-apps-ready - certs - - health-test + - cluster-health - integration-test variables: @@ -221,7 +194,7 @@ ci-test-image-build: - *debug_information after_script: - | - echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env + echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" | tee .ci.env artifacts: paths: - .ci.env @@ -256,13 +229,14 @@ report-ci-image-tag: - *debug_information script: - | - TAG_INFORMATION=$(curl https://open.greenhost.net/api/v4/projects/stackspin%2Fstackspin/registry/repositories/73/tags/${CI_COMMIT_REF_SLUG}); + TAG_INFORMATION=$(curl -sS https://open.greenhost.net/api/v4/projects/stackspin%2Fstackspin/registry/repositories/73/tags/${CI_COMMIT_REF_SLUG}); echo "Tag information: ${TAG_INFORMATION}" if [ "$TAG_INFORMATION" == '{"message":"404 Tag Not Found"}' ]; then - echo "CI_CONTAINER_TAG=main" > .ci.env + CI_CONTAINER_TAG="main" else - echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env + CI_CONTAINER_TAG="${CI_COMMIT_REF_SLUG}" fi + echo "CI_CONTAINER_TAG=${CI_CONTAINER_TAG}" | tee .ci.env artifacts: paths: - .ci.env @@ -286,13 +260,14 @@ report-ci-image-tag: interruptible: true -# Stage: create-vps -# ================= +# Stage: install-cluster +# ====================== # -# Creates the vps for the pipeline +# * Creates the vps for the pipeline +# * Installs k8s with ansible create-vps: - stage: create-vps + stage: install-cluster variables: SUBDOMAIN: "${CI_COMMIT_REF_SLUG}.ci" DOMAIN: "stackspin.net" @@ -304,7 +279,6 @@ create-vps: # Make sure .ci.env variables are not lost - cat .ci.env >> ${CLUSTER_DIR}/.cluster.env extends: - - .ssh_setup - .report_artifacts - .general_rules environment: @@ -314,16 +288,11 @@ create-vps: auto_stop_in: 1 week interruptible: true - -# Stage: setup-cluster -# ==================== -# -# Installs Stackspin - test-dns: - stage: setup-cluster + stage: install-cluster needs: - job: create-vps + # Needs a pytest ansible connection to get the configured system resolvers script: - *debug_information - cd ansible/ @@ -332,8 +301,10 @@ test-dns: - .general_rules interruptible: true -setup-stackspin: - stage: setup-cluster +install-k8s: + stage: install-cluster + needs: + - job: create-vps script: - *debug_information # Copy inventory files to ansible folder for use in install-apps step @@ -341,8 +312,52 @@ setup-stackspin: - cp ${CLUSTER_DIR}/inventory.yml ansible/ # Set up cluster - python3 -m stackspin $HOSTNAME install + extends: + - .ssh_setup + - .report_artifacts + - .general_rules + interruptible: true + +# Terminates a droplet and deletes the branch container image once the MR for it is merged +terminate-droplet: + # Stage has to be the same as the step that created the VPS + # https://docs.gitlab.com/ee/ci/environments.html#automatically-stopping-an-environment + stage: install-cluster + # Gets triggered by on_stop of create-vps job + when: manual + variables: + GIT_STRATEGY: none + script: + - *debug_information + # Delete droplet + - python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^${CI_COMMIT_REF_SLUG}\")" + # Delete container image if one was created + - > + "if [ \"$CI_CONTAINER_TAG\" != \"main\" ]; then + curl --request DELETE --header \"PRIVATE-TOKEN: ${CLEANER_TOKEN}\" + https://open.greenhost.net/api/v4/projects/stackspin%2Fstackspin/registry/repositories/73/tags/${CI_CONTAINER_TAG}; + fi" + environment: + name: $CI_COMMIT_REF_SLUG + action: stop + +# Stage: install-stackspin +# ======================== +# +# Installs flux and stackspin with it + +install-stackspin: + stage: install-stackspin + needs: + - job: test-dns + - job: install-k8s + script: + - *debug_information # Customize env file, remove all comments and empty lines - - sed "s/1.2.3.4/$IP_ADDRESS/; s/example.org/$FQDN/; s/acme_staging=false/acme_staging=true/; s/acme-v02/acme-staging-v02/; /^\s*#.*$/d; /^\s*$/d" install/.flux.env.example >> ${CLUSTER_DIR}/.flux.env + - cp install/.flux.env.example ${CLUSTER_DIR}/.flux.env + - sed -i "s/1.2.3.4/$IP_ADDRESS/" ${CLUSTER_DIR}/.flux.env + - sed -i "s/example.org/$FQDN/" ${CLUSTER_DIR}/.flux.env + - sed -i "/^\s*#.*$/d; /^\s*$/d" ${CLUSTER_DIR}/.flux.env # Deploy secret/stackspin-cluster-variables - cp install/kustomization.yaml ${CLUSTER_DIR} - kubectl create namespace flux-system @@ -353,48 +368,73 @@ setup-stackspin: # Install flux and general, non-app specific secrets - bash ./install/install-stackspin.sh extends: - - .ssh_setup - .report_artifacts - .general_rules interruptible: true - -# Stage: configure-cluster -# ==================== -# -# Configure cluster after basic installation -# i.e. CI-related config like zerossl clusterIssuer - -configure-stackspin: - stage: configure-cluster +.enable_app_template: + stage: install-stackspin + needs: + - job: install-stackspin script: - *debug_information - # Install custom ClusterIssuer for ZeroSSL production certificates - - bash ./.gitlab/ci_scripts/install_zerossl_issuer.sh - extends: - - .report_artifacts - - .general_rules + # Add optional override values we need for the CI pipeline only + - > + [ -f ./install/overrides/stackspin-${RESOURCE}-override.yaml ] && + kubectl apply -n stackspin-apps -f ./install/overrides/stackspin-${RESOURCE}-override.yaml + - bash ./install/install-app.sh ${RESOURCE} interruptible: true +enable-monitoring: + variables: + RESOURCE: "monitoring" + extends: + - .enable_app_template + - .monitoring_rules +enable-nextcloud: + variables: + RESOURCE: "nextcloud" + extends: + - .enable_app_template + - .nextcloud_rules +enable-wekan: + variables: + RESOURCE: "wekan" + extends: + - .enable_app_template + - .wekan_rules -# Stage: kustomization +enable-wordpress: + variables: + RESOURCE: "wordpress" + extends: + - .enable_app_template + - .wordpress_rules + +enable-zulip: + variables: + RESOURCE: "zulip" + extends: + - .enable_app_template + - .zulip_rules + +# Stage: base-ready # ==================== # -# Tests if all kustomizations are ready +# Test if base kustomizations are ready, before configuration can get applied +# that makes use of CRDs, i.e. clusterIssuer .kustomization-ready: - stage: kustomization + stage: base-ready needs: - - job: configure-stackspin - - job: test-dns + - job: install-stackspin script: - *debug_information - - cd ansible/ + - cd test/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/kube_config_cluster.yml" - - pytest -v -s -m 'kustomizations' --resource="$RESOURCE" --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 20 + - pytest -v -s -m 'kustomizations' --resource="$RESOURCE" --reruns 120 --reruns-delay 20 extends: - - .ssh_setup - .general_rules interruptible: true @@ -458,70 +498,40 @@ stackspin-kustomization-ready: extends: - .kustomization-ready -# Stage: install-apps -# ================== +# Stage: configure-stackspin # -# Checks if application needs to get installed - -.enable_app_template: - stage: install-apps +# Configure cluster after basic installation +# i.e. CI-related config like zerossl clusterIssuer +# +configure-zerossl-issuer: + stage: configure-stackspin + needs: + - job: install-stackspin + - job: cert-manager-kustomization-ready script: - *debug_information - # Add optional override values we need for the CI pipeline only - - '[ -f ./install/overrides/stackspin-${RESOURCE}-override.yaml ] && kubectl apply -n stackspin-apps -f ./install/overrides/stackspin-${RESOURCE}-override.yaml' - - bash ./install/install-app.sh ${RESOURCE} + # Install custom ClusterIssuer for ZeroSSL production certificates + - bash ./.gitlab/ci_scripts/install_zerossl_issuer.sh extends: - - .ssh_setup + - .report_artifacts + - .general_rules interruptible: true -enable-monitoring: - variables: - RESOURCE: "monitoring" - extends: - - .enable_app_template - - .monitoring_rules - -enable-nextcloud: - variables: - RESOURCE: "nextcloud" - extends: - - .enable_app_template - - .nextcloud_rules -enable-wekan: - variables: - RESOURCE: "wekan" - extends: - - .enable_app_template - - .wekan_rules -enable-wordpress: - variables: - RESOURCE: "wordpress" - extends: - - .enable_app_template - - .wordpress_rules - -enable-zulip: - variables: - RESOURCE: "zulip" - extends: - - .enable_app_template - - .zulip_rules - -# Stage: apps-kustomizations-ready +# Stage: optional-apps-ready # ================ # # Check that the kustomizations of all installed apps are ready. .app-kustomization-ready: - stage: apps-kustomizations-ready + stage: optional-apps-ready extends: - .kustomization-ready monitoring-kustomization-ready: needs: - - job: configure-stackspin + - job: install-stackspin - job: enable-monitoring variables: RESOURCE: "monitoring" @@ -531,7 +541,7 @@ monitoring-kustomization-ready: nextcloud-kustomization-ready: needs: - - job: configure-stackspin + - job: install-stackspin - job: enable-nextcloud variables: RESOURCE: "nextcloud" @@ -541,7 +551,7 @@ nextcloud-kustomization-ready: wekan-kustomization-ready: needs: - - job: configure-stackspin + - job: install-stackspin - job: enable-wekan variables: RESOURCE: "wekan" @@ -551,7 +561,7 @@ wekan-kustomization-ready: wordpress-kustomization-ready: needs: - - job: configure-stackspin + - job: install-stackspin - job: enable-wordpress variables: RESOURCE: "wordpress" @@ -561,7 +571,7 @@ wordpress-kustomization-ready: zulip-kustomization-ready: needs: - - job: configure-stackspin + - job: install-stackspin - job: enable-zulip variables: RESOURCE: "zulip" @@ -578,18 +588,16 @@ zulip-kustomization-ready: stage: certs script: - *debug_information - - cd ansible/ - - pytest -v -s -m 'certs' --resource="$RESOURCE" --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 - extends: - - .ssh_setup + - cd test/ + - pytest -v -s -m 'certs' --resource="$RESOURCE" --reruns 120 --reruns-delay 10 interruptible: true nextcloud-cert: variables: RESOURCE: "nextcloud" needs: - - job: enable-nextcloud - - job: configure-stackspin + - job: configure-zerossl-issuer + - job: install-stackspin extends: - .apps-cert - .nextcloud_rules @@ -598,18 +606,18 @@ kube-prometheus-stack-cert: variables: RESOURCE: "kube-prometheus-stack" needs: - - job: enable-monitoring - - job: configure-stackspin + - job: configure-zerossl-issuer + - job: install-stackspin extends: - .apps-cert - - .kube_prometheus_stack_rules + - .monitoring_rules single-sign-on-cert: variables: RESOURCE: "single-sign-on" needs: - - job: single-sign-on-kustomization-ready - - job: configure-stackspin + - job: configure-zerossl-issuer + - job: install-stackspin extends: - .apps-cert - .general_rules @@ -618,8 +626,8 @@ dashboard-cert: variables: RESOURCE: "dashboard" needs: - - job: dashboard-kustomization-ready - - job: configure-stackspin + - job: configure-zerossl-issuer + - job: install-stackspin extends: - .apps-cert - .general_rules @@ -628,8 +636,8 @@ wekan-cert: variables: RESOURCE: "wekan" needs: - - job: enable-wekan - - job: configure-stackspin + - job: configure-zerossl-issuer + - job: install-stackspin extends: - .apps-cert - .wekan_rules @@ -638,8 +646,8 @@ wordpress-cert: variables: RESOURCE: "wordpress" needs: - - job: enable-wordpress - - job: configure-stackspin + - job: configure-zerossl-issuer + - job: install-stackspin extends: - .apps-cert - .wordpress_rules @@ -648,20 +656,22 @@ zulip-cert: variables: RESOURCE: "zulip" needs: - - job: enable-zulip - - job: configure-stackspin + - job: configure-zerossl-issuer + - job: install-stackspin extends: - .apps-cert - .zulip_rules -# Stage: health-test -# ================== +# Stage: cluster-health +# ===================== # # General cluster health checks testinfra: - stage: health-test + stage: cluster-health + needs: + - job: install-stackspin script: - *debug_information - cd ansible/ @@ -671,22 +681,21 @@ testinfra: - .general_rules interruptible: true -kube-prometheus-stack-alerts: - stage: health-test +prometheus-alerts: + stage: cluster-health + needs: + - job: install-stackspin + - job: kube-prometheus-stack-cert variables: - # RESOURCE var is used in job specific rules (i.e. .kube_prometheus_stack_rules) + # RESOURCE var is used in job specific rules (i.e. ..monitoring_rules) RESOURCE: "kube-prometheus-stack" - # Enforce python requests using the system cert store, where LE staging - # cacert is added - REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt" script: - *debug_information - export BASIC_AUTH_PW=$(python3 -m stackspin $HOSTNAME secrets | grep stackspin-prometheus-basic-auth | cut -d'=' -f2) - cd test/ - - bash ../.gitlab/ci_scripts/retry_cmd_until_success.sh 30 10 pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' + - bash ../.gitlab/ci_scripts/retry_cmd_until_success.sh 30 10 pytest -s -m prometheus extends: - - .ssh_setup - - .kube_prometheus_stack_rules + - .monitoring_rules interruptible: true @@ -714,7 +723,7 @@ dashboard-taiko: variables: RESOURCE: "dashboard" needs: - - job: configure-stackspin + - job: install-stackspin - job: dashboard-cert - job: dashboard-kustomization-ready extends: @@ -725,18 +734,18 @@ grafana-taiko: variables: RESOURCE: "grafana" needs: - - job: configure-stackspin + - job: install-stackspin - job: kube-prometheus-stack-cert - job: monitoring-kustomization-ready extends: - .taiko - - .kube_prometheus_stack_rules + - .monitoring_rules nextcloud-taiko: variables: RESOURCE: "nextcloud" needs: - - job: configure-stackspin + - job: install-stackspin - job: nextcloud-cert - job: nextcloud-kustomization-ready extends: @@ -747,7 +756,7 @@ wekan-taiko: variables: RESOURCE: "wekan" needs: - - job: configure-stackspin + - job: install-stackspin - job: wekan-cert - job: wekan-kustomization-ready extends: @@ -758,7 +767,7 @@ wordpress-taiko: variables: RESOURCE: "wordpress" needs: - - job: configure-stackspin + - job: install-stackspin - job: wordpress-cert - job: wordpress-kustomization-ready extends: @@ -769,33 +778,9 @@ zulip-taiko: variables: RESOURCE: "zulip" needs: - - job: configure-stackspin + - job: install-stackspin - job: zulip-cert - job: zulip-kustomization-ready extends: - .taiko - .zulip_rules - - -# Etc -# === - - -# Terminates a droplet and deletes the branch container image once the MR for it is merged -terminate-droplet: - # Stage has to be the same as the step that created the VPS - # https://docs.gitlab.com/ee/ci/environments.html#automatically-stopping-an-environment - stage: create-vps - # Gets triggered by on_stop of create-vps job - when: manual - variables: - GIT_STRATEGY: none - script: - - *debug_information - # Delete droplet - - python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^${CI_COMMIT_REF_SLUG}\")" - # Delete container image if one was created - - "if [ \"$CI_CONTAINER_TAG\" != \"main\" ]; then curl --request DELETE --header \"PRIVATE-TOKEN: ${CLEANER_TOKEN}\" https://open.greenhost.net/api/v4/projects/stackspin%2Fstackspin/registry/repositories/73/tags/${CI_CONTAINER_TAG}; fi" - environment: - name: $CI_COMMIT_REF_SLUG - action: stop diff --git a/.gitlab/ci_scripts/install_zerossl_issuer.sh b/.gitlab/ci_scripts/install_zerossl_issuer.sh index d6f41df3a48e28de5f7ac9a39ce25e318ac53393..beed27a206c3dc695f297ebeb33c9503ea700d84 100755 --- a/.gitlab/ci_scripts/install_zerossl_issuer.sh +++ b/.gitlab/ci_scripts/install_zerossl_issuer.sh @@ -11,9 +11,6 @@ set -euo pipefail # Create secret with HMAC key b64tlskey=$(echo -n "${ZEROSSL_TLS_KEY}" | base64 -w0) -# Wait until cert-manager is ready -"$(dirname "$0")/retry_cmd_until_success.sh" 30 10 "flux get kustomization --status-selector ready=true --no-header | grep '^cert-manager'" - # Add ZeroSSL ClusterIssuer kubectl apply -n cert-manager -f - <<EOF --- diff --git a/.gitlab/commit_template.txt b/.gitlab/commit_template.txt index de8d77085d583220987c6e2fba3afb5edfed0b1b..f87d232ce6dc7be34f585b3a324685abe0943e14 100644 --- a/.gitlab/commit_template.txt +++ b/.gitlab/commit_template.txt @@ -9,7 +9,7 @@ # TRIGGER_JOBS=enable-nextcloud # # or trigger all jobs: -# TRIGGER_JOBS=enable-monitoring,enable-nextcloud,enable-single-sign-on,enable-wekan,enable-wordpress,enable-zulip +# TRIGGER_JOBS=enable-monitoring,enable-nextcloud,enable-wekan,enable-wordpress,enable-zulip # # Reference issue number with one of: # diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 21366c08015344a4d137f4e95ed17e97109d59f4..03dfb78c0492f6f92ce82508ed58b7eefccd189f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.0.1 hooks: - id: check-added-large-files - id: check-ast diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bbfba419c31ecb76cfc24339962c8de8ffa8fed..6cbcca3829a62dae86c10e1f8863ce0b4c7c6e5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # Changelog + All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). @@ -7,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Changed project name from OpenAppStack to Stackspin * Update: - * Kube-prometheus-stack to helm chart version 18.0.12 + * Kube-prometheus-stack to helm chart version 22.0.0 ## [0.7.0] - 2021-08-19 diff --git a/test/pytest/test_resources.py b/test/pytest/test_resources.py index 83d29ae2cd6e14947994196bfc353dba280f56c9..35373ed245ef5dacaa17445d3ba67cf21c8a6670 100644 --- a/test/pytest/test_resources.py +++ b/test/pytest/test_resources.py @@ -10,10 +10,10 @@ Documentation for the kubernetes client: * https://github.com/kubernetes-client/python/tree/master/examples """ import os -from kubernetes import client, config -from kubernetes.client.rest import ApiException import pytest +from kubernetes import client, config +from kubernetes.client.rest import ApiException # Helper functions @@ -134,7 +134,10 @@ def run_around_tests(): Prepare kube config before running a test """ cluster_dir = os.environ.get("CLUSTER_DIR") - kubeconfig = os.path.join('..', str(cluster_dir), 'kube_config_cluster.yml') + if cluster_dir: + kubeconfig = os.path.join('..', str(cluster_dir), 'kube_config_cluster.yml') + else: + kubeconfig = os.path.join(os.environ.get("KUBECONFIG")) config.load_kube_config(config_file=kubeconfig) yield