--- include: - /.gitlab/ci_templates/kaniko.yml - /.gitlab/ci_templates/ssh_setup.yml - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' # Global templates and YAML anchors # ================================= # # Used in various stages/job definitions # We don't use a `before_script` definition here because `extend` doesn't merge # `before_script` but rather overwrites it. # So we rather use [yaml anchors](https://docs.gitlab.com/ce/ci/yaml/README.html#anchors) # here. Unfortunatly, anchors can't get included from files so we need to # define them here. .debug_information: &debug_information - | echo "Env vars:" echo echo "HOSTNAME: $HOSTNAME" echo "IP_ADDRESS: $IP_ADDRESS" echo "Uptime: $(uptime)" echo "CLUSTER_DIR: $CLUSTER_DIR" echo "ANSIBLE_HOST_KEY_CHECKING: $ANSIBLE_HOST_KEY_CHECKING" echo "KANIKO_BUILD_IMAGENAME: $KANIKO_BUILD_IMAGENAME" echo "KANIKO build image ref: ${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}" echo "SSH_KEY_ID: $SSH_KEY_ID" echo [ -d $CLUSTER_DIR ] && find $CLUSTER_DIR || echo "directory ${CLUSTER_DIR} not found" echo echo # The dotenv report requires us to report the artifacts in every job that is # required with a `needs:` from another job. .report_artifacts: artifacts: paths: - clusters expire_in: 1 month when: always reports: dotenv: $CLUSTER_DIR/.cluster.env # Rules that enable the cluster to be built and are applied to most steps # (except for application-specific steps) .general_rules: rules: - changes: - .gitlab-ci.yml - .gitlab/ci_scripts/* - Dockerfile - ansible/**/* - flux/**/* - flux2/**/* - install/**/* - test/**/* - openappstack/**/* - requirements.txt # app rules # # Define the rules when/if app specific jobs are run. # Just add the variable RESOURCE to the job like this: # variables: # RESOURCE: "eventrouter" # and import the templates with i.e. # extends: .eventrouter_rules # .eventrouter_rules will ensure that the job is only executed: # - when files related to the app changed in the repo # - A pipeline gets started from the UI and the job name is included in the # CI variable `TRIGGER_JOBS` # - A commit is pushed containing the pattern TRIGGER_JOBS=.*<job name> # (i.e. TRIGGER_JOBS=ci-test-image-build,enable-nextcloud) # # Gitlab CI allows pushing CI vars via `git push` but a bug prevents this when # using merge request pipelines (see https://gitlab.com/gitlab-org/gitlab/-/issues/326098) .eventrouter_rules: extends: - .general_rules .loki_rules: extends: - .general_rules .promtail_rules: extends: - .general_rules .nextcloud_rules: rules: - changes: - flux2/apps/$RESOURCE/*.yaml - flux2/cluster/optional/$RESOURCE/*.yaml - flux2/infrastructure/sources/nextcloud.yaml - install/install-app.sh - test/taiko/* - if: '$TRIGGER_JOBS =~ /enable-nextcloud/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-nextcloud/' - if: '$CI_COMMIT_BRANCH == "master"' .kube_prometheus_stack_rules: extends: - .general_rules .cert_manager_rules: extends: - .general_rules .local_path_provisioner_rules: extends: - .general_rules .rocketchat_rules: rules: - changes: - flux2/apps/$RESOURCE/*.yaml - flux2/cluster/optional/$RESOURCE/*.yaml - flux2/infrastructure/sources/helm-stable.yaml - install/install-app.sh - test/taiko/* - if: '$TRIGGER_JOBS =~ /enable-rocketchat/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-rocketchat/' - if: '$CI_COMMIT_BRANCH == "master"' .single_sign_on_rules: rules: - changes: - flux2/core/base/$RESOURCE/*.yaml - flux2/infrastructure/sources/single-sign-on.yaml - install/install-openappstack.sh - test/taiko/* - if: '$TRIGGER_JOBS =~ /enable-single-sign-on/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-single-sign-on/' - if: '$CI_COMMIT_BRANCH == "master"' .wekan_rules: rules: - changes: - flux2/apps/$RESOURCE/*.yaml - flux2/cluster/optional/$RESOURCE/*.yaml - flux2/infrastructure/sources/wekan.yaml - install/install-app.sh - test/taiko/* - if: '$TRIGGER_JOBS =~ /enable-wekan/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wekan/' - if: '$CI_COMMIT_BRANCH == "master"' .wordpress_rules: rules: - changes: - flux2/apps/$RESOURCE/*.yaml - flux2/cluster/optional/$RESOURCE/*.yaml - flux2/infrastructure/sources/wordpress.yaml - install/install-app.sh - test/taiko/* - if: '$TRIGGER_JOBS =~ /enable-wordpress/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wordpress/' - if: '$CI_COMMIT_BRANCH == "master"' # Global declarations # =================== # https://docs.gitlab.com/ee/ci/yaml/README.html#workflowrules-templates stages: - build - create-vps - setup-cluster - kustomization - base-helm-release - install-apps - apps-helm-release - apps-ready - certs - health-test - integration-test variables: SSH_KEY_ID: "411" HOSTNAME: "${CI_COMMIT_REF_SLUG}" ANSIBLE_HOST_KEY_CHECKING: "False" KANIKO_BUILD_IMAGENAME: "openappstack-ci" CLUSTER_DIR: "/builds/openappstack/openappstack/clusters/${CI_COMMIT_REF_SLUG}" default: image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}" # Stage: build # ============ # # Builds CI test container image # There are 2 moments in which we (re)build the container image. If some files are # changed, or when the job is triggered with TRIGGER_JOBS. ci-test-image-build: stage: build before_script: - *debug_information after_script: - | echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env artifacts: paths: - .ci.env expire_in: 1 month when: always reports: dotenv: .ci.env environment: name: image/$CI_COMMIT_REF_SLUG url: https://open.greenhost.net:4567/openappstack/openappstack/openappstack-ci:${CI_COMMIT_REF_SLUG} on_stop: delete-image auto_stop_in: 3 weeks rules: # Automatically rebuild the container image if this file, the Dockerfile, # the installed requirements or the kaniko template change - changes: - Dockerfile - requirements.txt - .gitlab/ci_templates/kaniko.yml # Also rebuild when the CI variable contain this jobs name # or commit msg contains /TRIGGER_JOBS=.*ci-test-image-build/ - if: '$TRIGGER_JOBS =~ /ci-test-image-build/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/' extends: - .kaniko_build interruptible: true report-ci-image-tag: stage: build image: "curlimages/curl" before_script: - *debug_information script: - | TAG_INFORMATION=$(curl https://open.greenhost.net/api/v4/projects/openappstack%2Fopenappstack/registry/repositories/2/tags/${CI_COMMIT_REF_SLUG}); echo "Tag information: ${TAG_INFORMATION}" if [ "$TAG_INFORMATION" == '{"message":"404 Tag Not Found"}' ]; then echo "CI_CONTAINER_TAG=master" > .ci.env else echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env fi artifacts: paths: - .ci.env expire_in: 1 month when: always reports: dotenv: .ci.env rules: # Make sure this job does not run if ci-test-image-build runs - changes: - Dockerfile - requirements.txt when: never # Never run on file changes that trigger ci-test-image-build - if: '$TRIGGER_JOBS =~ /ci-test-image-build/' when: never # Never run when ci-test-image is triggered manually - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/' when: never # Never run when ci-test-image is triggered manually - when: always interruptible: true # Stage: create-vps # ================= # # Creates the vps for the pipeline create-vps: stage: create-vps variables: SUBDOMAIN: "${CI_COMMIT_REF_SLUG}.ci" DOMAIN: "openappstack.net" script: - *debug_information # Creates a VPS based on a custom CI image for which the ansible playbook # has already run. See CONTRIBUTING.md#ci-pipeline-image for more info - bash .gitlab/ci_scripts/create_vps.sh # Make sure .ci.env variables are not lost - cat .ci.env >> ${CLUSTER_DIR}/.cluster.env extends: - .ssh_setup - .report_artifacts - .general_rules environment: name: $CI_COMMIT_REF_SLUG url: https://$FQDN on_stop: terminate-droplet auto_stop_in: 1 week interruptible: true # Stage: setup-cluster # ==================== # # Installs OAS test-dns: stage: setup-cluster needs: - job: create-vps script: - *debug_information - cd ansible/ - pytest -v -s -m 'dns' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' extends: - .general_rules interruptible: true setup-openappstack: stage: setup-cluster script: - *debug_information # Copy inventory files to ansible folder for use in install-apps step - chmod 700 ansible - cp ${CLUSTER_DIR}/inventory.yml ansible/ # Set up cluster - python3 -m openappstack $HOSTNAME install # Customize env file, remove all comments and empty lines - sed "s/1.2.3.4/$IP_ADDRESS/; s/example.org/$FQDN/; s/acme_staging=false/acme_staging=true/; s/acme-v02/acme-staging-v02/; /^\s*#.*$/d; /^\s*$/d" install/.flux.env.example >> ${CLUSTER_DIR}/.flux.env # Deploy secret/oas-cluster-variables - cp install/kustomization.yaml ${CLUSTER_DIR} - kubectl apply -k ${CLUSTER_DIR} - bash ./install/install-openappstack.sh # TODO: Should also be removed or made up-to-date # Show versions of installed apps/binaries # - cd ansible # - ansible master -m shell -a 'oas-version-info.sh 2>&1' extends: - .ssh_setup - .report_artifacts - .general_rules interruptible: true # Stage: kustomization # ==================== # # Tests if all kustomizations are ready .kustomization-ready: stage: kustomization needs: - job: setup-openappstack - job: test-dns script: - *debug_information - cd ansible/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/kube_config_cluster.yml" - pytest -v -s -m 'kustomizations' --resource="$RESOURCE" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 20 extends: - .ssh_setup - .general_rules interruptible: true core-kustomizations-ready: variables: RESOURCE: "core" extends: - .kustomization-ready infrastructure-kustomizations-ready: variables: RESOURCE: "infrastructure" extends: - .kustomization-ready monitoring-kustomizations-ready: variables: RESOURCE: "monitoring" extends: - .kustomization-ready openappstack-kustomizations-ready: variables: RESOURCE: "openappstack" extends: - .kustomization-ready .helm-release: script: - *debug_information - cd ansible/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/kube_config_cluster.yml" - pytest -v -s -m 'helmreleases' --resource="$RESOURCE" --namespace="$NAMESPACE" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 20 extends: - .ssh_setup interruptible: true # Stage: base-helm-release # ================== # # Checks helmreleases for oas base are ready .base-helm-release: stage: base-helm-release needs: - job: core-kustomizations-ready - job: infrastructure-kustomizations-ready - job: openappstack-kustomizations-ready - job: setup-openappstack - job: test-dns extends: - .helm-release - .general_rules cert-manager-helm-release: variables: RESOURCE: "cert-manager" NAMESPACE: "cert-manager" extends: - .base-helm-release - .cert_manager_rules eventrouter-helm-release: variables: RESOURCE: "eventrouter" NAMESPACE: "oas" extends: - .base-helm-release - .eventrouter_rules local-path-provisioner-helm-release: variables: RESOURCE: "local-path-provisioner" NAMESPACE: "kube-system" extends: - .base-helm-release - .local_path_provisioner_rules loki-helm-release: variables: RESOURCE: "loki" NAMESPACE: "oas" extends: - .base-helm-release - .loki_rules promtail-helm-release: variables: RESOURCE: "promtail" NAMESPACE: "oas" extends: - .base-helm-release - .promtail_rules kube-prometheus-stack-helm-release: variables: RESOURCE: "kube-prometheus-stack" NAMESPACE: "oas" extends: - .base-helm-release - .kube_prometheus_stack_rules single-sign-on-helm-release: variables: RESOURCE: "single-sign-on" NAMESPACE: "oas" extends: - .base-helm-release - .single_sign_on_rules # Stage: install-apps # ================== # # Checks if application needs to get installed .enable_app_template: stage: install-apps script: - *debug_information # Add optional override values we need for the CI pipeline only - '[ -f ./install/overrides/oas-${RESOURCE}-override.yaml ] && kubectl apply -n oas-apps -f ./install/overrides/oas-${RESOURCE}-override.yaml' - bash ./install/install-app.sh ${RESOURCE} extends: - .ssh_setup interruptible: true enable-nextcloud: variables: RESOURCE: "nextcloud" extends: - .enable_app_template - .nextcloud_rules enable-rocketchat: variables: RESOURCE: "rocketchat" extends: - .enable_app_template - .rocketchat_rules enable-wekan: variables: RESOURCE: "wekan" extends: - .enable_app_template - .wekan_rules enable-wordpress: variables: RESOURCE: "wordpress" extends: - .enable_app_template - .wordpress_rules # Stage: apps-helm-release # ==================== # # Tests if all helm releases are ready .apps-helm-release: stage: apps-helm-release extends: - .helm-release interruptible: true nextcloud-helm-release: variables: RESOURCE: "nextcloud" NAMESPACE: "oas-apps" needs: - job: core-kustomizations-ready - job: infrastructure-kustomizations-ready - job: openappstack-kustomizations-ready - job: setup-openappstack - job: test-dns - job: enable-nextcloud extends: - .apps-helm-release - .nextcloud_rules rocketchat-helm-release: variables: RESOURCE: "rocketchat" NAMESPACE: "oas-apps" needs: - job: core-kustomizations-ready - job: infrastructure-kustomizations-ready - job: openappstack-kustomizations-ready - job: setup-openappstack - job: test-dns - job: enable-rocketchat extends: - .apps-helm-release - .rocketchat_rules wekan-helm-release: variables: RESOURCE: "wekan" NAMESPACE: "oas-apps" needs: - job: core-kustomizations-ready - job: infrastructure-kustomizations-ready - job: openappstack-kustomizations-ready - job: setup-openappstack - job: test-dns - job: enable-wekan extends: - .apps-helm-release - .wekan_rules wordpress-helm-release: variables: RESOURCE: "wordpress" NAMESPACE: "oas-apps" needs: - job: core-kustomizations-ready - job: infrastructure-kustomizations-ready - job: openappstack-kustomizations-ready - job: setup-openappstack - job: test-dns - job: enable-wordpress extends: - .apps-helm-release - .wordpress_rules # Stage: apps-ready # ====================== # # Tests apps for readiness state .apps-deployment: stage: apps-ready script: - *debug_information - cd ansible/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/kube_config_cluster.yml" - pytest -v -s -m 'deployments' --resource="$RESOURCE" --namespace="$NAMESPACE" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 extends: - .ssh_setup interruptible: true .apps-statefulset: stage: apps-ready script: - *debug_information - cd ansible/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/kube_config_cluster.yml" - pytest -v -s -m 'statefulsets' --resource="$RESOURCE" --namespace="$NAMESPACE" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 extends: - .ssh_setup interruptible: true .apps-daemonset: stage: apps-ready script: - *debug_information - cd ansible/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/kube_config_cluster.yml" - pytest -v -s -m 'daemonsets' --resource="$RESOURCE" --namespace="$NAMESPACE" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 extends: - .ssh_setup interruptible: true cert-manager-deployment: variables: RESOURCE: "cert-manager" NAMESPACE: "cert-manager" needs: - job: cert-manager-helm-release - job: setup-openappstack # Needs makes sure the artifacts from that job are downloaded extends: - .apps-deployment - .cert_manager_rules eventrouter-deployment: variables: RESOURCE: "eventrouter" NAMESPACE: "oas" needs: - job: eventrouter-helm-release - job: setup-openappstack extends: - .apps-deployment - .eventrouter_rules local-path-provisioner-deployment: variables: RESOURCE: "local-path-provisioner" NAMESPACE: "kube-system" needs: - job: local-path-provisioner-helm-release - job: setup-openappstack extends: - .apps-deployment - .local_path_provisioner_rules loki-statefulset: variables: RESOURCE: "loki" NAMESPACE: "oas" needs: - job: loki-helm-release - job: setup-openappstack extends: - .apps-statefulset - .loki_rules promtail-daemonset: variables: RESOURCE: "promtail" NAMESPACE: "oas" needs: - job: promtail-helm-release - job: setup-openappstack extends: - .apps-daemonset - .promtail_rules nextcloud-deployment: variables: RESOURCE: "nextcloud" NAMESPACE: "oas-apps" needs: - job: nextcloud-helm-release - job: setup-openappstack extends: - .apps-deployment - .nextcloud_rules kube-prometheus-stack-deployment: variables: RESOURCE: "kube-prometheus-stack" NAMESPACE: "oas" needs: - job: kube-prometheus-stack-helm-release - job: setup-openappstack extends: - .apps-deployment - .kube_prometheus_stack_rules rocketchat-deployment: variables: RESOURCE: "rocketchat" NAMESPACE: "oas-apps" needs: - job: rocketchat-helm-release - job: setup-openappstack extends: - .apps-deployment - .rocketchat_rules single-sign-on-deployment: variables: RESOURCE: "single-sign-on" NAMESPACE: "oas" needs: - job: single-sign-on-helm-release - job: setup-openappstack extends: - .apps-deployment - .single_sign_on_rules wekan-deployment: variables: RESOURCE: "wekan" NAMESPACE: "oas-apps" needs: - job: wekan-helm-release - job: setup-openappstack extends: - .apps-deployment - .wekan_rules wordpress-statefulset: variables: RESOURCE: "wordpress" NAMESPACE: "oas-apps" needs: - job: wordpress-helm-release - job: setup-openappstack extends: - .apps-statefulset - .wordpress_rules # Stage: certs # ================ # # Test each app for proper certs .apps-cert: stage: certs script: - *debug_information - cd ansible/ - pytest -v -s -m 'certs' --resource="$RESOURCE" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 extends: - .ssh_setup interruptible: true nextcloud-cert: variables: RESOURCE: "nextcloud" needs: - job: nextcloud-deployment - job: setup-openappstack extends: - .apps-cert - .nextcloud_rules kube-prometheus-stack-cert: variables: RESOURCE: "kube-prometheus-stack" needs: - job: kube-prometheus-stack-deployment - job: setup-openappstack extends: - .apps-cert - .kube_prometheus_stack_rules rocketchat-cert: variables: RESOURCE: "rocketchat" needs: - job: rocketchat-deployment - job: setup-openappstack extends: - .apps-cert - .rocketchat_rules single-sign-on-cert: variables: RESOURCE: "single-sign-on" needs: - job: single-sign-on-deployment - job: setup-openappstack extends: - .apps-cert - .single_sign_on_rules wekan-cert: variables: RESOURCE: "wekan" needs: - job: wekan-deployment - job: setup-openappstack extends: - .apps-cert - .wekan_rules wordpress-cert: variables: RESOURCE: "wordpress" needs: - job: wordpress-statefulset - job: setup-openappstack extends: - .apps-cert - .wordpress_rules # Stage: health-test # ================== # # General cluster health checks testinfra: stage: health-test script: - *debug_information - cd ansible/ - pytest -v -s -m 'testinfra' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' extends: - .ssh_setup - .general_rules interruptible: true kube-prometheus-stack-alerts: stage: health-test variables: # RESOURCE var is used in job specific rules (i.e. .kube_prometheus_stack_rules) RESOURCE: "kube-prometheus-stack" # Enforce python requests using the system cert store, where LE staging # cacert is added REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt" allow_failure: true script: - *debug_information - export BASIC_AUTH_PW=$(python3 -m openappstack $HOSTNAME secrets | grep oas-prometheus-basic-auth | cut -d'=' -f2) - cd test/ - pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' extends: - .ssh_setup - .kube_prometheus_stack_rules needs: - job: kube-prometheus-stack-deployment - job: setup-openappstack interruptible: true # Stage: integration-test # ======================= # # Runs integration tests for most apps using taiko .taiko: stage: integration-test script: - *debug_information # Run the taiko tests for specific app - python3 -m openappstack $HOSTNAME test --apps $RESOURCE retry: 2 artifacts: paths: - test/taiko/Screenshot* expire_in: 1 month when: on_failure extends: - .ssh_setup interruptible: true grafana-taiko: variables: RESOURCE: "grafana" needs: - job: kube-prometheus-stack-cert - job: setup-openappstack extends: - .taiko - .kube_prometheus_stack_rules nextcloud-taiko: variables: RESOURCE: "nextcloud" needs: - job: nextcloud-cert - job: setup-openappstack extends: - .taiko - .nextcloud_rules rocketchat-taiko: variables: RESOURCE: "rocketchat" needs: - job: rocketchat-cert - job: setup-openappstack extends: - .taiko - .rocketchat_rules allow_failure: true wekan-taiko: variables: RESOURCE: "wekan" needs: - job: wekan-cert - job: setup-openappstack extends: - .taiko - .wekan_rules wordpress-taiko: variables: RESOURCE: "wordpress" needs: - job: wordpress-cert - job: setup-openappstack extends: - .taiko - .wordpress_rules # Etc # === # Terminates a droplet once the branch for it is deleted terminate-droplet: # Stage has to be the same as the step that created the VPS # https://docs.gitlab.com/ee/ci/environments.html#automatically-stopping-an-environment stage: create-vps # Gets triggered by on_stop of create-vps job when: manual variables: GIT_STRATEGY: none script: - *debug_information - python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^${CI_COMMIT_REF_SLUG}\")" environment: name: $CI_COMMIT_REF_SLUG action: stop # Deletes a container image once a branch is deleted. # Careful! When you run this step manually, you might have to trigger container # image re-build as well delete-image: stage: build when: manual variables: GIT_STRATEGY: none script: - *debug_information - "curl --request DELETE --header \"PRIVATE-TOKEN: ${CLEANER_TOKEN}\" https://open.greenhost.net/api/v4/projects/openappstack%2Fopenappstack/registry/repositories/2/tags/${CI_COMMIT_REF_SLUG}" environment: name: image/$CI_COMMIT_REF_SLUG action: stop