.dockerignore

+.direnv
+.direnv
+.dockerignore
+.envrc
+.git
+.gitignore
+.gitlab
+.gitlab-ci.yml
+.gitlint
+.pre-commit-config.yaml
+.renovaterc.json
+.sourceignore
+CHANGELOG.md
+Dockerfile
+Makefile
+README.md
+VERSION
+ansible
+clusters
+docs
+flux2
+install
+stackspin
+upgrade-scripts
+venv

.gitignore

 # Ignore custom files created when following the README.md
-!ansible/group_vars/all/settings.yml.example
 /ansible/group_vars/all/settings*
 !/ansible/inventory.yml.example
 /ansible/inventory*
@@ -11,21 +10,30 @@
 /env
 /venv
 /.direnv
+.envrc
 
-# Ignore files created during CI using test/ci-bootstrap.py
+# Ignore files created during CI or development
 /test/group_vars/all/
 /test/inventory*
-/test/behave/behave.ini
-/test/behave/rerun_failing.features
+/test/cypress/videos/
+/test/cypress/screenshots/
+/test/cypress/logs/
 /clusters
+/install/installation-kustomization/*.txt
 
 # Ignore files created during tests
-/test/behave/**/screenshots/
+Screenshot-*.png
 
 # Etc
 __pycache__
 *.swp
 *.pyc
 
+# Personal taskfile(s)
+Taskfile.yaml
+
 # Documentation files
 /docs/_build
+
+# node modules for cypress
+node_modules

.gitlab-ci.yml

-include:
-  - .gitlab/ci_templates/kaniko.yml
-  - .gitlab/ci_templates/ssh_setup.yml
-stages:
-  - build
-  - create-vps
-  - setup-cluster
-  - install-apps
-  - wait-for-deployments
-  - health-test
-  - integration-test
-  - cleanup
-variables:
-  SSH_KEY_ID: "411"
-  HOSTNAME: "${CI_COMMIT_REF_SLUG}"
-  # Repeated values, because we're not allowed to use a variable in a variable
-  SUBDOMAIN: "${CI_COMMIT_REF_SLUG}.ci"
-  DOMAIN: "openappstack.net"
-  ADDRESS: "${CI_COMMIT_REF_SLUG}.ci.openappstack.net"
-  ANSIBLE_HOST_KEY_CHECKING: "False"
-  KANIKO_BUILD_IMAGENAME: "openappstack-ci"
-
-default:
-  image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_COMMIT_REF_SLUG}"
-
-
-ci_test_image:
-  stage: build
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - Dockerfile
-      - requirements.txt
-      - .gitlab/ci_templates/kaniko.yml
-  extends: .kaniko_build
-
-create-vps:
-  stage: create-vps
-  script:
-    - echo "hostname $HOSTNAME, subdomain $SUBDOMAIN, domain $DOMAIN, address $ADDRESS";
-    - ls clusters/${HOSTNAME} || echo "directory clusters/${HOSTNAME} not found"
-    # Creates the VPS only if an old VPS for this branch is not re-usable
-    - sh .gitlab/ci_scripts/create_vps.sh
-  artifacts:
-    paths:
-    - clusters
-    expire_in: 1 month
-    when: always
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - .gitlab/ci_scripts/*
-      - ansible/**/*
-      - flux/**/*
-      - test/**/*
-      - openappstack/**/*
-  extends: .ssh_setup
-  # Cache the cluster secrets so the next job can use it too
-  cache:
-    paths:
-      - clusters/$HOSTNAME/**
-    key: ${CI_COMMIT_REF_SLUG}
-
-setup-openappstack:
-  stage: setup-cluster
-  script:
-    # Copy inventory files to ansible folder for use in install-apps step
-    - chmod 700 ansible
-    - cp clusters/${CI_COMMIT_REF_SLUG}/inventory.yml ansible/
-    - cp clusters/${CI_COMMIT_REF_SLUG}/settings.yml ansible/group_vars/all/
-    # Set up cluster
-    - python3 -m openappstack $HOSTNAME install
-    # Show versions of installed apps/binaries
-    - chmod 700 ansible
-    - cd ansible
-    - ansible master -m shell -a 'oas-version-info.sh 2>&1'
-  artifacts:
-    paths:
-    - ./clusters
-    - ansible/inventory.yml
-    - ansible/group_vars/all/settings.yml
-    expire_in: 1 month
-    when: always
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - ansible/**/*
-      - flux/**/*
-      - test/**/*
-      - openappstack/**/*
-  extends: .ssh_setup
-  # Cache the cluster secrets so the next job can use them
-  cache:
-    paths:
-      - clusters/$HOSTNAME/**
-    key: ${CI_COMMIT_REF_SLUG}
-
-helmreleases:
-  stage: wait-for-deployments
-  script:
-    - cd ansible/
-    - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml"
-    - pytest -v -s -m 'helmreleases' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - ansible/**/*
-      - flux/**/*
-      - test/**/*
-      - openappstack/**/*
-  extends: .ssh_setup
+---
 
-testinfra:
-  stage: health-test
-  script:
-    - cd ansible/
-    - pytest -v -m 'testinfra' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*'
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - ansible/**/*
-      - flux/**/*
-      - test/**/*
-      - openappstack/**/*
-  extends: .ssh_setup
+workflow:
+  rules:
+  # Only run our pipelines for merge requests.
+  - if: $CI_MERGE_REQUEST_IID
+  # Or when the pipeline was scheduled (nightly
+  # run on main in particular).
+  - if: '$CI_PIPELINE_SOURCE == "schedule"'
 
-certs:
-  stage: health-test
-  allow_failure: true
-  script:
-    - cd ansible/
-    - pytest -s -m 'certs' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*'
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - ansible/**/*
-      - flux/**/*
-      - test/**/*
-      - openappstack/**/*
-  extends: .ssh_setup
-
-prometheus-alerts:
-  stage: health-test
-  variables:
-    OAS_DOMAIN: '${CI_COMMIT_REF_SLUG}.ci.openappstack.net'
-  allow_failure: true
-  script:
-    - cd test/
-    - pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*'
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - ansible/**/*
-      - flux/**/*
-      - test/**/*
-  extends: .ssh_setup
-
-behave-nextcloud:
-  stage: integration-test
-  script:
-    # Wait until flux creates the NextCloud HelmRelease.
-    - ssh root@$ADDRESS '/bin/bash -c "while true; do kubectl get hr -n oas-apps nextcloud; if [ \$? -eq 0 ]; then break; fi; sleep 20; done"'
-    # Wait until NextCloud is ready.
-    - ssh root@$ADDRESS '/bin/bash -c "kubectl wait -n oas-apps hr/nextcloud --for condition=Released --timeout=20m"'
-    # Run the behave tests for NextCloud.
-    - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags nextcloud || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags nextcloud
-  artifacts:
-    paths:
-    - test/behave/screenshots/
-    expire_in: 1 month
-    when: on_failure
-  retry: 2
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - ansible/**/*
-      - flux/**/*
-      - test/**/*
-      - openappstack/**/*
-  extends: .ssh_setup
-
-behave-grafana:
-  stage: integration-test
-  script:
-    - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags grafana || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags grafana
-  artifacts:
-    paths:
-    - test/behave/screenshots/
-    expire_in: 1 month
-    when: on_failure
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - ansible/**/*
-      - flux/**/*
-      - test/**/*
-      - openappstack/**/*
-
-terminate_mr_droplet_after_merge:
-  stage: cleanup
-  before_script:
-    - echo "We leave MR droplets running even when the pipeline is successful \
-      to be able to investigate a MR. We need to terminate them when the MR \
-      is merged into master."
-  script: |
-    if [ "$(git show -s --pretty=%p HEAD | wc -w)" -gt 1 ]
-    then
-      commit_message="$(git show -s --format=%s)"
-      tmp="${commit_message#*\'}"
-      merged_branch="${tmp%%\'*}"
-      echo "Current HEAD is a merge commit, removing droplet from related merge request branch name '#${merged_branch}'."
-      python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^${merged_branch}\.\")"
-    else
-      echo "Current HEAD is NOT a merge commit, nothing to do."
-    fi
-  only:
-    refs:
-      - master
-
-terminate_old_droplets:
-  stage: cleanup
-  script:
-    - echo "Terminate droplets 5 days after creation. Branches that exist longer than 5 days will get a new droplet when CI runs again."
-    - python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name('\d+-.*', 5)"
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - ansible/**/*
-      - flux/**/*
-      - test/**/*
-      - openappstack/**/*
-
-# We need one job that run every time (without any `only:` limitation).
-# This works around a Gitlab bug: if no job runs at all due to
-# `only`, Gitlab gets confused and doesn't allow you to merge the MR:
-# https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html#limitations
-gitlab-merge-workaround:
-  stage: cleanup
-  script:
-    - echo "That went well"
+include:
+  - local: /.gitlab/ci_pipelines/default.yml
+    rules:
+      - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME !~ /pre-release*/'
+  - local: /.gitlab/ci_pipelines/upgrade_test.yml
+    rules:
+      - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /pre-release*/'
+      - if: '$CI_COMMIT_MESSAGE =~ /UPGRADE_TEST/'

.gitlab/ci_pipelines/apps_ready.yml

+---
+
+workflow:
+  rules:
+  # Only run our pipelines for merge requests.
+  - if: $CI_MERGE_REQUEST_IID
+  # Or when the pipeline was scheduled (nightly
+  # run on main in particular).
+  - if: '$CI_PIPELINE_SOURCE == "schedule"'
+
+include:
+  - local: .gitlab/ci_templates/stackspin_common.yml
+
+stages:
+  - get-artifacts
+  - base-ready
+  - integration-test
+
+# Stage: optional-apps-ready
+# ================
+#
+# Check that the kustomizations of all installed apps are ready.
+
+# Downloads the artifacts from the parent pipeline, used by all the other jobs
+# in this pipeline. This needs to happen in a separate job, because all the
+# other jobs are optional.
+get-artifacts:
+  stage:
+    get-artifacts
+  # This job needs a script, otherwise GitLab CI lint will complain.
+  script:
+    - echo "This job only exists to get artifacts from the parent pipeline"
+  extends:
+    - .child_require_artifacts
+    - .report_artifacts
+
+kustomization-ready:
+  extends:
+    - .kustomization_ready
+  rules:
+    - if: '$SKIP_KUSTOMIZATION_READY == "true"'
+      when: never
+    - when: always
+
+cert:
+  stage: base-ready
+  script:
+    - cd test/
+    - pytest -v -s -m 'certs' --resource="$RESOURCE" --desired-revision "${STACKSPIN_BRANCH}@sha1:${STACKSPIN_COMMIT_SHA}" --reruns 120 --reruns-delay 10
+  interruptible: true
+  rules:
+    - if: '$SKIP_CERT == "true"'
+      when: never
+    - when: always
+
+cypress:
+  stage: integration-test
+  image:
+    # https://hub.docker.com/r/cypress/included/tags
+    name: cypress/included:14.1.0
+    entrypoint: [""]
+  variables:
+    CYPRESS_USE_SSO_LOGIN: "true"
+    CYPRESS_SSO_USER: "admin@ci.stackspin.net"
+    CYPRESS_SSO_PW: "$SSO_PASSWORD"
+    CYPRESS_SSO_LOGOUT: "https://sso.$FQDN/oauth2/sessions/logout"
+    GIT_STRATEGY: "${INSTALL_GIT_STRATEGY}"
+  script:
+    - !reference [.checkout_branch, script]
+    - cd test
+    - npm install
+    - cypress run --record --key "$CYPRESS_RECORD_KEY" --spec "cypress/e2e/$RESOURCE.cy.js"
+  interruptible: true
+  rules:
+    - if: '$CYPRESS_TEST == "true"'
+      when: always
+    - when: never
+
+helm-test:
+  stage: integration-test
+  image:
+    name: alpine/helm:3.17.1
+    entrypoint: ["/bin/sh", "-c"]
+  script:
+    - export KUBECONFIG="${PWD}/clusters/${HOSTNAME}/kube_config_cluster.yml"
+    - time helm test -n $HELM_RELEASE_NAMESPACE --logs --timeout 30m $HELM_RELEASE_NAME
+  interruptible: true
+  rules:
+    - if: '$HELM_TEST == "true"'
+      when: always
+    - when: never

.gitlab/ci_pipelines/default.yml

+---
+
+workflow:
+  rules:
+  # Only run our pipelines for merge requests.
+  - if: $CI_MERGE_REQUEST_IID
+  # Or when the pipeline was scheduled (nightly
+  # run on main in particular).
+  - if: '$CI_PIPELINE_SOURCE == "schedule"'
+
+include:
+  - /.gitlab/ci_templates/kaniko.yml
+  - /.gitlab/ci_templates/stackspin_common.yml
+
+stages:
+  - build
+  - install-cluster
+  - install-stackspin
+  - apps-ready
+  - cluster-health
+
+variables:
+  # Useful for debugging
+  # !!! CAREFUL - this will expose private keys in the CI logs !!!
+  # !!! Make sure to delete the resulting job output from the  !!!
+  # !!! job's output site (Trash can symbol at the top)        !!!
+  # CI_DEBUG_TRACE: "true"
+  # This decides which branch gets used by the create-vps and install-stackspin
+  # jobs. It also indirectly influences the commit sha that's checked for in the
+  # kustomization ready jobs
+  INSTALL_STACKSPIN_BRANCH: ${CI_COMMIT_REF_NAME}
+  # This pipeline uses the "fetch" (default) git strategy, but the
+  # "upgrade_test" pipeline needs the slower "clone" git strategy to be able to
+  # install an older version of Stackspin
+  INSTALL_GIT_STRATEGY: "fetch"
+  HOSTNAME: ${CI_COMMIT_REF_SLUG}
+
+# Stage: build
+# ============
+#
+# Builds CI test container image
+# There are 2 moments in which we (re)build the container image. If some files are
+# changed, or when the job is triggered with TRIGGER_JOBS.
+
+ci-test-image-build:
+  stage: build
+  after_script:
+    - |
+      echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" | tee .ci.env
+  artifacts:
+    paths:
+      - .ci.env
+    expire_in: 1 month
+    when: always
+    reports:
+      dotenv:
+        .ci.env
+  environment:
+    name: image/${CI_COMMIT_REF_SLUG}
+    url:
+      https://open.greenhost.net:4567/stackspin/stackspin/stackspin-ci:${CI_COMMIT_REF_SLUG}
+    auto_stop_in: 3 weeks
+  rules:
+    # Automatically rebuild the container image if this file, the Dockerfile,
+    # the installed requirements or the kaniko template change
+    - changes:
+        - Dockerfile
+        - requirements.txt
+        - .gitlab/ci_templates/kaniko.yml
+    # Also rebuild when the CI variable contain this jobs name
+    # or commit msg contains /TRIGGER_JOBS=.*ci-test-image-build/
+    - if: '$TRIGGER_JOBS =~ /ci-test-image-build/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/'
+  extends:
+    - .kaniko_build
+  interruptible: true
+
+report-ci-image-tag:
+  stage: build
+  image: "curlimages/curl"
+  script:
+    - |
+      TAG_INFORMATION=$(curl -sS https://open.greenhost.net/api/v4/projects/stackspin%2Fstackspin/registry/repositories/73/tags/${CI_COMMIT_REF_SLUG});
+      echo "Tag information: ${TAG_INFORMATION}"
+      if [ "$TAG_INFORMATION" == '{"message":"404 Tag Not Found"}' ]; then
+        CI_CONTAINER_TAG="$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
+      else
+        CI_CONTAINER_TAG="${CI_COMMIT_REF_SLUG}"
+      fi
+      echo "CI_CONTAINER_TAG=${CI_CONTAINER_TAG}" | tee .ci.env
+  artifacts:
+    paths:
+      - .ci.env
+    expire_in: 1 month
+    when: always
+    reports:
+      dotenv:
+        .ci.env
+  rules:
+    # Make sure this job does not run if ci-test-image-build runs
+    - changes:
+        - Dockerfile
+        - requirements.txt
+        - .gitlab/ci_templates/kaniko.yml
+      when: never  # Never run on file changes that trigger ci-test-image-build
+    - if: '$TRIGGER_JOBS =~ /ci-test-image-build/'
+      when: never  # Never run when ci-test-image is triggered manually
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/'
+      when: never  # Never run when ci-test-image is triggered manually
+    - when: always
+  interruptible: true
+
+
+# Stage: install-cluster
+# ======================
+#
+# * Creates the vps for the pipeline
+# * Installs k8s with ansible
+
+create-vps:
+  stage: install-cluster
+  variables:
+    SUBDOMAIN: "${HOSTNAME}.ci"
+    DOMAIN: "stackspin.net"
+    GIT_STRATEGY: "${INSTALL_GIT_STRATEGY}"
+    API_URL: "https://open.greenhost.net/api/v4"
+  script:
+    # Find out which apps will get installed so the create_vps.sh script
+    # can create a droplet with proper memory size.
+    # Then create a VPS based on a custom CI image for which the ansible playbook
+    # has already run.
+    - |
+      curl --silent --show-error --header "PRIVATE-TOKEN: $CREATE_VPS_TOKEN" "${API_URL}/projects/6/pipelines/${CI_PIPELINE_ID}/bridges" -o bridges.json
+      export ENABLED_APPS=$(jq -r '.[] | select(.name | contains("app-ready")) | .name' bridges.json | sed 's/-app-ready//' | tr '\n' ',' | sed 's/,$//')
+      # See CONTRIBUTING.md#ci-pipeline-image for more info
+      bash .gitlab/ci_scripts/create_vps.sh
+  extends:
+    - .ssh_setup
+    - .report_artifacts
+    - .general_rules
+  environment:
+    name: $CI_COMMIT_REF_SLUG
+    url: https://$FQDN
+    on_stop: terminate-droplet
+    auto_stop_in: 1 week
+  retry: 1
+  interruptible: true
+
+test-dns:
+  stage: install-cluster
+  needs:
+    - job: create-vps
+  # Needs a pytest ansible connection to get the configured system resolvers
+  script:
+    - cd ansible/
+    - pytest -v -s -m 'dns' --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
+  extends:
+    - .general_rules
+  interruptible: true
+
+# Terminates a droplet and deletes the branch container image once the MR for it is merged
+terminate-droplet:
+  # Stage has to be the same as the step that created the VPS
+  # https://docs.gitlab.com/ee/ci/environments.html#automatically-stopping-an-environment
+  stage: install-cluster
+  # Gets triggered by on_stop of create-vps job
+  when: manual
+  variables:
+    GIT_STRATEGY: none
+  before_script:
+    - echo "Default before script is overwritten because this job has no access to the Git repo"
+  script:
+    # Delete droplet
+    - python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^${HOSTNAME}\")"
+    # Delete container image if one was created. Do not delete `main` nor `v*`.
+    - >
+      if [[ "$CI_CONTAINER_TAG" != "main" && ! "$CI_CONTAINER_TAG" =~ v\d+ ]]; then
+        curl --request DELETE --header "PRIVATE-TOKEN: ${CLEANER_TOKEN}" https://open.greenhost.net/api/v4/projects/stackspin%2Fstackspin/registry/repositories/73/tags/${CI_CONTAINER_TAG};
+      fi
+  # We have problems with cosmos not allowing deletion even though the droplet
+  # was just stopped.
+  retry: 2
+  environment:
+    name: $CI_COMMIT_REF_SLUG
+    action: stop
+
+install-stackspin:
+  stage: install-stackspin
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+  trigger:
+    include:
+      - local: .gitlab/ci_pipelines/install_stackspin.yml
+    strategy: depend
+  extends:
+    - .general_rules
+
+# Stage: apps
+# ================
+#
+# Check that the kustomizations of all installed apps are ready.
+
+hedgedoc-app-ready:
+  stage: apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    RESOURCE: "hedgedoc"
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    CYPRESS_TEST: "true"
+    CYPRESS_BASE_URL: "https://hedgedoc.$FQDN"
+  extends:
+    - .hedgedoc_rules
+    - .trigger_apps_ready_pipeline
+
+monitoring-app-ready:
+  stage: apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    RESOURCE: "monitoring"
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    CYPRESS_TEST: "true"
+    CYPRESS_BASE_URL: "https://grafana.$FQDN"
+  extends:
+    - .monitoring_rules
+    - .trigger_apps_ready_pipeline
+
+nextcloud-app-ready:
+  stage: apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    RESOURCE: "nextcloud"
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    # Use `helm test` to run Cypress tests instead of directly in CI image
+    HELM_TEST: "true"
+    HELM_RELEASE_NAME: "nc"
+    HELM_RELEASE_NAMESPACE: "stackspin-apps"
+  extends:
+    - .nextcloud_rules
+    - .trigger_apps_ready_pipeline
+
+velero-app-ready:
+  stage: apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    RESOURCE: "velero"
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    # Do not run `cert` and cypress tests, because we don't have them for velero
+    SKIP_CERT: "true"
+  extends:
+    - .velero_rules
+    - .trigger_apps_ready_pipeline
+
+wekan-app-ready:
+  stage: apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    RESOURCE: "wekan"
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    CYPRESS_TEST: "true"
+    CYPRESS_BASE_URL: "https://wekan.$FQDN"
+  extends:
+    - .wekan_rules
+    - .trigger_apps_ready_pipeline
+
+wordpress-app-ready:
+  stage: apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    RESOURCE: "wordpress"
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    CYPRESS_TEST: "true"
+    CYPRESS_BASE_URL: "https://www.$FQDN"
+  extends:
+    - .wordpress_rules
+    - .trigger_apps_ready_pipeline
+
+zulip-app-ready:
+  stage: apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    RESOURCE: "zulip"
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    CYPRESS_TEST: "true"
+    CYPRESS_BASE_URL: "https://zulip.$FQDN"
+  extends:
+    - .zulip_rules
+    - .trigger_apps_ready_pipeline
+
+single-sign-on-app-ready:
+  stage: apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    RESOURCE: "single-sign-on"
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    # kustomization-ready has already run as part of the base-ready in
+    # the install-stackspin pipeline
+    SKIP_KUSTOMIZATION_READY: "true"
+  extends:
+    - .general_rules
+    - .trigger_apps_ready_pipeline
+
+dashboard-app-ready:
+  stage: apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: create-vps
+    RESOURCE: "dashboard"
+    STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    # kustomization-ready has already run as part of the base-ready in
+    # the install-stackspin pipeline
+    SKIP_KUSTOMIZATION_READY: "true"
+    HELM_TEST: "true"
+    HELM_RELEASE_NAME: "dashboard"
+    HELM_RELEASE_NAMESPACE: "stackspin"
+  extends:
+    - .general_rules
+    - .trigger_apps_ready_pipeline
+
+
+# Stage: cluster-health
+# =====================
+#
+# General cluster health checks
+
+testinfra:
+  stage: cluster-health
+  extends:
+    - .testinfra
+
+prometheus-alerts:
+  stage: cluster-health
+  extends:
+    - .prometheus_alerts

.gitlab/ci_pipelines/install_stackspin.yml

+---
+
+workflow:
+  rules:
+  # Only run our pipelines for merge requests.
+  - if: $CI_MERGE_REQUEST_IID
+  # Or when the pipeline was scheduled (nightly
+  # run on main in particular).
+  - if: '$CI_PIPELINE_SOURCE == "schedule"'
+
+include:
+  - local: .gitlab/ci_templates/stackspin_common.yml
+  - local: .gitlab/ci_templates/base_ready.yml
+
+stages:
+  - install-stackspin
+  - base-ready
+  - enable-apps
+  - configure-stackspin
+
+# Stage: install-stackspin
+# ========================
+#
+# Installs flux and stackspin with it
+
+install-stackspin:
+  stage: install-stackspin
+  variables:
+    GIT_STRATEGY: "${INSTALL_GIT_STRATEGY}"
+    SSO_PASSWORD: "$SSO_PASSWORD"
+  script:
+    # For upgrade_test pipelines, we install an older version (latest stable)
+    - !reference [.checkout_branch, script]
+    # Customize env file, remove all comments and empty lines
+    - cp install/.flux.env.example ${CLUSTER_DIR}/.flux.env
+    - sed -i "s/1.2.3.4/$IP_ADDRESS/" ${CLUSTER_DIR}/.flux.env
+    # Set admin user email address
+    - sed -i "s/admin_email=admin@example.org/admin_email=admin@ci.stackspin.net/" ${CLUSTER_DIR}/.flux.env
+    # Delete comments
+    - sed -i "/^\s*#.*$/d; /^\s*$/d" ${CLUSTER_DIR}/.flux.env
+    # Disable outgoing mail
+    - sed -i "s/outgoing_mail_enabled=true/outgoing_mail_enabled=false/" ${CLUSTER_DIR}/.flux.env
+    # Some apps (looking at you Zulip) don't have a way to disable email, so we
+    # set the mail domain like this so we don't contact the Greenhost server
+    # anyway (with known wrong credentials).
+    - sed -i "s/outgoing_mail_smtp_host=.*/outgoing_mail_smtp_host=localhost/" ${CLUSTER_DIR}/.flux.env
+    - sed -i "s/example.org/$FQDN/"  ${CLUSTER_DIR}/.flux.env
+    # Deploy secret/stackspin-cluster-variables
+    - cp install/kustomization.yaml ${CLUSTER_DIR}
+    - |
+      kubectl create namespace flux-system  -o yaml --dry-run=client --save-config | kubectl apply -f -
+      kubectl create namespace cert-manager -o yaml --dry-run=client --save-config | kubectl apply -f -
+      kubectl create namespace stackspin -o yaml --dry-run=client --save-config | kubectl apply -f -
+    - kubectl apply -k ${CLUSTER_DIR}
+    # Add an override so cert-manager uses the SSL.com ClusterIssuer
+    - kubectl apply -f ./install/overrides/ci/stackspin-cert-manager-override.yaml
+    # TODO: this is only necessary for the upgrade pipeline between 2.6 and
+    # 2.7, and can be removed after 2.7 is released.
+    - |
+      if [ ! -e ./install/overrides/ci/stackspin-dashboard-override.yaml ]; then cat > ./install/overrides/ci/stackspin-dashboard-override.yaml <<EOF
+      ---
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        namespace: stackspin
+        name: stackspin-dashboard-override
+      data:
+        values.yaml: |
+      EOF
+      fi
+    # Add an override to populate the dashboard helm test credentials.
+    - |
+      cat >> ./install/overrides/ci/stackspin-dashboard-override.yaml <<EOF
+          tests:
+            credentials:
+              user: 'admin@ci.stackspin.net'
+              password: '$SSO_PASSWORD'
+      EOF
+    - cat ./install/overrides/ci/stackspin-dashboard-override.yaml
+    - kubectl apply -f ./install/overrides/ci/stackspin-dashboard-override.yaml
+    # Install flux and general, non-app specific secrets
+    - ./install/install-stackspin.sh
+  extends:
+    - .general_rules
+    - .child_require_artifacts
+    - .report_artifacts
+  interruptible: true
+
+
+.enable_app_template:
+  stage: enable-apps
+  needs:
+    - install-stackspin
+    - dashboard-kustomization-ready
+  # Make sure only one enable-app job runs against a given cluster at the same
+  # time.
+  # TODO: this actually doesn't work, not sure why
+  resource_group: $CI_ENVIRONMENT_NAME
+  variables:
+    GIT_STRATEGY: "${INSTALL_GIT_STRATEGY}"
+  before_script:
+    # For upgrade_test pipelines, we install an older version (latest stable)
+    - |
+      if [[ "$CI_COMMIT_REF_NAME" != "$STACKSPIN_BRANCH" ]]; then
+        # NOTE: this command will fail if GIT_STRATEGY is not set to "clone"
+        git checkout "$STACKSPIN_BRANCH"
+      fi
+  script:
+    # Add optional override values we need for the CI pipeline only
+    - >
+      [ -f ./install/overrides/ci/stackspin-${RESOURCE}-override.yaml ] &&
+        kubectl apply -f ./install/overrides/ci/stackspin-${RESOURCE}-override.yaml
+    - kubectl exec -n stackspin deploy/dashboard-backend -- flask cli app install ${RESOURCE}
+  interruptible: true
+
+enable-monitoring:
+  variables:
+    RESOURCE: "monitoring"
+  extends:
+    - .enable_app_template
+    - .monitoring_rules
+
+enable-nextcloud:
+  variables:
+    RESOURCE: "nextcloud"
+  before_script:
+    # First do the git checkout, otherwise that will refuse to override the
+    # changes we make to the override file.
+    - !reference [.enable_app_template, before_script]
+    # Add Cypress record key so screenshots/videos of helm test are uploaded to
+    # the Cypress dashboard
+    - |
+      cat >> ./install/overrides/ci/stackspin-nextcloud-override.yaml<< EOF
+          tests:
+            ssoLogin:
+              enabled: true
+              username: 'admin@ci.stackspin.net'
+              password: '$SSO_PASSWORD'
+            cypress:
+              projectId: '$CYPRESS_PROJECT_ID'
+              recordKey: '$CYPRESS_RECORD_KEY'
+              commitInfo:
+                branch: '$CI_COMMIT_REF_NAME'
+                message: |
+                  $CI_COMMIT_TITLE
+                author: |
+                  $CI_COMMIT_AUTHOR
+                sha: '$CI_COMMIT_SHORT_SHA'
+      EOF
+  extends:
+    - .enable_app_template
+    - .nextcloud_rules
+
+enable-velero:
+  variables:
+    RESOURCE: "velero"
+  extends:
+    - .enable_app_template
+    - .velero_rules
+
+enable-wekan:
+  variables:
+    RESOURCE: "wekan"
+  extends:
+    - .enable_app_template
+    - .wekan_rules
+
+enable-wordpress:
+  variables:
+    RESOURCE: "wordpress"
+  extends:
+    - .enable_app_template
+    - .wordpress_rules
+
+enable-zulip:
+  variables:
+    RESOURCE: "zulip"
+  extends:
+    - .enable_app_template
+    - .zulip_rules
+
+enable-hedgedoc:
+  variables:
+    RESOURCE: "hedgedoc"
+  extends:
+    - .enable_app_template
+    - .hedgedoc_rules
+
+
+# Stage: configure-stackspin
+#
+# Configure cluster after basic installation
+# i.e. CI-related config like clusterIssuer
+
+configure-sslcom-issuer:
+  stage: configure-stackspin
+  needs:
+    - cert-manager-kustomization-ready
+    - install-stackspin
+  script:
+    # Install custom ClusterIssuer for SSL.com production certificates
+    - bash ./.gitlab/ci_scripts/install_sslcom_issuer.sh
+  extends:
+    - .general_rules
+  interruptible: true
+
+configure-zerossl-issuer:
+  stage: configure-stackspin
+  needs:
+    - cert-manager-kustomization-ready
+    - install-stackspin
+  script:
+    # Install custom ClusterIssuer for zerossl.com production certificates
+    - bash ./.gitlab/ci_scripts/install_zerossl_issuer.sh
+  extends:
+    - .general_rules
+  interruptible: true
+
+# Set the password for the initial Stackspin admin user to a fixed value, for
+# easy testing and debugging.
+admin-user-password:
+  stage: configure-stackspin
+  needs:
+  - install-stackspin
+  - dashboard-kustomization-ready
+  script:
+  - kubectl exec -n stackspin deploy/dashboard-backend -- flask cli user setpassword admin@ci.stackspin.net $SSO_PASSWORD

.gitlab/ci_pipelines/upgrade_test.yml

+---
+# This pipeline first runs the whole `default.yml` pipeline, but instead of
+# installing the newest version of Stackspin, it will install the latest stable
+# (defined in the INSTALL_STACKSPIN_BRANCH variable).
+#
+# After the default pipeline is finished, the stackspin `GitRepository` object
+# is patched to track the current branch and commit. Then, the apps and
+# cluster-health jobs are run again, to make sure that the upgrade broke
+# nothing.
+
+include:
+  - /.gitlab/ci_pipelines/default.yml
+  - local: .gitlab/ci_templates/base_ready.yml
+
+stages:
+  # First 5 stages are just the Default pipeline with different variables
+  - build
+  - install-cluster
+  - install-stackspin
+  - apps-ready
+  - cluster-health
+  # Upgrade the cluster to the current version
+  - upgrade-stackspin
+  # Run tests again
+  - base-ready
+  - upgraded-apps-ready
+  - upgrade-stackspin-health
+
+# Overrides stackspin branch used for create-vps and install-stackspin jobs
+variables:
+  INSTALL_STACKSPIN_BRANCH: v2
+  INSTALL_GIT_STRATEGY: clone
+  UPGRADE_STACKSPIN_BRANCH: $CI_COMMIT_REF_NAME
+  UPGRADE_STACKSPIN_COMMIT_SHA: $CI_COMMIT_SHA
+  # This variable is used by the `base_ready` template and the `app_tests` child
+  # pipelines after the upgrade has been applied. It is overridden in all jobs
+  # that rely on the INSTALL_STACKSPIN_BRANCH.
+  STACKSPIN_BRANCH: $UPGRADE_STACKSPIN_BRANCH
+  HOSTNAME: "upgrade-test"
+  CLUSTER_DIR: "clusters/upgrade-test"
+  # Increase disk size to 40G for upgrade test, see
+  # https://open.greenhost.net/stackspin/stackspin/-/issues/1271
+  # Outdated as we've grown the default disk size.
+  # DISK_SIZE: "40"
+
+# override environment name for upgrade-test, otherwise it will be `main`
+create-vps:
+  environment:
+    name: upgrade-test
+
+terminate-droplet:
+  environment:
+    name: upgrade-test
+
+
+upgrade-stackspin:
+  stage: upgrade-stackspin
+  # Make sure this has the create-vps artifacts
+  dependencies:
+    - create-vps
+  script:
+    # Change the `stackspin` GitRepo to switch to the new branch.
+    - kubectl patch -n flux-system gitrepository stackspin -p "{\"spec\":{\"ref\":{\"branch\":\"$UPGRADE_STACKSPIN_BRANCH\",\"commit\":\"$UPGRADE_STACKSPIN_COMMIT_SHA\"}}}" --type="merge"
+    # Make sure flux acts on that change right away, instead of waiting for the
+    # next nightly maintenance window to start upgrading.
+    - flux resume source git stackspin
+    # Set new STACKSPIN_COMMIT_SHA variable
+    - sed -i "s/STACKSPIN_COMMIT_SHA=.*/STACKSPIN_COMMIT_SHA=${UPGRADE_STACKSPIN_COMMIT_SHA}/" "${CLUSTER_DIR}/.cluster.env"
+  extends:
+    - .general_rules
+    - .report_artifacts
+  interruptible: true
+
+# These are imported, but we have to override `dependencies` for them in the
+# upgrade scenario, so that gitlab knows which exported environment (dotenv) to
+# use for these jobs. We have two jobs that export the `cluster.env` file:
+# first the create-vps one that first creates the cluster, but then later the
+# upgrade-stackspin one above which changes the `STACKSPIN_COMMIT_SHA`. We need
+# the latter for these jobs, because they should check that the new versions
+# (after the upgrade) of the kustomizations are ready.
+.kustomization_ready:
+  dependencies:
+  - upgrade-stackspin
+
+upgraded-monitoring-app-ready:
+  stage: upgraded-apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: upgrade-stackspin
+    RESOURCE: "monitoring"
+    CYPRESS_TEST: "true"
+    CYPRESS_BASE_URL: "https://grafana.$FQDN"
+  extends:
+    - .monitoring_rules
+    - .trigger_apps_ready_pipeline
+
+# Override the script in the report-ci-image-tag job from
+# .gitlab/ci_pipelines/default.yml to always report current stable (`v2`) as
+# the image tag.
+report-ci-image-tag:
+  script:
+    - echo "CI_CONTAINER_TAG=v2" | tee .ci.env
+
+# Similar to report-ci-image-tag: we override the script to use the `v2`
+# container tag in pre-release pipelines.
+ci-test-image-build:
+  script:
+    - echo "CI_CONTAINER_TAG=v2" | tee .ci.env
+  after_script:
+    - echo "Removed default after_script behavior"
+  environment: null
+
+upgraded-nextcloud-app-ready:
+  stage: upgraded-apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: upgrade-stackspin
+    RESOURCE: "nextcloud"
+  extends:
+    - .nextcloud_rules
+    - .trigger_apps_ready_pipeline
+
+upgraded-velero-app-ready:
+  stage: upgraded-apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: upgrade-stackspin
+    RESOURCE: "velero"
+    # Do not run `cert` and cypress tests, because we don't have them for velero
+    SKIP_CERT: "true"
+  extends:
+    - .velero_rules
+    - .trigger_apps_ready_pipeline
+
+upgraded-wekan-app-ready:
+  stage: upgraded-apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: upgrade-stackspin
+    RESOURCE: "wekan"
+  extends:
+    - .wekan_rules
+    - .trigger_apps_ready_pipeline
+
+upgraded-wordpress-app-ready:
+  stage: upgraded-apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: upgrade-stackspin
+    RESOURCE: "wordpress"
+  extends:
+    - .wordpress_rules
+    - .trigger_apps_ready_pipeline
+
+upgraded-zulip-app-ready:
+  stage: upgraded-apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: upgrade-stackspin
+    RESOURCE: "zulip"
+  extends:
+    - .zulip_rules
+    - .trigger_apps_ready_pipeline
+
+upgraded-hedgedoc-app-ready:
+  stage: upgraded-apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: upgrade-stackspin
+    RESOURCE: "hedgedoc"
+  extends:
+    - .hedgedoc_rules
+    - .trigger_apps_ready_pipeline
+
+upgraded-single-sign-on-app-ready:
+  stage: upgraded-apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: upgrade-stackspin
+    RESOURCE: "single-sign-on"
+    # kustomization-ready has already run as part of the base-ready in
+    # the install-stackspin pipeline
+    SKIP_KUSTOMIZATION_READY: "true"
+  extends:
+    - .general_rules
+    - .trigger_apps_ready_pipeline
+
+upgraded-dashboard-app-ready:
+  stage: upgraded-apps-ready
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    ARTIFACT_JOB: upgrade-stackspin
+    RESOURCE: "dashboard"
+    # kustomization-ready has already run as part of the base-ready in
+    # the install-stackspin pipeline
+    SKIP_KUSTOMIZATION_READY: "true"
+  extends:
+    - .general_rules
+    - .trigger_apps_ready_pipeline
+
+# Stage: upgrade-cluster-health
+# =====================
+#
+# Check if cluster is still healthy after upgrade
+
+upgrade-testinfra:
+  stage: upgrade-stackspin-health
+  extends:
+    - .testinfra
+
+upgrade-prometheus-alerts:
+  stage: upgrade-stackspin-health
+  extends:
+    - .prometheus_alerts

.gitlab/ci_scripts/can_reuse_machine.py

-#!/usr/bin/env python3
-"""
-This script takes 2 positional arguments, HOSTNAME and IP_ADDRESS.
-It polls the Greenhost API for a VPS with the hostname HOSTNAME and checks if
-that machine has the IP address provided in the second argument. If
-everything's OK, the machine can be reused for the current CI pipeline and the
-script exits with 0. If the machine can't be reused the script exits with 1
-"""
-
-import sys
-import greenhost_cloud
-
-HOSTNAME = sys.argv[1]
-IP_ADDRESS = sys.argv[2]
-MACHINES = greenhost_cloud.get_droplets_by_name("^{}$".format(HOSTNAME))
-
-if MACHINES == []:
-    print("No machines found with hostname {}".format(HOSTNAME))
-    sys.exit(1)
-
-for machine in MACHINES:
-    print("Machine status is {}".format(machine['status']))
-    print("Comparing IP address '{}' with IP address '{}'".format(
-        machine['networks']['v4'][0]['ip_address'], IP_ADDRESS))
-    if machine['status'] == 'running' and \
-            machine['networks']['v4'][0]['ip_address'] == IP_ADDRESS:
-        sys.exit(0)
-sys.exit(1)

.gitlab/ci_scripts/create_vps.sh

-#!/usr/bin/env sh
-# Check if cluster directory was available from cache
-set -v
-
-if [ -d clusters/$HOSTNAME/secrets ]
-then
-  ip_address=$(python -m openappstack ${HOSTNAME} info --ip-address)
-  echo "Running python script"
-  # Run a python script that checks if the cached machine still exists. The
-  # script exits with 0 (true) if the machine can be reused
-  if python3 $(dirname "$0")/can_reuse_machine.py $HOSTNAME $ip_address
-  then
-    echo "Will reuse machine with hostname '$HOSTNAME' and ip address '$ip_address'"
-    exit 0
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+mem_requirements() {
+  # Adds all mem requirements from all value configmaps of a given app
+  # This is not perfect but ok for a rough estimate
+  app="$1"
+  if [[ "$app" =~ (single-sign-on|dashboard) ]]; then
+    root_dir='flux2/core/base'
+  else
+    root_dir='flux2/apps'
   fi
+  echo $(($(grep -A9999 'values.yaml' \
+    "$root_dir"/"$app"/*-values-configmap.yaml |
+    grep ' *requests:' -iA2 | grep ' *memory:' | cut -d':' -f2 | numfmt \
+    --from=auto | tr '\n' '+' | sed 's/$/0/')))
+}
+
+# Calculate droplet mem size
+echo "ENABLED_APPS: $ENABLED_APPS"
+total_apps_mem=0
+IFS="," read -ra apps <<<"$ENABLED_APPS"
+for app in "${apps[@]}"; do
+  app_mem=$(mem_requirements "$app")
+  total_apps_mem=$((total_apps_mem + app_mem))
+  echo "Memory request for $app: $(echo "$app_mem" | numfmt --to=iec-i)"
+done
+
+# Total mem utilization for a barebone stackspin cluster without any optional apps
+# 2.8 GiB
+total_core_mem=3006477107
+total_mem=$((total_core_mem + total_apps_mem))
+echo -e "\nMemory requests for core components: $(echo "$total_core_mem" | numfmt --to=iec-i)"
+echo "Memory request for all additional apps: ""$(echo "$total_apps_mem" | numfmt --to=iec-i)"
+echo "Total memory requests: ""$(echo "$total_mem" | numfmt --to=iec-i)"
+
+total_mb=$((total_mem / 1024 / 1024))
+echo "Creating a droplet with $total_mb MB."
+
+# shellcheck disable=SC2039,SC3028
+VPS_HOSTNAME="$HOSTNAME"
+
+if [[ "$CI_COMMIT_REF_NAME" != "$INSTALL_STACKSPIN_BRANCH" ]]; then
+  # NOTE: this command will fail if GIT_STRATEGY is not set to "clone"
+  git checkout "$INSTALL_STACKSPIN_BRANCH"
 fi
+
 # Delete old machine if it still exists
 echo "Deleting old machine"
-python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^${HOSTNAME}$\")"
+python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^$VPS_HOSTNAME$\")"
 echo "Creating new machine"
-python3 -m openappstack $HOSTNAME create \
-  --acme-staging \
-  --create-droplet $DOMAIN \
-  --create-hostname $HOSTNAME \
-  --ssh-key-id $SSH_KEY_ID \
-  --create-domain-records \
-  --subdomain $SUBDOMAIN
+# Uses a custom disk image built from commit
+# 33ccc519e5d239395e94e655a3fbd5cfc3640a60 (around 2.12) on 2024-06-18. See
+# CONTRIBUTING.md#ci-pipeline-image for more info.
+args=(
+  --create-droplet "$DOMAIN"
+  --create-hostname "$VPS_HOSTNAME"
+  --ssh-key-id "$SSH_KEY_ID"
+  --mem "$total_mb"
+  --create-domain-records
+  --subdomain "$SUBDOMAIN"
+  --disk-image-id '-22353'
+  --truncate-subdomain
+)
+
+if [ "${DISK_SIZE:-}" != "" ]; then
+  args+=(--disk-size "$DISK_SIZE")
+fi
+
+python3 -m stackspin "$VPS_HOSTNAME" create "${args[@]}"
+# Disabled for now, see https://open.greenhost.net/stackspin/stackspin/-/issues/1057
+# --docker-mirror-server="${CI_DEPENDENCY_PROXY_SERVER}" \
+# --docker-mirror-endpoint="${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}" \
+# --docker-mirror-username="${CI_RUNNER_PERSONAL_ACCESS_USERNAME}" \
+# --docker-mirror-password="${CI_RUNNER_PERSONAL_ACCESS_TOKEN}"
+
+# Install Kubernetes, firewalle, etc.
+python3 -m stackspin "$VPS_HOSTNAME" install
+
+# Make sure .ci.env variables are not lost
+cat .ci.env >>"$CLUSTER_DIR/.cluster.env"
+
+# Save with which git commit we're installing Stackspin, so we can make sure
+# our Kustomizations are up to date
+echo "STACKSPIN_COMMIT_SHA=$(git rev-parse HEAD)" >>"$CLUSTER_DIR/.cluster.env"

.gitlab/ci_scripts/install_sslcom_issuer.sh

+#!/usr/bin/env bash
+#
+# Waits until cert-manager HelmRelease is ready, and then installs a SSL.com
+# ClusterIssuer with our credentials into the cert-manager namespace
+#
+# Usage:
+#
+#   ./install_sslcom_issuer.sh
+set -euo pipefail
+
+# Create secret with HMAC key
+b64secret=$(echo -n "${SSL_COM_EAB_HMAC_KEY}" | base64 -w0)
+
+# Add SSL.com ClusterIssuer
+kubectl apply -n cert-manager -f - <<EOF
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  namespace: cert-manager
+  name: sslcom-eabsecret
+data:
+  secret: ${b64secret}
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: sslcom-issuer
+spec:
+  acme:
+    # The email address used for signing up with ssl.com
+    email: ${SSL_COM_EMAIL_ADDRESS}
+    # The ACME server URL
+    server: https://acme.ssl.com/sslcom-dv-rsa
+    externalAccountBinding:
+      keyID: ${SSL_COM_EAB_KID}
+      keySecretRef:
+        name: sslcom-eabsecret
+        key: secret
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: sslcom-prod
+    solvers:
+      - http01:
+          ingress:
+            class: nginx
+EOF

.gitlab/ci_scripts/install_zerossl_issuer.sh

+#!/usr/bin/env bash
+#
+# Installs a ClusterIssuer for zerossl.com with our credentials.
+#
+# Usage:
+#
+#   ./install_zerossl_issuer.sh
+set -euo pipefail
+
+# Create secret with HMAC key
+b64secret=$(echo -n "${ZEROSSL_EAB_HMAC_KEY}" | base64 -w0)
+
+# Add SSL.com ClusterIssuer
+kubectl apply -n cert-manager -f - <<EOF
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  namespace: cert-manager
+  name: zerossl-eabsecret
+data:
+  secret: ${b64secret}
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: zerossl-issuer
+spec:
+  acme:
+    # The email address used for signing up with zerossl.com
+    email: ${ZEROSSL_EMAIL_ADDRESS}
+    # The ACME server URL
+    server: "https://acme.zerossl.com/v2/DV90"
+    externalAccountBinding:
+      keyID: ${ZEROSSL_EAB_KID}
+      keySecretRef:
+        name: zerossl-eabsecret
+        key: secret
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: zerossl-prod
+    solvers:
+      - http01:
+          ingress:
+            class: nginx
+EOF

.gitlab/ci_scripts/print_debug_info.sh

+#!/usr/bin/env sh
+
+echo "Env vars:"
+echo
+env | grep -E '^(HOSTNAME|CLUSTER_NAME|FQDN|IP_ADDRESS|CLUSTER_DIR|ANSIBLE_HOST_KEY_CHECKING|KANIKO_BUILD_IMAGENAME|SSH_KEY_ID|SHELL|CI_PROJECT_DIR|STACKSPIN_BRANCH|STACKSPIN_COMMIT_SHA)='
+echo
+echo "Uptime:                    $(uptime)"
+echo "KANIKO build image ref:    ${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}"
+echo
+if [ -f .ci.env ]; then
+  echo "Content of .ci.env:"
+  cat .ci.env
+fi
+if [ -f .cluster.env ]; then
+  echo "Content of .cluster.env:"
+  cat .cluster.env
+fi

.gitlab/ci_scripts/retry_cmd_until_success.sh

+#!/usr/bin/env bash
+#
+# Executes given cms multiple times until it is successful
+#
+# Usage:
+#
+#   ./retry_cmd_until_success.sh [retries] [sleep] [cmd]
+#
+# Example:
+#
+#   ./retry_cmd_until_success.sh 10 20 ping ix.de
+
+retries=$1
+if [[ ! $retries =~ ^-?[0-9]+$ ]]
+then
+  echo "Please specify retries count."
+  exit 1
+fi
+
+shift
+sleep_secs=$1
+if [[ ! $sleep_secs =~ ^-?[0-9]+$ ]]
+then
+  echo "Please specify sleep interval in seconds."
+  exit 1
+fi
+
+shift
+cmd=$*
+if [[ -z "$cmd" ]]
+then
+  echo "Please specify a cmd."
+  exit 1
+fi
+echo "Retrying \"$cmd\" ${retries} times."
+
+i=0
+until eval $cmd
+do
+  echo -e "return code: $?\n"
+  if [[ $i -ge $retries ]]
+  then
+    exit 1
+  fi
+  (( i++ ))
+  sleep $sleep_secs
+  echo "${i}. retry:"
+done

.gitlab/ci_templates/base_ready.yml

+---
+
+# Stage: base-ready
+# ====================
+#
+# Test if base kustomizations are ready, before configuration can get applied
+# that makes use of CRDs, i.e. clusterIssuer
+
+cert-manager-kustomization-ready:
+  variables:
+    RESOURCE: "cert-manager"
+  extends:
+    - .kustomization_ready
+
+dashboard-kustomization-ready:
+  variables:
+    RESOURCE: "dashboard"
+  extends:
+    - .kustomization_ready
+
+letsencrypt-issuer-kustomization-ready:
+  variables:
+    RESOURCE: "letsencrypt-issuer"
+  extends:
+    - .kustomization_ready
+
+local-path-provisioner-kustomization-ready:
+  variables:
+    RESOURCE: "local-path-provisioner"
+  extends:
+    - .kustomization_ready
+
+metallb-kustomization-ready:
+  variables:
+    RESOURCE: "metallb"
+  extends:
+    - .kustomization_ready
+
+namespaces-kustomization-ready:
+  variables:
+    RESOURCE: "namespaces"
+  extends:
+    - .kustomization_ready
+
+nginx-kustomization-ready:
+  variables:
+    RESOURCE: "nginx"
+  extends:
+    - .kustomization_ready
+
+single-sign-on-kustomization-ready:
+  variables:
+    RESOURCE: "single-sign-on"
+  extends:
+    - .kustomization_ready
+
+sources-kustomization-ready:
+  variables:
+    RESOURCE: "sources"
+  extends:
+    - .kustomization_ready
+
+stackspin-kustomization-ready:
+  variables:
+    RESOURCE: "stackspin"
+  extends:
+    - .kustomization_ready
+
+system-upgrade-config-ready:
+  variables:
+    RESOURCE: "system-upgrade-config"
+  extends:
+    - .kustomization_ready
+
+system-upgrade-controller-ready:
+  variables:
+    RESOURCE: "system-upgrade-controller"
+  extends:
+    - .kustomization_ready

.gitlab/ci_templates/dummy_job.yml

+dummy-job:
+  stage: dummy
+  script:
+  - echo "This is a dummy job which does nothing else but triggers a pipeline"
+  - echo "nonetheless, useful for situations where CI rules would not trigger"
+  - echo "a pipeline automatically, i.e. changes to renovate.json"
+  - echo "(see https://open.greenhost.net/stackspin/wordpress-helm/-/merge_requests/127#note_38728)"
+  when: manual

.gitlab/ci_templates/helm_package.yml

+# Include this file if you want to package your helm chart to a helm registry.
+# You'll need to set two variables:
+#
+# 1. CHART_NAME: the name of the helm chart. Should correspond to the name in
+#    Chart.yaml
+# 2. CHART_DIR (optional): the directory where your helm chart is in you
+#    repository. HAS TO END WITH A SLASH if you choose to override it.
+
+variables:
+  CHART_DIR: ""
+
+.chart_release_rules:
+  rules:
+    - changes:
+      - ${CHART_DIR}Chart.yaml
+
+.chart_changes_rules:
+  rules:
+    - changes:
+      - ${CHART_DIR}templates/**/*
+      - ${CHART_DIR}values-local.yaml.example
+      - ${CHART_DIR}values.yaml
+      - ${CHART_DIR}Chart.yaml
+
+lint-helm:
+  stage: lint-helm-chart
+  image:
+    name: alpine/helm:3.17.1
+    entrypoint: ["/bin/sh", "-c"]
+  script:
+    - cd ${CHART_DIR:-"."}
+    - helm dep update
+    - helm lint .
+  artifacts:
+    paths:
+      - '${CHART_DIR}charts/**'
+    expire_in: 1 week
+    # Even if lint fails, upload the charts/ folder as artifact
+    when: always
+  extends:
+    - .chart_changes_rules
+
+package-chart:
+  stage: package-helm-chart
+  image:
+    name: alpine/helm:3.17.1
+    entrypoint: ["/bin/sh", "-c"]
+  script:
+    - cd ${CHART_DIR:-"."}
+    - helm package .
+  artifacts:
+    paths:
+      - ${CHART_DIR}${CHART_NAME}-*
+    expire_in: 1 week
+  extends:
+    - .chart_release_rules
+
+# Push helm chart. Charts on the default branch are pushed to `stable`, others
+# are pushed to the `unstable` channel.
+release-helm:
+  image: "rancher/curlimages-curl:7.73.0"
+  stage: release-helm-chart
+  script:
+    - cd ${CHART_DIR:-"."}
+    - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ] || [ -n "$CI_COMMIT_TAG" ]; then export HELM_CHANNEL='stable'; else export HELM_CHANNEL='unstable'; fi
+    - export CHART_FILE=$(ls ${CHART_NAME}-*.tgz)
+    - curl --fail --request POST --user gitlab-ci-token:$CI_JOB_TOKEN --form "chart=@${CHART_FILE}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/api/${HELM_CHANNEL}/charts"
+    - echo "Chart '${CHART_FILE}' published to helm repository '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/api/${HELM_CHANNEL}/charts'"
+
+  extends:
+    - .chart_release_rules

.gitlab/ci_templates/kaniko.yml

@@ -17,9 +17,11 @@
   image:
     # We need a shell to provide the registry credentials, so we need to use the
     # kaniko debug image (https://github.com/GoogleContainerTools/kaniko#debug-image)
-    name: gcr.io/kaniko-project/executor:debug
+    # https://console.cloud.google.com/gcr/images/kaniko-project/global/executor
+    name: gcr.io/kaniko-project/executor:v1.23.2-debug
     entrypoint: [""]
   script:
+    - mkdir -p /kaniko/.docker/
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
     - if [ -n "${KANIKO_BUILD_IMAGENAME}" ]; then  export IMAGENAME="/${KANIKO_BUILD_IMAGENAME}"; fi
     - /kaniko/executor --cache=true --context ${CI_PROJECT_DIR}/${KANIKO_CONTEXT:-.} --dockerfile ${CI_PROJECT_DIR}/${KANIKO_CONTEXT:-.}/Dockerfile --destination ${CI_REGISTRY_IMAGE}${IMAGENAME}:${CI_COMMIT_REF_SLUG} ${KANIKO_ADDITIONAL_ARGS}

.gitlab/ci_templates/ssh_setup.yml

-.ssh_setup:
-  before_script:
-    - mkdir ~/.ssh
-    - echo -e 'Host *\n  stricthostkeychecking no' > ~/.ssh/config
-    - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null

.gitlab/ci_templates/stackspin_common.yml

+# General rules
+# =============
+#
+# Rules that enable the cluster to be built and are applied to most steps
+# (except for application-specific steps)
+
+.general_rules:
+  rules:
+    - changes:
+      - .gitlab-ci.yml
+      - .gitlab/ci_scripts/*
+      - .gitlab/ci_templates/*
+      - .gitlab/ci_pipelines/*
+      - Dockerfile
+      - ansible/**/*
+      - flux/**/*
+      - flux2/**/*
+      - install/**/*
+      - test/**/*
+      - stackspin/**/*
+      - requirements.txt
+      - VERSION
+      - CHANGELOG.md
+    - if: '$TRIGGER_JOBS =~ /enable-.*/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-/'
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+
+# app rules
+#
+# Define the rules when/if app specific jobs are run.
+# Just add the variable RESOURCE to the job like this:
+#   variables:
+#     RESOURCE: "nextcloud"
+# and import the templates with i.e.
+#   extends: .nextcloud_rules
+# .nextcloud_rules will ensure that the job is only executed:
+# - when files related to the app changed in the repo
+# - A pipeline gets started from the UI and the job name is included in the
+#   CI variable `TRIGGER_JOBS`
+# - A commit is pushed containing the pattern TRIGGER_JOBS=.*<job name>
+#   (i.e. TRIGGER_JOBS=ci-test-image-build,enable-nextcloud)
+#
+# Gitlab CI allows pushing CI vars via `git push` but a bug prevents this when
+# using merge request pipelines (see https://gitlab.com/gitlab-org/gitlab/-/issues/326098)
+
+.monitoring_rules:
+  rules:
+    - changes:
+        - flux2/apps/monitoring/*.yaml
+        - flux2/cluster/optional/monitoring/*.yaml
+        - flux2/core/base/sources/grafana-helmrepository.yaml
+        - flux2/core/base/sources/wikimedia-helmrepository.yaml
+        - flux2/core/base/sources/prometheus-community-helmrepository.yaml
+        - flux2/config/monitoring/*.yaml
+        # Install and test if we change anything about SSO.
+        - flux2/core/base/single-sign-on/*.yaml
+        - install/install-app.sh
+        - install/flux-version-check.sh
+        - test/cypress/e2e/monitoring.cy.js
+        - .gitlab/ci_pipelines/apps_ready.yml
+    - if: '$TRIGGER_JOBS =~ /enable-monitoring/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-monitoring/'
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+
+.nextcloud_rules:
+  rules:
+    - changes:
+        - flux2/apps/$RESOURCE/*.yaml
+        - flux2/cluster/optional/$RESOURCE/*.yaml
+        - flux2/core/base/sources/nextcloud-helmrepository.yaml
+        # Install and test if we change anything about SSO.
+        - flux2/core/base/single-sign-on/*.yaml
+        - install/install-app.sh
+        - install/flux-version-check.sh
+        - .gitlab/ci_pipelines/apps_ready.yml
+    - if: '$TRIGGER_JOBS =~ /enable-nextcloud/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-nextcloud/'
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+
+.single_sign_on_rules:
+  rules:
+    - changes:
+        - flux2/core/base/$RESOURCE/*.yaml
+        - flux2/infrastructure/sources/single-sign-on-helmrepository.yaml
+        - install/install-stackspin.sh
+        - .gitlab/ci_pipelines/apps_ready.yml
+    - if: '$TRIGGER_JOBS =~ /enable-single-sign-on/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-single-sign-on/'
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+
+.velero_rules:
+  rules:
+    - changes:
+        - flux2/apps/$RESOURCE/*.yaml
+        - flux2/cluster/optional/$RESOURCE/*.yaml
+        - flux2/core/base/sources/vmware-tanzu-helmrepository.yaml
+        - install/install-app.sh
+        - install/flux-version-check.sh
+        - .gitlab/ci_pipelines/apps_ready.yml
+    - if: '$TRIGGER_JOBS =~ /enable-velero/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-velero/'
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+
+.wekan_rules:
+  rules:
+    - changes:
+        - flux2/apps/$RESOURCE/*.yaml
+        - flux2/cluster/optional/$RESOURCE/*.yaml
+        - flux2/core/base/sources/wekan-helmrepository.yaml
+        # Install and test if we change anything about SSO.
+        - flux2/core/base/single-sign-on/*.yaml
+        - install/install-app.sh
+        - install/flux-version-check.sh
+        - test/cypress/e2e/wekan.cy.js
+        - .gitlab/ci_pipelines/apps_ready.yml
+    - if: '$TRIGGER_JOBS =~ /enable-wekan/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wekan/'
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+
+.wordpress_rules:
+  rules:
+    - changes:
+        - flux2/apps/$RESOURCE/*.yaml
+        - flux2/cluster/optional/$RESOURCE/*.yaml
+        - flux2/core/base/sources/wordpress-helmrepository.yaml
+        # Install and test if we change anything about SSO.
+        - flux2/core/base/single-sign-on/*.yaml
+        - install/install-app.sh
+        - install/flux-version-check.sh
+        - test/cypress/e2e/wordpress.cy.js
+        - .gitlab/ci_pipelines/apps_ready.yml
+    - if: '$TRIGGER_JOBS =~ /enable-wordpress/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wordpress/'
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+
+.zulip_rules:
+  rules:
+    - changes:
+        - flux2/apps/$RESOURCE/*.yaml
+        - flux2/cluster/optional/$RESOURCE/*.yaml
+        - flux2/core/base/sources/zulip-helmrepository.yaml
+        # Install and test if we change anything about SSO.
+        - flux2/core/base/single-sign-on/*.yaml
+        - install/install-app.sh
+        - install/flux-version-check.sh
+        - test/cypress/e2e/zulip.cy.js
+        - .gitlab/ci_pipelines/apps_ready.yml
+    - if: '$TRIGGER_JOBS =~ /enable-zulip/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-zulip/'
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+
+.hedgedoc_rules:
+  rules:
+    - changes:
+        - flux2/apps/$RESOURCE/*.yaml
+        - flux2/cluster/optional/$RESOURCE/*.yaml
+        - flux2/core/base/sources/hedgedoc-helmrepository.yaml
+        # Install and test if we change anything about SSO.
+        - flux2/core/base/single-sign-on/*.yaml
+        - install/install-app.sh
+        - install/flux-version-check.sh
+        - test/cypress/e2e/hedgedoc.cy.js
+        - .gitlab/ci_pipelines/apps_ready.yml
+    - if: '$TRIGGER_JOBS =~ /enable-hedgedoc/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-hedgedoc/'
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+
+# Common variables
+# ================
+#
+# Note on variables: some variables are defined in the .cluster.env file
+# generated by the `create-vps` job.
+variables:
+  SSH_KEY_ID: "411"
+  ANSIBLE_HOST_KEY_CHECKING: "False"
+  KANIKO_BUILD_IMAGENAME: "stackspin-ci"
+  CLUSTER_DIR: "clusters/${CI_COMMIT_REF_SLUG}"
+
+default:
+  image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}"
+  before_script:
+    - sh .gitlab/ci_scripts/print_debug_info.sh
+
+# Common job definitions
+# ======================
+#
+# Re-usable job definitions
+
+.kustomization_ready:
+  stage: base-ready
+  script:
+    - cd test/
+    - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/kube_config_cluster.yml"
+    - pytest -v -s -m 'kustomizations' --resource="$RESOURCE" --desired-revision "${STACKSPIN_BRANCH}@sha1:${STACKSPIN_COMMIT_SHA}" --reruns 120 --reruns-delay 20
+  extends:
+    - .general_rules
+  interruptible: true
+
+# The dotenv report requires us to report the artifacts in every job that is
+# required with a `needs:` from another job.
+.report_artifacts:
+  artifacts:
+    paths:
+      - clusters
+    expire_in: 1 month
+    when: always
+    reports:
+      dotenv:
+        $CLUSTER_DIR/.cluster.env
+
+.ssh_setup:
+  before_script:
+    - bash .gitlab/ci_scripts/print_debug_info.sh
+    - mkdir ~/.ssh
+    - echo -e 'Host *\n  stricthostkeychecking no' > ~/.ssh/config
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
+
+# Get create-vps artifacts in a child pipeline. Requires you to set a
+# PARENT_PIPELINE_ID from the parent pipeline
+.child_require_artifacts:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: $ARTIFACT_JOB
+
+.require_artifacts:
+  dependencies:
+    - create-vps
+
+.trigger_apps_ready_pipeline:
+  trigger:
+    include:
+      - local: .gitlab/ci_pipelines/apps_ready.yml
+    strategy: depend
+
+.testinfra:
+  script:
+    - cd ansible/
+    - pytest -v -s -m 'testinfra' --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
+  extends:
+    - .ssh_setup
+    - .general_rules
+    - .require_artifacts
+  interruptible: true
+
+.prometheus_alerts:
+  variables:
+    # RESOURCE var is used in job specific rules (i.e. ..monitoring_rules)
+    RESOURCE: "kube-prometheus-stack"
+  script:
+    - export BASIC_AUTH_PW=$(python3 -m stackspin $HOSTNAME secrets | grep stackspin-prometheus-basic-auth | awk '{ print $4 }')
+    - cd test/
+    - bash ../.gitlab/ci_scripts/retry_cmd_until_success.sh 30 10 pytest -s -m prometheus
+  extends:
+    - .monitoring_rules
+    - .require_artifacts
+  interruptible: true
+
+# Checkout older stackspin branch
+# used i.e. in upgrade pipelines where an older version branch of Stackspin
+# (defined by the STACKSPIN_BRANCH env var)
+# is installed and tested before an upgrade to the current version is performed
+.checkout_branch: &checkout_branch
+  script:
+    - |
+      if [[ "$CI_COMMIT_REF_NAME" != "$STACKSPIN_BRANCH" ]]; then
+        # NOTE: this command will fail if GIT_STRATEGY is not set to "clone"
+        git checkout "$STACKSPIN_BRANCH"
+      fi

.gitlab/commit_template.txt

+
+# Subject above, 50 chars max. 50 chars end here ↑
+# Description below:
+
+# Additional CI directives
+#
+# Trigger upgrade-test pipeline:
+# UPGRADE_TEST
+#
+# Include apps to install even there's no code change that would
+# trigger an atomated installation, i.e.:
+# TRIGGER_JOBS=enable-nextcloud
+#
+# or trigger all jobs:
+# TRIGGER_JOBS=enable-monitoring,enable-nextcloud,enable-velero,enable-wekan,enable-wordpress,enable-zulip
+#
+# Reference issue number with one of:
+#
+# Fixes: #
+# Closes: #
+# Related: #
+#
+# Install this commit template from git root with: `git config commit.template .gitlab/commit_template.txt`