.gitlab-ci.yml

---
include:
  - /.gitlab/ci_templates/kaniko.yml
  - /.gitlab/ci_templates/ssh_setup.yml
  - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'

# Global templates and YAML anchors
# =================================
#
# Used in various stages/job definitions

# We don't use a `before_script` definition here because `extend` doesn't merge
# `before_script` but rather overwrites it.
# So we rather use [yaml anchors](https://docs.gitlab.com/ce/ci/yaml/README.html#anchors)
# here. Unfortunatly, anchors can't get included from files so we need to
# define them here.
.debug_information: &debug_information
  - |
    echo "Env vars:"
    echo
    echo "HOSTNAME:                  $HOSTNAME"
    echo "IP_ADDRESS:                $IP_ADDRESS"
    echo "Uptime:                    $(uptime)"
    echo "ANSIBLE_HOST_KEY_CHECKING: $ANSIBLE_HOST_KEY_CHECKING"
    echo "KANIKO_BUILD_IMAGENAME:    $KANIKO_BUILD_IMAGENAME"
    echo "KANIKO build image ref:    ${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}"
    echo "SSH_KEY_ID:                $SSH_KEY_ID"
    echo "SHELL                      $SHELL"
    echo "CI_PROJECT_DIR:            $CI_PROJECT_DIR"

# The dotenv report requires us to report the artifacts in every job that is
# required with a `needs:` from another job.
.report_artifacts:
  artifacts:
    paths:
      - clusters
    expire_in: 1 month
    when: always
    reports:
      dotenv:
        $CLUSTER_DIR/.cluster.env

# Rules that enable the cluster to be built and are applied to most steps
# (except for application-specific steps)
.general_rules:
  rules:
    - changes:
      - .gitlab-ci.yml
      - .gitlab/ci_scripts/*
      - Dockerfile
      - ansible/**/*
      - flux/**/*
      - flux2/**/*
      - install/**/*
      - test/**/*
      - openappstack/**/*
      - requirements.txt
    - if: '$TRIGGER_JOBS =~ /enable-.*/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-/'
    - if: '$CI_COMMIT_BRANCH == "master"'

# app rules
#
# Define the rules when/if app specific jobs are run.
# Just add the variable RESOURCE to the job like this:
#   variables:
#     RESOURCE: "eventrouter"
# and import the templates with i.e.
#   extends: .eventrouter_rules
# .eventrouter_rules will ensure that the job is only executed:
# - when files related to the app changed in the repo
# - A pipeline gets started from the UI and the job name is included in the
#   CI variable `TRIGGER_JOBS`
# - A commit is pushed containing the pattern TRIGGER_JOBS=.*<job name>
#   (i.e. TRIGGER_JOBS=ci-test-image-build,enable-nextcloud)
#
# Gitlab CI allows pushing CI vars via `git push` but a bug prevents this when
# using merge request pipelines (see https://gitlab.com/gitlab-org/gitlab/-/issues/326098)

.monitoring_rules:
  rules:
    - changes:
        - flux2/apps/monitoring/*.yaml
        - flux2/cluster/optional/monitoring/*.yaml
        - flux2/infrastructure/sources/grafana.yaml
        - flux2/infrastructure/sources/wikimedia.yaml
        - flux2/infrastructure/sources/prometheus-community.yaml
        - install/install-app.sh
        - test/taiko/*
    - if: '$TRIGGER_JOBS =~ /enable-monitoring/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-monitoring/'
    - if: '$CI_COMMIT_BRANCH == "master"'

.eventrouter_rules:
  extends:
    - .monitoring_rules

.loki_rules:
  extends:
    - .monitoring_rules

.promtail_rules:
  extends:
    - .monitoring_rules

.kube_prometheus_stack_rules:
  extends:
    - .monitoring_rules

.nextcloud_rules:
  rules:
    - changes:
        - flux2/apps/$RESOURCE/*.yaml
        - flux2/cluster/optional/$RESOURCE/*.yaml
        - flux2/infrastructure/sources/nextcloud.yaml
        - install/install-app.sh
        - test/taiko/*
    - if: '$TRIGGER_JOBS =~ /enable-nextcloud/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-nextcloud/'
    - if: '$CI_COMMIT_BRANCH == "master"'

.cert_manager_rules:
  extends:
    - .general_rules

.local_path_provisioner_rules:
  extends:
    - .general_rules

.rocketchat_rules:
  rules:
    - changes:
        - flux2/apps/$RESOURCE/*.yaml
        - flux2/cluster/optional/$RESOURCE/*.yaml
        - install/install-app.sh
        - test/taiko/*
    - if: '$TRIGGER_JOBS =~ /enable-rocketchat/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-rocketchat/'
    - if: '$CI_COMMIT_BRANCH == "master"'

.single_sign_on_rules:
  rules:
    - changes:
        - flux2/core/base/$RESOURCE/*.yaml
        - flux2/infrastructure/sources/single-sign-on.yaml
        - install/install-openappstack.sh
        - test/taiko/*
    - if: '$TRIGGER_JOBS =~ /enable-single-sign-on/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-single-sign-on/'
    - if: '$CI_COMMIT_BRANCH == "master"'

.wekan_rules:
  rules:
    - changes:
        - flux2/apps/$RESOURCE/*.yaml
        - flux2/cluster/optional/$RESOURCE/*.yaml
        - flux2/infrastructure/sources/wekan.yaml
        - install/install-app.sh
        - test/taiko/*
    - if: '$TRIGGER_JOBS =~ /enable-wekan/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wekan/'
    - if: '$CI_COMMIT_BRANCH == "master"'

.wordpress_rules:
  rules:
    - changes:
        - flux2/apps/$RESOURCE/*.yaml
        - flux2/cluster/optional/$RESOURCE/*.yaml
        - flux2/infrastructure/sources/wordpress.yaml
        - install/install-app.sh
        - test/taiko/*
    - if: '$TRIGGER_JOBS =~ /enable-wordpress/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wordpress/'
    - if: '$CI_COMMIT_BRANCH == "master"'


# Global declarations
# ===================

# https://docs.gitlab.com/ee/ci/yaml/README.html#workflowrules-templates

stages:
  - build
  - create-vps
  - setup-cluster
  - kustomization
  - install-apps
  - apps-kustomizations-ready
  - certs
  - health-test
  - integration-test

variables:
  SSH_KEY_ID: "411"
  HOSTNAME: "${CI_COMMIT_REF_SLUG}"
  ANSIBLE_HOST_KEY_CHECKING: "False"
  KANIKO_BUILD_IMAGENAME: "openappstack-ci"
  CLUSTER_DIR: "clusters/${CI_COMMIT_REF_SLUG}"

default:
  image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}"


# Stage: build
# ============
#
# Builds CI test container image
# There are 2 moments in which we (re)build the container image. If some files are
# changed, or when the job is triggered with TRIGGER_JOBS.

ci-test-image-build:
  stage: build
  before_script:
    - *debug_information
  after_script:
    - |
      echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env
  artifacts:
    paths:
      - .ci.env
    expire_in: 1 month
    when: always
    reports:
      dotenv:
        .ci.env
  environment:
    name: image/$CI_COMMIT_REF_SLUG
    url: https://open.greenhost.net:4567/openappstack/openappstack/openappstack-ci:${CI_COMMIT_REF_SLUG}
    on_stop: delete-image
    auto_stop_in: 3 weeks
  rules:
    # Automatically rebuild the container image if this file, the Dockerfile,
    # the installed requirements or the kaniko template change
    - changes:
        - Dockerfile
        - requirements.txt
        - .gitlab/ci_templates/kaniko.yml
    # Also rebuild when the CI variable contain this jobs name
    # or commit msg contains /TRIGGER_JOBS=.*ci-test-image-build/
    - if: '$TRIGGER_JOBS =~ /ci-test-image-build/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/'
  extends:
    - .kaniko_build
  interruptible: true

report-ci-image-tag:
  stage: build
  image: "curlimages/curl"
  before_script:
    - *debug_information
  script:
    - |
      TAG_INFORMATION=$(curl https://open.greenhost.net/api/v4/projects/openappstack%2Fopenappstack/registry/repositories/2/tags/${CI_COMMIT_REF_SLUG});
      echo "Tag information: ${TAG_INFORMATION}"
      if [ "$TAG_INFORMATION" == '{"message":"404 Tag Not Found"}' ]; then
        echo "CI_CONTAINER_TAG=master" > .ci.env
      else
        echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env
      fi
  artifacts:
    paths:
      - .ci.env
    expire_in: 1 month
    when: always
    reports:
      dotenv:
        .ci.env
  rules:
    # Make sure this job does not run if ci-test-image-build runs
    - changes:
        - Dockerfile
        - requirements.txt
        - .gitlab/ci_templates/kaniko.yml
      when: never  # Never run on file changes that trigger ci-test-image-build
    - if: '$TRIGGER_JOBS =~ /ci-test-image-build/'
      when: never  # Never run when ci-test-image is triggered manually
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/'
      when: never  # Never run when ci-test-image is triggered manually
    - when: always
  interruptible: true


# Stage: create-vps
# =================
#
# Creates the vps for the pipeline

create-vps:
  stage: create-vps
  variables:
    SUBDOMAIN: "${CI_COMMIT_REF_SLUG}.ci"
    DOMAIN: "openappstack.net"
  script:
    - *debug_information
    # Creates a VPS based on a custom CI image for which the ansible playbook
    # has already run. See CONTRIBUTING.md#ci-pipeline-image for more info
    - bash .gitlab/ci_scripts/create_vps.sh
    # Make sure .ci.env variables are not lost
    - cat .ci.env >> ${CLUSTER_DIR}/.cluster.env
  extends:
    - .ssh_setup
    - .report_artifacts
    - .general_rules
  environment:
    name: $CI_COMMIT_REF_SLUG
    url: https://$FQDN
    on_stop: terminate-droplet
    auto_stop_in: 1 week
  interruptible: true


# Stage: setup-cluster
# ====================
#
# Installs OAS

test-dns:
  stage: setup-cluster
  needs:
    - job: create-vps
  script:
    - *debug_information
    - cd ansible/
    - pytest -v -s -m 'dns' --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
  extends:
    - .general_rules
  interruptible: true

setup-openappstack:
  stage: setup-cluster
  script:
    - *debug_information
    # Copy inventory files to ansible folder for use in install-apps step
    - chmod 700 ansible
    - cp ${CLUSTER_DIR}/inventory.yml ansible/
    # Set up cluster
    - python3 -m openappstack $HOSTNAME install
    # Customize env file, remove all comments and empty lines
    - sed "s/1.2.3.4/$IP_ADDRESS/; s/example.org/$FQDN/; s/acme_staging=false/acme_staging=true/; s/acme-v02/acme-staging-v02/; /^\s*#.*$/d; /^\s*$/d" install/.flux.env.example >> ${CLUSTER_DIR}/.flux.env
    # Deploy secret/oas-cluster-variables
    - cp install/kustomization.yaml ${CLUSTER_DIR}
    - kubectl create namespace flux-system
    - kubectl apply -k ${CLUSTER_DIR}
    - bash ./install/install-openappstack.sh
  extends:
    - .ssh_setup
    - .report_artifacts
    - .general_rules
  interruptible: true


# Stage: kustomization
# ====================
#
# Tests if all kustomizations are ready
.kustomization-ready:
  stage: kustomization
  needs:
    - job: setup-openappstack
    - job: test-dns
  script:
    - *debug_information
    - cd ansible/
    - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/kube_config_cluster.yml"
    - pytest -v -s -m 'kustomizations' --resource="$RESOURCE" --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 20
  extends:
    - .ssh_setup
    - .general_rules
  interruptible: true

core-kustomizations-ready:
  variables:
    RESOURCE: "core"
  extends:
    - .kustomization-ready

infrastructure-kustomizations-ready:
  variables:
    RESOURCE: "infrastructure"
  extends:
    - .kustomization-ready

openappstack-kustomizations-ready:
  variables:
    RESOURCE: "openappstack"
  extends:
    - .kustomization-ready

# Stage: install-apps
# ==================
#
# Checks if application needs to get installed

.enable_app_template:
  stage: install-apps
  script:
    - *debug_information
    # Add optional override values we need for the CI pipeline only
    - '[ -f ./install/overrides/oas-${RESOURCE}-override.yaml ] && kubectl apply -n oas-apps -f ./install/overrides/oas-${RESOURCE}-override.yaml'
    - bash ./install/install-app.sh ${RESOURCE}
  extends:
    - .ssh_setup
  interruptible: true

enable-monitoring:
  variables:
    RESOURCE: "monitoring"
  extends:
    - .enable_app_template
    - .monitoring_rules

enable-nextcloud:
  variables:
    RESOURCE: "nextcloud"
  extends:
    - .enable_app_template
    - .nextcloud_rules

enable-rocketchat:
  variables:
    RESOURCE: "rocketchat"
  extends:
    - .enable_app_template
    - .rocketchat_rules

enable-wekan:
  variables:
    RESOURCE: "wekan"
  extends:
    - .enable_app_template
    - .wekan_rules

enable-wordpress:
  variables:
    RESOURCE: "wordpress"
  extends:
    - .enable_app_template
    - .wordpress_rules

# Stage: apps-kustomizations-ready
# ================
#
# Check that the kustomizations of all installed apps are ready.

.app-kustomization-ready:
  stage: apps-kustomizations-ready
  extends:
    - .kustomization-ready

monitoring-kustomization-ready:
  needs:
    - job: setup-openappstack
    - job: enable-monitoring
  variables:
    RESOURCE: "monitoring"
  extends:
    - .app-kustomization-ready
    - .monitoring_rules

nextcloud-kustomization-ready:
  needs:
    - job: setup-openappstack
    - job: enable-nextcloud
  variables:
    RESOURCE: "nextcloud"
  extends:
    - .app-kustomization-ready
    - .nextcloud_rules

rocketchat-kustomization-ready:
  needs:
    - job: setup-openappstack
    - job: enable-rocketchat
  variables:
    RESOURCE: "rocketchat"
  extends:
    - .app-kustomization-ready
    - .rocketchat_rules

wekan-kustomization-ready:
  needs:
    - job: setup-openappstack
    - job: enable-wekan
  variables:
    RESOURCE: "wekan"
  extends:
    - .app-kustomization-ready
    - .wekan_rules

wordpress-kustomization-ready:
  needs:
    - job: setup-openappstack
    - job: enable-wordpress
  variables:
    RESOURCE: "wordpress"
  extends:
    - .app-kustomization-ready
    - .wordpress_rules

# Stage: certs
# ================
#
# Test each app for proper certs

.apps-cert:
  stage: certs
  script:
    - *debug_information
    - cd ansible/
    - pytest -v -s -m 'certs' --resource="$RESOURCE" --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10
  extends:
    - .ssh_setup
  interruptible: true

nextcloud-cert:
  variables:
    RESOURCE: "nextcloud"
  needs:
    - job: enable-nextcloud
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .nextcloud_rules

kube-prometheus-stack-cert:
  variables:
    RESOURCE: "kube-prometheus-stack"
  needs:
    - job: enable-monitoring
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .kube_prometheus_stack_rules

rocketchat-cert:
  variables:
    RESOURCE: "rocketchat"
  needs:
    - job: enable-rocketchat
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .rocketchat_rules

single-sign-on-cert:
  variables:
    RESOURCE: "single-sign-on"
  needs:
    - job: core-kustomizations-ready
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .single_sign_on_rules

wekan-cert:
  variables:
    RESOURCE: "wekan"
  needs:
    - job: enable-wekan
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .wekan_rules

wordpress-cert:
  variables:
    RESOURCE: "wordpress"
  needs:
    - job: enable-wordpress
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .wordpress_rules


# Stage: health-test
# ==================
#
# General cluster health checks

testinfra:
  stage: health-test
  script:
    - *debug_information
    - cd ansible/
    - pytest -v -s -m 'testinfra' --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
  extends:
    - .ssh_setup
    - .general_rules
  interruptible: true

kube-prometheus-stack-alerts:
  stage: health-test
  variables:
    # RESOURCE var is used in job specific rules (i.e. .kube_prometheus_stack_rules)
    RESOURCE: "kube-prometheus-stack"
    # Enforce python requests using the system cert store, where LE staging
    # cacert is added
    REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt"
  script:
    - *debug_information
    - export BASIC_AUTH_PW=$(python3 -m openappstack $HOSTNAME secrets | grep oas-prometheus-basic-auth | cut -d'=' -f2)
    - cd test/
    - bash ../.gitlab/ci_scripts/retry_cmd_until_success.sh 30 10 pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=../${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
  extends:
    - .ssh_setup
    - .kube_prometheus_stack_rules
  interruptible: true


# Stage: integration-test
# =======================
#
# Runs integration tests for most apps using taiko

.taiko:
  stage: integration-test
  before_script:
    - *debug_information
  script:
    # Retry taiko tests 60 times until they succeed,
    # with a sleep interval of 10s in between tests
    - bash ./.gitlab/ci_scripts/retry_cmd_until_success.sh 60 10 unbuffer python3 -m openappstack $HOSTNAME test --apps $RESOURCE | ts -i | ts
  artifacts:
    paths:
      - test/taiko/Screenshot*
    expire_in: 1 month
    when: on_failure
  interruptible: true

dashboard-taiko:
  variables:
    RESOURCE: "dashboard"
  needs:
    - job: setup-openappstack
    - job: core-kustomizations-ready
  extends:
    - .taiko
    - .general_rules

grafana-taiko:
  variables:
    RESOURCE: "grafana"
  needs:
    - job: kube-prometheus-stack-cert
    - job: setup-openappstack
    - job: monitoring-kustomization-ready
  extends:
    - .taiko
    - .kube_prometheus_stack_rules

nextcloud-taiko:
  variables:
    RESOURCE: "nextcloud"
  needs:
    - job: nextcloud-cert
    - job: setup-openappstack
    - job: nextcloud-kustomization-ready
  extends:
    - .taiko
    - .nextcloud_rules

rocketchat-taiko:
  variables:
    RESOURCE: "rocketchat"
  needs:
    - job: rocketchat-cert
    - job: setup-openappstack
    - job: rocketchat-kustomization-ready
  extends:
    - .taiko
    - .rocketchat_rules

wekan-taiko:
  variables:
    RESOURCE: "wekan"
  needs:
    - job: wekan-cert
    - job: setup-openappstack
    - job: wekan-kustomization-ready
  extends:
    - .taiko
    - .wekan_rules

wordpress-taiko:
  variables:
    RESOURCE: "wordpress"
  needs:
    - job: wordpress-cert
    - job: setup-openappstack
    - job: wordpress-kustomization-ready
  extends:
    - .taiko
    - .wordpress_rules


# Etc
# ===


# Terminates a droplet once the branch for it is deleted
terminate-droplet:
  # Stage has to be the same as the step that created the VPS
  # https://docs.gitlab.com/ee/ci/environments.html#automatically-stopping-an-environment
  stage: create-vps
  # Gets triggered by on_stop of create-vps job
  when: manual
  variables:
    GIT_STRATEGY: none
  script:
    - *debug_information
    - python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^${CI_COMMIT_REF_SLUG}\")"
  environment:
    name: $CI_COMMIT_REF_SLUG
    action: stop

# Deletes a container image once a branch is deleted.
# Careful! When you run this step manually, you might have to trigger container
# image re-build as well
delete-image:
  stage: build
  when: manual
  variables:
    GIT_STRATEGY: none
  script:
    - *debug_information
    - "curl --request DELETE --header \"PRIVATE-TOKEN: ${CLEANER_TOKEN}\" https://open.greenhost.net/api/v4/projects/openappstack%2Fopenappstack/registry/repositories/2/tags/${CI_COMMIT_REF_SLUG}"
  environment:
    name: image/$CI_COMMIT_REF_SLUG
    action: stop