---
include:
  - /.gitlab/ci_templates/kaniko.yml
  - /.gitlab/ci_templates/ssh_setup.yml
  - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'

# Global templates and YAML anchors
# =================================
#
# Used in various stages/job definitions

# We don't use a `before_script` definition here because `extend` doesn't merge
# `before_script` but rather overwrites it.
# So we rather use [yaml anchors](https://docs.gitlab.com/ce/ci/yaml/README.html#anchors)
# here. Unfortunatly, anchors can't get included from files so we need to
# define them here.
.debug_information: &debug_information
  - |
    echo "Env vars:"
    echo
    echo "HOSTNAME:                  $HOSTNAME"
    echo "IP_ADDRESS:                $IP_ADDRESS"
    echo "Uptime:                    $(uptime)"
    echo "CLUSTER_DIR:               $CLUSTER_DIR"
    echo "ANSIBLE_HOST_KEY_CHECKING: $ANSIBLE_HOST_KEY_CHECKING"
    echo "KANIKO_BUILD_IMAGENAME:    $KANIKO_BUILD_IMAGENAME"
    echo "KANIKO build image ref:    ${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}"
    echo "SSH_KEY_ID:                $SSH_KEY_ID"
    echo
    [ -d $CLUSTER_DIR ] && find $CLUSTER_DIR || echo "directory ${CLUSTER_DIR} not found"
    echo
    echo

# The dotenv report requires us to report the artifacts in every job that is
# required with a `needs:` from another job.
.report_artifacts:
  artifacts:
    paths:
      - clusters
      - ./enabled_apps/$APP
    expire_in: 1 month
    when: always
    reports:
      dotenv:
        $CLUSTER_DIR/.cluster.env

# Rules that enable the cluster to be built and are applied to most steps
# (except for application-specific steps)
.general_rules:
  rules:
    - changes:
        - .gitlab-ci.yml
        - .gitlab/ci_scripts/*
        - Dockerfile
        - ansible/**/*
        - flux/**/*
        - test/**/*
        - openappstack/**/*
        - requirements.txt

# app rules
#
# Define the rules when/if app specific jobs are run.
# Just add the variable APP to the job like this:
#   variables:
#     APP: "eventrouter"
# and import the templates with i.e.
#   extends: .eventrouter_rules
# .eventrouter_rules will ensure that the job is only executed:
# - when files related to the app changed in the repo
# - A pipeline gets started from the UI and the job name is included in the
#   CI variable `TRIGGER_JOBS`
# - A commit is pushed containing the pattern TRIGGER_JOBS=.*<job name>
#   (i.e. TRIGGER_JOBS=ci-test-image-build,enable-nextcloud)
#
# Gitlab CI allows pushing CI vars via `git push` but a bug prevents this when
# using merge request pipelines (see https://gitlab.com/gitlab-org/gitlab/-/issues/326098)
.eventrouter_rules:
  extends:
    - .general_rules

.loki_rules:
  extends:
    - .general_rules

.promtail_rules:
  extends:
    - .general_rules

.nextcloud_rules:
  rules:
    - changes:
        - flux/**/$APP*.yaml
        - ansible/roles/apps/templates/settings/$APP.yaml
        - ansible/roles/apps/tasks/$APP.yaml
        - test/behave/features/$APP.feature
    - if: '$TRIGGER_JOBS =~ /enable-nextcloud/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-nextcloud/'
    - if: '$CI_COMMIT_BRANCH == "master"'

.prometheus_stack_rules:
  extends:
    - .general_rules

.cert_manager_rules:
  extends:
    - .general_rules

.local_path_provisioner_rules:
  extends:
    - .general_rules

.rocketchat_rules:
  rules:
    - changes:
        - flux/**/$APP*.yaml
        - ansible/roles/apps/templates/settings/$APP.yaml
        - ansible/roles/apps/tasks/$APP.yaml
        - test/behave/features/$APP.feature
    - if: '$TRIGGER_JOBS =~ /enable-rocketchat/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-rocketchat/'
    - if: '$CI_COMMIT_BRANCH == "master"'

.single_sign_on_rules:
  rules:
    - changes:
        - flux/**/$APP*.yaml
        - ansible/roles/apps/templates/settings/$APP.yaml
        - ansible/roles/apps/tasks/$APP.yaml
    - if: '$TRIGGER_JOBS =~ /enable-single-sign-on/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-single-sign-on/'
    - if: '$CI_COMMIT_BRANCH == "master"'

.wordpress_rules:
  rules:
    - changes:
        - flux/**/$APP*.yaml
        - ansible/roles/apps/templates/settings/$APP.yaml
        - ansible/roles/apps/tasks/$APP.yaml
        - test/behave/features/$APP.feature
    - if: '$TRIGGER_JOBS =~ /enable-wordpress/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wordpress/'
    - if: '$CI_COMMIT_BRANCH == "master"'


# Global declarations
# ===================

# https://docs.gitlab.com/ee/ci/yaml/README.html#workflowrules-templates

stages:
  - build
  - create-vps
  - enable-apps
  - setup-cluster
  - helm-release
  - apps-ready
  - certs
  - health-test
  - integration-test

variables:
  SSH_KEY_ID: "411"
  HOSTNAME: "${CI_COMMIT_REF_SLUG}"
  # Repeated values, because we're not allowed to use a variable in a variable
  SUBDOMAIN: "${CI_COMMIT_REF_SLUG}.ci"
  DOMAIN: "openappstack.net"
  ANSIBLE_HOST_KEY_CHECKING: "False"
  KANIKO_BUILD_IMAGENAME: "openappstack-ci"
  CLUSTER_DIR: "/builds/openappstack/openappstack/clusters/${CI_COMMIT_REF_SLUG}"

default:
  image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}"


# Stage: build
# ============
#
# Builds CI test container image
# There are 2 moments in which we (re)build the container image. If some files are
# changed, or when the job is triggered with TRIGGER_JOBS.

ci-test-image-build:
  stage: build
  before_script:
    - *debug_information
  after_script:
    - |
      echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env
  artifacts:
    paths:
      - .ci.env
    expire_in: 1 month
    when: always
    reports:
      dotenv:
        .ci.env
  environment:
    name: image/$CI_COMMIT_REF_SLUG
    url: https://open.greenhost.net:4567/openappstack/openappstack/openappstack-ci:${CI_COMMIT_REF_SLUG}
    on_stop: delete-image
    auto_stop_in: 3 weeks
  rules:
    # Automatically rebuild the container image if this file, the Dockerfile,
    # the installed requirements or the kaniko template change
    - changes:
        - Dockerfile
        - requirements.txt
        - .gitlab/ci_templates/kaniko.yml
    # Also rebuild when the CI variable contain this jobs name
    # or commit msg contains /TRIGGER_JOBS=.*ci-test-image-build/
    - if: '$TRIGGER_JOBS =~ /ci-test-image-build/'
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/'
  extends:
    - .kaniko_build
  interruptible: true

report-ci-image-tag:
  stage: build
  image: "curlimages/curl"
  before_script:
    - *debug_information
  script:
    - |
      TAG_INFORMATION=$(curl https://open.greenhost.net/api/v4/projects/openappstack%2Fopenappstack/registry/repositories/2/tags/${CI_COMMIT_REF_SLUG});
      echo "Tag information: ${TAG_INFORMATION}"
      if [ "$TAG_INFORMATION" == '{"message":"404 Tag Not Found"}' ]; then
        echo "CI_CONTAINER_TAG=master" > .ci.env
      else
        echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env
      fi
  artifacts:
    paths:
      - .ci.env
    expire_in: 1 month
    when: always
    reports:
      dotenv:
        .ci.env
  rules:
    # Make sure this job does not run if ci-test-image-build runs
    - changes:
        - Dockerfile
        - requirements.txt
      when: never  # Never run on file changes that trigger ci-test-image-build
    - if: '$TRIGGER_JOBS =~ /ci-test-image-build/'
      when: never  # Never run when ci-test-image is triggered manually
    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/'
      when: never  # Never run when ci-test-image is triggered manually
    - when: always
  interruptible: true


# Stage: create-vps
# =================
#
# Creates the vps for the pipeline

create-vps:
  stage: create-vps
  script:
    - *debug_information
    # Creates a VPS based on a custom CI image for which --install-kubernetes
    # has already run. See CONTRIBUTING.md#ci-pipeline-image for more info
    - sh .gitlab/ci_scripts/create_vps.sh
    # Make sure .ci.env variables are not lost
    - cat .ci.env >> ${CLUSTER_DIR}/.cluster.env
  extends:
    - .ssh_setup
    - .report_artifacts
    - .general_rules
  environment:
    name: $CI_COMMIT_REF_SLUG
    url: https://$FQDN
    on_stop: terminate-droplet
    auto_stop_in: 1 week
  interruptible: true

# Stage: enable-apps
# ==================
#
# Checks if application needs to get installed

.enable_app_template:
  stage: enable-apps
  before_script:
    - *debug_information
  script:
    - |
      [ ! -d ./enabled_apps ] && mkdir enabled_apps || /bin/true
      touch ./enabled_apps/$APP
  needs:
    - job: create-vps
  extends:
    - .report_artifacts
  interruptible: true

enable-cert-manager:
  variables:
    APP: "cert-manager"
  extends:
    - .enable_app_template
    - .cert_manager_rules

enable-eventrouter:
  variables:
    APP: "eventrouter"
  extends:
    - .enable_app_template
    - .eventrouter_rules

enable-loki:
  variables:
    APP: "loki"
  extends:
    - .enable_app_template
    - .loki_rules

enable-promtail:
  variables:
    APP: "promtail"
  extends:
    - .enable_app_template
    - .promtail_rules

enable-nextcloud:
  variables:
    APP: "nextcloud"
  extends:
    - .enable_app_template
    - .nextcloud_rules

enable-prometheus-stack:
  variables:
    APP: "prometheus-stack"
  extends:
    - .enable_app_template
    - .prometheus_stack_rules

enable-rocketchat:
  variables:
    APP: "rocketchat"
  extends:
    - .enable_app_template
    - .rocketchat_rules

enable-single-sign-on:
  variables:
    APP: "single-sign-on"
  extends:
    - .enable_app_template
    - .single_sign_on_rules

enable-wordpress:
  variables:
    APP: "wordpress"
  extends:
    - .enable_app_template
    - .wordpress_rules


# Stage: setup-cluster
# ====================
#
# Installs OAS

test-dns:
  stage: setup-cluster
  needs:
    - job: create-vps
  script:
    - *debug_information
    - cd ansible/
    - pytest -v -s -m 'dns' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
  extends:
    - .general_rules
  interruptible: true

setup-openappstack:
  stage: setup-cluster
  script:
    - *debug_information
    # Copy inventory files to ansible folder for use in install-apps step
    - chmod 700 ansible
    # For every app we add the fully qualified path to an array named enabled_applications which we then save as settings.yml
    # Refer to ansible/group_vars/all/settings.yml.example to see an example
    - for app in enabled_apps/*; do yq -i eval ".enabled_applications += [\"$(basename $app)\"]" clusters/${CI_COMMIT_REF_SLUG}/group_vars/all/settings.yml; done
    - cp clusters/${CI_COMMIT_REF_SLUG}/inventory.yml ansible/
    - cp clusters/${CI_COMMIT_REF_SLUG}/group_vars/all/settings.yml ansible/group_vars/all/
    # Set up cluster
    - python3 -m openappstack $HOSTNAME install --install-kubernetes
    # Show versions of installed apps/binaries
    - cd ansible
    - ansible master -m shell -a 'oas-version-info.sh 2>&1'
  extends:
    - .ssh_setup
    - .report_artifacts
    - .general_rules
  interruptible: true


# Stage: helm-release
# ====================
#
# Tests if all helmreleases are in `deployed` state


.helm-release:
  stage: helm-release
  needs:
    - job: setup-openappstack
    - job: test-dns
  script:
    - *debug_information
    - cd ansible/
    - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml"
    - pytest -v -s -m 'helmreleases' --app="$APP" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 20
  extends:
    - .ssh_setup
  interruptible: true

cert-manager-helm-release:
  variables:
    APP: "cert-manager"
  extends:
    - .helm-release
    - .cert_manager_rules

eventrouter-helm-release:
  variables:
    APP: "eventrouter"
  extends:
    - .helm-release
    - .eventrouter_rules

local-path-provisioner-helm-release:
  variables:
    APP: "local-path-provisioner"
  extends:
    - .helm-release
    - .local_path_provisioner_rules

loki-helm-release:
  variables:
    APP: "loki"
  extends:
    - .helm-release
    - .loki_rules

promtail-helm-release:
  variables:
    APP: "promtail"
  extends:
    - .helm-release
    - .promtail_rules

nextcloud-helm-release:
  variables:
    APP: "nextcloud"
  extends:
    - .helm-release
    - .nextcloud_rules

prometheus-stack-helm-release:
  variables:
    APP: "prometheus-stack"
  extends:
    - .helm-release
    - .prometheus_stack_rules

rocketchat-helm-release:
  variables:
    APP: "rocketchat"
  extends:
    - .helm-release
    - .rocketchat_rules

single-sign-on-helm-release:
  variables:
    APP: "single-sign-on"
  extends:
    - .helm-release
    - .single_sign_on_rules

wordpress-helm-release:
  variables:
    APP: "wordpress"
  extends:
    - .helm-release
    - .wordpress_rules


# Stage: app-ready
# ================
#
# Tests apps for readiness state


.apps-ready:
  stage: apps-ready
  script:
    - *debug_information
    - cd ansible/
    - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml"
    - pytest -v -s -m 'apps_running' --app="$APP" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10
  extends:
    - .ssh_setup
  interruptible: true

cert-manager-ready:
  variables:
    APP: "cert-manager"
  needs:
    - job: cert-manager-helm-release
    - job: setup-openappstack  # Needs makes sure the artifacts from that job are downloaded
  extends:
    - .apps-ready
    - .cert_manager_rules

eventrouter-ready:
  variables:
    APP: "eventrouter"
  needs:
    - job: eventrouter-helm-release
    - job: setup-openappstack
  extends:
    - .apps-ready
    - .eventrouter_rules

local-path-provisioner-ready:
  variables:
    APP: "local-path-provisioner"
  needs:
    - job: local-path-provisioner-helm-release
    - job: setup-openappstack
  extends:
    - .apps-ready
    - .local_path_provisioner_rules

loki-ready:
  variables:
    APP: "loki"
  needs:
    - job: loki-helm-release
    - job: setup-openappstack
  extends:
    - .apps-ready
    - .loki_rules

promtail-ready:
  variables:
    APP: "promtail"
  needs:
    - job: promtail-helm-release
    - job: setup-openappstack
  extends:
    - .apps-ready
    - .promtail_rules

nextcloud-ready:
  variables:
    APP: "nextcloud"
  needs:
    - job: nextcloud-helm-release
    - job: setup-openappstack
  extends:
    - .apps-ready
    - .nextcloud_rules

prometheus-stack-ready:
  variables:
    APP: "prometheus-stack"
  needs:
    - job: prometheus-stack-helm-release
    - job: setup-openappstack
  extends:
    - .apps-ready
    - .prometheus_stack_rules

rocketchat-ready:
  variables:
    APP: "rocketchat"
  needs:
    - job: rocketchat-helm-release
    - job: setup-openappstack
  extends:
    - .apps-ready
    - .rocketchat_rules

single-sign-on-ready:
  variables:
    APP: "single-sign-on"
  needs:
    - job: single-sign-on-helm-release
    - job: setup-openappstack
  extends:
    - .apps-ready
    - .single_sign_on_rules

wordpress-ready:
  variables:
    APP: "wordpress"
  needs:
    - job: wordpress-helm-release
    - job: setup-openappstack
  extends:
    - .apps-ready
    - .wordpress_rules

# Stage: certs
# ================
#
# Test each app for proper certs

.apps-cert:
  stage: certs
  script:
    - *debug_information
    - cd ansible/
    - pytest -v -s -m 'certs' --app="$APP" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10
  extends:
    - .ssh_setup
  interruptible: true

nextcloud-cert:
  variables:
    APP: "nextcloud"
  needs:
    - job: nextcloud-ready
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .nextcloud_rules

prometheus-stack-cert:
  variables:
    APP: "prometheus-stack"
  needs:
    - job: prometheus-stack-ready
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .prometheus_stack_rules

rocketchat-cert:
  variables:
    APP: "rocketchat"
  needs:
    - job: rocketchat-ready
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .rocketchat_rules

single-sign-on-cert:
  variables:
    APP: "single-sign-on"
  needs:
    - job: single-sign-on-ready
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .single_sign_on_rules

wordpress-cert:
  variables:
    APP: "wordpress"
  needs:
    - job: wordpress-ready
    - job: setup-openappstack
  extends:
    - .apps-cert
    - .wordpress_rules


# Stage: health-test
# ==================
#
# General cluster health checks

testinfra:
  stage: health-test
  script:
    - *debug_information
    - cd ansible/
    - pytest -v -s -m 'testinfra' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
  extends:
    - .ssh_setup
    - .general_rules
  interruptible: true

prometheus-stack-alerts:
  stage: health-test
  variables:
    # APP var is used in job specific rules (i.e. .prometheus_stack_rules)
    APP: "prometheus-stack"
  allow_failure: true
  script:
    - *debug_information
    - cd test/
    - pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
  extends:
    - .ssh_setup
    - .prometheus_stack_rules
  needs:
    - job: prometheus-stack-ready
    - job: setup-openappstack
  interruptible: true


# Stage: integration-test
# =======================
#
# Runs integration tests for most apps using behave

.behave:
  stage: integration-test
  script:
    - *debug_information
    # Run the behave tests for specific app
    - python3 -m openappstack $HOSTNAME test --behave-headless --behave-ignore-certificate-errors --behave-tags $APP || python3 -m openappstack $HOSTNAME test --behave-headless --behave-ignore-certificate-errors --behave-tags $APP --behave-rerun-failing
  retry: 2
  artifacts:
    paths:
      - test/behave/screenshots/
    expire_in: 1 month
    when: on_failure
  extends:
    - .ssh_setup
  interruptible: true

prometheus-stack-behave:
  variables:
    APP: "prometheus-stack"
  needs:
    - job: prometheus-stack-cert
    - job: setup-openappstack
  extends:
    - .behave
    - .prometheus_stack_rules

nextcloud-behave:
  variables:
    APP: "nextcloud"
  needs:
    - job: nextcloud-cert
    - job: setup-openappstack
  extends:
    - .behave
    - .nextcloud_rules

rocketchat-behave:
  variables:
    APP: "rocketchat"
  needs:
    - job: rocketchat-cert
    - job: setup-openappstack
  extends:
    - .behave
    - .rocketchat_rules


wordpress-behave:
  variables:
    APP: "wordpress"
  needs:
    - job: wordpress-cert
    - job: setup-openappstack
  extends:
    - .behave
    - .wordpress_rules


# Etc
# ===


# Terminates a droplet once the branch for it is deleted
terminate-droplet:
  # Stage has to be the same as the step that created the VPS
  # https://docs.gitlab.com/ee/ci/environments.html#automatically-stopping-an-environment
  stage: create-vps
  # Gets triggered by on_stop of create-vps job
  when: manual
  variables:
    GIT_STRATEGY: none
  script:
    - *debug_information
    - python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^${CI_COMMIT_REF_SLUG}\")"
  environment:
    name: $CI_COMMIT_REF_SLUG
    action: stop

# Deletes a container image once a branch is deleted.
# Careful! When you run this step manually, you might have to trigger container
# image re-build as well
delete-image:
  stage: build
  when: manual
  variables:
    GIT_STRATEGY: none
  script:
    - *debug_information
    - "curl --request DELETE --header \"PRIVATE-TOKEN: ${CLEANER_TOKEN}\" https://open.greenhost.net/api/v4/projects/openappstack%2Fopenappstack/registry/repositories/2/tags/${CI_COMMIT_REF_SLUG}"
  environment:
    name: image/$CI_COMMIT_REF_SLUG
    action: stop
