Merge branch '734-option-to-trigger-app-deployments-from-ci-variables' into 'master'

Resolve "Option to trigger app deployments from CI variables"

Closes #739, #736, and #734

See merge request openappstack/openappstack!372

.gitlab-ci.yml

 ---
 
-# YAML anchors
-# ============
+# Global templates and YAML anchors
+# =================================
 #
+# Used in various stages/job definitions
+
 # We don't use a `before_script` definition here because `extend` doesn't merge
 # `before_script` but rather overwrites it.
 # So we rather use [yaml anchors](https://docs.gitlab.com/ce/ci/yaml/README.html#anchors)
 # here. Unfortunatly, anchors can't get included from files so we need to
 # define them here.
-
 .debug_information: &debug_information
   - |
     echo "Env vars:"
     echo
     echo "HOSTNAME:                  $HOSTNAME"
     echo "IP_ADDRESS:                $IP_ADDRESS"
-    echo "Uptime:                    $(uptime -p)"
+    echo "Uptime:                    $(uptime)"
     echo "CLUSTER_DIR:               $CLUSTER_DIR"
     echo "ANSIBLE_HOST_KEY_CHECKING: $ANSIBLE_HOST_KEY_CHECKING"
     echo "KANIKO_BUILD_IMAGENAME:    $KANIKO_BUILD_IMAGENAME"
@@ -26,55 +27,93 @@
     echo
     echo
 
-.image_build_template: &image_build_template
-  stage: build
-  before_script:
-    - *debug_information
-  extends:
-    - .kaniko_build
-  environment:
-    name: image/$CI_COMMIT_REF_SLUG
-    url: https://open.greenhost.net:4567/openappstack/openappstack/openappstack-ci:${CI_COMMIT_REF_SLUG}
-    on_stop: delete-image
-    auto_stop_in: 3 weeks
 
-# YAML extends
-# ============
+# app rules
 #
-
-# the .app_rules should be used whenever an app-specific job is executed.
-# just add the variable app to the job like this:
-
+# Define the rules when/if app specific jobs are run.
+# Just add the variable APP to the job like this:
 #   variables:
 #     APP: "eventrouter"
-
-# and import the template with
-
-#   extends: .app_rules
-
-# .app_rules will ensure that the job is only executed when files related to the app changed in the repo
-.app_rules:
-  before_script:
-    - *debug_information
+# and import the templates with i.e.
+#   extends: .eventrouter_rules
+# .eventrouter_rules will ensure that the job is only executed:
+# - when files related to the app changed in the repo
+# - A pipeline gets started from the UI and the job name is included in the
+#   CI variable `TRIGGER_JOBS`
+# - A commit is pushed containing the pattern TRIGGER_JOBS=.*<job name>
+#   (i.e. TRIGGER_JOBS=ci-test-image-build,enable-grafana)
+#
+# Gitlab CI allows pushing CI vars via `git push` but a bug prevents this when
+# using merge request pipelines (see https://gitlab.com/gitlab-org/gitlab/-/issues/326098)
+.eventrouter_rules:
   rules:
     - changes:
         - flux/**/$APP*.yaml
         - ansible/roles/apps/templates/settings/$APP.yaml
         - ansible/roles/apps/tasks/$APP.yaml
+    - if: '$TRIGGER_JOBS =~ /enable-eventrouter/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-eventrouter/'
+    - if: '$CI_COMMIT_BRANCH == "master"'
 
-
-.enable_app_template:
-  extends: .app_rules
-  stage: enable-apps
-  script:
-    - |
-      [ ! -d ./enabled_apps ] && mkdir enabled_apps || /bin/true
-      touch ./enabled_apps/$APP
-  artifacts:
-    paths:
-      - ./clusters
-      - ./enabled_apps/$APP
-
+.grafana_rules:
+  rules:
+    - changes:
+      - flux/**/$APP*.yaml
+      - ansible/roles/apps/templates/settings/$APP.yaml
+      - ansible/roles/apps/tasks/$APP.yaml
+    - if: '$TRIGGER_JOBS =~ /enable-grafana/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-grafana/'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+
+.nextcloud_rules:
+  rules:
+    - changes:
+      - flux/**/$APP*.yaml
+      - ansible/roles/apps/templates/settings/$APP.yaml
+      - ansible/roles/apps/tasks/$APP.yaml
+    - if: '$TRIGGER_JOBS =~ /enable-nextcloud/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-nextcloud/'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+
+.prometheus_rules:
+  rules:
+    - changes:
+      - flux/**/$APP*.yaml
+      - ansible/roles/apps/templates/settings/$APP.yaml
+      - ansible/roles/apps/tasks/$APP.yaml
+    - if: '$TRIGGER_JOBS =~ /enable-prometheus/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-prometheus/'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+
+.rocketchat_rules:
+  rules:
+    - changes:
+      - flux/**/$APP*.yaml
+      - ansible/roles/apps/templates/settings/$APP.yaml
+      - ansible/roles/apps/tasks/$APP.yaml
+    - if: '$TRIGGER_JOBS =~ /enable-rocketchat/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-rocketchat/'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+
+.single_sign_on_rules:
+  rules:
+    - changes:
+      - flux/**/$APP*.yaml
+      - ansible/roles/apps/templates/settings/$APP.yaml
+      - ansible/roles/apps/tasks/$APP.yaml
+    - if: '$TRIGGER_JOBS =~ /enable-single-sign-on/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-single-sign-on/'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+
+.wordpress_rules:
+  rules:
+    - changes:
+      - flux/**/$APP*.yaml
+      - ansible/roles/apps/templates/settings/$APP.yaml
+      - ansible/roles/apps/tasks/$APP.yaml
+    - if: '$TRIGGER_JOBS =~ /enable-wordpress/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wordpress/'
+    - if: '$CI_COMMIT_BRANCH == "master"'
 
 
 # Global declarations
@@ -82,9 +121,8 @@
 
 # https://docs.gitlab.com/ee/ci/yaml/README.html#workflowrules-templates
 include:
-  - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml'
-  - .gitlab/ci_templates/kaniko.yml
-  - .gitlab/ci_templates/ssh_setup.yml
+  - /.gitlab/ci_templates/kaniko.yml
+  - /.gitlab/ci_templates/ssh_setup.yml
   - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'
 
 stages:
@@ -121,25 +159,26 @@ default:
 # Write "REBUILD_CONTAINER" in your commit message to force rebuilding the container.
 
 ci-test-image-build:
-  <<: *image_build_template
+  stage: build
+  before_script:
+    - *debug_information
+  environment:
+    name: image/$CI_COMMIT_REF_SLUG
+    url: https://open.greenhost.net:4567/openappstack/openappstack/openappstack-ci:${CI_COMMIT_REF_SLUG}
+    on_stop: delete-image
+    auto_stop_in: 3 weeks
   rules:
     # Automatically rebuild the container image if this file, the Dockerfile,
     # the installed requirements or the kaniko template change
     - changes:
-        - .gitlab-ci.yml
         - Dockerfile
         - requirements.txt
-        - .gitlab/ci_templates/**
-        # These  changes need to be tracked because subsequent jobs will try to
-        # use the image that is tagged by this job.
-        - .gitlab/ci_scripts/*
-        - ansible/**/*
-        - flux/**/*
-        - test/**/*
-        - openappstack/**/*
-    # Also rebuild when the CI message contains REBUILD_CONTAINER
-    - if: '$CI_COMMIT_MESSAGE =~ /REBUILD_CONTAINER/'
-
+    # Also rebuild when the CI variable contain this jobs name
+    # or commit msg contains /TRIGGER_JOBS=.*ci-test-image-build/
+    - if: '$TRIGGER_JOBS =~ /ci-test-image-build/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/'
+  extends:
+    - .kaniko_build
 
 # Stage: create-vps
 # =================
@@ -182,40 +221,67 @@ create-vps:
 #
 # Checks if application needs to get installed
 
+.enable_app_template:
+  stage: enable-apps
+  before_script:
+    - *debug_information
+  script:
+    - |
+      [ ! -d ./enabled_apps ] && mkdir enabled_apps || /bin/true
+      touch ./enabled_apps/$APP
+  artifacts:
+    paths:
+      - ./clusters
+      - ./enabled_apps/$APP
+
 enable-eventrouter:
   variables:
     APP: "eventrouter"
-  extends: .enable_app_template
+  extends:
+    - .enable_app_template
+    - .eventrouter_rules
 
 enable-grafana:
   variables:
     APP: "grafana"
-  extends: .enable_app_template
+  extends:
+    - .enable_app_template
+    - .grafana_rules
 
 enable-nextcloud:
   variables:
     APP: "nextcloud"
-  extends: .enable_app_template
+  extends:
+    - .enable_app_template
+    - .nextcloud_rules
 
 enable-prometheus:
   variables:
     APP: "prometheus"
-  extends: .enable_app_template
+  extends:
+    - .enable_app_template
+    - .prometheus_rules
 
 enable-rocketchat:
   variables:
     APP: "rocketchat"
-  extends: .enable_app_template
+  extends:
+    - .enable_app_template
+    - .rocketchat_rules
 
 enable-single-sign-on:
   variables:
     APP: "single-sign-on"
-  extends: .enable_app_template
+  extends:
+    - .enable_app_template
+    - .single_sign_on_rules
 
 enable-wordpress:
   variables:
     APP: "wordpress"
-  extends: .enable_app_template
+  extends:
+    - .enable_app_template
+    - .wordpress_rules
 
 
 # Stage: setup-cluster
@@ -291,42 +357,56 @@ setup-openappstack:
     when: always
   extends:
     - .ssh_setup
-    - .app_rules
 
 eventrouter-helm-release:
   variables:
     APP: "eventrouter"
-  extends: .helm-release
+  extends:
+    - .helm-release
+    - .eventrouter_rules
 
 grafana-helm-release:
   variables:
     APP: "grafana"
-  extends: .helm-release
+  extends:
+    - .helm-release
+    - .grafana_rules
 
 nextcloud-helm-release:
   variables:
     APP: "nextcloud"
-  extends: .helm-release
+  extends:
+    - .helm-release
+    - .nextcloud_rules
 
 prometheus-helm-release:
   variables:
     APP: "prometheus"
-  extends: .helm-release
+  extends:
+    - .helm-release
+    - .prometheus_rules
 
 rocketchat-helm-release:
   variables:
     APP: "rocketchat"
-  extends: .helm-release
+  extends:
+    - .helm-release
+    - .rocketchat_rules
 
 single-sign-on-helm-release:
   variables:
     APP: "single-sign-on"
-  extends: .helm-release
+  extends:
+    - .helm-release
+    - .single_sign_on_rules
 
 wordpress-helm-release:
   variables:
     APP: "wordpress"
-  extends: .helm-release
+  extends:
+    - .helm-release
+    - .wordpress_rules
+
 
 # Stage: app-ready
 # ================
@@ -348,56 +428,69 @@ wordpress-helm-release:
     when: always
   extends:
     - .ssh_setup
-    - .app_rules
 
 eventrouter-ready:
   variables:
     APP: "eventrouter"
   needs:
     - job: eventrouter-helm-release
-  extends: .apps-ready
+  extends:
+    - .apps-ready
+    - .eventrouter_rules
 
 grafana-ready:
   variables:
     APP: "grafana"
   needs:
     - job: grafana-helm-release
-  extends: .apps-ready
+  extends:
+    - .apps-ready
+    - .grafana_rules
 
 nextcloud-ready:
   variables:
     APP: "nextcloud"
   needs:
     - job: nextcloud-helm-release
-  extends: .apps-ready
+  extends:
+    - .apps-ready
+    - .nextcloud_rules
 
 prometheus-ready:
   variables:
     APP: "prometheus"
   needs:
     - job: prometheus-helm-release
-  extends: .apps-ready
+  extends:
+    - .apps-ready
+    - .prometheus_rules
 
 rocketchat-ready:
   variables:
     APP: "rocketchat"
   needs:
     - job: rocketchat-helm-release
-  extends: .apps-ready
+  extends:
+    - .apps-ready
+    - .rocketchat_rules
 
 single-sign-on-ready:
   variables:
     APP: "single-sign-on"
   needs:
     - job: single-sign-on-helm-release
-  extends: .apps-ready
+  extends:
+    - .apps-ready
+    - .single_sign_on_rules
 
 wordpress-ready:
   variables:
     APP: "wordpress"
   needs:
     - job: wordpress-helm-release
-  extends: .apps-ready
+  extends:
+    - .apps-ready
+    - .wordpress_rules
 
 # Stage: certs
 # ================
@@ -417,49 +510,60 @@ wordpress-ready:
     when: always
   extends:
     - .ssh_setup
-    - .app_rules
 
 grafana-cert:
   variables:
     APP: "grafana"
   needs:
     - job: grafana-ready
-  extends: .apps-cert
+  extends:
+    - .apps-cert
+    - .grafana_rules
 
 nextcloud-cert:
   variables:
     APP: "nextcloud"
   needs:
     - job: nextcloud-ready
-  extends: .apps-cert
+  extends:
+    - .apps-cert
+    - .nextcloud_rules
 
 prometheus-cert:
   variables:
     APP: "prometheus"
   needs:
     - job: prometheus-ready
-  extends: .apps-cert
+  extends:
+    - .apps-cert
+    - .prometheus_rules
 
 rocketchat-cert:
   variables:
     APP: "rocketchat"
   needs:
     - job: rocketchat-ready
-  extends: .apps-cert
+  extends:
+    - .apps-cert
+    - .rocketchat_rules
 
 single-sign-on-cert:
   variables:
     APP: "single-sign-on"
   needs:
     - job: single-sign-on-ready
-  extends: .apps-cert
+  extends:
+    - .apps-cert
+    - .single_sign_on_rules
 
 wordpress-cert:
   variables:
     APP: "wordpress"
   needs:
     - job: wordpress-ready
-  extends: .apps-cert
+  extends:
+    - .apps-cert
+    - .wordpress_rules
 
 
 # Stage: health-test
@@ -486,7 +590,7 @@ testinfra:
 prometheus-alerts:
   stage: health-test
   variables:
-    # Adding the app var hier in combination with .app_rules applies app specific gitlab-ci rules
+    # APP var is used in job specific rules (i.e. .grafana_rules)
     APP: "prometheus"
   allow_failure: true
   script:
@@ -495,7 +599,7 @@ prometheus-alerts:
     - pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
   extends:
     - .ssh_setup
-    - .app_rules
+    - .prometheus_rules
   needs:
     - job: prometheus-ready
 
@@ -519,35 +623,43 @@ prometheus-alerts:
     when: on_failure
   extends:
     - .ssh_setup
-    - .app_rules
 
 grafana-behave:
   variables:
     APP: "grafana"
   needs:
     - job: grafana-cert
-  extends: .behave
+  extends:
+    - .behave
+    - .grafana_rules
 
 nextcloud-behave:
   variables:
     APP: "nextcloud"
   needs:
     - job: nextcloud-cert
-  extends: .behave
+  extends:
+    - .behave
+    - .nextcloud_rules
 
 rocketchat-behave:
   variables:
     APP: "rocketchat"
   needs:
     - job: rocketchat-cert
-  extends: .behave
+  extends:
+    - .behave
+    - .rocketchat_rules
+
 
 wordpress-behave:
   variables:
     APP: "wordpress"
   needs:
     - job: wordpress-cert
-  extends: .behave
+  extends:
+    - .behave
+    - .wordpress_rules
 
 
   # Etc
@@ -593,5 +705,4 @@ gitlab-merge-workaround:
   stage: build
   image: busybox
   script:
-    - *debug_information
     - echo "Not building anything, no changes."

ansible/group_vars/all/settings.yml.example

@@ -91,7 +91,7 @@ enabled_applications:
   - 'metallb'
   - 'ingress'
   - 'local-path-provisioner'
-  - 'single-sign-on'
+  # - 'single-sign-on'
   # The backup system Velero is disabled by default, see settings under `backup` above.
   # - 'velero'
   #