diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cac7c3b1967a3ae2fb7bff785648539a8cfb025b..95f34531d6b1894bb2792729c2d450115d42f583 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,8 @@ --- +include: + - /.gitlab/ci_templates/kaniko.yml + - /.gitlab/ci_templates/ssh_setup.yml + - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' # Global templates and YAML anchors # ================================= @@ -20,13 +24,37 @@ echo "CLUSTER_DIR: $CLUSTER_DIR" echo "ANSIBLE_HOST_KEY_CHECKING: $ANSIBLE_HOST_KEY_CHECKING" echo "KANIKO_BUILD_IMAGENAME: $KANIKO_BUILD_IMAGENAME" - echo "KANIKO build image ref: ${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_COMMIT_REF_SLUG}" + echo "KANIKO build image ref: ${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}" echo "SSH_KEY_ID: $SSH_KEY_ID" echo [ -d $CLUSTER_DIR ] && find $CLUSTER_DIR || echo "directory ${CLUSTER_DIR} not found" echo echo +# The dotenv report requires us to report the artifacts in every job that is +# required with a `needs:` from another job. +.report_artifacts: + artifacts: + paths: + - clusters + - ./enabled_apps/$APP + expire_in: 1 month + when: always + reports: + dotenv: + $CLUSTER_DIR/.cluster.env + +# Rules that enable the cluster to be built and are applied to most steps +# (except for application-specific steps) +.general_rules: + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/ci_scripts/* + - ansible/**/* + - flux/**/* + - test/**/* + - openappstack/**/* # app rules # @@ -46,36 +74,36 @@ # Gitlab CI allows pushing CI vars via `git push` but a bug prevents this when # using merge request pipelines (see https://gitlab.com/gitlab-org/gitlab/-/issues/326098) .eventrouter_rules: - rules: - - when: always + extends: + - .general_rules .loki_stack_rules: - rules: - - when: always + extends: + - .general_rules .nextcloud_rules: rules: - changes: - - flux/**/$APP*.yaml - - ansible/roles/apps/templates/settings/$APP.yaml - - ansible/roles/apps/tasks/$APP.yaml - - test/behave/features/$APP.feature + - flux/**/$APP*.yaml + - ansible/roles/apps/templates/settings/$APP.yaml + - ansible/roles/apps/tasks/$APP.yaml + - test/behave/features/$APP.feature - if: '$TRIGGER_JOBS =~ /enable-nextcloud/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-nextcloud/' - if: '$CI_COMMIT_BRANCH == "master"' .prometheus_stack_rules: - rules: - - when: always + extends: + - .general_rules .rocketchat_rules: rules: - changes: - - flux/**/$APP*.yaml - - ansible/roles/apps/templates/settings/$APP.yaml - - ansible/roles/apps/tasks/$APP.yaml - - test/behave/features/$APP.feature + - flux/**/$APP*.yaml + - ansible/roles/apps/templates/settings/$APP.yaml + - ansible/roles/apps/tasks/$APP.yaml + - test/behave/features/$APP.feature - if: '$TRIGGER_JOBS =~ /enable-rocketchat/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-rocketchat/' - if: '$CI_COMMIT_BRANCH == "master"' @@ -83,9 +111,9 @@ .single_sign_on_rules: rules: - changes: - - flux/**/$APP*.yaml - - ansible/roles/apps/templates/settings/$APP.yaml - - ansible/roles/apps/tasks/$APP.yaml + - flux/**/$APP*.yaml + - ansible/roles/apps/templates/settings/$APP.yaml + - ansible/roles/apps/tasks/$APP.yaml - if: '$TRIGGER_JOBS =~ /enable-single-sign-on/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-single-sign-on/' - if: '$CI_COMMIT_BRANCH == "master"' @@ -93,10 +121,10 @@ .wordpress_rules: rules: - changes: - - flux/**/$APP*.yaml - - ansible/roles/apps/templates/settings/$APP.yaml - - ansible/roles/apps/tasks/$APP.yaml - - test/behave/features/$APP.feature + - flux/**/$APP*.yaml + - ansible/roles/apps/templates/settings/$APP.yaml + - ansible/roles/apps/tasks/$APP.yaml + - test/behave/features/$APP.feature - if: '$TRIGGER_JOBS =~ /enable-wordpress/' - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wordpress/' - if: '$CI_COMMIT_BRANCH == "master"' @@ -106,10 +134,6 @@ # =================== # https://docs.gitlab.com/ee/ci/yaml/README.html#workflowrules-templates -include: - - /.gitlab/ci_templates/kaniko.yml - - /.gitlab/ci_templates/ssh_setup.yml - - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' stages: - build @@ -133,7 +157,7 @@ variables: CLUSTER_DIR: "/builds/openappstack/openappstack/clusters/${CI_COMMIT_REF_SLUG}" default: - image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_COMMIT_REF_SLUG}" + image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_CONTAINER_TAG}" # Stage: build @@ -141,13 +165,23 @@ default: # # Builds CI test container image # There are 2 moments in which we (re)build the container image. If some files are -# changed, or when it needs to be done manually -# Write "REBUILD_CONTAINER" in your commit message to force rebuilding the container. +# changed, or when the job is triggered with TRIGGER_JOBS. ci-test-image-build: stage: build before_script: - *debug_information + after_script: + - | + echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env + artifacts: + paths: + - .ci.env + expire_in: 1 month + when: always + reports: + dotenv: + .ci.env environment: name: image/$CI_COMMIT_REF_SLUG url: https://open.greenhost.net:4567/openappstack/openappstack/openappstack-ci:${CI_COMMIT_REF_SLUG} @@ -159,6 +193,7 @@ ci-test-image-build: - changes: - Dockerfile - requirements.txt + - .gitlab/ci_templates/kaniko.yml # Also rebuild when the CI variable contain this jobs name # or commit msg contains /TRIGGER_JOBS=.*ci-test-image-build/ - if: '$TRIGGER_JOBS =~ /ci-test-image-build/' @@ -167,6 +202,44 @@ ci-test-image-build: - .kaniko_build interruptible: true +report-ci-image-tag: + stage: build + image: "curlimages/curl" + before_script: + - *debug_information + script: + - | + TAG_INFORMATION=$(curl https://open.greenhost.net/api/v4/projects/openappstack%2Fopenappstack/registry/repositories/2/tags/${CI_COMMIT_REF_SLUG}); + echo "Tag information: ${TAG_INFORMATION}" + if [ "$TAG_INFORMATION" == '{"message":"404 Tag Not Found"}' ]; then + echo "CI_CONTAINER_TAG=master" > .ci.env + else + echo "CI_CONTAINER_TAG=${CI_COMMIT_REF_SLUG}" > .ci.env + fi + artifacts: + paths: + - .ci.env + expire_in: 1 month + when: always + reports: + dotenv: + .ci.env + rules: + # Make sure this job does not run if ci-test-image-build runs + - changes: + - Dockerfile + - requirements.txt + when: never # Never run on file changes that trigger ci-test-image-build + - if: '$TRIGGER_JOBS =~ /ci-test-image-build/' + when: never # Never run when ci-test-image is triggered manually + - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/' + when: never # Never run when ci-test-image is triggered manually + - if: '$CI_COMMIT_BRANCH == "master"' + when: never # Never run on master branch + - when: always + interruptible: true + + # Stage: create-vps # ================= # @@ -179,24 +252,12 @@ create-vps: # Creates a VPS based on a custom CI image for which --install-kubernetes # has already run. See CONTRIBUTING.md#ci-pipeline-image for more info - sh .gitlab/ci_scripts/create_vps.sh - artifacts: - paths: - - clusters - expire_in: 1 month - when: always - reports: - dotenv: - $CLUSTER_DIR/.cluster.env - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/ci_scripts/* - - ansible/**/* - - flux/**/* - - test/**/* - - openappstack/**/* + # Make sure .ci.env variables are not lost + - cat .ci.env >> ${CLUSTER_DIR}/.cluster.env extends: - .ssh_setup + - .report_artifacts + - .general_rules environment: name: $CI_COMMIT_REF_SLUG url: https://$FQDN @@ -217,10 +278,10 @@ create-vps: - | [ ! -d ./enabled_apps ] && mkdir enabled_apps || /bin/true touch ./enabled_apps/$APP - artifacts: - paths: - - ./clusters - - ./enabled_apps/$APP + needs: + - job: create-vps + extends: + - .report_artifacts interruptible: true enable-eventrouter: @@ -281,6 +342,8 @@ enable-wordpress: test-dns: stage: setup-cluster + needs: + - job: create-vps script: - *debug_information - cd ansible/ @@ -310,20 +373,10 @@ setup-openappstack: # Show versions of installed apps/binaries - cd ansible - ansible master -m shell -a 'oas-version-info.sh 2>&1' - artifacts: - paths: - - ./clusters - expire_in: 1 month - when: always - rules: - - changes: - - .gitlab-ci.yml - - ansible/**/* - - flux/**/* - - test/**/* - - openappstack/**/* extends: - .ssh_setup + - .report_artifacts + - .general_rules interruptible: true @@ -335,18 +388,14 @@ setup-openappstack: .helm-release: stage: helm-release + needs: + - job: setup-openappstack + - job: test-dns script: - *debug_information - cd ansible/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml" - pytest -v -s -m 'helmreleases' --app="$APP" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 20 - artifacts: - paths: - - ./clusters - - ansible/inventory.yml - - ansible/group_vars/all/settings.yml - expire_in: 1 month - when: always extends: - .ssh_setup interruptible: true @@ -415,11 +464,6 @@ wordpress-helm-release: - cd ansible/ - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml" - pytest -v -s -m 'apps_running' --app="$APP" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 - artifacts: - paths: - - ./clusters - expire_in: 1 month - when: always extends: - .ssh_setup interruptible: true @@ -429,6 +473,7 @@ eventrouter-ready: APP: "eventrouter" needs: - job: eventrouter-helm-release + - job: setup-openappstack # Needs makes sure the artifacts from that job are downloaded extends: - .apps-ready - .eventrouter_rules @@ -438,6 +483,7 @@ loki-stack-ready: APP: "loki-stack" needs: - job: loki-stack-helm-release + - job: setup-openappstack extends: - .apps-ready - .loki_stack_rules @@ -447,6 +493,7 @@ nextcloud-ready: APP: "nextcloud" needs: - job: nextcloud-helm-release + - job: setup-openappstack extends: - .apps-ready - .nextcloud_rules @@ -456,6 +503,7 @@ prometheus-stack-ready: APP: "prometheus-stack" needs: - job: prometheus-stack-helm-release + - job: setup-openappstack extends: - .apps-ready - .prometheus_stack_rules @@ -465,6 +513,7 @@ rocketchat-ready: APP: "rocketchat" needs: - job: rocketchat-helm-release + - job: setup-openappstack extends: - .apps-ready - .rocketchat_rules @@ -474,6 +523,7 @@ single-sign-on-ready: APP: "single-sign-on" needs: - job: single-sign-on-helm-release + - job: setup-openappstack extends: - .apps-ready - .single_sign_on_rules @@ -483,6 +533,7 @@ wordpress-ready: APP: "wordpress" needs: - job: wordpress-helm-release + - job: setup-openappstack extends: - .apps-ready - .wordpress_rules @@ -498,11 +549,6 @@ wordpress-ready: - *debug_information - cd ansible/ - pytest -v -s -m 'certs' --app="$APP" --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10 - artifacts: - paths: - - ./clusters - expire_in: 1 month - when: always extends: - .ssh_setup interruptible: true @@ -512,6 +558,7 @@ nextcloud-cert: APP: "nextcloud" needs: - job: nextcloud-ready + - job: setup-openappstack extends: - .apps-cert - .nextcloud_rules @@ -521,6 +568,7 @@ prometheus-stack-cert: APP: "prometheus-stack" needs: - job: prometheus-stack-ready + - job: setup-openappstack extends: - .apps-cert - .prometheus_stack_rules @@ -530,6 +578,7 @@ rocketchat-cert: APP: "rocketchat" needs: - job: rocketchat-ready + - job: setup-openappstack extends: - .apps-cert - .rocketchat_rules @@ -539,6 +588,7 @@ single-sign-on-cert: APP: "single-sign-on" needs: - job: single-sign-on-ready + - job: setup-openappstack extends: - .apps-cert - .single_sign_on_rules @@ -548,6 +598,7 @@ wordpress-cert: APP: "wordpress" needs: - job: wordpress-ready + - job: setup-openappstack extends: - .apps-cert - .wordpress_rules @@ -590,6 +641,7 @@ prometheus-stack-alerts: - .prometheus_stack_rules needs: - job: prometheus-stack-ready + - job: setup-openappstack interruptible: true @@ -619,6 +671,7 @@ prometheus-stack-behave: APP: "prometheus-stack" needs: - job: prometheus-stack-cert + - job: setup-openappstack extends: - .behave - .prometheus_stack_rules @@ -628,6 +681,7 @@ nextcloud-behave: APP: "nextcloud" needs: - job: nextcloud-cert + - job: setup-openappstack extends: - .behave - .nextcloud_rules @@ -637,6 +691,7 @@ rocketchat-behave: APP: "rocketchat" needs: - job: rocketchat-cert + - job: setup-openappstack extends: - .behave - .rocketchat_rules @@ -647,13 +702,14 @@ wordpress-behave: APP: "wordpress" needs: - job: wordpress-cert + - job: setup-openappstack extends: - .behave - .wordpress_rules - # Etc - # === +# Etc +# === # Terminates a droplet once the branch for it is deleted @@ -686,14 +742,3 @@ delete-image: environment: name: image/$CI_COMMIT_REF_SLUG action: stop - -# We need one job that run every time (without any `only:` limitation). -# This works around a Gitlab bug: if no job runs at all due to -# `only`, Gitlab gets confused and doesn't allow you to merge the MR: -# https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html#limitations -gitlab-merge-workaround: - stage: build - image: busybox - script: - - echo "Not building anything, no changes." - interruptible: true diff --git a/Dockerfile b/Dockerfile index 61fc57812b8842d743b8565f86f277ea9bbaaeb8..23feb62a32ab5302a4865074961cacb66bd3c0ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,3 +32,4 @@ RUN \ update-ca-certificates && \ pip install --no-cache-dir -r /requirements.txt && \ ln -s /usr/bin/python3 /usr/bin/python + diff --git a/README.md b/README.md index 42f751a771eb218fbfec277be28def8516eadf16..5ff2f585eb635d86b99f48ebddaa1318c4258d5e 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,4 @@ a single-node kubernetes cluster. Please refer to https://docs.openappstack.net for further details, and to [the installation tutorial](docs/installation_instructions.md) for a step by step installation tutorial how to install your cluster. +