diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7cb68360f4e14993afeee4b789928f9f336adabd..ec64e887f2fd3dfd161fb43cf93d2fc379c0d261 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,21 +1,22 @@
---
-# YAML anchors
-# ============
+# Global templates and YAML anchors
+# =================================
#
+# Used in various stages/job definitions
+
# We don't use a `before_script` definition here because `extend` doesn't merge
# `before_script` but rather overwrites it.
# So we rather use [yaml anchors](https://docs.gitlab.com/ce/ci/yaml/README.html#anchors)
# here. Unfortunatly, anchors can't get included from files so we need to
# define them here.
-
.debug_information: &debug_information
- |
echo "Env vars:"
echo
echo "HOSTNAME: $HOSTNAME"
echo "IP_ADDRESS: $IP_ADDRESS"
- echo "Uptime: $(uptime -p)"
+ echo "Uptime: $(uptime)"
echo "CLUSTER_DIR: $CLUSTER_DIR"
echo "ANSIBLE_HOST_KEY_CHECKING: $ANSIBLE_HOST_KEY_CHECKING"
echo "KANIKO_BUILD_IMAGENAME: $KANIKO_BUILD_IMAGENAME"
@@ -26,55 +27,93 @@
echo
echo
-.image_build_template: &image_build_template
- stage: build
- before_script:
- - *debug_information
- extends:
- - .kaniko_build
- environment:
- name: image/$CI_COMMIT_REF_SLUG
- url: https://open.greenhost.net:4567/openappstack/openappstack/openappstack-ci:${CI_COMMIT_REF_SLUG}
- on_stop: delete-image
- auto_stop_in: 3 weeks
-# YAML extends
-# ============
+# app rules
#
-
-# the .app_rules should be used whenever an app-specific job is executed.
-# just add the variable app to the job like this:
-
+# Define the rules when/if app specific jobs are run.
+# Just add the variable APP to the job like this:
# variables:
# APP: "eventrouter"
-
-# and import the template with
-
-# extends: .app_rules
-
-# .app_rules will ensure that the job is only executed when files related to the app changed in the repo
-.app_rules:
- before_script:
- - *debug_information
+# and import the templates with i.e.
+# extends: .eventrouter_rules
+# .eventrouter_rules will ensure that the job is only executed:
+# - when files related to the app changed in the repo
+# - A pipeline gets started from the UI and the job name is included in the
+# CI variable `TRIGGER_JOBS`
+# - A commit is pushed containing the pattern TRIGGER_JOBS=.*<job name>
+# (i.e. TRIGGER_JOBS=ci-test-image-build,enable-grafana)
+#
+# Gitlab CI allows pushing CI vars via `git push` but a bug prevents this when
+# using merge request pipelines (see https://gitlab.com/gitlab-org/gitlab/-/issues/326098)
+.eventrouter_rules:
rules:
- changes:
- flux/**/$APP*.yaml
- ansible/roles/apps/templates/settings/$APP.yaml
- ansible/roles/apps/tasks/$APP.yaml
+ - if: '$TRIGGER_JOBS =~ /enable-eventrouter/'
+ - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-eventrouter/'
+ - if: '$CI_COMMIT_BRANCH == "master"'
-
-.enable_app_template:
- extends: .app_rules
- stage: enable-apps
- script:
- - |
- [ ! -d ./enabled_apps ] && mkdir enabled_apps || /bin/true
- touch ./enabled_apps/$APP
- artifacts:
- paths:
- - ./clusters
- - ./enabled_apps/$APP
-
+.grafana_rules:
+ rules:
+ - changes:
+ - flux/**/$APP*.yaml
+ - ansible/roles/apps/templates/settings/$APP.yaml
+ - ansible/roles/apps/tasks/$APP.yaml
+ - if: '$TRIGGER_JOBS =~ /enable-grafana/'
+ - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-grafana/'
+ - if: '$CI_COMMIT_BRANCH == "master"'
+
+.nextcloud_rules:
+ rules:
+ - changes:
+ - flux/**/$APP*.yaml
+ - ansible/roles/apps/templates/settings/$APP.yaml
+ - ansible/roles/apps/tasks/$APP.yaml
+ - if: '$TRIGGER_JOBS =~ /enable-nextcloud/'
+ - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-nextcloud/'
+ - if: '$CI_COMMIT_BRANCH == "master"'
+
+.prometheus_rules:
+ rules:
+ - changes:
+ - flux/**/$APP*.yaml
+ - ansible/roles/apps/templates/settings/$APP.yaml
+ - ansible/roles/apps/tasks/$APP.yaml
+ - if: '$TRIGGER_JOBS =~ /enable-prometheus/'
+ - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-prometheus/'
+ - if: '$CI_COMMIT_BRANCH == "master"'
+
+.rocketchat_rules:
+ rules:
+ - changes:
+ - flux/**/$APP*.yaml
+ - ansible/roles/apps/templates/settings/$APP.yaml
+ - ansible/roles/apps/tasks/$APP.yaml
+ - if: '$TRIGGER_JOBS =~ /enable-rocketchat/'
+ - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-rocketchat/'
+ - if: '$CI_COMMIT_BRANCH == "master"'
+
+.single_sign_on_rules:
+ rules:
+ - changes:
+ - flux/**/$APP*.yaml
+ - ansible/roles/apps/templates/settings/$APP.yaml
+ - ansible/roles/apps/tasks/$APP.yaml
+ - if: '$TRIGGER_JOBS =~ /enable-single-sign-on/'
+ - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-single-sign-on/'
+ - if: '$CI_COMMIT_BRANCH == "master"'
+
+.wordpress_rules:
+ rules:
+ - changes:
+ - flux/**/$APP*.yaml
+ - ansible/roles/apps/templates/settings/$APP.yaml
+ - ansible/roles/apps/tasks/$APP.yaml
+ - if: '$TRIGGER_JOBS =~ /enable-wordpress/'
+ - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wordpress/'
+ - if: '$CI_COMMIT_BRANCH == "master"'
# Global declarations
@@ -82,9 +121,8 @@
# https://docs.gitlab.com/ee/ci/yaml/README.html#workflowrules-templates
include:
- - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml'
- - .gitlab/ci_templates/kaniko.yml
- - .gitlab/ci_templates/ssh_setup.yml
+ - /.gitlab/ci_templates/kaniko.yml
+ - /.gitlab/ci_templates/ssh_setup.yml
- template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'
stages:
@@ -121,25 +159,26 @@ default:
# Write "REBUILD_CONTAINER" in your commit message to force rebuilding the container.
ci-test-image-build:
- <<: *image_build_template
+ stage: build
+ before_script:
+ - *debug_information
+ environment:
+ name: image/$CI_COMMIT_REF_SLUG
+ url: https://open.greenhost.net:4567/openappstack/openappstack/openappstack-ci:${CI_COMMIT_REF_SLUG}
+ on_stop: delete-image
+ auto_stop_in: 3 weeks
rules:
# Automatically rebuild the container image if this file, the Dockerfile,
# the installed requirements or the kaniko template change
- changes:
- - .gitlab-ci.yml
- Dockerfile
- requirements.txt
- - .gitlab/ci_templates/**
- # These changes need to be tracked because subsequent jobs will try to
- # use the image that is tagged by this job.
- - .gitlab/ci_scripts/*
- - ansible/**/*
- - flux/**/*
- - test/**/*
- - openappstack/**/*
- # Also rebuild when the CI message contains REBUILD_CONTAINER
- - if: '$CI_COMMIT_MESSAGE =~ /REBUILD_CONTAINER/'
-
+ # Also rebuild when the CI variable contain this jobs name
+ # or commit msg contains /TRIGGER_JOBS=.*ci-test-image-build/
+ - if: '$TRIGGER_JOBS =~ /ci-test-image-build/'
+ - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*ci-test-image-build/'
+ extends:
+ - .kaniko_build
# Stage: create-vps
# =================
@@ -182,35 +221,60 @@ create-vps:
#
# Checks if application needs to get installed
+.enable_app_template:
+ stage: enable-apps
+ before_script:
+ - *debug_information
+ script:
+ - |
+ [ ! -d ./enabled_apps ] && mkdir enabled_apps || /bin/true
+ touch ./enabled_apps/$APP
+ artifacts:
+ paths:
+ - ./clusters
+ - ./enabled_apps/$APP
+
enable-eventrouter:
variables:
APP: "eventrouter"
- extends: .enable_app_template
+ extends:
+ - .enable_app_template
+ - .eventrouter_rules
enable-grafana:
variables:
APP: "grafana"
- extends: .enable_app_template
+ extends:
+ - .enable_app_template
+ - .grafana_rules
enable-nextcloud:
variables:
APP: "nextcloud"
- extends: .enable_app_template
+ extends:
+ - .enable_app_template
+ - .nextcloud_rules
enable-rocketchat:
variables:
APP: "rocketchat"
- extends: .enable_app_template
+ extends:
+ - .enable_app_template
+ - .rocketchat_rules
enable-single-sign-on:
variables:
APP: "single-sign-on"
- extends: .enable_app_template
+ extends:
+ - .enable_app_template
+ - .single_sign_on_rules
enable-wordpress:
variables:
APP: "wordpress"
- extends: .enable_app_template
+ extends:
+ - .enable_app_template
+ - .wordpress_rules
# Stage: setup-cluster
@@ -286,42 +350,56 @@ setup-openappstack:
when: always
extends:
- .ssh_setup
- - .app_rules
eventrouter-helm-release:
variables:
APP: "eventrouter"
- extends: .helm-release
+ extends:
+ - .helm-release
+ - .eventrouter_rules
grafana-helm-release:
variables:
APP: "grafana"
- extends: .helm-release
+ extends:
+ - .helm-release
+ - .grafana_rules
nextcloud-helm-release:
variables:
APP: "nextcloud"
- extends: .helm-release
+ extends:
+ - .helm-release
+ - .nextcloud_rules
prometheus-helm-release:
variables:
APP: "prometheus"
- extends: .helm-release
+ extends:
+ - .helm-release
+ - .prometheus_rules
rocketchat-helm-release:
variables:
APP: "rocketchat"
- extends: .helm-release
+ extends:
+ - .helm-release
+ - .rocketchat_rules
single-sign-on-helm-release:
variables:
APP: "single-sign-on"
- extends: .helm-release
+ extends:
+ - .helm-release
+ - .single_sign_on_rules
wordpress-helm-release:
variables:
APP: "wordpress"
- extends: .helm-release
+ extends:
+ - .helm-release
+ - .wordpress_rules
+
# Stage: app-ready
# ================
@@ -343,56 +421,69 @@ wordpress-helm-release:
when: always
extends:
- .ssh_setup
- - .app_rules
eventrouter-ready:
variables:
APP: "eventrouter"
needs:
- job: eventrouter-helm-release
- extends: .apps-ready
+ extends:
+ - .apps-ready
+ - .eventrouter_rules
grafana-ready:
variables:
APP: "grafana"
needs:
- job: grafana-helm-release
- extends: .apps-ready
+ extends:
+ - .apps-ready
+ - .grafana_rules
nextcloud-ready:
variables:
APP: "nextcloud"
needs:
- job: nextcloud-helm-release
- extends: .apps-ready
+ extends:
+ - .apps-ready
+ - .nextcloud_rules
prometheus-ready:
variables:
APP: "prometheus"
needs:
- job: prometheus-helm-release
- extends: .apps-ready
+ extends:
+ - .apps-ready
+ - .prometheus_rules
rocketchat-ready:
variables:
APP: "rocketchat"
needs:
- job: rocketchat-helm-release
- extends: .apps-ready
+ extends:
+ - .apps-ready
+ - .rocketchat_rules
single-sign-on-ready:
variables:
APP: "single-sign-on"
needs:
- job: single-sign-on-helm-release
- extends: .apps-ready
+ extends:
+ - .apps-ready
+ - .single_sign_on_rules
wordpress-ready:
variables:
APP: "wordpress"
needs:
- job: wordpress-helm-release
- extends: .apps-ready
+ extends:
+ - .apps-ready
+ - .wordpress_rules
# Stage: certs
# ================
@@ -412,49 +503,60 @@ wordpress-ready:
when: always
extends:
- .ssh_setup
- - .app_rules
grafana-cert:
variables:
APP: "grafana"
needs:
- job: grafana-ready
- extends: .apps-cert
+ extends:
+ - .apps-cert
+ - .grafana_rules
nextcloud-cert:
variables:
APP: "nextcloud"
needs:
- job: nextcloud-ready
- extends: .apps-cert
+ extends:
+ - .apps-cert
+ - .nextcloud_rules
prometheus-cert:
variables:
APP: "prometheus"
needs:
- job: prometheus-ready
- extends: .apps-cert
+ extends:
+ - .apps-cert
+ - .prometheus_rules
rocketchat-cert:
variables:
APP: "rocketchat"
needs:
- job: rocketchat-ready
- extends: .apps-cert
+ extends:
+ - .apps-cert
+ - .rocketchat_rules
single-sign-on-cert:
variables:
APP: "single-sign-on"
needs:
- job: single-sign-on-ready
- extends: .apps-cert
+ extends:
+ - .apps-cert
+ - .single_sign_on_rules
wordpress-cert:
variables:
APP: "wordpress"
needs:
- job: wordpress-ready
- extends: .apps-cert
+ extends:
+ - .apps-cert
+ - .wordpress_rules
# Stage: health-test
@@ -481,7 +583,7 @@ testinfra:
prometheus-alerts:
stage: health-test
variables:
- # Adding the app var hier in combination with .app_rules applies app specific gitlab-ci rules
+ # APP var is used in job specific rules (i.e. .grafana_rules)
APP: "prometheus"
allow_failure: true
script:
@@ -490,7 +592,7 @@ prometheus-alerts:
- pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=${CLUSTER_DIR}/inventory.yml --hosts='ansible://*'
extends:
- .ssh_setup
- - .app_rules
+ - .prometheus_rules
needs:
- job: prometheus-ready
@@ -514,35 +616,43 @@ prometheus-alerts:
when: on_failure
extends:
- .ssh_setup
- - .app_rules
grafana-behave:
variables:
APP: "grafana"
needs:
- job: grafana-cert
- extends: .behave
+ extends:
+ - .behave
+ - .grafana_rules
nextcloud-behave:
variables:
APP: "nextcloud"
needs:
- job: nextcloud-cert
- extends: .behave
+ extends:
+ - .behave
+ - .nextcloud_rules
rocketchat-behave:
variables:
APP: "rocketchat"
needs:
- job: rocketchat-cert
- extends: .behave
+ extends:
+ - .behave
+ - .rocketchat_rules
+
wordpress-behave:
variables:
APP: "wordpress"
needs:
- job: wordpress-cert
- extends: .behave
+ extends:
+ - .behave
+ - .wordpress_rules
# Etc
@@ -588,5 +698,4 @@ gitlab-merge-workaround:
stage: build
image: busybox
script:
- - *debug_information
- echo "Not building anything, no changes."
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index cb3547645076f8af153568445c83a6755dcd27d0..096db8e0cb29b80115ab08b3b35f270ebf775ddf 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -9,6 +9,7 @@ repos:
- id: detect-private-key
- id: end-of-file-fixer
- id: trailing-whitespace
+ - id: debug-statements
- repo: https://github.com/jumanjihouse/pre-commit-hooks
rev: 2.1.5
hooks:
@@ -18,3 +19,13 @@ repos:
rev: v1.23.0
hooks:
- id: hadolint
+ - repo: https://github.com/jazzband/pip-tools
+ # Makes sure the requirements.txt file is up to date
+ rev: 5.2.1
+ hooks:
+ - id: pip-compile
+ - repo: https://github.com/timothycrosley/isort
+ # Sorts import statements for you
+ rev: 5.0.9
+ hooks:
+ - id: isort
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000000000000000000000000000000000000..0a8b888b993e43910a9fb9b9b8ff0e35523bd64f
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,48 @@
+# How to contribute
+
+## Preparing the development environment
+
+Make sure you have development dependencies installed in your development environment.
+
+```
+pip install -r requirements-dev.txt
+```
+
+## pre-commit hooks
+
+We use [pre-commit](https://pre-commit.com/) to maintain and install pre-commit
+hooks that should be executed before each commit.
+
+
+Please install these required tools on your system:
+
+* [hadolint](https://github.com/hadolint/hadolint) for linting the `Dockerfile`
+* [shellcheck](https://www.shellcheck.net/) and
+* [shfmt](https://github.com/mvdan/sh) to lint and validate shell scripts
+
+Then install pre-commit hooks:
+
+```
+pre-commit install
+```
+
+Running `git commmit` for the first time after installing the hook usually takes a
+little longer because `pre-commit` pulls it's hooks from upstream repositories.
+You can find all hooks in `.pre-commit-config.yaml`.
+
+In case you need to skip the execution of the pre-commit hooks (please don't!),
+use `git commit --no-verify`.
+
+
+## Adding dependencies
+
+Make sure you update our `requirements.txt` file before you push your changes.
+Whenever you need a new python package, add it to requirements.in and run
+
+`pip-compile`
+
+to generate an new `requirements.txt` which does not only pin the new package
+but also its dependencies.
+
+If the new package you are adding is only used by developers,
+please add it to the `requirements-dev.txt` file.
diff --git a/flux/oas-apps/nextcloud_hr.yaml b/flux/oas-apps/nextcloud_hr.yaml
index 39945192f39af8984668ef44003f750163a8b5ea..0f6c5c50e24c8236d06d4f26197103f83affb100 100644
--- a/flux/oas-apps/nextcloud_hr.yaml
+++ b/flux/oas-apps/nextcloud_hr.yaml
@@ -1,3 +1,4 @@
+---
apiVersion: helm.fluxcd.io/v1
kind: HelmRelease
metadata:
@@ -11,7 +12,7 @@ spec:
releaseName: nc
chart:
git: https://open.greenhost.net/openappstack/nextcloud
- ref: 0.2.5
+ ref: 0.2.6
path: .
valuesFrom:
- secretKeyRef:
diff --git a/openappstack/cluster.py b/openappstack/cluster.py
index 79a7bb51538afa7946005675b655d81281231ec1..d106dfff89ccaff31beaee13f1fc112af035b29d 100644
--- a/openappstack/cluster.py
+++ b/openappstack/cluster.py
@@ -170,11 +170,22 @@ class Cluster:
settings['ip_address'] = self.ip_address
settings['domain'] = self.domain
settings['admin_email'] = 'admin@{0}'.format(self.domain)
- settings['acme_staging'] = self.acme_staging
settings['flux']['local_flux'] = self.local_flux
settings['cluster_dir'] = self.cluster_dir
settings['prometheus_enable_ingress'] = self.prometheus_enable_ingress
+ # Configure apps to handle invalid certs i.e. from
+ # Letsencrypt staging API
+ settings['acme_staging'] = self.acme_staging
+ nextcloud_extra_values = """
+ onlyoffice:
+ unauthorizedStorage: true
+ httpsHstsEnabled: false
+ """
+ if self.acme_staging:
+ settings['nextcloud_extra_values'] = \
+ yaml.load(nextcloud_extra_values)
+
file_contents = yaml.safe_dump(settings, default_flow_style=False)
log.debug(file_contents)
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 0575c31c09525e2224ba89dc7c8a6120ea7adf8e..918a023277633eb412d3c1e1f333191bf2f9a051 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,3 +1,4 @@
# Add development requirements here
# Please pin releases manually if necessary
pip-tools
+pre-commit
diff --git a/test/behave/features/nextcloud.feature b/test/behave/features/nextcloud.feature
index e9a9d9c8f2f3b6ddd8ce00f201adebea13a6b7fe..e1d77c0f259a18a87d7552143c8475caf582a03d 100644
--- a/test/behave/features/nextcloud.feature
+++ b/test/behave/features/nextcloud.feature
@@ -39,5 +39,5 @@ Scenario: Create a new document in OnlyOffice
And I click on the element "input.icon-confirm"
And I focus the last opened tab
Then I expect a new tab has been opened
- And I expect that element "div.type-error" does not exist
+ And I expect that element "div.toast-error" does not exist
And I wait on element "iframe[name='frameEditor']" for 20000ms to be visible
diff --git a/test/pytest/test_certs.py b/test/pytest/test_certs.py
index 460692589bde2fed6271b978fa5aa8c96a702244..c3fde585dcdd61139ed96fb8a9508c6727751bdd 100755
--- a/test/pytest/test_certs.py
+++ b/test/pytest/test_certs.py
@@ -99,6 +99,7 @@ def test_cert_validation(host, app): # pylint: disable=too-many-statements
# prometheus is only exposed if opted-in in the settings
'prometheus': 'prometheus',
'rocketchat': 'chat',
+ 'single-sign-on': 'sso',
'wordpress': 'www'
}