From 516e749386aade25d913f467a68eb94b5cf45a8f Mon Sep 17 00:00:00 2001
From: Arie Peterson <arie@greenhost.nl>
Date: Fri, 15 Nov 2019 17:20:25 +0100
Subject: [PATCH] Set up flux and helm-operator using a cluster-local repo
---
ansible/bootstrap.yml | 3 +
ansible/group_vars/all/settings.yml.example | 2 +-
ansible/roles/apps/tasks/flux.yml | 14 +++
ansible/roles/apps/tasks/init.yml | 17 ++++
ansible/roles/apps/tasks/main.yml | 10 +--
ansible/roles/apps/tasks/nextcloud.yml | 42 ---------
.../apps/templates/secrets.nextcloud.yaml | 88 +++++++++++++++++++
ansible/roles/local-flux/files/git-hook.sh | 3 +
ansible/roles/local-flux/files/nginx.yaml | 69 +++++++++++++++
ansible/roles/local-flux/tasks/main.yml | 52 +++++++++++
flux/nextcloud.yaml | 19 ++++
helmfiles/helmfile.d/00-flux.yaml | 41 +++++++++
helmfiles/helmfile.d/20-nextcloud.yaml | 15 ----
13 files changed, 312 insertions(+), 63 deletions(-)
create mode 100644 ansible/roles/apps/tasks/flux.yml
delete mode 100644 ansible/roles/apps/tasks/nextcloud.yml
create mode 100644 ansible/roles/apps/templates/secrets.nextcloud.yaml
create mode 100644 ansible/roles/local-flux/files/git-hook.sh
create mode 100644 ansible/roles/local-flux/files/nginx.yaml
create mode 100644 ansible/roles/local-flux/tasks/main.yml
create mode 100644 flux/nextcloud.yaml
create mode 100644 helmfiles/helmfile.d/00-flux.yaml
delete mode 100644 helmfiles/helmfile.d/20-nextcloud.yaml
diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml
index 59a93f363..35cd4f6bf 100644
--- a/ansible/bootstrap.yml
+++ b/ansible/bootstrap.yml
@@ -60,6 +60,9 @@
- import_role:
name: apps
tags: ['apps']
+ - import_role:
+ name: local-flux
+ tags: ['flux']
always:
- import_role:
name: finalize
diff --git a/ansible/group_vars/all/settings.yml.example b/ansible/group_vars/all/settings.yml.example
index bf9c3ce75..f5ad5d725 100644
--- a/ansible/group_vars/all/settings.yml.example
+++ b/ansible/group_vars/all/settings.yml.example
@@ -15,10 +15,10 @@ acme_staging: false
# Which apps to install from the helmfile.d/ dir
helmfiles:
- 00-storage
+ - 00-flux
- 05-cert-manager
- 10-nginx
- 15-monitoring
- - 20-nextcloud
# Optional, custom rke config.
# I.e. you can set the desired Kubernetes version but please be aware of
diff --git a/ansible/roles/apps/tasks/flux.yml b/ansible/roles/apps/tasks/flux.yml
new file mode 100644
index 000000000..efb88fe10
--- /dev/null
+++ b/ansible/roles/apps/tasks/flux.yml
@@ -0,0 +1,14 @@
+---
+- name: Install flux
+ tags:
+ - helmfile
+ - flux
+ include_role:
+ name: "helmfile"
+ tasks_from: "apply"
+ apply:
+ tags:
+ - helmfile
+ - flux
+ vars:
+ helmfile: '00-flux'
diff --git a/ansible/roles/apps/tasks/init.yml b/ansible/roles/apps/tasks/init.yml
index d02a739d3..17376e311 100644
--- a/ansible/roles/apps/tasks/init.yml
+++ b/ansible/roles/apps/tasks/init.yml
@@ -59,6 +59,23 @@
- "prometheus"
- "nextcloud"
+- name: Create Kubernetes secret
+ tags:
+ - config
+ - flux
+ - oas
+ - nextcloud
+ k8s:
+ state: present
+ definition:
+ api_version: v1
+ kind: Secret
+ metadata:
+ namespace: "oas-apps"
+ name: "oas"
+ data:
+ nextcloud.yaml: "{{ lookup('template','secrets.nextcloud.yaml') | b64encode }}"
+
- name: Check if there are failed helm deployments
tags:
- helm
diff --git a/ansible/roles/apps/tasks/main.yml b/ansible/roles/apps/tasks/main.yml
index 2c1afae58..859cd7dcc 100644
--- a/ansible/roles/apps/tasks/main.yml
+++ b/ansible/roles/apps/tasks/main.yml
@@ -3,6 +3,11 @@
import_tasks: init.yml
tags: [ helmfile ]
+- name: Install flux
+ import_tasks: flux.yml
+ tags: [ helmfile ]
+ when: '"00-flux" in helmfiles'
+
- name: Install local-storage
import_tasks: local-storage.yml
tags: [ helmfile ]
@@ -22,8 +27,3 @@
import_tasks: prometheus.yml
tags: [ helmfile ]
when: '"15-monitoring" in helmfiles'
-
-- name: Install nextcloud
- import_tasks: nextcloud.yml
- tags: [ helmfile ]
- when: '"20-nextcloud" in helmfiles'
diff --git a/ansible/roles/apps/tasks/nextcloud.yml b/ansible/roles/apps/tasks/nextcloud.yml
deleted file mode 100644
index acb05aedd..000000000
--- a/ansible/roles/apps/tasks/nextcloud.yml
+++ /dev/null
@@ -1,42 +0,0 @@
----
-- name: Clone nextcloud repo
- tags:
- - git
- - helmfile
- - nextcloud
- git:
- repo: 'https://open.greenhost.net/openappstack/nextcloud'
- dest: '{{ data_directory }}/source/repos/nextcloud'
- version: '{{ git_nextcloud_version }}'
-
-- name: Remove requirements.lock file
- tags:
- - git
- - nextcloud
- - helmfile
- file:
- path: '{{ data_directory }}/source/repos/nextcloud/nextcloud-onlyoffice/requirements.lock'
- state: absent
-
-- name: Install nextcloud and onlyoffice
- tags:
- - helmfile
- - nextcloud
- - onlyoffice
- include_role:
- name: "helmfile"
- tasks_from: "apply"
- apply:
- tags:
- - helmfile
- - nextcloud
- - onlyoffice
- environment:
- - NEXTCLOUD_PASSWORD: "{{ nextcloud_password }}"
- - NEXTCLOUD_MARIADB_PASSWORD: "{{ nextcloud_mariadb_password }}"
- - NEXTCLOUD_MARIADB_ROOT_PASSWORD: "{{ nextcloud_mariadb_root_password }}"
- - ONLYOFFICE_JWT_SECRET: "{{ onlyoffice_jwt_secret }}"
- - ONLYOFFICE_POSTGRESQL_PASSWORD: "{{ onlyoffice_postgresql_password }}"
- - ONLYOFFICE_RABBITMQ_PASSWORD: "{{ onlyoffice_rabbitmq_password }}"
- vars:
- helmfile: '20-nextcloud'
diff --git a/ansible/roles/apps/templates/secrets.nextcloud.yaml b/ansible/roles/apps/templates/secrets.nextcloud.yaml
new file mode 100644
index 000000000..2669785ed
--- /dev/null
+++ b/ansible/roles/apps/templates/secrets.nextcloud.yaml
@@ -0,0 +1,88 @@
+nextcloud:
+ nextcloud:
+ host: "files.{{ domain }}"
+ password: "{{ nextcloud_password }}"
+
+ ingress:
+ enabled: true
+ annotations:
+ # Tell cert-manager to automatically get a TLS certificate
+ kubernetes.io/tls-acme: "true"
+ # Set max body size high to allow big NextCloud uploads
+ nginx.ingress.kubernetes.io/proxy-body-size: 1G
+ nginx.ingress.kubernetes.io/server-snippet: |-
+ server_tokens off;
+ proxy_hide_header X-Powered-By;
+ rewrite ^/.well-known/webfinger /public.php?service=webfinger last;
+ rewrite ^/.well-known/host-meta /public.php?service=host-meta last;
+ rewrite ^/.well-known/host-meta.json /public.php?service=host-meta-json;
+ location = /.well-known/carddav {
+ return 301 $scheme://$host/remote.php/dav;
+ }
+ location = /.well-known/caldav {
+ return 301 $scheme://$host/remote.php/dav;
+ }
+ location = /robots.txt {
+ allow all;
+ log_not_found off;
+ access_log off;
+ }
+ hosts:
+ - "files.{{ domain }}"
+ tls:
+ - hosts:
+ - "files.{{ domain }}"
+ secretName: oas-nextcloud-files
+
+ # Use 2 GB of storage for NC storage (maybe make configurable later?)
+ persistence:
+ enabled: true
+ size: 2Gi
+
+ # Explicitly disable use of internal database
+ internalDatabase:
+ enabled: false
+
+ livenessProbe:
+ initialDelaySeconds: 120
+ readinessProbe:
+ initialDelaySeconds: 120
+
+# Enable and configure MariaDB chart
+mariadb:
+ db:
+ password: "{{ nextcloud_mariadb_password }}"
+ enabled: true
+ master:
+ persistence:
+ ## Enable PostgreSQL persistence using Persistent Volume Claims.
+ enabled: true
+ size: 512Mi
+ replication:
+ enabled: false
+ rootUser:
+ password: "{{ nextcloud_mariadb_root_password }}"
+
+onlyoffice:
+ server_name: "office.{{ domain }}"
+ ingress:
+ enabled: true
+ annotations:
+ # Tell cert-manager to automatically get a TLS certificate
+ kubernetes.io/tls-acme: "true"
+ paths:
+ - "/"
+ hosts:
+ - "office.{{ domain }}"
+ tls:
+ - hosts:
+ - "office.{{ domain }}"
+ secretName: oas-nextcloud-office
+ jwtSecret: "{{ onlyoffice_jwt_secret }}"
+
+postgresql:
+ postgresqlPassword: "{{ onlyoffice_postgresql_password }}"
+
+rabbitmq:
+ rabbitmq:
+ password: "{{ onlyoffice_rabbitmq_password }}"
diff --git a/ansible/roles/local-flux/files/git-hook.sh b/ansible/roles/local-flux/files/git-hook.sh
new file mode 100644
index 000000000..dac4c5319
--- /dev/null
+++ b/ansible/roles/local-flux/files/git-hook.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec git update-server-info
diff --git a/ansible/roles/local-flux/files/nginx.yaml b/ansible/roles/local-flux/files/nginx.yaml
new file mode 100644
index 000000000..d9926bd17
--- /dev/null
+++ b/ansible/roles/local-flux/files/nginx.yaml
@@ -0,0 +1,69 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: local-flux-data
+ labels:
+ type: local
+spec:
+ storageClassName: ""
+ capacity:
+ storage: 10Mi
+ accessModes:
+ - ReadOnlyMany
+ hostPath:
+ path: "/var/lib/OpenAppStack/local-flux"
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ namespace: oas
+ name: local-flux-data
+spec:
+ storageClassName: ""
+ volumeName: "local-flux-data"
+ accessModes:
+ - ReadOnlyMany
+ resources:
+ requests:
+ storage: 10Mi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: oas
+ name: local-flux
+spec:
+ selector:
+ matchLabels:
+ app: local-flux
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: local-flux
+ spec:
+ containers:
+ - name: local-flux-nginx
+ image: nginx:1.17-alpine
+ ports:
+ - containerPort: 80
+ volumeMounts:
+ - mountPath: "/usr/share/nginx/html"
+ name: local-flux-data
+ volumes:
+ - name: local-flux-data
+ persistentVolumeClaim:
+ claimName: local-flux-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+ namespace: oas
+ name: local-flux
+spec:
+ selector:
+ app: local-flux
+ ports:
+ - protocol: TCP
+ port: 80
+ targetPort: 80
diff --git a/ansible/roles/local-flux/tasks/main.yml b/ansible/roles/local-flux/tasks/main.yml
new file mode 100644
index 000000000..6a42539fd
--- /dev/null
+++ b/ansible/roles/local-flux/tasks/main.yml
@@ -0,0 +1,52 @@
+---
+
+- block:
+
+ - name: Copy HelmRelease files to server's local flux repo
+ tags:
+ - flux
+ copy:
+ src: "../../../flux/"
+ dest: "{{ repo }}"
+ register: helmreleases
+ become: true
+
+ - name: Create local flux repo
+ tags:
+ - flux
+ command: git init "{{ repo }}"
+ args:
+ creates: "{{ repo }}/.git"
+ become: true
+
+ - name: Enable post-update hook in local flux repo
+ tags:
+ - flux
+ copy:
+ src: "git-hook.sh"
+ dest: "{{ repo }}/.git/hooks/{{ item }}"
+ mode: "0755"
+ with_items:
+ - "post-update"
+ - "post-commit"
+ become: true
+
+ - name: Add HelmRelease files to local flux commit
+ tags:
+ - flux
+ shell: git add . && git -c "user.name=OpenAppStack automation" -c "user.email=tech@openappstack.net" commit --allow-empty --author="OpenAppStack automation <>" -m "Local flux via ansible"
+ args:
+ chdir: "{{ repo }}"
+ when: helmreleases.changed
+ become: true
+
+ - name: Deploy local flux nginx
+ tags:
+ - flux
+ k8s:
+ state: present
+ definition: "{{ lookup('file', 'nginx.yaml') }}"
+
+ vars:
+ repo: "/var/lib/OpenAppStack/local-flux"
+ when: local_flux
diff --git a/flux/nextcloud.yaml b/flux/nextcloud.yaml
new file mode 100644
index 000000000..13994d409
--- /dev/null
+++ b/flux/nextcloud.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: helm.fluxcd.io/v1
+kind: HelmRelease
+metadata:
+ name: nextcloud
+ namespace: oas-apps
+ annotations:
+ flux.weave.works/automated: "false"
+spec:
+ # calling the release "nextcloud" runs into a bug in the helm chart
+ releaseName: nc
+ chart:
+ git: https://open.greenhost.net/openappstack/nextcloud
+ ref: master
+ path: .
+ valuesFrom:
+ - secretKeyRef:
+ name: oas
+ key: nextcloud.yaml
diff --git a/helmfiles/helmfile.d/00-flux.yaml b/helmfiles/helmfile.d/00-flux.yaml
new file mode 100644
index 000000000..6cf4a4bf5
--- /dev/null
+++ b/helmfiles/helmfile.d/00-flux.yaml
@@ -0,0 +1,41 @@
+environments:
+ oas:
+ values:
+ - "/etc/OpenAppStack/values/local.yaml"
+
+repositories:
+ - name: "fluxcd"
+ url: "https://charts.fluxcd.io"
+
+releases:
+ - name: "oas-{{ .Environment.Values.releaseName }}-flux"
+ namespace: "oas"
+ chart: "fluxcd/flux"
+ values:
+ - git:
+ # This is the url to the "local-flux" nginx pod that is running
+ # inside the cluster, and is serving the git repo with HelmRelease
+ # files over http.
+ url: "http://local-flux.oas.svc.cluster.local/.git"
+ readonly: true
+ registry:
+ # Do not do follow updates of upstream docker images automatically.
+ excludeImage: "*"
+ sync:
+ # Necessary for read-only mode.
+ state: "secret"
+ syncGarbageCollection:
+ # Delete resources originally created by Flux when their manifests
+ # are removed from the git repo.
+ enabled: true
+ # bug in memcached chart?
+ memcached:
+ securityContext:
+ allowPrivilegeEscalation: null
+ wait: false
+ - name: "oas-{{ .Environment.Values.releaseName }}-helm-operator"
+ namespace: "oas"
+ chart: "fluxcd/helm-operator"
+ values:
+ - createCRD: true
+ wait: false
diff --git a/helmfiles/helmfile.d/20-nextcloud.yaml b/helmfiles/helmfile.d/20-nextcloud.yaml
deleted file mode 100644
index 6167231fe..000000000
--- a/helmfiles/helmfile.d/20-nextcloud.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-environments:
- oas:
- values:
- - "/etc/OpenAppStack/values/local.yaml"
-
-releases:
- - name: "oas-{{ .Environment.Values.releaseName }}-files"
- namespace: "oas-apps"
- # Install from file path, so you don't run into https://github.com/roboll/helmfile/issues/726
- chart: "../../repos/nextcloud"
- values:
- - "../values/nextcloud.yaml.gotmpl"
- - "/etc/OpenAppStack/values/apps/nextcloud.yaml.gotmpl"
- wait: true
- timeout: 900
--
GitLab