diff --git a/ansible/group_vars/all/oas.yml b/ansible/group_vars/all/oas.yml index 19a85c813274f9da00609f7bbe31e2e2cd1abaac..cbfe158640e4143c182d315f62fa4f78b05878a4 100644 --- a/ansible/group_vars/all/oas.yml +++ b/ansible/group_vars/all/oas.yml @@ -14,7 +14,6 @@ collabora_password: "{{ lookup('password', '{{ secret_directory }}/collabora_adm grafana_admin_password: "{{ lookup('password', '{{ secret_directory }}/grafana_admin_password chars=ascii_letters') }}" # git repo versions -git_helmfiles_version: '00e4239375f4aab8779168e78d03f3f6fcb44a6f' git_charts_version: 'HEAD' git_local_storage_version: 'HEAD' git_nextcloud_version: 'bd748d9a11111411d3c9e536f90d9909b01b5b72' diff --git a/ansible/roles/configure/tasks/main.yml b/ansible/roles/configure/tasks/main.yml index 8aa27239019947fabb71e1129a44ebd8b2876dec..2ebc8d6d5801edce3339b80aca5414fafe47064b 100644 --- a/ansible/roles/configure/tasks/main.yml +++ b/ansible/roles/configure/tasks/main.yml @@ -12,6 +12,7 @@ - curl - git - nftables + - rsync - snapd - unattended-upgrades # Update again after 1 day diff --git a/ansible/roles/setup/tasks/helmfiles.yml b/ansible/roles/setup/tasks/helmfiles.yml index 670e01565e153484ad04d61a5c558dab6d62b697..e4b137ae830f0445826358441eec2ef3be93d31f 100644 --- a/ansible/roles/setup/tasks/helmfiles.yml +++ b/ansible/roles/setup/tasks/helmfiles.yml @@ -1,14 +1,22 @@ --- -- name: Clone helmfiles repo +- name: Remove old helmfiles repo + tags: + - helm + - helmfile + file: + path: '/oas/source/repos/helmfiles' + state: absent + +- name: Synchronize helmfiles directory tags: - git - helm - helmfile - git: - repo: 'https://code.greenhost.net/openappstack/helmfiles' - dest: '/oas/source/repos/helmfiles' - version: '{{ git_helmfiles_version }}' + synchronize: + src: '../../helmfiles' + dest: '/oas/source' + delete: true - name: Clone charts repo tags: @@ -59,6 +67,7 @@ - name: Touch config file locations tags: - config + - helmfile - oas file: state: touch @@ -85,10 +94,11 @@ - name: Apply helmfiles tags: - helmfile + - tmp environment: - COLLABORA_PASSWORD: "{{ collabora_password }}" - NEXTCLOUD_PASSWORD: "{{ nextcloud_password }}" - NEXTCLOUD_MARIADB_PASSWORD: "{{ nextcloud_mariadb_password }}" - NEXTCLOUD_MARIADB_ROOT_PASSWORD: "{{ nextcloud_mariadb_root_password }}" - GRAFANA_ADMIN_PASSWORD: "{{ grafana_admin_password }}" - command: '/usr/local/bin/helmfile -b /snap/bin/helm -e oas -f /oas/source/repos/helmfiles/helmfile.d/ apply --suppress-secrets' + command: '/usr/local/bin/helmfile -b /snap/bin/helm -e oas -f /oas/source/helmfiles/helmfile.d/ apply --suppress-secrets' diff --git a/helmfiles/README.md b/helmfiles/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4388b502753fe20bd9e6ce51a0a4fa84be6bfadb --- /dev/null +++ b/helmfiles/README.md @@ -0,0 +1,93 @@ +## Introduction + +This directory describes the applications that are deployed to a new OpenAppStack +instance. The `helmfile.d` subdirectory contains information about which helm +charts need to be deployed. The `values` subdirectory contains values.yml overrides +for these charts. + +Use [helmfile](https://github.com/roboll/helmfile) to install these applications +to a cluster. + +## Usage + +The data in this directory is typically used by the Ansible playbooks located +in the `ansible/` top level directory. Check the tasks tagged `helmfile` to +get more information. + +If you have a cluster already, and do not want to use our bootstrap script to +install these applications, follow these steps: + +### Prerequisites + +Make sure you follow the installation instructions of +[helmfile](https://github.com/roboll/helmfile) before you try this! Also note +that helmfile requires `helm diff` to be installed. Install it by running +`helm plugin install https://github.com/databus23/helm-diff` + +### Preparation + +Do these three steps to prepare the installation process: + +1. Get the local-storage chart locally + + ```bash + $ git clone https://open.greenhost.net/openappstack/local-storage ../local-storage + ``` + +1. You need to have a configuration file called `local.yaml` in the + following directory relative to this directory: + `../../../config/values/local.yaml`. Use our template at + https://code.greenhost.net/openappstack/bootstrap/blob/master/ansible/roles/configure_helmfile/templates/local.yaml.j2 + and fill in the variables. + +1. You need to set some environment variables: + - `$NEXTCLOUD_PASSWORD` to set the Nextcloud administrator password + - `$COLLABORA_PASSWORD` to set the Collabora administrator password + - `$NEXTCLOUD_MARIADB_ROOT_PASSWORD` for the MariaDB that NextCloud uses + - `$GRAFANA_ADMIN_PASSWORD` for the admin password of grafana + +1. OAS allows you to override the nginx configuration by setting variables in + a file at `/oas/config/values/apps/nginx.yaml` on the cluster. You can + leave this file empty, but it *has* to exist. + +1. Certmanager will get installed, which uses some custom resource definitions. + You need to add these resource definitions like so: + + ``` + kubectl apply -f https://raw.githubusercontent.com/jetstack/cert-manager/release-0.7/deploy/manifests/00-crds.yaml + ``` + +### Installation + +Install all the applications by running: + +``` +$ helmfile -e oas -f helmfile.d/ apply +``` + +Where: + +- `-e oas` means that you are running this for the `oas` environment defined in + the files +- `-f helmfile.d` means you want to use the description in files in the local + `helmfile.d` directory +- `apply` syncs your kubernetes cluster state to the one desired by the files. + +**NOTE:** If you have applied these helmfiles before, check if you still have +old `pvc`'s for mariadb lying around. They can mess up the installation process, +especially if you use different passwords than before. + +For example: + +``` +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +data-oas-test-files-mariadb-0 Bound pvc-2a0dfd8f-7176-11e9-8ea4-00160a765c00 512Mi RWO local 9m +``` + +The mariadb pvc can collide with your installation. Remove it (note, this also +removes all the data that was in that database!) by running: + +``` +kubectl delete pvc data-oas-test-files-mariadb-0 +``` diff --git a/helmfiles/helmfile.d/00-storage.yaml b/helmfiles/helmfile.d/00-storage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aea03b10af5e2895f05d52d713d20e6b6d5e0348 --- /dev/null +++ b/helmfiles/helmfile.d/00-storage.yaml @@ -0,0 +1,11 @@ +environments: + oas: + values: + - "../../../config/values/local.yaml" + +releases: + - name: "oas-{{ .Environment.Values.releaseName }}-local-storage" + namespace: "oas" + chart: "../../repos/local-storage/" + values: + - "../values/local-storage.yaml" diff --git a/helmfiles/helmfile.d/05-cert-manager.yaml b/helmfiles/helmfile.d/05-cert-manager.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d45c0b1f052f848987d171dae0bf935cbb9fc9ec --- /dev/null +++ b/helmfiles/helmfile.d/05-cert-manager.yaml @@ -0,0 +1,17 @@ +environments: + oas: + values: + - "../../../config/values/local.yaml" + +repositories: + - name: jetstack + url: https://charts.jetstack.io + +releases: + - name: "oas-{{ .Environment.Values.releaseName }}-cert-manager" + namespace: "cert-manager" + chart: "jetstack/cert-manager" + version: "0.8" + values: + - "../values/cert-manager.yaml.gotmpl" + wait: false diff --git a/helmfiles/helmfile.d/10-nginx.yaml b/helmfiles/helmfile.d/10-nginx.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9a8cfe2a2bca2207e5a2ab3074aba76f7bfbf09 --- /dev/null +++ b/helmfiles/helmfile.d/10-nginx.yaml @@ -0,0 +1,13 @@ +environments: + oas: + values: + - "../../../config/values/local.yaml" + +releases: + - name: "oas-{{ .Environment.Values.releaseName }}-proxy" + namespace: "oas" + chart: "stable/nginx-ingress" + values: + - "../values/nginx.yaml.gotmpl" + - "/oas/config/values/apps/nginx.yaml.gotmpl" + wait: false diff --git a/helmfiles/helmfile.d/15-monitoring.yaml b/helmfiles/helmfile.d/15-monitoring.yaml new file mode 100644 index 0000000000000000000000000000000000000000..888521b34906c0cdb52d2039a1ba88ac0920d541 --- /dev/null +++ b/helmfiles/helmfile.d/15-monitoring.yaml @@ -0,0 +1,13 @@ +environments: + oas: + values: + - "../../../config/values/local.yaml" + +releases: + - name: "oas-{{ .Environment.Values.releaseName }}-prometheus" + namespace: "oas" + chart: "stable/prometheus-operator" + values: + - "../values/prometheus.yaml.gotmpl" + - "/oas/config/values/apps/prometheus.yaml.gotmpl" + wait: false diff --git a/helmfiles/helmfile.d/20-nextcloud.yaml b/helmfiles/helmfile.d/20-nextcloud.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6f32401cf30804946b5c6ac22d174a69fc7f323 --- /dev/null +++ b/helmfiles/helmfile.d/20-nextcloud.yaml @@ -0,0 +1,21 @@ +environments: + oas: + values: + - "../../../config/values/local.yaml" + +# Note: needs helm-git plugin (https://github.com/aslafy-z/helm-git) +repositories: + - name: onlyoffice-documentserver + url: git+https://code.greenhost.net/openappstack/nextcloud@onlyoffice-documentserver?ref=master + + + +releases: + - name: "oas-{{ .Environment.Values.releaseName }}-files" + namespace: "oas-apps" + # Install from file path, so you don't run into https://github.com/roboll/helmfile/issues/726 + chart: "../../repos/nextcloud/nextcloud-onlyoffice" + values: + - "../values/nextcloud.yaml.gotmpl" + wait: true + timeout: 600 diff --git a/helmfiles/values/cert-manager.yaml.gotmpl b/helmfiles/values/cert-manager.yaml.gotmpl new file mode 100644 index 0000000000000000000000000000000000000000..1e3749018de49c1e8ff4ca4c3ded45339b74bb2f --- /dev/null +++ b/helmfiles/values/cert-manager.yaml.gotmpl @@ -0,0 +1,7 @@ +ingressShim: + {{- if .Environment.Values.acmeStaging | default false }} + defaultIssuerName: "letsencrypt-staging" + {{- else }} + defaultIssuerName: "letsencrypt-production" + {{- end }} + defaultIssuerKind: ClusterIssuer diff --git a/helmfiles/values/local-storage.yaml b/helmfiles/values/local-storage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2c04142952c14814494479086ad3d5e775467e8 --- /dev/null +++ b/helmfiles/values/local-storage.yaml @@ -0,0 +1,2 @@ +storageDirectory: "/var/lib/OpenAppStack/local-storage" +defaultStorageClass: true diff --git a/helmfiles/values/nextcloud.yaml.gotmpl b/helmfiles/values/nextcloud.yaml.gotmpl new file mode 100644 index 0000000000000000000000000000000000000000..5edca0dc51ddbd1b5f40317a661b4eb95c7fc11f --- /dev/null +++ b/helmfiles/values/nextcloud.yaml.gotmpl @@ -0,0 +1,64 @@ +nextcloud: + nextcloud: + host: "files.{{ .Environment.Values.domain }}" + password: "{{ requiredEnv "NEXTCLOUD_PASSWORD" }}" + + ingress: + enabled: true + annotations: + # Tell cert-manager to automatically get a TLS certificate + kubernetes.io/tls-acme: "true" + # Set max body size high to allow big NextCloud uploads + nginx.ingress.kubernetes.io/proxy-body-size: 1G + hosts: + - "files.{{ .Environment.Values.domain }}" + tls: + - hosts: + - "files.{{ .Environment.Values.domain }}" + secretName: oas-{{ .Environment.Values.releaseName }}-files + + # Use 2 GB of storage for NC storage (maybe make configurable later?) + persistence: + enabled: true + size: 2Gi + + # Explicitly disable use of internal database + internalDatabase: + enabled: false + + # Enable and configure MariaDB chart + mariadb: + db: + password: "{{ requiredEnv "NEXTCLOUD_MARIADB_PASSWORD" }}" + enabled: true + master: + persistence: + ## Enable PostgreSQL persistence using Persistent Volume Claims. + enabled: true + size: 512Mi + replication: + enabled: false + rootUser: + password: "{{ requiredEnv "NEXTCLOUD_MARIADB_ROOT_PASSWORD" }}" + livenessProbe: + initialDelaySeconds: 120 + readinessProbe: + initialDelaySeconds: 120 + +onlyoffice-documentserver: + ingress: + enabled: true + annotations: + # Tell cert-manager to automatically get a TLS certificate + kubernetes.io/tls-acme: "true" + paths: + - "/" + hosts: + - "office.{{ .Environment.Values.domain }}" + tls: + - hosts: + - "office.{{ .Environment.Values.domain }}" + secretName: oas-{{ .Environment.Values.releaseName }}-office + + onlyoffice: + server_name: "office.{{ .Environment.Values.domain }}" diff --git a/helmfiles/values/nginx.yaml.gotmpl b/helmfiles/values/nginx.yaml.gotmpl new file mode 100644 index 0000000000000000000000000000000000000000..e4bfa621df0280f6420e42fcf853b6d9730eebd2 --- /dev/null +++ b/helmfiles/values/nginx.yaml.gotmpl @@ -0,0 +1,13 @@ +controller: + # scope: + # enabled: true + # namespace: oas-apps + service: + externalIPs: ["{{ .Environment.Values.ip }}"] + resources: + limits: + cpu: 100m + memory: 1Gi + requests: + cpu: 50m + memory: 64Mi diff --git a/helmfiles/values/prometheus.yaml.gotmpl b/helmfiles/values/prometheus.yaml.gotmpl new file mode 100644 index 0000000000000000000000000000000000000000..c9ea726fe5bacb7c720021618003dcb8f05fcb89 --- /dev/null +++ b/helmfiles/values/prometheus.yaml.gotmpl @@ -0,0 +1,55 @@ +# Depending on which DNS solution you have installed in your cluster enable the right exporter +coreDns: + enabled: false + +kubeDns: + enabled: true + +alertmanager: + alertmanagerSpec: + storage: + volumeClaimTemplate: + metadata: + name: alertmanager + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi + +prometheus: + prometheusSpec: + storageSpec: + volumeClaimTemplate: + metadata: + name: prometheus + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi + # https://github.com/rancher/rancher/issues/14836 prevents 'subPath' to work + # with non-privileged users. This is an insecure workaround for the time + # being... + securityContext: + runAsUser: 0 + fsGroup: 0 + runAsNonRoot: false + +grafana: + adminPassword: "{{ requiredEnv "GRAFANA_ADMIN_PASSWORD" }}" + + ingress: + enabled: true + annotations: + kubernetes.io/tls-acme: "true" + hosts: + - "grafana.{{ .Environment.Values.domain }}" + tls: + - secretName: grafana-tls + hosts: + - "grafana.{{ .Environment.Values.domain }}" + persistence: + enabled: true + size: 2Gi + accessModes: ["ReadWriteOnce"] diff --git a/test/Dockerfile b/test/Dockerfile index ead704cb4460689886a9932c6a41559c5662e00d..a5f6c7e9cea98e1c5a9ec450657f8f72ec99957c 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -16,8 +16,9 @@ RUN apk --no-cache add \ musl-dev \ openssh-client \ openssl-dev \ - python3-dev + python3-dev \ + rsync COPY ./requirements.txt /requirements.txt RUN pip3 install -r /requirements.txt - +RUN ln -s /usr/bin/python3 /usr/bin/python diff --git a/test/ci-bootstrap.py b/test/ci-bootstrap.py index 0ae70eb4386631118d3a069f6da18354a88984f4..547ebdea57b835511c26ae859d3e3078c79946f4 100755 --- a/test/ci-bootstrap.py +++ b/test/ci-bootstrap.py @@ -143,7 +143,7 @@ def main(): # pylint: disable=too-many-statements,too-many-branches ssh_key_id=args.ssh_key_id, region='ams1', size=8192, - disk=15, + disk=20, image=19) droplet_id = droplet['droplet']['id'] log.info('Created droplet id: %s', droplet_id)