diff --git a/CHANGELOG.md b/CHANGELOG.md
index bb25e69adaa1c726bbbf98e0a04f88eecafccc4b..defb8bf78b3c431c6a1d26e4a45cd94c8f659cf6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,14 @@
# Changelog
+## [0.7.0] - Unreleased
+
+* **BREAKING**: the `oas-secrets` helm chart has been removed. Because of this
+ `flux` will remove your existing secrets when you change the tracking branch
+ from `v0.6` to `v0.7`. Follow our [upgrade
+ guide](https://docs.openappstack.net/en/latest/upgrading.html) to prevent data
+ loss
+* TODO INSERT MORE HERE
+
## [0.6.0] - 2021-07-14
### Added
diff --git a/Dockerfile b/Dockerfile
index 7f88afb1fe134464d2ee9e7d5a27538cfde5ed98..48b5e5eaad456340916da7c2b74d3a346916b799 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,7 +16,7 @@ COPY ./requirements.txt /requirements.txt
RUN \
apk --no-cache add \
bash=~5.1.4-r0 \
- cargo=~1.52.0-r0 \
+ cargo=~1.52.1-r1 \
chromium=~91.0.4472.164-r0 \
curl=~7.78.0-r0 \
# needed for installing pycurl python module
diff --git a/charts/oas-secrets/.helmignore b/charts/oas-secrets/.helmignore
deleted file mode 100644
index 0e8a0eb36f4ca2c939201c0d54b5d82a1ea34778..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/.helmignore
+++ /dev/null
@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
diff --git a/charts/oas-secrets/Chart.yaml b/charts/oas-secrets/Chart.yaml
deleted file mode 100644
index cbc26985a3f96d4f83f47a443b7e1d8a9ce5a149..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/Chart.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
----
-apiVersion: v2
-name: secrets
-description: |
- A helm chart to generate secrets that are needed by other helm charts. Values
- inside the secret will only be generated if they do not exist yet, so it is
- safe to run `helm upgrade` on this chart.
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.9
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-# appVersion: "0.1.0"
diff --git a/charts/oas-secrets/templates/NOTES.txt b/charts/oas-secrets/templates/NOTES.txt
deleted file mode 100644
index 5e349b352c73698680859520c3556ce353f94de0..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/NOTES.txt
+++ /dev/null
@@ -1 +0,0 @@
-Secrets generated
diff --git a/charts/oas-secrets/templates/_helpers.tpl b/charts/oas-secrets/templates/_helpers.tpl
deleted file mode 100644
index d791991eac699d079688804ab0447aac72ee8bf0..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/_helpers.tpl
+++ /dev/null
@@ -1,84 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "secrets.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "secrets.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "secrets.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "secrets.labels" -}}
-helm.sh/chart: {{ include "secrets.chart" . }}
-{{ include "secrets.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "secrets.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "secrets.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "secrets.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create }}
-{{- default (include "secrets.fullname" .) .Values.serviceAccount.name }}
-{{- else }}
-{{- default "default" .Values.serviceAccount.name }}
-{{- end }}
-{{- end }}
-
-
-{{/*
-Returns a secret if it already in Kubernetes, otherwise it creates
-it randomly.
-Special thanks to @iTaybb: https://github.com/helm/charts/issues/5167#issuecomment-840698657
-*/}}
-{{- define "getOrGeneratePass" }}
-{{- $len := (default 16 .Length) | int -}}
-{{- $obj := (lookup "v1" .Kind .Namespace .Name).data -}}
-{{- $val := "" -}}
-{{- if $obj }}
-{{- $val := (index $obj .Key) -}}
-{{- end -}}
-{{- if $val }}
-{{- $val -}}
-{{- else if (eq (lower .Kind) "secret") -}}
-{{- randAlphaNum $len | b64enc -}}
-{{- else -}}
-{{- randAlphaNum $len -}}
-{{- end -}}
-{{- end }}
\ No newline at end of file
diff --git a/charts/oas-secrets/templates/oas-alertmanager-basic-auth.yaml b/charts/oas-secrets/templates/oas-alertmanager-basic-auth.yaml
deleted file mode 100644
index f4471fda9ead646670925392fefc4e7730a5aabe..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/oas-alertmanager-basic-auth.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
- namespace: "oas"
- name: "oas-alertmanager-basic-auth"
-data:
- {{- $pass := (include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-alertmanager-basic-auth" "Key" "pass")) | b64dec -}}
- {{/* Readable version of the password for humans who want to log in */}}
- pass: {{ $pass | b64enc }}
- {{/* Encoded version of the password for nginx ingress */}}
- auth: {{ htpasswd "admin" $pass | b64enc }}
diff --git a/charts/oas-secrets/templates/oas-kube-prometheus-stack-variables.yaml b/charts/oas-secrets/templates/oas-kube-prometheus-stack-variables.yaml
deleted file mode 100644
index 0781a9b19205a9b44d26d3d5080e04e3eefd9728..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/oas-kube-prometheus-stack-variables.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
- name: oas-kube-prometheus-stack-variables
-data:
- grafana_admin_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-kube-prometheus-stack-variables" "Key" "grafana_admin_password") }}"
diff --git a/charts/oas-secrets/templates/oas-nextcloud-variables.yaml b/charts/oas-secrets/templates/oas-nextcloud-variables.yaml
deleted file mode 100644
index 017cc80830ec4f554aa37586bb5d3149b13eda32..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/oas-nextcloud-variables.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
- name: oas-nextcloud-variables
-data:
- nextcloud_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-nextcloud-variables" "Key" "nextcloud_password") }}"
- nextcloud_mariadb_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-nextcloud-variables" "Key" "nextcloud_mariadb_password") }}"
- nextcloud_mariadb_root_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-nextcloud-variables" "Key" "nextcloud_mariadb_root_password") }}"
- onlyoffice_jwt_secret: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-nextcloud-variables" "Key" "onlyoffice_jwt_secret") }}"
- onlyoffice_postgresql_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-nextcloud-variables" "Key" "onlyoffice_postgresql_password") }}"
- onlyoffice_rabbitmq_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-nextcloud-variables" "Key" "onlyoffice_rabbitmq_password") }}"
diff --git a/charts/oas-secrets/templates/oas-oauth-variables.yaml b/charts/oas-secrets/templates/oas-oauth-variables.yaml
deleted file mode 100644
index 4f4f67a8d836957af71e50443157b9e48329a915..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/oas-oauth-variables.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
- name: oas-oauth-variables
-data:
- grafana_oauth_client_secret: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-oauth-variables" "Key" "grafana_oauth_client_secret") }}"
- nextcloud_oauth_client_secret: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-oauth-variables" "Key" "nextcloud_oauth_client_secret") }}"
- rocketchat_oauth_client_secret: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-oauth-variables" "Key" "rocketchat_oauth_client_secret") }}"
- userpanel_oauth_client_secret: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-oauth-variables" "Key" "userpanel_oauth_client_secret") }}"
- wekan_oauth_client_secret: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-oauth-variables" "Key" "wekan_oauth_client_secret") }}"
- wordpress_oauth_client_secret: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-oauth-variables" "Key" "wordpress_oauth_client_secret") }}"
diff --git a/charts/oas-secrets/templates/oas-prometheus-basic-auth.yaml b/charts/oas-secrets/templates/oas-prometheus-basic-auth.yaml
deleted file mode 100644
index f094270a286fc3345cd2b1f71501d573422c2ead..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/oas-prometheus-basic-auth.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
- namespace: "oas"
- name: "oas-prometheus-basic-auth"
-data:
- {{- $pass := (include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-prometheus-basic-auth" "Key" "pass")) | b64dec -}}
- {{/* Readable version of the password for humans who want to log in */}}
- pass: {{ $pass | b64enc }}
- {{/* Encoded version of the password for nginx ingress */}}
- auth: {{ htpasswd "admin" $pass | b64enc }}
diff --git a/charts/oas-secrets/templates/oas-rocketchat-variables.yaml b/charts/oas-secrets/templates/oas-rocketchat-variables.yaml
deleted file mode 100644
index 747ed200b009ee6bd48b7de162265a633c8cdb11..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/oas-rocketchat-variables.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
- name: oas-rocketchat-variables
-data:
- mongodb_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-rocketchat-variables" "Key" "mongodb_password") }}"
- mongodb_root_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-rocketchat-variables" "Key" "mongodb_root_password") }}"
- rocketchat_admin_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-rocketchat-variables" "Key" "rocketchat_admin_password") }}"
diff --git a/charts/oas-secrets/templates/oas-single-sign-on-variables.yaml b/charts/oas-secrets/templates/oas-single-sign-on-variables.yaml
deleted file mode 100644
index 55b29a84bd5a0ac36be99d083d9e5905dbaddcff..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/oas-single-sign-on-variables.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
- name: oas-single-sign-on-variables
-data:
- userbackend_admin_username: {{ b64enc "admin" }}
- userbackend_admin_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-single-sign-on-variables" "Key" "userbackend_admin_password") }}"
- userbackend_postgres_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-single-sign-on-variables" "Key" "userbackend_postgres_password") }}"
- hydra_system_secret: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-single-sign-on-variables" "Key" "hydra_system_secret") }}"
diff --git a/charts/oas-secrets/templates/oas-wekan-variables.yaml b/charts/oas-secrets/templates/oas-wekan-variables.yaml
deleted file mode 100644
index 1f0ffccc1d3acdf7200e7d1daa47a8465f3a5d70..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/oas-wekan-variables.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
- name: oas-wekan-variables
-data:
- mongodb_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-wekan-variables" "Key" "mongodb_password") }}"
- mongodb_root_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-wekan-variables" "Key" "mongodb_root_password") }}"
diff --git a/charts/oas-secrets/templates/oas-wordpress-variables.yaml b/charts/oas-secrets/templates/oas-wordpress-variables.yaml
deleted file mode 100644
index 37df564996fa95d8e0610f4a11711676a3cf0f21..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/templates/oas-wordpress-variables.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
- name: oas-wordpress-variables
-data:
- wordpress_admin_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-wordpress-variables" "Key" "wordpress_admin_password") }}"
- wordpress_mariadb_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-wordpress-variables" "Key" "wordpress_mariadb_password") }}"
- wordpress_mariadb_root_password: "{{ include "getOrGeneratePass" (dict "Namespace" .Release.Namespace "Kind" "Secret" "Name" "oas-wordpress-variables" "Key" "wordpress_mariadb_root_password") }}"
diff --git a/charts/oas-secrets/values.yaml b/charts/oas-secrets/values.yaml
deleted file mode 100644
index 449682fa418728aacfada08e962f59a297aca5c2..0000000000000000000000000000000000000000
--- a/charts/oas-secrets/values.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-# The domain OpenAppStack runs on. There needs to be an A record to
-# oas.example.com and an additional record *.oas.example.com, that both point to
-# this machine
-domain: oas.example.com
-# The email address of the cluster administrator
-admin_email: admin@oas.example.com
\ No newline at end of file
diff --git a/docs/upgrading.rst b/docs/upgrading.rst
index 17bc83dcd4a630615ab2a077b2d462c83699ac62..26d2da8c0283d68bc10997286d1540d7fcf6f61b 100644
--- a/docs/upgrading.rst
+++ b/docs/upgrading.rst
@@ -1,6 +1,154 @@
Upgrading OpenAppStack
======================
+Upgrading to 0.7.0
+------------------
+
+Because of `problems with Helm and secret management
+<https://open.greenhost.net/openappstack/openappstack/-/issues/891>`__
+we had to move away from using a helm chart for secrets, and now use scripts
+that run during installation to manage secrets. Because we have removed the
+``oas-secrets`` helm chart, Flux will remove the secrets that it has
+generated. **It is important that you back up these secrets before switching
+from ``v0.6`` to ``v0.7``!**
+
+Before you start, please ensure that you have the right ``yq`` tool installed,
+because you will need it later. There are two very different versions of
+``yq``. The one you need is the go based [yq from Mike
+Farah](http://mikefarah.github.io/yq/), which installs the same binary name ``yq``
+as the [python-yq](https://github.com/kislyuk/yq), while both have different
+command sets.
+
+The yq needed here can be installed by running ``sudo snap install yq``,
+``brew install yq`` or with other methods from the [``yq`` installation
+instructions](http://mikefarah.github.io/yq/#install).
+
+
+To back-up your secrets, run the following script:
+
+.. code:: bash
+
+ bash
+ #!/usr/bin/env bash
+
+ mkdir secrets-backup
+
+ kubectl get secret -o yaml -n flux-system oas-cluster-variables > secrets-backup/oas-cluster-variables.yaml
+ kubectl get secret -o yaml -n flux-system oas-wordpress-variables > secrets-backup/oas-wordpress-variables.yaml
+ kubectl get secret -o yaml -n flux-system oas-wekan-variables > secrets-backup/oas-wekan-variables.yaml
+ kubectl get secret -o yaml -n flux-system oas-single-sign-on-variables > secrets-backup/oas-single-sign-on-variables.yaml
+ kubectl get secret -o yaml -n flux-system oas-rocketchat-variables > secrets-backup/oas-rocketchat-variables.yaml
+ kubectl get secret -o yaml -n flux-system oas-kube-prometheus-stack-variables > secrets-backup/oas-kube-prometheus-stack-variables.yaml
+ kubectl get secret -o yaml -n oas oas-prometheus-basic-auth > secrets-backup/oas-prometheus-basic-auth.yaml
+ kubectl get secret -o yaml -n oas oas-alertmanager-basic-auth > secrets-backup/oas-alertmanager-basic-auth.yaml
+ kubectl get secret -o yaml -n flux-system oas-oauth-variables > secrets-backup/oas-oauth-variables.yaml
+ kubectl get secret -o yaml -n flux-system oas-nextcloud-variables > secrets-backup/oas-nextcloud-variables.yaml
+
+This script assumes you have all applications enabled. You might get an error
+like:
+
+.. code:: bash
+
+ Error from server (NotFound): secrets "oas-wekan-variables" not found
+
+This is not a problem, but it *does* mean you need to add an oauth secret for
+Wekan to the file ``secrets-backup/oas-oauth-variables.yaml``. Copy one of the
+lines under "data:", rename the field to ``wekan_oauth_client_secret`` and enter
+a different random password. Make sure to base64 encode it (``echo "<your random
+password>" | base64``).
+
+If you get the error several times, that means ther
+
+This script creates a directory called ``secrets-backup`` and places the secrets
+that have been generated by Helm in it as ``yaml`` files.
+
+Now you can upgrade your cluster by running ``kubectl edit gitrepository -n
+flux-system openappstack`` and setting ``spec.ref.branch`` to ``v0.7``
+
+Flux will now start updating your cluster to version 0.7. This process will fail,
+because it will remove the secrets that you just backed up. Make
+sure that the ``oas-secrets`` helmrelease has been removed by running ``flux get
+hr -A``. You might also see that some helmreleases start failing to be installed
+because important secrets do not exist anymore.
+
+As soon as the ``oas-secrets`` helmrelease does not exist anymore, you can run
+the following code:
+
+.. code:: bash
+
+ #!/usr/bin/env bash
+
+ # Again: make sure you use https://github.com/mikefarah/yq -- install with `snap install yq`
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-cluster-variables.yaml | kubectl apply -f - -n flux-system
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-wordpress-variables.yaml | kubectl apply -f - -n flux-system
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-wekan-variables.yaml | kubectl apply -f - -n flux-system
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-single-sign-on-variables.yaml | kubectl apply -f - -n flux-system
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-rocketchat-variables.yaml | kubectl apply -f - -n flux-system
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-kube-prometheus-stack-variables.yaml | kubectl apply -f - -n flux-system
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-prometheus-basic-auth.yaml | kubectl apply -f - -n flux-system
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-alertmanager-basic-auth.yaml | kubectl apply -f - -n flux-system
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-oauth-variables.yaml | kubectl apply -f - -n flux-system
+ yq eval 'del(.metadata.annotations,.metadata.labels,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid)' secrets-backup/oas-nextcloud-variables.yaml | kubectl apply -f - -n flux-system
+
+Again this script assumes you have all applications installed. If you get the
+following error, you can ignore it:
+
+.. code:: bash
+
+ error: error validating "STDIN": error validating data: [apiVersion not set, kind not set]; if you choose to ignore these errors, turn validation off with --validate=false
+
+Now Flux should succeed in finishing the update. Some helmreleases or
+kustomizations might have already failed because the secrets did not exist. Once
+failed, you can retrigger reconciliation of a kustomization using the commands
+``flux reconcile kustomization ...`` or ``flux reconcile helmrelease ...``. This
+can take quite a while (over an hour some times), because Flux waits for some
+long timeouts before giving up and re-starting a reconciliation.
+
+Some errors we've seen during our own upgrade process, and how to solve them
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+SSO helm upgrade failed
+'''''''''''''''''''''''
+
+.. code::
+
+ oas single-sign-on False Helm upgrade failed: template: single-sign-on/templates/secret-oauth2-clients.yaml:9:55: executing "single-sign-on/templates/secret-oauth2-clients.yaml" at <b64enc>: invalid value; expected string 0.2.2 False
+
+This means that the ``single-sign-on`` helmrelease was created with empty oauth
+secrets. The secrets will get a value once the ``core`` *kustomization* is
+reconciled: ``flux reconcile ks core`` should solve the problem.
+
+If that does not solve the problem, you should check if the secret contains a
+value for all the apps:
+
+.. code::
+
+ # kubectl get secret -n flux-system oas-oauth-variables -o yaml
+ apiVersion: v1
+ data:
+ grafana_oauth_client_secret: <redacted>
+ nextcloud_oauth_client_secret: <redacted>
+ rocketchat_oauth_client_secret: <redacted>
+ userpanel_oauth_client_secret: <redacted>
+ wekan_oauth_client_secret: <redacted>
+ wordpress_oauth_client_secret: <redacted>
+ ...
+
+If your secret lacks one of these variables, use ``kubectl edit`` to add them.
+You can use any password generator to generate a password for it. Make sure to
+base64 encode the data before you enter it in the secret.
+
+Loki upgrade retries exhausted
+''''''''''''''''''''''''''''''
+
+While running ``flux get helmrelease -A``, you'll see:
+
+.. code::
+ oas loki False upgrade retries exhausted 2.5.2 False
+
+This happens sometimes because Loki takes a long time to upgrade. Usually it is
+solved by running ``flux reconcile hr loki -n oas`` again.
+
Upgrading to 0.6.0
------------------
@@ -19,7 +167,7 @@ production.
Upgrading from 0.4.0 to 0.5.0
-----------------------------
-Unfortunatly we can’t ensure a smooth upgrade for this version neither.
+Unfortunately we can’t ensure a smooth upgrade for this version neither.
Please read the section below on how to do an upgrade by installing a
the new OAS version from scratch after backing up your data.
diff --git a/flux2/README.md b/flux2/README.md
index c6f022574531811150d4222006d6bd6296861c49..6c7fd51a0933999802c3b7ad46b0d796375a64a7 100644
--- a/flux2/README.md
+++ b/flux2/README.md
@@ -33,6 +33,5 @@ flux2
├── local-path-provisioner # (default) storage class to safe files on disk
├── namespaces # namespaces used by OAS
├── nginx # Ingress
- ├── secrets # Auto-generated secrets for applications
└── sources # Helm repositories needed for HelmReleases from other folders
```
diff --git a/flux2/cluster/base/infrastructure.yaml b/flux2/cluster/base/infrastructure.yaml
index 5de7ecd6ee7b75e5f19cb89006e4e13bf118bf08..53c949da1bfd9a140c56c273b6a20f21bbefd3be 100644
--- a/flux2/cluster/base/infrastructure.yaml
+++ b/flux2/cluster/base/infrastructure.yaml
@@ -21,7 +21,3 @@ spec:
kind: HelmRelease
name: local-path-provisioner
namespace: kube-system
- - apiVersion: helm.toolkit.fluxcd.io/v1beta1
- kind: HelmRelease
- name: oas-secrets
- namespace: flux-system
diff --git a/flux2/infrastructure/secrets/kustomization.yaml b/flux2/infrastructure/secrets/kustomization.yaml
deleted file mode 100644
index c6fd78466ae94c2384b8537ba6d4edc538326709..0000000000000000000000000000000000000000
--- a/flux2/infrastructure/secrets/kustomization.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# The secrets will be used by Flux in the postBuild Kustomization step and need
-# to be in the flux-system namespace
-namespace: flux-system
-resources:
- - release.yaml
\ No newline at end of file
diff --git a/flux2/infrastructure/secrets/release.yaml b/flux2/infrastructure/secrets/release.yaml
deleted file mode 100644
index 4073c1768bdba6a4e64880fbe108d1e15e4dd62d..0000000000000000000000000000000000000000
--- a/flux2/infrastructure/secrets/release.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
- name: oas-secrets
-spec:
- releaseName: oas-secrets
- chart:
- spec:
- chart: ./charts/oas-secrets/
- # NOTE: Change the GitRepository yaml file if you want a different version
- sourceRef:
- kind: GitRepository
- name: openappstack
- namespace: flux-system
- interval: 1h0m0s
- # Allow custom values either by configMap or by secret
- valuesFrom:
- - kind: ConfigMap
- name: oas-secrets-override
- optional: true
- - kind: Secret
- name: oas-secrets-override
- optional: true
diff --git a/install/generate_secrets.py b/install/generate_secrets.py
new file mode 100644
index 0000000000000000000000000000000000000000..93f5702c779a42061d9b3e15b70d33a1310adf69
--- /dev/null
+++ b/install/generate_secrets.py
@@ -0,0 +1,148 @@
+"""
+Generates Kubernetes secrets based on a provided app name.
+
+If the `templates` directory contains a secret called `oas-{app}-variables`, it
+will check if that secret already exists in the cluster, and if not: generate
+it. It does the same for an `oas-{app}-basic-auth` secret that will contain a
+password as well as a htpasswd encoded version of it.
+
+See https://open.greenhost.net/openappstack/openappstack/-/issues/891 for the
+context why we use this script and not a helm chart to generate secrets.
+
+usage: python generate_secrets.py template_filename.j2
+"""
+
+import crypt
+import os
+import secrets
+import string
+import sys
+import yaml
+
+import jinja2
+import jinja2_base64_filters # pylint: disable=unused-import
+from kubernetes import client, config
+from kubernetes.client.exceptions import ApiException
+from kubernetes.utils import create_from_yaml
+
+
+def main():
+ """Run everything"""
+ # Add jinja filters we want to use
+ env = jinja2.Environment(extensions=["jinja2_base64_filters.Base64Filters"])
+ env.filters["generate_password"] = generate_password
+
+ if len(sys.argv) < 2:
+ print("Please provide an app name as an argument")
+ sys.exit(1)
+ app_name = sys.argv[1]
+
+ create_variables_secret(app_name, env)
+ create_basic_auth_secret(app_name, env)
+
+
+def get_templates_dir():
+ """Returns directory that contains the Jinja templates used to create app
+ secrets"""
+ return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'templates')
+
+
+def create_variables_secret(app_name, env):
+ """Checks if a variables secret for app_name already exists, generates it if necessary"""
+ variables_filename = \
+ os.path.join(get_templates_dir(), f"oas-{app_name}-variables.yaml.jinja")
+ if os.path.exists(variables_filename):
+ # Check if k8s secret already exists, if not, generate it
+ with open(variables_filename) as template_file:
+ lines = template_file.read()
+ secret_name, secret_namespace = get_secret_metadata(lines)
+ if get_kubernetes_secret(secret_name, secret_namespace) is None:
+ print(f"Adding secret {secret_name} in namespace"
+ f" {secret_namespace} to cluster.")
+ template = env.from_string(lines)
+ store_kubernetes_secret(template.render(), secret_namespace)
+ else:
+ print(f"Secret {secret_name} in namespace {secret_namespace}"
+ " already exists. Not generating new secrets.")
+
+
+def create_basic_auth_secret(app_name, env):
+ """Checks if a basic auth secret for app_name already exists, generates it if necessary"""
+ basic_auth_filename = \
+ os.path.join(get_templates_dir(), f"oas-{app_name}-basic-auth.yaml.jinja")
+ if os.path.exists(basic_auth_filename):
+ with open(basic_auth_filename) as template_file:
+ lines = template_file.read()
+ secret_name, secret_namespace = get_secret_metadata(lines)
+
+ if get_kubernetes_secret(secret_name, secret_namespace) is None:
+ basic_auth_username = 'admin'
+ basic_auth_password = generate_password(32)
+ basic_auth_htpasswd = gen_htpasswd(
+ basic_auth_username,
+ basic_auth_password)
+ print(f"Adding secret {secret_name} in namespace"
+ f" {secret_namespace} to cluster.")
+ template = env.from_string(
+ lines,
+ globals={
+ 'pass': basic_auth_password,
+ 'htpasswd': basic_auth_htpasswd
+ })
+ store_kubernetes_secret(template.render(), secret_namespace)
+ else:
+ print(f"Secret {secret_name} in namespace {secret_namespace}"
+ " already exists. Not generating new secrets.")
+
+def get_secret_metadata(yaml_string):
+ """Returns secret name and namespace from metadata field in a yaml string"""
+ secret_info = yaml.safe_load(yaml_string)
+ secret_name = secret_info['metadata']['name']
+ # default namespace is flux-system, but other namespace can be
+ # provided in secret metadata
+ if 'namespace' in secret_info['metadata']:
+ secret_namespace = secret_info['metadata']['namespace']
+ else:
+ secret_namespace = 'flux-system'
+ return secret_name, secret_namespace
+
+
+def get_kubernetes_secret(secret_name, namespace):
+ """Returns the contents of a kubernetes secret or None if the secret does not exist."""
+ try:
+ secret = API.read_namespaced_secret(secret_name, namespace).data
+ except ApiException as ex:
+ # 404 is expected when the optional secret does not exist.
+ if ex.status != 404:
+ raise ex
+ return None
+ return secret
+
+def store_kubernetes_secret(secret_string, namespace):
+ """Converts secret_string into a yaml object and adds it to the cluster"""
+ secret_yaml = yaml.safe_load(secret_string)
+ api_client = client.api_client.ApiClient()
+ api_response = create_from_yaml(
+ api_client,
+ yaml_objects=[secret_yaml],
+ namespace=namespace)
+ print(f"Secret created with api response: {api_response}")
+
+def generate_password(length):
+ """Generates a password of "length" characters"""
+ length = int(length)
+ password = ''.join((secrets.choice(string.ascii_letters) for i in range(length)))
+ return password
+
+
+def gen_htpasswd(user, password):
+ """generate htpasswd entry for user with password"""
+ return "{}:{}".format(user, crypt.crypt(
+ password, crypt.mksalt(crypt.METHOD_SHA512)
+ ),
+ )
+
+if __name__ == "__main__":
+ config.load_kube_config()
+ API = client.CoreV1Api()
+ main()
diff --git a/install/install-app.sh b/install/install-app.sh
index a3f7b47cfc36ab8e3ef343540a3bc2f33807f3ff..bd9d58a7ad03ed01133eded5ba2fdcf3cf55fde7 100755
--- a/install/install-app.sh
+++ b/install/install-app.sh
@@ -7,6 +7,10 @@ app=$1
# shellcheck disable=SC1090
. "$(dirname "$0")/flux-version-check.sh"
+# Check if the secrets for the app already exist in the cluster, and if not,
+# generate them and add them to the cluster.
+python "$(dirname "$0")/generate_secrets.py" $app
+
# This kustomization's only purpose is to add the kustomization that is in the
# flxu2/cluster/optional/$app folder. After this kustomization is applied
# an `add-$app` kustomization will be present on the cluster, as well as a
diff --git a/install/install-openappstack.sh b/install/install-openappstack.sh
index ee2fb2e44e7281cf1ba08d4e162a2d0ced65d80c..2056236eec5dacedb769925f68fa4bac77987217 100755
--- a/install/install-openappstack.sh
+++ b/install/install-openappstack.sh
@@ -15,6 +15,18 @@ branch=${CI_COMMIT_REF_NAME:-}
echo "Tracking branch $branch for https://open.greenhost.net/openappstack/openappstack flux repo"
+# Create oas and oas-apps namespaces
+kubectl create namespace oas
+kubectl create namespace oas-apps
+
+# Generate oauth and SSO secrets
+python "$(dirname "$0")/generate_secrets.py" single-sign-on
+python "$(dirname "$0")/generate_secrets.py" oauth
+
+# Generate secrets for monitoring
+python "$(dirname "$0")/generate_secrets.py" kube-prometheus-stack
+python "$(dirname "$0")/generate_secrets.py" alertmanager
+
flux create source git openappstack \
--url=https://open.greenhost.net/openappstack/openappstack \
--branch=$branch \
diff --git a/install/templates/oas-alertmanager-basic-auth.yaml.jinja b/install/templates/oas-alertmanager-basic-auth.yaml.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..fedb272ab91f0794367040944c361f9bccd94690
--- /dev/null
+++ b/install/templates/oas-alertmanager-basic-auth.yaml.jinja
@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ namespace: "oas"
+ name: "oas-alertmanager-basic-auth"
+data:
+ pass: "{{ pass | b64encode }}"
+ auth: "{{ htpasswd | b64encode }}"
diff --git a/install/templates/oas-kube-prometheus-stack-variables.yaml.jinja b/install/templates/oas-kube-prometheus-stack-variables.yaml.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..035baf54b938c396a8012ca9e36625c26e73750c
--- /dev/null
+++ b/install/templates/oas-kube-prometheus-stack-variables.yaml.jinja
@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: oas-kube-prometheus-stack-variables
+data:
+ grafana_admin_password: "{{ 32 | generate_password | b64encode }}"
diff --git a/install/templates/oas-nextcloud-variables.yaml.jinja b/install/templates/oas-nextcloud-variables.yaml.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..6ef7809d889c53c9761eeb1b0c771301653b2cb0
--- /dev/null
+++ b/install/templates/oas-nextcloud-variables.yaml.jinja
@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: oas-nextcloud-variables
+data:
+ nextcloud_password: "{{ 32 | generate_password | b64encode }}"
+ nextcloud_mariadb_password: "{{ 32 | generate_password | b64encode }}"
+ nextcloud_mariadb_root_password: "{{ 32 | generate_password | b64encode }}"
+ onlyoffice_jwt_secret: "{{ 32 | generate_password | b64encode }}"
+ onlyoffice_postgresql_password: "{{ 32 | generate_password | b64encode }}"
+ onlyoffice_rabbitmq_password: "{{ 32 | generate_password | b64encode }}"
diff --git a/install/templates/oas-oauth-variables.yaml.jinja b/install/templates/oas-oauth-variables.yaml.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..befd2f774e40402a86caf20ea51c7173342165f7
--- /dev/null
+++ b/install/templates/oas-oauth-variables.yaml.jinja
@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: oas-oauth-variables
+data:
+ grafana_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"
+ nextcloud_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"
+ rocketchat_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"
+ userpanel_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"
+ wekan_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"
+ wordpress_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"
diff --git a/install/templates/oas-prometheus-basic-auth.yaml b/install/templates/oas-prometheus-basic-auth.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d4d2d35521890ed6a3f53951b8a77070a1b8d2f6
--- /dev/null
+++ b/install/templates/oas-prometheus-basic-auth.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ namespace: "oas"
+ name: "oas-prometheus-basic-auth"
+data:
+ # Readable version of the password for humans who want to log in
+ pass: "{{ pass | b64encode }}"
+ # Encoded version of the password for nginx ingress
+ auth: "{{ htpasswd | b64encode }}"
diff --git a/install/templates/oas-rocketchat-variables.yaml.jinja b/install/templates/oas-rocketchat-variables.yaml.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..4bce7f2911c2375e567b9a16333e89f7ff0c4453
--- /dev/null
+++ b/install/templates/oas-rocketchat-variables.yaml.jinja
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ name: oas-rocketchat-variables
+data:
+ mongodb_password: "{{ 32 | generate_password | b64encode }}"
+ mongodb_root_password: "{{ 32 | generate_password | b64encode }}"
+ rocketchat_admin_password: "{{ 32 | generate_password | b64encode }}"
diff --git a/install/templates/oas-single-sign-on-variables.yaml.jinja b/install/templates/oas-single-sign-on-variables.yaml.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..7424dd4ed4eb672b4030ff632b560f2b19852e80
--- /dev/null
+++ b/install/templates/oas-single-sign-on-variables.yaml.jinja
@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: oas-single-sign-on-variables
+data:
+ userbackend_admin_username: '{{ "admin" | b64encode }}'
+ userbackend_admin_password: "{{ 32 | generate_password | b64encode }}"
+ userbackend_postgres_password: "{{ 32 | generate_password | b64encode }}"
+ hydra_system_secret: "{{ 32 | generate_password | b64encode }}"
diff --git a/install/templates/oas-wekan-variables.yaml.jinja b/install/templates/oas-wekan-variables.yaml.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..f3cc5a366cd1b39a18a41903a43f3cff7dee61e0
--- /dev/null
+++ b/install/templates/oas-wekan-variables.yaml.jinja
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ name: oas-wekan-variables
+data:
+ mongodb_password: "{{ 32 | generate_password | b64encode }}"
+ mongodb_root_password: "{{ 32 | generate_password | b64encode }}"
diff --git a/install/templates/oas-wordpress-variables.yaml.jinja b/install/templates/oas-wordpress-variables.yaml.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..18bf4339ca9afa86d1583f9a6ab0716a578fa00d
--- /dev/null
+++ b/install/templates/oas-wordpress-variables.yaml.jinja
@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: oas-wordpress-variables
+data:
+ wordpress_admin_password: "{{ 32 | generate_password | b64encode }}"
+ wordpress_mariadb_password: "{{ 32 | generate_password | b64encode }}"
+ wordpress_mariadb_root_password: "{{ 32 | generate_password | b64encode }}"
diff --git a/requirements.in b/requirements.in
index 6708866660e6ef43d0948e81b7c06ec841e23271..cf4e84c5723c94bd8f6df12a9eabffd59baf573a 100644
--- a/requirements.in
+++ b/requirements.in
@@ -15,7 +15,12 @@
ansible<2.10
# needed for test_dns.py
dnspython
-kubernetes
+# `install/generate_secrets.py` needs the `create_from_yaml` function that accepts
+# the `yaml_objects` parameter which was added in commit
+# 13814c0f7e0e587c46512386da3d08c64fc83e04. Until that is merged into a stable
+# release, we install from Git. Should be replaced with `kubernetes>18` some time
+# in the future.
+-e git+git://github.com/kubernetes-client/python.git@2f9643eac71535f7802cd16a078ce50e8866f1ad#egg=kubernetes
# Needed for testinfra using the ansible module
paramiko
psutil
@@ -28,3 +33,6 @@ setuptools
wheel
-e git+https://open.greenhost.net/greenhost/cloud-api#egg=greenhost_cloud
passlib
+# Needed for secrets generation in install/generate_secrets.py
+jinja2-base64-filters
+htpasswd
diff --git a/requirements.txt b/requirements.txt
index b56d665d4b1d51b96755cbc494d8ad2cb4a36997..036e5ae92ed16a265dbd7734c2bb448cdefb5001 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,9 @@
#
-e git+https://open.greenhost.net/greenhost/cloud-api#egg=greenhost_cloud
# via -r requirements.in
-ansible==2.9.23
+-e git+git://github.com/kubernetes-client/python.git@2f9643eac71535f7802cd16a078ce50e8866f1ad#egg=kubernetes
+ # via -r requirements.in
+ansible==2.9.24
# via -r requirements.in
attrs==21.2.0
# via pytest
@@ -23,7 +25,7 @@ cffi==1.14.6
# bcrypt
# cryptography
# pynacl
-charset-normalizer==2.0.1
+charset-normalizer==2.0.4
# via requests
cryptography==3.4.7
# via
@@ -32,20 +34,26 @@ cryptography==3.4.7
# pyopenssl
dnspython==2.1.0
# via -r requirements.in
-google-auth==1.32.1
+google-auth==1.34.0
# via kubernetes
+htpasswd==2.3
+ # via -r requirements.in
idna==3.2
# via requests
iniconfig==1.1.1
# via pytest
jinja2==3.0.1
- # via ansible
-kubernetes==17.17.0
+ # via
+ # ansible
+ # jinja2-base64-filters
+jinja2-base64-filters==0.1.4
# via -r requirements.in
markupsafe==2.0.1
# via jinja2
oauthlib==3.1.1
# via requests-oauthlib
+orderedmultidict==1.0.1
+ # via htpasswd
packaging==21.0
# via pytest
paramiko==2.7.2
@@ -102,6 +110,7 @@ six==1.16.0
# bcrypt
# google-auth
# kubernetes
+ # orderedmultidict
# pynacl
# pyopenssl
# python-dateutil
@@ -115,9 +124,9 @@ urllib3==1.26.6
# via
# kubernetes
# requests
-websocket-client==1.1.0
+websocket-client==1.2.0
# via kubernetes
-wheel==0.36.2
+wheel==0.37.0
# via -r requirements.in
# The following packages are considered to be unsafe in a requirements file: