Merge branch 'master' into '207-use-proper-release-versions'

# Conflicts:
#   VERSION

.gitignore

@@ -26,3 +26,6 @@
 # Etc
 __pycache__
 *.swp
+
+# Documentation files
+/docs/_build

.gitlab-ci.yml

@@ -45,6 +45,11 @@ bootstrap:
     - test/group_vars/all/settings.yml
     expire_in: 1 month
     when: always
+  only:
+    changes:
+      - ansible
+      - helmfiles
+      - test
 
 install:
   stage: install-apps
@@ -64,6 +69,11 @@ install:
     paths:
     - test/behave/behave.ini
     expire_in: 1 month
+  only:
+    changes:
+      - ansible
+      - helmfiles
+      - test
 
 testinfra:
   stage: health-test
@@ -72,6 +82,11 @@ testinfra:
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
     - cd test/
     - py.test -v -m 'testinfra' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
+  only:
+    changes:
+      - ansible
+      - helmfiles
+      - test
 
 certs:
   stage: health-test
@@ -83,6 +98,11 @@ certs:
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
     - cd test/
     - py.test -s -m 'certs' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
+  only:
+    changes:
+      - ansible
+      - helmfiles
+      - test
 
 behave-nextcloud:
   stage: integration-test
@@ -99,6 +119,11 @@ behave-nextcloud:
     expire_in: 1 month
     when: on_failure
   retry: 2
+  only:
+    changes:
+      - ansible
+      - helmfiles
+      - test
 
 behave-grafana:
   stage: integration-test
@@ -114,6 +139,11 @@ behave-grafana:
     - test/behave/screenshots/
     expire_in: 1 month
     when: on_failure
+  only:
+    changes:
+      - ansible
+      - helmfiles
+      - test
 
 terminate:
   stage: cleanup
@@ -123,3 +153,8 @@ terminate:
     - echo "$CI_COMMIT_MESSAGE" | grep '!ci_dont_terminate' && echo 'Termination of droplet disabled in commit message.' || python3 -u ./ci-bootstrap.py --use-existing-inventory --terminate
     # Remove droplet older than 2 days
     - python3 -c "import cosmos; cosmos.terminate_droplets_by_name(\"^ci-\", 2)"
+  only:
+    changes:
+      - ansible
+      - helmfiles
+      - test

docs/Makefile

+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.md

+## Documentation
+
+This folder contains the documentation. You can find the documentation at
+https://openappstack.readthedocs.io/. It is also possible to build the
+documentation, using Sphinx:
+
+```
+pip install sphinx
+make html
+```

docs/_static/diagrams/application-build-process.xml

+<mxfile userAgent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/69.0.3497.81 Chrome/69.0.3497.81 Safari/537.36" version="9.1.8" editor="www.draw.io" type="device"><diagram name="Page-1" id="9f46799a-70d6-7492-0946-bef42562c5a5">7Vxtc6O2Fv41nrn9UA/vLx/jZLftNHe6s2m3vR9lkDFdGTEgx0l/fY9AwmAJm8TYyd3Fk9mFYwnB0XOOnkeSmdm3m6efCpSv/0tjTGaWET/N7LuZZZmO5cF/3PJcWwInrA1Jkcai0N7wkP6DhdEQ1m0a47JTkFFKWJp3jRHNMhyxjg0VBd11i60o6baaowQrhocIEdX6ZxqzdW31DWNv/xmnyZodfLFE0dekoNtMNDez7FX1qb/eIHkpUb5co5juWib7w8y+LShl9dHm6RYT7lrptbrex55vm9sucMaGVPCDusYjIlvx6L9ul7jIMAP3g3/JtmS4mFkegestlvwo4Ufi9tmz9Fi5SzcEZXC2WLMNAaMJhyuasQdRiJ8jkiYZHEdwf3BZe/GIC5aC32/EF4zmYI3WKYnv0TPd8qcoGXhVni3WtEj/gcsi2QZ8XTABIcBdu8QDrwlmA6wFLqHMJ+kaszHdo5KJMhElBOVlumxueIOKJM0WlDG6EYXkk35MCbmlhBaVA2Q/w1V59+NYlpY9XF9/k0bimKAlJosGL/JKGa18WLKCfsWty8d+uDSM5huJSos7+fBGsBdFwvkf0SYlPAi/4CJGGZJ9UrvLtMR5q7pRfVrPeZ9m0hsJQWUp7l8AB/oPP/Wiz2wwDakC0w1mxTMUERUcy5m7dSWRJxxPAHK3jzrLtmvbuhVxliEKIhHpSXP1psHPkBlQlkBf7lv0jU57tm8r7QWGpjmv2xoiAN8MMbzgfVe2YwwOWs+6N1WRp49C11Ki8CbPCcQFS2kGXzzg4jGN8BSGbx6Gy8B13GFhGET4amHYjrkmpx+GphqHz110nwo603BeE3RhN+aCdxFythJyP8Lp523Ghz3Ujr5bXo1b47TMaYk4Kk8HIriedYOwiyUBsDZohEkGJ8Er1huaZY6iNEvuqzJ3zt7yWXiQm3brlOEHsPN72gFTAxuF661IFQrrNI5xVoUKQwzV0cYxldM0Y5Xb3QX8QefcGpCo3TvuDHdh7s/hjxcvALkZPB9KKxBiCOUd5uGsCRMVsE36Gx2whxhq47UDnmNIcbRIgQdepcm2gAzD+SUAwdiWg5jShIwLIkPWuAYyArcPGVyTVMBgFP6hbM2BwcUKAz/gApLJx5LRgouQCS9n4uUEA60B0YsXfyjfHCOTqGLrlywB9lNOxG4idpcgdqZtDIO3/yo19R6JXWBqk/Kngj5WU0qW0ZVWPIvx6jxeMsymfHzpfBy8G2YX+FqoQDx+VUSAAdjY0eLrkFQ9AeSCALkmwXNMpW+7g0+ri//GjD2LtI+2wPvA3wVb04RmiNxT3nNVOXjO4vkvXn9uGbY0/K8yuIYnDZ9wkcL98jG8agnHCZYDPSZLuvuwN2g8jbP4hk+DcyrAxxQ+KnIjH1MldeiOatiMXezrRsbQ823kaUck6R+6LSIsR5LQM003XBkeDgMc/GiKKAMWkWBRTw5L/KmOjmMFJhCAj91Z+XO6tPf2eufexMNNDO37Y2i9gB/AwMS3ln8ww22rgsP0NCzJdIxvhZIpMWfpOdodziFF4SxKcTkNqq+fNenPcedMsGlROsYoq8JDPzcrZ9zqxGzf6FKyhza8U7JlmVfnRsX9wZ9a3qYtzddaKkmwpiXL0Eab+rU1u+NG1Lnd4a1HBeZ3a/wnneN5PfucwwgO5DMuf9BeZ+Kj7yl0NARVGzrBEck9NHQ8dUZJ6f02c9wz0jZ9jLaA+Fgygj6Cq1LK39dpptDKoTz45CA+FiG1VPrpuGPTT1H1E0frfjC2A+9wfds/WEGrb0vUO2uUtRQoKPmonn2eqOx3SGVfPNn4cqZrHWBdmwl9HdW1v5nZRz1z6RJbI4XxCRHCF4imsXu8sdsajNgjJFeHzzFIrqcFxoOYW5gg8U4gMTRljbIcqIXEF1Sk9RaT0wpiAsbVgKFZJ74YMPTLEp0NJ9W+ApKWDGfyjLtkgsQVIdHshrsGJhxb6dvzlyJepgzHUGz4KWV/yebhWKx7DNVyct2gLeas62g5JzAVfnvQr/WdjqHlfE/p7dOddUS5N0tO7qy94HQNVd+PhT70dECn7dyTOPFCFSe16L/CEpNsvJW/f8Zkwwf0NSqG7DCY5Pd3Kb/P/C2For/tcNj4ZHrfzLZu+XiHOivHUbqq1TfK8/kLp8MmFnXOho46H74LFe7rFx4bas3x8WtQz9EwnEzK6z0A5Jqa3D++9MizX062SaJX6MYSlZU8q04ehZLvXcueMFJhpBU19ThOixgXBz4YA0hDNbwE3FlA0s/uNDPBz1Wbe1Y47St84yyj2yp9sTQTWEf1nbLBr1faKXz9InpOp96Gzwb0UOfXKTu5dNtWdn7PjtHRlV2gjg5KP3IFkvcCUryvQATGrHkPwHCeb7vqjjJpaEHXNrQLbf75TtDMWy+2IDFfyKknjXtBjdsx9sWoHC6uvLDcBNHglWXPmXuua/qWAwAObdPpwD+cW4HX+vi6ND73grC5RqiJDDOYO75rNJ/T84MaRWy63ciEq6qReX1R3LP4+Eu95ghHcWd9euIhIy4quIPBflQOn8KvZc0tTzZ2VnbXLz7tsdL5VcwElbeEihO0Ov2t8KLuQKx1zgptCVMl8oSYt0QMAMN9Y8DYOg7d3Qpd7dluzbIAIvhoSEZE09iODa25H7T4g/kqH7tQwh/Dx+osuMq/1yjHFVnGeQ/kW17bv9fsty0je6rXYatt3tdmroqIbMijJoqH9M8LKORVJJGjviJE8XetC4c9X9/eyz79OPzXR67h6MSi5+o8MwISA5V69njmHMVc+7/XC77f8YE/t9tM31Cnl+GKGn9ACoQMuK9njwAc1T13OCf0WQ3XSS5/p3LZ6wmNo7luGIK/JZHr6IXLH9U7mYx8//qH1rLMREZfS0ZXJM2/9GLWGY7ZI6990EHYMuehPQZFcvS65bebhy5aCpTFkJrUH+uV8od6E2TeBjKO34LC9XCjbu7iuBEbvMQ2frHNhP+bxfX8WrXlZFroe2cY8sJ5cHUI+fqRSu4RVPei1G8XnCbeLgShE+vGNQHrFRfG3LD5S2sMz3FNz7QC0+zyoxBw47XmBtRtciDA5r4Rer7vmKHtOoFGjY2XwNTlO1Vs/B/PDTRa/GVzAz0KeG60Py9XfHC6f5l7zWr3L8y3P/wL</diagram></mxfile>
\ No newline at end of file

docs/conf.py

+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'OpenAppStack'
+copyright = '2019, Greenhost'
+author = 'Greenhost'
+
+# The full version, including alpha/beta/rc tags
+with open('../VERSION') as version_file:
+    release = version_file.read()
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'recommonmark'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'README.md']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Readthedocs.io needs us to tell it what the index file is. This defaults to
+# 'contents'
+master_doc = 'index'

docs/design.md

+OpenAppStack Design
+===================
+
+This article covers the basic design of OpenAppStack.
+
+## Application build pipeline
+
+The following diagram explains the process to go from an application's source
+code to a deployment on OpenAppStack.
+
+![Application build process](_static/diagrams/application-build-process.png)
+
+These are the steps in more detail:
+
+- Build container (this process should be maintained by application developer by
+  providing a Dockerfile with the application)
+   1. Get application package (source code, installation package, etc.)
+      1. If not part of the package: get default configuration for the
+         application
+   1. Build container with application package installed
+      1. Install application dependencies
+      1. Install application package
+      1. Setup default configuration
+      1. Setup pluggable configuration override, can be:
+         - Reading environment variables
+         - Extra configuration file mounted into the container elsewhere
+- Helm chart
+   - Deployment configuration to specify:
+     - The container(s) that should be deployed.
+     - The port(s) that they expose.
+     - Volume mounts for configuration files and secrets.
+     - Live/readyness probes
+     - Persistent storage locations and methods
+     - A lot of other things
+   - Service configuration to specify:
+     - Ports exposed to the user of the application
+   - Ingress configuration to specify:
+     - How to proxy to the application (which hostname or URL)
+     - Some authentication plugins (http auth, for example)
+   - Custom files:
+     - Add file templates for mountable application configuration files
+     - Files that specify integrations with other services
+- Deploy
+   1. Create `values.yaml` file with the variables for the Helm deployment to
+      the Kubernetes cluster
+   1. "Manually" add secrets to the Kubernetes cluster.
+   1. Run `helm install` to install the customised application.
+
+
+## Configuration
+
+As can be seen in the images, applications are expected to have two different
+types of configuration. Containers should provide a default configuration,
+that at least configures things like the port the application runs on, the
+locations for log files, etc.
+
+What we call the *external configuration* is provided by the user. This includes
+overrides of the default application, as well as variables like the hostname
+that the application will run on and listen to and the title of the web
+interface.
+
+OpenAppStack will use Helm charts to provide the external configuration for the
+"Deploy" step. Helm charts can contain configuration file templates with
+default values that can be overridden during the installation or upgrade of a
+helm chart.
+
+## Application containers
+
+For inclusion in OpenAppStack, it is required that the application developers
+provide Docker containers for their applications. There are several reasons for
+this:
+
+- If application developers do not provide a container, chances are they also do
+  not think about how their application would update itself after a new
+  container is deployed. This can lead to problems with things like database
+  migrations.
+- Maintaining the containerisation for an application can, in most cases, not be
+  fully automated.
+
+### Container updates
+
+When an application update is available, these updates need to be rolled out to
+OpenAppStack instances. This will be done according the following steps:
+
+1. Application container is built with new application source and tagged for
+   testing.
+4. Helm chart for application is updated to provide new container.
+3. Helm chart is deployed to an OpenAppStack test cluster following the steps in
+   the diagram above.
+2. Application is tested with automated tests
+3. If tests succeed, new container is tagged for release.
+4. OpenAppStack automated update job fetches new Helm chart and upgrades current
+   instance using Helm.
+
+Most of these steps can be developed by configuring a CI system and configuring
+Kubernetes and Helm correctly. The automated update job that will run on
+OpenAppStack clusters will be developed by us.
+
+## Persistent data
+
+Containerised applications are normally "stateless" (meaning no data is saved
+inside the containers). However, it is possible to mount persistent volumes to
+specific directories in the container, basically adding a persistent layer on
+top of the containerised application. To provide this in OAS's simple setup, we
+have developed a [local storage
+provisioner](https://open.greenhost.net/openappstack/local-storage) that
+automatically provides persistent data on the VPS running OAS to an application
+that requests it.

docs/index.rst

+.. OpenAppStack documentation master file, created by
+   sphinx-quickstart on Wed Jul 31 15:18:33 2019.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to OpenAppStack's documentation!
+========================================
+
+OpenAppStack (OAS) is a platform that will offer self-managed, click-and-play
+provisioning of online applications for Civil Society Organisations (CSOs).
+Users will be able to easily set up a self-hosted instance of OpenAppStack, so
+they can keep control over the data that gets entered into these applications.
+
+OpenAppStack is:
+
+- Open Source
+- Self updating
+- Easy to deploy
+- Integrated
+
+For more information, go to `the OpenAppStack website`_.
+
+.. _the OpenAppStack website: https://openappstack.net
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   installation_instructions
+   design

docs/installation_instructions.md

+# OpenAppStack Tutorial
+
+Here's how you can set up a single-node OpenAppStack cluster. Support for
+multi-node clusters will come in the future.
+
+## Warnings
+
+* OpenAppStack is still under heavy development, and is not ready for
+  production use! We anticipate major changes and do not guarantee a
+  data-preserving upgrade path from current installations. Feel free to try
+  OpenAppStack for testing though, and please
+  [report](https://openappstack.net/contact.html) any issues you encounter.
+* When you install OpenAppStack on a server, the installation process will make
+  some substantial changes to the server's configuration, so please do not use
+  a server that functions as anything other than a testing ground.
+
+## Prerequisites
+
+* A virtual machine or bare metal server with:
+
+  * current Debian stable "buster";
+  * a public IP address;
+  * 8GB of RAM;
+  * at least 20GB of disk space for installation, plus more for application
+    data;
+  * root ssh access.
+
+### DNS entries
+
+First, begin with creating DNS records for your cluster. It's important to
+start with configuring DNS because depending on your DNS setup/provider, it
+takes a while to propagate. You need one dedicated subdomain entry and a
+wildcard entry for everything inside it. For example, create an A record for
+these domains:
+
+- oas.example.org
+- \*.oas.example.org
+
+and make them point to your machine's public IP address.
+
+## Configure your cluster
+
+Clone the OAS bootstrap repo:
+
+    git clone https://open.greenhost.net/openappstack/openappstack.git
+
+Copy `ansible/inventory.yml.example` to `ansible/inventory.yml` and edit it to
+reflect your cluster.
+
+Also copy `ansible/group_vars/all/settings.yml.example` to
+`ansible/group_vars/all/settings.yml` and edit as you see fit.
+
+### Prerequisites
+
+* You need `ansible >= 2.7` installed on your workstation to run the bootstrap
+  scripts. Please install it using your system package manager.
+
+  In the case your system package manager doesn't provide that particular
+  ansible version, you can install it via the python package manager like this:
+
+  ```
+  pip3 install --user -r ansible/requirements.txt
+  ```
+
+  Hint: if you have several python projects on your computer, consider using
+  [virtualenv](https://virtualenv.pypa.io/en/stable/)
+
+  Hint: if you get a [segmentation
+  fault](https://bitbucket.org/cffi/cffi/issues/272/segfault-while-installing-via-pip)
+  using above command, you can add `--no-use-wheel` to it.
+
+
+### Installation
+
+The bootstrap process sets up a single-node kubernetes cluster on the machine
+and installs the utility tools `helmfile`, `helm`, `kubectl` and `rke`.
+
+To run the bootstrap process, you need to move into the `ansible/` directory,
+then run
+
+```
+ansible-playbook bootstrap.yml
+```
+
+It will take approximately 5 to 10 minutes to set up your cluster. Please
+[report](https://openappstack.net/contact.html) any installation issues.
+
+## Usage
+
+Right now, there are two applications installed:
+* [Nextcloud](https://nextcloud.com/), a file sharing and communication platform;
+* [ONLYOFFICE](https://www.onlyoffice.com/connectors-nextcloud.aspx), an online document
+  editing suite.
+
+You can access Nextcloud via https://files.oas.example.org. Use the username
+`admin` with the automatically generated Nextcloud password that you can find in the `ansible/cluster_data/secrets/`
+folder on your workstation.
+ONLYOFFICE is already integrated in your Nextcloud installation which allows you to create and
+share ONLYOFFICE documents within Nextcloud.
+
+Besides these applications, some other auxiliary components are installed:
+* `local-storage` provides an easy way for the cluster to use a directory on
+  the node (by default `/var/lib/OpenAppStack/local-storage`) for storage;
+* nginx is a webserver that functions as a so-called ingress controller,
+  routing web traffic that enters the cluster to the various applications;
+* `cert-manager` acquires and stores [Let's Encrypt](https://letsencrypt.org/)
+  certificates, enabling encrypted web traffic to all applications running in
+  the cluster;
+* Prometheus and Grafana together provide metrics displayed in nice visual
+  dashboards.
+
+### Monitoring
+
+You should be able to access the visual interface to the monitoring system at
+`https://grafana.oas.example.org/`. A user `admin` is created at installation
+time; the password that was generated during installation is stored in the file
+`ansible/cluster_data/secrets/prometheus_grafana_admin_password` on your workstation.
+
+## Managing an existing cluster
+
+Log in to your cluster with:
+
+    ssh USER@oas.example.org
+
+where `USER` is the `ansible_user` you configured in `ansible/inventory.yml`.
+
+Some programs that are installed on your cluster:
+
+* `kubectl` is the Kubernetes control program.
+  For example, run `kubectl get pods -n oas` to see a list of pods that exist
+  in the `oas` namespace (system applications like nginx),
+  and `kubectl get pods -n oas-apps` for all other OpenAppStack applications
+  (like Nextcloud and ONLYOFFICE).
+  Run `kubectl --help` for help.
+* `helm` is the "Kubernetes package manager". Use `helm ls` to see what apps are
+  installed in your cluster. You can also use it to perform manual upgrades;
+  see `helm --help`.
+* `helmfile` is a high-level tool to manage your app installations.
+  Its manual usage is a bit tricky since [current helmfile config depends on
+  environmental variables to be
+  present](https://open.greenhost.net/openappstack/openappstack/issues/101).