diff --git a/.gitignore b/.gitignore
index d223adc4c1fad0349410a317098a58afd2ffb32d..519ccbc11536d0325eff0fbb31639c1056560db1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,3 +26,6 @@
# Etc
__pycache__
*.swp
+
+# Documentation files
+/docs/_build
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d3bff357f5d3b3e780821462e23ccb11a1848966..4a3f4ec2eec757c8c45edf92668775e8b1a452ed 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,6 +45,11 @@ bootstrap:
- test/group_vars/all/settings.yml
expire_in: 1 month
when: always
+ only:
+ changes:
+ - ansible
+ - helmfiles
+ - test
install:
stage: install-apps
@@ -64,6 +69,11 @@ install:
paths:
- test/behave/behave.ini
expire_in: 1 month
+ only:
+ changes:
+ - ansible
+ - helmfiles
+ - test
testinfra:
stage: health-test
@@ -72,6 +82,11 @@ testinfra:
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
- cd test/
- py.test -v -m 'testinfra' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
+ only:
+ changes:
+ - ansible
+ - helmfiles
+ - test
certs:
stage: health-test
@@ -83,6 +98,11 @@ certs:
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
- cd test/
- py.test -s -m 'certs' --connection=ansible --ansible-inventory=./inventory.yml --hosts='ansible://*'
+ only:
+ changes:
+ - ansible
+ - helmfiles
+ - test
behave-nextcloud:
stage: integration-test
@@ -99,6 +119,11 @@ behave-nextcloud:
expire_in: 1 month
when: on_failure
retry: 2
+ only:
+ changes:
+ - ansible
+ - helmfiles
+ - test
behave-grafana:
stage: integration-test
@@ -114,6 +139,11 @@ behave-grafana:
- test/behave/screenshots/
expire_in: 1 month
when: on_failure
+ only:
+ changes:
+ - ansible
+ - helmfiles
+ - test
terminate:
stage: cleanup
@@ -123,3 +153,8 @@ terminate:
- echo "$CI_COMMIT_MESSAGE" | grep '!ci_dont_terminate' && echo 'Termination of droplet disabled in commit message.' || python3 -u ./ci-bootstrap.py --use-existing-inventory --terminate
# Remove droplet older than 2 days
- python3 -c "import cosmos; cosmos.terminate_droplets_by_name(\"^ci-\", 2)"
+ only:
+ changes:
+ - ansible
+ - helmfiles
+ - test
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..d4bb2cbb9eddb1bb1b4f366623044af8e4830919
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS ?=
+SPHINXBUILD ?= sphinx-build
+SOURCEDIR = .
+BUILDDIR = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..e61fda96567a8e17a316d8edab2f7b857440edf4
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,10 @@
+## Documentation
+
+This folder contains the documentation. You can find the documentation at
+https://openappstack.readthedocs.io/. It is also possible to build the
+documentation, using Sphinx:
+
+```
+pip install sphinx
+make html
+```
diff --git a/docs/_static/diagrams/application-build-process.png b/docs/_static/diagrams/application-build-process.png
new file mode 100644
index 0000000000000000000000000000000000000000..7aeb810053dbf584a4c55499bb78051f355d6d2d
Binary files /dev/null and b/docs/_static/diagrams/application-build-process.png differ
diff --git a/docs/_static/diagrams/application-build-process.xml b/docs/_static/diagrams/application-build-process.xml
new file mode 100644
index 0000000000000000000000000000000000000000..f5b669b79d58973cee1a4d4dd513aca7710feb28
--- /dev/null
+++ b/docs/_static/diagrams/application-build-process.xml
@@ -0,0 +1 @@
+<mxfile userAgent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/69.0.3497.81 Chrome/69.0.3497.81 Safari/537.36" version="9.1.8" editor="www.draw.io" type="device"><diagram name="Page-1" id="9f46799a-70d6-7492-0946-bef42562c5a5">7Vxtc6O2Fv41nrn9UA/vLx/jZLftNHe6s2m3vR9lkDFdGTEgx0l/fY9AwmAJm8TYyd3Fk9mFYwnB0XOOnkeSmdm3m6efCpSv/0tjTGaWET/N7LuZZZmO5cF/3PJcWwInrA1Jkcai0N7wkP6DhdEQ1m0a47JTkFFKWJp3jRHNMhyxjg0VBd11i60o6baaowQrhocIEdX6ZxqzdW31DWNv/xmnyZodfLFE0dekoNtMNDez7FX1qb/eIHkpUb5co5juWib7w8y+LShl9dHm6RYT7lrptbrex55vm9sucMaGVPCDusYjIlvx6L9ul7jIMAP3g3/JtmS4mFkegestlvwo4Ufi9tmz9Fi5SzcEZXC2WLMNAaMJhyuasQdRiJ8jkiYZHEdwf3BZe/GIC5aC32/EF4zmYI3WKYnv0TPd8qcoGXhVni3WtEj/gcsi2QZ8XTABIcBdu8QDrwlmA6wFLqHMJ+kaszHdo5KJMhElBOVlumxueIOKJM0WlDG6EYXkk35MCbmlhBaVA2Q/w1V59+NYlpY9XF9/k0bimKAlJosGL/JKGa18WLKCfsWty8d+uDSM5huJSos7+fBGsBdFwvkf0SYlPAi/4CJGGZJ9UrvLtMR5q7pRfVrPeZ9m0hsJQWUp7l8AB/oPP/Wiz2wwDakC0w1mxTMUERUcy5m7dSWRJxxPAHK3jzrLtmvbuhVxliEKIhHpSXP1psHPkBlQlkBf7lv0jU57tm8r7QWGpjmv2xoiAN8MMbzgfVe2YwwOWs+6N1WRp49C11Ki8CbPCcQFS2kGXzzg4jGN8BSGbx6Gy8B13GFhGET4amHYjrkmpx+GphqHz110nwo603BeE3RhN+aCdxFythJyP8Lp523Ghz3Ujr5bXo1b47TMaYk4Kk8HIriedYOwiyUBsDZohEkGJ8Er1huaZY6iNEvuqzJ3zt7yWXiQm3brlOEHsPN72gFTAxuF661IFQrrNI5xVoUKQwzV0cYxldM0Y5Xb3QX8QefcGpCo3TvuDHdh7s/hjxcvALkZPB9KKxBiCOUd5uGsCRMVsE36Gx2whxhq47UDnmNIcbRIgQdepcm2gAzD+SUAwdiWg5jShIwLIkPWuAYyArcPGVyTVMBgFP6hbM2BwcUKAz/gApLJx5LRgouQCS9n4uUEA60B0YsXfyjfHCOTqGLrlywB9lNOxG4idpcgdqZtDIO3/yo19R6JXWBqk/Kngj5WU0qW0ZVWPIvx6jxeMsymfHzpfBy8G2YX+FqoQDx+VUSAAdjY0eLrkFQ9AeSCALkmwXNMpW+7g0+ri//GjD2LtI+2wPvA3wVb04RmiNxT3nNVOXjO4vkvXn9uGbY0/K8yuIYnDZ9wkcL98jG8agnHCZYDPSZLuvuwN2g8jbP4hk+DcyrAxxQ+KnIjH1MldeiOatiMXezrRsbQ823kaUck6R+6LSIsR5LQM003XBkeDgMc/GiKKAMWkWBRTw5L/KmOjmMFJhCAj91Z+XO6tPf2eufexMNNDO37Y2i9gB/AwMS3ln8ww22rgsP0NCzJdIxvhZIpMWfpOdodziFF4SxKcTkNqq+fNenPcedMsGlROsYoq8JDPzcrZ9zqxGzf6FKyhza8U7JlmVfnRsX9wZ9a3qYtzddaKkmwpiXL0Eab+rU1u+NG1Lnd4a1HBeZ3a/wnneN5PfucwwgO5DMuf9BeZ+Kj7yl0NARVGzrBEck9NHQ8dUZJ6f02c9wz0jZ9jLaA+Fgygj6Cq1LK39dpptDKoTz45CA+FiG1VPrpuGPTT1H1E0frfjC2A+9wfds/WEGrb0vUO2uUtRQoKPmonn2eqOx3SGVfPNn4cqZrHWBdmwl9HdW1v5nZRz1z6RJbI4XxCRHCF4imsXu8sdsajNgjJFeHzzFIrqcFxoOYW5gg8U4gMTRljbIcqIXEF1Sk9RaT0wpiAsbVgKFZJ74YMPTLEp0NJ9W+ApKWDGfyjLtkgsQVIdHshrsGJhxb6dvzlyJepgzHUGz4KWV/yebhWKx7DNVyct2gLeas62g5JzAVfnvQr/WdjqHlfE/p7dOddUS5N0tO7qy94HQNVd+PhT70dECn7dyTOPFCFSe16L/CEpNsvJW/f8Zkwwf0NSqG7DCY5Pd3Kb/P/C2For/tcNj4ZHrfzLZu+XiHOivHUbqq1TfK8/kLp8MmFnXOho46H74LFe7rFx4bas3x8WtQz9EwnEzK6z0A5Jqa3D++9MizX062SaJX6MYSlZU8q04ehZLvXcueMFJhpBU19ThOixgXBz4YA0hDNbwE3FlA0s/uNDPBz1Wbe1Y47St84yyj2yp9sTQTWEf1nbLBr1faKXz9InpOp96Gzwb0UOfXKTu5dNtWdn7PjtHRlV2gjg5KP3IFkvcCUryvQATGrHkPwHCeb7vqjjJpaEHXNrQLbf75TtDMWy+2IDFfyKknjXtBjdsx9sWoHC6uvLDcBNHglWXPmXuua/qWAwAObdPpwD+cW4HX+vi6ND73grC5RqiJDDOYO75rNJ/T84MaRWy63ciEq6qReX1R3LP4+Eu95ghHcWd9euIhIy4quIPBflQOn8KvZc0tTzZ2VnbXLz7tsdL5VcwElbeEihO0Ov2t8KLuQKx1zgptCVMl8oSYt0QMAMN9Y8DYOg7d3Qpd7dluzbIAIvhoSEZE09iODa25H7T4g/kqH7tQwh/Dx+osuMq/1yjHFVnGeQ/kW17bv9fsty0je6rXYatt3tdmroqIbMijJoqH9M8LKORVJJGjviJE8XetC4c9X9/eyz79OPzXR67h6MSi5+o8MwISA5V69njmHMVc+7/XC77f8YE/t9tM31Cnl+GKGn9ACoQMuK9njwAc1T13OCf0WQ3XSS5/p3LZ6wmNo7luGIK/JZHr6IXLH9U7mYx8//qH1rLMREZfS0ZXJM2/9GLWGY7ZI6990EHYMuehPQZFcvS65bebhy5aCpTFkJrUH+uV8od6E2TeBjKO34LC9XCjbu7iuBEbvMQ2frHNhP+bxfX8WrXlZFroe2cY8sJ5cHUI+fqRSu4RVPei1G8XnCbeLgShE+vGNQHrFRfG3LD5S2sMz3FNz7QC0+zyoxBw47XmBtRtciDA5r4Rer7vmKHtOoFGjY2XwNTlO1Vs/B/PDTRa/GVzAz0KeG60Py9XfHC6f5l7zWr3L8y3P/wL</diagram></mxfile>
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..f78b7e86997cd22b914ad6d5d68be9aeb7890283
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,60 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'OpenAppStack'
+copyright = '2019, Greenhost'
+author = 'Greenhost'
+
+# The full version, including alpha/beta/rc tags
+with open('../VERSION') as version_file:
+ release = version_file.read()
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'recommonmark'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'README.md']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Readthedocs.io needs us to tell it what the index file is. This defaults to
+# 'contents'
+master_doc = 'index'
diff --git a/docs/design.md b/docs/design.md
new file mode 100644
index 0000000000000000000000000000000000000000..1afba3ec946eb734d16cbe145107353c6c6dd72f
--- /dev/null
+++ b/docs/design.md
@@ -0,0 +1,108 @@
+OpenAppStack Design
+===================
+
+This article covers the basic design of OpenAppStack.
+
+## Application build pipeline
+
+The following diagram explains the process to go from an application's source
+code to a deployment on OpenAppStack.
+
+
+
+These are the steps in more detail:
+
+- Build container (this process should be maintained by application developer by
+ providing a Dockerfile with the application)
+ 1. Get application package (source code, installation package, etc.)
+ 1. If not part of the package: get default configuration for the
+ application
+ 1. Build container with application package installed
+ 1. Install application dependencies
+ 1. Install application package
+ 1. Setup default configuration
+ 1. Setup pluggable configuration override, can be:
+ - Reading environment variables
+ - Extra configuration file mounted into the container elsewhere
+- Helm chart
+ - Deployment configuration to specify:
+ - The container(s) that should be deployed.
+ - The port(s) that they expose.
+ - Volume mounts for configuration files and secrets.
+ - Live/readyness probes
+ - Persistent storage locations and methods
+ - A lot of other things
+ - Service configuration to specify:
+ - Ports exposed to the user of the application
+ - Ingress configuration to specify:
+ - How to proxy to the application (which hostname or URL)
+ - Some authentication plugins (http auth, for example)
+ - Custom files:
+ - Add file templates for mountable application configuration files
+ - Files that specify integrations with other services
+- Deploy
+ 1. Create `values.yaml` file with the variables for the Helm deployment to
+ the Kubernetes cluster
+ 1. "Manually" add secrets to the Kubernetes cluster.
+ 1. Run `helm install` to install the customised application.
+
+
+## Configuration
+
+As can be seen in the images, applications are expected to have two different
+types of configuration. Containers should provide a default configuration,
+that at least configures things like the port the application runs on, the
+locations for log files, etc.
+
+What we call the *external configuration* is provided by the user. This includes
+overrides of the default application, as well as variables like the hostname
+that the application will run on and listen to and the title of the web
+interface.
+
+OpenAppStack will use Helm charts to provide the external configuration for the
+"Deploy" step. Helm charts can contain configuration file templates with
+default values that can be overridden during the installation or upgrade of a
+helm chart.
+
+## Application containers
+
+For inclusion in OpenAppStack, it is required that the application developers
+provide Docker containers for their applications. There are several reasons for
+this:
+
+- If application developers do not provide a container, chances are they also do
+ not think about how their application would update itself after a new
+ container is deployed. This can lead to problems with things like database
+ migrations.
+- Maintaining the containerisation for an application can, in most cases, not be
+ fully automated.
+
+### Container updates
+
+When an application update is available, these updates need to be rolled out to
+OpenAppStack instances. This will be done according the following steps:
+
+1. Application container is built with new application source and tagged for
+ testing.
+4. Helm chart for application is updated to provide new container.
+3. Helm chart is deployed to an OpenAppStack test cluster following the steps in
+ the diagram above.
+2. Application is tested with automated tests
+3. If tests succeed, new container is tagged for release.
+4. OpenAppStack automated update job fetches new Helm chart and upgrades current
+ instance using Helm.
+
+Most of these steps can be developed by configuring a CI system and configuring
+Kubernetes and Helm correctly. The automated update job that will run on
+OpenAppStack clusters will be developed by us.
+
+## Persistent data
+
+Containerised applications are normally "stateless" (meaning no data is saved
+inside the containers). However, it is possible to mount persistent volumes to
+specific directories in the container, basically adding a persistent layer on
+top of the containerised application. To provide this in OAS's simple setup, we
+have developed a [local storage
+provisioner](https://open.greenhost.net/openappstack/local-storage) that
+automatically provides persistent data on the VPS running OAS to an application
+that requests it.
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..76d3ccfe2fb897383d27fa7e5f7f4318f9b81ebe
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,30 @@
+.. OpenAppStack documentation master file, created by
+ sphinx-quickstart on Wed Jul 31 15:18:33 2019.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+Welcome to OpenAppStack's documentation!
+========================================
+
+OpenAppStack (OAS) is a platform that will offer self-managed, click-and-play
+provisioning of online applications for Civil Society Organisations (CSOs).
+Users will be able to easily set up a self-hosted instance of OpenAppStack, so
+they can keep control over the data that gets entered into these applications.
+
+OpenAppStack is:
+
+- Open Source
+- Self updating
+- Easy to deploy
+- Integrated
+
+For more information, go to `the OpenAppStack website`_.
+
+.. _the OpenAppStack website: https://openappstack.net
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+ installation_instructions
+ design
diff --git a/docs/installation_instructions.md b/docs/installation_instructions.md
new file mode 100644
index 0000000000000000000000000000000000000000..6e2935291970709ccff87d72a2cc377dd771329f
--- /dev/null
+++ b/docs/installation_instructions.md
@@ -0,0 +1,141 @@
+# OpenAppStack Tutorial
+
+Here's how you can set up a single-node OpenAppStack cluster. Support for
+multi-node clusters will come in the future.
+
+## Warnings
+
+* OpenAppStack is still under heavy development, and is not ready for
+ production use! We anticipate major changes and do not guarantee a
+ data-preserving upgrade path from current installations. Feel free to try
+ OpenAppStack for testing though, and please
+ [report](https://openappstack.net/contact.html) any issues you encounter.
+* When you install OpenAppStack on a server, the installation process will make
+ some substantial changes to the server's configuration, so please do not use
+ a server that functions as anything other than a testing ground.
+
+## Prerequisites
+
+* A virtual machine or bare metal server with:
+
+ * current Debian stable "buster";
+ * a public IP address;
+ * 8GB of RAM;
+ * at least 20GB of disk space for installation, plus more for application
+ data;
+ * root ssh access.
+
+### DNS entries
+
+First, begin with creating DNS records for your cluster. It's important to
+start with configuring DNS because depending on your DNS setup/provider, it
+takes a while to propagate. You need one dedicated subdomain entry and a
+wildcard entry for everything inside it. For example, create an A record for
+these domains:
+
+- oas.example.org
+- \*.oas.example.org
+
+and make them point to your machine's public IP address.
+
+## Configure your cluster
+
+Clone the OAS bootstrap repo:
+
+ git clone https://open.greenhost.net/openappstack/openappstack.git
+
+Copy `ansible/inventory.yml.example` to `ansible/inventory.yml` and edit it to
+reflect your cluster.
+
+Also copy `ansible/group_vars/all/settings.yml.example` to
+`ansible/group_vars/all/settings.yml` and edit as you see fit.
+
+### Prerequisites
+
+* You need `ansible >= 2.7` installed on your workstation to run the bootstrap
+ scripts. Please install it using your system package manager.
+
+ In the case your system package manager doesn't provide that particular
+ ansible version, you can install it via the python package manager like this:
+
+ ```
+ pip3 install --user -r ansible/requirements.txt
+ ```
+
+ Hint: if you have several python projects on your computer, consider using
+ [virtualenv](https://virtualenv.pypa.io/en/stable/)
+
+ Hint: if you get a [segmentation
+ fault](https://bitbucket.org/cffi/cffi/issues/272/segfault-while-installing-via-pip)
+ using above command, you can add `--no-use-wheel` to it.
+
+
+### Installation
+
+The bootstrap process sets up a single-node kubernetes cluster on the machine
+and installs the utility tools `helmfile`, `helm`, `kubectl` and `rke`.
+
+To run the bootstrap process, you need to move into the `ansible/` directory,
+then run
+
+```
+ansible-playbook bootstrap.yml
+```
+
+It will take approximately 5 to 10 minutes to set up your cluster. Please
+[report](https://openappstack.net/contact.html) any installation issues.
+
+## Usage
+
+Right now, there are two applications installed:
+* [Nextcloud](https://nextcloud.com/), a file sharing and communication platform;
+* [ONLYOFFICE](https://www.onlyoffice.com/connectors-nextcloud.aspx), an online document
+ editing suite.
+
+You can access Nextcloud via https://files.oas.example.org. Use the username
+`admin` with the automatically generated Nextcloud password that you can find in the `ansible/cluster_data/secrets/`
+folder on your workstation.
+ONLYOFFICE is already integrated in your Nextcloud installation which allows you to create and
+share ONLYOFFICE documents within Nextcloud.
+
+Besides these applications, some other auxiliary components are installed:
+* `local-storage` provides an easy way for the cluster to use a directory on
+ the node (by default `/var/lib/OpenAppStack/local-storage`) for storage;
+* nginx is a webserver that functions as a so-called ingress controller,
+ routing web traffic that enters the cluster to the various applications;
+* `cert-manager` acquires and stores [Let's Encrypt](https://letsencrypt.org/)
+ certificates, enabling encrypted web traffic to all applications running in
+ the cluster;
+* Prometheus and Grafana together provide metrics displayed in nice visual
+ dashboards.
+
+### Monitoring
+
+You should be able to access the visual interface to the monitoring system at
+`https://grafana.oas.example.org/`. A user `admin` is created at installation
+time; the password that was generated during installation is stored in the file
+`ansible/cluster_data/secrets/prometheus_grafana_admin_password` on your workstation.
+
+## Managing an existing cluster
+
+Log in to your cluster with:
+
+ ssh USER@oas.example.org
+
+where `USER` is the `ansible_user` you configured in `ansible/inventory.yml`.
+
+Some programs that are installed on your cluster:
+
+* `kubectl` is the Kubernetes control program.
+ For example, run `kubectl get pods -n oas` to see a list of pods that exist
+ in the `oas` namespace (system applications like nginx),
+ and `kubectl get pods -n oas-apps` for all other OpenAppStack applications
+ (like Nextcloud and ONLYOFFICE).
+ Run `kubectl --help` for help.
+* `helm` is the "Kubernetes package manager". Use `helm ls` to see what apps are
+ installed in your cluster. You can also use it to perform manual upgrades;
+ see `helm --help`.
+* `helmfile` is a high-level tool to manage your app installations.
+ Its manual usage is a bit tricky since [current helmfile config depends on
+ environmental variables to be
+ present](https://open.greenhost.net/openappstack/openappstack/issues/101).