...
 
Commits (102)
......@@ -174,6 +174,28 @@ behave-nextcloud:
- openappstack/**/*
extends: .ssh_setup
behave-wordpress:
stage: integration-test
script:
# Wait for wordpress pod to be Ready
- ssh root@$FQDN '/bin/bash -c "kubectl wait -n oas-apps pod -l release=wordpress --for condition=Ready --timeout=20m"'
# Run behave tests against wordpress
- python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags wordpress || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags wordpress
artifacts:
paths:
- test/behave/screenshots/
expire_in: 1 month
when: on_failure
retry: 2
only:
changes:
- .gitlab-ci.yml
- ansible/**/*
- flux/**/*
- test/**/*
- openappstack/**/*
extends: .ssh_setup
behave-rocketchat:
stage: integration-test
script:
......
# Template for integrating a new app
## Single sign-on
* [ ] Integrate the new app into the signle sign-on system
## Documentation
* [ ] Add app to `docs/installation_instructions.md`
* [ ] Add app to `docs/testing_instructions.md`
## Tests
* [ ] Add behave feature (`tests/behave/feature`)
* [ ] Check for successful helmrelease (`test/pytest/test_helmreleases.py`)
* [ ] Test cert (`test/pytest/test_certs.py`)
## CI
* [ ] Run new tests in CI (`.gitlab-ci.yml`)
......@@ -4,7 +4,7 @@
No unreleased changes yet.
## [0.3.0] - 2019-12-24
## [0.3.0] - 2020-01-20
* Release candidate for beta tests
* Added features:
......@@ -13,6 +13,7 @@ No unreleased changes yet.
* Rocket.chat version 2.1.1 at chat.oas.yourdomain
* Use Flux instead of Helmfile to install and update applications
* Flux version 1.16.0 and helm-operator version 1.0.0-rc4
* WordPress at www.oas.example.org from https://code.greenhost.net/open/wordpress-helm
* Application updates:
- cert-manager 0.11.0
- prometheus-operator 0.34.0
......@@ -23,6 +24,22 @@ No unreleased changes yet.
- kubernetes 1.15.5
* Fix many many bugs
Known issues:
* There are some [initial prometheus alerts](https://open.greenhost.net/openappstack/openappstack/issues/286)
after a fresh OAS installation
* [Prometheus web interface is publicly available](https://open.greenhost.net/openappstack/openappstack/issues/294)
* Single sign-on has some issues:
* Rocket.chat SSO does not work out of the box (see docs for setup
instructions)
* WordPress and Grafana SSO do not give admin permissions to users with the
"admin" enabled in the user-panel
* Nextcloud requires email (SMTP) to be set up before you log in with new
users using SSO
These issues will be addressed in the next version.
## [0.2.1] - 2019-10-29
This is the release for the second user test.
......
.DEFAULT_GOAL := help
help:
# See Makefile for available targets.
freeze:
# pip wrongly adds `pkg-resources==0.0.0` which will break some systems, see
# https://open.greenhost.net/openappstack/openappstack/issues/498#note_15057
pip3 freeze | grep -v "pkg-resources" > requirements-stable.txt
......@@ -61,6 +61,9 @@
name: local-flux
tags: ['flux']
when: local_flux
- import_role:
name: additional_config
tags: ['additional']
always:
- import_role:
name: finalize
......
......@@ -25,11 +25,20 @@ rocketchat_mongodb_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/r
rocketchat_mongodb_root_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/rocketchat_mongodb_root_password chars=ascii_letters') }}"
rocketchat_admin_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/rocketchat_admin_password chars=ascii_letters') }}"
# WordPress credentials
wordpress_admin_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/wordpress_admin_password chars=ascii_letters') }}"
wordpress_mariadb_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/wordpress_mariadb_password chars=ascii_letters') }}"
wordpress_mariadb_root_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/wordpress_mariadb_root_password chars=ascii_letters') }}"
# Grafana credentials
grafana_admin_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/grafana_admin_password chars=ascii_letters') }}"
# Single sign-on passwords
userpanel_oauth_client_secret: "{{ lookup('password', '{{ cluster_dir }}/secrets/userpanel_oauth_client_secret chars=ascii_letters') }}"
nextcloud_oauth_client_secret: "{{ lookup('password', '{{ cluster_dir }}/secrets/nextcloud_oauth_client_secret chars=ascii_letters') }}"
grafana_oauth_client_secret: "{{ lookup('password', '{{ cluster_dir }}/secrets/grafana_oauth_client_secret chars=ascii_letters') }}"
wordpress_oauth_client_secret: "{{ lookup('password', '{{ cluster_dir }}/secrets/wordpress_oauth_client_secret chars=ascii_letters') }}"
rocketchat_oauth_client_secret: "{{ lookup('password', '{{ cluster_dir }}/secrets/rocketchat_oauth_client_secret chars=ascii_letters') }}"
userbackend_postgres_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/userbackend_postgres_password chars=ascii_letters') }}"
userbackend_admin_username: "admin"
userbackend_admin_password: "{{ lookup('password', '{{ cluster_dir }}/secrets/userbackend_admin_password chars=ascii_letters') }}"
......@@ -76,4 +85,4 @@ local_flux: false
# mechanism (flux). Not used if local_flux is set to true.
flux_source:
repo: "https://open.greenhost.net/openappstack/openappstack"
branch: "0.3"
branch: "master"
This role configures the cluster after app installation has started via
flux.
All resources that are not needed for flux should go here, so we save time by
starting the flux install process early in the background.
---
- name: Use core-dns service for system resolver
tags:
- dns
- core-dns
lineinfile:
path: /etc/resolv.conf
insertbefore: BOF
line: nameserver 10.43.0.10
---
- name: Logrotate config for OAS logs
copy:
src: etc/logrotate.d/OpenAppStack
dest: /etc/logrotate.d/
mode: '0644'
---
- import_tasks: logging.yml
- import_tasks: dns.yml
......@@ -54,4 +54,10 @@
flux:
name: "cert-manager-settings"
namespace: "oas"
include_tasks: flux_secret.yml
include_tasks:
file: flux_secret.yml
apply:
tags:
- config
- flux
- cert-manager
......@@ -58,13 +58,4 @@
# helm-operator
# # Chart name
# helm-operator
# # Enable custom Helm repositories
# --set configureRepositories.enable=true
# # Add the stable default Helm repository
# --set configureRepositories.repositories[0].name=stable
# --set configureRepositories.repositories[0].url=https://kubernetes-charts.storage.googleapis.com
# # Add the ory repository that is required for the single-sign-on helmchart
# --set configureRepositories.repositories[1].name=ory
# --set configureRepositories.repositories[1].url=https://k8s.ory.sh/helm/charts
shell: helm upgrade --install --repo "https://charts.fluxcd.io" --namespace oas --version 0.3.0 --set createCRD=true --set chartsSyncInterval=20m helm-operator helm-operator --set configureRepositories.enable=true --set configureRepositories.repositories[0].name=stable --set configureRepositories.repositories[0].url=https://kubernetes-charts.storage.googleapis.com --set configureRepositories.repositories[1].name=ory --set configureRepositories.repositories[1].url=https://k8s.ory.sh/helm/charts
shell: helm upgrade --install --repo "https://charts.fluxcd.io" --namespace oas --version 0.3.0 --set createCRD=true --set chartsSyncInterval=20m helm-operator helm-operator
......@@ -8,4 +8,10 @@
flux:
name: "local-storage-settings"
namespace: "oas"
include_tasks: flux_secret.yml
include_tasks:
file: flux_secret.yml
apply:
tags:
- config
- flux
- local-storage
......@@ -24,3 +24,5 @@
- name: Tasks pertaining to NextCloud
import_tasks: nextcloud.yml
- name: Tasks pertaining to WordPress
import_tasks: wordpress.yml
---
- name: Create Kubernetes secret with nextcloud settings
tags:
- config
......@@ -8,7 +7,13 @@
vars:
flux:
name: "nextcloud-settings"
include_tasks: flux_secret.yml
include_tasks:
file: flux_secret.yml
apply:
tags:
- config
- flux
- nextcloud
- name: Create persistent volumes for nextcloud data and metadata
tags:
......@@ -20,9 +25,19 @@
name: "nextcloud-{{ item.name }}"
namespace: "oas-apps"
size: "{{ item.size }}"
include_tasks: pvc.yml
include_tasks:
file: pvc.yml
apply:
tags:
- config
- nextcloud
- storage
with_items:
- name: "files"
size: 2Gi
- name: "mariadb"
size: 512Mi
- name: "rabbitmq"
size: 8Gi
- name: "redis"
size: 8Gi
......@@ -9,4 +9,10 @@
flux:
name: "ingress-settings"
namespace: "oas"
include_tasks: flux_secret.yml
include_tasks:
file: flux_secret.yml
apply:
tags:
- config
- flux
- nginx
......@@ -40,4 +40,11 @@
flux:
name: "monitoring-settings"
namespace: "oas"
include_tasks: flux_secret.yml
include_tasks:
file: flux_secret.yml
apply:
tags:
- config
- flux
- monitoring
- prometheus
---
- name: Create Persistent Volume Claim for application
tags:
- storage
k8s:
state: present
definition:
......
......@@ -8,4 +8,10 @@
vars:
flux:
name: "rocketchat-settings"
include_tasks: flux_secret.yml
include_tasks:
file: flux_secret.yml
apply:
tags:
- config
- flux
- rocketchat
......@@ -9,4 +9,10 @@
flux:
name: "single-sign-on-settings"
namespace: "oas"
include_tasks: flux_secret.yml
include_tasks:
file: flux_secret.yml
apply:
tags:
- config
- flux
- single-sign-on
---
- name: Create Kubernetes secret with wordpress settings
tags:
- config
- flux
- wordpress
vars:
flux:
name: "wordpress-settings"
include_tasks:
file: flux_secret.yml
apply:
tags:
- config
- flux
- wordpress
- name: Create persistent volumes for wordpress data and metadata
tags:
- config
- wordpress
- storage
vars:
pvc:
name: "wordpress-{{ item.name }}"
namespace: "oas-apps"
size: "{{ item.size }}"
include_tasks:
file: pvc.yml
apply:
tags:
- config
- wordpress
- storage
with_items:
- name: "files"
size: 2Gi
- name: "mariadb"
size: 512Mi
......@@ -43,8 +43,6 @@ prometheus:
resources:
requests:
storage: 5Gi
service:
type: NodePort
prometheusOperator:
createCustomResource: false
......@@ -121,7 +119,19 @@ additionalPrometheusRulesMap:
grafana:
adminPassword: "{{ grafana_admin_password }}"
grafana.ini:
server:
root_url: "https://grafana.{{ domain }}"
auth.generic_oauth:
name: OpenAppStack
enabled: true
client_id: grafana
client_secret: "{{ grafana_oauth_client_secret }}"
scopes: "openid profile email openappstack_roles"
auth_url: "https://sso.{{ domain }}/oauth2/auth"
token_url: "https://sso.{{ domain }}/oauth2/token"
api_url: "https://sso.{{ domain }}/userinfo"
role_attribute_path: contains(openappstack_roles[*], 'admin') && 'Admin' || 'Editor'
ingress:
enabled: true
annotations:
......
......@@ -90,3 +90,15 @@ postgresql:
rabbitmq:
rabbitmq:
password: "{{ onlyoffice_rabbitmq_password }}"
persistence:
existingClaim: "nextcloud-rabbitmq"
redis:
persistence:
existingClaim: "nextcloud-redis"
sociallogin:
server_name: "sso.{{ domain }}"
client_id: nextcloud
client_secret: "{{ nextcloud_oauth_client_secret }}"
groups_claim: "openappstack_roles"
replicaCount: 1
consentProviderImage:
<< : &IMAGE_DEFAULTS_SSO { tag: "integration-user-panel", pullPolicy: "Always" }
<< : &IMAGE_DEFAULTS_SSO { tag: "0.2.0", pullPolicy: "Always" }
repository: "open.greenhost.net:4567/openappstack/single-sign-on/consent_provider"
loginProviderImage:
<< : *IMAGE_DEFAULTS_SSO
......@@ -10,17 +10,28 @@ loginProviderImage:
singleSignOnHost: &SSO_HOST "sso.{{ domain }}"
userpanel:
applicationName: &USER_PANEL user-panel
image:
<< : &IMAGE_DEFAULTS_USER_PANEL { tag: "master", pullPolicy: "Always" }
<< : &IMAGE_DEFAULTS_USER_PANEL { tag: "1.2.0", pullPolicy: "Always" }
repository: "open.greenhost.net:4567/openappstack/user-panel/frontend"
ingress:
host: "admin.{{ domain }}"
oAuthClientSecret: "{{ userpanel_oauth_client_secret }}"
userbackend:
image:
<< : *IMAGE_DEFAULTS_USER_PANEL
repository: "open.greenhost.net:4567/openappstack/user-panel/backend"
applications:
- name: *USER_PANEL
description: Administration interface to manage user accounts
- name: &NEXTCLOUD nextcloud
description: "Nextcloud Files offers an on-premise Universal File Access and sync platform with powerful collaboration capabilities and desktop, mobile and web interfaces."
- name: &WORDPRESS wordpress
description: "WordPress website hosting."
- name: &ROCKETCHAT rocketchat
description: "Communicate and collaborate using team chat and switch to video or audio calls with screen sharing for more efficient teamwork."
- name: &GRAFANA grafana
description: "Grafana allows you to query, visualize, alert on and understand metrics generated by OpenAppStack. It can be used to create explore and share dashboards."
username: "{{ userbackend_admin_username }}"
password: "{{ userbackend_admin_password }}"
email: "{{ userbackend_admin_email }}"
......@@ -34,9 +45,7 @@ userbackend:
hydra:
hydra:
dangerousForceHttp: true
config:
dsn: memory
urls:
self:
issuer: "https://sso.{{ domain }}"
......@@ -58,3 +67,73 @@ hydra:
secretName: hydra-public.tls
admin:
enabled: false
oAuthClients:
- clientName: *USER_PANEL
clientSecret: "{{ userpanel_oauth_client_secret }}"
redirectUri: "https://admin.{{ domain }}/callback"
scopes: "openid profile email openappstack_roles"
clientUri: "https://admin.{{ domain }}"
clientLogoUri: "https://admin.{{ domain }}/favicon.ico"
tokenEndpointAuthMethod: "client_secret_basic"
responseTypes:
- "token"
grantTypes:
- "implicit"
- clientName: *NEXTCLOUD
clientSecret: "{{ nextcloud_oauth_client_secret }}"
redirectUri: "https://files.{{ domain }}/apps/sociallogin/custom_oidc/oas"
scopes: "openid profile email openappstack_roles"
clientUri: "https://files.{{ domain }}"
clientLogoUri: "https://files.{{ domain }}/core/img/favicon-touch.png"
tokenEndpointAuthMethod: "client_secret_post"
responseTypes:
- "code"
- "id_token"
grantTypes:
- "authorization_code"
- "refresh_token"
- "client_credentials"
- clientName: *WORDPRESS
clientSecret: "{{ wordpress_oauth_client_secret }}"
redirectUri: "https://www.{{ domain }}/wp-admin/admin-ajax.php?action=openid-connect-authorize"
scopes: "openid profile email openappstack_roles offline_access"
clientUri: "https://www.{{ domain }}"
clientLogoUri: "https://www.{{ domain }}/wp-admin/images/wordpress-logo.svg"
tokenEndpointAuthMethod: "client_secret_post"
responseTypes:
- "code"
- "id_token"
grantTypes:
- "authorization_code"
- "refresh_token"
- "client_credentials"
- "implicit"
- clientName: *ROCKETCHAT
clientSecret: "{{ rocketchat_oauth_client_secret }}"
redirectUri: "https://chat.{{ domain }}/_oauth/openappstack"
scopes: "openid profile email openappstack_roles"
clientUri: "https://chat.{{ domain }}"
clientLogoUri: "https://chat.{{ domain }}/images/logo/logo.svg"
tokenEndpointAuthMethod: "client_secret_post"
responseTypes:
- "code"
- "id_token"
grantTypes:
- "authorization_code"
- "refresh_token"
- "client_credentials"
- clientName: *GRAFANA
clientSecret: "{{ grafana_oauth_client_secret }}"
redirectUri: "https://grafana.{{ domain }}/login/generic_oauth"
scopes: "openid profile email openappstack_roles"
clientUri: "https://grafana.{{ domain }}"
clientLogoUri: "https://grafana.{{ domain }}/public/img/grafana_icon.svg"
tokenEndpointAuthMethod: "client_secret_post"
responseTypes:
- "code"
- "id_token"
grantTypes:
- "authorization_code"
- "refresh_token"
- "client_credentials"
image:
tag: 34-add-openid-connect-plugin-and-configuration
initImage:
tag: 34-add-openid-connect-plugin-and-configuration
wordpress:
config:
db:
prefix: wp_
adm:
usid: admin
pssw: "{{ wordpress_admin_password }}"
site:
alt_path: openappstack-login
# NOTE: Use a theme *slug* here
theme: twentynineteen
# NOTE: Make sure you use underscore and that the localisation is in full caps
locale: en_US
version: 5.3.2
url: "https://www.{{ domain }}"
title: "OpenAppStack website"
persistence:
existingClaim: wordpress-files
openid_connect_settings:
enabled: true
client_secret: {{ wordpress_oauth_client_secret }}
endpoint_login: https://sso.{{ domain }}/oauth2/auth
endpoint_userinfo: https://sso.{{ domain }}/userinfo
endpoint_token: https://sso.{{ domain }}/oauth2/token
endpoint_end_session: ""
# After our SSO supports it, we should set this as the logout URL
# https://open.greenhost.net/openappstack/single-sign-on/issues/28
# endpoint_end_session: https://sso.{{ domain }}/oauth2/sessions/logout
no_sslverify: "0"
http_request_timeout: "15"
enable_logging: "1"
scope: email profile openid openappstack_roles offline_access
database:
db:
user: wordpress
password: "{{ wordpress_mariadb_password }}"
rootUser:
password: "{{ wordpress_mariadb_root_password }}"
persistence:
## Enable PostgreSQL persistence using Persistent Volume Claims.
enabled: true
existingClaim: "wordpress-mariadb"
replication:
enabled: false
# It's advisable to set resource limits to prevent your K8s cluster from
# crashing
# resources:
# limits:
# cpu: 100m
# memory: 512Mi
# requests:
# cpu: 50m
# memory: 256Mi
ingress:
enabled: true
annotations:
kubernetes.io/tls-acme: "true"
path: /
hosts:
- "www.{{ domain }}"
tls:
- hosts:
- "www.{{ domain }}"
secretName: oas-wordpress
......@@ -10,6 +10,7 @@
name:
- apt-transport-https
- curl
- dnsutils
- git
- nftables
- rsync
......@@ -30,12 +31,6 @@
- python3-wheel
- python3-psutil
- name: Logrotate config for OAS logs
copy:
src: etc/logrotate.d/OpenAppStack
dest: /etc/logrotate.d/
mode: '0644'
- name: Install python packages via pip3
tags:
- package
......
......@@ -18,7 +18,7 @@
# -- Project information -----------------------------------------------------
project = 'OpenAppStack'
copyright = '2019, Greenhost'
copyright = '2020, Greenhost'
author = 'Greenhost'
# The full version, including alpha/beta/rc tags
......
......@@ -27,9 +27,10 @@ For more information, go to `the OpenAppStack website`_.
:caption: Contents:
installation_instructions
upgrading
testing_instructions
usage
troubleshooting
maintenance
upgrading
design
reference
This diff is collapsed.
......@@ -17,6 +17,61 @@ The [Loki documentation](https://github.com/grafana/loki#documentation) is a
good starting point how this setup works, and the [Using Loki in Grafana](https://grafana.com/docs/grafana/latest/features/datasources/loki/)
gets you started with querying your cluster logs with grafana.
You will find the loki grafana integration on your cluster at https://grafana.oas.example.org/explore
together with some generic query examples.
### LogQL query examples
Please also refer to the [LogQL documentation](https://github.com/grafana/loki/blob/master/docs/logql.md).
#### Flux
Flux is responsible for installing applications. It used `helm-operator` to
deploy the desired helm releases.
Query all messages from `flux`:
{app="flux"}
Query all messages from `flux` and `helm-operator`:
{app=~"(flux|helm-operator)"}
`flux` messages containing `wordpress`:
{app = "flux"} |= "wordpress"
`flux` messages containing `wordpress` without `unchanged` events (to only show
the installation messages):
{app = "flux"} |= "wordpress" != "unchanged"
Filter out redundant `flux` messages:
{ app = "flux" } !~ "(unchanged | event=refreshed | method=Sync | component=checkpoint)"
Debug oauth2 single sign-on with rocketchat:
{container_name=~"(hydra|rocketchat)"}
#### Cert-manager
Cert manager is responsible for requesting Let's Encrypt TLS certificates.
Query `cert-manager` messages containing `chat`:
{app="cert-manager"} |= "chat"
#### Hydra
Hydra is the single sign-on system.
Show only warnings and errors from `hydra`:
{container_name="hydra"} != "level=info"
## Backup
Please take care to backup the following locations:
......
......@@ -21,26 +21,20 @@ First we'd like you to setup an OpenAppStack cluster by yourself, following the
Please run the [command line tests](troubleshooting.md) which checks the overall
functionality of your cluster and include the output in your feedback.
## User panel
## Usage
Please open https://admin.oas.example.org in the browser. You should see
"Welcome to OpenAppStack" and a Login button. Please try logging in.
An admin user was generated with the username `admin`. The password is saved in
`clusters/my-cluster/secrets/userbackend_admin_password` in the OpenAppStack
directory on your local machine.
After logging in to the user panel, please try to make a new user. Don't forget
to give it a username and password, and press "Save" afterwards.
Please go through the [Usage documentation](./usage.md) and make sure you
complete all steps.
## Nextcloud
### Logging into Nextcloud
Please browse to https://files.oas.example.org and try to log in. You should
have a buttin saying "Login with OpenAppStack". Try that button. Please try
logging in with your admin account, as well as the user you created in the user
panel.
Please browse to https://files.oas.example.org and try to log in using single
sign-on. Use the button labeled `Login with OpenAppStack`.
Please try logging in with your admin account and configure the email settings
as shown in the Usage doc.
After that please login with the user you created in the user panel.
### Nextcloud client application
......@@ -48,7 +42,7 @@ panel.
* If you feel like it, please try the [Nextcloud mobile client](https://nextcloud.com/clients/) for your smartphone, connect it to your OpenAppStack instance, and use it to download and/or open some files, upload a new file, etc.
## ONLYOFFICE
## Onlyoffice
### Creating a new office document
......@@ -59,17 +53,30 @@ From the main Nextcloud webpage, please try to create a new office document, by
This part of the test requires the cooperation of another person; feel free to skip it now if that's not convenient at this point.
* First, try to share your document with a different user.
* Then, try to open the shared document from a few different user accounts simultaneously, and let all participants edit the document mercilessly. There are also some collaboration features that you may want to try: on the left of the OnlyOffice screen there are buttons for chat and for text comments.
* Then, try to open the shared document from a few different user accounts simultaneously, and let all participants edit the document mercilessly.
There are also some collaboration features that you may want to try: on the left of the Onlyoffice screen there are buttons for chat and for text comments.
## Rocketchat
You can find Rocketchat at https://chat.oas.example.org.
Once you configured Rocketchat for single sign-on as desribed in the Usage docs
please login using single sign-on as `admin` and afterwards as the user you
created.
## Wordpress
## Rocket.Chat
You can find Wordpress at https://www.oas.example.org.
Please try to login as the new user you created earlier by pressing "Log in" and
using the `Login with OpenID Connect` button.
You can find Rocket.Chat at https://chat.oas.example.org. Please go there and
try to log in.
At the moment Administrator privileges will not be available for single sign-on
users of WordPress. You can sign in with the automatically created administrator
account. The username is `admin` and the password can be found in the
`wordpress_admin_password` file in the `secrets` folder of your provisioning
machine's config directory.
Note that at the moment we have not integrated Rocket.Chat with the single
sign-on system yet. You can only sign in with the automatically created
administrator account. The username is `admin` and the password can be found in
`clusters/my-cluster/secrets/rocketchat_admin_password`
## Providing feedback
......
# Troubleshooting
Note: `cluster$` indicates that the commands should be run as root on your OAS cluster.
If you run into problems, there are a few things you can do to research the
problem. This document describes what you can do.
> **NOTE:** `cluster$` indicates that the commands should be run as root on your
> OAS machine.
## Known issues
Take a look if the problem you have encountered is already in our [issue
tracker](https://open.greenhost.net/groups/openappstack/-/issues).
## Run the cli tests
......@@ -106,7 +115,7 @@ then:
#### Behave tests
##### Using Firefox instead of Chromium
##### Using Firefox instead of Chromium
If you want to use Firefox instead of Chromium, you need to install the gecko
driver
......@@ -141,22 +150,46 @@ applications.
behave -D nextcloud.url=https://files.ci-20410.ci.openappstack.net \
-D nextcloud.admin.password=…
## SSH access
You can SSH login to your VPS. Some programs that are available to the root user
on the VPS:
* `kubectl`, the Kubernetes control program. The root user is connected to the
cluster automatically.
* `helm` is the "Kubernetes package manager". Use `helm ls` to see what apps are
installed in your cluster. You can also use it to perform manual upgrades;
see `helm --help`.
## Using kubectl to debug your cluster
You can use `kubectl`, the Kubernetes control program, to find and manipulate
your Kubernetes cluster. Once you have installed `kubectl`, to get access to your
cluster with the OAS CLI:
$ python -m openappstack my-cluster info
## Upgrading
Look for these lines:
If you encounter problems when you upgrade your cluster, please make sure first
to include all potential new values of `ansible/group_vars/all/settings.yml.example`
to your `clusters/YOUR_CLUSTERNAME/group_vars/all/settings.yml`, and rerun the installation
script.
To use kubectl with this cluster, copy-paste this in your terminal:
export KUBECONFIG=/home/you/projects/openappstack/clusters/my-cluster/secrets/kube_config_cluster.yml
Copy the whole `export` line into your terminal. In *the same terminal window*,
kubectl will connect to your cluster.
## HTTPS Certificates
OAS uses [cert-manager](http://docs.cert-manager.io/en/latest/) to automatically
fetch [Let's Encrypt](https://letsencrypt.org/) certificates for all deployed
services. If you experience invalid SSL certificates (i.e. your browser warns you
when visiting Nextcloud (`https://files.YOUR.CLUSTER.DOMAIN`) here's how to
debug this:
services. If you experience invalid SSL certificates, i.e. your browser warns you
when visiting Rocketchat (`https://chat.example.org`), here's how to
debug this. A useful resource for troubleshooting is also the official cert-manager
[Troubleshooting Issuing ACME Certificates](https://cert-manager.io/docs/faq/acme/)
documentation.
In this example we fix a failed certificate request for `chat.example.org`.
We will start by checking if `cert-manager` is set up correctly.
Did you create your cluster using the `--acme-staging` argument?
Please check the resulting value of the `acme_staging` key in
......@@ -166,26 +199,72 @@ which can't be validated by default in your browser.
Are all cert-manager pods in the `oas` namespace in the `READY` state ?
cluster$ kubectl -n oas get pods | grep cert-manager
$ kubectl -n oas get pods | grep cert-manager
Are there any `cm-acme-http-solver-*` pods still running, indicating that there
are unfinished certificate requests ?
cluster$ kubectl get pods --all-namespaces | grep cm-acme-http-solver
$ kubectl get pods --all-namespaces | grep cm-acme-http-solver
Show the logs of the main `cert-manager` pod:
cluster$ kubectl -n oas logs -l "app.kubernetes.io/name=cert-manager"
$ kubectl -n oas logs -l "app.kubernetes.io/name=cert-manager"
You can `grep` for your cluster domain or for any specific subdomain to narrow
down results.
Query for failed certificates, -requests, challenges or orders:
$ kubectl get --all-namespaces certificate,certificaterequest,challenge,order | grep -iE '(false|pending)'
oas-apps certificate.cert-manager.io/oas-rocketchat False oas-rocketchat 15h
oas-apps certificaterequest.cert-manager.io/oas-rocketchat-2045852889 False 15h
oas-apps challenge.acme.cert-manager.io/oas-rocketchat-2045852889-1775447563-837515681 pending chat.example.org 15h
oas-apps order.acme.cert-manager.io/oas-rocketchat-2045852889-1775447563 pending 15h
We see that the Rocketchat certificate resources are in a bad state since 15h.
Show certificate resource status message:
$ kubectl -n oas-apps get certificate oas-rocketchat -o jsonpath="{.status.conditions[*]['message']}"
Waiting for CertificateRequest "oas-rocketchat-2045852889" to complete
We see that the `certificate` is waiting for the `certificaterequest`, lets
query it's status message:
$ kubectl -n oas-apps get certificaterequest oas-rocketchat-2045852889 -o jsonpath="{.status.conditions[*]['message']}"
Waiting on certificate issuance from order oas-apps/oas-rocketchat-2045852889-1775447563: "pending"
Show the related order resource and look at the status and events:
kubectl -n oas-apps describe order oas-rocketchat-2045852889-1775447563
Show the failed challenge resource reason:
$ kubectl -n oas-apps get challenge oas-rocketchat-2045852889-1775447563-837515681 -o jsonpath='{.status.reason}'
Waiting for http-01 challenge propagation: wrong status code '503', expected '200'
In this example, deleting the challenge fixed the issue and a proper certificate
could get fetched:
$ kubectl -n oas-apps delete challenges.acme.cert-manager.io oas-rocketchat-2045852889-1775447563-837515681
## Application installation fails
Find applications that fail to install:
helm ls --all | grep -i -v DEPLOYED
kubectl get helmreleases --all-namespaces | grep -i -v DEPLOYED
Especially the nextcloud installation process is brittle and error-prone.
Lets take it as an example how to debug the root cause.
## Purge OAS and install from scratch
If ever things fail beyond possible recovery, here's how to completely purge an OAS installation in order to start from scratch:
cluster$ apt purge docker-ce-cli containerd.io
cluster$ mount | egrep '^(tmpfs.*kubelet|nsfs.*docker)' | cut -d' ' -f 3 | xargs umount
cluster$ systemctl reboot
cluster$ mount | egrep '^(.*kubelet|nsfs.*docker)' | cut -d' ' -f 3 | xargs umount
cluster$ rm -rf /var/lib/docker /var/lib/OpenAppStack /etc/kubernetes /var/lib/etcd /var/lib/rancher /var/lib/kubelet /var/log/OpenAppStack /var/log/containers /var/log/pods
cluster$ systemctl reboot
......@@ -9,13 +9,13 @@ intervention.
```
cd CLUSTER_DIR
mkdir ./group_vars/all
mkdir -p ./group_vars/all/
mv settings.yml ./group_vars/all/
```
* [Flux](https://fluxcd.io) is now used to install and update applications. For
that reason, we need you to remove all helm charts (WARNING: You will lose
your data!):
your data!):
```
helm delete --purge oas-test-cert-manager oas-test-local-storage \
......@@ -26,4 +26,10 @@ intervention.
for these applications. Find all the pvcs by running `kubectl get pvc
--namespace oas-apps` and `kubectl get pvc --namespace oas`
## Troubleshooting
If you encounter problems when you upgrade your cluster, please make sure first
to include all potential new values of
`ansible/group_vars/all/settings.yml.example` to your
`clusters/YOUR_CLUSTERNAME/group_vars/all/settings.yml`, and rerun the
installation script.
# Usage
After all the applications are installed, the first thing to do is log into
https://admin.oas.example.org. Here you can find the "user panel", a place where
you can create, edit and delete users. You can log in with the user "admin". The
password can be found in
`clusters/my-cluster/secrets/userbackend_admin_password`. After logging in, you
will see an overview of all the applications your user has access to. For more
information on how to create users and give them access to applications, take a
look at the [user panel
documentation](https://docs.openappstack.net/projects/user-panel/en/latest/).
> **NOTE:** at the moment none of the applications are available at
> `oas.example.org`, we only provide applications in subdomains. In the future
> this might change.
## Applications
These applications are available after the installation is completed
successfully:
### OAS User panel
The [OAS user panel](https://open.greenhost.net/openappstack/user-panel/)
can be used to create and edit users. These users can be used to log into the
applications listed below.
The user panel is available at https://admin.oas.example.org. You can login
as `admin` using the `userbackend_admin_password` password from your secrets
folder.
After logging in to the user panel follow the [user panel documentation](https://docs.openappstack.net/projects/user-panel/en/latest/#creating-a-new-user)
to create a new user.
*Note*: The email address is important because some applications need a valid
email address for notification mails.
Single sign-on with Grafana will fail for users lacking an email address.
You can now use the new user to log in to all apps which were granted access to
in the last step using single sign-on.
### Nextcloud
[Nextcloud](https://nextcloud.com/) is a file sharing and communication
platform and is available at https://files.oas.example.org.
#### Single sign-on
Nextcloud needs to be configured to properly send out emails.
You can do so by logging in as `admin` using signle sign-on and then going to
`Settings -> Basic settings -> Email server` and entering your SMTP email
config and credentials.
Please complete this configuration before you login as non-admin user using
single sign-on, otherwise the [first login will not succeed](https://open.greenhost.net/openappstack/openappstack/issues/508).
### Onlyoffice
[Onlyoffice](https://www.onlyoffice.com/connectors-nextcloud.aspx) is an online
document editing suite. Your can open documents in Onlyoffice by clicking them in Nextcloud. You can open new documents by clicking the "Plus" button in Nextcloud and selecting Document, Spreadsheet or Presentation.
### Rocketchat
[Rocketchat](https://rocket.chat/) is a team chat application and available at
https://chat.oas.example.org.
#### Single sign-on
Until we [fully automate SSO integration for Rocketchat](https://open.greenhost.net/openappstack/openappstack/issues/516)
manual intervention is neccessary to activate it. You need to follow these steps once:
- Log in as `admin` using the `rocketchat_admin_password` from your secrets
folder.
- On the top left side click on the `Options` button (three dots) and then click
on `Administration`
- In the left menu scroll down and click on `OAuth` (not `oauth apps`)
- Click on `add custom oauth` and enter `Openappstack`
- Click on the newly added `Custom OAuth: Openappstack` provider
- Change the following settings (leave all others like they are):
- Enable: `True`
- URL: `https://sso.oas.example.org` (change `oas.example.org` to your domain)
- Token Path: `/oauth2/token`
- Identity Path: `/userinfo`
- Authorize Path: `/oauth2/auth`
- Scope: `openid profile openappstack_roles email`
- Id: `rocketchat`
- Secret: Paste the `rocketchat_oauth_client_secret` from your secrets folder
- Login Style: `Redirect`
- Button Text: `Login with OpenAppStack`
- Username field: `preferred_username`
- Name field: `name`
- Roles/Groups field name: `openappstack_roles`
- Merge roles from SSO: `True`
- Merge users: `True`
- Click `Save changes`, log out and you are done.
Next time you log in to Rocketchat you will be able to use single sign-on using
the `Login` button.
### Known issues
- [Rocketchat isn't configured yet to send out email notifications](https://open.greenhost.net/openappstack/openappstack/issues/510)
### Wordpress
[Wordpress](https://wordpress.com) is a website content management system and
available at https://www.oas.example.org.
Click the `Log in` button and then click `Login with OpenID Connect` to use
single sign-on.
#### Single sign-on
- If you [log in as `admin` using single sign-on, you will not have
admin rights within Wordpress](https://open.greenhost.net/openappstack/single-sign-on/issues/33).
In order to use admin rights you need to log in without single sign-on using the
`wordpress_admin_password` password in the `secrets` folder.
### Grafana
[Grafana](https://grafana.com) that shows you information about the status of
your cluster.
Read more about Grafana in the [monitoring chapter below](#monitoring)
#### Single sign-on
- If you [log in as `admin` using single sign-on, you will not have
admin rights within Grafana](https://open.greenhost.net/openappstack/single-sign-on/issues/32).
In order to use admin rights you need to log in without signgle sign-on using the
`grafana_admin_password` password in the `secrets` folder.
### Other applications installed into the cluster
Besides these applications, some other components are installed.
These are part of the OpenAppStack back end and they dont't have a user facing
web interfaces, but we like to list them here for reference:
* [OAS local-storage](https://open.greenhost.net/openappstack/local-storage) provides an easy way for the cluster to use a directory on
the node (by default `/var/lib/OpenAppStack/local-storage`) for storage;
* [NGINX](https://www.nginx.com) is a webserver that functions as a so-called ingress controller,
routing web traffic that enters the cluster to the various applications;
* [cert-manager](https://cert-manager.io) acquires and stores [Let's
Encrypt](https://letsencrypt.org/) certificates, enabling encrypted web
traffic to all applications running in the cluster;
* [Flux](https://fluxcd.io) checks for application updates approved by the
OpenAppStack team and installs them automatically.
......@@ -11,7 +11,7 @@ spec:
releaseName: nc
chart:
git: https://open.greenhost.net/openappstack/nextcloud
ref: bef4c1940082df69a9dab931f5b02665228be733
ref: 7a493d320b7a41b08ca78c1b785365239a23ed62
path: .
valuesFrom:
- secretKeyRef:
......
......@@ -10,7 +10,7 @@ spec:
releaseName: single-sign-on
chart:
git: https://open.greenhost.net/openappstack/single-sign-on
ref: d0a4553a2b7e01bf7a0cde58f093cb78d85222aa
ref: 0.2.0
path: ./helmchart/single-sign-on/
valuesFrom:
- secretKeyRef:
......
apiVersion: helm.fluxcd.io/v1
kind: HelmRelease
metadata:
name: wordpress
namespace: oas-apps
annotations:
flux.weave.works/automated: "false"
spec:
releaseName: wordpress
chart:
git: https://code.greenhost.net/open/wordpress-helm
ref: 858527ebcdc57401939491ca1a4d7bf7823818df
path: .
valuesFrom:
- secretKeyRef:
name: wordpress-settings
key: values.yaml
timeout: 1800
......@@ -216,6 +216,10 @@ class Cluster:
secret_directory, 'rocketchat_admin_password'), 'r') as stream:
rocketchat_admin_password = yaml.safe_load(stream)
with open(os.path.join(
secret_directory, 'wordpress_admin_password'), 'r') as stream:
wordpress_admin_password = yaml.safe_load(stream)
with open(os.path.join(
secret_directory, 'grafana_admin_password'), 'r') as stream:
grafana_admin_password = yaml.safe_load(stream)
......@@ -242,6 +246,12 @@ class Cluster:
behave_config['behave.userdata']['rocketchat.password'] = \
rocketchat_admin_password
behave_config['behave.userdata']['wordpress.url'] = \
'https://www.{}/openappstack-login'.format(self.domain)
behave_config['behave.userdata']['wordpress.username'] = 'admin'
behave_config['behave.userdata']['wordpress.password'] = \
wordpress_admin_password
behave_config['behave.userdata']['grafana.url'] = \
'https://grafana.{}'.format(self.domain)
behave_config['behave.userdata']['grafana.username'] = 'admin'
......
......@@ -24,7 +24,6 @@ packaging==19.2
paramiko==2.7.1
parse==1.12.1
parse-type==0.5.2
pkg-resources==0.0.0
pluggy==0.13.1
psutil==5.6.7
py==1.8.0
......
......@@ -13,3 +13,6 @@ def before_tag(context, tag):
if tag == 'rocketchat':
context.rocketchat = get_values(userdata, 'rocketchat')
if tag == 'wordpress':
context.wordpress = get_values(userdata, 'wordpress')
......@@ -6,3 +6,9 @@ from oas_behave.common_steps import *
def step_impl(context):
"""Open rocketchat URL."""
context.behave_driver.get(context.rocketchat['url'])
@when(u'I open the wordpress URL')
@given(u'I open the wordpress URL')
def step_impl(context):
"""Open wordpress URL."""
context.behave_driver.get(context.wordpress['url'])
@wordpress
Feature: Test WordPress admin login
As an OAS admin
I want to be able to login to WordPress as the user admin
Scenario: Open WordPress
When I open the wordpress URL
Then I wait on element "#user_login" for 25000ms to be visible
And I expect that element "#user_pass" is visible
Scenario: Login to WordPress
Given the element "#user_login" is visible
When I enter the "wordpress" "username" in the inputfield "#user_login"
And I enter the "wordpress" "password" in the inputfield "#user_pass"
And I click on the button "#wp-submit"
Then I wait on element "#wpwrap" for 25000ms to be visible
......@@ -5,6 +5,8 @@ import re
@pytest.mark.prometheus
def test_prometheus_alerts(host):
"""Test prometheus for firing alerts."""
def summarize_alerts(alerts):
"""Print a alert summary."""
print('Total alerts: %s' % len(alerts))
......@@ -12,7 +14,8 @@ def test_prometheus_alerts(host):
print("Starting prometheus test...")
url = 'http://127.0.0.1:30090/api/v1/alerts'
url = 'http://monitoring-prometheus-oper-prometheus.oas.svc.cluster.local:9090/api/v1/alerts'
alert_json = json.loads(host.check_output('curl ' + url))
status = alert_json["status"]
alerts = alert_json["data"]["alerts"]
......@@ -38,32 +41,54 @@ def test_prometheus_alerts(host):
assert status == "success", "Failure queriying the prometheus api at" + url
assert count == 0, "Firing alerts: {0}".format(count)
def ignore_alert(alert):
"""
Returns true if an alert should be ignored. This is when the "severity"
equals "none", or in some application specific cases.
"""
if alert["labels"]["severity"] == "none":
return True
# Ignore `KubeAPILatencyHigh` fom high load during installation
# phase
# Ignore `KubeAPILatencyHigh` fom high load during installation
# phase
if alert["labels"]["alertname"] == "KubeAPILatencyHigh":
return True
# Filter out failing Nextcloud installation jobs since a lot of
# them fail until they succeed during installation
if "nextcloud" in alert["labels"]["pod"]:
if alert["labels"]["alertname"] in [
"KubeJobFailed",
"KubeJobCompletion",
"KubePodNotReady"]:
return True
# Filter out when RocketChat pods take very long to start
if "rocketchat" in alert["labels"]["pod"]:
if alert["labels"]["alertname"] in [
"KubePodNotReady",
"KubeDeploymentReplicasMismatch",
"KubeStatefulSetReplicasMismatch"]:
return True
if 'pod' in alert["labels"]:
# Filter out failing Nextcloud installation jobs since a lot of
# them fail until they succeed during installation
if "nextcloud" in alert["labels"]["pod"]:
if alert["labels"]["alertname"] in [
"KubeJobFailed",
"KubeJobCompletion",
"KubePodNotReady"]:
return True
# Filter out when some apps take too long to start
if re.search("(rocketchat|wordpress)", alert["labels"]["pod"]):
if alert["labels"]["alertname"] in [
"KubePodNotReady",
"KubeDeploymentReplicasMismatch",
"KubeStatefulSetReplicasMismatch"]:
return True
# Filter out failed signgle-sign-on pods until we fix
# https://open.greenhost.net/openappstack/single-sign-on/issues/26
if "single-sign-on-create-" in alert["labels"]["pod"]:
if alert["labels"]["alertname"] in ["KubePodNotReady"]:
return True
if 'job_name' in alert["labels"]:
# Filter out failed signgle-sign-on jobs until we fix
# https://open.greenhost.net/openappstack/single-sign-on/issues/26
if "single-sign-on-create-" in alert["labels"]["job_name"]:
if alert["labels"]["alertname"] in [
"KubeJobFailed",
"KubeJobCompletion"]:
return True
return False
......