Merge branch 'pre-release/v0.8' into 'v0.8'

Release version 0.8.5

See merge request !1326

.gitlab/ci_pipelines/apps_ready.yml

@@ -63,6 +63,24 @@ taiko:
       when: never
     - when: always
 
+cypress:
+  stage: integration-test
+  image:
+    name: cypress/included:10.3.0
+    entrypoint: [""]
+  variables:
+    CYPRESS_BASE_URL: "https://grafana.$FQDN"
+    CYPRESS_USE_SSO_LOGIN: "true"
+    CYPRESS_SSO_USER: "admin@$FQDN"
+    CYPRESS_SSO_PW: "$SSO_PASSWORD"
+  script:
+    - "cd test && cypress run --record --key $CYPRESS_RECORD_KEY"
+  interruptible: true
+  rules:
+    - if: '$CYPRESS_TEST == "true"'
+      when: always
+    - when: never
+
 helm-test:
   stage: integration-test
   image:
@@ -76,3 +94,8 @@ helm-test:
     - if: '$HELM_TEST == "true"'
       when: always
     - when: never
+  # TODO: REMOVE THIS
+  # Had to allow failure again because the new onlyoffice plugin makes helm test
+  # fail. This has been fixed in a newer chart version, but not in the v0.8
+  # branch
+  allow_failure: true

.gitlab/ci_pipelines/default.yml

@@ -198,6 +198,8 @@ monitoring-app-ready:
     ARTIFACT_JOB: create-vps
     RESOURCE: "monitoring"
     STACKSPIN_BRANCH: $INSTALL_STACKSPIN_BRANCH
+    SKIP_TAIKO: "true"
+    CYPRESS_TEST: "true"
   extends:
     - .monitoring_rules
     - .trigger_apps_ready_pipeline

.gitlab/ci_pipelines/install_stackspin.yml

@@ -19,6 +19,7 @@ install-stackspin:
   stage: install-stackspin
   variables:
     GIT_STRATEGY: "${INSTALL_GIT_STRATEGY}"
+    SSO_PASSWORD: "$SSO_PASSWORD"
   script:
     # For upgrade_test pipelines, we install an older version (latest stable)
     - |
@@ -41,6 +42,10 @@ install-stackspin:
     - kubectl apply -k ${CLUSTER_DIR}
     # Add an override so cert-manager uses the SSL.com ClusterIssuer
     - kubectl apply -f ./install/overrides/ci/stackspin-cert-manager-override.yaml
+    # Pre-create this secret with only a single key, so we have a static SSO
+    # password. This will be augmented by generate_secrets.py, generating all
+    # keys we're not setting here.
+    - kubectl create secret generic -n flux-system stackspin-single-sign-on-variables --from-literal=userbackend_admin_password=$SSO_PASSWORD
     # Install flux and general, non-app specific secrets
     # FIXME: We override the CI_COMMIT_REF_NAME variable here because it was
     # used by the v0.8 install script. This can be removed after 0.9 has been

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/zricethezav/gitleaks
-    rev: v8.8.12
+    rev: v8.9.0
     hooks:
       - id: gitleaks-docker
   # Enable if you want to lint your commit msgs according to

CHANGELOG.md

 # Changelog
 
+## [0.8.5]
+
+### Features
+
+* Override Kratos' default account recovery email (!1293)
+* Migrate grafana taiko tests to cypress (!1300)
+* Batch user import in the Dashboard
+* A new "User" in the dashboard now gets "User" roles for all apps by default
+
+### Known issues
+
+* Although it is now possible to host applications on custom URLs, the
+  dashboard's home page still points to the default URLs.
+
+### Bug fixes
+
+* Add wekan and zulip to stackspin secrets cli subcmd (!1311)
+* Headers / reverse proxy settings do not read the IP address (in at least Zulip) (!1299)
+* Investigate why Wekan test fails some times for main CI pipeline (!1287)
+
+### Documentation
+
+* docs: Move relevant SSO documentation to stackspin/stackspin (!1294)
+
+### Updates
+
+* chore(deps): update dependency alpine_3_16/chromium to v102.0.5005.167-r0 (!1309)
+* chore(deps): update dependency alpine_3_16/yq to v4.25.1-r2 (!1304)
+* chore(deps): update dependency sphinx to v5.1.1 (!1317)
+* chore(deps): update helm release cert-manager to v1.9.0 (!1306)
+* chore(deps): update helm release cert-manager to v1.9.1 (!1315)
+* chore(deps): update helm release kube-prometheus-stack to v38.0.3 (!1322)
+* chore(deps): update helm release kube-prometheus-stack to v38 (!1307)
+* chore(deps): update helm release loki to v2.13.3 (!1318)
+* chore(deps): update helm release mariadb to v11.1.1 (!1316)
+* chore(deps): update helm release nextcloud-onlyoffice to v0.10.4 (!1308)
+* chore(deps): update helm release nextcloud-onlyoffice to v0.10.6 (!1319)
+* chore(deps): update helm release promtail to v6.2.2 (!1297)
+* chore(deps): update helm release stackspin-dashboard to v1.2.0 (!1323)
+* chore(deps): update helm release wordpress to v0.7.2 (!1320)
+* chore(deps): update pre-commit hook zricethezav/gitleaks to v8.9.0 (!1321)
+
+### Current application versions:
+
+| name                      | chart  | app_version           |
+|---------------------------|--------|-----------------------|
+| cert-manager              | v1.9.1 | v1.9.1                |
+| dashboard                 | 1.2.0  | 0.2.8                 |
+| eventrouter               | 0.4.0  | 0.3                   |
+| hydra                     | 0.23.3 | v1.10.5               |
+| ingress-nginx             | 4.2.0  | 1.3.0                 |
+| kratos                    | 0.23.3 | v0.9.0-alpha.2        |
+| kube-prometheus-stack     | 38.0.3 | 0.57.0                |
+| local-path-provisioner    | 0.0.22 | v0.0.22               |
+| loki                      | 2.13.3 | v2.6.1                |
+| metallb                   | 3.0.11 | 0.12.1                |
+| missing-container-metrics | 0.25.0 | 0.25.0                |
+| nextcloud & onlyoffice    | 0.10.6 | NC-23.0.3-OO-7.1.1.23 |
+| promtail                  | 6.2.2  | 2.6.1                 |
+| single-sign-on-database   | 11.1.1 | 10.6.8                |
+| velero                    | 2.30.1 | 1.9.0                 |
+| wekan                     | 1.1.1  | 5.93                  |
+| wordpress                 | 0.7.2  | 6.0.1                 |
+| zulip                     | 0.2.1  | 5.3-0                 |
+
 ## [0.8.4]
 
 ### Features
@@ -62,7 +127,7 @@ Current application versions:
 | loki                      | 2.13.1  | v2.6.0                |
 | metallb                   | 3.0.10  | 0.12.1                |
 | missing-container-metrics | 0.25.0  | 0.25.0                |
-| nc                        | 0.10.1  | NC-23.0.3-OO-7.0.1.37 |
+| nextcloud & onlyoffice    | 0.10.1  | NC-23.0.3-OO-7.0.1.37 |
 | promtail                  | 6.2.1   | 2.6.0                 |
 | single-sign-on-database   | 11.0.14 | 10.6.8                |
 | velero                    | 2.30.1  | 1.9.0                 |

Dockerfile

@@ -16,7 +16,7 @@ ENV BASH_VERSION="5.1.16-r2"
 # renovate: datasource=repology depName=alpine_3_16/cargo
 ENV CARGO_VERSION="1.60.0-r2"
 # renovate: datasource=repology depName=alpine_3_16/chromium versioning=loose
-ENV CHROMIUM_VERSION="102.0.5005.158-r0"
+ENV CHROMIUM_VERSION="102.0.5005.167-r0"
 # renovate: datasource=repology depName=alpine_3_16/coreutils version=loose
 ENV COREUTILS_VERSION="9.1-r0"
 # renovate: datasource=repology depName=alpine_3_16/curl
@@ -30,7 +30,7 @@ ENV GIT_VERSION="2.36.2-r0"
 # renovate: datasource=repology depName=alpine_3_16/jq versioning=loose
 ENV JQ_VERSION="1.6-r1"
 # renovate: datasource=repology depName=alpine_edge/kubectl
-ENV KUBECTL_VERSION="1.24.2-r1"
+ENV KUBECTL_VERSION="1.24.3-r1"
 # renovate: datasource=repology depName=alpine_3_16/libffi-dev
 ENV LIBFFI_DEV_VERSION="3.4.2-r1"
 # renovate: datasource=repology depName=alpine_3_16/libsodium-dev
@@ -42,15 +42,15 @@ ENV MOREUTILS_VERSION="0.67-r0"
 # renovate: datasource=repology depName=alpine_3_16/npm
 ENV NPM_VERSION="8.10.0-r0"
 # renovate: datasource=repology depName=alpine_3_16/openssh-client-default versioning=loose
-ENV OPENSSH_CLIENT_DEFAULT_VERSION="9.0_p1-r1"
+ENV OPENSSH_CLIENT_DEFAULT_VERSION="9.0_p1-r2"
 # renovate: datasource=repology depName=alpine_3_16/py3-pip
 ENV PY3_PIP_VERSION="22.1.1-r0"
 # renovate: datasource=repology depName=alpine_3_16/python3-dev
-ENV PYTHON3_DEV_VERSION="3.10.4-r0"
+ENV PYTHON3_DEV_VERSION="3.10.5-r0"
 # renovate: datasource=repology depName=alpine_3_16/rsync
-ENV RSYNC_VERSION="3.2.4-r0"
+ENV RSYNC_VERSION="3.2.4-r1"
 # renovate: datasource=repology depName=alpine_3_16/yq
-ENV YQ_VERSION="4.25.1-r1"
+ENV YQ_VERSION="4.25.1-r2"
 
 # Makes pynacl use system SODIUM
 ENV SODIUM_INSTALL=system

VERSION

-0.8.4
+0.8.5

docs/conf.py

@@ -33,6 +33,7 @@ with open('../VERSION') as version_file:
 extensions = [
     'recommonmark',
     'sphinx.ext.autosectionlabel',
+    'sphinxcontrib.mermaid',
     'sphinx_design',
     'versionwarning.extension',
 ]

docs/index.rst

@@ -59,3 +59,4 @@ For more information, go to `the Stackspin website`_.
 
    reference/reference
    reference/comparable_projects
+   reference/sso_architecture

docs/reference/sso_architecture.rst

+Single sign-on Architecture
+===========================
+
+The single sign-on system consists of a few components:
+
+1. The *OAuth 2* and *OpenID Connect (OIDC) provider*, `Hydra`_
+2. The *Identity provider*, `Kratos`_
+3. The Login application which serves as a login panel, consent application and
+   a settings screen for the Kratos settings. `The login application code is
+   part of the Dashboard Backend repository
+   <https://open.greenhost.net/stackspin/dashboard-backend/-/tree/main/web>`__
+
+.. _Hydra: https://www.ory.sh/hydra/docs/
+.. _Kratos: https://www.ory.sh/kratos/docs/
+
+Overview
+--------
+
+The single sign-on system is a combination of applications that serve as a
+central user database and authenticates them to other applications. Users are
+stored inside Kratos's database.  Kratos also serves an API that helps us
+generate interfaces for many user-related tasks, such as:
+
+1. Setting your name and username
+2. The whole password reset flow
+3. The login form
+4. 2FA (not implemented in the login application yet)
+
+The Login application is mostly a front-end that uses the Kratos API to generate
+interfaces for logging in, resetting the password and setting some user data.
+
+
+Flows
+-----
+
+Logging in
+~~~~~~~~~~
+
+The Kratos login flow is documented `in the Kratos documentation
+<https://www.ory.sh/kratos/docs/self-service/flows/user-login#login-for-client-side-ajax-browser-clients>`__.
+Our implementation is slightly different from what you see there:
+
+.. mermaid::
+
+   sequenceDiagram
+      participant B as Browser
+      participant L as Login application
+      participant K as Kratos
+
+      B->>L: User clicks "Login with Stackspin"
+      L->L: Check if cookie for current session exists
+      alt Cookie does not exist
+        L-->>+K: Start `/self-service/login/browser` flow
+        K-->>-B: At end of login flow, sets session cookie
+      else Cookie exists
+        L->>B: Shows "you are already logged in" screen
+        B->B: If cookie has `flow_state == auth`, redirect to Cookie's `auth_url` and remove `flow_state`.
+      end
+
+User creation
+~~~~~~~~~~~~~
+
+We have not implemented Kratos's *Registration* flow, because users cannot
+self-register with a Stackspin cluster. An administrator can make new users
+using the Dashboard application. When a user is created, an email address always
+needs to be provided.
+
+Once a user has been created, they can start the `Account Recovery and Password
+Reset flow
+<https://www.ory.sh/kratos/docs/self-service/flows/account-recovery>`__ in order
+to set or reset their password. We use the "Recovery ``link`` Method" described
+in the Kratos documentation.
+
+User settings
+~~~~~~~~~~~~~
+
+Although users can change their settings through the `Dashboard application
+<https://open.greenhost.net/stackspin/dashboard>`__, the login application also
+implements the `user-settings Kratos flow
+<https://www.ory.sh/kratos/docs/next/self-service/flows/user-settings/>`__.
+Users that receive a password recovery link use this flow to reset their
+passwords. It also allows them to change their username and full name.
+
+Authentication
+~~~~~~~~~~~~~~
+
+The following is an adaptation of the sequence diagram provided in the `Hydra
+documentation <https://www.ory.sh/docs/hydra/concepts/login>`__
+
+.. mermaid::
+
+   sequenceDiagram
+      OAuth 2 Client->>Ory Hydra: Initiates OAuth 2 Authorize Code or Implicit Flow
+      Ory Hydra-->>Ory Hydra: No end user session available (not authenticated)
+      opt Login Application as Login Provider
+        Ory Hydra->>Login Application: Redirects end user with login challenge
+        Login Application-->Ory Hydra: Fetches login info
+        Login Application-->>Login Application: Authenticates user w/ Kratos
+        Login Application-->Ory Hydra: Transmits login info and receives redirect url with login verifier
+        Login Application->>Ory Hydra: Redirects end user to redirect url with login verifier
+      end
+      Ory Hydra-->>Ory Hydra: First time that client asks user for permissions
+      opt Login Application as Consent Provider
+        Ory Hydra->>Login Application: Redirects end user with consent challenge
+        Login Application-->Ory Hydra: Fetches consent info (which user, what app, what scopes)
+        Note over Ory Hydra, Login Application: Not implemented: user is asked to grant app access<br />default: access granted
+        Login Application-->Ory Hydra: Transmits consent result and receives redirect url with consent verifier
+        Login Application->>Ory Hydra: Redirects to redirect url with consent verifier
+      end
+      Ory Hydra-->>Ory Hydra: Verifies grant
+      Ory Hydra->>OAuth 2 Client: Transmits authorization code/token`
+
+Configuring OIDC clients
+------------------------
+
+If you have installed the SSO system using the Helm chart, following this
+documentation, these are the settings that you usually need to for setting up
+new OIDC clients:
+
+- OAuth 2 server URL: ``https://sso.stackspin.example.org``
+- OAuth 2 Auth Endpoint: ``https://sso.stackspin.example.org/oauth2/auth``
+- OAuth 2 Userinfo endpoint: ``https://sso.stackspin.example.org/userinfo``
+- OAuth 2 Token endpoint: ``https://sso.stackspin.example.org/oauth2/token``
+
+In addition you'll need to add the client to Hydra. This happens with `Hydra
+Maester
+<https://www.ory.sh/hydra/docs/guides/kubernetes-helm-chart/#hydra-maester>`__,
+a helper application that reads ``oauth2clients.hydra.ory.sh`` Kubernetes
+objects and synchronises them with Hydra.
+
+An example ``oauth2client``:
+
+.. code:: yaml
+
+   apiVersion: hydra.ory.sh/v1alpha1
+   kind: OAuth2Client
+   metadata:
+     name: dashboard-oauth-client
+   spec:
+     grantTypes:
+       - authorization_code
+       - refresh_token
+       - client_credentials
+       - implicit
+     hydraAdmin: {}
+     metadata: null
+     redirectUris:
+     - https://dashboard.stackspin.example.org/login-callback
+     responseTypes:
+     - id_token
+     - code
+     scope: openid profile email stackspin_roles
+     secretName: stackspin-dashboard-oauth-variables
+     tokenEndpointAuthMethod: client_secret_post

docs/requirements.in

@@ -6,7 +6,8 @@
 # Stackspin to requirements-dev.txt!
 #
 recommonmark==0.7.1
-sphinx==5.0.2
+sphinx==5.1.1
 sphinx-design==0.2.0
 sphinx-rtd-theme==1.0.0
 sphinx-version-warning==1.1.2
+sphinxcontrib-mermaid==0.7.1

docs/requirements.txt

@@ -41,7 +41,7 @@ requests==2.27.1
     # via sphinx
 snowballstemmer==2.2.0
     # via sphinx
-sphinx==5.0.2
+sphinx==5.1.1
     # via
     #   -r requirements.in
     #   recommonmark
@@ -62,6 +62,8 @@ sphinxcontrib-htmlhelp==2.0.0
     # via sphinx
 sphinxcontrib-jsmath==1.0.1
     # via sphinx
+sphinxcontrib-mermaid==0.7.1
+    # via -r requirements.in
 sphinxcontrib-qthelp==1.0.3
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.5

flux2/apps/monitoring/kube-prometheus-stack-release.yaml

@@ -10,7 +10,7 @@ spec:
     spec:
       # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
       chart: kube-prometheus-stack
-      version: 37.2.0
+      version: 38.0.3
       sourceRef:
         kind: HelmRepository
         name: prometheus-community

flux2/apps/monitoring/loki-release.yaml

@@ -10,7 +10,7 @@ spec:
     spec:
       # https://artifacthub.io/packages/helm/grafana/loki
       chart: loki
-      version: 2.13.1
+      version: 2.13.3
       sourceRef:
         kind: HelmRepository
         name: grafana

flux2/apps/monitoring/promtail-release.yaml

@@ -10,7 +10,7 @@ spec:
     spec:
       # https://artifacthub.io/packages/helm/grafana/promtail
       chart: promtail
-      version: 6.2.1
+      version: 6.2.2
       sourceRef:
         kind: HelmRepository
         name: grafana

flux2/apps/nextcloud/release.yaml

@@ -9,7 +9,7 @@ spec:
   chart:
     spec:
       chart: nextcloud-onlyoffice
-      version: 0.10.1
+      version: 0.10.6
       sourceRef:
         kind: HelmRepository
         name: nextcloud

flux2/apps/wordpress/release.yaml

@@ -9,7 +9,7 @@ spec:
   chart:
     spec:
       chart: wordpress
-      version: 0.6.53
+      version: 0.7.2
       sourceRef:
         kind: HelmRepository
         name: wordpress-helm

flux2/apps/zulip/zulip-values-configmap.yaml

@@ -73,6 +73,13 @@ data:
         SECRETS_social_auth_oidc_secret: "${client_secret}"
         # Enable "low memory mode", queue workers run 1 multithreaded process
         QUEUE_WORKERS_MULTIPROCESS: 'False'
+        # Let Zulip know the IP address of our reverse proxy (ingress
+        # controller), so it knows to trust the `X-Forwarded-For` header in
+        # that case.
+        # 10.42.0.0/16 contains all ip addresses that are assigned to
+        # kubernetes pods which includes the ip address of the ingress
+        # controller.
+        LOADBALANCER_IPS: '10.42.0.0/16'
       persistence:
         existingClaim: zulip-data
 

flux2/core/base/cert-manager/release.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       # https://artifacthub.io/packages/helm/cert-manager/cert-manager
       chart: cert-manager
-      version: v1.8.2
+      version: v1.9.1
       sourceRef:
         kind: HelmRepository
         name: jetstack

flux2/core/base/dashboard/dashboard-release.yaml

@@ -14,7 +14,7 @@ spec:
   chart:
     spec:
       chart: stackspin-dashboard
-      version: 1.1.0
+      version: 1.2.0
       sourceRef:
         kind: HelmRepository
         name: dashboard