.gitignore

@@ -3,7 +3,9 @@
 # dependencies
 /node_modules
 /frontend/node_modules
+/backend/venv
 /backend/web/node_modules
+/backend/venv
 /.pnp
 .pnp.js
 

.gitlab-ci.yml

@@ -9,13 +9,13 @@ stages:
   - package-helm-chart
   - release-helm-chart
 
-image: node:18-alpine
+image: node:20-alpine
 
 variables:
   CHART_NAME: stackspin-dashboard
   CHART_DIR: deployment/helmchart/
 
-build-project:
+yarn:
   stage: build-project
   before_script: []
   script:
@@ -25,13 +25,14 @@ build-project:
     - echo "REACT_APP_API_URL=/api/v1" > .env
     - echo "EXTEND_ESLINT=true" >> .env
     - yarn build
-    - mv build web-build
+    - mkdir docker
+    - mv build docker/html
     - echo "Build successful"
   artifacts:
     expire_in: 1 hour
     name: web-build
     paths:
-      - frontend/web-build
+      - frontend/docker/html
 
 .kaniko-build:
   script:
@@ -42,6 +43,8 @@ build-project:
 
 build-frontend-image:
   stage: build-image
+  needs:
+  - yarn
   image:
     # We need a shell to provide the registry credentials, so we need to use the
     # kaniko debug image (https://github.com/GoogleContainerTools/kaniko#debug-image)
@@ -49,7 +52,7 @@ build-frontend-image:
     entrypoint: [""]
   variables:
     KANIKO_BUILD_IMAGENAME: dashboard
-    DIRECTORY: frontend/web-build
+    DIRECTORY: frontend/docker
   before_script:
     - cp deployment/Dockerfile $DIRECTORY
     - cp deployment/nginx.conf $DIRECTORY
@@ -58,6 +61,7 @@ build-frontend-image:
 
 build-backend-image:
   stage: build-image
+  needs: []
   variables:
     KANIKO_BUILD_IMAGENAME: dashboard-backend
     DIRECTORY: backend
@@ -71,6 +75,7 @@ build-backend-image:
 
 build-test-image:
   stage: build-image
+  needs: []
   variables:
     KANIKO_BUILD_IMAGENAME: cypress-test
     DIRECTORY: tests

CHANGELOG.md

 # Changelog
 
-## [0.7.6]
+## 0.13.3
+
+- Fix creating app roles for users created via the CLI.
+
+## 0.13.2
+
+- Update hydra client library to v2 and adapt to its changed API.
+- Change the jwt identity claim because that's strictly required to be a string
+  now, and we put a json object in there before.
+
+## 0.13.1
+
+- Add the `cryptography` python library as dependency of the backend. This is
+  necessary for sha256_password and caching_sha2_password auth methods.
+
+## 0.13.0
+
+- Fix the initial recovery flow created automatically for new users, which was
+  broken by the kratos client lib upgrade.
+- Add support for serving arbitrary files from the frontend pod, provided by a
+  persistent volume claim. Reorganize the assets to make it easier to include
+  custom icons this way.
+- Add support for theming. Customizing colours, logo and background image is
+  now particularly easy, but mostly anything can be changed through a custom
+  stylesheet.
+- Only show Stackspin version info to admin users.
+
+## 0.12.4
+
+- Prevent database connections from being shared between worker processes.
+  That sharing may cause intermittent database errors.
+
+## 0.12.3
+
+- Fix broken kratos hooks tracking last recovery and login times.
+- Upgrade python to 3.13.
+
+## 0.12.2
+
+- Fix consent page for `ENFORCE_2FA` instances.
+
+## 0.12.1
+
+- Add back `jinja2-base64-filters` to backend for templating kustomizations
+  during app installation.
+
+## 0.12.0
+
+- Add basic support for WebAuthn as second-factor authentication.
+- Only show app version numbers in the dashboard tiles to admin users.
+- Do not serve Dockerfile and nginx config from frontend.
+- Start renovating python dependencies of the backend. Upgrade many direct and
+  indirect dependencies.
+
+## 0.11.1
+
+- Fix password reset form in case no email address is pre-filled.
+
+## 0.11.0
+
+- Allow pre-filling user's email address in a link to the password (re)set
+  form. This is useful when creating new user accounts.
+- Fix user provisioning after installing new apps.
+
+## 0.10.5
+
+- Look up users from Kratos by email address using the proper (new) API
+  mechanism for that, instead of iterating over all users.
+- Compare email addresses case insensitively to deal with Stackspin apps
+  changing case of email address strings.
+- Fix broken user accounts when created via the flask CLI.
+- Replace slightly off-spec usage of `__repr__` by `__str__`.
+
+## 0.10.4
+
+- Disable Zulip accounts when deleting users, because Zulip doesn't allow
+  deleting accounts via SCIM.
+
+## 0.10.3
+
+- Fix setting successful provisioning status.
+
+## 0.10.2
+
+- Fine-tune logging levels, and introduce a new environment variable
+  `LOG_LEVEL` to set the log level at runtime.
+- Track when a user's full name has been changed, and only include the name in
+  the SCIM provisioning call when it has changed, or for newly provisioned
+  users.
+
+## 0.10.1
+
+- Watch dashboard configmaps with lists of apps and oauthclients, and
+  reload config on changes. This also makes sure that we always load the config
+  at dashboard start-up, even when there are no (SCIM-supporting) apps
+  installed.
+
+## 0.10.0
+
+- Include new "System resources" module with basic stats.
+- Implement basic (manual/static) SCIM functionality for automatic user provisioning.
+- Implement dynamic (i.e., arbitrary apps) SCIM functionality, tested and
+  tailored for Nextcloud and Zulip.
+- Upgrade to tailwind v3, and update several other javascript dependencies.
+- Make info modals slightly wider, to make sure you can see the full contents
+  also for slightly larger fonts. In particular, this fixes a partially
+  invisible reset link.
+- Add a CLI command for deleting older unused accounts.
+- Add logo for Gitea.
+
+## 0.9.2
+
+- Fix saving user properties, which was broken because of the partial tags
+  implementation.
+
+## 0.9.1
+
+- Do not autocomplete totp input field.
+- Allow removing user app roles from CLI.
+
+## 0.9.0
+
+- Improve user listing: show label for admin users, show last login and
+  password reset times, improved layout.
+- Fix rare bug in frontend's idea of admin status in the face of custom apps.
+- Prepare backend for user tags.
+
+## 0.8.4
+
+- Allow enforcing 2fa.
+- Add button for admin users to reset 2FA of users. Also improve UX of this and
+  other dangerous operations in the user edit screen.
+- Fix logout to include hydra post-logout.
+- Do not show link to recovery on TOTP form.
+- Fix css of demo sign-up.
+- Upgrade to python 3.12.
+
+## 0.8.3
+
+- Introduce backend code for resetting 2FA, and add cli command for that.
+- Upgrade Kratos api library `ory-kratos-client` to 1.0.0.
+- Patch our usage of Kratos api pagination of identities list.
+
+## 0.8.2
+
+- End the Kratos session in prelogout. This makes sure that we end the "SSO
+  session" also when logging out from an app. We used to rely on hydra's
+  post-logout url to get at the kratos logout, but apps sometimes override that
+  url via an oidc parameter.
+
+## 0.8.1
+
+- Add a couple of attributes to our OIDC tokens to support our switch to
+  another Nextcloud app for OIDC.
+
+## 0.8.0
+
+- Add feature to easily edit app permissions for multiple users at once.
+- Change the way secrets are created for apps, creating them in the stackspin
+  project (using an existing secrets controller). So remove support for
+  generating app secrets in the dashboard.
+- Fix password reset when 2FA is enabled.
+- Fix bug that all Wekan users get admin permissions in Wekan regardless of
+  role set in Stackspin.
+- Enable "pre-ping" for all database connections managed by sqlalchemy in the
+  dashboard backend, hoping to eliminate or reduce dropped database
+  connections.
+- Fix listing of Velero in app permissions when batch-creating users.
+
+## 0.7.6
 
 - Add Forgejo metadata for use as custom app.
 
-## [0.7.5]
+## 0.7.5
 
 - Add Jitsi and Mattermost metadata for use as custom apps.
 
-## [0.7.4]
+## 0.7.4
 
 - Make the sign-in UI less wide.
 
-## [0.7.3]
+## 0.7.3
 
 Only changes to the helm chart.
 
-## [0.7.2]
+## 0.7.2
 
 - Apply Stackspin styling to the login component. This covers the login pages,
   recovery page, and profile/authentication settings.
 
-## [0.7.1]
+## 0.7.1
 
 - Load the flask_migrate flask extension in dev/cli mode so we may run `flask
   db` commands from the cli again.
 
-## [0.7.0]
+## 0.7.0
 
 - Improve the UX of the dashboard tiles: adding help texts in modals, add a
   status dropdown with version info, add alerts before and after automatic
@@ -39,28 +208,28 @@ Only changes to the helm chart.
   This was fixed by upgrading Kratos; we're happy to see that the default
   Kratos behaviour was changed in this regard.
 
-## [0.6.7]
+## 0.6.7
 
 Only changes to the helm chart.
 
-## [0.6.6]
+## 0.6.6
 
 Only changes to the helm chart.
 
-## [0.6.5]
+## 0.6.5
 
 - Further improve (error) message handling. In particular, show feedback when
   saving profile settings. Some of the previous error message changes have been
   reverted pending further consideration of the design.
 - Disable changing the email address as this is not supported right now.
 
-## [0.6.4]
+## 0.6.4
 
 - Fix error messages that were not shown, in particular when providing wrong
   credentials when logging in. We redesigned the error handling, considering
   that these messages may be translated later on.
 
-## [0.6.3]
+## 0.6.3
 
 - Add support for Hedgedoc.
 - Add a button for admins for creating a recovery link for a user.
@@ -71,11 +240,11 @@ Only changes to the helm chart.
 - Show the user UUID in user modal.
 - Only show installed apps when configuring roles.
 
-## [0.6.2]
+## 0.6.2
 
 - Fix submit button label in the form for verifying your TOTP code.
 
-## [0.6.1]
+## 0.6.1
 
 - Add TOTP as second factor authentication. Please note that you'll need to set
   a `backend.dashboardUrl` value instead of the old `backend.loginPanelUrl` one
@@ -83,13 +252,13 @@ Only changes to the helm chart.
 - Create a new backend endpoint for providing some environment variables to the
   frontend, with the URLs of the Kratos and Hydra APIs.
 
-## [0.6.0]
+## 0.6.0
 
 - Make it easier to add apps, by reading apps and oauthclients from configmaps
   at startup.
 - Reset alembic migration history.
 
-## [0.5.2]
+## 0.5.2
 
 - Fix login welcome message
 - Clarify "set new password" button (#94)
@@ -97,11 +266,11 @@ Only changes to the helm chart.
   entered (#96)
 - Fix access checking for monitoring (#105)
 
-## [0.5.1]
+## 0.5.1
 
 - Fix bug of missing "Monitoring" app access when creating a new user.
 - Add Velero to the list of installable apps, but hide it from the dashboard
 
-## [0.5.0]
+## 0.5.0
 
 - Merge dashboard-backend repository into this repository, released as 0.5.0

README.md

@@ -52,98 +52,112 @@ it is based on traditional Bootstrap + JQuery.
 
 ## Development environment
 
-After this process is finished, the following will run in local docker containers:
-
-- the dashboard frontend
-- the dashboard backend
-
-The following will be available through proxies running in local docker containers and port-forwards:
-
-- Hydra admin API
-- Kratos admin API and public API
-- The MariaDB database
-
-These need to be available locally, because Kratos wants to run on the same
-domain as the front-end that serves the login interface.
-
-### Setup
-
-Before you start, make sure your machine has the required software installed, as per official documentation: https://docs.stackspin.net/en/v2/installation/install_cli.html#preparing-the-provisioning-machine.
-
-Please read through all subsections to set up your environment before
-attempting to run the dashboard locally.
-
-#### 1. Stackspin cluster
-
-To develop the Dashboard, you need a Stackspin cluster that is set up as a
-development environment. Follow the instructions [in the
-dashboard-dev-overrides
-repository](https://open.greenhost.net/stackspin/dashboard-dev-overrides#dashboard-dev-overrides)
-in order to set up a development-capable cluster. The Dashboard, as well as
-Kratos and Hydra, will be configured to point their endpoints to
-`http://stackspin_proxy:8081` in that cluster. As a result, you can run
-components using the `docker-compose.yml` file in this repository, and still log
-into Stackspin applications that run on the cluster.
-
-#### 2. Environment for frontend
-
-The frontend needs to know where the backend API and hydra can be reached. To
-configure it, create a `local.env` file in the `frontend` directory:
-
-    cp local.env.example local.env
-
-#### 3. Setup hosts file
-
-The application will run on `http://stackspin_proxy`. Add the following line to
-`/etc/hosts` to be able to access that from your browser:
-
-```
-127.0.0.1	stackspin_proxy
-```
-
-#### 4. Kubernetes access
-
-The `./run_app.sh` script needs to access the Kubernetes cluster that runs your Stackspin instance. If you followed the setup as above, you will have a YAML configuration file somewhere on your machine -- usually in the `clusters` directory of your Stackspin local repository -- called `kube_config_cluster.yml`. This file holds all the configuration information (URLs, domain names, certificate data) needed to connect to the instance.
-
-Copy that file into the `backend/kubeconfig` directory.
-
-If you wish to connect this dashboard to another Stackspin cluster, you can replace the `kube_config_cluster.yml` file with the one that's in that Stackspin's `clusters` directory.
-
-## 5. Build and run
-
-To recap, you now have:
-
-- All the software and configurations as described above
-- A running Stackspin cluster (a VPS somewhere in The Cloud)
-- A `kube_config_cluster.yml` file in the `backend/kubeconfig` that will tell the script how to connect to your Stackspin cluster of choice
-- Overrides for local dashboard development (by installing and running the [Dashboard Dev Overrides](https://open.greenhost.net/stackspin/dashboard-dev-overrides) repository, editing your `/etc/hosts` file, etc)
-- A copy of the [Stackspin Dashboard repository](https://open.greenhost.net/stackspin/dashboard) on your device.
-
-That's a lot of work! Good job.
-
-### Setup your local dev environment
-
-Before you actually run the main script, `cd` into the `/frontend` directory and run`yarn install`.
-
-This is not strictly necessary for development; the script already builds and installs all the necessary modules in the dashboard's docker container. But running `yarn install` locally will let your IDE enable all of its bells and whistles like linting, autocorrecting, intellisense etc. Without this step, your IDE will most probably complain it cannot find any modules to `import`, as there is no `node_modules` folder.
-
-### Let's Run this App
-
-After you've finished all setup steps, you can run everything using:
-
-```
-./run_app.sh
-```
-
-This script
-
-- sets a few environment variables based on the content in your cluster
-  secrets, and
-- runs `docker compose up` to build and run all necessary components, including a reverse proxy and the backend flask application.
-
-If you're curious about what `docker compose up` does, you can check out the `docker-compose.yml` file. If you are curious about what `docker compose up` _means,_ you can start here: https://github.com/docker/compose or even here: https://en.wikipedia.org/wiki/Infrastructure_as_code.
-
-This should be it, congratulations!! If you're having issues, or if something is not working properly, please open an issue or get in touch: info@stackspin.net
+The development environment is a hybrid one, where one or both of the dashboard
+frontend and backend run locally, but the rest of the cluster runs on a remote
+machine.
+
+The remote should be a regular Stackspin cluster, though preferably one that's
+dedicated to development purposes.
+
+The local dashboard frontend and/or backend can run in a docker container or
+directly ("native mode"). (At this time it's not possible to mix the two, for
+example by having the dashboard backend run directly and the frontend in a
+docker container.)
+
+The connection between the local and remote parts is set up by a tool called
+telepresence. If you want to develop the frontend for example, telepresence
+intercepts traffic that goes into the remote's frontend pod and redirects it to
+your copy that's running locally on your machine; responses from your local
+frontend are led back via the remote. This interception happens invisibly to
+your browser, which you just point at the remote cluster.
+
+### Prerequisites
+
+#### Set up telepresence on your local development machine
+
+You need to do this once for every development machine you're using
+(workstation, laptop).
+
+* You need root on your machine and at some point allow telepresence to perform
+  actions as root, in order to make network changes to allow the two-way
+  tunnel. If this is not possible or not desirable, you can try to run your
+  local dashboard in a docker container instead.
+* Set `user_allow_other` in `/etc/fuse.conf`. This is necessary when
+  telepresence adds (FUSE-based) sshfs mounts so your local code can access
+  volumes from the kubernetes cluster, in particular the one with the service
+  account token (credentials for calling the kubernetes api), to let the
+  dashboard interact with the cluster.
+  - MacOS users may have to do a little extra work to get a working current
+    sshfs: see [telepresence
+    docs](https://www.getambassador.io/docs/telepresence-oss/latest/troubleshooting#volume-mounts-are-not-working-on-macos).
+* Download and install the telepresence binary on your development machine:
+  https://www.getambassador.io/docs/telepresence-oss/latest/install
+
+#### Access to development cluster
+
+You need `kubectl` and `helm` binaries, and a `kubectl` configuration file
+(often called "kubeconfig") containing credentials needed to authenticate
+against your cluster. If the `KUBECONFIG` environment variable is set and
+points to the config file, this will be picked up by the various programs.
+
+#### Set up telepresence on your development cluster
+
+You need to do this once for every cluster you want to use as a development cluster.
+
+* Install telepresence on your development cluster:
+  ```
+  telepresence helm install -f telepresence-values.yaml
+  ```
+
+#### Install local dependencies
+
+Before running the frontend in native mode:
+* Make sure you have nodejs installed. You may want to use [Node Version
+  Manager](https://github.com/nvm-sh/nvm) to make it easy to install several
+  versions side by side.
+* Install necessary javascript dependencies (will be placed in
+  `frontend/node_modules`) using `./dev.sh frontend setup`.
+
+Before running the backend in native mode:
+* Make sure you have python3 installed.
+* Install necessary python dependencies (in a virtualenv in `backend/venv`)
+  using `./dev.sh backend setup`.
+
+### Run
+
+From the root `dashboard` directory, run for example `./dev.sh frontend`. This
+will set up the telepresence tunnel to the cluster, and start the dashboard
+frontend server in native mode. `./dev.sh backend` will do the same but for the
+backend. You can run both at the same time (in separate terminal windows) if
+you want to make changes to both frontend and backend.
+
+If you want to run the local dashboard in docker instead, use `./dev.sh
+frontend docker` and/or `./dev.sh backend docker`. Please note that due to a
+telepresence limitation it's not currently possible to run the frontend
+natively and the backend in docker at the same time, or vice versa.
+
+#### Known issues
+
+* Running the dashboard backend locally with telepresence in docker mode
+  currently doesn't work because of dns resolution issues in the docker
+  container: https://github.com/telepresenceio/telepresence/issues/1492 . We
+  could work around this by using a fully qualified domain name for the
+  database service -- which doesn't agree with the goal of making the stackspin
+  namespace variable -- or using the service env vars, but we're hoping that
+  telepresence will fix this in time.
+* Telepresence intercepts traffic to a pod, but the original pod is still
+  running. In case of the backend, this is sometimes problematic, for example
+  when you're adding database migrations which the original pod then doesn't
+  know about and crashes, or with SCIM which involves timer-based actions which
+  are then performed both by your modified local instance and by the original
+  remote one. There is some work in progress to allow scaling down the
+  intercepted pod: https://github.com/telepresenceio/telepresence/issues/1608 .
+* If telepresence is giving errors, in particular ones about "an intercept with
+  the same name already existing" on repeated runs, it may help to reset the
+  telepresence state by doing `./dev.sh reset`. This will stop the local
+  telepresence daemon so it can be cleanly restarted on the next try, and will
+  also restart the "traffic manager" on the remote so it will discard any old
+  lingering intercepts.
 
 ---
 

backend/Dockerfile

-FROM python:3.11-slim
+FROM python:3.13-slim
 
 # set "app" as the working directory from which CMD, RUN, ADD references
 WORKDIR /app

backend/app.py

+from apscheduler.schedulers.background import BackgroundScheduler
 from flask import Flask, jsonify
 from flask_cors import CORS
 from flask_jwt_extended import JWTManager
 import flask_migrate
 from jsonschema.exceptions import ValidationError
 from NamedAtomicLock import NamedAtomicLock
+import threading
+import traceback
 from werkzeug.exceptions import BadRequest
 
 # These imports are required
@@ -12,10 +15,16 @@ from cliapp import cli
 from web import web
 
 from areas import users
-from areas import apps
+from areas.apps.apps import *
 from areas import auth
+from areas import resources
 from areas import roles
+from areas import tags
 from cliapp import cliapp
+import config
+import helpers.kubernetes
+import helpers.provision
+import helpers.threads
 from web import login
 
 from database import db
@@ -41,22 +50,28 @@ import os
 import sys
 
 # Configure logging.
+log_level = logging.getLevelName(config.LOG_LEVEL or 'INFO')
 from logging.config import dictConfig
 dictConfig({
     'version': 1,
     'formatters': {'default': {
-        'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+        'format': '[%(asctime)s] %(levelname)s in %(name)s (%(filename)s+%(lineno)d): %(message)s',
     }},
     'handlers': {'wsgi': {
         'class': 'logging.StreamHandler',
         'stream': 'ext://flask.logging.wsgi_errors_stream',
         'formatter': 'default',
+        'level': log_level,
     }},
     'root': {
-        'level': 'INFO',
         'handlers': ['wsgi'],
-    }
+        'level': log_level,
+    },
+    # Loggers are created also by alembic, flask_migrate, etc. Without this
+    # setting, those loggers seem to be ignored.
+    'disable_existing_loggers': False,
 })
+logging.getLogger("kubernetes.client.rest").setLevel(logging.WARNING)
 
 app = Flask(__name__)
 
@@ -65,19 +80,27 @@ app.config["SQLALCHEMY_DATABASE_URI"] = SQLALCHEMY_DATABASE_URI
 app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {'pool_pre_ping': True}
 app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = SQLALCHEMY_TRACK_MODIFICATIONS
 
-app.logger.setLevel(logging.INFO)
-
 cors = CORS(app)
 
 db.init_app(app)
 
+with app.app_context():
+    provisioner = helpers.provision.Provision()
+
 def init_routines():
     with app.app_context():
         # We have reset the alembic migration history at Stackspin version 2.2.
         # This checks whether we need to prepare the database to follow that
         # change.
         migration_reset.reset()
+    app.logger.info("Loading flask_migrate.")
+    # flask_migrate exits the program when it encounters an error, for example
+    # when the version set in the database is not found in the
+    # `migrations/versions` directory. We could prevent that by catching the
+    # `SystemExit` exception here, but actually it's not safe to continue in
+    # that case.
     flask_migrate.Migrate(app, db)
+    app.logger.info("Attempting flask_migrate database upgrade.")
     try:
         with app.app_context():
             flask_migrate.upgrade()
@@ -87,20 +110,59 @@ def init_routines():
         app.logger.info(f"upgrade failed: {type(e)}: {e}")
         sys.exit(2)
 
-    # We need this app context in order to talk the database, which is managed by
-    # flask-sqlalchemy, which assumes a flask app context.
+    def reload():
+        # We need this app context in order to talk the database, which is managed by
+        # flask-sqlalchemy, which assumes a flask app context.
+        with app.app_context():
+            app.logger.info("Reloading dashboard config from cluster resources.")
+            # Load the list of apps from a configmap and store any missing ones in the
+            # database.
+            app_slugs = cluster_config.populate_apps()
+            # Same for the list of oauthclients.
+            cluster_config.populate_oauthclients()
+            # Load per-app scim config if present.
+            cluster_config.populate_scim_config(app_slugs)
+    # We could call `reload` here manually, but actually the watch also at its
+    # start creates events for existing secrets so we don't need to.
     with app.app_context():
-        # Load the list of apps from a configmap and store any missing ones in the
-        # database.
-        cluster_config.populate_apps()
-        # Same for the list of oauthclients.
-        cluster_config.populate_oauthclients()
-
-# `init_routines`should only run once per dashboard instance. To enforce this we
-# have different behaviour for production and development mode:
-# * we have "preload" on for gunicorn, so this file is loaded only once, before
-#   workers are forked (production);
-# * we make sure that in development mode we run this only once, even though
+        # Set watch for dashboard SCIM config secrets. Any time those change,
+        # we reload so we can do SCIM for newly installed apps.
+        try:
+            helpers.kubernetes.watch_dashboard_config(app, reload)
+        except Exception as e:
+            app.logger.error(f"Error watching dashboard config: {e}")
+
+    # Set up a generic task scheduler (cron-like).
+    scheduler = BackgroundScheduler()
+    scheduler.start()
+    # Add a job to run the provisioning reconciliation routine regularly.
+    # We'll also run it when we make changes that should be propagated
+    # immediately.
+    scheduler.add_job(helpers.threads.request_provision, 'interval', id='provision', hours=24)
+    # We'll run this in a separate thread so it can be done in the background.
+    # We have this single "provisioning worker" so there will be only one
+    # provisioning operation at a time. We use an Event to signal a
+    # provisioning request.
+    def provisioning_loop():
+        while True:
+            app.logger.info("Waiting for next provisioning run.")
+            # helpers.threads.provision_event.wait()
+            # helpers.threads.provision_event.clear()
+            helpers.threads.wait_provision()
+            app.logger.info("Starting provisioning.")
+            with app.app_context():
+                try:
+                    provisioner.reconcile()
+                except Exception as e:
+                    app.logger.warn(f"Exception during user provisioning:")
+                    app.logger.warn(traceback.format_exc())
+    threading.Thread(target=provisioning_loop).start()
+
+# `init_routines` should only run once per dashboard instance. To enforce this
+# we have different behaviour for production and development mode:
+# * We have "preload" on for gunicorn, so this file is loaded only once, before
+#   workers are forked (production).
+# * We make sure that in development mode we run this only once, even though
 #   this file is loaded twice by flask for some reason.
 if RUN_BY_GUNICORN:
     logging.info("Running initialization code (production mode).")
@@ -117,6 +179,13 @@ else:
         # cli.
         flask_migrate.Migrate(app, db)
 
+# Now that we've done all database interactions in the initialisation routines,
+# we need to drop all connections to the database to prevent those from being
+# shared among different worker processes.
+logging.info("Disposing of database connections.")
+with app.app_context():
+    db.engine.dispose()
+
 app.register_blueprint(api_v1)
 app.register_blueprint(web)
 app.register_blueprint(cli)
@@ -133,10 +202,19 @@ jwt = JWTManager(app)
 
 # When token is not valid or missing handler
 @jwt.invalid_token_loader
+def invalid_token_callback(reason):
+    logging.info(f"Invalid token: {reason}.")
+    return jsonify({"errorMessage": "Unauthorized (invalid token)"}), 401
+
 @jwt.unauthorized_loader
+def unauthorized_callback(reason):
+    logging.info(f"No token: {reason}.")
+    return jsonify({"errorMessage": "Unauthorized (no token)"}), 401
+
 @jwt.expired_token_loader
-def expired_token_callback(*args):
-    return jsonify({"errorMessage": "Unauthorized"}), 401
+def expired_token_callback(reason):
+    logging.info(f"Expired token: {reason}.")
+    return jsonify({"errorMessage": "Unauthorized (expired token)"}), 401
 
 @app.route("/")
 def index():

backend/areas/__init__.py

@@ -18,6 +18,7 @@ def api_environment():
     environment = {
         "HYDRA_PUBLIC_URL": HYDRA_PUBLIC_URL,
         "KRATOS_PUBLIC_URL": KRATOS_PUBLIC_URL,
+        "TELEPRESENCE": TELEPRESENCE,
     }
     return jsonify(environment)
 

backend/areas/apps/apps_service.py

+import threading
+
 from flask import current_app
 from flask_jwt_extended import get_jwt
 import ory_kratos_client
 from ory_kratos_client.api import identity_api
 
 from .models import App, AppRole
+from areas.roles.models import Role
+from areas.users.models import User
 from config import *
+from database import db
 from helpers.access_control import user_has_access
 from helpers.kratos_user import KratosUser
+import helpers.kubernetes as k8s
+from helpers.threads import request_provision
 
 class AppsService:
     @staticmethod
@@ -18,7 +25,7 @@ class AppsService:
     def get_accessible_apps():
         apps = App.query.all()
 
-        kratos_admin_api_configuration = ory_kratos_client.Configuration(host=KRATOS_ADMIN_URL, discard_unknown_keys=True)
+        kratos_admin_api_configuration = ory_kratos_client.Configuration(host=KRATOS_ADMIN_URL)
         with ory_kratos_client.ApiClient(kratos_admin_api_configuration) as kratos_admin_client:
             kratos_identity_api = identity_api.IdentityApi(kratos_admin_client)
 
@@ -42,3 +49,39 @@ class AppsService:
     def get_app_roles():
         app_roles = AppRole.query.all()
         return [{"user_id": app_role.user_id, "app_id": app_role.app_id, "role_id": app_role.role_id} for app_role in app_roles]
+
+    @classmethod
+    def install_app(cls, app):
+        app.install()
+        # Create app roles for the new app for all admins, and reprovision. We
+        # do this asynchronously, because we first need to wait for the app
+        # installation to be finished -- otherwise the SCIM config for user
+        # provisioning is not ready yet.
+        current_app.logger.info("Starting thread for creating app roles.")
+        # We need to pass the app context to the thread, because it needs that
+        # for database operations.
+        ca = current_app._get_current_object()
+        threading.Thread(target=cls.create_admin_app_roles, args=(ca, app,)).start()
+
+    @staticmethod
+    def create_admin_app_roles(ca, app):
+        """Create AppRole objects for the given app for all admins."""
+        with ca.app_context():
+            ca.logger.info("Waiting for kustomizations to be ready.")
+            k8s.wait_kustomization_ready(app)
+            for user in User.get_all():
+                if not user['stackspin_data']['stackspin_admin']:
+                    # We are only dealing with admin users here.
+                    continue
+                existing_app_role = AppRole.query.filter_by(app_id=app.id, user_id=user['id']).first()
+                if existing_app_role is None:
+                    ca.logger.info(f"Creating app role for app {app.slug} for admin user {user['id']}")
+                    app_role = AppRole(
+                        user_id=user['id'],
+                        app_id=app.id,
+                        role_id=Role.ADMIN_ROLE_ID
+                    )
+                    db.session.add(app_role)
+                    db.session.commit()
+            ca.logger.info("Requesting user provisioning.")
+            request_provision()

backend/areas/apps/models.py

 """Everything to do with Apps"""
 
-import os
 import base64
+import enum
+import os
 
-from sqlalchemy import ForeignKey, Integer, String, Boolean
+from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Integer, String, Unicode
 from sqlalchemy.orm import relationship
 
 from database import db
@@ -29,6 +30,9 @@ class App(db.Model):
     # The URL is only stored in the DB for external applications; otherwise the
     # URL is stored in a configmap (see get_url)
     url = db.Column(String(length=128), unique=False)
+    scim_url = db.Column(String(length=1024), nullable=True)
+    scim_token = db.Column(String(length=1024), nullable=True)
+    scim_group_support = db.Column(Boolean, nullable=False, server_default='0')
     oauthclients = relationship("OAuthClientApp", back_populates="app")
 
     def __init__(self, slug, name, external=False, url=None):
@@ -37,7 +41,7 @@ class App(db.Model):
         self.external = external
         self.url = url
 
-    def __repr__(self):
+    def __str__(self):
         return f"{self.id} <{self.name}>"
 
     def get_url(self):
@@ -87,20 +91,17 @@ class App(db.Model):
 
     def install(self):
         """Creates a Kustomization in the Kubernetes cluster that installs this application"""
-        # Generate the necessary passwords, etc. from a template
-        self.__generate_secrets()
         # Create add-<app> kustomization
         self.__create_kustomization()
 
     def uninstall(self):
         """
-        Delete the app kustomization.
+        Delete the `add-$app` kustomization.
 
-        In our case, this triggers a deletion of the app's PVCs (so deletes all
-        data), as well as any other Kustomizations and HelmReleases related to
-        the app. It also triggers a deletion of the OAuth2Client object, but
-        does not delete the secrets generated by the `install` command. It also
-        does not remove the TLS secret generated by cert-manager.
+        This triggers a deletion of the app's PVCs (so deletes all data), as
+        well as any other Kustomizations and HelmReleases related to the app.
+        It also triggers a deletion of the OAuth2Client object. It does not
+        remove the TLS secret generated by cert-manager.
         """
         self.__delete_kustomization()
 
@@ -123,20 +124,6 @@ class App(db.Model):
         db.session.delete(self)
         return db.session.commit()
 
-    def __generate_secrets(self):
-        """Generates passwords for app installation"""
-        # Create app variables secret
-        if self.variables_template_filepath:
-            k8s.create_variables_secret(self.slug, self.variables_template_filepath)
-
-        k8s.create_variables_secret(
-            self.slug,
-            os.path.join(
-                self.__get_templates_dir(),
-                "stackspin-oauth-variables.yaml.jinja"
-            )
-        )
-
     def __create_kustomization(self):
         """Creates the `add-{app_slug}` kustomization in the Kubernetes cluster"""
         kustomization_template_filepath = \
@@ -148,16 +135,6 @@ class App(db.Model):
         """Deletes kustomization for this app"""
         k8s.delete_kustomization(f"add-{self.slug}")
 
-
-    @property
-    def variables_template_filepath(self):
-        """Path to the variables template used to generate secrets the app needs"""
-        variables_template_filepath = os.path.join(self.__get_templates_dir(),
-                f"stackspin-{self.slug}-variables.yaml.jinja")
-        if os.path.exists(variables_template_filepath):
-            return variables_template_filepath
-        return None
-
     @property
     def namespace(self):
         """
@@ -203,10 +180,28 @@ class App(db.Model):
 
     @staticmethod
     def __get_templates_dir():
-        """Returns directory that contains the Jinja templates used to create app secrets."""
+        """Returns directory that contains the Jinja templates for kubernetes manifests."""
         return os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates")
 
 
+class ProvisionStatus(enum.Enum):
+    SyncNeeded = "SyncNeeded"
+    Provisioned = "Provisioned"
+    # Provisioning is not necessary for this user/role, for
+    # example because the user has no access to this app.
+    NotProvisioned = "NotProvisioned"
+    # SCIM Provisioning is not supported for this particular app.
+    NotSupported = "NotSupported"
+    # This user needs to be deleted from this app.
+    ToDelete = "ToDelete"
+    # This app role entry corresponds to a Stackspin user that no longer
+    # exists.
+    Orphaned = "Orphaned"
+    # Something went wrong; more information can be found in the
+    # `last_provision_message`.
+    Error = "Error"
+
+
 class AppRole(db.Model):  # pylint: disable=too-few-public-methods
     """
     The AppRole object, stores the roles Users have on Apps
@@ -215,10 +210,25 @@ class AppRole(db.Model):  # pylint: disable=too-few-public-methods
     user_id = db.Column(String(length=64), primary_key=True)
     app_id = db.Column(Integer, ForeignKey("app.id"), primary_key=True)
     role_id = db.Column(Integer, ForeignKey("role.id"))
+    provision_status = db.Column(
+        Enum(
+            ProvisionStatus,
+            native_enum=False,
+            length=32,
+            values_callable=lambda _: [str(member.value) for member in ProvisionStatus]
+        ),
+        nullable=False,
+        default=ProvisionStatus.SyncNeeded,
+        server_default=ProvisionStatus.SyncNeeded.value
+    )
+    last_provision_attempt = db.Column(DateTime, nullable=True)
+    last_provision_message = db.Column(Unicode(length=256), nullable=True)
+    scim_id = db.Column(Unicode(length=256), nullable=True)
 
     role = relationship("Role")
+    app = relationship("App")
 
-    def __repr__(self):
+    def __str__(self):
         return (f"role_id: {self.role_id}, user_id: {self.user_id},"
                 f" app_id: {self.app_id}, role: {self.role}")
 
@@ -248,7 +258,7 @@ class AppStatus():  # pylint: disable=too-few-public-methods
 
         kustomization = app.kustomization
         if kustomization is not None and "status" in kustomization:
-            ks_ready, ks_message = AppStatus.check_condition(kustomization['status'])
+            ks_ready, ks_message = k8s.check_condition(kustomization['status'])
             self.installed = True
             if ks_ready:
                 self.ready = ks_ready
@@ -265,7 +275,7 @@ class AppStatus():  # pylint: disable=too-few-public-methods
         helmreleases = app.helmreleases
         for helmrelease in helmreleases:
             hr_status = helmrelease['status']
-            hr_ready, hr_message = AppStatus.check_condition(hr_status)
+            hr_ready, hr_message = k8s.check_condition(hr_status)
 
             # For now, only show the message of the first HR that isn't ready
             if not hr_ready:
@@ -277,29 +287,9 @@ class AppStatus():  # pylint: disable=too-few-public-methods
         self.ready = ks_ready
         self.message = f"App Kustomization status: {ks_message}"
 
-    def __repr__(self):
+    def __str__(self):
         return f"Installed: {self.installed}\tReady: {self.ready}\tMessage: {self.message}"
 
-    @staticmethod
-    def check_condition(status):
-        """
-        Returns a tuple that has true/false for readiness and a message
-
-        Ready, in this case means that the condition's type == "Ready" and its
-        status == "True". If the condition type "Ready" does not occur, the
-        status is interpreted as not ready.
-
-        The message that is returned is the message that comes with the
-        condition with type "Ready"
-
-        :param status: Kubernetes resource's "status" object.
-        :type status: dict
-        """
-        for condition in status["conditions"]:
-            if condition["type"] == "Ready":
-                return condition["status"] == "True", condition["message"]
-        return False, "Condition with type 'Ready' not found"
-
     def to_dict(self):
         """Represents this app status as a dict"""
         return {
@@ -323,6 +313,20 @@ class OAuthClientApp(db.Model):  # pylint: disable=too-few-public-methods
 
     app = relationship("App", back_populates="oauthclients")
 
-    def __repr__(self):
+    def __str__(self):
         return (f"oauthclient_id: {self.oauthclient_id}, app_id: {self.app_id},"
                 f" app: {self.app}")
+
+class ScimAttribute(db.Model):  # pylint: disable=too-few-public-methods
+    """
+    The ScimAttribute object records that a certain user attribute needs to be
+    set in a certain app via SCIM.
+    """
+
+    user_id = db.Column(String(length=64), primary_key=True)
+    app_id = db.Column(Integer, ForeignKey("app.id"), primary_key=True)
+    attribute = db.Column(String(length=64), primary_key=True)
+
+    def __str__(self):
+        return (f"attribute: {self.attribute}, user_id: {self.user_id},"
+                f" app_id: {self.app_id}")

backend/areas/apps/templates/stackspin-hedgedoc-variables.yaml.jinja

-apiVersion: v1
-kind: Secret
-metadata:
-  name: stackspin-hedgedoc-variables
-data:
-  hedgedoc_mariadb_password: "{{ 32 | generate_password | b64encode }}"
-  hedgedoc_mariadb_root_password: "{{ 32 | generate_password | b64encode }}"

backend/areas/apps/templates/stackspin-nextcloud-variables.yaml.jinja

----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: stackspin-nextcloud-variables
-data:
-  nextcloud_password: "{{ 32 | generate_password | b64encode }}"
-  nextcloud_mariadb_password: "{{ 32 | generate_password | b64encode }}"
-  nextcloud_mariadb_root_password: "{{ 32 | generate_password | b64encode }}"
-  nextcloud_redis_password: "{{ 32 | generate_password | b64encode }}"
-  onlyoffice_database_password: "{{ 32 | generate_password | b64encode }}"
-  onlyoffice_jwt_secret: "{{ 32 | generate_password | b64encode }}"
-  onlyoffice_rabbitmq_password: "{{ 32 | generate_password | b64encode }}"

backend/areas/apps/templates/stackspin-oauth-variables.yaml.jinja

----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: stackspin-{{ app }}-oauth-variables
-data:
-  client_id: "{{ app | b64encode }}"
-  client_secret: "{{ 32 | generate_password | b64encode }}"

backend/areas/apps/templates/stackspin-wekan-variables.yaml.jinja

-apiVersion: v1
-kind: Secret
-metadata:
-  name: stackspin-wekan-variables
-data:
-  mongodb_password: "{{ 32 | generate_password | b64encode }}"
-  mongodb_root_password: "{{ 32 | generate_password | b64encode }}"

backend/areas/apps/templates/stackspin-wordpress-variables.yaml.jinja

----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: stackspin-wordpress-variables
-data:
-  wordpress_admin_password: "{{ 32 | generate_password | b64encode }}"
-  wordpress_mariadb_password: "{{ 32 | generate_password | b64encode }}"
-  wordpress_mariadb_root_password: "{{ 32 | generate_password | b64encode }}"

backend/areas/apps/templates/stackspin-zulip-variables.yaml.jinja

-apiVersion: v1
-kind: Secret
-metadata:
-  name: stackspin-zulip-variables
-data:
-  admin_password: "{{ 32 | generate_password | b64encode }}"
-  memcached_password: "{{ 32 | generate_password | b64encode }}"
-  rabbitmq_password: "{{ 32 | generate_password | b64encode }}"
-  rabbitmq_erlang_cookie: "{{ 32 | generate_password | b64encode }}"
-  redis_password: "{{ 32 | generate_password | b64encode }}"
-  postgresql_password: "{{ 32 | generate_password | b64encode }}"
-  zulip_password: "{{ 32 | generate_password | b64encode }}"

backend/areas/auth/auth.py

@@ -4,7 +4,7 @@ from flask_cors import cross_origin
 from datetime import timedelta
 
 from areas import api_v1
-from areas.apps import App, AppRole
+from areas.apps.models import App, AppRole
 from config import *
 from helpers import HydraOauth, BadRequest, KratosApi
 
@@ -28,6 +28,7 @@ def hydra_callback():
         raise BadRequest("Missing code query param")
 
     token = HydraOauth.get_token(state, code)
+    token_id = token["access_token"]
     user_info = HydraOauth.get_user_info()
     kratos_id = user_info["sub"]
 
@@ -35,7 +36,7 @@ def hydra_callback():
 
     try:
         access_token = create_access_token(
-            identity=token, expires_delta=timedelta(hours=1), additional_claims={"user_id": kratos_id}
+            identity=token_id, expires_delta=timedelta(hours=1), additional_claims={"user_id": kratos_id}
         )
     except Exception as e:
         raise BadRequest("Error with creating auth token between backend and frontend")

backend/areas/resources/__init__.py

+from .resources import *
+from .resources_service import *

backend/areas/resources/resources.py

+from flask import jsonify, request
+from flask_cors import cross_origin
+from flask_expects_json import expects_json
+from flask_jwt_extended import get_jwt, jwt_required
+
+from areas import api_v1
+from helpers.auth_guard import admin_required
+
+from .resources_service import ResourcesService
+
+
+@api_v1.route("/resources", methods=["GET"])
+@jwt_required()
+@cross_origin()
+@admin_required()
+def get_resources():
+    res = ResourcesService.get_resources()
+    return jsonify(res)

backend/areas/resources/resources_service.py

+# from config import KRATOS_ADMIN_URL
+
+# from database import db
+
+# from kubernetes.client import CustomObjectsApi
+# import re
+import requests
+from requests.exceptions import ConnectionError
+
+from flask import current_app
+# import helpers.kubernetes
+
+
+class ResourcesService:
+    @classmethod
+    def get_resources(cls):
+        # custom = CustomObjectsApi()
+        # raw_nodes = custom.list_cluster_custom_object('metrics.k8s.io', 'v1beta1', 'nodes')
+        # raw_pods = custom.list_cluster_custom_object('metrics.k8s.io', 'v1beta1', 'pods')
+        # nodes = []
+        # for node in raw_nodes["items"]:
+        #     nodes.append({
+        #         "name": node["metadata"]["name"],
+        #         "cpu_raw": node["usage"]["cpu"],
+        #         "cpu": cls.parse_cpu(node["usage"]["cpu"]),
+        #         "memory_raw": node["usage"]["memory"],
+        #         "memory_used": cls.parse_memory(node["usage"]["memory"]),
+        #     })
+        try:
+            cores = cls.get_prometheus('machine_cpu_cores')
+            current_app.logger.info(f"Number of cores: {cores}")
+            result = {
+                "success": True,
+                # Number of cores in use. So average usage times number of cores.
+                "cpu": cores * cls.get_prometheus('1 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[8m])))', 'float'),
+                "cpu_total": cores,
+            #         "memory_raw": node["usage"]["memory"],
+            #         "memory_used": cls.parse_memory(node["usage"]["memory"]),
+                "memory_total": cls.get_prometheus('machine_memory_bytes'),
+                "memory_available": cls.get_prometheus('node_memory_MemAvailable_bytes'),
+                "disk_free": cls.get_prometheus('node_filesystem_free_bytes{mountpoint="/"}'),
+                "disk_total": cls.get_prometheus('node_filesystem_size_bytes{mountpoint="/"}'),
+            }
+        except ConnectionError:
+            return {
+                "success": False,
+                "message": "Could not contact prometheus; perhaps monitoring is not enabled.",
+            }
+        return result
+
+    # @staticmethod
+    # def parse_cpu(s):
+    #     result = re.match(r"^(\d+)([mun]?)$", s)
+    #     if result is None:
+    #         raise Exception("cpu data does not match known patterns")
+    #     number = result.group(1)
+    #     suffix = result.group(2)
+    #     multipliers = {"": 1, "m": 1e-3, "u": 1e-6, "n": 1e-9}
+    #     return (int(number) * multipliers[suffix])
+
+    # @staticmethod
+    # def parse_memory(s):
+    #     result = re.match(r"^(\d+)(|Ki|Mi|Gi)$", s)
+    #     if result is None:
+    #         raise Exception("memory data does not match known patterns")
+    #     number = result.group(1)
+    #     suffix = result.group(2)
+    #     multipliers = {"": 1, "Ki": 1024, "Mi": 1024*1024, "Gi": 1024*1024*1024}
+    #     return (int(number) * multipliers[suffix])
+
+    @staticmethod
+    def get_prometheus(query, cast='int'):
+        try:
+            params = {
+                "query": query,
+            }
+            result = requests.get("http://kube-prometheus-stack-prometheus:9090/api/v1/query", params=params)
+            current_app.logger.info(query)
+            current_app.logger.info(result.json())
+            value = result.json()["data"]["result"][0]["value"][1]
+        except AttributeError:
+            return None
+        if cast == 'float':
+            converted = float(value)
+        else:
+            converted = int(value)
+        return converted
+
+  # "pods": {
+  #   "apiVersion": "metrics.k8s.io/v1beta1",
+  #   "items": [
+  #     {
+  #       "containers": [
+  #         {
+  #           "name": "traffic-manager",
+  #           "usage": {
+  #             "cpu": "2839360n",
+  #             "memory": "24696Ki"
+  #           }
+  #         }
+  #       ],
+  #       "metadata": {
+  #         "creationTimestamp": "2023-11-30T15:10:10Z",
+  #         "labels": {
+  #           "app": "traffic-manager",
+  #           "pod-template-hash": "5cd7cc7fd6",
+  #           "telepresence": "manager"
+  #         },
+  #         "name": "traffic-manager-5cd7cc7fd6-mp7td",
+  #         "namespace": "ambassador"
+  #       },
+  #       "timestamp": "2023-11-30T15:10:00Z",
+  #       "window": "12.942s"
+  #     },

backend/areas/roles/models.py

@@ -9,5 +9,5 @@ class Role(db.Model):
     id = db.Column(Integer, primary_key=True)
     name = db.Column(String(length=64))
 
-    def __repr__(self):
+    def __str__(self):
         return f"Role {self.name}"