diff --git a/backend/app.py b/backend/app.py
index 896f1035fd748df697776c66690ada75f16bd637..40e07302edfa3c0e63b8ded02a57581f5bd63f07 100644
--- a/backend/app.py
+++ b/backend/app.py
@@ -1,3 +1,4 @@
+from apscheduler.schedulers.background import BackgroundScheduler
from flask import Flask, jsonify
from flask_cors import CORS
from flask_jwt_extended import JWTManager
@@ -16,6 +17,7 @@ from areas import apps
from areas import auth
from areas import roles
from cliapp import cliapp
+import helpers.provision
from web import login
from database import db
@@ -37,6 +39,7 @@ import cluster_config
from config import *
import logging
import migration_reset
+import os
import sys
# Configure logging.
@@ -66,38 +69,63 @@ app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = SQLALCHEMY_TRACK_MODIFICATIONS
app.logger.setLevel(logging.INFO)
cors = CORS(app)
+provisioner = helpers.provision.Provision()
db.init_app(app)
-# We'll now perform some initialization routines. Because these have to be done
-# once at startup, not for every gunicorn worker, we take a machine-wide lock
-# for this.
-init_lock = NamedAtomicLock('dashboard_init')
-if init_lock.acquire():
- try:
- with app.app_context():
- # We have reset the alembic migration history at Stackspin version 2.2.
- # This checks whether we need to prepare the database to follow that
- # change.
- migration_reset.reset()
- flask_migrate.Migrate(app, db)
+def init_routines():
+ # We'll now perform some initialization routines. Because these have to be done
+ # once at startup, not for every gunicorn worker, we take a machine-wide lock
+ # for this.
+ init_lock = NamedAtomicLock('dashboard_init')
+ if init_lock.acquire():
try:
with app.app_context():
- flask_migrate.upgrade()
- except Exception as e:
- app.logger.info(f"upgrade failed: {type(e)}: {e}")
- sys.exit(2)
-
- # We need this app context in order to talk the database, which is managed by
- # flask-sqlalchemy, which assumes a flask app context.
+ # We have reset the alembic migration history at Stackspin version 2.2.
+ # This checks whether we need to prepare the database to follow that
+ # change.
+ migration_reset.reset()
+ flask_migrate.Migrate(app, db)
+ try:
+ with app.app_context():
+ flask_migrate.upgrade()
+ except Exception as e:
+ app.logger.info(f"upgrade failed: {type(e)}: {e}")
+ sys.exit(2)
+
+ # We need this app context in order to talk the database, which is managed by
+ # flask-sqlalchemy, which assumes a flask app context.
+ with app.app_context():
+ # Load the list of apps from a configmap and store any missing ones in the
+ # database.
+ cluster_config.populate_apps()
+ # Same for the list of oauthclients.
+ cluster_config.populate_oauthclients()
+ finally:
+ init_lock.release()
+
+ # We define this wrapper because the SCIM provisioning code needs to access the
+ # database, which needs a flask app context.
+ def provision():
with app.app_context():
- # Load the list of apps from a configmap and store any missing ones in the
- # database.
- cluster_config.populate_apps()
- # Same for the list of oauthclients.
- cluster_config.populate_oauthclients()
- finally:
- init_lock.release()
+ provisioner.reconcile()
+ # Set up a generic task scheduler (cron-like).
+ # TODO: there should really only be one of those per flask instance, while now
+ # this is executed once for every worker thread -- and also for flask CLI runs.
+ scheduler = BackgroundScheduler()
+ scheduler.start()
+ # Add a job to run the provisioning reconciliation routine regularly.
+ scheduler.add_job(provision, 'interval', minutes=1)
+
+if RUN_INIT:
+ logging.info("WERKZEUG_RUN_MAIN: {}".format(os.environ.get("WERKZEUG_RUN_MAIN", "unset")))
+ # In development mode, the app code is loaded twice, but only one of these
+ # times this variable is set.
+ if os.environ.get("WERKZEUG_RUN_MAIN") == "true":
+ logging.info("Running initialization code.")
+ init_routines()
+else:
+ logging.info("Not running initialization code.")
app.register_blueprint(api_v1)
app.register_blueprint(web)
diff --git a/backend/areas/apps/models.py b/backend/areas/apps/models.py
index 069c73751643f42af0619e65c7d6425d4802138d..50e7da7b5b9d2804f013c06695bc1ae89ba51794 100644
--- a/backend/areas/apps/models.py
+++ b/backend/areas/apps/models.py
@@ -1,9 +1,10 @@
"""Everything to do with Apps"""
-import os
import base64
+import enum
+import os
-from sqlalchemy import ForeignKey, Integer, String, Boolean
+from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Integer, String, Unicode
from sqlalchemy.orm import relationship
from database import db
@@ -202,6 +203,19 @@ class App(db.Model):
return os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates")
+class ProvisionStatus(enum.Enum):
+ SyncNeeded = "SyncNeeded"
+ Provisioned = "Provisioned"
+ # Provisioning is not necessary for this user/role, for
+ # example because the user has no access to this app.
+ NotProvisioned = "NotProvisioned"
+ # SCIM Provisioning is not supported for this particular app.
+ NotSupported = "NotSupported"
+ # Something went wrong; more information can be found in the
+ # `last_provision_message`.
+ Error = "Error"
+
+
class AppRole(db.Model): # pylint: disable=too-few-public-methods
"""
The AppRole object, stores the roles Users have on Apps
@@ -210,8 +224,22 @@ class AppRole(db.Model): # pylint: disable=too-few-public-methods
user_id = db.Column(String(length=64), primary_key=True)
app_id = db.Column(Integer, ForeignKey("app.id"), primary_key=True)
role_id = db.Column(Integer, ForeignKey("role.id"))
+ provision_status = db.Column(
+ Enum(
+ ProvisionStatus,
+ native_enum=False,
+ length=32,
+ values_callable=lambda _: [str(member.value) for member in ProvisionStatus]
+ ),
+ nullable=False,
+ default=ProvisionStatus.SyncNeeded,
+ server_default=ProvisionStatus.SyncNeeded.value
+ )
+ last_provision_attempt = db.Column(DateTime, nullable=True)
+ last_provision_message = db.Column(Unicode(length=256), nullable=True)
role = relationship("Role")
+ app = relationship("App")
def __repr__(self):
return (f"role_id: {self.role_id}, user_id: {self.user_id},"
diff --git a/backend/config.py b/backend/config.py
index 6e5d37ac1221925e8403612674225bd0ccf5314d..f8cbf03ce46ecf4f0aa75b6ff4e51caa386c4d25 100644
--- a/backend/config.py
+++ b/backend/config.py
@@ -23,3 +23,8 @@ SQLALCHEMY_TRACK_MODIFICATIONS = False
LOAD_INCLUSTER_CONFIG = os.environ.get("LOAD_INCLUSTER_CONFIG").lower() == "true"
DEMO_INSTANCE = os.environ.get("DASHBOARD_DEMO_INSTANCE", "False").lower() in ('true', '1')
+
+SCIM_URL = os.environ.get("SCIM_URL")
+SCIM_TOKEN = os.environ.get("SCIM_TOKEN")
+# Use this to help our flask code figure out when to run its initialization routines.
+RUN_INIT = os.environ.get("DASHBOARD_RUN_INIT", "False").lower() in ('true', '1')
diff --git a/backend/helpers/provision.py b/backend/helpers/provision.py
new file mode 100644
index 0000000000000000000000000000000000000000..2f606485c23cda95bbc724a9aefdbb8ed2d00726
--- /dev/null
+++ b/backend/helpers/provision.py
@@ -0,0 +1,277 @@
+from datetime import datetime
+import json.decoder
+import logging
+import ory_kratos_client
+from ory_kratos_client.api import identity_api
+import requests
+
+from areas.apps.models import App, AppRole, ProvisionStatus
+from areas.roles.models import Role
+import config
+from database import db
+from helpers.kratos_user import KratosUser
+
+class ProvisionError(Exception):
+ pass
+
+class User:
+ def __init__(self, kratos_id, scim_id, displayName):
+ self.kratos_id = kratos_id
+ self.scim_id = scim_id
+ self.displayName = displayName
+
+ def ref(self):
+ return f"{config.SCIM_URL}Users/{self.scim_id}"
+
+class Group:
+ def __init__(self, scim_id, displayName, members):
+ self.scim_id = scim_id
+ self.displayName = displayName
+ self.members = members
+ self.modified = False
+
+ def add_member(self, user):
+ if user.scim_id not in self.members:
+ logging.info(f"Adding user to dict: {user.displayName} ({user.scim_id})")
+ self.members[user.scim_id] = user
+ self.modified = True
+
+ def remove_member(self, user):
+ if user.scim_id in self.members:
+ logging.info(f"Found user to remove from dict: {user.displayName} ({user.scim_id})")
+ del self.members[user.scim_id]
+ self.modified = True
+
+ def debug(self):
+ logging.info(f"Group {self.displayName} ({self.scim_id})")
+ for _, member in self.members.items():
+ logging.info(f" with user {member.displayName} ({member.scim_id})")
+
+# Read from the database which users need to be provisioned on which apps, and
+# do the corresponding SCIM calls to those apps to do the actual provisioning.
+class Provision:
+ def __init__(self):
+ # Set up kratos API client.
+ kratos_admin_api_configuration = ory_kratos_client.Configuration(host=config.KRATOS_ADMIN_URL, discard_unknown_keys=True)
+ kratos_admin_client = ory_kratos_client.ApiClient(kratos_admin_api_configuration)
+ self.kratos_identity_api = identity_api.IdentityApi(kratos_admin_client)
+
+ def reconcile(self):
+ logging.info("Reconciling user provisioning")
+ # logging.info(f"SCIM URL: {config.SCIM_URL}")
+ # logging.info(f"SCIM token: {config.SCIM_TOKEN}")
+ existing_users = self._get_existing_users()
+ for userId, u in existing_users.items():
+ logging.info(f"Existing user: {u.displayName} ({userId})")
+ existing_groups = self._get_existing_groups()
+ for _, g in existing_groups.items():
+ g.debug()
+ # We will modify this group in-memory over the course of the user reconciliation.
+ # After we have reconciled all users, we will update the group in a single operation.
+ # We could also use PATCH operations instead to add single users as we
+ # go, but the Nextcloud SCIM app does not support that.
+ admin_group = existing_groups.get('admin', None)
+ if admin_group is None:
+ raise ProvisionError("Admin group could not be found, aborting.")
+ # TODO: we later need to retry Error ones as well.
+ app_roles = db.session.query(AppRole).filter(
+ AppRole.provision_status == ProvisionStatus.SyncNeeded
+ )
+ for app_role in app_roles:
+ # logging.info(f"found app_role: {app_role}")
+ app_role.last_provision_attempt = datetime.now()
+ existing_user = existing_users.get(app_role.user_id, None)
+ if existing_user is not None:
+ logging.info(f"User {app_role.user_id} already exists in the app.")
+ if app_role.role_id == Role.NO_ACCESS_ROLE_ID:
+ if existing_user is None:
+ logging.info("User without access does not exist yet, so nothing to do.")
+ # We have not provisioned this user in this app yet, so we
+ # don't have to do anything at this point.
+ app_role.provision_status = ProvisionStatus.Provisioned
+ app_role.last_provision_message = "Provisioning not required for user without access."
+ db.session.commit()
+ continue
+ else:
+ logging.info("User without access exists in the app; we continue so we can disable the user in the app.")
+ app = app_role.app
+ # logging.info(f"which has app: {app}")
+ if app.slug != "nextcloud":
+ # logging.info(f"we don't support that one")
+ app_role.provision_status = ProvisionStatus.NotSupported
+ app_role.last_provision_message = f"App does not support automatic user provisioning."
+ db.session.commit()
+ continue
+ try:
+ self._provision_user(app_role, existing_user, admin_group)
+ new_status = ProvisionStatus.Provisioned
+ new_message = "User successfully provisioned."
+ except ProvisionError as ex:
+ new_status = ProvisionStatus.Error
+ new_message = str(ex)
+ app_role.provision_status = new_status
+ app_role.last_provision_message = new_message
+ db.session.commit()
+ # let's do only one at a time for now
+ # break
+ logging.info("Before provisioning admin group:")
+ admin_group.debug()
+ if admin_group.modified:
+ logging.info("Admin group was modified, so updating it via SCIM.")
+ self._provision_group(admin_group)
+ else:
+ logging.info("Admin group was not modified.")
+
+ # Provision the user via SCIM PUT or POST, based on the user and role
+ # information in `app_role`, and on the existing user object in the app
+ # `existing_user` (previously obtained via SCIM GET). Also update the
+ # `admin_group` object so we may later add or remove the user to/from the
+ # admin group.
+ def _provision_user(self, app_role, existing_user, admin_group):
+ logging.info(f"Reconciling user {app_role.user_id}")
+ # Get the related user object
+ logging.info(f"Info: Getting user data from Kratos.")
+ kratos_user = KratosUser(self.kratos_identity_api, app_role.user_id)
+ headers = {
+ 'Authorization': 'Bearer ' + config.SCIM_TOKEN
+ }
+ if app_role.role_id == Role.NO_ACCESS_ROLE_ID:
+ active = False
+ else:
+ active = True
+ logging.info(f"Active user: {active}")
+ data = {
+ 'schemas': ['urn:ietf:params:scim:schemas:core:2.0:User'],
+ 'externalId': app_role.user_id,
+ 'active': active,
+ # Sadly Nextcloud doesn't allow changing the userName, so we set it
+ # to something unique and stable.
+ # https://github.com/nextcloud/server/issues/5488
+ # We add the `stackspin-` prefix to make this compatible with the
+ # username generated by the sociallogin (SSO) Nextcloud app.
+ 'userName': f"stackspin-{app_role.user_id}",
+ 'displayName': kratos_user.name,
+ 'emails': [{
+ 'value': kratos_user.email,
+ 'primary': True
+ }],
+ }
+ if existing_user is None:
+ url = f"{config.SCIM_URL}bearer/Users"
+ response = requests.post(url, headers=headers, json=data)
+ else:
+ url = f"{config.SCIM_URL}bearer/Users/{existing_user.scim_id}"
+ response = requests.put(url, headers=headers, json=data)
+ logging.info(f"SCIM url: {url}")
+ logging.info(f"SCIM http status: {response.status_code}")
+ try:
+ response_json = response.json()
+ except json.decoder.JSONDecodeError as e:
+ logging.info("SCIM result was not json")
+ logging.info(response.content)
+ raise ProvisionError("App returned non-json data in SCIM user put/post.")
+ logging.info(f"got: {response_json}")
+ # {'schemas': ['urn:ietf:params:scim:schemas:core:2.0:User'], 'id':
+ # 'Greenhostie', 'externalId': '316cbd5c-7b69-4a27-8a3b-96b3ec056e99',
+ # 'meta': None, 'userName': 'Greenhostie', 'name': {'formatted':
+ # 'Greenhostie', 'familyName': None, 'givenName': None, 'middleName': None,
+ # 'honorificPrefix': None, 'honorificSuffix': None}, 'displayName':
+ # 'Greenhostie', 'nickName': None, 'profileUrl': None, 'title': None,
+ # 'userType': None, 'preferredLanguage': None, 'locale': None, 'timezone':
+ # None, 'active': True, 'password': None, 'emails': [{'type': None,
+ # 'primary': True, 'display': None, 'value': 'arie+scim@greenhost.nl',
+ # '$ref': None}], 'phoneNumbers': None, 'ims': None, 'photos': None,
+ # 'addresses': None, 'groups': None, 'entitlements': None, 'roles': None,
+ # 'x509Certificates': None}
+ user = User(app_role.user_id, response_json['id'], kratos_user.name)
+ if app_role.role_id == Role.ADMIN_ROLE_ID:
+ logging.info(f"Adding user to admin group: {user.displayName} ({user.kratos_id})")
+ admin_group.add_member(user)
+ else:
+ logging.info(f"Removing user from admin group: {user.displayName} ({user.kratos_id})")
+ admin_group.remove_member(user)
+ logging.info("After adding/removing user:")
+ admin_group.debug()
+
+ def _get_existing_users(self):
+ logging.info(f"Info: Getting list of current users from app via SCIM.")
+ url = f"{config.SCIM_URL}bearer/Users"
+ headers = {
+ 'Authorization': 'Bearer ' + config.SCIM_TOKEN
+ }
+ response = requests.get(url, headers=headers)
+ logging.info(f"SCIM http status: {response.status_code}")
+ try:
+ response_json = response.json()
+ except json.decoder.JSONDecodeError as e:
+ logging.info("SCIM result was not json")
+ logging.info(response.content)
+ raise ProvisionError("Failed to get existing users from Nextcloud")
+ # logging.info(f"got: {response_json}")
+ # Make a dictionary of the users, using their externalId as key, which
+ # is the kratos user ID.
+ users = {}
+ for u in response_json['Resources']:
+ kratos_id = u['externalId']
+ if not kratos_id:
+ logging.info(f"Got user without externalId: {u}")
+ # Users that were created in Nextcloud by SSO, before SCIM was
+ # introduced in Stackspin, will not have `externalId` set, so
+ # we get the Kratos ID from the `id` attribute instead.
+ if u['id'].startswith('stackspin-'):
+ kratos_id = u['id'][len('stackspin-'):]
+ else:
+ # This is a user that was not created by Stackspin (either
+ # SSO or SCIM), so we'll ignore it.
+ continue
+ users[kratos_id] = User(kratos_id, u['id'], u['displayName'])
+ return users
+
+ def _get_existing_groups(self):
+ logging.info(f"Info: Getting list of current groups from Nextcloud via SCIM.")
+ url = f"{config.SCIM_URL}bearer/Groups"
+ headers = {
+ 'Authorization': 'Bearer ' + config.SCIM_TOKEN
+ }
+ response = requests.get(url, headers=headers)
+ logging.info(f"SCIM http status: {response.status_code}")
+ try:
+ response_json = response.json()
+ except json.decoder.JSONDecodeError as e:
+ logging.info("SCIM result was not json")
+ logging.info(response.content)
+ raise ProvisionError("Failed to get existing groups from Nextcloud")
+ logging.info(f"got: {response_json}")
+ groups = {}
+ for group in response_json['Resources']:
+ members = {}
+ for member in group['members']:
+ scim_id = member['value']
+ members[scim_id] = User(None, scim_id, member['display'])
+ groups[group['id']] = Group(group['id'], group['displayName'], members)
+ return groups
+
+ def _provision_group(self, group):
+ logging.info(f"Reconciling group {group.scim_id}")
+ headers = {
+ 'Authorization': 'Bearer ' + config.SCIM_TOKEN
+ }
+ member_data = [
+ {'value': member.scim_id, 'display': member.displayName, '$ref': member.ref()}
+ for _, member in group.members.items()]
+ logging.info(f"Will update admin group with member data {member_data}")
+ data = {
+ 'schemas': ['urn:ietf:params:scim:schemas:core:2.0:Group'],
+ 'displayName': group.displayName,
+ 'members': member_data,
+ }
+ url = f"{config.SCIM_URL}bearer/Groups/{group.scim_id}"
+ response = requests.put(url, headers=headers, json=data)
+ logging.info(f"SCIM http status: {response.status_code}")
+ try:
+ response_json = response.json()
+ except json.decoder.JSONDecodeError as e:
+ logging.info("SCIM result was not json")
+ logging.info(response.content)
+ raise ProvisionError("App returned non-json data in SCIM group put.")
+ logging.info(f"got: {response_json}")
diff --git a/backend/migrations/versions/7d27395c892a_new_migration.py b/backend/migrations/versions/7d27395c892a_new_migration.py
index 94a1d844c0918712970147183025150a28ccd37c..5c1a3332615421e89f20395d866d992aa5881ee3 100644
--- a/backend/migrations/versions/7d27395c892a_new_migration.py
+++ b/backend/migrations/versions/7d27395c892a_new_migration.py
@@ -16,11 +16,11 @@ down_revision = None
branch_labels = None
depends_on = None
-conn = op.get_bind()
-inspector = Inspector.from_engine(conn)
-tables = inspector.get_table_names()
-
def upgrade():
+ conn = op.get_bind()
+ inspector = Inspector.from_engine(conn)
+ tables = inspector.get_table_names()
+
if "app" not in tables:
op.create_table(
"app",
@@ -69,4 +69,4 @@ def downgrade():
op.drop_table("oauthclient_app")
op.drop_table("app_role")
op.drop_table("role")
- op.drop_table("app")
\ No newline at end of file
+ op.drop_table("app")
diff --git a/backend/migrations/versions/825262488cd9_add_scim_support.py b/backend/migrations/versions/825262488cd9_add_scim_support.py
new file mode 100644
index 0000000000000000000000000000000000000000..f60bce5c71157a4b6d42259ddc4ed464d8425d09
--- /dev/null
+++ b/backend/migrations/versions/825262488cd9_add_scim_support.py
@@ -0,0 +1,57 @@
+"""Add SCIM support for user provisioning
+
+Revision ID: 825262488cd9
+Revises: 7d27395c892a
+Create Date: 2023-03-08 10:50:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+from sqlalchemy.engine.reflection import Inspector
+
+from helpers.provision import ProvisionStatus
+
+# revision identifiers, used by Alembic.
+revision = '825262488cd9'
+down_revision = '7d27395c892a'
+branch_labels = None
+depends_on = None
+
+def upgrade():
+ op.add_column(
+ "app_role",
+ sa.Column(
+ "provision_status",
+ sa.Enum(
+ ProvisionStatus,
+ native_enum=False,
+ length=32,
+ values_callable=lambda _: [str(member.value) for member in ProvisionStatus]
+ ),
+ nullable=False,
+ default=ProvisionStatus.SyncNeeded,
+ server_default=ProvisionStatus.SyncNeeded.value
+ ),
+ )
+ op.add_column(
+ "app_role",
+ sa.Column(
+ "last_provision_attempt",
+ sa.DateTime,
+ nullable=True
+ ),
+ )
+ op.add_column(
+ "app_role",
+ sa.Column(
+ "last_provision_message",
+ sa.Unicode(length=256),
+ nullable=True
+ ),
+ )
+
+def downgrade():
+ op.drop_column("app_role", "provision_status")
+ op.drop_column("app_role", "last_provision_attempt")
+ op.drop_column("app_role", "last_provision_message")
diff --git a/backend/requirements.txt b/backend/requirements.txt
index fc2836ad096a8f50d217569cc0a9b8eccdc5db34..6c939b7b02f3fb43b7d409fdf28af36294e84985 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -1,3 +1,4 @@
+APScheduler==3.9.1
attrs==21.4.0
black==22.1.0
certifi==2021.10.8
diff --git a/docker-compose.yml b/docker-compose.yml
index 655be3e58c31e6d70fefdab2b1f40398e6fee636..ac0f1dea0d649558007942089e03aed8fbb84078 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -54,7 +54,7 @@ services:
- "$KUBECONFIG:/.kube/config"
depends_on:
- kube_port_mysql
- entrypoint: ["bash", "-c", "flask run --host $$(hostname -i)"]
+ entrypoint: ["bash", "-c", "DASHBOARD_RUN_INIT=true flask run --host $$(hostname -i)"]
kube_port_kratos_admin:
image: bitnami/kubectl:1.26.2
user: "${KUBECTL_UID}:${KUBECTL_GID}"