add Zulip

.gitlab-ci.yml

@@ -173,6 +173,17 @@ include:
     - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-wordpress/'
     - if: '$CI_COMMIT_BRANCH == "master"'
 
+.zulip_rules:
+  rules:
+    - changes:
+        - flux2/apps/$RESOURCE/*.yaml
+        - flux2/cluster/optional/$RESOURCE/*.yaml
+        - flux2/infrastructure/sources/zulip.yaml
+        - install/install-app.sh
+        - test/taiko/*
+    - if: '$TRIGGER_JOBS =~ /enable-zulip/'
+    - if: '$CI_COMMIT_MESSAGE =~ /TRIGGER_JOBS=.*enable-zulip/'
+    - if: '$CI_COMMIT_BRANCH == "master"'
 
 # Global declarations
 # ===================
@@ -436,6 +447,13 @@ enable-wordpress:
     - .enable_app_template
     - .wordpress_rules
 
+enable-zulip:
+  variables:
+    RESOURCE: "zulip"
+  extends:
+    - .enable_app_template
+    - .zulip_rules
+
 # Stage: apps-kustomizations-ready
 # ================
 #
@@ -496,6 +514,16 @@ wordpress-kustomization-ready:
     - .app-kustomization-ready
     - .wordpress_rules
 
+zulip-kustomization-ready:
+  needs:
+    - job: setup-openappstack
+    - job: enable-zulip
+  variables:
+    RESOURCE: "zulip"
+  extends:
+    - .app-kustomization-ready
+    - .zulip_rules
+
 # Stage: certs
 # ================
 #
@@ -571,6 +599,16 @@ wordpress-cert:
     - .apps-cert
     - .wordpress_rules
 
+zulip-cert:
+  variables:
+    RESOURCE: "zulip"
+  needs:
+    - job: enable-zulip
+    - job: setup-openappstack
+  extends:
+    - .apps-cert
+    - .zulip_rules
+
 
 # Stage: health-test
 # ==================
@@ -692,6 +730,17 @@ wordpress-taiko:
     - .taiko
     - .wordpress_rules
 
+zulip-taiko:
+  variables:
+    RESOURCE: "zulip"
+  needs:
+    - job: zulip-cert
+    - job: setup-openappstack
+    - job: zulip-kustomization-ready
+  extends:
+    - .taiko
+    - .zulip_rules
+
 
 # Etc
 # ===

.gitlab/issue_templates/new_app.md

@@ -5,21 +5,21 @@
 
 * [ ] Create new source if needed in `flux2/infrastructure/sources/APP.yaml`
 * [ ] Include `APP.yaml` in `flux2/infrastructure/sources/kustomization.yaml`
-* [ ] Add app secret: `charts/oas-secrets/templates/oas-APP-variables.yaml`
-* Add kustomizations:
+* [ ] Add app secret: `install/templates/oas-APP-variables.yaml.jinja`
+* Add `Kustomizations`:
    * [ ] `flux2/cluster/optional/APP/APP.yaml`
    * [ ] `flux2/apps/APP/kustomization.yaml`
    * [ ] If needed, add PVCs in `flux2/apps/APP/pvc.yaml`
-   * [ ] Add helmrelease in `flux2/apps/APP/release.yaml`
+   * [ ] Add `HelmRelease` in `flux2/apps/APP/release.yaml`
 
 ### Single sign-on
 
 * Integrate the new app into the single sign-on system
-  * Add OAuth client secret to `charts/oas-secrets/templates/oas-oauth-variables.yaml`
+  * [ ] Add OAuth client secret to `install/templates/oas-oauth-variables.yaml.jinja`
   * In `flux2/core/base/single-sign-on/release.yaml`:
     * [ ] Add app `userbackend.applications`
     * [ ] Add app to `oAuthClients`
-  * Confgure app OIDC settings in helmrelease `flux2/apps/APP/release.yaml`
+  * [ ] Configure app OIDC settings in `HelmRelease` `flux2/apps/APP/release.yaml`
   * [ ] Disable user/pw login if possible
   * [ ] Admin-login should grant admin privileges
   * [ ] Non-admin should not grant admin privileges
@@ -28,21 +28,21 @@
 
 * [ ] Make sure testing app resources work (`test/pytest/test_resources.py`)
 * [ ] Make sure testing app cert works (`test/pytest/test_certs.py`)
-* [ ] Add taiko test (`tests/taiko`)
+* [ ] Add Taiko test (`tests/taiko`)
 
 ## CI
 
-Add app to following stages in `.gitlab-ci.yml`:
+Add the following elements to `.gitlab-ci.yml`:
 
-* [ ] install-apps
-* [ ] apps-helm-release
-* [ ] apps-ready
-* [ ] certs
-* [ ] integration-tests
+* [ ] `.APP-rules` partial
+* [ ] `enable-APP` job
+* [ ] `APP-kustomization-ready` job
+* [ ] `APP-cert` job
+* [ ] `APP-taiko` test job
 
 ## Renovatebot
 
-* [ ] Make sure the needed helmRelease fields for renovatebot are in place and
+* [ ] Make sure the needed `HelmRelease` fields for renovatebot are in place and
       order, i.e.
       ```
       # renovate: registryUrl=https://helm-charts.wikimedia.org/stable/

flux2/apps/zulip/kustomization.yaml

+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: oas-apps
+resources:
+  - release.yaml
+  - zulip-values-configmap.yaml

flux2/apps/zulip/release.yaml

+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: zulip
+  namespace: oas-apps
+spec:
+  releaseName: zulip
+  chart:
+    spec:
+      chart: zulip
+      # NOTE: Change the GitRepository yaml file if you want a different version
+      sourceRef:
+        kind: GitRepository
+        name: zulip-helm-chart
+        namespace: flux-system
+  interval: 1h
+  install:
+    timeout: 15m
+  valuesFrom:
+    - kind: ConfigMap
+      name: oas-zulip-values
+      optional: false
+    # Allow overriding values by ConfigMap or Secret
+    - kind: ConfigMap
+      name: oas-zulip-override
+      optional: true
+    - kind: Secret
+      name: oas-zulip-override
+      optional: true

flux2/apps/zulip/zulip-values-configmap.yaml

+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: oas-rocketchat-values
+data:
+  values.yaml: |
+    ingress:
+      hosts:
+        - zulip.${domain}
+      annotations:
+        # Tell cert-manager to automatically get a TLS certificate
+        kubernetes.io/tls-acme: "true"
+      tls:
+        - hosts:
+            - "zulip.${domain}"
+          secretName: oas-zulip
+
+    memcached:
+      password: "${memcached_password}"
+      resources:
+        limits:
+          cpu: 200m
+          memory: 256Mi
+        requests:
+          cpu: 100m
+          memory: 128Mi
+
+    redis:
+      password: "${redis_password}"
+      resources:
+        limits:
+          cpu: 200m
+          memory: 64Mi
+        requests:
+          cpu: 100m
+          memory: 32Mi
+
+    postgresql:
+      password: "${postgresql_password}"
+      resources:
+        limits:
+          cpu: 400m
+          memory: 256Mi
+        requests:
+          cpu: 200m
+          memory: 128Mi
+
+    zulip:
+      image: open.greenhost.net:4567/openappstack/openappstack/zulip:f60b8cc
+      environment:
+        DISABLE_HTTPS: true
+        SSL_CERTIFICATE_GENERATION: self-signed
+        SETTING_EXTERNAL_HOST: zulip.${domain}
+        SETTING_ZULIP_ADMINISTRATOR: "${admin_email}"
+        SECRETS_email_password: "${outgoing_mail_smtp_password}"
+        SETTING_EMAIL_HOST: '${outgoing_mail_smtp_host}'
+        SETTING_EMAIL_HOST_USER: '${outgoing_mail_smtp_user}'
+        SETTING_EMAIL_PORT: '${outgoing_mail_smtp_host}'
+        SETTING_EMAIL_USE_SSL: 'False'
+        SETTING_EMAIL_USE_TLS: 'True'
+        ZULIP_AUTH_BACKENDS: 'EmailAuthBackend'
+        # NOTE: Needs to be a Python Tuple
+        SETTING_AUTHENTICATION_BACKENDS: '("zproject.backends.GenericOpenIdConnectBackend",)'
+        # NOTE: Needs adjusted entrypoint that's currently only in our Docker container
+        # (https://github.com/greenhost/docker-zulip/commit/d583a2d28707a3b77bf610bedc2c2bb81f2a5f88)
+        # NOTE: This is a Python object, not JSON
+        SETTING_SOCIAL_AUTH_OIDC_ENABLED_IDPS: '{"openappstack": { "oidc_url": "https://sso.${domain}/", "display_name": "OpenAppStack", "display_icon": None, "client_id": "zulip", "secret": get_secret("social_auth_oidc_secret"), "auto_signup": True }}'
+        SECRETS_social_auth_oidc_secret: "${zulip_oauth_client_secret}"
+        # Enable "low memory mode", queue workers run 1 multithreaded process
+        QUEUE_WORKERS_MULTIPROCESS: 'False'
+      resources:
+        limits:
+          cpu: 400m
+          memory: 1.5Gi
+        requests:
+          cpu: 100m
+          memory: 1Gi

flux2/cluster/optional/zulip/zulip.yaml

+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+kind: Kustomization
+metadata:
+  name: zulip
+  namespace: flux-system
+spec:
+  interval: 1h
+  dependsOn:
+    - name: core
+    - name: infrastructure
+  sourceRef:
+    kind: GitRepository
+    name: openappstack
+  path: ./flux2/apps/zulip
+  prune: true
+  validation: client
+  healthChecks:
+    - apiVersion: helm.toolkit.fluxcd.io/v1beta1
+      kind: HelmRelease
+      name: zulip
+      namespace: oas-apps
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: zulip-zulip
+      namespace: oas-apps
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: zulip-postgresql
+      namespace: oas-apps
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: zulip-redis
+      namespace: oas-apps
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: zulip-rabbitmq
+      namespace: oas-apps
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: zulip-zulip
+      namespace: oas-memcached
+  postBuild:
+    substituteFrom:
+      - kind: Secret
+        name: oas-zulip-variables
+      - kind: Secret
+        name: oas-oauth-variables
+      - kind: Secret
+        name: oas-cluster-variables

flux2/core/base/single-sign-on/single-sign-on-values-configmap.yaml

@@ -26,6 +26,8 @@ data:
           description: "Grafana allows you to query, visualize, alert on and understand metrics generated by OpenAppStack. It can be used to create explore and share dashboards."
         - name: &WEKAN wekan
           description: "Wekan Kanban board."
+        - name: &ZULIP zulip
+          description: "Communicate and collaborate using team chat and switch to video or audio calls with screen sharing for more efficient teamwork."
       username: "${userbackend_admin_username}"
       password: "${userbackend_admin_password}"
       email: "${admin_email}"
@@ -150,3 +152,19 @@ data:
         - "refresh_token"
         - "client_credentials"
         - "implicit"
+    # https://zulip.readthedocs.io/en/latest/production/authentication-methods.html#openid-connect
+    - clientName: *ZULIP
+      clientSecret: "${zulip_oauth_client_secret}"
+      redirectUri: "https://zulip.${domain}/complete/oidc/"
+      scopes: "openid profile email"
+      clientUri: "https://zulip.${domain}"
+      clientLogoUri: "https://zulip.${domain}/static/images/zulip-logo.svg"
+      tokenEndpointAuthMethod: "client_secret_post"
+      responseTypes:
+        - "code"
+        - "id_token"
+      grantTypes:
+        - "authorization_code"
+        - "refresh_token"
+        - "client_credentials"
+        - "implicit"

flux2/infrastructure/sources/kustomization.yaml

@@ -16,3 +16,4 @@ resources:
   - wekan.yaml
   - wikimedia.yaml
   - wordpress.yaml
+  - zulip.yaml

flux2/infrastructure/sources/zulip.yaml

+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: GitRepository
+metadata:
+  name: zulip-helm-chart
+  namespace: flux-system
+spec:
+  # The interval at which to check the upstream for updates
+  interval: 1h
+  # The repository URL, can be a HTTP/S or SSH address
+  url: https://github.com/greenhost/docker-zulip
+  # The Git reference to checkout and monitor for changes
+  # (defaults to master)
+  # For all available options, see:
+  # https://toolkit.fluxcd.io/components/source/api/#source.toolkit.fluxcd.io/v1beta1.GitRepositoryRef
+  ref:
+    branch: helm-chart-stackspin

install/templates/oas-oauth-variables.yaml.jinja

@@ -10,3 +10,4 @@ data:
   userpanel_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"
   wekan_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"
   wordpress_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"
+  zulip_oauth_client_secret: "{{ 32 | generate_password | b64encode }}"

install/templates/oas-zulip-variables.yaml.jinja

+apiVersion: v1
+kind: Secret
+metadata:
+  name: oas-zulip-variables
+data:
+  memcached_password: "{{ 32 | generate_password | b64encode }}"
+  redis_password: "{{ 32 | generate_password | b64encode }}"
+  postgresql_password: "{{ 32 | generate_password | b64encode }}"

test/pytest/test_certs.py

@@ -2,14 +2,16 @@
 """Test if application ingress uses a valid certificate."""
 
 import os
-import socket
 import shutil
+import socket
 import sys
+
 import certifi
 import pytest
 import requests
 from OpenSSL import SSL
 
+
 @pytest.mark.resource
 @pytest.mark.certs
 def test_cert_validation(host, resource): # pylint: disable=too-many-statements
@@ -101,7 +103,8 @@ def test_cert_validation(host, resource): # pylint: disable=too-many-statements
         'rocketchat': 'chat',
         'single-sign-on': 'sso',
         'wekan': 'wekan',
-        'wordpress': 'www'
+        'wordpress': 'www',
+        'zulip': 'zulip',
     }
 
     if resource == 'all':

test/taiko/apps.js

@@ -186,6 +186,15 @@ const assert = require('assert');
       await goto(dashboardUrl)
     }
 
+    // Zulip
+    if (taikoTests.includes('zulip') || taikoTests === 'all') {
+      const dashboardUrl = 'https://zulip.' + domain
+
+      console.log('• Zulip')
+      await goto(zulipUrl)
+      await click("Log in with OpenAppStack")
+    }
+
   } catch (error) {
     await screenshot()
     console.error(error)