Merge branch '709-add-proper-labels-to-eventrouter-logs' into 'master'

Resolve "Add proper labels to eventrouter logs"

Closes #709

See merge request openappstack/openappstack!408

.gitlab-ci.yml

@@ -79,7 +79,11 @@ include:
   extends:
     - .general_rules
 
-.loki_stack_rules:
+.loki_rules:
+  extends:
+    - .general_rules
+
+.promtail_rules:
   extends:
     - .general_rules
 
@@ -305,12 +309,19 @@ enable-eventrouter:
     - .enable_app_template
     - .eventrouter_rules
 
-enable-loki-stack:
+enable-loki:
   variables:
-    APP: "loki-stack"
+    APP: "loki"
   extends:
     - .enable_app_template
-    - .loki_stack_rules
+    - .loki_rules
+
+enable-promtail:
+  variables:
+    APP: "promtail"
+  extends:
+    - .enable_app_template
+    - .promtail_rules
 
 enable-nextcloud:
   variables:
@@ -429,12 +440,19 @@ local-path-provisioner-helm-release:
     - .helm-release
     - .local_path_provisioner_rules
 
-loki-stack-helm-release:
+loki-helm-release:
   variables:
-    APP: "loki-stack"
+    APP: "loki"
   extends:
     - .helm-release
-    - .loki_stack_rules
+    - .loki_rules
+
+promtail-helm-release:
+  variables:
+    APP: "promtail"
+  extends:
+    - .helm-release
+    - .promtail_rules
 
 nextcloud-helm-release:
   variables:
@@ -519,15 +537,25 @@ local-path-provisioner-ready:
     - .apps-ready
     - .local_path_provisioner_rules
 
-loki-stack-ready:
+loki-ready:
+  variables:
+    APP: "loki"
+  needs:
+    - job: loki-helm-release
+    - job: setup-openappstack
+  extends:
+    - .apps-ready
+    - .loki_rules
+
+promtail-ready:
   variables:
-    APP: "loki-stack"
+    APP: "promtail"
   needs:
-    - job: loki-stack-helm-release
+    - job: promtail-helm-release
     - job: setup-openappstack
   extends:
     - .apps-ready
-    - .loki_stack_rules
+    - .promtail_rules
 
 nextcloud-ready:
   variables:

.gitlab/issue_templates/update_all_components.md

@@ -21,7 +21,8 @@ Installed by flux:
 * [ ] ./oas/prometheus.yaml
 * [ ] ./oas/letsencrypt-staging.yaml
 * [ ] ./oas/ingress.yaml
-* [ ] ./oas/loki-stack.yaml
+* [ ] ./oas/loki.yaml
+* [ ] ./oas/promtail.yaml
 * [ ] ./velero/velero.yaml
 * [ ] ./oas-custom/flux-custom.yaml
 * [ ] ./cert-manager/cert-manager.yaml

ansible/group_vars/all/settings.yml.example

@@ -95,7 +95,8 @@ enabled_applications:
   #
   # Monitoring components (auto-enabled by GitLab CI)
   # - 'prometheus-stack'
-  # - 'loki-stack'
+  # - 'loki'
+  # - 'promtail'
   # - 'eventrouter'
   #
   # The backup system Velero is disabled by default, see settings under `backup` above.

ansible/roles/apps/tasks/loki-stack.yml → ansible/roles/apps/tasks/loki.yml

-- name: Create Kubernetes secret with loki-stack settings
+- name: Create Kubernetes secret with loki settings
   tags:
     - config
     - flux
-    - loki-stack
+    - loki
   vars:
     flux_secret:
-      name: "loki-stack"
+      name: "loki"
       namespace: "oas"
   include_tasks:
     file: flux_secret.yml
@@ -13,4 +13,4 @@
       tags:
         - config
         - flux
-        - loki-stack
+        - loki

ansible/roles/apps/tasks/main.yml

@@ -12,8 +12,11 @@
 - name: Tasks pertaining to prometheus and grafana
   import_tasks: prometheus-stack.yml
 
-- name: Tasks pertaining to loki-stack
-  import_tasks: loki-stack.yml
+- name: Tasks pertaining to loki
+  import_tasks: loki.yml
+
+- name: Tasks pertaining to promtail
+  import_tasks: promtail.yml
 
 - name: Tasks pertaining to eventrouter
   import_tasks: eventrouter.yml

ansible/roles/apps/tasks/promtail.yml

+- name: Create Kubernetes secret with promtail settings
+  tags:
+    - config
+    - flux
+    - promtail
+  vars:
+    flux_secret:
+      name: "promtail"
+      namespace: "oas"
+  include_tasks:
+    file: flux_secret.yml
+    apply:
+      tags:
+        - config
+        - flux
+        - promtail

ansible/roles/apps/templates/settings/loki-stack.yaml

----
-# https://github.com/grafana/helm-charts/blob/main/charts/loki-stack/values.yaml
-loki:
-  # https://github.com/grafana/helm-charts/blob/main/charts/loki/values.yaml
-  enabled: true
-  resources:
-    limits:
-      cpu: 400m
-      memory: 180Mi
-    requests:
-      cpu: 200m
-      memory: 90Mi
-  persistence:
-    enabled: true
-    accessModes:
-    - ReadWriteOnce
-    size: 10Gi
-    annotations: {}
-    # existingClaim:
-  config:
-    # https://github.com/grafana/loki/blob/main/cmd/loki/loki-local-config.yaml
-    # https://grafana.com/docs/loki/latest/operations/storage/retention
-    schema_config:
-      configs:
-      - from: 2021-05-31
-        store: boltdb-shipper
-        object_store: filesystem
-        schema: v11
-        index:
-          prefix: index_
-          period: 24h
-    storage_config:
-      boltdb_shipper:
-        active_index_directory: /data/loki/boltdb-shipper-active
-        cache_location: /data/loki/boltdb-shipper-cache
-        cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
-        shared_store: filesystem
-      filesystem:
-        directory: /data/loki/chunks
-    compactor:
-      working_directory: /data/loki/boltdb-shipper-compactor
-      shared_store: filesystem
-    limits_config:
-      reject_old_samples: true
-      reject_old_samples_max_age: 168h  # 7 days
-    chunk_store_config:
-      max_look_back_period: 672h # 28 days
-    table_manager:
-      retention_deletes_enabled: true
-      retention_period: 672h  # 28 days
-
-promtail:
-  # https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml
-  enabled: true
-  initContainer:
-    enabled: true
-    fsInotifyMaxUserInstances: 512
-  resources:
-    limits:
-      cpu: 400m
-      memory: 256Mi
-    requests:
-      cpu: 300m
-      memory: 128Mi

ansible/roles/apps/templates/settings/loki.yaml

+---
+enabled: true
+resources:
+  limits:
+    cpu: 400m
+    memory: 180Mi
+  requests:
+    cpu: 200m
+    memory: 90Mi

ansible/roles/apps/templates/settings/promtail.yaml

+---
+enabled: true
+initContainer:
+  enabled: true
+  fsInotifyMaxUserInstances: 512
+resources:
+  limits:
+    cpu: 400m
+    memory: 256Mi
+  requests:
+    cpu: 300m
+    memory: 128Mi
+config:
+  lokiAddress: http://loki:3100/loki/api/v1/push
+  # https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml#L217
+  snippets:
+    # https://grafana.com/docs/loki/latest/clients/promtail/pipelines/
+    pipelineStages:
+      - cri: {}
+      - match:
+          selector: '{app="eventrouter"}'
+          stages:
+            - json:
+                expressions:
+                  event_verb: verb
+                  event_kind: event.involvedObject.kind
+                  event_reason: event.reason
+                  event_namespace: event.involvedObject.namespace
+                  event_name: event.metadata.name
+                  event_source_host: event.source.host
+                  event_source_component: event.source.component
+            - labels:
+                event_verb:
+                event_kind:
+                event_reason:
+                event_namespace:
+                event_name:
+                event_source_host:
+                event_source_component:

docs/upgrading.md

@@ -81,7 +81,8 @@ enabled_applications:
   # - 'velero'
   # Applications.
   - 'grafana'
-  - 'loki-stack'
+  - 'loki'
+  - 'promtail'
   - 'nextcloud'
   - 'prometheus'
   - 'rocketchat'

flux/oas/loki_cm.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: loki
+    grafana_datasource: "1"
+    release: loki
+  name: loki-datasource
+  namespace: oas
+data:
+  loki-stack-datasource.yaml: |-
+    apiVersion: 1
+    datasources:
+    - name: Loki
+      type: loki
+      access: proxy
+      url: http://loki:3100
+      version: 1

flux/oas/loki-stack_hr.yaml → flux/oas/loki_hr.yaml

@@ -2,18 +2,18 @@
 apiVersion: helm.fluxcd.io/v1
 kind: HelmRelease
 metadata:
-  name: loki-stack
+  name: loki
   namespace: oas
   annotations:
     flux.weave.works/automated: "false"
 spec:
-  releaseName: loki-stack
+  releaseName: loki
   chart:
-    # https://github.com/grafana/helm-charts/tree/main/charts/loki-stack
+    # https://github.com/grafana/helm-charts/tree/main/charts/loki
     repository: https://grafana.github.io/helm-charts
-    name: loki-stack
-    version: 2.4.1
+    name: loki
+    version: 2.5.0
   valuesFrom:
   - secretKeyRef:
-      name: loki-stack-settings
+      name: loki-settings
       key: values.yaml

flux/oas/promtail_hr.yaml

+---
+apiVersion: helm.fluxcd.io/v1
+kind: HelmRelease
+metadata:
+  name: promtail
+  namespace: oas
+  annotations:
+    flux.weave.works/automated: "false"
+spec:
+  releaseName: promtail
+  chart:
+    # https://github.com/grafana/helm-charts/tree/main/charts/promtail
+    repository: https://grafana.github.io/helm-charts
+    name: promtail
+    version: 3.5.1
+  valuesFrom:
+  - secretKeyRef:
+      name: promtail-settings
+      key: values.yaml

test/pytest/test_app_deployments.py

@@ -17,7 +17,8 @@ EXPECTED_RELEASES = {
     'oas': [
         'ingress',
         'prometheus-stack',
-        'loki-stack',
+        'loki',
+        'promtail',
         'eventrouter',
         'single-sign-on'
     ],
@@ -35,9 +36,12 @@ EXPECTED_APP_LABELS = {
     'local-path-provisioner': {
         'namespace': 'kube-system',
         'label_selector': 'app.kubernetes.io/instance=local-path-provisioner'},
-    'loki-stack': {
+    'loki': {
         'namespace': 'oas',
         'label_selector': 'app=loki'},
+    'promtail': {
+        'namespace': 'oas',
+        'label_selector': 'app=promtail'},
     'nextcloud': {
         'namespace': 'oas-apps',
         'label_selector': 'app.kubernetes.io/instance=nc'},