Increase loki retention time
Currently the log retention time is only one hour:
$ kubectl --namespace oas port-forward service/loki-stack 3100
Forwarding from 127.0.0.1:3100 -> 3100
Forwarding from [::1]:3100 -> 3100
...
❯ logcli query -q --limit=30000 '{job="oas/eventrouter"}' | sed -e 1b -e '$!d'
2021-05-31T17:57:07+02:00 {} 2021-05-31T15:57:07.332946183Z stdout F {"verb":"UPDATED","event":{"metadata":{"name":"wordpress-0.1682ee0e5357c92e","namespace":"oas-apps","uid":"a3243fce-6bd0-4ff8-a798-873f38b7501a","resourceVersion":"59173907","creationTimestamp":"2021-05-27T13:02:41Z","managedFields":[{"manager":"k3s","operation":"Update","apiVersion":"v1","time":"2021-05-27T13:02:41Z"}]},"involvedObject":{"kind":"Pod","namespace":"oas-apps","name":"wordpress-0","uid":"3c03f75e-d1ae-48df-a492-bcf44559271a","apiVersion":"v1","resourceVersion":"58646519","fieldPath":"spec.initContainers{init-wordpress}"},"reason":"BackOff","message":"Back-off restarting failed container","source":{"component":"kubelet","host":"oas.greenhost.net"},"firstTimestamp":"2021-05-27T13:02:41Z","lastTimestamp":"2021-05-31T15:57:07Z","count":24936,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"old_event":{"metadata":{"name":"wordpress-0.1682ee0e5357c92e","namespace":"oas-apps","uid":"a3243fce-6bd0-4ff8-a798-873f38b7501a","resourceVersion":"59173466","creationTimestamp":"2021-05-27T13:02:41Z","managedFields":[{"manager":"k3s","operation":"Update","apiVersion":"v1","time":"2021-05-27T13:02:41Z"}]},"involvedObject":{"kind":"Pod","namespace":"oas-apps","name":"wordpress-0","uid":"3c03f75e-d1ae-48df-a492-bcf44559271a","apiVersion":"v1","resourceVersion":"58646519","fieldPath":"spec.initContainers{init-wordpress}"},"reason":"BackOff","message":"Back-off restarting failed container","source":{"component":"kubelet","host":"oas.greenhost.net"},"firstTimestamp":"2021-05-27T13:02:41Z","lastTimestamp":"2021-05-31T15:52:06Z","count":24915,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""}}
2021-05-31T17:01:36+02:00 {} 2021-05-31T15:01:35.880633793Z stdout F {"verb":"UPDATED","event":{"metadata":{"name":"single-sign-on-recreate-oauth2-clients.1682edff0fc549d8","namespace":"oas","uid":"da83ae84-5cd8-4da2-b8bc-b05e07fd14c6","resourceVersion":"59169104","creationTimestamp":"2021-05-27T13:01:35Z","managedFields":[{"manager":"k3s","operation":"Update","apiVersion":"v1","time":"2021-05-27T13:01:35Z"}]},"involvedObject":{"kind":"CronJob","namespace":"oas","name":"single-sign-on-recreate-oauth2-clients","uid":"201cc5d3-610c-48b3-9082-a93dae4e8824","apiVersion":"batch/v1beta1","resourceVersion":"56910312"},"reason":"FailedNeedsStart","message":"Cannot determine if job needs to be started: too many missed start time (\u003e 100). Set or decrease .spec.startingDeadlineSeconds or check clock skew","source":{"component":"cronjob-controller"},"firstTimestamp":"2021-05-27T13:01:35Z","lastTimestamp":"2021-05-31T15:01:35Z","count":34847,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"old_event":{"metadata":{"name":"single-sign-on-recreate-oauth2-clients.1682edff0fc549d8","namespace":"oas","uid":"da83ae84-5cd8-4da2-b8bc-b05e07fd14c6","resourceVersion":"59168693","creationTimestamp":"2021-05-27T13:01:35Z","managedFields":[{"manager":"k3s","operation":"Update","apiVersion":"v1","time":"2021-05-27T13:01:35Z"}]},"involvedObject":{"kind":"CronJob","namespace":"oas","name":"single-sign-on-recreate-oauth2-clients","uid":"201cc5d3-610c-48b3-9082-a93dae4e8824","apiVersion":"batch/v1beta1","resourceVersion":"56910312"},"reason":"FailedNeedsStart","message":"Cannot determine if job needs to be started: too many missed start time (\u003e 100). Set or decrease .spec.startingDeadlineSeconds or check clock skew","source":{"component":"cronjob-controller"},"firstTimestamp":"2021-05-27T13:01:35Z","lastTimestamp":"2021-05-31T14:56:40Z","count":34818,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""}}
This makes our whole log aggregation useless, and impossible to properly debug issues like #782 (closed) i.e.
Loki retention can be configured like this: https://grafana.com/docs/loki/latest/operations/storage/retention/
Edited by Varac