diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f54fe6407d6fa02681ef3fce9c3482d873302bc5..081150576c883dc02c58791debe86a6cd33721e0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -162,6 +162,11 @@ prometheus-alerts: behave-nextcloud: stage: integration-test script: + # Wait until flux creates the NextCloud HelmRelease. + - ssh root@$ADDRESS '/bin/bash -c "while true; do kubectl get hr -n oas-apps nextcloud; if [ \$? -eq 0 ]; then break; fi; sleep 20; done"' + # Wait until NextCloud is ready. + - ssh root@$ADDRESS '/bin/bash -c "kubectl wait -n oas-apps hr/nextcloud --for condition=Released --timeout=20m"' + # Run the behave tests for NextCloud. - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags nextcloud || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags nextcloud artifacts: paths: @@ -176,6 +181,7 @@ behave-nextcloud: - helmfiles/**/* - test/**/* - openappstack/**/* + extends: .ssh_setup behave-grafana: stage: integration-test diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml index 59a93f3634a3f22f26df147017194791da8b6169..f895ea5d37ad794d1192a56f3241dcf14d41806c 100644 --- a/ansible/bootstrap.yml +++ b/ansible/bootstrap.yml @@ -60,6 +60,10 @@ - import_role: name: apps tags: ['apps'] + - import_role: + name: local-flux + tags: ['flux'] + when: local_flux always: - import_role: name: finalize diff --git a/ansible/group_vars/all/settings.yml.example b/ansible/group_vars/all/settings.yml.example index bf9c3ce75e816c1f866a27a301674c631447e590..f5ad5d7252966b3b7726b17fd2c4dbf585e00cf0 100644 --- a/ansible/group_vars/all/settings.yml.example +++ b/ansible/group_vars/all/settings.yml.example @@ -15,10 +15,10 @@ acme_staging: false # Which apps to install from the helmfile.d/ dir helmfiles: - 00-storage + - 00-flux - 05-cert-manager - 10-nginx - 15-monitoring - - 20-nextcloud # Optional, custom rke config. # I.e. you can set the desired Kubernetes version but please be aware of diff --git a/ansible/roles/apps/tasks/flux.yml b/ansible/roles/apps/tasks/flux.yml new file mode 100644 index 0000000000000000000000000000000000000000..efb88fe105df177b0a7c2e3ffaf65e8e4afbd5bf --- /dev/null +++ b/ansible/roles/apps/tasks/flux.yml @@ -0,0 +1,14 @@ +--- +- name: Install flux + tags: + - helmfile + - flux + include_role: + name: "helmfile" + tasks_from: "apply" + apply: + tags: + - helmfile + - flux + vars: + helmfile: '00-flux' diff --git a/ansible/roles/apps/tasks/init.yml b/ansible/roles/apps/tasks/init.yml index d02a739d32e962297ca21b93f7994fea21e0aa75..a1cd66afc8b512e55566f1be081f71679bb6cf17 100644 --- a/ansible/roles/apps/tasks/init.yml +++ b/ansible/roles/apps/tasks/init.yml @@ -17,20 +17,6 @@ delete: true become: true -- name: Create OAS namespaces - tags: - - kubernetes - - namespace - k8s: - name: '{{ item }}' - api_version: v1 - kind: Namespace - state: present - with_items: - - 'oas' - - 'oas-apps' - - - name: Create value overrides directory tags: - config diff --git a/ansible/roles/apps/tasks/main.yml b/ansible/roles/apps/tasks/main.yml index 2c1afae58ae53bd7ec8bbee5105d471b00f65380..10de1819a8f9b1c23c40f549f10e2262c94f2207 100644 --- a/ansible/roles/apps/tasks/main.yml +++ b/ansible/roles/apps/tasks/main.yml @@ -3,6 +3,11 @@ import_tasks: init.yml tags: [ helmfile ] +- name: Install flux + import_tasks: flux.yml + tags: [ helmfile ] + when: '"00-flux" in helmfiles' + - name: Install local-storage import_tasks: local-storage.yml tags: [ helmfile ] @@ -23,7 +28,5 @@ tags: [ helmfile ] when: '"15-monitoring" in helmfiles' -- name: Install nextcloud +- name: Tasks pertaining to NextCloud import_tasks: nextcloud.yml - tags: [ helmfile ] - when: '"20-nextcloud" in helmfiles' diff --git a/ansible/roles/apps/tasks/nextcloud.yml b/ansible/roles/apps/tasks/nextcloud.yml index acb05aedd10b04940321766eafe33fb1238cda9f..1b6047647c34a46d0a7f88d6fc044f3266f783ea 100644 --- a/ansible/roles/apps/tasks/nextcloud.yml +++ b/ansible/roles/apps/tasks/nextcloud.yml @@ -1,42 +1,18 @@ --- -- name: Clone nextcloud repo - tags: - - git - - helmfile - - nextcloud - git: - repo: 'https://open.greenhost.net/openappstack/nextcloud' - dest: '{{ data_directory }}/source/repos/nextcloud' - version: '{{ git_nextcloud_version }}' - -- name: Remove requirements.lock file - tags: - - git - - nextcloud - - helmfile - file: - path: '{{ data_directory }}/source/repos/nextcloud/nextcloud-onlyoffice/requirements.lock' - state: absent -- name: Install nextcloud and onlyoffice +- name: Create Kubernetes secret with NextCloud values tags: - - helmfile + - config + - flux + - oas - nextcloud - - onlyoffice - include_role: - name: "helmfile" - tasks_from: "apply" - apply: - tags: - - helmfile - - nextcloud - - onlyoffice - environment: - - NEXTCLOUD_PASSWORD: "{{ nextcloud_password }}" - - NEXTCLOUD_MARIADB_PASSWORD: "{{ nextcloud_mariadb_password }}" - - NEXTCLOUD_MARIADB_ROOT_PASSWORD: "{{ nextcloud_mariadb_root_password }}" - - ONLYOFFICE_JWT_SECRET: "{{ onlyoffice_jwt_secret }}" - - ONLYOFFICE_POSTGRESQL_PASSWORD: "{{ onlyoffice_postgresql_password }}" - - ONLYOFFICE_RABBITMQ_PASSWORD: "{{ onlyoffice_rabbitmq_password }}" - vars: - helmfile: '20-nextcloud' + k8s: + state: present + definition: + api_version: v1 + kind: Secret + metadata: + namespace: "oas-apps" + name: "oas" + data: + nextcloud.yaml: "{{ lookup('template','secrets.nextcloud.yaml') | b64encode }}" diff --git a/ansible/roles/apps/templates/secrets.nextcloud.yaml b/ansible/roles/apps/templates/secrets.nextcloud.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8de162654e14adfde8423b96163983717934407 --- /dev/null +++ b/ansible/roles/apps/templates/secrets.nextcloud.yaml @@ -0,0 +1,89 @@ +nextcloud: + nextcloud: + host: "files.{{ domain }}" + password: "{{ nextcloud_password }}" + + ingress: + enabled: true + annotations: + # Tell cert-manager to automatically get a TLS certificate + kubernetes.io/tls-acme: "true" + # Set max body size high to allow big NextCloud uploads + nginx.ingress.kubernetes.io/proxy-body-size: 1G + nginx.ingress.kubernetes.io/server-snippet: |- + server_tokens off; + proxy_hide_header X-Powered-By; + rewrite ^/.well-known/webfinger /public.php?service=webfinger last; + rewrite ^/.well-known/host-meta /public.php?service=host-meta last; + rewrite ^/.well-known/host-meta.json /public.php?service=host-meta-json; + location = /.well-known/carddav { + return 301 $scheme://$host/remote.php/dav; + } + location = /.well-known/caldav { + return 301 $scheme://$host/remote.php/dav; + } + location = /robots.txt { + allow all; + log_not_found off; + access_log off; + } + hosts: + - "files.{{ domain }}" + tls: + - hosts: + - "files.{{ domain }}" + secretName: oas-nextcloud-files + + # Use 2 GB of storage for NC storage (maybe make configurable later?) + persistence: + enabled: true + size: 2Gi + + # Explicitly disable use of internal database + internalDatabase: + enabled: false + + livenessProbe: + initialDelaySeconds: 120 + failureThreshold: 20 + readinessProbe: + initialDelaySeconds: 120 + +# Enable and configure MariaDB chart +mariadb: + db: + password: "{{ nextcloud_mariadb_password }}" + enabled: true + master: + persistence: + ## Enable PostgreSQL persistence using Persistent Volume Claims. + enabled: true + size: 512Mi + replication: + enabled: false + rootUser: + password: "{{ nextcloud_mariadb_root_password }}" + +onlyoffice: + server_name: "office.{{ domain }}" + ingress: + enabled: true + annotations: + # Tell cert-manager to automatically get a TLS certificate + kubernetes.io/tls-acme: "true" + paths: + - "/" + hosts: + - "office.{{ domain }}" + tls: + - hosts: + - "office.{{ domain }}" + secretName: oas-nextcloud-office + jwtSecret: "{{ onlyoffice_jwt_secret }}" + +postgresql: + postgresqlPassword: "{{ onlyoffice_postgresql_password }}" + +rabbitmq: + rabbitmq: + password: "{{ onlyoffice_rabbitmq_password }}" diff --git a/ansible/roles/local-flux/files/Chart.yaml b/ansible/roles/local-flux/files/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65f31ac167e1b6a9c9be73a6a091373d2dd7fb0e --- /dev/null +++ b/ansible/roles/local-flux/files/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +name: local-flux +version: 0.1.0 +description: | + Sets up an nginx deployment to serve a git repo (stored on local + disk) via http diff --git a/ansible/roles/local-flux/files/git-hook.sh b/ansible/roles/local-flux/files/git-hook.sh new file mode 100644 index 0000000000000000000000000000000000000000..dac4c53199531b3b62211e8f587d1d6288e0fb91 --- /dev/null +++ b/ansible/roles/local-flux/files/git-hook.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec git update-server-info diff --git a/ansible/roles/local-flux/files/nginx.yaml b/ansible/roles/local-flux/files/nginx.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a61cd82cd37ddf6f8e1489f2c554a76f8ad0ff71 --- /dev/null +++ b/ansible/roles/local-flux/files/nginx.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ .Release.Name }}-data + labels: + type: local +spec: + storageClassName: "" + capacity: + storage: 10Mi + accessModes: + - ReadOnlyMany + hostPath: + path: "{{ .Values.local_flux_directory }}" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + namespace: oas + name: {{ .Release.Name }}-data +spec: + storageClassName: "" + # Make sure this claims uses the persistent volume describe above. + volumeName: {{ .Release.Name }}-data + accessModes: + - ReadOnlyMany + resources: + requests: + storage: 10Mi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: oas + name: {{ .Release.Name }} +spec: + selector: + matchLabels: + app: {{ .Release.Name }} + replicas: 1 + template: + metadata: + labels: + app: {{ .Release.Name }} + spec: + containers: + - name: {{ .Release.Name }}-nginx + image: nginx:{{ .Values.nginx.image.tag }} + ports: + - containerPort: 80 + volumeMounts: + - mountPath: "/usr/share/nginx/html" + name: {{ .Release.Name }}-data + volumes: + - name: {{ .Release.Name }}-data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-data +--- +apiVersion: v1 +kind: Service +metadata: + namespace: oas + name: {{ .Release.Name }} +spec: + selector: + app: {{ .Release.Name }} + ports: + - protocol: TCP + port: 80 + targetPort: 80 diff --git a/ansible/roles/local-flux/files/values.yaml b/ansible/roles/local-flux/files/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9d504f4bc356be2e58c70f65b16f97db911cf2a --- /dev/null +++ b/ansible/roles/local-flux/files/values.yaml @@ -0,0 +1,6 @@ +# Directory on node where the HelmRelease files are stored. +local_flux_directory: "/var/lib/OpenAppStack/local-flux" + +nginx: + image: + tag: "1.17-alpine" diff --git a/ansible/roles/local-flux/tasks/main.yml b/ansible/roles/local-flux/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..00e5a4257ff302478c78db13ab37d3269de8550e --- /dev/null +++ b/ansible/roles/local-flux/tasks/main.yml @@ -0,0 +1,72 @@ +--- + +- block: + + - name: Copy HelmRelease files to server's local flux repo + tags: + - flux + copy: + src: "../../../flux/" + dest: "{{ repo }}" + register: helmreleases + become: true + + - name: Create local flux repo + tags: + - flux + command: git init "{{ repo }}" + args: + creates: "{{ repo }}/.git" + become: true + + - name: Enable post-update hook in local flux repo + tags: + - flux + copy: + src: "git-hook.sh" + dest: "{{ repo }}/.git/hooks/{{ item }}" + mode: "0755" + with_items: + - "post-update" + - "post-commit" + become: true + + - name: Add HelmRelease files to local flux commit + tags: + - flux + shell: git add . && git -c "user.name=OpenAppStack automation" -c "user.email=tech@openappstack.net" commit --allow-empty --author="OpenAppStack automation <>" -m "Local flux via ansible" + args: + chdir: "{{ repo }}" + when: helmreleases.changed + become: true + + - name: Create local-flux helm chart directory + tags: + - flux + file: + path: "/var/lib/OpenAppStack/source/local-flux/templates" + state: directory + become: true + + - name: Copy local-flux helm chart to server + tags: + - flux + copy: + src: "{{ item.file }}" + dest: "/var/lib/OpenAppStack/source/local-flux/{{ item.subdir }}/{{ item.file }}" + with_items: + - file: "nginx.yaml" + subdir: "templates" + - file: "values.yaml" + subdir: "." + - file: "Chart.yaml" + subdir: "." + become: true + + - name: Install local-flux helm chart + tags: + - flux + shell: helm install --namespace=oas --name=local-flux /var/lib/OpenAppStack/source/local-flux + + vars: + repo: "/var/lib/OpenAppStack/local-flux" diff --git a/ansible/roles/setup/tasks/main.yml b/ansible/roles/setup/tasks/main.yml index 4ac4ed4cfafb0824d1a1d98fc658db63cb40829b..e6ef312353f85abe9a7a8c7dd98c75ec41079b83 100644 --- a/ansible/roles/setup/tasks/main.yml +++ b/ansible/roles/setup/tasks/main.yml @@ -3,3 +3,4 @@ - import_tasks: rke.yml - import_tasks: tiller.yml - import_tasks: krew.yml +- import_tasks: namespaces.yml diff --git a/ansible/roles/setup/tasks/namespaces.yml b/ansible/roles/setup/tasks/namespaces.yml new file mode 100644 index 0000000000000000000000000000000000000000..d9435fe92e5ec12f6372cdc13de625d75294607e --- /dev/null +++ b/ansible/roles/setup/tasks/namespaces.yml @@ -0,0 +1,14 @@ +--- + +- name: Create OAS namespaces + tags: + - kubernetes + - namespace + k8s: + name: '{{ item }}' + api_version: v1 + kind: Namespace + state: present + with_items: + - 'oas' + - 'oas-apps' diff --git a/flux/nextcloud.yaml b/flux/nextcloud.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1df101280d86dcbdf5009d2ab99440bae05dc985 --- /dev/null +++ b/flux/nextcloud.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: helm.fluxcd.io/v1 +kind: HelmRelease +metadata: + name: nextcloud + namespace: oas-apps + annotations: + flux.weave.works/automated: "false" +spec: + # Calling the release "nextcloud" runs into a bug in the helm chart. + # See https://open.greenhost.net/openappstack/nextcloud/issues/3 for details. + releaseName: nc + chart: + git: https://open.greenhost.net/openappstack/nextcloud + ref: master + path: . + valuesFrom: + - secretKeyRef: + name: oas + key: nextcloud.yaml + timeout: 900 diff --git a/helmfiles/helmfile.d/00-flux.yaml b/helmfiles/helmfile.d/00-flux.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d535369ee74fa66ca01a0478eb2a647ac88c3d1d --- /dev/null +++ b/helmfiles/helmfile.d/00-flux.yaml @@ -0,0 +1,37 @@ +environments: + oas: + values: + - "/etc/OpenAppStack/values/local.yaml" + +repositories: + - name: "fluxcd" + url: "https://charts.fluxcd.io" + +releases: + - name: "oas-{{ .Environment.Values.releaseName }}-flux" + namespace: "oas" + chart: "fluxcd/flux" + values: + - git: + # This is the url to the "local-flux" nginx pod that is running + # inside the cluster, and is serving the git repo with HelmRelease + # files over http. + url: "http://local-flux.oas.svc.cluster.local/.git" + readonly: true + registry: + # Do not do follow updates of upstream docker images automatically. + excludeImage: "*" + sync: + # Necessary for read-only mode. + state: "secret" + syncGarbageCollection: + # Delete resources originally created by Flux when their manifests + # are removed from the git repo. + enabled: true + wait: false + - name: "oas-{{ .Environment.Values.releaseName }}-helm-operator" + namespace: "oas" + chart: "fluxcd/helm-operator" + values: + - createCRD: true + wait: false diff --git a/helmfiles/helmfile.d/20-nextcloud.yaml b/helmfiles/helmfile.d/20-nextcloud.yaml deleted file mode 100644 index 6167231fe2a8874203c3087e007fe44a6cb37e5a..0000000000000000000000000000000000000000 --- a/helmfiles/helmfile.d/20-nextcloud.yaml +++ /dev/null @@ -1,15 +0,0 @@ -environments: - oas: - values: - - "/etc/OpenAppStack/values/local.yaml" - -releases: - - name: "oas-{{ .Environment.Values.releaseName }}-files" - namespace: "oas-apps" - # Install from file path, so you don't run into https://github.com/roboll/helmfile/issues/726 - chart: "../../repos/nextcloud" - values: - - "../values/nextcloud.yaml.gotmpl" - - "/etc/OpenAppStack/values/apps/nextcloud.yaml.gotmpl" - wait: true - timeout: 900 diff --git a/openappstack/cluster.py b/openappstack/cluster.py index 51cf07f4f49b1fbd28d0bfae6677acd08cc9f4a6..a3927bd8743b01527ccdd53e3db227075569c879 100644 --- a/openappstack/cluster.py +++ b/openappstack/cluster.py @@ -41,6 +41,9 @@ class Cluster: self.domain = None # By default, use Let's Encrypt's live environment self.acme_staging = False + # Let the auto-update mechanism (flux) follow a cluster-local git repo, + # not one hosted on open.greenhost.net. + self.local_flux = True # Set this to False if the data needs to be (re)loaded from file self.data_loaded = False # Load data from inventory.yml and settings.yml @@ -147,6 +150,7 @@ class Cluster: settings['domain'] = self.domain settings['admin_email'] = 'admin@{0}'.format(self.domain) settings['acme_staging'] = self.acme_staging + settings['local_flux'] = self.local_flux settings['cluster_dir'] = self.cluster_dir file_contents = yaml.safe_dump(settings, default_flow_style=False) diff --git a/test/letsencrypt_staging_bundle.pem b/test/letsencrypt_staging_bundle.pem deleted file mode 100644 index 5f5342293f258c4cd2ff216f00c371b70e4bcf3b..0000000000000000000000000000000000000000 --- a/test/letsencrypt_staging_bundle.pem +++ /dev/null @@ -1,56 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw -GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2 -MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0 -8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym -oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0 -ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN -xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56 -dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9 -AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw -HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0 -BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu -b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu -Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq -hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF -UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9 -AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp -DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7 -IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf -zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI -PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w -SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em -2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0 -WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt -n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU= ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIFATCCAumgAwIBAgIRAKc9ZKBASymy5TLOEp57N98wDQYJKoZIhvcNAQELBQAw -GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDMyMzIyNTM0NloXDTM2 -MDMyMzIyNTM0NlowGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMIICIjANBgkq -hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA+pYHvQw5iU3v2b3iNuYNKYgsWD6KU7aJ -diddtZQxSWYzUI3U0I1UsRPTxnhTifs/M9NW4ZlV13ZfB7APwC8oqKOIiwo7IwlP -xg0VKgyz+kT8RJfYr66PPIYP0fpTeu42LpMJ+CKo9sbpgVNDZN2z/qiXrRNX/VtG -TkPV7a44fZ5bHHVruAxvDnylpQxJobtCBWlJSsbIRGFHMc2z88eUz9NmIOWUKGGj -EmP76x8OfRHpIpuxRSCjn0+i9+hR2siIOpcMOGd+40uVJxbRRP5ZXnUFa2fF5FWd -O0u0RPI8HON0ovhrwPJY+4eWKkQzyC611oLPYGQ4EbifRsTsCxUZqyUuStGyp8oa -aoSKfF6X0+KzGgwwnrjRTUpIl19A92KR0Noo6h622OX+4sZiO/JQdkuX5w/HupK0 -A0M0WSMCvU6GOhjGotmh2VTEJwHHY4+TUk0iQYRtv1crONklyZoAQPD76hCrC8Cr -IbgsZLfTMC8TWUoMbyUDgvgYkHKMoPm0VGVVuwpRKJxv7+2wXO+pivrrUl2Q9fPe -Kk055nJLMV9yPUdig8othUKrRfSxli946AEV1eEOhxddfEwBE3Lt2xn0hhiIedbb -Ftf/5kEWFZkXyUmMJK8Ra76Kus2ABueUVEcZ48hrRr1Hf1N9n59VbTUaXgeiZA50 -qXf2bymE6F8CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB -Af8wHQYDVR0OBBYEFMEmdKSKRKDm+iAo2FwjmkWIGHngMA0GCSqGSIb3DQEBCwUA -A4ICAQBCPw74M9X/Xx04K1VAES3ypgQYH5bf9FXVDrwhRFSVckria/7dMzoF5wln -uq9NGsjkkkDg17AohcQdr8alH4LvPdxpKr3BjpvEcmbqF8xH+MbbeUEnmbSfLI8H -sefuhXF9AF/9iYvpVNC8FmJ0OhiVv13VgMQw0CRKkbtjZBf8xaEhq/YqxWVsgOjm -dm5CAQ2X0aX7502x8wYRgMnZhA5goC1zVWBVAi8yhhmlhhoDUfg17cXkmaJC5pDd -oenZ9NVhW8eDb03MFCrWNvIh89DDeCGWuWfDltDq0n3owyL0IeSn7RfpSclpxVmV -/53jkYjwIgxIG7Gsv0LKMbsf6QdBcTjhvfZyMIpBRkTe3zuHd2feKzY9lEkbRvRQ -zbh4Ps5YBnG6CKJPTbe2hfi3nhnw/MyEmF3zb0hzvLWNrR9XW3ibb2oL3424XOwc -VjrTSCLzO9Rv6s5wi03qoWvKAQQAElqTYRHhynJ3w6wuvKYF5zcZF3MDnrVGLbh1 -Q9ePRFBCiXOQ6wPLoUhrrbZ8LpFUFYDXHMtYM7P9sc9IAWoONXREJaO08zgFtMp4 -8iyIYUyQAbsvx8oD2M8kRvrIRSrRJSl6L957b4AFiLIQ/GgV2curs0jje7Edx34c -idWw1VrejtwclobqNMVtG3EiPUIpJGpbMcJgbiLSmKkrvQtGng== ------END CERTIFICATE-----