.gitlab-ci.yml 6.94 KB
Newer Older
Varac's avatar
Varac committed
1
2
include:
  - .gitlab/ci_templates/kaniko.yml
Varac's avatar
Varac committed
3
  - .gitlab/ci_templates/ssh_setup.yml
Varac's avatar
Varac committed
4
5
stages:
  - build
6
  - create-vps
7
8
  - setup-cluster
  - install-apps
Varac's avatar
Varac committed
9
  - wait-for-deployments
10
11
  - health-test
  - integration-test
Varac's avatar
Varac committed
12
  - cleanup
13
14
variables:
  SSH_KEY_ID: "411"
15
  HOSTNAME: "${CI_COMMIT_REF_SLUG}"
Maarten de Waard's avatar
Maarten de Waard committed
16
  # Repeated values, because we're not allowed to use a variable in a variable
17
  SUBDOMAIN: "${CI_COMMIT_REF_SLUG}.ci"
18
  DOMAIN: "openappstack.net"
19
  ADDRESS: "${CI_COMMIT_REF_SLUG}.ci.openappstack.net"
Varac's avatar
Varac committed
20
  ANSIBLE_HOST_KEY_CHECKING: "False"
21
22
23
  KANIKO_BUILD_IMAGENAME: "openappstack-ci"

default:
24
25
  image: "${CI_REGISTRY_IMAGE}/${KANIKO_BUILD_IMAGENAME}:${CI_COMMIT_REF_SLUG}"

Varac's avatar
Varac committed
26

Varac's avatar
Varac committed
27
28
29
30
ci_test_image:
  stage: build
  only:
    changes:
31
      - .gitlab-ci.yml
32
      - Dockerfile
33
      - requirements.txt
34
      - .gitlab/ci_templates/kaniko.yml
Varac's avatar
Varac committed
35
  extends: .kaniko_build
Varac's avatar
Varac committed
36

37
38
create-vps:
  stage: create-vps
Varac's avatar
Varac committed
39
  script:
40
41
    - echo "hostname $HOSTNAME, subdomain $SUBDOMAIN, domain $DOMAIN, address $ADDRESS";
    - ls clusters/${HOSTNAME} || echo "directory clusters/${HOSTNAME} not found"
Maarten de Waard's avatar
Maarten de Waard committed
42
43
    # Creates the VPS only if an old VPS for this branch is not re-usable
    - sh .gitlab/ci_scripts/create_vps.sh
44
45
46
47
48
49
50
51
  artifacts:
    paths:
    - clusters
    expire_in: 1 month
    when: always
  only:
    changes:
      - .gitlab-ci.yml
Maarten de Waard's avatar
Maarten de Waard committed
52
      - .gitlab/ci_scripts/*
53
      - ansible/**/*
Arie Peterson's avatar
Arie Peterson committed
54
      - flux/**/*
55
56
57
58
59
60
61
62
63
      - test/**/*
      - openappstack/**/*
  extends: .ssh_setup
  # Cache the cluster secrets so the next job can use it too
  cache:
    paths:
      - clusters/$HOSTNAME/**
    key: ${CI_COMMIT_REF_SLUG}

64
setup-openappstack:
65
66
67
  stage: setup-cluster
  script:
    # Copy inventory files to ansible folder for use in install-apps step
Varac's avatar
Varac committed
68
    - chmod 700 ansible
69
70
    - cp clusters/${CI_COMMIT_REF_SLUG}/inventory.yml ansible/
    - cp clusters/${CI_COMMIT_REF_SLUG}/settings.yml ansible/group_vars/all/
71
    # Set up cluster
72
    - python3 -m openappstack $HOSTNAME install
Varac's avatar
Varac committed
73
    # Show versions of installed apps/binaries
Varac's avatar
Varac committed
74
75
    - chmod 700 ansible
    - cd ansible
Varac's avatar
Varac committed
76
    - ansible master -m shell -a 'oas-version-info.sh 2>&1'
Varac's avatar
Varac committed
77
78
  artifacts:
    paths:
Maarten de Waard's avatar
Maarten de Waard committed
79
    - ./clusters
Varac's avatar
Varac committed
80
81
    - ansible/inventory.yml
    - ansible/group_vars/all/settings.yml
Varac's avatar
Varac committed
82
    expire_in: 1 month
83
    when: always
84
85
  only:
    changes:
86
      - .gitlab-ci.yml
87
      - ansible/**/*
88
      - flux/**/*
89
      - test/**/*
90
      - openappstack/**/*
Varac's avatar
Varac committed
91
  extends: .ssh_setup
92
93
94
95
96
  # Cache the cluster secrets so the next job can use them
  cache:
    paths:
      - clusters/$HOSTNAME/**
    key: ${CI_COMMIT_REF_SLUG}
97

Varac's avatar
Varac committed
98
99
100
101
102
103
104
105
106
107
helmreleases:
  stage: wait-for-deployments
  script:
    - cd ansible/
    - export KUBECONFIG="${PWD}/../clusters/${HOSTNAME}/secrets/kube_config_cluster.yml"
    - pytest -v -s -m 'helmreleases' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*' --reruns 120 --reruns-delay 10
  only:
    changes:
      - .gitlab-ci.yml
      - ansible/**/*
Arie Peterson's avatar
Arie Peterson committed
108
      - flux/**/*
Varac's avatar
Varac committed
109
110
111
112
      - test/**/*
      - openappstack/**/*
  extends: .ssh_setup

Varac's avatar
Varac committed
113
testinfra:
114
  stage: health-test
Varac's avatar
Varac committed
115
  script:
116
    - cd ansible/
117
    - pytest -v -m 'testinfra' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*'
118
119
  only:
    changes:
120
      - .gitlab-ci.yml
121
      - ansible/**/*
Arie Peterson's avatar
Arie Peterson committed
122
      - flux/**/*
123
      - test/**/*
124
      - openappstack/**/*
Varac's avatar
Varac committed
125
  extends: .ssh_setup
126
127

certs:
128
  stage: health-test
Varac's avatar
Varac committed
129
  allow_failure: true
130
  script:
131
    - cd ansible/
132
    - pytest -s -m 'certs' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*'
133
134
  only:
    changes:
135
      - .gitlab-ci.yml
136
      - ansible/**/*
Arie Peterson's avatar
Arie Peterson committed
137
      - flux/**/*
138
      - test/**/*
139
      - openappstack/**/*
Varac's avatar
Varac committed
140
  extends: .ssh_setup
Varac's avatar
Varac committed
141

Varac's avatar
Varac committed
142
143
144
prometheus-alerts:
  stage: health-test
  variables:
145
    OAS_DOMAIN: '${CI_COMMIT_REF_SLUG}.ci.openappstack.net'
Varac's avatar
Varac committed
146
147
148
  allow_failure: true
  script:
    - cd test/
149
    - pytest -s -m 'prometheus' --connection=ansible --ansible-inventory=../clusters/${HOSTNAME}/inventory.yml --hosts='ansible://*'
Varac's avatar
Varac committed
150
151
  only:
    changes:
152
      - .gitlab-ci.yml
Varac's avatar
Varac committed
153
      - ansible/**/*
Arie Peterson's avatar
Arie Peterson committed
154
      - flux/**/*
Varac's avatar
Varac committed
155
      - test/**/*
Varac's avatar
Varac committed
156
  extends: .ssh_setup
Varac's avatar
Varac committed
157

Varac's avatar
Varac committed
158
behave-nextcloud:
159
  stage: integration-test
Varac's avatar
Varac committed
160
  script:
161
162
163
164
165
    # Wait until flux creates the NextCloud HelmRelease.
    - ssh root@$ADDRESS '/bin/bash -c "while true; do kubectl get hr -n oas-apps nextcloud; if [ \$? -eq 0 ]; then break; fi; sleep 20; done"'
    # Wait until NextCloud is ready.
    - ssh root@$ADDRESS '/bin/bash -c "kubectl wait -n oas-apps hr/nextcloud --for condition=Released --timeout=20m"'
    # Run the behave tests for NextCloud.
Varac's avatar
Varac committed
166
    - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags nextcloud || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags nextcloud
Varac's avatar
Varac committed
167
168
169
170
171
  artifacts:
    paths:
    - test/behave/screenshots/
    expire_in: 1 month
    when: on_failure
Varac's avatar
Varac committed
172
  retry: 2
173
174
  only:
    changes:
175
      - .gitlab-ci.yml
176
      - ansible/**/*
Arie Peterson's avatar
Arie Peterson committed
177
      - flux/**/*
178
      - test/**/*
179
      - openappstack/**/*
180
  extends: .ssh_setup
Varac's avatar
Varac committed
181

Varac's avatar
Varac committed
182
183
184
behave-grafana:
  stage: integration-test
  script:
Varac's avatar
Varac committed
185
    - python3 -m openappstack $HOSTNAME test --behave-headless --behave-tags grafana || python3 -m openappstack $HOSTNAME test --behave-headless --behave-rerun-failing --behave-tags grafana
Varac's avatar
Varac committed
186
187
188
189
190
  artifacts:
    paths:
    - test/behave/screenshots/
    expire_in: 1 month
    when: on_failure
191
192
  only:
    changes:
193
      - .gitlab-ci.yml
194
      - ansible/**/*
Arie Peterson's avatar
Arie Peterson committed
195
      - flux/**/*
196
      - test/**/*
197
      - openappstack/**/*
Varac's avatar
Varac committed
198

199
200
201
202
203
204
terminate_mr_droplet_after_merge:
  stage: cleanup
  before_script:
    - echo "We leave MR droplets running even when the pipeline is successful \
      to be able to investigate a MR. We need to terminate them when the MR \
      is merged into master."
205
  script: |
206
    if [ "$(git show -s --pretty=%p HEAD | wc -w)" -gt 1 ]
207
    then
208
209
210
211
      commit_message="$(git show -s --format=%s)"
      tmp="${commit_message#*\'}"
      merged_branch="${tmp%%\'*}"
      echo "Current HEAD is a merge commit, removing droplet from related merge request branch name '#${merged_branch}'."
212
      python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name(\"^${merged_branch}\.\")"
213
214
215
216
217
218
219
220
    else
      echo "Current HEAD is NOT a merge commit, nothing to do."
    fi
  only:
    refs:
      - master

terminate_old_droplets:
Varac's avatar
Varac committed
221
222
  stage: cleanup
  script:
223
    - echo "Terminate droplets 5 days after creation. Branches that exist longer than 5 days will get a new droplet when CI runs again."
224
    - python3 -c "import greenhost_cloud; greenhost_cloud.terminate_droplets_by_name('\d+-.*', 5)"
225
226
  only:
    changes:
227
      - .gitlab-ci.yml
228
      - ansible/**/*
Arie Peterson's avatar
Arie Peterson committed
229
      - flux/**/*
230
      - test/**/*
231
      - openappstack/**/*
232

233
234
# We need one job that run every time (without any `only:` limitation).
# This works around a Gitlab bug: if no job runs at all due to
235
236
237
238
239
# `only`, Gitlab gets confused and doesn't allow you to merge the MR:
# https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html#limitations
gitlab-merge-workaround:
  stage: cleanup
  script:
Ana Aviles's avatar
Ana Aviles committed
240
    - echo "That went well"