diff --git a/.gitignore b/.gitignore index 26c584815647e31869c8bbd432944f2911828a8d..625cf3967d28477f527e0a06f30bf5d3ff87201a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,11 +4,16 @@ # Ignore files created by ansible-playbook *.retry +/ansible/secrets/ +ansible/rke.log # Ignore files created during CI using test/ci-bootstrap.py /test/group_vars/ /test/secrets/ /test/inventory.yml +# Ignore files created during tests +/test/behave/**/screenshots/ + # Etc __pycache__ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1815dc8de5cb6802830bc15e318adec9dee608fc..ea45b4d9477f3e6ab57bc104f8d4fdfe8e59e29a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,26 @@ +stages: + - build + - deploy + - test + - cleanup + +ci_test_image: + stage: build + variables: + DOCKER_DRIVER: overlay2 + image: docker:stable + services: + - docker:dind + before_script: + - docker info + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker build -t ${CI_REGISTRY_IMAGE}/bootstrap-ci test/ + - docker push ${CI_REGISTRY_IMAGE}/bootstrap-ci + only: + changes: + - test/Dockerfile + control_image: stage: build variables: @@ -9,31 +32,56 @@ control_image: - docker info script: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - docker build -t openappstack/bootstrap/oas-control control/ - - docker tag openappstack/bootstrap/oas-control docker.greenhost.net/openappstack/bootstrap/oas-control - - docker push docker.greenhost.net/openappstack/bootstrap/oas-control + - docker build -t ${CI_REGISTRY_IMAGE}/oas-control control/ + - docker push ${CI_REGISTRY_IMAGE}/oas-control only: changes: - control/**/* bootstrap: stage: deploy - image: alpine + image: docker.greenhost.net/openappstack/bootstrap/bootstrap-ci script: - - apk update - - apk add ansible musl-dev linux-headers gcc py3-psutil openssh-client - - pip3 install requests tabulate testinfra # Ensure test/ is not world-writable otherwise ansible-playbook refuses to run, see # https://docs.ansible.com/ansible/devel/reference_appendices/config.html#cfg-in-world-writable-dir - chmod 755 test/ - cd test/ - eval $(ssh-agent -s) - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - # - mkdir -p --mode 700 ~/.ssh - ANSIBLE_HOST_KEY_CHECKING=False python3 -u ./ci-bootstrap.py --create_droplet - - py.test -v --ansible-inventory=./inventory.yml --connection=ssh --hosts='ansible://*' - - python3 -c "import cosmos; cosmos.terminate_droplets_by_name(\"^ci-${CI_JOB_ID}\$\")" + # Wait for proper LE cert to get served + - timeout -t 1200 sh -c 'while ! curl --cacert ./letsencrypt_staging_bundle.pem -s https://auth.ci-${CI_JOB_ID}.ci.openappstack.net/auth/ > /dev/null; do date; echo "Waiting for LE cert..."; sleep 5; done' artifacts: paths: - ansible/rke.log + - test/behave/behave.ini + expire_in: 1 month + when: always + +testinfra: + stage: test + image: docker.greenhost.net/openappstack/bootstrap/bootstrap-ci + script: + - cd test/ + - py.test -v --ansible-inventory=./inventory.yml --connection=ssh --hosts='ansible://*' + +behave: + stage: test + image: docker.greenhost.net/openappstack/bootstrap/bootstrap-ci + script: + # Run behave tests + - cd test/behave/ + - behave + artifacts: + paths: + - test/behave/screenshots/ expire_in: 1 month + when: always + +terminate: + stage: cleanup + image: docker.greenhost.net/openappstack/bootstrap/bootstrap-ci + script: + # Remove droplet after successful tests + - cd test/ + - python3 -c "import cosmos; cosmos.terminate_droplets_by_name(\"^ci-${CI_JOB_ID}\$\")" diff --git a/control/files/bin/control b/control/files/bin/control index 842fac36934bfea66bf50682d8a15fd5d00cacf4..652228b175763860318abd0d47e767830e46b4b8 100755 --- a/control/files/bin/control +++ b/control/files/bin/control @@ -56,7 +56,7 @@ getRepos() configureKeycloak() { - kubectl create secret generic realm-secret "--from-file=/control/k8s-config/realm.json" + kubectl create secret generic realm-secret "--from-file=/control/k8s-config/realm.json" --dry-run -o yaml | kubectl apply -f - } configFiles() diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..302ecf6984b1b053d0a4c64915d7281f63882d7a --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,22 @@ +FROM alpine:3.9 + +LABEL Name="Openappstack bootstrap CI test image" +LABEL version="3.9" +LABEL vendor1="Greenhost" + +RUN apk --no-cache add \ + ansible \ + curl \ + musl-dev \ + linux-headers \ + gcc \ + py3-psutil \ + py3-requests \ + openssh-client \ + chromium \ + chromium-chromedriver + +# p3-tabulate is not installable via pip3, +# see https://code.greenhost.net/openappstack/bootstrap/issues/54 +# There is no alpine package for testinfra and behave(-webdriver) +RUN pip3 install behave-webdriver tabulate testinfra diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000000000000000000000000000000000000..31069677b2026a7c79b40a03ac49fe2e9ea19806 --- /dev/null +++ b/test/README.md @@ -0,0 +1,9 @@ +# Run behave tests in bootstrap-ci docker image + + docker run --rm -it docker.greenhost.net/openappstack/bootstrap/bootstrap-ci sh + + apk --no-cache add git + git clone https://code.greenhost.net/openappstack/bootstrap.git + cd bootstrap/test/behave + behave -D keycloak.admin.url=https://auth.ci-20410.ci.openappstack.net/auth/admin/master/console/ \ + -D keycloak.admin.password=… diff --git a/test/behave/behave.ini b/test/behave/behave.ini new file mode 100644 index 0000000000000000000000000000000000000000..ea68e693f6011d6746dda6550c6551c2da812fac --- /dev/null +++ b/test/behave/behave.ini @@ -0,0 +1,13 @@ +[behave] +# Stop after first failure, see +# https://behave.readthedocs.io/en/latest/behave.html?highlight=--stop#command-line-arguments +stop=true + +[behave.userdata] +# url and password will differ for each cluster +# so we need to pass them on the command line like: +# +# behave -D keycloak.admin.url=https://auth.ci-20397.ci.openappstack.net/auth/admin/master/console/ \ +# -D keycloak.admin.password=… +# +keycloak.admin.username=keycloak diff --git a/test/behave/features/environment.py b/test/behave/features/environment.py new file mode 100644 index 0000000000000000000000000000000000000000..5b2268fdc0a8e5b7708c351893a7da6653ae6727 --- /dev/null +++ b/test/behave/features/environment.py @@ -0,0 +1,47 @@ +"""Basic setup for behave and chromedriver.""" +import behave_webdriver +import os +import re +import time +from behave_webdriver.driver import ChromeOptions + + +def save_screenshot(context, step): + """Save a screenshot to ./screenshots.""" + timestamp = time.strftime("%Y-%m-%dT%H:%M:%S") + filename = re.sub('\W', '-', '{} failed {}'.format(timestamp, + str(step.name))) + filepath = os.path.join('screenshots', filename + '.png') + if not os.path.exists('screenshots'): + os.mkdir('screenshots') + + print('Saving screenshot to %s' % filepath) + context.behave_driver.save_screenshot(filepath) + + +def before_all(context): + """Run at the very beginning.""" + userdata = context.config.userdata + context.keycloak = {} + context.keycloak['admin'] = {} + context.keycloak['admin']['url'] = userdata.get('keycloak.admin.url') + context.keycloak['admin']['username'] = userdata.get('keycloak.admin.username') + context.keycloak['admin']['password'] = userdata.get('keycloak.admin.password') + + chrome_options = ChromeOptions() + chrome_options.add_argument('--headless') + chrome_options.add_argument('--no-sandbox') + chrome_options.add_argument('--disable-dev-shm-usage') + context.behave_driver = behave_webdriver.Chrome( + chrome_options=chrome_options) + + +def after_all(context): + """Cleanup after tests run.""" + context.behave_driver.quit() + + +def after_step(context, step): + """Save screeshot if step fails.""" + if step.status == 'failed': + save_screenshot(context, step) diff --git a/test/behave/features/keycloak.feature b/test/behave/features/keycloak.feature new file mode 100644 index 0000000000000000000000000000000000000000..845f0cf472928de3604ffd9a40cd7ca0f6ad7803 --- /dev/null +++ b/test/behave/features/keycloak.feature @@ -0,0 +1,16 @@ +Feature: Test keycloak admin login + As an OAS admin + I want to be able to login to the keycloak admin console + +Scenario: Open keycloak admin console + Given the title is not "Log in to Keycloak" + When I open the keycloak admin console url + Then I wait on element "input#username" for 25000ms to be visible + And I expect that the title is "Log in to Keycloak" + +Scenario: Login to keycloak + Given the title is "Log in to Keycloak" + When I enter the "keycloak" "admin" "username" in the inputfield "input#username" + When I enter the "keycloak" "admin" "password" in the inputfield "input#password" + And I click on the button "input#kc-login" + Then I wait on element "input#displayName" for 25000ms to be visible diff --git a/test/behave/features/steps/login.py b/test/behave/features/steps/login.py new file mode 100644 index 0000000000000000000000000000000000000000..a8d1ee722649a6810766b6e897bb3507447d7c3f --- /dev/null +++ b/test/behave/features/steps/login.py @@ -0,0 +1,22 @@ +"""Custom steps for login tests.""" + +from behave import given, when +from behave_webdriver.steps import * + + +@when(u'I open the keycloak admin console url') +@given(u'I open the keycloak admin console url') +def step_impl(context): + """Login to the keycloak admin console.""" + context.behave_driver.get(context.keycloak['admin']['url']) + + +@when(u'I enter the "{section}" "{user}" "{cred_type}" in the inputfield "{element}"') +def step_impl(context, section, user, cred_type, element): + """Enter username/password into login inputfields.""" + elem = context.behave_driver.get_element(element) + elem.clear() + + context_section = getattr(context, section) + value = context_section[user][cred_type] + elem.send_keys(value) diff --git a/test/ci-bootstrap.py b/test/ci-bootstrap.py index 2372188f2c1e4620c01355f92450d03631468479..cb36d10198668a3f83b2ce444da0762870e47cad 100755 --- a/test/ci-bootstrap.py +++ b/test/ci-bootstrap.py @@ -2,25 +2,33 @@ r""" Used by CI to bootstrap a new cluster and run tests. -Prerequisites -- Ansible > 2.2 (at least stretch-backports needed) -- External python3 libraries: - - ansible-runner - - requests - - tabulate - - psutil - Env vars needed: - COSMOS_API_TOKEN -In Debian: +Install requirements: + +- Alpine using `requirements.txt`: + + apk --no-cache add python3-dev build-base libffi-dev linux-headers \ + openssl-dev openssh-client + pip3 install -r requirements.txt + +- Apline using packages (much faster): + + apk --no-cache add ansible musl-dev linux-headers gcc py3-psutil \ + openssh-client + pip3 install requests tabulate testinfra + + +- Debian (using deb packages): apt-get install -y --no-install-recommends ansible gcc libc6-dev \ python3-distutils python3-pip python3-setuptools python3-wheel \ python3-psutil - pip3 install ansible-runner requests tabulate + pip3 install requests tabulate testinfra """ import argparse +import configparser import cosmos import logging import os @@ -190,7 +198,7 @@ if __name__ == "__main__": # Bootstrap # playbook path here is relative to private_data_dir/project, see # https://ansible-runner.readthedocs.io/en/latest/intro.html#inputdir - playbook='./bootstrap.yml' + playbook = './bootstrap.yml' ansible_playbook_cmd = 'ansible-playbook %s' % playbook log.info('Running %s', ansible_playbook_cmd) @@ -205,5 +213,19 @@ if __name__ == "__main__": traceback.print_exc() sys.exit(result.returncode) + # Write behave config file for later use + with open('./secrets/keycloak_admin_password', 'r') as stream: + keycloak_admin_password = yaml.load(stream) + behave_config = configparser.ConfigParser() + behave_config['behave'] = {'stop': True} + behave_config['behave.userdata'] = {} + behave_config['behave.userdata']['keycloak.admin.url'] = \ + 'https://auth.{}/auth/admin/master/console/'.format(settings['domain']) + behave_config['behave.userdata']['keycloak.admin.username'] = 'keycloak' + behave_config['behave.userdata']['keycloak.admin.password'] = \ + keycloak_admin_password + with open('./behave/behave.ini', 'w') as configfile: + behave_config.write(configfile) + if args.terminate: cosmos.terminate_droplet(id) diff --git a/test/letsencrypt_staging_bundle.pem b/test/letsencrypt_staging_bundle.pem new file mode 100644 index 0000000000000000000000000000000000000000..5f5342293f258c4cd2ff216f00c371b70e4bcf3b --- /dev/null +++ b/test/letsencrypt_staging_bundle.pem @@ -0,0 +1,56 @@ +-----BEGIN CERTIFICATE----- +MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw +GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2 +MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0 +8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym +oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0 +ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN +xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56 +dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9 +AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw +HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0 +BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu +b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu +Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq +hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF +UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9 +AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp +DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7 +IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf +zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI +PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w +SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em +2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0 +WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt +n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFATCCAumgAwIBAgIRAKc9ZKBASymy5TLOEp57N98wDQYJKoZIhvcNAQELBQAw +GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDMyMzIyNTM0NloXDTM2 +MDMyMzIyNTM0NlowGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMIICIjANBgkq +hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA+pYHvQw5iU3v2b3iNuYNKYgsWD6KU7aJ +diddtZQxSWYzUI3U0I1UsRPTxnhTifs/M9NW4ZlV13ZfB7APwC8oqKOIiwo7IwlP +xg0VKgyz+kT8RJfYr66PPIYP0fpTeu42LpMJ+CKo9sbpgVNDZN2z/qiXrRNX/VtG +TkPV7a44fZ5bHHVruAxvDnylpQxJobtCBWlJSsbIRGFHMc2z88eUz9NmIOWUKGGj +EmP76x8OfRHpIpuxRSCjn0+i9+hR2siIOpcMOGd+40uVJxbRRP5ZXnUFa2fF5FWd +O0u0RPI8HON0ovhrwPJY+4eWKkQzyC611oLPYGQ4EbifRsTsCxUZqyUuStGyp8oa +aoSKfF6X0+KzGgwwnrjRTUpIl19A92KR0Noo6h622OX+4sZiO/JQdkuX5w/HupK0 +A0M0WSMCvU6GOhjGotmh2VTEJwHHY4+TUk0iQYRtv1crONklyZoAQPD76hCrC8Cr +IbgsZLfTMC8TWUoMbyUDgvgYkHKMoPm0VGVVuwpRKJxv7+2wXO+pivrrUl2Q9fPe +Kk055nJLMV9yPUdig8othUKrRfSxli946AEV1eEOhxddfEwBE3Lt2xn0hhiIedbb +Ftf/5kEWFZkXyUmMJK8Ra76Kus2ABueUVEcZ48hrRr1Hf1N9n59VbTUaXgeiZA50 +qXf2bymE6F8CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB +Af8wHQYDVR0OBBYEFMEmdKSKRKDm+iAo2FwjmkWIGHngMA0GCSqGSIb3DQEBCwUA +A4ICAQBCPw74M9X/Xx04K1VAES3ypgQYH5bf9FXVDrwhRFSVckria/7dMzoF5wln +uq9NGsjkkkDg17AohcQdr8alH4LvPdxpKr3BjpvEcmbqF8xH+MbbeUEnmbSfLI8H +sefuhXF9AF/9iYvpVNC8FmJ0OhiVv13VgMQw0CRKkbtjZBf8xaEhq/YqxWVsgOjm +dm5CAQ2X0aX7502x8wYRgMnZhA5goC1zVWBVAi8yhhmlhhoDUfg17cXkmaJC5pDd +oenZ9NVhW8eDb03MFCrWNvIh89DDeCGWuWfDltDq0n3owyL0IeSn7RfpSclpxVmV +/53jkYjwIgxIG7Gsv0LKMbsf6QdBcTjhvfZyMIpBRkTe3zuHd2feKzY9lEkbRvRQ +zbh4Ps5YBnG6CKJPTbe2hfi3nhnw/MyEmF3zb0hzvLWNrR9XW3ibb2oL3424XOwc +VjrTSCLzO9Rv6s5wi03qoWvKAQQAElqTYRHhynJ3w6wuvKYF5zcZF3MDnrVGLbh1 +Q9ePRFBCiXOQ6wPLoUhrrbZ8LpFUFYDXHMtYM7P9sc9IAWoONXREJaO08zgFtMp4 +8iyIYUyQAbsvx8oD2M8kRvrIRSrRJSl6L957b4AFiLIQ/GgV2curs0jje7Edx34c +idWw1VrejtwclobqNMVtG3EiPUIpJGpbMcJgbiLSmKkrvQtGng== +-----END CERTIFICATE----- diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4d492ae34ef747ec960b78f60ea5ba8921ed4bd9 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,7 @@ +ansible>2.2 +behave-webdriver +psutil +requests +tabulate +setuptools +wheel