diff --git a/.gitignore b/.gitignore index 93ea8ac213d062495152fb6a1ea52f02a1977fb1..8d3754031d39003b6015869e58cbe3ad1520eb9b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ /ansible/secrets/ ansible/rke.log +# Virtualenvs for the project +/env +/venv + # Ignore files created during CI using test/ci-bootstrap.py /test/group_vars/ /test/secrets/ @@ -18,3 +22,4 @@ ansible/rke.log # Etc __pycache__ +*.swp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a46c63bddca73c1b6887098775233cac50a329d1..cbc618cfb09759db922be2c70e380d7b3207a506 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ stages: - build - - deploy + - setup-cluster + - install-apps - test - e2e-test - cleanup @@ -24,7 +25,7 @@ ci_test_image: - test/requirements.txt bootstrap: - stage: deploy + stage: setup-cluster image: "${CI_REGISTRY_IMAGE}/bootstrap-ci" script: # Ensure test/ is not world-writable otherwise ansible-playbook refuses to run, see @@ -33,18 +34,33 @@ bootstrap: - cd test/ - eval $(ssh-agent -s) - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - - ANSIBLE_HOST_KEY_CHECKING=False python3 -u ./ci-bootstrap.py --create_droplet + - ANSIBLE_HOST_KEY_CHECKING=False python3 -u ./ci-bootstrap.py --create-droplet --create-domain-records --run-ansible --ansible-param skip-tags=helmfile cache: key: "$CI_PIPELINE_ID" paths: - - test/behave/behave.ini - test/inventory.yml + - test/group_vars/all.yml artifacts: paths: - ansible/rke.log expire_in: 1 month when: always +install: + stage: install-apps + image: "${CI_REGISTRY_IMAGE}/bootstrap-ci" + cache: + key: "$CI_PIPELINE_ID" + paths: + - test/inventory.yml + - test/group_vars/all.yml + - test/behave/behave.ini + script: + - cd test/ + - eval $(ssh-agent -s) + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null + - ANSIBLE_HOST_KEY_CHECKING=False python3 -u ./ci-bootstrap.py --use-existing-inventory --run-ansible --ansible-param tags=helmfile --write-behave-config + testinfra: stage: test image: "${CI_REGISTRY_IMAGE}/bootstrap-ci" @@ -88,7 +104,13 @@ behave: terminate: stage: cleanup image: "${CI_REGISTRY_IMAGE}/bootstrap-ci" + cache: + key: "$CI_PIPELINE_ID" + paths: + - test/inventory.yml + policy: pull script: # Remove droplet after successful tests - cd test/ - - python3 -c "import cosmos; cosmos.terminate_droplets_by_name(\"^ci-${CI_PIPELINE_ID}\$\")" + # - python3 -c "import cosmos; cosmos.terminate_droplets_by_name(\"^ci-${CI_PIPELINE_ID}\$\")" + - python3 -u ./ci-bootstrap.py --use-existing-inventory --terminate diff --git a/ansible/group_vars/cluster/settings.yml.example b/ansible/group_vars/cluster/settings.yml.example index f2133a3e3beca97fef9d5339f216bd2b633c7469..b46ea3744a5cc28983ddceb333ce58854e671b03 100644 --- a/ansible/group_vars/cluster/settings.yml.example +++ b/ansible/group_vars/cluster/settings.yml.example @@ -11,10 +11,10 @@ release_name: "test" # Keycloak administrator password. If you do not change this value, it gets # generated and stored in ./secrets/keycloak_admin_password. You can also choose # your own password and fill it in here instead. -keycloak_password: "{{ lookup('password', './secrets/keycloak_admin_password') }}" +keycloak_password: "{{ lookup('password', './secrets/keycloak_admin_password chars=ascii_letters') }}" # Nextcloud administrator password. Works the same as keycloak password, except # it is stored in `secrets/nextcloud_admin_password`. -nextcloud_password: "{{ lookup('password', './secrets/nextcloud_admin_password') }}" +nextcloud_password: "{{ lookup('password', './secrets/nextcloud_admin_password chars=ascii_letters') }}" # If this is "true" TLS certificates will be requested at the Let's Encrypt # staging server. If this is "false", you use Let's Encrypt's production server. # Note that LE's production server has stricter rate limits, so set this to diff --git a/test/ci-bootstrap.py b/test/ci-bootstrap.py index 858533c774b5f7e5e1fdd1f50d23839688c956b1..c686bf79b22d07f9ae7c2d430261b9eacaa1362f 100755 --- a/test/ci-bootstrap.py +++ b/test/ci-bootstrap.py @@ -29,7 +29,6 @@ Install requirements: import argparse import configparser -import cosmos import logging import os import random @@ -40,28 +39,14 @@ import sys import traceback import yaml +import cosmos -def init_logging(log, loglevel): - """ - Configure logging. - - - debug and info go to stdout - - warning and above go to stderr - """ - log.setLevel(loglevel) - stdout = logging.StreamHandler(sys.stdout) - stdout.setLevel(loglevel) - stdout.addFilter(lambda record: record.levelno <= logging.INFO) - - stderr = logging.StreamHandler() - stderr.setLevel(logging.WARNING) - - log.addHandler(stdout) - log.addHandler(stderr) - +SETTINGS_FILE = './group_vars/all.yml' +ANSIBLE_INVENTORY = './inventory.yml' -if __name__ == "__main__": +def main(): # pylint: disable=too-many-statements,too-many-branches + """ does everything """ # Parse command line arguments parser = argparse.ArgumentParser( description='Run bootstrap script' @@ -69,17 +54,22 @@ if __name__ == "__main__": group = parser.add_mutually_exclusive_group(required=True) group.add_argument( - '--create_droplet', + '--create-droplet', action='store_true', help='Create droplet automatically') group.add_argument( - '--droplet_id', + '--droplet-id', metavar='ID', type=int, help='ID of droplet to deploy to') + group.add_argument( + '--use-existing-inventory', + action='store_true', + help='Assumes inventory.yml has already been generated') + parser.add_argument( - '--ssh_key_id', + '--ssh-key-id', metavar='ID', type=int, default=411, @@ -95,114 +85,166 @@ if __name__ == "__main__": action='store_true', help='Be more verbose') + parser.add_argument( + '--ansible-param', + metavar=['PARAM[=VALUE]'], + action='append', + nargs=1, + help=('forward ansible parameters to the ansible-playbook call ' + '(two dashes are prepended to PARAM)')) + parser.add_argument( + '--run-ansible', + action='store_true', + help='Runs the ansible bootstrap process') + + parser.add_argument( + '--create-domain-records', + action='store_true', + help='Creates DNS entries for the cluster') + + parser.add_argument( + '--write-behave-config', + action='store_true', + help='Writes a configuration file for behave with cluster information') + args = parser.parse_args() verbose = args.verbose loglevel = logging.DEBUG if verbose else logging.INFO - - # Setup logging for this script - log = logging.getLogger(__name__) init_logging(log, loglevel) # Setup logging for cosmos module log_cosmos = logging.getLogger('cosmos') init_logging(log_cosmos, loglevel) - # Start bootstrapping - if args.create_droplet: - # Create droplet - - # image: 18 = Ubuntu xenial 18.04 x64 - # ssh_keys - # - 411: ci, ed25519 - # - 407: varac - - if "CI_PIPELINE_ID" in os.environ: - instance_id = os.environ['CI_PIPELINE_ID'] + if not args.use_existing_inventory: + # Start bootstrapping + if args.create_droplet: + # Create droplet + + # image: 18 = Ubuntu xenial 18.04 x64 + # ssh_keys + # - 411: ci, ed25519 + # - 407: varac + + if "CI_PIPELINE_ID" in os.environ: + instance_id = os.environ['CI_PIPELINE_ID'] + else: + # Use random generated ID in case we're not running in gitlab CI + # and there's no CI_PIPELINE_ID env var + instance_id = ''.join( + random.choice(string.ascii_lowercase + string.digits) + for _ in range(10)) + + droplet = cosmos.create_droplet( + name='ci-' + instance_id, + ssh_key_id=args.ssh_key_id, + region='ams1', + size=4096, + disk=8, + image=18) + droplet_id = droplet['droplet']['id'] + log.info('Created droplet id: %s', droplet_id) + cosmos.wait_for_state(droplet_id, 'running') else: - # Use random generated ID in case we're not running in gitlab CI - # and there's no CI_PIPELINE_ID env var - instance_id = ''.join( - random.choice(string.ascii_lowercase + string.digits) - for _ in range(10)) - - droplet = cosmos.create_droplet( - name='ci-' + instance_id, - ssh_key_id=args.ssh_key_id, - region='ams1', - size=4096, - disk=8, - image=18) - id = droplet['droplet']['id'] - log.info('Created droplet id: %s', id) - cosmos.wait_for_state(id, 'running') + droplet_id = args.droplet_id + + if verbose: + cosmos.list_droplets() + + # Get droplet ip + droplet = cosmos.get_droplet(droplet_id) + droplet_ip = droplet['networks']['v4'][0]['ip_address'] + droplet_name = droplet['name'] + + # Create inventory + with open('../ansible/inventory.yml.example', 'r') as stream: + inventory = yaml.safe_load(stream) + inventory['all']['hosts'][droplet_name] = inventory['all']['hosts']['oas-dev'] + del inventory['all']['hosts']['oas-dev'] + + inventory['all']['hosts'][droplet_name]['ansible_host'] = droplet_ip + inventory['all']['hosts'][droplet_name]['ansible_ssh_extra_args'] = \ + '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + inventory['all']['children']['cluster']['hosts'] = droplet_name + inventory['all']['children']['master']['hosts'] = droplet_name + inventory['all']['children']['worker']['hosts'] = droplet_name + + with open(ANSIBLE_INVENTORY, 'w') as stream: + yaml.safe_dump(inventory, stream, default_flow_style=False) + + # Create settings + with open('../ansible/group_vars/cluster/settings.yml.example', + 'r') as stream: + settings = yaml.safe_load(stream) + + settings['ip_address'] = droplet_ip + settings['domain'] = droplet_name + '.ci.openappstack.net' + settings['admin_email'] = "admin@{0}".format(settings['domain']) + settings['acme_staging'] = "true" + + if not os.path.exists('./group_vars'): + os.mkdir('./group_vars') + with open(SETTINGS_FILE, 'w') as stream: + yaml.safe_dump(settings, stream, default_flow_style=False) + + log.debug(yaml.safe_dump(inventory, default_flow_style=False)) + log.debug(yaml.safe_dump(settings, default_flow_style=False)) + + # Wait for ssh + cosmos.wait_for_ssh(droplet_ip) else: - id = args.droplet_id - - if verbose: - cosmos.list_droplets() - - # Get droplet ip - droplet = cosmos.get_droplet(id) - ip = droplet['networks']['v4'][0]['ip_address'] - name = droplet['name'] - - # Create domain records - domain_record = cosmos.create_domain_record( - domain='openappstack.net', name=name + '.ci', data=ip, record_type='A', - update=True) - log.info("Domain record: %s", domain_record) - - domain_record = cosmos.create_domain_record( - domain='openappstack.net', name='*.' + name + '.ci', data=name + '.ci', - record_type='CNAME', update=True) - log.info("Domain record: %s", domain_record) - - if verbose: - cosmos.list_domain_records('openappstack.net') - - # Wait for ssh - cosmos.wait_for_ssh(ip) - - # Create inventory - with open('../ansible/inventory.yml.example', 'r') as stream: - inventory = yaml.load(stream) - inventory['all']['hosts'][name] = inventory['all']['hosts']['oas-dev'] - del inventory['all']['hosts']['oas-dev'] - - inventory['all']['hosts'][name]['ansible_host'] = ip - inventory['all']['hosts'][name]['ansible_ssh_extra_args'] = \ - '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' - inventory['all']['children']['cluster']['hosts'] = name - inventory['all']['children']['master']['hosts'] = name - inventory['all']['children']['worker']['hosts'] = name - - with open('./inventory.yml', 'w') as stream: - yaml.dump(inventory, stream, default_flow_style=False) - - # Create settings - with open('../ansible/group_vars/cluster/settings.yml.example', - 'r') as stream: - settings = yaml.load(stream) - - settings['ip_address'] = ip - settings['domain'] = name + '.ci.openappstack.net' - settings['admin_email'] = "admin@{0}".format(settings['domain']) - settings['acme_staging'] = "true" - - if not os.path.exists('./group_vars'): - os.mkdir('./group_vars') - with open('./group_vars/all', 'w') as stream: - yaml.dump(settings, stream, default_flow_style=False) - - log.debug(yaml.dump(inventory, default_flow_style=False)) - log.debug(yaml.dump(settings, default_flow_style=False)) - - # Bootstrap + # Work with the master node from the inventory + with open(ANSIBLE_INVENTORY, 'r') as stream: + inventory = yaml.safe_load(stream) + droplet_name = inventory['all']['children']['master']['hosts'] + droplet_ip = inventory['all']['hosts'][droplet_name]['ansible_host'] + log.info("Read data from inventory:\n\tname: %s\n\tip: %s", + droplet_name, droplet_ip) + + # For if write_behave_config is called later: + settings = None + + if args.create_domain_records: + # Create domain records + domain_record = cosmos.create_domain_record( + domain='openappstack.net', name=droplet_name + '.ci', data=droplet_ip, record_type='A', + update=True) + log.info("Domain record: %s", domain_record) + + domain_record = cosmos.create_domain_record( + domain='openappstack.net', name='*.' + droplet_name + '.ci', data=droplet_name + '.ci', + record_type='CNAME', update=True) + log.info("Domain record: %s", domain_record) + + if verbose: + cosmos.list_domain_records('openappstack.net') + + + if args.run_ansible: + + run_ansible('./bootstrap.yml', args.ansible_param) + + if args.write_behave_config: + write_behave_config(settings=settings) + + if args.terminate: + cosmos.terminate_droplets_by_name(droplet_name) + +def run_ansible(playbook, ansible_params): + """ Calls `ansible-playbook` directly to run the specified playbook. """ # playbook path here is relative to private_data_dir/project, see # https://ansible-runner.readthedocs.io/en/latest/intro.html#inputdir - playbook = './bootstrap.yml' ansible_playbook_cmd = 'ansible-playbook %s' % playbook + if ansible_params: + for param in ansible_params: + if len(param) > 1: + log.warning('More than 1 parameter. Ignoring the rest! Use ' + '--ansible-param several times to supply ' + 'more than 1 parameter') + param = param[0] + ansible_playbook_cmd += ' --' + param + log.info('Running %s', ansible_playbook_cmd) result = subprocess.run(shlex.split(ansible_playbook_cmd)) @@ -215,9 +257,13 @@ if __name__ == "__main__": traceback.print_exc() sys.exit(result.returncode) - # Write behave config file for later use +def write_behave_config(settings=None): + """ Write behave config file for later use """ + if settings is None: + with open(SETTINGS_FILE) as stream: + settings = yaml.safe_load(stream) with open('./secrets/keycloak_admin_password', 'r') as stream: - keycloak_admin_password = yaml.load(stream) + keycloak_admin_password = yaml.safe_load(stream) behave_config = configparser.ConfigParser() behave_config['behave'] = {'stop': True} behave_config['behave.userdata'] = {} @@ -229,5 +275,26 @@ if __name__ == "__main__": with open('./behave/behave.ini', 'w') as configfile: behave_config.write(configfile) - if args.terminate: - cosmos.terminate_droplet(id) +def init_logging(logger, loglevel): + """ + Configure logging. + + - debug and info go to stdout + - warning and above go to stderr + """ + logger.setLevel(loglevel) + stdout = logging.StreamHandler(sys.stdout) + stdout.setLevel(loglevel) + stdout.addFilter(lambda record: record.levelno <= logging.INFO) + + stderr = logging.StreamHandler() + stderr.setLevel(logging.WARNING) + + logger.addHandler(stdout) + logger.addHandler(stderr) + +if __name__ == "__main__": + # Setup logging for this script + log = logging.getLogger(__name__) # pylint: disable=invalid-name + + main() diff --git a/test/cosmos.py b/test/cosmos.py index 8a45d830687b0b6e54d3fcc20df471f87c7b5251..a4449e60c8a6a573adb3d27925d428c75f95eb2b 100755 --- a/test/cosmos.py +++ b/test/cosmos.py @@ -19,7 +19,7 @@ def request_api(resource: str, request_type: str = 'GET', if 'COSMOS_API_TOKEN' in os.environ: api_token = os.environ['COSMOS_API_TOKEN'] else: - raise ValueError('Please export the COSMOS_API_TOKEN' + raise ValueError('Please export the COSMOS_API_TOKEN ' 'environment variable.') headers = {'Content-Type': 'application/json',