Resolve "Add login panel CI/CD to create (docker) image"

Closes #103 (closed)

.gitlab-ci.yml

@@ -3,146 +3,229 @@ include:
 
 stages:
   - build
-  - build-test-images
-  # TODO: Re-enable after fixing #82
-  # - lint
-  - application-test
+  - lint
+  # TODO: Re-enable after fixing #73
+  # - application-test
   - integration-test
 
-integration_test_app:
-  stage: build-test-images
+#login_test:
+#  image: python:3.8
+#  stage: unittest
+#  needs: []
+#  variables:
+#    HYDRA_ADMIN_URL: http://localhost/ignored
+#    KRATOS_PUBLIC_URL: http://localhost/ignored
+#    PUBLIC_URL: http://localhost/ignored
+#  cache:
+#    paths:
+#      - "$CI_PROJECT_DIR/pip-cache"
+#    key: "$CI_PROJECT_ID"
+#    before_script:
+#      - cd login
+#      - python -V
+#      - pip install -r requirements.txt
+#    script:
+#      - pytest -v --cov=login --cov-report=term --cov-report=xml tests
+#    artifacts:
+#      reports:
+#        cobertura: login/coverage.xml
+#    coverage: '/^TOTAL.+?(\d+\%)$/'
+
+
+
+# Stackspin login panel
+login:
+  stage: build
+  needs: []
+  extends: .kaniko_build
+  variables:
+    KANIKO_CONTEXT: "login"
+    KANIKO_BUILD_IMAGENAME: $CI_JOB_NAME
+  before_script:
+    - ls -l ${CI_PROJECT_DIR}/${KANIKO_CONTEXT:-.}/Dockerfile 
+  only:
+    changes:
+      - login/**/*
+
+
+# For full integration testing we need to build the full stack in CI/CD
+#
+# - postgres    : Backend database, the Dockerfile is based on the upstream
+#                 with a small addition to create inital databases
+# - kratos      : Identity manager, based on the upstream. We set the DSN
+#                 in the Dockerimage, as its different from hydra's DSN, 
+#                 however, gitlab-ci will apply the same enviroment for all
+#                 services.
+# - hydra       : See above, the identiy provider
+# - login       : Our login panel
+# - sso_testapp : A test app which implements OIDC for testing
+# - behave      : Image to do behave testing
+
+# Build our own postgres image, which is based on upstream
+postgres:
+  stage: build
+  needs: []
   variables:
-    KANIKO_CONTEXT: "test/integration_tests"
+    KANIKO_CONTEXT: ".gitlab/ci/postgres"
     KANIKO_BUILD_IMAGENAME: $CI_JOB_NAME
   extends: .kaniko_build
   only:
     changes:
-      - test/integration_tests/**/*
-      - .gitlab-ci.yml
+      - .gitlab/ci/postgres/**/*
+
 
-integration_test:
-  stage: build-test-images
+# Build our own kratos image, which is based on upstream
+kratos:
+  stage: build
+  needs: []
   variables:
-    KANIKO_CONTEXT: "test/integration_tests/test"
+    KANIKO_CONTEXT: ".gitlab/ci/kratos"
     KANIKO_BUILD_IMAGENAME: $CI_JOB_NAME
   extends: .kaniko_build
   only:
     changes:
-      - test/integration_tests/test/**/*
-      - test/integration_tests/test/Dockerfile
-      - .gitlab-ci.yml
+      - .gitlab/ci/kratos/**/*
 
-pylint:
-  stage: build-test-images
+# Build our own hydra image, which is based on upstream
+hydra:
+  stage: build
+  needs: []
   variables:
-    KANIKO_CONTEXT: "test/lint/pylint"
+    KANIKO_CONTEXT: ".gitlab/ci/hydra"
     KANIKO_BUILD_IMAGENAME: $CI_JOB_NAME
   extends: .kaniko_build
   only:
     changes:
-      - test/lint/pylint/Dockerfile
-      - test/lint/pylint/requirements.txt
-      - .gitlab-ci.yml
+      - .gitlab/ci/hydra/**/*
+
+# A Fake SSO app to test the behaviour
+sso_testapp:
+  stage: build
+  variables:
+    KANIKO_CONTEXT: "test/sso_testapp/"
+    KANIKO_BUILD_IMAGENAME: $CI_JOB_NAME
+  extends: .kaniko_build
+  only:
+    changes:
+      - test/sso_testapp/**/*
+
+# Image to test behaviour (web)
+behave:
+  stage: build
+  variables:
+    KANIKO_CONTEXT: "test/behave"
+    KANIKO_BUILD_IMAGENAME: $CI_JOB_NAME
+  extends: .kaniko_build
+  only:
+    changes:
+      - test/behave/**/*
 
 
 
-# TODO: This integration testing is the old test. This need to be modified
-#       and will happen in the issue which is about integration the login 
-#       image & testing that
 behave-integration:
   stage: integration-test
   services:
-    - name: postgres:latest
+    - name: ${CI_REGISTRY_IMAGE}/postgres:${CI_COMMIT_REF_NAME}
       alias: postgres
-    - name: docker.io/oryd/hydra:v1.8
+    - name: ${CI_REGISTRY_IMAGE}/kratos:${CI_COMMIT_REF_NAME}
+      alias: kratosmigrate
+      command:
+        - migrate
+        - sql
+        - -e
+        - -y
+    - name: ${CI_REGISTRY_IMAGE}/kratos:${CI_COMMIT_REF_NAME}
+      alias: kratos
+      command:
+        - serve
+        - --config
+        - /etc/config/kratos.yaml
+    - name: ${CI_REGISTRY_IMAGE}/hydra:${CI_COMMIT_REF_NAME}
+      alias: hydramigrate
+      command:
+        - migrate
+        - sql
+        - -e
+        - -y
+    - name: ${CI_REGISTRY_IMAGE}/hydra:${CI_COMMIT_REF_NAME}
       alias: hydra
       command:
         - serve
         - all
         - --dangerous-force-http
         - --dangerous-allow-insecure-redirect-urls
-        - "http://oidc:5000/callback"
-    - name: ${CI_REGISTRY_IMAGE}/integration_test_app:${CI_COMMIT_REF_NAME}
+        - http://oidc:5000/login
+    - name: ${CI_REGISTRY_IMAGE}/login:${CI_COMMIT_REF_NAME}
       alias: oidc
+    - name: ${CI_REGISTRY_IMAGE}/sso_testapp:${CI_COMMIT_REF_NAME}
+      alias: ssoapp
   variables:
     # Feature Flag FF_NETWORK_PER_BUILD Enables creation of a docker network per build
     # with the docker executor of the gitlab-runner. This is required for service
     # interconnection. Requires gitlab-runner v12.9.0
     FF_NETWORK_PER_BUILD: 1
-    DATABASE_HOST: "postgres"
-    URLS_LOGIN: "http://login:5000/login"
-    URLS_LOGOUT: "http://logout:5002/logout"
-    URLS_CONSENT: "http://consent:5001/consent"
-    URLS_SELF_ISSUER: "http://hydra:4444/"
-    BASE_URL: "http://hydra:4444/"
-    HYDRA_ADMIN_URL: "http://hydra:4445"
-    ACCESS_TOKEN_URL: "http://hydra:4444/oauth2/token"
-    AUTHORIZE_URL: "http://hydra:4444/oauth2/auth"
-    USERINFO_URL: "http://hydra:4444/userinfo"
-    GRAPHQL_URL: "http://backend:5000/graphql"
-    GIT_SUBMODULE_STRATEGY: "recursive"
-    TESTUSER_USERNAME: "testuser"
-    TESTUSER_USERNAME2: "testuser2"
-    TESTUSER_PASSWORD: "password"
-    TESTUSER_EMAIL: "testuser@example.net"
-    TESTUSER_EMAIL2: "testuser2@example.net"
-    ROLE: "admin"
-    DSN: "memory"
-    SECRETS_SYSTEM: "youReallyNeedToChangeThis"
-    DATABASE_USER: postgres
-    DATABASE_PASSWORD: secret
-    DATABASE_NAME: postgres
-    POSTGRES_PASSWORD: secret
+    OAUTHLIB_INSECURE_TRANSPORT: "true"
+    # For hydra
+    URLS_SELF_ISSUER: http://hydra:4445/
+    URLS_CONSENT: http://oidc:5000/login
+    URLS_LOGIN: http://oidc:5000/consent
+    SECRETS_SYSTEM: RandomSecretForTesting
+    # For postgres image super user credentials, additional credentials are
+    # created from `.gitlab/ci/postgres/setup.sql` during initialization
+    POSTGRES_PASSWORD: postgres
     POSTGRES_USER: postgres
     POSTGRES_DB: postgres
-    OAUTHLIB_INSECURE_TRANSPORT: "true"
-    KEY: "testapp"
-    SECRET: "secret"
+    # For Login image
+    FLASK_RUN_HOST: "0.0.0.0"
+    FLASK_RUN_PORT: "5000"
+    PUBLIC_URL: "http://localhost:5000/"
+    HYDRA_ADMIN_URL: "http://hydra:4445"
+    KRATOS_PUBLIC_URL: "http://kratos:4433"
+    KRATOS_ADMIN_URL: "http://kratos:4434"
+    DATABASE_URL: "postgresql://stackspin:stackspin@localhost/stackspin"
+    APP_SETTINGS: "config.DevelopmentConfig"
+    # General flask
     DEBUG: "true"
     FLASK_ENV: "development"
-  image: ${CI_REGISTRY_IMAGE}/integration_test:${CI_COMMIT_REF_NAME}
+    # For sso test app
+    BASE_URL: "http://hydra:4444/"
+    KEY: "testapp"
+    SECRET: "secret"
+  image: ${CI_REGISTRY_IMAGE}/behave:${CI_COMMIT_REF_NAME}
   script:
-    # TODO: Fixed by https://open.greenhost.net/stackspin/single-sign-on/-/issues/103
-    - echo "The panel code is not included in CI/CD build. It can not do testing"
-    # Create user and application objects
-    #- /bin/bash user-panel/backend/utils/create-user.bash ${TESTUSER_USERNAME} ${TESTUSER_PASSWORD} ${TESTUSER_EMAIL} backend:5000
-    #- /bin/bash user-panel/backend/utils/create-user.bash ${TESTUSER_USERNAME2} ${TESTUSER_PASSWORD} ${TESTUSER_EMAIL2} backend:5000
-    #- /bin/bash user-panel/backend/utils/create-application.bash ${KEY} "Application used for testing" backend:5000
-    #- /bin/bash user-panel/backend/utils/create-role.bash ${ROLE} backend:5000
-    #- /bin/bash user-panel/backend/utils/grant-access.bash ${TESTUSER_USERNAME} ${KEY} backend:5000
-    #- /bin/bash user-panel/backend/utils/assign-role.bash ${TESTUSER_USERNAME} ${ROLE} backend:5000
-    # Wait for 60s for hydra to become available. Then create the oauth2 client object
-    #- while [[ $HYDRAADMINSTATUS -ne "200" &&  60 -ge $TIMER ]]; do HYDRAADMINSTATUS=`curl http://hydra:4445/health/ready -o /dev/null -w "%{http_code}"` || TIMER=$TIMER+5 && sleep 5 ; done
-    #- /bin/bash test/create-hydra-client.bash ${KEY} ${SECRET} http://hydra:4445 http://oidc:5000/callback http://oidc:5000/ http://oidc:5000/logout
-    #- cd test/integration_tests/test/behave/
-    #- TIMER=0
-    #- while [[ $HYDRASTATUS -ne "200" &&  60 -ge $TIMER ]]; do HYDRASTATUS=`curl http://hydra:4444/health/ready -o /dev/null -w "%{http_code}"` || TIMER=$TIMER+5 && sleep 5 ; done
-    #- >
-    #    python3 -m behave
-    #    -D headless=True
-    #    -D url=http://oidc:5000
-    #    -D username=${TESTUSER_USERNAME}
-    #    -D username2=${TESTUSER_USERNAME2}
-    #    -D password=${TESTUSER_PASSWORD}
-    #    -D email=${TESTUSER_EMAIL}
-    #    -D role=${ROLE}
+    - curl -s http://hydra:4445/health/alive
+    - curl -s http://kratos:4433/health/alive
+    - curl -s http://oidc:5000/status
+    # Steps to do:
+    # - create user & access roles & grant access
+    # - add client applition ID + key for testing
+    # - test login etc
+
   artifacts:
     paths:
       - test/integration_tests/test/behave/screenshots/
     expire_in: 1 month
     when: on_failure
 
-# TODO: Fix https://open.greenhost.net/stackspin/single-sign-on/-/issues/82
-#
-# pylint-lint:
-#   stage: lint
-#   variables:
-#     GIT_AUTHOR_NAME: "RUNNER"
-#     GIT_AUTHOR_EMAIL: "runner@greenhost.net"
-#     PYLINT_PLUGINS: "pylint_flask pylint_flask_sqlalchemy"
-#   image: ${CI_REGISTRY_IMAGE}/pylint:${CI_COMMIT_REF_NAME}
-#   script:
-#     - echo "Reverting back to main to squash commits"
-#     - git reset --soft main
-#     - git lint
 
+pylint:
+  stage: build
+  variables:
+    KANIKO_CONTEXT: "test/lint/pylint"
+    KANIKO_BUILD_IMAGENAME: $CI_JOB_NAME
+  extends: .kaniko_build
+  only:
+    changes:
+      - test/lint/pylint/Dockerfile
+      - test/lint/pylint/requirements.txt
+      - .gitlab-ci.yml
+
+pylint-lint:
+  stage: lint
+  variables:
+    GIT_STRATEGY: clone
+    GIT_DEPTH: 0
+    PYLINT_PLUGINS: "pylint_flask pylint_flask_sqlalchemy"
+  image: ${CI_REGISTRY_IMAGE}/pylint:${CI_COMMIT_REF_NAME}
+  script:
+    - darker --check -i -L pylint --diff --revision remotes/origin/main .