.gitlab-ci.yml

variables:
  CI: "true"
  DOCKER_DRIVER: overlay2
  # Make sure sbt cache are captured completely
  COURSIER_CACHE: $CI_PROJECT_DIR/sbt-cache/coursier
  SBT_OPTS: >
    -Dsbt.global.base=$CI_PROJECT_DIR/sbt-cache/sbtboot
    -Dsbt.boot.directory=$CI_PROJECT_DIR/sbt-cache/boot
    -Dsbt.ivy.home=$CI_PROJECT_DIR/sbt-cache/ivy
  # If you changed dependencies (especially for scala), you might want to prevent your branch from using old cache by increase the version blew
  # gitlab actually use the same technique when you click the `Clear Cache Button`
  CACHE_VERSION: ts-12-scala-7
  CRYPTOGRAPHY_DONT_BUILD_RUST: 1

stages:
  - builders
  - sbt-prebuild
  - prebuild
  - buildtest
  - dockerize
  - preview
  - deploy-dev
  - pre-release
  - release
  - deploy-staging
  - deploy-prod

builders-and-yarn:
  stage: builders
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:master
  retry: 1
  needs: []
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - .yarn
  before_script:
    - yarn install --no-progress --child-concurrency 1 --cache-folder .yarn
  services:
    - docker:dind
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.builder=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master
  artifacts:
    paths:
      - "node_modules"
      - "*/node_modules"
      - "packages/*/node_modules"
      - "yarn.lock"
    expire_in: 7 days

# Make sure sbt depenencies, plugins are in place, cached (only for this job) and pass to following stage as artifacts
sbt-prebuild:
  stage: sbt-prebuild
  needs: []
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:master
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/sbt-cache"
  script:
    - |
      if [ ! -d "$CI_PROJECT_DIR/sbt-cache" ]; then
        cp -R /sbt-cache "$CI_PROJECT_DIR/sbt-cache"
        mkdir -p "$CI_PROJECT_DIR/sbt-cache/coursier"
        cp -R "$CI_PROJECT_DIR/dep-jars/." "$CI_PROJECT_DIR/sbt-cache/coursier/https/"
      fi
    - sbt update
  artifacts:
    paths:
      - "sbt-cache"
      - "target"
      - "project/target"
      - "project/project/target"
      - "*/target"
      - "*/project/target"
      - "*/project/project/target"
    expire_in: 7 days

check-scala-formatting:
  stage: prebuild
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:master
  needs: []
  script:
    - sbt scalafmtCheckAll

registry-typescript-api:
  stage: prebuild
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  needs: 
    - builders-and-yarn
    - sbt-prebuild
  dependencies:
    - builders-and-yarn
    - sbt-prebuild
  script:
    - lerna run generate --scope=@magda/typescript-common --stream
  artifacts:
    paths:
      - "magda-typescript-common/src/generated"
    expire_in: 7 days

buildtest:search-with-index-cache:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:master
  retry: 2
  timeout: 18 minutes
  needs: [
    "sbt-prebuild"
  ]
  dependencies:
    - sbt-prebuild
  before_script:
    - |
      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
        export DOCKER_HOST='tcp://localhost:2375'
      fi
    - docker info
    - apk update
    - apk upgrade
    - apk add python3 python3-dev py3-pip gcc musl-dev build-base libressl-dev libffi-dev libgcc
    - pip3 install --upgrade pip
    - pip3 install wheel
    - pip3 install docker-compose
    - docker-compose --version
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/pip-cache"
  services:
    - docker:dind
  variables:
    # allow openssl 1.02 until we upgrade builder to use node12/alpine3.9 
    CRYPTOGRAPHY_ALLOW_OPENSSL_102: 1
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
    # 2.5gb heap
    # unload classes we don't need
    # aim for < 1000ms gc pauses (largely ignored)
    # use the next-gen collector that every blog on the internet says not to use because we know better than them
    # use < 25% of time for GC
    # put the ivy cache in the workspace where gitlab can cache it
    SBT_OPTS: >
      -Xms2500M -Xmx2500M
      -XX:+CMSClassUnloadingEnabled
      -XX:MaxGCPauseMillis=1000
      -XX:+UseG1GC
      -XX:GCTimeRatio=3
      -Dsbt.global.base=$CI_PROJECT_DIR/sbt-cache/sbtboot
      -Dsbt.boot.directory=$CI_PROJECT_DIR/sbt-cache/boot
      -Dsbt.ivy.home=$CI_PROJECT_DIR/sbt-cache/ivy
  tags:
    - heavy
  script:
    - cd magda-elastic-search
    - docker-compose up -d
    - cd ..
    - sbt -DelasticSearch.serverUrl=elasticsearch://docker:9200 "intTest/testOnly au.csiro.data61.magda.api.*Spec"
    - cd magda-elastic-search
    - docker-compose down
  artifacts:
    paths:
      - "target"
      - "project/target"
      - "project/project/target"
      - "*/target"
      - "*/project/target"
      - "*/project/project/target"
    expire_in: 7 days

buildtest:search-no-index-cache:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:master
  retry: 2
  timeout: 18 minutes
  needs: [
    "sbt-prebuild"
  ]
  dependencies:
    - sbt-prebuild
  before_script:
    - |
      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
        export DOCKER_HOST='tcp://localhost:2375'
      fi
    - docker info
    - apk update
    - apk upgrade
    - apk add python3 python3-dev py3-pip gcc musl-dev build-base libressl-dev libffi-dev libgcc
    - pip3 install --upgrade pip
    - pip3 install wheel
    - pip3 install docker-compose
    - docker-compose --version
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/pip-cache"
  services:
    - postgres:13.3
    - docker:dind
  variables:
    # allow openssl 1.02 until we upgrade builder to use node12/alpine3.9 
    CRYPTOGRAPHY_ALLOW_OPENSSL_102: 1
    POSTGRES_URL: "jdbc:postgresql://postgres/postgres"
    POSTGRES_DB: postgres
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: "password"
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
    # 2.5gb heap
    # unload classes we don't need
    # aim for < 1000ms gc pauses (largely ignored)
    # use the next-gen collector that every blog on the internet says not to use because we know better than them
    # use < 25% of time for GC
    # put the ivy cache in the workspace where gitlab can cache it
    SBT_OPTS: >
      -Xms2500M -Xmx2500M
      -XX:+CMSClassUnloadingEnabled
      -XX:MaxGCPauseMillis=1000
      -XX:+UseG1GC
      -XX:GCTimeRatio=3
      -Dsbt.global.base=$CI_PROJECT_DIR/sbt-cache/sbtboot
      -Dsbt.boot.directory=$CI_PROJECT_DIR/sbt-cache/boot
      -Dsbt.ivy.home=$CI_PROJECT_DIR/sbt-cache/ivy
  tags:
    - heavy
  script:
    - cd magda-elastic-search
    - docker-compose up -d
    - cd ..
    - sbt -DelasticSearch.serverUrl=elasticsearch://docker:9200 "intTest/testOnly au.csiro.data61.magda.crawler.*Spec au.csiro.data61.magda.indexer.*Spec au.csiro.data61.magda.spatial.*Spec"
    - cd magda-elastic-search
    - docker-compose down
  artifacts:
    paths:
      - "target"
      - "project/target"
      - "project/project/target"
      - "*/target"
      - "*/project/target"
      - "*/project/project/target"
    expire_in: 7 days

buildtest:ui:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
  retry: 1
  needs: ["builders-and-yarn"]
  dependencies:
    - builders-and-yarn
  cache:
    paths: []
  variables:
    REACT_APP_SHA1: "$CI_COMMIT_SHA"
  script:
    - yarn install
    - yarn run eslint
    - cd magda-typescript-common && yarn build && cd ..
    - yarn run in-submodules -- -f categories.ui=true -- run build --include-filtered-dependencies
    - yarn run in-submodules -- -f categories.ui=true -- run test
  artifacts:
    paths:
      - "*/build"
      - "*/dist"
    expire_in: 7 days

buildtest:registry:
  stage: buildtest
  retry: 2
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:master
  needs: [
    "sbt-prebuild"
  ]
  timeout: 30 minutes
  dependencies:
    - sbt-prebuild
  services:
    - postgres:13.3
  variables:
    POSTGRES_URL: "jdbc:postgresql://postgres/postgres"
    POSTGRES_DB: postgres
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: "password"
  script:
    - sbt "registryApi/testOnly au.csiro.data61.magda.registry.*"
  artifacts:
    paths:
      - "target"
      - "project/target"
      - "project/project/target"
      - "*/target"
      - "*/project/target"
      - "*/project/project/target"
    expire_in: 7 days

buildtest:scala-others:
  stage: buildtest
  retry: 2
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:master
  needs: [
    "sbt-prebuild"
  ]
  timeout: 30 minutes
  dependencies:
    - sbt-prebuild
  script:
    - sbt "common/test"
    - sbt "indexer/test"
  artifacts:
    paths:
      - "target"
      - "project/target"
      - "project/project/target"
      - "*/target"
      - "*/project/target"
      - "*/project/project/target"
    expire_in: 7 days

buildtest:typescript-apis-stateless:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
  retry: 1
  needs: [
    "builders-and-yarn",
    "registry-typescript-api"
  ]
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
  script:
    - cd magda-typescript-common && yarn build && yarn test && cd ..
    - cd magda-minion-framework && yarn build && yarn test && cd ..
    - yarn run in-submodules -- -f language=typescript -f categories.api=true -f categories.stateless=true -- run build --include-filtered-dependencies
    - yarn run in-submodules -- -f language=typescript -f categories.api=true -f categories.stateless=true -- run test --include-filtered-dependencies
    - yarn run in-submodules -- -f categories.npmPackage=true -f categories.useCommonLib=true -- run build
    - yarn run in-submodules -- -f categories.npmPackage=true -f categories.useMinionLib=true -- run build
  artifacts:
    paths:
      - "*/dist"
      - "packages/*/dist"
      - "packages/*/bin"
    expire_in: 30 days

buildtest:typescript-apis-with-pg:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 1
  needs: [
    "builders-and-yarn",
    "registry-typescript-api"
  ]
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
  before_script:
    - |
      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
        export DOCKER_HOST='tcp://localhost:2375'
      fi
    - docker info
    - apk update
    - apk upgrade
    - apk add python3 python3-dev py3-pip gcc musl-dev build-base libressl-dev libffi-dev libgcc
    - pip3 install --upgrade pip
    - pip3 install wheel
    - pip3 install docker-compose
    - docker-compose --version
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/pip-cache"
  services:
    - postgres:13.3
    - docker:dind
  variables:
    # allow openssl 1.02 until we upgrade builder to use node12/alpine3.9 
    CRYPTOGRAPHY_ALLOW_OPENSSL_102: 1
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
    POSTGRES_HOST: postgres
    POSTGRES_DB: postgres
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: "password"
    PGUSER: postgres
    PGPASSWORD: password
    OPA_URL: "http://docker:8181/"
  script:
    - cd deploy/helm/internal-charts/opa
    - docker-compose up -d
    - cd ../../../../
    - cd magda-typescript-common && yarn build && yarn test && cd ..
    - cd magda-minion-framework && yarn build && yarn test && cd ..
    - yarn run in-submodules -- -f language=typescript -f categories.api=true -f categories.uses-pg=true -- run build --include-filtered-dependencies
    - yarn run in-submodules -- -f language=typescript -f categories.api=true -f categories.uses-pg=true -- run test --include-filtered-dependencies
    - yarn run in-submodules -- -f categories.npmPackage=true -f categories.useAuthApi=true -- run build
    - cd deploy/helm/internal-charts/opa
    - docker-compose down
  artifacts:
    paths:
      - "*/dist"
      - "packages/*/dist"
      - "packages/*/bin"
    expire_in: 7 days

buildtest:typescript-apis-with-es:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  needs: ["builders-and-yarn"]
  dependencies:
    - builders-and-yarn
  before_script:
    - |
      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
        export DOCKER_HOST='tcp://localhost:2375'
      fi
    - docker info
    - apk update
    - apk upgrade
    - apk add --no-cache curl
    - apk add python3 python3-dev py3-pip gcc musl-dev build-base libressl-dev libffi-dev libgcc
    - pip3 install --upgrade pip
    - pip3 install wheel
    - pip3 install docker-compose
    - docker-compose --version
  cache:
    key: $CI_JOB_NAME
    paths:
      - "$CI_PROJECT_DIR/pip-cache"
  services:
    - docker:dind
  variables:
    # allow openssl 1.02 until we upgrade builder to use node12/alpine3.9 
    CRYPTOGRAPHY_ALLOW_OPENSSL_102: 1
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
    TEST_ES_URL: "http://docker:9200"
  script:
    - cd magda-elastic-search
    - docker-compose up -d
    - cd ..
    - yarn run in-submodules -- -f language=typescript -f categories.api=true -f categories.uses-es=true -- run build --include-filtered-dependencies
    # Wait for es to come up
    - until $(curl --output /dev/null --silent --head --fail http://docker:9200); do printf '.' sleep 5; done
    - curl http://docker:9200
    - yarn run in-submodules -- -f language=typescript -f categories.api=true -f categories.uses-es=true -- run test --include-filtered-dependencies
    - cd magda-elastic-search
    - docker-compose down
  artifacts:
    paths:
      - "*/dist"
    expire_in: 30 days

buildtest:storage-api:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 1
  needs: [
    "builders-and-yarn",
    "registry-typescript-api"
  ]
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
  before_script:
    - |
      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
        export DOCKER_HOST='tcp://localhost:2375'
      fi
    - docker info
    - apk update
    - apk upgrade
    - apk add python3 python3-dev py3-pip gcc musl-dev build-base libressl-dev libffi-dev libgcc
    - pip3 install --upgrade pip
    - pip3 install wheel
    - pip3 install docker-compose
    - docker-compose --version
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/pip-cache"
  services:
    - docker:dind
  variables:
    # allow openssl 1.02 until we upgrade builder to use node12/alpine3.9 
    CRYPTOGRAPHY_ALLOW_OPENSSL_102: 1
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
    MINIO_HOST: "docker"
    MINIO_PORT: "9000"
  script:
    - cd magda-storage-api
    - docker-compose up -d
    - yarn run build
    - yarn run test
    - docker-compose down
  artifacts:
    paths:
      - "*/dist"
    expire_in: 7 days

buildtest:opa-policies:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 1
  needs: ["builders-and-yarn"]
  dependencies:
    - builders-and-yarn
  cache:
    paths: []
  services:
    - docker:dind
  script:
    - docker run -v $PWD/deploy/helm/internal-charts/opa/policies:/policies openpolicyagent/opa:0.17.1 test -v ./policies

buildtest:helm-charts:
  stage: buildtest
  image: dtzar/helm-kubectl:3.1.1
  needs: []
  cache:
    paths: []
  script:
    # Init helm
    # - helm init --client-only
    - helm repo add magda-io https://charts.magda.io
    - helm dep up deploy/helm/internal-charts/storage-api
    - helm dep up deploy/helm/magda-core
    - helm dep up deploy/helm/magda
    - helm dep up deploy/helm/local-deployment
    - helm dep up deploy/helm/local-auth-test-deployment

    - echo "helm lint magda chart using magda-dev.yml"
    - helm lint -f deploy/helm/magda-dev.yml deploy/helm/local-deployment

    - echo "helm lint magda chart using minikube-dev.yml"
    - helm lint -f deploy/helm/minikube-dev.yml deploy/helm/local-deployment

    - echo "helm lint magda chart using preview.yml"
    - helm lint -f deploy/helm/preview.yml deploy/helm/local-deployment

    - echo "helm lint magda chart using preview-multi-tenant.yml"
    - helm lint -f deploy/helm/preview-multi-tenant.yml deploy/helm/local-deployment

  artifacts:
    paths:
      - "deploy/helm/magda/charts"
      - "deploy/helm/magda-core/charts"
      - "deploy/helm/local-deployment/charts"
      - "deploy/helm/local-auth-test-deployment/charts"
    expire_in: 7 days

buildtest:helm-docs-check:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:master
  needs: []
  cache:
    paths: []
  services:
    - docker:dind
  script:
    - code=0
    - docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.5.0 || code=$?;
      if [ "$code" != "0" ]; then 
        echo "Failed to run helm-docs!";
        exit 1;
      fi;
    - cd deploy
    - code=0
    - git ls-files -m | grep -i readme.md || code=$?;
      if [ "$code" == "0" ]; then
        echo -e "Some of helm chart docs are required to be updated using the [helm-docs](https://github.com/norwoodj/helm-docs) tool. \n
        Please run helm-docs (v1.5.0) at project root, review & commit docs changes and push a new commit.";
        exit 1;
      else 
        echo -e "helm docs check passed. helm docs update is not required.";
      fi;

dockerize:scala:
  stage: dockerize
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  retry: 2
  needs: [
    "registry-typescript-api",
    "sbt-prebuild",
    "buildtest:registry",
    "buildtest:search-no-index-cache",
    "buildtest:search-with-index-cache"
  ]
  dependencies:
    - registry-typescript-api
    - sbt-prebuild
    - buildtest:registry
    - buildtest:search-no-index-cache
    - buildtest:search-with-index-cache
  services:
    - docker:dind
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - cat /dev/null | sbt -Drepository=$CI_REGISTRY/magda-data/magda -Dversion=$CI_COMMIT_REF_SLUG dockerBuildAndPush

dockerize:ui:
  stage: dockerize
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  needs: [
    "builders-and-yarn",
    "registry-typescript-api",
    "buildtest:ui"
  ]
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
    - buildtest:ui
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.ui=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

dockerize:typescript-apis:
  stage: dockerize
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  needs: [
    "builders-and-yarn",
    "registry-typescript-api",
    "buildtest:typescript-apis-stateless",
    "buildtest:typescript-apis-with-pg",
    "buildtest:typescript-apis-with-es",
    "buildtest:storage-api"
  ]
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
    - buildtest:typescript-apis-stateless
    - buildtest:typescript-apis-with-pg
    - buildtest:typescript-apis-with-es
    - buildtest:storage-api
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.api=true -f language=typescript -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

dockerize:migrators:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.migrator=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

dockerize:dockerExtensions:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.dockerExtension=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

inttest:registryAuth:
  # https://github.com/kind-ci/examples/blob/master/.gitlab-ci.yml
  stage: preview
  retry: 1
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  allow_failure: true
  needs: [
    "sbt-prebuild",
    "builders-and-yarn",
    "buildtest:registry",
    "buildtest:helm-charts"
  ]
  dependencies:
    - sbt-prebuild
    - builders-and-yarn
    - buildtest:registry
    - buildtest:helm-charts
  services:
    - docker:dind
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - /usr/local/bin/kind
      - /usr/local/bin/kubectl
      - /usr/local/bin/helm
  variables:
    KUBECTL: v1.19.14
    KIND: v0.11.1
  script: |
    apk add --update postgresql-client sed curl
    curl -Lo /usr/local/bin/kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND}/kind-$(uname)-amd64 -C -
    chmod +x /usr/local/bin/kind
    curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL}/bin/linux/amd64/kubectl -C -
    chmod +x /usr/local/bin/kubectl
    curl -L https://get.helm.sh/helm-v3.5.2-linux-amd64.tar.gz -C - | tar -xzO linux-amd64/helm > /usr/local/bin/helm
    chmod +x /usr/local/bin/helm
    kind create cluster --config=./kind-config.yaml --wait 20m --image kindest/node:v1.16.15@sha256:83067ed51bf2a3395b24687094e283a7c7c865ccc12a8b1d7aa673ba0c5e8861
    # Set up KUBECONFIG that links to kind - we use sed to edit it so it's pointing to gitlab's docker host
    kind get kubeconfig | sed -E -e 's/localhost|0\.0\.0\.0/docker/g' > ./kubeconfig
    export KUBECONFIG="$(pwd)/kubeconfig"
    export JWT_SECRET=udIsbcYaKs1G4n6AdiMSIvPx5KpxQAy8FA2aIcD46iCipNAZvds4jeXFLZKhVvSJZvhYb5Pvgvmtonk7UFfhGnYcd3DXM7KzHG7gBmGO8PCsOZ4t7icqZoJbpdDqYWMmd9XnrVXtJhR6HVFBmEmbk9AmFJ1Gz9ipYPGYLoFcavPs9iZ63KPXgdt4aBdWQcmICkGPYiY8CQOvqOoiU7hUhKDTkJgRRTSaax6UQDOveTQvQnd5uyXuV4os0tlahzRX
    kubectl create ns test2
    echo "{ \"apiVersion\": \"v1\", \"kind\": \"Secret\", \"metadata\": {\"name\": \"auth-secrets\"}, \"type\": \"Opaque\", \"data\": {\"jwt-secret\": \"dWRJc2JjWWFLczFHNG42QWRpTVNJdlB4NUtweFFBeThGQTJhSWNENDZpQ2lwTkFadmRzNGplWEZMWktoVnZTSlp2aFliNVB2Z3ZtdG9uazdVRmZoR25ZY2QzRFhNN0t6SEc3Z0JtR084UENzT1o0dDdpY3Fab0picGREcVlXTW1kOVhuclZYdEpoUjZIVkZCbUVtYms5QW1GSjFHejlpcFlQR1lMb0ZjYXZQczlpWjYzS1BYZ2R0NGFCZFdRY21JQ2tHUFlpWThDUU92cU9vaVU3aFVoS0RUa0pnUlJUU2FheDZVUURPdmVUUXZRbmQ1dXlYdVY0b3MwdGxhaHpSWA==\"}}" | kubectl apply --namespace test2 -f -
    echo "{ \"apiVersion\": \"v1\", \"kind\": \"Secret\", \"metadata\": {\"name\": \"regcred\"}, \"type\": \"kubernetes.io/dockerconfigjson\", \"data\": { \".dockerconfigjson\": \"$DOCKERCONFIGJSON\" }}" | kubectl apply --namespace test2 -f -
    helm upgrade test2 deploy/helm/local-auth-test-deployment --namespace test2 --install --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,magda-core.combined-db.magda-postgres.postgresql.image.registry=registry.gitlab.com,magda-core.combined-db.magda-postgres.postgresql.image.repository=magda-data/magda/data61/magda-postgres,magda-core.combined-db.magda-postgres.postgresql.image.tag=$CI_COMMIT_REF_SLUG
    # Forward local ports to kind
    kubectl port-forward combined-db-postgresql-0 5432 --namespace test2 >/dev/null 2>&1 < /dev/null &
    kubectl port-forward deployment/authorization-api 6104:80 --namespace test2 >/dev/null 2>&1 < /dev/null &
    # Wait for port forwarding to start
    timeout -t 22 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 5432
    timeout -t 22 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 6104
    # Set up some values
    psql -h localhost -p 5432 -U postgres -d auth -f magda-registry-api/src/test/resources/data/organizations.sql
    psql -h localhost -p 5432 -U postgres -d auth -f magda-registry-api/src/test/resources/data/users.sql
    # Test!
    sbt "registryApi/testOnly au.csiro.data61.magda.opa.*"

(Full) Run As Preview: &runAsPreview
  stage: preview
  when: manual
  only:
    - branches
  except:
    - master
  dependencies:
    - buildtest:helm-charts
  cache:
    paths: []
  image:
    name: dtzar/helm-kubectl:3.1.1
  retry: 1
  environment:
    name: preview/$CI_COMMIT_REF_NAME
    url: https://$CI_COMMIT_REF_SLUG.dev.magda.io
    on_stop: Stop Preview
  before_script:
    # Add PWGEN for generating passwords
    - apk add --update pwgen

    # Env vars
    - JWT_SECRET="${JWT_SECRET:-$(pwgen 32 1)}"
    - SESSION_SECRET="${SESSION_SECRET:-$(pwgen 32 1)}"
    - MINIO_HOST="${MINIO_HOST:-localhost}"
    - MINIO_PORT="${MINIO_PORT:-9000}"
    - MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-}"
    - MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-}"

    # Kube Config
    - echo "$KUBECTL_CONFIG" > kubectlconfig.yaml
    - export KUBECONFIG=kubectlconfig.yaml

    # Create kube namespace
    - kubectl get namespace $CI_COMMIT_REF_SLUG || kubectl create namespace $CI_COMMIT_REF_SLUG

    # Create kube secrets
    - kubectl create secret generic auth-secrets --from-literal=jwt-secret=$JWT_SECRET --from-literal=session-secret=$SESSION_SECRET --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - kubectl -n $CI_COMMIT_REF_SLUG annotate --overwrite secret auth-secrets replicator.v1.mittwald.de/replication-allowed=true replicator.v1.mittwald.de/replication-allowed-namespaces=${CI_COMMIT_REF_SLUG}-openfaas-fn
    - kubectl create secret docker-registry regcred --namespace $CI_COMMIT_REF_SLUG --docker-server=registry.gitlab.com --docker-username=gitlab-ci-token --docker-password=$CI_JOB_TOKEN --docker-email=alex.gilleran@data61.csiro.au --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - kubectl create secret generic oauth-secrets --from-literal=facebook-client-secret=$FACEBOOK_CLIENT_SECRET --from-literal=google-client-secret=$GOOGLE_CLIENT_SECRET --from-literal arcgis-client-secret=$ARCGIS_CLIENT_SECRET --from-literal vanguard-certificate="$VANGUARD_CERT" --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - kubectl create secret generic smtp-secret --from-literal=username=$SMTP_USERNAME --from-literal=password=$SMTP_PASSWORD --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - echo "$DB_SERVICE_ACCOUNT_PRIVATE_KEY" > backup-storage-account.json
    - kubectl create secret generic backup-storage-account --from-file backup-storage-account.json --from-literal=GOOGLE_APPLICATION_CREDENTIALS=/etc/wal-g.d/env/backup-storage-account.json --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - kubectl create secret generic storage-secrets --from-literal=accesskey=$MINIO_ACCESS_KEY --from-literal=secretkey=$MINIO_SECRET_KEY --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
  script:
    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/local-deployment --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600m --wait
    - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"

(UI) Run As Preview:
  <<: *runAsPreview
  script:
    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/local-deployment --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview.yml --set global.openfaas.enabled=false,global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,magda.magda-core.ingress.targetService=web,tags.all=false,tags.web-server=true,magda.magda-core.web-server.baseUrl=https://dev.magda.io,magda.magda-core.web-server.useLocalStyleSheet=true --timeout 3600m --wait
    - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"

(UI - Auto) Run As Preview:
  <<: *runAsPreview
  when: always
  only:
    variables:
      - $CI_COMMIT_MESSAGE =~ /#deploy-ui-preview/
  script:
    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/local-deployment --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview.yml --set global.openfaas.enabled=false,global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,magda.magda-core.ingress.targetService=web,tags.all=false,tags.web-server=true,magda.magda-core.web-server.baseUrl=https://dev.magda.io,magda.magda-core.web-server.useLocalStyleSheet=true --timeout 3600m --wait
    - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"

(No Data) Run As Preview:
  <<: *runAsPreview
  script:
    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/local-deployment --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,magda.magda-core.combined-db.magda-postgres.backupRestore.recoveryMode.enabled=false,magda.magda-core.elasticsearch.useGcsSnapshots=false,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600m --wait
    - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"
(No Data) Run As Multi-tenant Preview:
  <<: *runAsPreview
  script:
    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/local-deployment --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview-multi-tenant.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,magda.magda-core.combined-db.magda-postgres.backupRestore.recoveryMode.enabled=false,magda.magda-core.elasticsearch.useGcsSnapshots=false,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.magdaAdminPortalName=admin-$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600 --wait
    - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"


Stop Preview: &stopPreview
  stage: preview
  when: manual
  only:
    - branches
  except:
    - master
  dependencies: []
  cache:
    paths: []
  image:
    name: dtzar/helm-kubectl:3.1.1
  retry: 1
  before_script: []
  environment:
    name: preview/$CI_COMMIT_REF_NAME
    action: stop
  script:
    - echo "$KUBECTL_CONFIG" > kubectlconfig.yaml
    - export KUBECONFIG=kubectlconfig.yaml
    - helm --namespace $CI_COMMIT_REF_SLUG del $CI_COMMIT_REF_SLUG
    - kubectl delete namespace $CI_COMMIT_REF_SLUG

Deploy Master To Dev:
  stage: deploy-dev
  only:
    - master
  cache: {}
  dependencies:
    - buildtest:helm-charts
  image:
    name: dtzar/helm-kubectl:3.1.1
  retry: 1
  before_script: []
  environment:
    name: dev
    url: https://dev.magda.io
  script:
    - echo "$KUBECTL_CONFIG" > kubectlconfig.yaml
    - export KUBECONFIG=kubectlconfig.yaml
    - kubectl create secret docker-registry regcred --namespace default --docker-server=registry.gitlab.com --docker-username=gitlab-ci-token --docker-password=$CI_JOB_TOKEN --docker-email=alex.gilleran@data61.csiro.au --dry-run -o json | kubectl apply --namespace default -f -
    - helm upgrade magda deploy/helm/local-deployment --install --recreate-pods -f deploy/helm/magda-dev.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=master --timeout 3600m --wait

pre-release:check-release-version:
  stage: pre-release
  only:
    # Strict Semvar validation
    - /^v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$/
  except:
    - branches
    - triggers
  dependencies:
    - builders-and-yarn
  cache: {}
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
  script:
    - yarn check-release-version

Release Tags To Docker Hub:
  stage: release
  only:
    # Strict Semvar validation
    - /^v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$/
  except:
    - branches
    - triggers
  dependencies:
    - builders-and-yarn
  needs:
    - pre-release:check-release-version
    - builders-and-yarn
  cache: {}
  services:
    - docker:dind
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 1
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - docker login -u magdabot -p $DOCKER_HUB_PASSWORD
    - yarn install
    - yarn run retag-and-push -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG


Publish NPM Packages:
  stage: release
  only:
    # Strict Semvar validation
    - /^v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$/
  except:
    - branches
    - triggers
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
    - buildtest:typescript-apis-stateless
    - buildtest:typescript-apis-with-pg
    - buildtest:typescript-apis-with-es
  needs:
    - builders-and-yarn
    - registry-typescript-api
    - buildtest:typescript-apis-stateless
    - buildtest:typescript-apis-with-pg
    - buildtest:typescript-apis-with-es
    - pre-release:check-release-version
  image:
    name: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
  cache: {}
  script:
    # Setup NPM token & scoped registry
    - npm config set @magda:registry https://registry.npmjs.org/
    - npm config set '//registry.npmjs.org/:_authToken' "${NPM_TOKEN}"
    - yarn run in-submodules -f categories.npmPackage=true run release

Publish Helm Chart:
  stage: release
  only:
    # Strict Semvar validation
    - /^v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$/
  except:
    - branches
    - triggers
  dependencies:
    - buildtest:helm-charts
  needs:
    - buildtest:helm-charts
    - pre-release:check-release-version
  image:
    name: dtzar/helm-kubectl:3.1.1
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - deploy/helm/chart-repo
  retry: 1
  script:
    # Get AWS CLI
    - apk -v --update add python py-pip groff less mailcap
    - pip install --upgrade awscli==1.14.5 s3cmd==2.0.1 python-magic

    # Run deploy
    - cd deploy/helm
    - mkdir -p sync_dir
    - mkdir -p index_dir
    - if ! aws s3 cp s3://magda-charts/index.yaml index_dir/index.yaml; then echo "failed to copy current repo index" && exit 1; fi
    - helm package -d sync_dir magda-core
    - helm package -d sync_dir magda
    - helm repo index --merge "index_dir/index.yaml" sync_dir
    # avoid conflicts & make sure the packages are in place first; upload index at the end
    - mv -f sync_dir/index.yaml index_dir/index.yaml
    - aws s3 sync sync_dir s3://magda-charts/
    - aws s3 cp index_dir/index.yaml s3://magda-charts/index.yaml

# Update scripts:
#   stage: release
#   only:
#     # Strict Semvar validation
#     - /^v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$/
#   except:
#     - branches
#     - triggers
#   image: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
#   needs:
#     - builders-and-yarn
#     - pre-release:check-release-version
#   dependencies:
#     - builders-and-yarn
#   script:
#     - cd scripts
#     - yarn pkg create-secrets/index.js --out-path create-secrets/build
#     - cd ../../
#     - if [ ! -d magda-config/ ]; then git clone https://github.com/magda-io/magda-config.git; else cd magda-config && git pull && cd ..; fi
#     - cp magda/scripts/create-secrets/build/index-linux magda-config/create-secrets/index-linux
#     - cp magda/scripts/create-secrets/build/index-macos magda-config/create-secrets/index-macos
#     - cp magda/scripts/create-secrets/build/index-win.exe magda-config/create-secrets/index-win.exe
#     - cd magda-config
#     - cd create-secrets
#     - git add index-linux index-macos index-win.exe
#     - git config --global user.email "magdabot@magda.io"
#     - git config --global user.name "magdabot"
#     - git commit -m "Update create-secrets scripts `date`"
#     - git push "https://x-access-token:$GITHUB_ACCESS_TOKEN@github.com/magda-io/magda-config" master