diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c700f7d9c9c19e87e26a3451d3fca79a0d99b253..e9c16a1be251d942476c04b279d78e20dfbb4c64 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,3 @@
-stages:
- - build
- - test
- - deploy
test-python:
@@ -15,7 +11,6 @@ test-python:
- apt-get update && apt install -y libblas-dev liblapack-dev python3-dev
- pip install -r requirements.txt
- python -m unittest
-# - python setup.py test
parallel:
matrix:
- PYTHON_VERSION: [
@@ -26,47 +21,6 @@ test-python:
]
-test-docker-compose:
- image: docker:24
- needs: []
- # when: delayed
- # start_in: 3 minutes
- stage: test
- retry:
- max: 2
- when: runner_system_failure
- script:
- - apk update && apk upgrade && apk add jq bash curl
- - docker compose version
- - ./test_docker_compose.sh
- after_script:
- - docker compose logs back > log/docker-compose.back.log
- - docker compose logs > log/docker-compose.log
- artifacts:
- when: always
- paths:
- - log
- expire_in: 1 week
-
-
-#test-backend-image:
-# image: docker:24
-# needs:
-# - build-backend
-# stage: test
-# script:
-# - mkdir .eggs jass.egg-info && chmod -R 777 .eggs jass.egg-info
-# - touch data/initTable.hdf5
-# - >
-# docker run
-# -v $(pwd)/data:/data
-# -v $(pwd)/projects:/projects
-# -v $(pwd)/.eggs:/code/.eggs
-# -v $(pwd)/jass.egg-info:/code/jass.egg-info
-# "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/backend:$CI_COMMIT_SHA"
-# python setup.py test
-
-
pages:
needs: []
image: python:3.9
@@ -82,319 +36,3 @@ pages:
- public
only:
- master
-
-
-.build:
- stage: build
- image: docker:24
- retry:
- max: 2
- when: runner_system_failure
- script:
- - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
- # pull the latest build on master
- - docker pull "$CI_REGISTRY_IMAGE/master/${PATH_TAG}:latest" || true
- # pull the latest build on this branch
- - docker pull "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:latest" || true
- # build the image while passing commit SHA and tagging the image with it
- - docker build
- --target ${PATH_TAG}
- --build-arg CI_COMMIT_REF_SLUG
- --build-arg CI_COMMIT_SHA
- --build-arg API_URL
- --cache-from "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:latest"
- --cache-from "$CI_REGISTRY_IMAGE/master/${PATH_TAG}:latest"
- --tag "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:$CI_COMMIT_SHA"
- --tag "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:latest"
- .
- # push image as latest for the current branch
- - docker push "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:latest"
- # push image tagged with its sha
- - docker push "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:$CI_COMMIT_SHA"
-
-
-build-backend:
- needs: []
- extends: .build
- variables:
- PATH_TAG: backend
- after_script:
- - mkdir .eggs jass.egg-info && chmod -R 777 .eggs jass.egg-info
- - touch data/initTable.hdf5
- - >
- docker run
- -v $(pwd)/data:/data
- -v $(pwd)/projects:/projects
- -v $(pwd)/.eggs:/code/.eggs
- -v $(pwd)/jass.egg-info:/code/jass.egg-info
- "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:$CI_COMMIT_SHA"
- python -m unittest
-
-
-build-client-dev:
- except:
- - master
- needs: []
- extends: .build
- before_script:
- - cd client
- variables:
- PATH_TAG: client-static-serve
- API_URL: "https://jass-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud/api"
-
-
-build-client-prod:
- only:
- - master
- needs: []
- extends: .build
- before_script:
- - cd client
- variables:
- PATH_TAG: client-static-serve
- API_URL: "https://jass.pasteur.fr/api"
-
-
-
-.deploy:
- stage: deploy
- needs:
- - build-backend
- - build-client-dev
- - test-python
-# - test-backend-image
- image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
- retry:
- max: 2
- when: runner_system_failure
- script:
- # create if missing the shared pvc to save data
- - kubectl apply -n $NAMESPACE -f chart/pvc-shared-data.yaml
- - helm dependency update chart
- # shows some error on first deployment
- - kubectl get secret --namespace "${NAMESPACE}" ${CI_COMMIT_REF_SLUG}-rabbitmq || NO_SECRET=$?
- - >
- if [ $NO_SECRET ]; then
- RABBITMQ_PASSWORD=''
- RABBITMQ_ERLANG_COOKIE=''
- else
- RABBITMQ_PASSWORD=$(kubectl get secret --namespace "${NAMESPACE}" ${CI_COMMIT_REF_SLUG}-rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
- RABBITMQ_ERLANG_COOKIE=$(kubectl get secret --namespace "${NAMESPACE}" ${CI_COMMIT_REF_SLUG}-rabbitmq -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 -d)
- fi
- - >
- helm upgrade --install --namespace=${NAMESPACE}
- --set CI_REGISTRY_IMAGE=${CI_REGISTRY_IMAGE}
- --set image.tag=${CI_COMMIT_SHA}
- --set ingress.host.name=${PUBLIC_URL}
- --set registry.username=${DEPLOY_USER}
- --set registry.password=${DEPLOY_TOKEN}
- --set registry.host=${CI_REGISTRY}
- --set imagePullSecrets[0].name="registry-pull-secret-${CI_COMMIT_REF_SLUG}"
- --set rabbitmq.auth.password=$RABBITMQ_PASSWORD
- --set rabbitmq.auth.erlangCookie=$RABBITMQ_ERLANG_COOKIE
- --values ./chart/values.${NAMESPACE}.yaml
- ${CI_COMMIT_REF_SLUG} ./chart/
-
-
-deploy-dev:
- except:
- - master
- stage: deploy
- extends: .deploy
- variables:
- NAMESPACE: "jass-dev"
- PUBLIC_URL: "jass-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud"
- environment:
- name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}"
- url: "https://jass-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud"
- on_stop: delete-dev-deployment
-
-
-deploy-prod:
- only:
- - master
- stage: deploy
- needs:
- - build-backend
- - build-client-prod
- - test-python
- extends: .deploy
- variables:
- NAMESPACE: "jass-prod"
- PUBLIC_URL: "jass.pasteur.fr"
- environment:
- name: "k8sprod-02/jass-prod/${CI_COMMIT_REF_SLUG}"
- url: "https://jass.pasteur.fr"
-
-
-delete-dev-deployment:
- stage: deploy
- needs: []
- when: manual
- image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
- variables:
- GIT_STRATEGY: none # important to not checkout source when branch is deleted
- NAMESPACE: "jass-dev"
- environment:
- name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}"
- action: stop
- retry:
- max: 2
- when: runner_system_failure
- script:
- - echo "Removing $CI_COMMIT_REF_SLUG"
- - helm delete -n ${NAMESPACE} ${CI_COMMIT_REF_SLUG}
- - kubectl delete pvc -lapp.kubernetes.io/name=rabbitmq,app.kubernetes.io/instance=${CI_COMMIT_REF_SLUG}
-
-
-.upload-from-fex-to-somewhere:
- stage: deploy
- needs: []
- when: manual
- image: harbor.pasteur.fr/kube-system/helm-kubectl:3.14.0
- variables:
- FEX_URL: ""
- script:
- - mkdir -p removed added
- - >
- if [[ "$FEX_URL" =~ ^https://dl\.pasteur\.fr/.* ]]; then
- echo "Valid URL: $FEX_URL";
- else
- echo "FEX_URL invalide (FEX_URL=$FEX_URL), please provide a fex url starting with https://dl.pasteur.fr/...";
- exit 1;
- fi
- - wget $FEX_URL
- - ls -lah .
- - export HDF5_FILENAME=$(ls -1 *hdf5 | cut -d'/' -f2)
- - >
- if [ ! $HDF5_FILENAME ]; then
- echo "No hdf5 file fetched, exiting";
- exit 2;
- else
- echo "Valid File: $HDF5_FILENAME";
- fi
- - kubectl apply -f chart/sidekick.yaml -f chart/pvc-shared-data.yaml
- - kubectl wait -f chart/sidekick.yaml --for=condition=Ready --timeout=3600s
- - kubectl cp sidekick:/data/$HDF5_FILENAME ./removed/$HDF5_FILENAME || echo "no older version"
- - kubectl cp ./$HDF5_FILENAME sidekick:/data/$HDF5_FILENAME
- - mv ./$HDF5_FILENAME ./added/
- - kubectl exec sidekick -- ls -lah /data
- - kubectl delete -f chart/sidekick.yaml --force
- artifacts:
- when: always
- paths:
- - added
- - removed
- expire_in: 1 month
-
-
-upload-from-fex-to-dev:
- extends: .upload-from-fex-to-somewhere
- environment:
- name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}"
-
-
-upload-from-fex-to-prod:
- extends: .upload-from-fex-to-somewhere
- only:
- - master
- environment:
- name: "k8sprod-02/jass-prod/${CI_COMMIT_REF_SLUG}"
-
-
-.delete-data-from-somewhere:
- stage: deploy
- needs: []
- when: manual
- image: harbor.pasteur.fr/kube-system/helm-kubectl:3.14.0
- variables:
- HDF5_FILENAME: ""
- script:
- - kubectl apply -f chart/sidekick.yaml -f chart/pvc-shared-data.yaml
- - kubectl wait -f chart/sidekick.yaml --for=condition=Ready --timeout=3600s
- - kubectl exec sidekick -- ls -lah /data
- - >
- if [ ! $HDF5_FILENAME ]; then
- echo "No hdf5 file provided, exiting";
- exit 0;
- else
- echo "File to remove: $HDF5_FILENAME";
- fi
- - mkdir removed
- - >
- if [[ "$HDF5_FILENAME" =~ ^[^/]+\.hdf5$ ]]; then
- echo "Valid filename: $HDF5_FILENAME";
- else
- echo "HDF5_FILENAME invalide (HDF5_FILENAME=$HDF5_FILENAME), please provide an hdf5 file in /data";
- fi
- - kubectl cp sidekick:/data/$HDF5_FILENAME ./removed/$HDF5_FILENAME
- - kubectl exec sidekick -- rm -f /data/$HDF5_FILENAME
- - kubectl delete -f chart/sidekick.yaml --force
- artifacts:
- when: always
- paths:
- - removed
- expire_in: 1 month
-
-
-delete-data-from-dev:
- extends: .delete-data-from-somewhere
- environment:
- name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}"
-
-
-delete-data-from-prod:
- extends: .delete-data-from-somewhere
- only:
- - master
- environment:
- name: "k8sprod-02/jass-prod/${CI_COMMIT_REF_SLUG}"
-
-
-.log-fetcher:
- stage: deploy
- needs: []
- when: manual
- image: harbor.pasteur.fr/kube-system/helm-kubectl:3.12.0
- variables:
- GIT_STRATEGY: none # don't need to checkout sources
- LAST_ROWS_COUNT: "400" # you can override it when triggering the job
- CHART_LOCATION: "chart"
- script:
- - kubectl -n ${NAMESPACE} get quota
- - mkdir logs
- - |
- for POD in $(kubectl -n ${NAMESPACE} get po -l="app.kubernetes.io/instance=${CI_COMMIT_REF_SLUG}" --output=jsonpath='{.items[*].metadata.name}'); do
- echo "************************************************************"
- echo "************************************************************"
- echo "* ${POD}";
- echo "************************************************************"
- kubectl -n ${NAMESPACE} logs $POD --tail ${LAST_ROWS_COUNT} > logs/${POD}.log
- cat logs/${POD}.log
- done
- - echo "Check Job artifacts to download or browse log files. See "Job artifacts">"Browse" on right panel"
- artifacts:
- when: always
- paths:
- - logs
- expire_in: 1 month
-
-
-log-fetcher-in-dev:
- except:
- - master
- extends: .log-fetcher
- variables:
- NAMESPACE: "jass-dev"
- environment:
- name: "k8sdev-01/${NAMESPACE}/${CI_COMMIT_REF_SLUG}"
-
-
-log-fetcher-in-prod:
- only:
- - master
- extends: .log-fetcher
- variables:
- NAMESPACE: "jass-prod"
- environment:
- name: "k8sprod-02/${NAMESPACE}/${CI_COMMIT_REF_SLUG}"