diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6af21979300c53c77fade38c4e00bfb50babc841..d146105e451a11ed2135f0ac3cc843f85d84cff3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,12 +64,23 @@ build: -deploy_prod: +.deploy_step: + image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0 + before_script: + - helm repo add bitnami https://charts.bitnami.com/bitnami + - helm repo update + - pwd && ls -lah + - echo $NAMESPACE + - echo $CI_COMMIT_REF_SLUG + - echo $STORAGE_SUFFIX + - kubectl get pods -n $NAMESPACE + - apk add --no-cache gettext + + + +.prod_settings: only: - master - stage: deploy - needs: ["build"] - image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0 variables: CI_DEBUG_TRACE: "false" NAMESPACE: "viralhostrange-prod" @@ -81,15 +92,11 @@ deploy_prod: environment: name: "k8sprod-02/viralhostrange-prod/$CI_COMMIT_REF_SLUG" url: "https://viralhostrangedb.pasteur.cloud" + + + +.deploy_db: script: - - helm repo add bitnami https://charts.bitnami.com/bitnami - - helm repo update - - pwd && ls -lah - - echo $NAMESPACE - - echo $CI_COMMIT_REF_SLUG - - echo $STORAGE_SUFFIX - - kubectl get pods -n $NAMESPACE - - apk add --no-cache gettext - kubectl delete secret registry-gitlab -n $NAMESPACE --ignore-not-found=true - kubectl create secret docker-registry -n $NAMESPACE registry-gitlab --docker-server=$CI_REGISTRY --docker-username=$DEPLOY_USER --docker-password=$DEPLOY_TOKEN --docker-email=$GITLAB_USER_EMAIL - envsubst < k8s/config-map.yaml | kubectl apply -n $NAMESPACE -f - @@ -107,6 +114,11 @@ deploy_prod: --set primary.podLabels.branch=branch${STORAGE_SUFFIX} --set primary.podLabels.app=postgres-app --set fullnameOverride="db${STORAGE_SUFFIX}" + + + +.deploy_web: + script: - envsubst < k8s/kubernetes-storage.yaml | kubectl apply -n $NAMESPACE -f - - envsubst < k8s/manifest.yaml | kubectl apply -n $NAMESPACE -f - # - envsubst < k8s/manifest-webhost-autoscale.yaml | kubectl apply -n $NAMESPACE -f - @@ -114,10 +126,30 @@ deploy_prod: +deploy_prod_db: + needs: ["build"] + stage: deploy + extends: + - .deploy_step + - .prod_settings + - .deploy_db + + + +deploy_prod_web: + needs: ["build", "deploy_prod_db"] + stage: deploy + extends: + - .deploy_step + - .prod_settings + - .deploy_web + + + dump_prod_for_dev: except: - master - stage: build + stage: deploy needs: [] image: registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image variables: @@ -136,12 +168,9 @@ dump_prod_for_dev: -deploy_dev: +.dev_settings: except: - master - stage: deploy - needs: ["build", "dump_prod_for_dev"] - image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0 variables: CI_DEBUG_TRACE: "false" NAMESPACE: "viralhostrange-dev" @@ -154,40 +183,41 @@ deploy_dev: name: "k8sdev-01/viralhostrange-dev/$CI_COMMIT_REF_SLUG" url: "https://viralhostrangedb-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud" on_stop: stop_and_delete_in_dev + + + +deploy_dev_db: + needs: ["build"] + stage: deploy + extends: + - .deploy_step + - .dev_settings + - .deploy_db + after_script: + - . ./k8s/init_db_from_prod.sh + + + +init_dev_db: + needs: ["deploy_dev_db", "dump_prod_for_dev"] + stage: deploy + extends: + - .deploy_step + - .dev_settings script: - - helm repo add bitnami https://charts.bitnami.com/bitnami - - helm repo update - - pwd && ls -lah - - echo $NAMESPACE - - echo $CI_COMMIT_REF_SLUG - - echo $STORAGE_SUFFIX - - kubectl get pods -n $NAMESPACE - - apk add --no-cache gettext - - kubectl delete secret registry-gitlab -n $NAMESPACE --ignore-not-found=true - - kubectl create secret docker-registry -n $NAMESPACE registry-gitlab --docker-server=$CI_REGISTRY --docker-username=$DEPLOY_USER --docker-password=$DEPLOY_TOKEN --docker-email=$GITLAB_USER_EMAIL - - envsubst < k8s/config-map.yaml | kubectl apply -n $NAMESPACE -f - - - > - helm upgrade --install db${STORAGE_SUFFIX} bitnami/postgresql --version $CHART_VERSION --namespace=$NAMESPACE - --set image.tag=$POSTGRESQL_VERSION - --set persistence.storageClass='isilon' - --set persistence.enabled=true - --set persistence.size="5Gi" - --set image.pullPolicy='Always' - --set postgresqlUsername='postgres' - --set postgresqlPostgresPassword="$POSTGRES_PASSWORD" - --set postgresqlPassword="$POSTGRES_PASSWORD" - --set postgresqlDatabase='viralhostrangedb' - --set primary.podLabels.branch=branch${STORAGE_SUFFIX} - --set primary.podLabels.app=postgres-app - --set fullnameOverride="db${STORAGE_SUFFIX}" - . ./k8s/init_db_from_prod.sh - - envsubst < k8s/kubernetes-storage.yaml | kubectl apply -n $NAMESPACE -f - - - envsubst < k8s/manifest.yaml | kubectl apply -n $NAMESPACE -f - - # - envsubst < k8s/manifest-webhost-autoscale.yaml | kubectl apply -n $NAMESPACE -f - - - envsubst < k8s/kubernetes-cronjob.yaml | kubectl apply -n $NAMESPACE -f - +deploy_dev_web: + needs: ["init_dev_db"] + stage: deploy + extends: + - .deploy_step + - .dev_settings + - .deploy_web + + stop_and_delete_in_dev: except: - master diff --git a/src/viralhostrange/HOWTO.md b/src/viralhostrange/HOWTO.md index ad5ed56478f248e369e9e293b9ae373fd1d60478..53257f180358fcd8fc6babd8c407f811044ad8c3 100644 --- a/src/viralhostrange/HOWTO.md +++ b/src/viralhostrange/HOWTO.md @@ -6,4 +6,35 @@ docker run -v $(pwd)/persistent_volume:/code/persistent_volume -e "USE_SQLITE_AS ``` docker run --name db-test_db -e POSTGRES_PASSWORD=eee -e POSTGRES_DB=viralhostrangedb -e POSTGRES_USER=postgres -e LC_COLLATE=POSIX -p 5432:5432 -d postgres:10.1 -``` \ No newline at end of file +``` + +## How to load a specific dump in PostgreSQL + +Useful when upgrading PostgreSQL, or restoring the DB after cluster migration. + +### get a dump +Get it from the CI as it is an artifact + +### clean up +``` +export CI_COMMIT_REF_SLUG="split-deploy" + +# Stop web to prevent manage.py migrate +kubectl delete deployments -l branch=branch-$CI_COMMIT_REF_SLUG + +# Stop the db, and delete its storage +kubectl delete statefulsets,pvc -l app.kubernetes.io/instance=db-$CI_COMMIT_REF_SLUG +``` +Do keep the storage of the web container + +# load the data from the dump +``` +dev_db=$(kubectl --namespace=$NAMESPACE get po -l branch=branch-${CI_COMMIT_REF_SLUG},app=postgres-app --output jsonpath='{.items[0].metadata.name}' || echo "") +echo $dev_db +export POSTGRES_PASSWORD=toto +kubectl --namespace=$NAMESPACE exec -i $dev_db -- bash -c "PGPASSWORD=$POSTGRES_PASSWORD psql -U postgres viralhostrangedb" < prod.sql +``` + +# Restart + +Re-run CI job deploy_web_* \ No newline at end of file