From d975342266743badabc90654111ae32b262b82de Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Mon, 3 Jan 2022 14:52:13 +0100 Subject: [PATCH 1/9] split deploy in two steps --- .gitlab-ci.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6af2197..6d7540c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -136,11 +136,10 @@ dump_prod_for_dev: -deploy_dev: +.deploy_dev: except: - master stage: deploy - needs: ["build", "dump_prod_for_dev"] image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0 variables: CI_DEBUG_TRACE: "false" @@ -154,6 +153,12 @@ deploy_dev: name: "k8sdev-01/viralhostrange-dev/$CI_COMMIT_REF_SLUG" url: "https://viralhostrangedb-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud" on_stop: stop_and_delete_in_dev + + + +deploy_dev_db: + extends: .deploy_dev + needs: ["build", "dump_prod_for_dev"] script: - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo update @@ -181,6 +186,13 @@ deploy_dev: --set primary.podLabels.app=postgres-app --set fullnameOverride="db${STORAGE_SUFFIX}" - . ./k8s/init_db_from_prod.sh + + + +deploy_dev_web: + extends: .deploy_dev + needs: ["deploy_dev_db", ] + script: - envsubst < k8s/kubernetes-storage.yaml | kubectl apply -n $NAMESPACE -f - - envsubst < k8s/manifest.yaml | kubectl apply -n $NAMESPACE -f - # - envsubst < k8s/manifest-webhost-autoscale.yaml | kubectl apply -n $NAMESPACE -f - -- GitLab From 4ca377b34ad739c9bd7fd6c56fd2ec237ffd2dad Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Mon, 3 Jan 2022 15:11:36 +0100 Subject: [PATCH 2/9] split deploy steps in re-usable steps for dev and prod --- .gitlab-ci.yml | 105 ++++++++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d7540c..384bb7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,12 +64,22 @@ build: -deploy_prod: +.deploy_step: + image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0 + before_script: + - helm repo add bitnami https://charts.bitnami.com/bitnami + - helm repo update + - pwd && ls -lah + - echo $NAMESPACE + - echo $CI_COMMIT_REF_SLUG + - echo $STORAGE_SUFFIX + - kubectl get pods -n $NAMESPACE + - apk add --no-cache gettext + + +.prod_settings: only: - master - stage: deploy - needs: ["build"] - image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0 variables: CI_DEBUG_TRACE: "false" NAMESPACE: "viralhostrange-prod" @@ -81,15 +91,11 @@ deploy_prod: environment: name: "k8sprod-02/viralhostrange-prod/$CI_COMMIT_REF_SLUG" url: "https://viralhostrangedb.pasteur.cloud" + + + +.deploy_db: script: - - helm repo add bitnami https://charts.bitnami.com/bitnami - - helm repo update - - pwd && ls -lah - - echo $NAMESPACE - - echo $CI_COMMIT_REF_SLUG - - echo $STORAGE_SUFFIX - - kubectl get pods -n $NAMESPACE - - apk add --no-cache gettext - kubectl delete secret registry-gitlab -n $NAMESPACE --ignore-not-found=true - kubectl create secret docker-registry -n $NAMESPACE registry-gitlab --docker-server=$CI_REGISTRY --docker-username=$DEPLOY_USER --docker-password=$DEPLOY_TOKEN --docker-email=$GITLAB_USER_EMAIL - envsubst < k8s/config-map.yaml | kubectl apply -n $NAMESPACE -f - @@ -107,6 +113,11 @@ deploy_prod: --set primary.podLabels.branch=branch${STORAGE_SUFFIX} --set primary.podLabels.app=postgres-app --set fullnameOverride="db${STORAGE_SUFFIX}" + + + +.deploy_web: + script: - envsubst < k8s/kubernetes-storage.yaml | kubectl apply -n $NAMESPACE -f - - envsubst < k8s/manifest.yaml | kubectl apply -n $NAMESPACE -f - # - envsubst < k8s/manifest-webhost-autoscale.yaml | kubectl apply -n $NAMESPACE -f - @@ -114,6 +125,26 @@ deploy_prod: +deploy_prod_db: + needs: ["build"] + stage: deploy + extends: + - .deploy_step + - .prod_settings + - .deploy_db + + + +deploy_prod_web: + needs: ["build", "deploy_prod_db"] + stage: deploy + extends: + - .deploy_step + - .prod_settings + - .deploy_web + + + dump_prod_for_dev: except: - master @@ -136,10 +167,9 @@ dump_prod_for_dev: -.deploy_dev: +.dev_settings: except: - master - stage: deploy image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0 variables: CI_DEBUG_TRACE: "false" @@ -157,47 +187,24 @@ dump_prod_for_dev: deploy_dev_db: - extends: .deploy_dev needs: ["build", "dump_prod_for_dev"] - script: - - helm repo add bitnami https://charts.bitnami.com/bitnami - - helm repo update - - pwd && ls -lah - - echo $NAMESPACE - - echo $CI_COMMIT_REF_SLUG - - echo $STORAGE_SUFFIX - - kubectl get pods -n $NAMESPACE - - apk add --no-cache gettext - - kubectl delete secret registry-gitlab -n $NAMESPACE --ignore-not-found=true - - kubectl create secret docker-registry -n $NAMESPACE registry-gitlab --docker-server=$CI_REGISTRY --docker-username=$DEPLOY_USER --docker-password=$DEPLOY_TOKEN --docker-email=$GITLAB_USER_EMAIL - - envsubst < k8s/config-map.yaml | kubectl apply -n $NAMESPACE -f - - - > - helm upgrade --install db${STORAGE_SUFFIX} bitnami/postgresql --version $CHART_VERSION --namespace=$NAMESPACE - --set image.tag=$POSTGRESQL_VERSION - --set persistence.storageClass='isilon' - --set persistence.enabled=true - --set persistence.size="5Gi" - --set image.pullPolicy='Always' - --set postgresqlUsername='postgres' - --set postgresqlPostgresPassword="$POSTGRES_PASSWORD" - --set postgresqlPassword="$POSTGRES_PASSWORD" - --set postgresqlDatabase='viralhostrangedb' - --set primary.podLabels.branch=branch${STORAGE_SUFFIX} - --set primary.podLabels.app=postgres-app - --set fullnameOverride="db${STORAGE_SUFFIX}" + stage: deploy + extends: + - .deploy_step + - .dev_settings + - .deploy_db + after_script: - . ./k8s/init_db_from_prod.sh deploy_dev_web: - extends: .deploy_dev - needs: ["deploy_dev_db", ] - script: - - envsubst < k8s/kubernetes-storage.yaml | kubectl apply -n $NAMESPACE -f - - - envsubst < k8s/manifest.yaml | kubectl apply -n $NAMESPACE -f - - # - envsubst < k8s/manifest-webhost-autoscale.yaml | kubectl apply -n $NAMESPACE -f - - - envsubst < k8s/kubernetes-cronjob.yaml | kubectl apply -n $NAMESPACE -f - - + needs: ["build", "deploy_dev_db"] + stage: deploy + extends: + - .deploy_step + - .dev_settings + - .deploy_web stop_and_delete_in_dev: -- GitLab From e6ee0bfb253e1b6b9dd4c329305a2d36809deb05 Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Mon, 3 Jan 2022 15:16:32 +0100 Subject: [PATCH 3/9] put dev db initialisation alone --- .gitlab-ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 384bb7f..0a77748 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,6 +77,7 @@ build: - apk add --no-cache gettext + .prod_settings: only: - master @@ -198,8 +199,20 @@ deploy_dev_db: +init_dev_db: + needs: ["deploy_dev_db"] + stage: deploy + extends: + - .deploy_step + - .dev_settings + - .deploy_db + script: + - . ./k8s/init_db_from_prod.sh + + + deploy_dev_web: - needs: ["build", "deploy_dev_db"] + needs: ["init_dev_db"] stage: deploy extends: - .deploy_step -- GitLab From c7f53de49ae80520bc903e965d28478b5d4d02c4 Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Mon, 3 Jan 2022 15:20:01 +0100 Subject: [PATCH 4/9] SKIP TEST --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a77748..f09e7bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ build: stage: build needs: [] variables: - RUN_TEST: "1" + RUN_TEST: "0" POSTGRES_HOST: "db-test" STORAGE_SUFFIX: "-test" POSTGRES_PASSWORD: "example" -- GitLab From 04c6563ab20be43b59a1c9ff1b760f5f85674893 Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Mon, 3 Jan 2022 15:38:58 +0100 Subject: [PATCH 5/9] db dump is needed by init, not db setup --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f09e7bc..4797d05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -188,7 +188,7 @@ dump_prod_for_dev: deploy_dev_db: - needs: ["build", "dump_prod_for_dev"] + needs: ["build"] stage: deploy extends: - .deploy_step @@ -200,12 +200,11 @@ deploy_dev_db: init_dev_db: - needs: ["deploy_dev_db"] + needs: ["deploy_dev_db", "dump_prod_for_dev"] stage: deploy extends: - .deploy_step - .dev_settings - - .deploy_db script: - . ./k8s/init_db_from_prod.sh -- GitLab From 527901fe0b8780e1a3181a5d20cbc632191fec5e Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Mon, 3 Jan 2022 17:10:29 +0100 Subject: [PATCH 6/9] db dump is for deploy, as we use need it will be run as soon as possible --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4797d05..468200b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -149,7 +149,7 @@ deploy_prod_web: dump_prod_for_dev: except: - master - stage: build + stage: deploy needs: [] image: registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image variables: -- GitLab From 8c4fa93f07e5a468b68032d3aa979d6b6995e599 Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Mon, 3 Jan 2022 17:12:09 +0100 Subject: [PATCH 7/9] db dump is for deploy, as we use need it will be run as soon as possible --- src/viralhostrange/HOWTO.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/viralhostrange/HOWTO.md b/src/viralhostrange/HOWTO.md index ad5ed56..53257f1 100644 --- a/src/viralhostrange/HOWTO.md +++ b/src/viralhostrange/HOWTO.md @@ -6,4 +6,35 @@ docker run -v $(pwd)/persistent_volume:/code/persistent_volume -e "USE_SQLITE_AS ``` docker run --name db-test_db -e POSTGRES_PASSWORD=eee -e POSTGRES_DB=viralhostrangedb -e POSTGRES_USER=postgres -e LC_COLLATE=POSIX -p 5432:5432 -d postgres:10.1 -``` \ No newline at end of file +``` + +## How to load a specific dump in PostgreSQL + +Useful when upgrading PostgreSQL, or restoring the DB after cluster migration. + +### get a dump +Get it from the CI as it is an artifact + +### clean up +``` +export CI_COMMIT_REF_SLUG="split-deploy" + +# Stop web to prevent manage.py migrate +kubectl delete deployments -l branch=branch-$CI_COMMIT_REF_SLUG + +# Stop the db, and delete its storage +kubectl delete statefulsets,pvc -l app.kubernetes.io/instance=db-$CI_COMMIT_REF_SLUG +``` +Do keep the storage of the web container + +# load the data from the dump +``` +dev_db=$(kubectl --namespace=$NAMESPACE get po -l branch=branch-${CI_COMMIT_REF_SLUG},app=postgres-app --output jsonpath='{.items[0].metadata.name}' || echo "") +echo $dev_db +export POSTGRES_PASSWORD=toto +kubectl --namespace=$NAMESPACE exec -i $dev_db -- bash -c "PGPASSWORD=$POSTGRES_PASSWORD psql -U postgres viralhostrangedb" < prod.sql +``` + +# Restart + +Re-run CI job deploy_web_* \ No newline at end of file -- GitLab From 5a5dd77ea2a3038c04727a39f22f6df44366232b Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Tue, 4 Jan 2022 10:37:20 +0100 Subject: [PATCH 8/9] redundant with .deploy_step --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 468200b..e5e92b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -171,7 +171,6 @@ dump_prod_for_dev: .dev_settings: except: - master - image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0 variables: CI_DEBUG_TRACE: "false" NAMESPACE: "viralhostrange-dev" -- GitLab From 5192b84c9a29e32ed0d13831891296acb8c30e79 Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Tue, 4 Jan 2022 10:37:27 +0100 Subject: [PATCH 9/9] run the tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e5e92b6..d146105 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ build: stage: build needs: [] variables: - RUN_TEST: "0" + RUN_TEST: "1" POSTGRES_HOST: "db-test" STORAGE_SUFFIX: "-test" POSTGRES_PASSWORD: "example" -- GitLab