Commit d4e1f059 authored by Bryan  BRANCOTTE's avatar Bryan BRANCOTTE
Browse files

Merge branch 'split-deploy' into 'master'

split deploy in two steps

See merge request !89
parents 507f8200 5192b84c
......@@ -64,12 +64,23 @@ build:
deploy_prod:
.deploy_step:
image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
before_script:
- helm repo add bitnami https://charts.bitnami.com/bitnami
- helm repo update
- pwd && ls -lah
- echo $NAMESPACE
- echo $CI_COMMIT_REF_SLUG
- echo $STORAGE_SUFFIX
- kubectl get pods -n $NAMESPACE
- apk add --no-cache gettext
.prod_settings:
only:
- master
stage: deploy
needs: ["build"]
image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
variables:
CI_DEBUG_TRACE: "false"
NAMESPACE: "viralhostrange-prod"
......@@ -81,15 +92,11 @@ deploy_prod:
environment:
name: "k8sprod-02/viralhostrange-prod/$CI_COMMIT_REF_SLUG"
url: "https://viralhostrangedb.pasteur.cloud"
.deploy_db:
script:
- helm repo add bitnami https://charts.bitnami.com/bitnami
- helm repo update
- pwd && ls -lah
- echo $NAMESPACE
- echo $CI_COMMIT_REF_SLUG
- echo $STORAGE_SUFFIX
- kubectl get pods -n $NAMESPACE
- apk add --no-cache gettext
- kubectl delete secret registry-gitlab -n $NAMESPACE --ignore-not-found=true
- kubectl create secret docker-registry -n $NAMESPACE registry-gitlab --docker-server=$CI_REGISTRY --docker-username=$DEPLOY_USER --docker-password=$DEPLOY_TOKEN --docker-email=$GITLAB_USER_EMAIL
- envsubst < k8s/config-map.yaml | kubectl apply -n $NAMESPACE -f -
......@@ -107,6 +114,11 @@ deploy_prod:
--set primary.podLabels.branch=branch${STORAGE_SUFFIX}
--set primary.podLabels.app=postgres-app
--set fullnameOverride="db${STORAGE_SUFFIX}"
.deploy_web:
script:
- envsubst < k8s/kubernetes-storage.yaml | kubectl apply -n $NAMESPACE -f -
- envsubst < k8s/manifest.yaml | kubectl apply -n $NAMESPACE -f -
# - envsubst < k8s/manifest-webhost-autoscale.yaml | kubectl apply -n $NAMESPACE -f -
......@@ -114,10 +126,30 @@ deploy_prod:
deploy_prod_db:
needs: ["build"]
stage: deploy
extends:
- .deploy_step
- .prod_settings
- .deploy_db
deploy_prod_web:
needs: ["build", "deploy_prod_db"]
stage: deploy
extends:
- .deploy_step
- .prod_settings
- .deploy_web
dump_prod_for_dev:
except:
- master
stage: build
stage: deploy
needs: []
image: registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image
variables:
......@@ -136,12 +168,9 @@ dump_prod_for_dev:
deploy_dev:
.dev_settings:
except:
- master
stage: deploy
needs: ["build", "dump_prod_for_dev"]
image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
variables:
CI_DEBUG_TRACE: "false"
NAMESPACE: "viralhostrange-dev"
......@@ -154,40 +183,41 @@ deploy_dev:
name: "k8sdev-01/viralhostrange-dev/$CI_COMMIT_REF_SLUG"
url: "https://viralhostrangedb-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud"
on_stop: stop_and_delete_in_dev
deploy_dev_db:
needs: ["build"]
stage: deploy
extends:
- .deploy_step
- .dev_settings
- .deploy_db
after_script:
- . ./k8s/init_db_from_prod.sh
init_dev_db:
needs: ["deploy_dev_db", "dump_prod_for_dev"]
stage: deploy
extends:
- .deploy_step
- .dev_settings
script:
- helm repo add bitnami https://charts.bitnami.com/bitnami
- helm repo update
- pwd && ls -lah
- echo $NAMESPACE
- echo $CI_COMMIT_REF_SLUG
- echo $STORAGE_SUFFIX
- kubectl get pods -n $NAMESPACE
- apk add --no-cache gettext
- kubectl delete secret registry-gitlab -n $NAMESPACE --ignore-not-found=true
- kubectl create secret docker-registry -n $NAMESPACE registry-gitlab --docker-server=$CI_REGISTRY --docker-username=$DEPLOY_USER --docker-password=$DEPLOY_TOKEN --docker-email=$GITLAB_USER_EMAIL
- envsubst < k8s/config-map.yaml | kubectl apply -n $NAMESPACE -f -
- >
helm upgrade --install db${STORAGE_SUFFIX} bitnami/postgresql --version $CHART_VERSION --namespace=$NAMESPACE
--set image.tag=$POSTGRESQL_VERSION
--set persistence.storageClass='isilon'
--set persistence.enabled=true
--set persistence.size="5Gi"
--set image.pullPolicy='Always'
--set postgresqlUsername='postgres'
--set postgresqlPostgresPassword="$POSTGRES_PASSWORD"
--set postgresqlPassword="$POSTGRES_PASSWORD"
--set postgresqlDatabase='viralhostrangedb'
--set primary.podLabels.branch=branch${STORAGE_SUFFIX}
--set primary.podLabels.app=postgres-app
--set fullnameOverride="db${STORAGE_SUFFIX}"
- . ./k8s/init_db_from_prod.sh
- envsubst < k8s/kubernetes-storage.yaml | kubectl apply -n $NAMESPACE -f -
- envsubst < k8s/manifest.yaml | kubectl apply -n $NAMESPACE -f -
# - envsubst < k8s/manifest-webhost-autoscale.yaml | kubectl apply -n $NAMESPACE -f -
- envsubst < k8s/kubernetes-cronjob.yaml | kubectl apply -n $NAMESPACE -f -
deploy_dev_web:
needs: ["init_dev_db"]
stage: deploy
extends:
- .deploy_step
- .dev_settings
- .deploy_web
stop_and_delete_in_dev:
except:
- master
......
......@@ -6,4 +6,35 @@ docker run -v $(pwd)/persistent_volume:/code/persistent_volume -e "USE_SQLITE_AS
```
docker run --name db-test_db -e POSTGRES_PASSWORD=eee -e POSTGRES_DB=viralhostrangedb -e POSTGRES_USER=postgres -e LC_COLLATE=POSIX -p 5432:5432 -d postgres:10.1
```
\ No newline at end of file
```
## How to load a specific dump in PostgreSQL
Useful when upgrading PostgreSQL, or restoring the DB after cluster migration.
### get a dump
Get it from the CI as it is an artifact
### clean up
```
export CI_COMMIT_REF_SLUG="split-deploy"
# Stop web to prevent manage.py migrate
kubectl delete deployments -l branch=branch-$CI_COMMIT_REF_SLUG
# Stop the db, and delete its storage
kubectl delete statefulsets,pvc -l app.kubernetes.io/instance=db-$CI_COMMIT_REF_SLUG
```
Do keep the storage of the web container
# load the data from the dump
```
dev_db=$(kubectl --namespace=$NAMESPACE get po -l branch=branch-${CI_COMMIT_REF_SLUG},app=postgres-app --output jsonpath='{.items[0].metadata.name}' || echo "")
echo $dev_db
export POSTGRES_PASSWORD=toto
kubectl --namespace=$NAMESPACE exec -i $dev_db -- bash -c "PGPASSWORD=$POSTGRES_PASSWORD psql -U postgres viralhostrangedb" < prod.sql
```
# Restart
Re-run CI job deploy_web_*
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment