Merge branch 'split-deploy' into 'master'

split deploy in two steps

See merge request !89

.gitlab-ci.yml

@@ -64,12 +64,23 @@ build:
 
 
 
-deploy_prod:
+.deploy_step:
+  image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
+  before_script:
+    - helm repo add bitnami https://charts.bitnami.com/bitnami
+    - helm repo update
+    - pwd && ls -lah
+    - echo $NAMESPACE
+    - echo $CI_COMMIT_REF_SLUG
+    - echo $STORAGE_SUFFIX
+    - kubectl get pods -n $NAMESPACE
+    - apk add --no-cache gettext
+
+
+
+.prod_settings:
   only:
     - master
-  stage: deploy
-  needs: ["build"]
-  image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
   variables:
     CI_DEBUG_TRACE: "false"
     NAMESPACE: "viralhostrange-prod"
@@ -81,15 +92,11 @@ deploy_prod:
   environment:
     name: "k8sprod-02/viralhostrange-prod/$CI_COMMIT_REF_SLUG"
     url: "https://viralhostrangedb.pasteur.cloud"
+
+
+
+.deploy_db:
   script:
-    - helm repo add bitnami https://charts.bitnami.com/bitnami
-    - helm repo update
-    - pwd && ls -lah
-    - echo $NAMESPACE
-    - echo $CI_COMMIT_REF_SLUG
-    - echo $STORAGE_SUFFIX
-    - kubectl get pods -n $NAMESPACE
-    - apk add --no-cache gettext
     - kubectl delete secret registry-gitlab -n $NAMESPACE --ignore-not-found=true
     - kubectl create secret docker-registry -n $NAMESPACE registry-gitlab --docker-server=$CI_REGISTRY --docker-username=$DEPLOY_USER --docker-password=$DEPLOY_TOKEN --docker-email=$GITLAB_USER_EMAIL
     - envsubst < k8s/config-map.yaml | kubectl apply -n $NAMESPACE -f -
@@ -107,6 +114,11 @@ deploy_prod:
       --set primary.podLabels.branch=branch${STORAGE_SUFFIX}
       --set primary.podLabels.app=postgres-app
       --set fullnameOverride="db${STORAGE_SUFFIX}"
+
+
+
+.deploy_web:
+  script:
     - envsubst < k8s/kubernetes-storage.yaml | kubectl apply -n $NAMESPACE -f -
     - envsubst < k8s/manifest.yaml | kubectl apply -n $NAMESPACE -f -
     # - envsubst < k8s/manifest-webhost-autoscale.yaml | kubectl apply -n $NAMESPACE -f -
@@ -114,10 +126,30 @@ deploy_prod:
 
 
 
+deploy_prod_db:
+  needs: ["build"]
+  stage: deploy
+  extends:
+    - .deploy_step
+    - .prod_settings
+    - .deploy_db
+
+
+
+deploy_prod_web:
+  needs: ["build", "deploy_prod_db"]
+  stage: deploy
+  extends:
+    - .deploy_step
+    - .prod_settings
+    - .deploy_web
+
+
+
 dump_prod_for_dev:
   except:
     - master
-  stage: build
+  stage: deploy
   needs: []
   image: registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image
   variables:
@@ -136,12 +168,9 @@ dump_prod_for_dev:
 
 
 
-deploy_dev:
+.dev_settings:
   except:
     - master
-  stage: deploy
-  needs: ["build", "dump_prod_for_dev"]
-  image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
   variables:
     CI_DEBUG_TRACE: "false"
     NAMESPACE: "viralhostrange-dev"
@@ -154,40 +183,41 @@ deploy_dev:
     name: "k8sdev-01/viralhostrange-dev/$CI_COMMIT_REF_SLUG"
     url: "https://viralhostrangedb-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud"
     on_stop: stop_and_delete_in_dev
+
+
+
+deploy_dev_db:
+  needs: ["build"]
+  stage: deploy
+  extends:
+    - .deploy_step
+    - .dev_settings
+    - .deploy_db
+  after_script:
+    - . ./k8s/init_db_from_prod.sh
+
+
+
+init_dev_db:
+  needs: ["deploy_dev_db", "dump_prod_for_dev"]
+  stage: deploy
+  extends:
+    - .deploy_step
+    - .dev_settings
   script:
-    - helm repo add bitnami https://charts.bitnami.com/bitnami
-    - helm repo update
-    - pwd && ls -lah
-    - echo $NAMESPACE
-    - echo $CI_COMMIT_REF_SLUG
-    - echo $STORAGE_SUFFIX
-    - kubectl get pods -n $NAMESPACE
-    - apk add --no-cache gettext
-    - kubectl delete secret registry-gitlab -n $NAMESPACE --ignore-not-found=true
-    - kubectl create secret docker-registry -n $NAMESPACE registry-gitlab --docker-server=$CI_REGISTRY --docker-username=$DEPLOY_USER --docker-password=$DEPLOY_TOKEN --docker-email=$GITLAB_USER_EMAIL
-    - envsubst < k8s/config-map.yaml | kubectl apply -n $NAMESPACE -f -
-    - >
-      helm upgrade --install db${STORAGE_SUFFIX} bitnami/postgresql --version $CHART_VERSION --namespace=$NAMESPACE
-      --set image.tag=$POSTGRESQL_VERSION
-      --set persistence.storageClass='isilon'
-      --set persistence.enabled=true
-      --set persistence.size="5Gi"
-      --set image.pullPolicy='Always'
-      --set postgresqlUsername='postgres'
-      --set postgresqlPostgresPassword="$POSTGRES_PASSWORD"
-      --set postgresqlPassword="$POSTGRES_PASSWORD"
-      --set postgresqlDatabase='viralhostrangedb'
-      --set primary.podLabels.branch=branch${STORAGE_SUFFIX}
-      --set primary.podLabels.app=postgres-app
-      --set fullnameOverride="db${STORAGE_SUFFIX}"
     - . ./k8s/init_db_from_prod.sh
-    - envsubst < k8s/kubernetes-storage.yaml | kubectl apply -n $NAMESPACE -f -
-    - envsubst < k8s/manifest.yaml | kubectl apply -n $NAMESPACE -f -
-    # - envsubst < k8s/manifest-webhost-autoscale.yaml | kubectl apply -n $NAMESPACE -f -
-    - envsubst < k8s/kubernetes-cronjob.yaml | kubectl apply -n $NAMESPACE -f -
 
 
 
+deploy_dev_web:
+  needs: ["init_dev_db"]
+  stage: deploy
+  extends:
+    - .deploy_step
+    - .dev_settings
+    - .deploy_web
+
+
 stop_and_delete_in_dev:
   except:
     - master

src/viralhostrange/HOWTO.md

@@ -6,4 +6,35 @@ docker run -v $(pwd)/persistent_volume:/code/persistent_volume -e "USE_SQLITE_AS
 
 ```
 docker run --name db-test_db -e POSTGRES_PASSWORD=eee -e POSTGRES_DB=viralhostrangedb -e POSTGRES_USER=postgres -e LC_COLLATE=POSIX -p 5432:5432 -d postgres:10.1
-```
\ No newline at end of file
+```
+
+## How to load a specific dump in PostgreSQL
+
+Useful when upgrading PostgreSQL, or restoring the DB after cluster migration.
+
+### get a dump
+Get it from the CI as it is an artifact
+
+### clean up
+```
+export CI_COMMIT_REF_SLUG="split-deploy"
+
+# Stop web to prevent manage.py migrate
+kubectl delete deployments -l branch=branch-$CI_COMMIT_REF_SLUG
+
+# Stop the db, and delete its storage
+kubectl delete statefulsets,pvc -l app.kubernetes.io/instance=db-$CI_COMMIT_REF_SLUG
+```
+Do keep the storage of the web container
+
+# load the data from the dump
+```
+dev_db=$(kubectl --namespace=$NAMESPACE get po -l branch=branch-${CI_COMMIT_REF_SLUG},app=postgres-app --output jsonpath='{.items[0].metadata.name}' || echo "")
+echo $dev_db
+export POSTGRES_PASSWORD=toto
+kubectl --namespace=$NAMESPACE exec -i $dev_db -- bash -c "PGPASSWORD=$POSTGRES_PASSWORD psql -U postgres viralhostrangedb" < prod.sql
+```
+
+# Restart
+
+Re-run CI job deploy_web_*
\ No newline at end of file