Commit 913a7120 authored by Bryan  BRANCOTTE's avatar Bryan BRANCOTTE
Browse files

Merge branch 'dockerizing-app' into 'master'

Dockerize app, and k8s

Closes #269, #272, and #271

See merge request !38
parents 9d9beec9 954aa70d
Pipeline #83522 failed with stages
in 9 minutes and 13 seconds
......@@ -19,3 +19,5 @@ db.sqlite3
#bryan custom
*IPPIDB_PPI.xlsx*
#end bryan custom
/chart/Chart.lock
/chart/charts/
image: registry-gitlab.pasteur.fr/dsi-tools/docker-images/docker:latest
stages:
- build
- test
- deploy
test-style:
image: python:3.9
stage: test
needs: []
script:
- apt update && apt install -y apache2-dev graphviz graphviz-dev
- cd ippisite
- pip install flake8
- flake8 --config=.flake8
- flake8 --config=.flake8
.test:
image: python:3.9
stage: test
needs: ["build-ippisite"]
script:
- apt-get update && apt-get install -y nano wget gettext python3-dev cron apache2-dev
- python -m ensurepip
- apt-get update && apt-get install -y python3-openbabel
- export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages"
- export PYTHONUNBUFFERED=1
- cd ippisite
- pip install -r requirements.txt
- apt-get install -y libgraphviz-dev
- pip install -r requirements-dev.txt
- python manage.py migrate
- python manage.py test
- coverage run --source='.' manage.py test
- coverage report
- coverage html
- export LC_CTYPE="en_US.UTF-8"
- cd docs
- make html
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
- mkdir -p persistent && chmod -R 777 persistent
- >
docker run
--network host
-e USE_SQLITE_AS_DB=$USE_SQLITE_AS_DB
-e POSTGRES_PASSWORD=$POSTGRES_PASSWORD
-e POSTGRES_HOST=db-local
-e POSTGRES_DB=$POSTGRES_DB
-e POSTGRES_USER=$POSTGRES_USER
-v $(pwd)/persistent:/code/persistent
"$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/web-container:$CI_COMMIT_SHA"
test
- mv persistent/coverage.xml coverage.xml
- mv persistent/htmlcov htmlcov
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: coverage.xml
paths:
- htmlcov/*
- persistent/tests_http_cache.sqlite
expire_in: 1 week
cache:
key: all-branch
paths:
- ippisite/htmlcov
- ippisite/docs/build/html
- persistent/tests_http_cache.sqlite
test-sqlite:
# As testing for sqlite is only informative, we only run it for master
only:
- master
extends: ".test"
# needs: ["test-pg"] # needs should be set to test-pg when cache is not working in gitlab-ci to spare network resources
allow_failure: true # test is mostly informative, sqlite is not in the scope
variables:
USE_SQLITE_AS_DB: "true"
services:
- registry-gitlab.pasteur.fr/dsi-tools/docker-images/docker:dind
test-pg:
extends: ".test"
variables:
POSTGRES_NAME: "ippidb"
POSTGRES_DB: "ippidb"
POSTGRES_USER: "ippidb"
POSTGRES_PASSWORD: "ippidb"
POSTGRES_HOST: "postgres"
services:
- postgres:14
- name: postgres:14
alias: "db-local"
- registry-gitlab.pasteur.fr/dsi-tools/docker-images/docker:dind
pages:
stage: deploy
......@@ -65,3 +87,136 @@ pages:
only:
- master
.build:
stage: build
services:
- registry-gitlab.pasteur.fr/dsi-tools/docker-images/docker:dind
script:
- ls -lah
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
# pull the latest build on master
- docker pull "$CI_REGISTRY_IMAGE/master/web-container:latest" || true
- docker pull "$CI_REGISTRY_IMAGE/master/celery-container:latest" || true
# pull the latest build on this branch
- docker pull "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/web-container:latest" || true
- docker pull "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/celery-container:latest" || true
# build the image while passing commit SHA and tagging the image with it
- docker build
--target web-container
--build-arg CI_COMMIT_REF_SLUG
--build-arg CI_COMMIT_SHA
--cache-from "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/web-container:latest"
--cache-from "$CI_REGISTRY_IMAGE/master/web-container:latest"
--tag "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/web-container:$CI_COMMIT_SHA"
--tag "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/web-container:latest"
.
# push image as latest for the current branch
- docker push "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/web-container:latest"
# push image tagged with its sha
- docker push "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/web-container:$CI_COMMIT_SHA"
# build the image while passing commit SHA and tagging the image with it
- docker build
--target celery-container
--build-arg CI_COMMIT_REF_SLUG
--build-arg CI_COMMIT_SHA
--cache-from "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/celery-container:latest"
--cache-from "$CI_REGISTRY_IMAGE/master/celery-container:latest"
--cache-from "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/web-container:latest"
--cache-from "$CI_REGISTRY_IMAGE/master/web-container:latest"
--tag "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/celery-container:$CI_COMMIT_SHA"
--tag "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/celery-container:latest"
.
# push image as latest for the current branch
- docker push "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/celery-container:latest"
# push image tagged with its sha
- docker push "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/celery-container:$CI_COMMIT_SHA"
build-ippisite:
needs: []
extends: .build
before_script:
- cd ippisite
.deploy:
stage: deploy
needs:
- "test-pg"
image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
script:
- helm dependency update chart
- >
helm upgrade --install --namespace=${NAMESPACE}
--set CI_REGISTRY_IMAGE=${CI_REGISTRY_IMAGE}
--set image.tag=${CI_COMMIT_SHA}
--set ingress.host.name=${PUBLIC_URL}
--set ingress.annotations."kubernetes\.io/ingress\.class"=${INGRESS_CLASS}
--set registry.username=${DEPLOY_USER}
--set registry.password=${DEPLOY_TOKEN}
--set registry.host=${CI_REGISTRY}
--set imagePullSecrets[0].name="registry-pull-secret-${CI_COMMIT_REF_SLUG}"
--set djangoSecrets.MARVINJS_APIKEY=$(echo "$MARVINJS_APIKEY_master" | base64)
--set djangoSecrets.GACODE=$(echo "$GACODE_master" | base64)
--set djangoSecrets.GALAXY_APIKEY=$(echo "$GALAXY_APIKEY_master" | base64)
--set djangoSecrets.GALAXY_BASE_URL=$(echo "$GALAXY_BASE_URL_master" | base64)
--set djangoSecrets.GALAXY_COMPOUNDPROPERTIES_WORKFLOWID_master=$(echo "$GALAXY_COMPOUNDPROPERTIES_WORKFLOWID_master" | base64)
${CI_COMMIT_REF_SLUG} ./chart/
cache:
key: charts-all-branch
paths:
- chart/charts
deploy-dev:
stage: deploy
needs:
- "test-pg"
extends: .deploy
except:
- release
variables:
NAMESPACE: "ippidb-dev"
PUBLIC_URL: "ippidb-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud"
INGRESS_CLASS: "internal"
environment:
name: "k8sdev-01/ippidb-dev/${CI_COMMIT_REF_SLUG}"
url: "https://ippidb-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud"
on_stop: delete-dev-deployment
deploy-prod:
stage: deploy
needs:
- "test-pg"
extends: .deploy
only:
- release
variables:
NAMESPACE: "ippidb-prod"
PUBLIC_URL: "ippidb-internal.pasteur.cloud"
INGRESS_CLASS: "internal"
environment:
name: "k8sprod-02/ippidb-prod/${CI_COMMIT_REF_SLUG}"
url: "https://ippidb-internal.pasteur.cloud"
delete-dev-deployment:
stage: deploy
needs: []
except:
- release
when: manual
image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
variables:
GIT_STRATEGY: none # important to not checkout source when branch is deleted
NAMESPACE: "ippidb-dev"
environment:
name: "k8sdev-01/ippidb-dev/${CI_COMMIT_REF_SLUG}"
action: stop
script:
- echo "Removing $CI_COMMIT_REF_SLUG"
- helm delete -n ${NAMESPACE} ${CI_COMMIT_REF_SLUG}
- kubectl delete pvc -n ${NAMESPACE} -lapp.kubernetes.io/instance=${CI_COMMIT_REF_SLUG}
\ No newline at end of file
......@@ -11,4 +11,39 @@ pip install -q flake8
flake8 --install-hook git
git config --bool flake8.strict true
ln -s ippisite/.flake8 .flake8
```
## Copy data to helm container
```shell
# the first time, create a context, safer and easier than specifying the namespace a each command
export K8S_USER=$USER
kubectl config set-context ippidb-dev --cluster=k8sdev-01 --user=$K8S_USER@k8sdev-01 --namespace ippidb-dev
kubectl config set-context ippidb-prod --cluster=k8sprod-02 --user=$K8S_USER@k8sprod-02 --namespace ippidb-prod
```
```shell
kubectl config use-context ippidb-dev
POD=$(kubectl get po -lapp=web-deployment,app.kubernetes.io/instance=$(git branch --show) --output jsonpath='{.items[0].metadata.name}')
echo $POD
date
kubectl cp ippisite/persistent/media/ $POD:/code/persistent/media --container=django # 50 minutes
date
kubectl config use-context ippidb-dev
POD=$(kubectl get po -lapp.kubernetes.io/name=postgresqlbitnami,app.kubernetes.io/instance=$(git branch --show) --output jsonpath='{.items[0].metadata.name}'); echo $POD
date
kubectl cp ippisite/db-django-4.0.sql.gz $POD:/bitnami/postgresql/ # 1 minutes
# once in the pod run
# gzip -d -c /bitnami/postgresql/db-django-4.0.sql.gz | psql -U postgres
kubectl exec -it $POD -- bash
date
kubectl config use-context ippidb-dev
POD=$(kubectl get po -lapplication=spilo,cluster-name=$(git branch --show)-postgresql --output jsonpath='{.items[0].metadata.name}'); echo $POD
date
# 5 minutes
gzip -d -c ippisite/db-django-4.0.sql.gz | kubectl exec --stdin --tty $POD -- psql
date
```
\ No newline at end of file
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
apiVersion: v2
name: ippidb
description: ippidb Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 1.0.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
appVersion: 1.0.0
# list dependencies
dependencies:
- name: postgresql
version: "11"
repository: "https://charts.bitnami.com/bitnami"
condition: postgresqlBackend.bitnami.enabled
alias: postgresqlbitnami
- name: redis
version: "16.5"
repository: "https://charts.bitnami.com/bitnami"
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
http {
proxy_temp_path /tmp/proxy_temp;
client_body_temp_path /tmp/client_temp;
fastcgi_temp_path /tmp/fastcgi_temp;
uwsgi_temp_path /tmp/uwsgi_temp;
scgi_temp_path /tmp/scgi_temp;
include /etc/nginx/mime.types;
client_max_body_size 100m;
server {
listen 8080;
root /usr/src/app/;
location / {
autoindex on;
}
}
}
# Application will be available at http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .Values.ingress.host.name }}
# To delete app, do:
helm delete {{.Release.Name}}
kubectl delete pvc -lapp.kubernetes.io/name=postgresql
kubectl delete pvc -lapp.kubernetes.io/name=redis
# You also can delete media dir:
kubectl delete pvc -lapp.kubernetes.io/name=django
\ No newline at end of file
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "chart.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "chart.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "chart.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "chart.labels" -}}
helm.sh/chart: {{ include "chart.chart" . }}
{{ include "chart.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "chart.selectorLabels" -}}
app.kubernetes.io/name: {{ include "chart.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{- define "chart.fullnameImagePullSecret" -}}
{{- printf "%s-%s" (include "chart.fullname" .) .Values.imagePullSecret.name }}
{{- end }}
{{- define "imagePullSecret" }}
{{- if .Values.registry }}
{{- if and .Values.registry.password .Values.registry.username }}
{{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.registry.host (printf "%s:%s" .Values.registry.username .Values.registry.password | b64enc) | b64enc }}
{{- end }}
{{- end }}
{{- end }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "chart.fullname" . }}-nginx-config
data:
{{- $files := .Files }}
{{- range tuple "nginx.conf" }}
{{ . }}: |-
{{ $files.Get . | indent 4 }}
{{- end }}
\ No newline at end of file
{{- $root := . -}}
{{- range $task := .Values.djangoCronTasks.tasks }}
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: {{ include "chart.fullname" $root }}-{{ $task.command | replace "_" "-"}}
labels:
{{- include "chart.labels" $root | nindent 4 }}
app: {{ $root.Values.django.name }}
spec:
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 2
schedule: "{{ $task.periodicity }}"
concurrencyPolicy: Forbid
jobTemplate:
spec:
template:
spec:
restartPolicy: Never
containers:
- name: {{ printf "%s-%s" $root.Release.Name $root.Values.django.name }}
image: {{ printf "%s/%s/%s:%s" $root.Values.CI_REGISTRY_IMAGE $root.Release.Name $root.Values.django.name $root.Values.image.tag }}
imagePullPolicy: Always
securityContext:
{{- toYaml $root.Values.securityContext | nindent 16 }}
env:
- name: DJANGO_DEBUG
value: "true"
- name: POSTGRES_USER
value: "postgres"
- name: SECRET_KEY
value: {{ $root.Values.django.secretKey }}
- name: ALLOWED_HOSTS
value: {{ .Values.ingress.host.name | quote }}
command:
- "python"
- "manage.py"
- "{{ $task.command }}"
resources:
{{- if $task.resources -}}
{{- toYaml $task.resources | nindent 16 }}
{{- else if $root.Values.djangoCronTasks.resources -}}
{{- toYaml $root.Values.djangoCronTasks.resources | nindent 16 }}
{{- else -}}
{{- toYaml $root.Values.django.resources | nindent 16 }}
{{- end }}
volumeMounts:
- mountPath: /code/persistent/
name: persistent-files
volumes:
{{- if .Values.django.persistence.enabled }}
- name: persistent-files
persistentVolumeClaim:
claimName: {{ printf "%s-media-files" .Release.Name }}
{{- end }}
---
{{- end }}
\ No newline at end of file
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ printf "%s-%s" .Release.Name "celery" }}
labels:
{{- include "chart.labels" . | nindent 4 }}
app: celery-worker
spec:
selector:
matchLabels:
{{- include "chart.selectorLabels" . | nindent 6 }}
app: celery-worker
strategy:
type: Recreate
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "chart.selectorLabels" . | nindent 8 }}
app: celery-worker
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: celery
image: {{ printf "%s/%s/%s:%s" .Values.CI_REGISTRY_IMAGE .Release.Name "celery-container" .Values.image.tag }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
# command: ["/docker-celery-entrypoint.sh", "hold_on"]
env:
- name: DJANGO_DEBUG
value: "false"
- name: POSTGRES_HOST
value: {{ printf "%s-postgresql" .Release.Name }}{{- if .Values.postgresqlBackend.bitnami.enabled }}bitnami{{end}}
- name: POSTGRES_DB
value: "postgres"
{{- if .Values.postgresqlBackend.bitnami.enabled }}
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ printf "%s-postgresqlbitnami" .Release.Name }}
key: postgres-password
{{end}}
{{- if .Values.postgresqlBackend.zalando.enabled }}
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: {{ printf "postgres.%s-postgresql.credentials.postgresql.acid.zalan.do" .Release.Name }}
key: password
{{end}}
- name: POSTGRES_USER
value: "postgres"
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ printf "%s-redis" .Release.Name }}
key: redis-password
- name: ALLOWED_HOSTS
value: {{ .Values.ingress.host.name | quote }}
- name: REDIS_HOSTNAME
value: {{ printf "%s-redis-master" .Release.Name }}
{{- $root := . -}}
{{- range tuple "MARVINJS_APIKEY" "GALAXY_APIKEY" "GALAXY_BASE_URL" "GALAXY_COMPOUNDPROPERTIES_WORKFLOWID" "GACODE" "SECRET_KEY" }}
- name: {{ . }}
valueFrom:
secretKeyRef:
name: {{ printf "%s-django-secret" $root.Release.Name }}
key: {{ . }}
{{- end }}
resources:
{{- toYaml .Values.celery.resources | nindent 12 }}
volumeMounts:
{{- if .Values.django.persistence.enabled }}
- mountPath: /code/persistent
name: persistent-files
{{- end }}
volumes:
{{- if .Values.django.persistence.enabled }}
- name: persistent-files
persistentVolumeClaim:
claimName: {{ printf "%s-media-files" .Release.Name }}
{{- end }}
\ No newline at end of file
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ printf "%s-%s" .Release.Name "web" }}
labels:
{{- include "chart.labels" . | nindent 4 }}
app: web-deployment
spec:
selector:
matchLabels:
{{- include "chart.selectorLabels" . | nindent 6 }}
app: web-deployment
strategy:
type: Recreate
template:
metadata: