Merge branch 'dev-fastapi' into 'dev'

merge fast-api into dev

See merge request !51

.dockerignore

 data/
+client/
+projects/
+.venv/
\ No newline at end of file

.gitattributes

+__main__.py -crlf
+doc/source/install.rst -crlf

.gitignore

@@ -77,3 +77,4 @@ data/meta_short2.csv
 data/meta_table.csv
 data/out/
 data/rawGWASfiles/
+projects/project_*
\ No newline at end of file

.gitlab-ci.yml

 stages:
+  - build
   - test
   - deploy
 
-test-centos7:
-  image: centos:centos7
-  stage: test
-  script:
-  - yum install -y epel-release wget
-  - yum install -y https://repo.ius.io/ius-release-el7.rpm
-  - yum update -y
-  - yum provides python3.6
-  - yum install -y python36u python36u-libs python36u-devel python36u-pip
-  - yum install -y blas-devel lapack-devel
-  - pip3.6 install --user pip
-  - python3.6 -m pip install virtualenv
-  - virtualenv -p python3.6 venv
-  - . venv/bin/activate
-  - yum install -y git
-  - pip3.6 install -r requirements.txt
-  - yum install -y httpd httpd-devel gcc
-  - pip3.6 install mod_wsgi
-  - python3.6 setup.py test
-
-.test-python:
+
+test-python:
+  image: python:$PYTHON_VERSION
+  needs: []
   stage: test
   script:
   - apt-get update && apt install -y libblas-dev liblapack-dev python3-dev
   - pip install -r requirements.txt
   - python setup.py test
+  parallel:
+    matrix:
+      - PYTHON_VERSION: [
+          '3.7',
+          '3.8',
+          '3.9',
+          '3.10',
+      ]
 
-test-python36:
-  extends: ".test-python"
-  image: python:3.6
 
-test-python37:
-  extends: ".test-python"
-  image: python:3.7
+test-docker-compose:
+  image: docker/compose
+  services:
+    - registry-gitlab.pasteur.fr/dsi-tools/docker-images/docker:dind
+  needs: []
+  stage: test
+  script:
+  - apk update && apk upgrade && apk add jq bash curl
+  - ./test_docker_compose.sh
+  artifacts:
+    when: always
+    paths:
+    - log
+    expire_in: 1 week
+
 
-test-python38:
-  extends: ".test-python"
-  image: python:3.8
+#test-backend-image:
+#  image: registry-gitlab.pasteur.fr/dsi-tools/docker-images/docker:latest
+#  services:
+#    - registry-gitlab.pasteur.fr/dsi-tools/docker-images/docker:dind
+#  needs:
+#    - build-backend
+#  stage: test
+#  script:
+#    - mkdir .eggs jass.egg-info && chmod -R 777 .eggs jass.egg-info
+#    - touch data/initTable.hdf5
+#    - >
+#      docker run
+#      -v $(pwd)/data:/data
+#      -v $(pwd)/projects:/projects
+#      -v $(pwd)/.eggs:/code/.eggs
+#      -v $(pwd)/jass.egg-info:/code/jass.egg-info
+#      "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/backend:$CI_COMMIT_SHA"
+#      python setup.py test
 
-test-python39:
-  extends: ".test-python"
-  image: python:3.9
 
 pages:
-  image: python:3.6
+  needs: []
+  image: python:3.9
   stage: deploy
   script:
   - pip install sphinx sphinx-argparse sphinxcontrib.bibtex sphinx_rtd_theme
@@ -87,3 +101,127 @@ deploy-webserver:
     - ansible-playbook -vvv -i ./hosts_deploy deploy.yaml --extra-vars "@params-server.json"
   only:
   - master
+
+
+
+.build:
+  stage: build
+  image: registry-gitlab.pasteur.fr/dsi-tools/docker-images/docker:latest
+  services:
+    - registry-gitlab.pasteur.fr/dsi-tools/docker-images/docker:dind
+  script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+    # pull the latest build on master
+    - docker pull "$CI_REGISTRY_IMAGE/master/${PATH_TAG}:latest" || true
+    # pull the latest build on this branch
+    - docker pull "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:latest" || true
+    # build the image while passing commit SHA and tagging the image with it
+    - docker build
+      --target ${PATH_TAG}
+      --build-arg CI_COMMIT_REF_SLUG
+      --build-arg CI_COMMIT_SHA
+      --build-arg API_URL
+      --cache-from "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:latest"
+      --cache-from "$CI_REGISTRY_IMAGE/master/${PATH_TAG}:latest"
+      --tag "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:$CI_COMMIT_SHA"
+      --tag "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:latest"
+      .
+    # push image as latest for the current branch
+    - docker push "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:latest"
+    # push image tagged with its sha
+    - docker push "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:$CI_COMMIT_SHA"
+
+
+build-backend:
+  needs: []
+  extends: .build
+  variables:
+    PATH_TAG: backend
+  after_script:
+    - mkdir .eggs jass.egg-info && chmod -R 777 .eggs jass.egg-info
+    - touch data/initTable.hdf5
+    - >
+      docker run
+      -v $(pwd)/data:/data
+      -v $(pwd)/projects:/projects
+      -v $(pwd)/.eggs:/code/.eggs
+      -v $(pwd)/jass.egg-info:/code/jass.egg-info
+      "$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/${PATH_TAG}:$CI_COMMIT_SHA"
+      python setup.py test
+
+
+build-client:
+  needs: []
+  extends: .build
+  before_script:
+    - cd client
+  variables:
+    PATH_TAG: client-static-serve
+    API_URL: "https://jass-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud/api"
+
+
+.deploy:
+  stage: deploy
+  needs:
+    - build-backend
+    - build-client
+    - test-python
+#    - test-backend-image
+  image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
+  script:
+    # create if missing the shared pvc to save data
+    - kubectl apply -n $NAMESPACE -f chart/pvc-shared-data.yaml
+    - helm dependency update chart
+    # shows some error on first deployment
+    - kubectl get secret --namespace "jass-dev" ${CI_COMMIT_REF_SLUG}-rabbitmq || NO_SECRET=$?
+    - >
+      if [ $NO_SECRET ]; then
+        RABBITMQ_PASSWORD=''
+        RABBITMQ_ERLANG_COOKIE=''
+      else
+        RABBITMQ_PASSWORD=$(kubectl get secret --namespace "jass-dev" ${CI_COMMIT_REF_SLUG}-rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
+        RABBITMQ_ERLANG_COOKIE=$(kubectl get secret --namespace "jass-dev" ${CI_COMMIT_REF_SLUG}-rabbitmq -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 -d)
+      fi
+    - >
+      helm upgrade --install --namespace=${NAMESPACE}
+      --set CI_REGISTRY_IMAGE=${CI_REGISTRY_IMAGE}
+      --set image.tag=${CI_COMMIT_SHA}
+      --set ingress.host.name=${PUBLIC_URL}
+      --set ingress.annotations."kubernetes\.io/ingress\.class"=${INGRESS_CLASS}
+      --set registry.username=${DEPLOY_USER}
+      --set registry.password=${DEPLOY_TOKEN}
+      --set registry.host=${CI_REGISTRY}
+      --set imagePullSecrets[0].name="registry-pull-secret-${CI_COMMIT_REF_SLUG}"
+      --set rabbitmq.auth.password=$RABBITMQ_PASSWORD
+      --set rabbitmq.auth.erlangCookie=$RABBITMQ_ERLANG_COOKIE
+      ${CI_COMMIT_REF_SLUG} ./chart/
+
+
+deploy-dev:
+  stage: deploy
+  extends: .deploy
+  variables:
+    NAMESPACE: "jass-dev"
+    PUBLIC_URL: "jass-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud"
+    INGRESS_CLASS: "internal"
+  environment:
+    name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}"
+    url: "https://jass-${CI_COMMIT_REF_SLUG}.dev.pasteur.cloud"
+    on_stop: delete-dev-deployment
+
+
+delete-dev-deployment:
+  stage: deploy
+  needs: []
+  when: manual
+  image: harbor.pasteur.fr/kube-system/helm-kubectl:3.4.0
+  variables:
+    GIT_STRATEGY: none # important to not checkout source when branch is deleted
+    NAMESPACE: "jass-dev"
+  environment:
+    name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}"
+    action: stop
+  script:
+    - echo "Removing $CI_COMMIT_REF_SLUG"
+    - helm delete -n ${NAMESPACE} ${CI_COMMIT_REF_SLUG}
+    - kubectl delete pvc -lapp.kubernetes.io/name=rabbitmq,app.kubernetes.io/instance=${CI_COMMIT_REF_SLUG}
\ No newline at end of file

Dockerfile

-FROM centos:7
-
-RUN yum install -y epel-release wget gcc https://repo.ius.io/ius-release-el7.rpm
-RUN yum update -y
-RUN yum install -y python36u python36u-libs python36u-devel python36u-pip openssl-devel  libffi-devel
-RUN pip3.6 install --upgrade pip
-RUN pip3.6 install ansible
-RUN yum install -y python-pip
-RUN python -m pip install --upgrade pip
-COPY . /code
-WORKDIR /code/ansible
-RUN ansible-playbook -vvv -i ./hosts_local deploy.yaml --extra-vars "@params-docker.json"
-RUN pip3.6 install -U watchdog[watchmedo]
-CMD ["jass", "serve"]
+FROM python:3.9 as backend
+
+ENV JASS_DATA_DIR '/data'
+ENV JASS_PROJECTS_DIR '/projects'
+
+EXPOSE 8080
+
+CMD ["uvicorn", "jass.server:app", "--host", "0.0.0.0", "--port", "8080"]
+
+RUN addgroup --gid 1000 kiwi \
+ && adduser --disabled-password --gecos '' --uid 1000 --gid 1000 kiwi \
+ && apt-get update \
+ && apt-get install -y \
+        nano \
+        wget \
+        rsync \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip install --upgrade pip setuptools \
+ && mkdir /code \
+ && mkdir ${JASS_DATA_DIR} \
+ && chown kiwi:kiwi ${JASS_DATA_DIR} \
+ && mkdir ${JASS_PROJECTS_DIR} \
+ && chown kiwi:kiwi ${JASS_PROJECTS_DIR}
+WORKDIR /code
+
+COPY requirements*.txt /code/
+RUN pip install -r requirements.txt \
+ && pip3 install -U watchdog[watchmedo]
+
+COPY ./*-entrypoint.sh /
+RUN chmod a+x /*-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+COPY . /code/
+RUN pip install -e .
+
+USER kiwi
\ No newline at end of file

ansible/celeryd

 CELERYD_NODES="worker"
 CELERY_BIN="celery"
-CELERY_APP="jass.models.project"
+CELERY_APP="jass"
 CELERYD_CHDIR="/var/jass_data"
 CELERYD_OPTS="--time-limit=3000 --concurrency=1 --max-tasks-per-child=1"
 CELERYD_LOG_FILE="/var/jass_data/celery%N.log"

chart/Chart.yaml

+apiVersion: v2
+name: jass
+description: jass Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 1.0.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 2.1.0
+
+# list dependencies
+dependencies:
+- name: rabbitmq
+  version: "8.30"
+  repository: "https://charts.bitnami.com/bitnami"

chart/pvc-shared-data.yaml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "jass-shared-data-files"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 15Gi
+  storageClassName: isilon
+status: {}
\ No newline at end of file

chart/rabbitmq.conf

+log.console.level = error

chart/sidekick.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: sidekick
+spec:
+  containers:
+    - name: sidekick
+      image: nginxinc/nginx-unprivileged:latest
+      volumeMounts:
+        - mountPath: /data
+          name: shared-data-files
+  volumes:
+    - name: shared-data-files
+      persistentVolumeClaim:
+        claimName: "jass-shared-data-files"
\ No newline at end of file

chart/templates/NOTES.txt

+# Application will be available at http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .Values.ingress.host.name }}
+
+# To delete app, do:
+helm delete {{.Release.Name}}
\ No newline at end of file

chart/templates/_helpers.tpl

+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "chart.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "chart.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "chart.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "chart.labels" -}}
+helm.sh/chart: {{ include "chart.chart" . }}
+{{ include "chart.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "chart.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "chart.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{- define "chart.fullnameImagePullSecret" -}}
+{{- printf "%s-%s" (include "chart.fullname" .) .Values.imagePullSecret.name }}
+{{- end }}

chart/templates/_secretHelper.tpl

+{{- define "imagePullSecret" }}
+{{- if .Values.registry }}
+{{- if and .Values.registry.password .Values.registry.username }}
+{{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.registry.host (printf "%s:%s" .Values.registry.username .Values.registry.password | b64enc) | b64enc }}
+{{- end }}
+{{- end }}
+{{- end }}

chart/templates/cronjob-back-command.yaml

+{{- $root := . -}}
+{{- range $task := .Values.backCronTasks.tasks }}
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: {{ printf "%s-%s" (include "chart.fullname" $root) (join "-" $task.command | replace "_" "-") | trunc 52 | trimSuffix "-" }}
+  labels:
+    {{- include "chart.labels" $root | nindent 4 }}
+    app: back-cron-job
+spec:
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 2
+  schedule: "{{ $task.periodicity }}"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+            - name: "backend"
+              image: {{ printf "%s/%s/%s:%s" $root.Values.CI_REGISTRY_IMAGE $root.Release.Name "backend" $root.Values.image.tag }}
+              imagePullPolicy: Always
+              securityContext:
+                {{- toYaml $root.Values.securityContext | nindent 16 }}
+              env:
+                - name: RABBITMQ_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ printf "%s-rabbitmq" $root.Release.Name }}
+                      key: rabbitmq-password
+                - name: RABBITMQ_USER
+                  value: 'user'
+                - name: RABBITMQ_HOST
+                  value: {{ printf "%s-rabbitmq" $root.Release.Name }}
+                - name: RABBITMQ_PORT
+                  value: '5672'
+              command:
+                {{- range $cmd := $task.command }}
+                - '{{ $cmd }}'
+                {{- end }}
+              resources:
+                {{- if $task.resources -}}
+                {{- toYaml $task.resources | nindent 16 }}
+                {{- else if $root.Values.backCronTasks.resources -}}
+                {{- toYaml $root.Values.backCronTasks.resources | nindent 16 }}
+                {{- else -}}
+                {{- toYaml $root.Values.back.resources | nindent 16 }}
+                {{- end }}
+              volumeMounts:
+                - mountPath: /projects
+                  name: projects-files
+                - mountPath: /data
+                  name: shared-data-files
+          volumes:
+            - name: projects-files
+              persistentVolumeClaim:
+                claimName: {{ printf "%s-projects-files" $root.Release.Name }}
+            - name: shared-data-files
+              persistentVolumeClaim:
+                claimName: "jass-shared-data-files"
+---
+{{- end }}
\ No newline at end of file

chart/templates/deployment-back.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "back" }}
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+    app: back
+spec:
+  selector:
+    matchLabels:
+      {{- include "chart.selectorLabels" . | nindent 6 }}
+      app: back
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+    {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      labels:
+        {{- include "chart.selectorLabels" . | nindent 8 }}
+        app: back
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - name: back
+        image: {{ printf "%s/%s/%s:%s" .Values.CI_REGISTRY_IMAGE .Release.Name "backend" .Values.image.tag }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}
+        env:
+          - name: RABBITMQ_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: {{ printf "%s-rabbitmq" .Release.Name }}
+                key: rabbitmq-password
+          - name: RABBITMQ_USER
+            value: 'user'
+          - name: RABBITMQ_HOST
+            value: {{ printf "%s-rabbitmq" .Release.Name }}
+          - name: RABBITMQ_PORT
+            value: '5672'
+        resources:
+          {{- toYaml .Values.back.resources | nindent 12 }}
+        volumeMounts:
+          - mountPath: /projects
+            name: projects-files
+          - mountPath: /data
+            name: shared-data-files
+      volumes:
+        - name: projects-files
+          persistentVolumeClaim:
+            claimName: {{ printf "%s-projects-files" .Release.Name }}
+        - name: shared-data-files
+          persistentVolumeClaim:
+            claimName: "jass-shared-data-files"
\ No newline at end of file

chart/templates/deployment-celery-worker.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "celery" }}
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+    app: celery-worker
+spec:
+  selector:
+    matchLabels:
+      {{- include "chart.selectorLabels" . | nindent 6 }}
+      app: celery-worker
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+    {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      labels:
+        {{- include "chart.selectorLabels" . | nindent 8 }}
+        app: celery-worker
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - name: celery
+        image: {{ printf "%s/%s/%s:%s" .Values.CI_REGISTRY_IMAGE .Release.Name "backend" .Values.image.tag }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}
+        env:
+          - name: RABBITMQ_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: {{ printf "%s-rabbitmq" .Release.Name }}
+                key: rabbitmq-password
+          - name: RABBITMQ_USER
+            value: 'user'
+          - name: RABBITMQ_HOST
+            value: {{ printf "%s-rabbitmq" .Release.Name }}
+          - name: RABBITMQ_PORT
+            value: '5672'
+        command:
+          - "celery"
+          - "-A"
+          - "jass"
+          - "worker"
+          {{- if .Values.celery.concurrency }}
+          - "--concurrency={{ .Values.celery.concurrency }}"
+          {{- end }}
+          {{- if .Values.celery.maxTasksPerChild }}
+          - "--max-tasks-per-child={{ .Values.celery.maxTasksPerChild }}"
+          {{- end }}
+        resources:
+          {{- toYaml .Values.celery.resources | nindent 12 }}
+        volumeMounts:
+          - mountPath: /projects
+            name: projects-files