Change image link for test

get back to latest postgres

change service to test only

remove k8s

.gitignore

@@ -10,6 +10,7 @@ backend/public
 
 # Frontend
 node_modules/
+dist/
 
 # Jupyter notebook
 notebooks/

.gitlab-ci.yml

-image: python:3.7
-
 stages:
-  - lint_test_coverage_backend
+  - build
+  - test
+  - deploy
 
 # Pick zero or more services to be used on all builds.
 # Only needed when using a docker container to run your tests in.
 # Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
 services:
-  - postgres:latest
+  - docker:dind
 
 variables:
   POSTGRES_DB: postgres
+  DOCKER_HOST: tcp://localhost:2375/
 
 # This folder is cached between builds
 # http://docs.gitlab.com/ce/ci/yaml/README.html#cache
@@ -18,16 +19,45 @@ cache:
   paths:
   - ~/.cache/pip/
 
-before_script:
-  - pip install pipenv
-  - cd backend
-  - pipenv install --dev --system --deploy
+build_backend:
+  image: docker:latest
+  stage: build
+  script:
+    - sh ci/build/build_backend.sh
+  tags:
+    - k8s
+
+build_frontend:
+  image: docker:latest
+  stage: build
+  script:
+    - sh ci/build/build_frontend.sh
+  tags:
+    - k8s
 
 test-backend:
-  stage: lint_test_coverage_backend
+  image: $CI_REGISTRY_IMAGE/backend:${CI_COMMIT_REF_NAME}
+  services:
+    - postgres:latest
+  stage: test
   variables:
-    DATABASE_HOST: postgres
+    DATABASE_HOST: localhost
     DJANGO_SETTINGS_MODULE: "metagenedb.settings-gitlab-ci"
   script:
+    - cd backend
+    - pipenv install --dev --system --deploy
     - flake8 --max-line-length 120
     - pytest --cov .
+
+deploy:
+  stage: deploy
+  image: registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image
+  variables:
+    NAMESPACE: ${CI_PROJECT_NAME}-dev
+  environment:
+    name: ${CI_PROJECT_NAME}-dev
+    url: https://${CI_PROJECT_NAME}-dev.k8s-dev.pasteur.fr
+  script:
+    - sh ci/deploy/deploy.sh
+  tags:
+    - k8s

README.md

@@ -6,11 +6,11 @@
 Django based project to build genes catalog and tools
 to play with it and contact external services.
 
-## Run the services on your local machine
+----
 
-### Prerequisites
+## Setup the services on your local machine
 
-#### Dependencies
+### Dependencies
 
 The application depends on different services that run independently on docker images and all of this is
 orchestrated by `docker-compose`.
@@ -20,16 +20,82 @@ Therefore to run the application you need:
 * `Docker` : [Install instructions](https://docs.docker.com/install/)
 * `Docker Compose` : [Install instructions](https://docs.docker.com/compose/install/)
 
-#### Configuration
+### Configuration
 
-For `docker-compose`, you need to create a `.env` file: `touch .env`.
+For `docker-compose`, you need to create a `.env` file: `touch .env`. An example is available: `.env.sample`.
 
 The settings of the Django server is based on the `backend/.env` file. You can copy the sample file
 (`cp backend/.env.sample backend/.env`) and fill in the variables.
 
 You can of course customize more of the Django server settings in the `settings` module of metagenedb.
 
-### Run the application
+Now we will go through the different parts
+
+#### Secret key
+
+This is the Django `SECRET_KEY` and you need to specify your own. For instance you can use the command
+`openssl rand -base64 32` to generate one by command line.
+
+#### Create your own DB on postgresql
+
+The following variables have the default value:
+
+```bash
+DATABASE_HOST=postgresql
+DATABASE_USER=postgres
+DATABASE_NAME=postgres
+DATABASE_PASSWORD=""
+DATABASE_PORT=5432
+```
+
+It will work if you leave it as it is but you might face security issues having a by default database
+without credentials.
+
+What we recommand is to create your own database. Here is described one way to do it. To do that you need to
+first run the db image and identify its running ID:
+
+```bash
+khillion:~/metagenedb $ docker-compose up postgresql -d  # This runs only the postgresql service of your docker-compose in detached mode. You can also detached from you running screen using Ctrl+Z
+Creating postgresql ... done
+khillion:~/metagenedb $ docker ps  # List your running docker images
+CONTAINER ID        IMAGE                  COMMAND                  CREATED             STATUS              PORTS                    NAMES
+5002f210f9d8        postgres:11.4-alpine   "docker-entrypoint.s…"   1 minute ago      Up 1 minute       0.0.0.0:5433->5432/tcp   postgresql
+```
+
+Now that you have the `CONTAINER ID`, here `5002f210f9d8` you can run a `bash` terminal in this container and
+create your own database:
+
+```bash
+khillion:~/metagenedb $ docker exec -it 5002f210f9d8 bash
+bash-5.0# psql --user=postgres
+````
+
+This will open the `SQL` console where you can do what you need:
+
+```psql
+CREATE ROLE metagenedb WITH PASSWORD 'yourawesomepassword';
+ALTER ROLE metagenedb WITH CREATEDB;
+CREATE DATABASE metagenedb WITH OWNER metagenedb;
+exit
+```
+
+Now you have you own database, protected by a password and you need to update your `.env`:
+
+```bash
+DATABASE_HOST=postgresql
+DATABASE_USER=metagenedb
+DATABASE_NAME=metagenedb
+DATABASE_PASSWORD=yourawesomepassword
+DATABASE_PORT=5432
+```
+
+> **Note**: The by default port for postgres is `5432`. In the `docker-compose.yaml` you will notice that this
+port is redirected to `5433` on the `localhost`. This is done in order to not interfere with your local
+postgres if you have one. This means you need to change `DATABASE_HOST` to `localhost` and `DATABASE_POS
+
+----
+
+## Run the application
 
 For the moment, only the `docker-compose.dev.yaml` is used. To run the application simply run the command:
 
@@ -43,7 +109,7 @@ container to be built again.
 Since directories with source codes are mounted in the containers, changes you make locally should be
 directly reflected on the application.
 
-#### Populate the database
+### Populate the database
 
 You have a set of scripts available within the `backend/scripts` directory that you can execute directly
 from within the container. First identify the container ID corresponding to the backend with `docker ps` command. Then you can execute a bash terminal within the container and execute the scripts you want:
@@ -57,3 +123,7 @@ For the moment you can:
 
 * Import all kegg orthologies with `load_kegg.py`: It directly fetch all KEGGs KO from the KEGG REST API.
 * Import genes from IGC catalog from the [annotation file](ftp://ftp.cngb.org/pub/SciRAID/Microbiome/humanGut_9.9M/GeneAnnotation/IGC.annotation_OF.summary.gz). You can a small part of this annotation file in the `dev_data` folder.
+
+> **Note**: You can also execute the scripts locally from a `pipenv shell` for instance. You need to make
+sure that you change the way to log to postgres since the access is different from your machine compared to
+from a container.

backend/.env.sample

 DEBUG=True
-SECRET_KEY="your secret"
-DATABASES_HOST=db
-DATABASES_POSTGRES_PORT=5432
+PORT=8000
+DATABASE_HOST=postgresql
+DATABASE_USER=postgres
+DATABASE_NAME=postgres
+DATABASE_PASSWORD=""
+DATABASE_PORT=5432
+SECRET_KEY=YOUR_KEY

backend/Dockerfile

@@ -7,6 +7,7 @@ ENV PYTHONUNBUFFERED 1
 RUN pip install pipenv
 
 WORKDIR /code
+RUN rm -rf Dockerfile
 
 # Copy Pipfile and install
 COPY . /code/

backend/metagenedb/settings-gitlab-ci.py

@@ -6,7 +6,7 @@ DATABASES = {
         'NAME': 'ci',
         'USER': 'postgres',
         'PASSWORD': 'postgres',
-        'HOST': 'postgres',
+        'HOST': 'localhost',
         'PORT': '5432',
     },
 }

backend/metagenedb/settings/django.py

@@ -119,10 +119,11 @@ if DEBUG:
 DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql',
-       'NAME': 'postgres',
-       'USER': 'postgres',
-       'HOST': env.str('DATABASES_HOST', default='db'),
-       'PORT': env.int('DATABASES_POSTGRES_PORT', default=5432),
+       'NAME': env.str('DATABASE_NAME', default='postgres'),
+       'USER': env.str('DATABASE_USER', default='postgres'),
+       'PASSWORD': env.str('DATABASE_PASSWORD', default=''),
+       'HOST': env.str('DATABASE_HOST', default='postgresql'),
+       'PORT': env.int('DATABASE_PORT', default=5432),
    }
 }
 

backend/scripts/populate_db/import_ncbi_taxonomy.py

@@ -39,6 +39,7 @@ def create_taxo_nodes(taxonomy_nodes_file, taxo_name_dict):
     _LOGGER.info(f"Create taxonomy objects from {taxonomy_nodes_file}...")
     FOREIGN_KEY_FIELDS = ['parent_tax_id']
     with open(taxonomy_nodes_file, "r") as file:
+        cpt = 0
         for i in file:
             node = NCBITaxonomyLineParser.node(i)
             node['name'] = taxo_name_dict.get(node.get('tax_id'), "No name")
@@ -47,21 +48,31 @@ def create_taxo_nodes(taxonomy_nodes_file, taxo_name_dict):
             serializer = TaxonomySerializer(data=node)
             if serializer.is_valid():
                 serializer.save()
+                cpt += 1
             else:
                 _LOGGER.warning(f"Invalid data: {serializer.errors}. Insertion skipped. Data: {serializer.data}")
+            if cpt % 10000 == 0:
+                _LOGGER.info(f"{cpt} taxonomies created...")
+    _LOGGER.info(f"[DONE] {cpt} taxonomies created.")
 
 
 def update_taxo_nodes(taxonomy_nodes_file):
     _LOGGER.info(f"Linking taxonomy objects to direct parental node from {taxonomy_nodes_file}...")
+    all_taxo_count = Taxonomy.objects.select_related(SELECT_RELATED_PARENT).all().count()
     with open(taxonomy_nodes_file, "r") as file:
+        cpt = 0
         for i in file:
             node = NCBITaxonomyLineParser.node(i)
             taxo_obj = Taxonomy.objects.get(tax_id=node.get('tax_id'))
             serializer = TaxonomySerializer(taxo_obj, data=node)
             if serializer.is_valid():
                 serializer.save()
+                cpt += 1
             else:
                 _LOGGER.warning(f"Invalid data: {serializer.errors}. Link to parent skipped. Data: {serializer.data}")
+            if cpt % 10000 == 0:
+                _LOGGER.info(f"{cpt}/{all_taxo_count} taxonomies updated...")
+    _LOGGER.info(f"[DONE] {cpt}/{all_taxo_count} taxonomies updated.")
 
 
 def _build_hierarchy(taxo):
@@ -88,6 +99,7 @@ def build_all_hierarchy(chunk_size=8000):
         cpt += 1
         if cpt % 10000 == 0:
             _LOGGER.info(f"{cpt}/{all_taxo.count()} hierachies built...")
+    _LOGGER.info(f"[DONE] {cpt}/{all_taxo.count()} hierachies built.")
 
 
 def parse_arguments():

backend/scripts/start.sh

@@ -2,7 +2,9 @@
 
 SCRIPTS_PATH=scripts
 
+if [ -z $PORT ];then PORT=8000;fi  # Need to fix to get value from .env file
+
 python ${SCRIPTS_PATH}/manage.py collectstatic --no-input
-# python ${SCRIPTS_PATH}/manage.py makemigrations
+python ${SCRIPTS_PATH}/manage.py makemigrations
 python ${SCRIPTS_PATH}/manage.py migrate
-python ${SCRIPTS_PATH}/manage.py runserver 0.0.0.0:8000
+python ${SCRIPTS_PATH}/manage.py runserver 0.0.0.0:${PORT}

ci/build/build_backend.sh

+#!/bin/sh
+
+docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+docker build -t "$CI_REGISTRY_IMAGE/backend:${CI_COMMIT_REF_NAME}" backend/
+docker push "$CI_REGISTRY_IMAGE/backend:${CI_COMMIT_REF_NAME}"

ci/build/build_frontend.sh

+#!/bin/sh
+
+docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+docker build -t "$CI_REGISTRY_IMAGE/frontend:${CI_COMMIT_REF_NAME}" frontend/
+docker push "$CI_REGISTRY_IMAGE/frontend:${CI_COMMIT_REF_NAME}"

ci/deploy/deploy.sh

+#!/bin/sh
+
+yum install -y gettext
+
+# Secrets
+## Gitlab registry
+kubectl delete secret registry-gitlab -n ${NAMESPACE} --ignore-not-found=true
+kubectl create secret docker-registry -n ${NAMESPACE} registry-gitlab --docker-server=registry-gitlab.pasteur.fr --docker-username=${DEPLOY_USER} --docker-password=${DEPLOY_TOKEN} --docker-email=kubernetes@pasteur.fr
+## SECRET_KEY for Django
+kubectl delete secret backend-secret -n ${NAMESPACE} --ignore-not-found=true
+kubectl create secret generic backend-secret -n ${NAMESPACE} --from-literal=secret_key=${SECRET_KEY}
+
+# Deployement
+## DB
+envsubst < ci/kubernetes/postgresql.yaml | kubectl apply -n ${NAMESPACE} -f -
+kubectl -n ${NAMESPACE} wait --for=condition=available --timeout=600s deployment/postgresql
+## Backend
+envsubst < ci/kubernetes/backend.yaml | kubectl apply -n ${NAMESPACE} -f -
+kubectl -n ${NAMESPACE} patch deployment backend -p "{\"spec\":{\"template\":{\"metadata\":{\"labels\":{\"date\":\"`date +'%s'`\"}}}}}"
+## Frontend
+envsubst < ci/kubernetes/frontend.yaml | kubectl apply -n ${NAMESPACE} -f -
+kubectl -n ${NAMESPACE} patch deployment frontend -p "{\"spec\":{\"template\":{\"metadata\":{\"labels\":{\"date\":\"`date +'%s'`\"}}}}}"
+## Ingress
+envsubst < ci/kubernetes/ingress.yaml | kubectl apply -n ${NAMESPACE} -f -

ci/kubernetes/backend.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backend
+  labels:
+      app: backend
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: backend
+  template:
+    metadata:
+      labels:
+        app: backend
+    spec:
+      containers:
+      - name: backend-app
+        image: $CI_REGISTRY_IMAGE/backend:${CI_COMMIT_REF_NAME}
+        imagePullPolicy: Always
+        args:
+        - /code/scripts/start.sh
+        env:
+        - name: DATABASE_HOST
+          value: postgresql
+        - name: DATABASE_NAME
+          valueFrom:
+            secretKeyRef:
+              name: postgresql-credentials
+              key: database
+        - name: DATABASE_USER
+          valueFrom:
+            secretKeyRef:
+              name: postgresql-credentials
+              key: username
+        # - name: DATABASE_PASSWORD
+        #   valueFrom:
+        #     secretKeyRef:
+        #       name: postgresql-credentials
+        #       key: password
+        - name: SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: backend-secret
+              key: secret_key
+        - name: PORT
+          value: "8000"
+        - name: DEBUG
+          value: "True"
+        ports:
+        - containerPort: 8000
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "250m"
+          limits:
+            memory: "256Mi"
+            cpu: "500m"  
+      imagePullSecrets:
+        - name: registry-gitlab
+      volumes:
+      - name: postgresql-credentials
+        secret:
+          secretName: postgresql-credentials
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: backend
+  labels:
+    app: backend
+spec:
+  type: ClusterIP
+  ports:
+  - port: 8000
+  selector:
+    app: backend

ci/kubernetes/frontend.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frontend
+  labels:
+      app: frontend
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: frontend
+  template:
+    metadata:
+      labels:
+        app: frontend
+    spec:
+      containers:
+      - name: frontend-app
+        image: $CI_REGISTRY_IMAGE/frontend:${CI_COMMIT_REF_NAME}
+        imagePullPolicy: Always
+        env:
+        - name: NODE_ENV
+          value: "development"
+        ports:
+        - containerPort: 80
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "250m"
+          limits:
+            memory: "256Mi"
+            cpu: "500m"
+      imagePullSecrets:
+        - name: registry-gitlab
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: frontend
+  labels:
+    app: frontend
+spec:
+  type: ClusterIP
+  ports:
+  - port: 80
+  selector:
+    app: frontend

ci/kubernetes/ingress.yaml

+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  annotations:
+    kubernetes.io/ingress.class: traefik
+  labels:
+    app: metagenedb
+  name: metagenedb
+spec:
+  rules:
+  - host: ${CI_PROJECT_NAME}-dev.k8s-dev.pasteur.fr
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: frontend
+          servicePort: 80
+      - path: /admin
+        backend:
+          serviceName: backend
+          servicePort: 8000
+      - path: /static
+        backend:
+          serviceName: backend
+          servicePort: 8000
+      - path: /catalog
+        backend:
+          serviceName: backend
+          servicePort: 8000
\ No newline at end of file

ci/kubernetes/postgresql.yaml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-claim
+  labels:
+      app: postgresql
+spec:
+  accessModes:
+      - ReadWriteOnce
+  resources:
+      requests:
+        storage: 8Gi
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgresql-credentials
+type: Opaque
+data:
+  username: bWV0YWdlbmVkYl91c2VyCg==
+  password: bWV0YWdlbmVkYl9kZXYK
+  database: bWV0YWdlbmVkYl9kZXYK
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgresql
+  labels:
+    app: postgresql
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgresql
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: postgresql
+        tier: postgreSQL
+    spec:
+      containers:
+      - name: postgresql
+        image: postgres:11.4-alpine
+        env:
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: postgresql-credentials
+              key: username
+        - name: POSTGRES_DB
+          valueFrom:
+            secretKeyRef:
+              name: postgresql-credentials
+              key: database
+        # - name: POSTGRES_PASSWORD
+        #   valueFrom:
+        #     secretKeyRef:
+        #       name: postgresql-credentials
+        #       key: password
+        ports:
+        - containerPort: 5432
+          name: postgresql
+        resources:
+          requests:
+            memory: "64Mi"
+            cpu: "50m"
+          limits:
+            memory: "128Mi"
+            cpu: "100m"
+        volumeMounts:
+        - name: postgresql
+          mountPath: /var/lib/postgresql/data
+          subPath: data
+      volumes:
+      - name: postgresql
+        persistentVolumeClaim:
+          claimName: postgres-claim
+      - name: postgresql-credentials
+        secret:
+          secretName: postgresql-credentials
+      securityContext:
+        runAsUser: 65534
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgresql
+  labels:
+    app: postgresql
+spec:
+  ports:
+    - port: 5432
+  selector:
+    app: postgresql
+    tier: postgreSQL

docker-compose.yaml

-version: '3.7'
+version: '3'
 
 services:
   backend:
@@ -14,25 +14,26 @@ services:
     environment:
       DEBUG: "true"
     depends_on:
-      - db
+      - postgresql
     command: /code/scripts/start.sh
     networks:
       - main
 
-  db:
+  postgresql:
+    container_name: postgresql
+    image: postgres:11.4-alpine
     shm_size: '2gb'
-    container_name: db
-    image: postgres:latest
     ports:
       - "5433:5432"
     volumes:
-      - db-data:/var/lib/postgresql/data