Merge branch 'dev' into 'master'

Prod

See merge request !59

.gitignore

+.DS_Store
 __pycache__/
 *.egg-info/
-.env
+*.env
 .env_dev
 .idea/
 .vscode
@@ -8,6 +9,9 @@ __pycache__/
 # Backend static files
 backend/public
 
+# Backend debugging folder for logs and profiling
+debugging
+
 # Frontend
 node_modules/
 dist/
@@ -18,3 +22,4 @@ notebooks/
 
 # Tests files
 .coverage
+.pytest_cache

.gitlab-ci.yml

@@ -12,6 +12,7 @@ services:
 variables:
   POSTGRES_DB: postgres
   DOCKER_HOST: tcp://localhost:2375/
+  DOCKER_API_VERSION: "1.39"
 
 # This folder is cached between builds
 # http://docs.gitlab.com/ce/ci/yaml/README.html#cache
@@ -27,37 +28,66 @@ build_backend:
   tags:
     - k8s
 
-build_frontend:
+build_frontend_dev:
   image: docker:latest
   stage: build
   script:
     - sh ci/build/build_frontend.sh
+  variables:
+    VUE_APP_TITLE: "[DEV] MetageneDB"
   tags:
     - k8s
+  only:
+    refs:
+      - dev
 
 test-backend:
   image: $CI_REGISTRY_IMAGE/backend:${CI_COMMIT_REF_NAME}
   services:
-    - postgres:latest
+    - postgres:11.4
+    - redis:alpine
   stage: test
   variables:
     DATABASE_HOST: localhost
     DJANGO_SETTINGS_MODULE: "metagenedb.settings-gitlab-ci"
+    REDIS_HOST: "localhost"
+    CACHE_TTL: "0"
   script:
+    - apt-get install -y postgresql-client
     - cd backend
     - pipenv install --dev --system --deploy
     - flake8 --max-line-length 120
+    - until pg_isready -h ${DATABASE_HOST}; do echo waiting; sleep 2; done;
     - pytest --cov .
 
-deploy:
+deploy-dev:
   stage: deploy
   image: registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image
   variables:
     NAMESPACE: ${CI_PROJECT_NAME}-dev
   environment:
     name: ${CI_PROJECT_NAME}-dev
-    url: https://${CI_PROJECT_NAME}-dev.k8s-dev.pasteur.fr
+    url: https://${CI_PROJECT_NAME}-dev.pasteur.cloud
+  script:
+    - sh ci/deploy/deploy.sh
+  tags:
+    - k8s
+  only:
+    refs:
+      - dev
+
+deploy-prod:
+  stage: deploy
+  image: registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image
+  variables:
+    NAMESPACE: ${CI_PROJECT_NAME}
+  environment:
+    name: ${CI_PROJECT_NAME}
+    url: https://${CI_PROJECT_NAME}.pasteur.cloud
   script:
     - sh ci/deploy/deploy.sh
   tags:
     - k8s
+  only:
+    refs:
+      - master

LICENSE

+MIT License
+
+Copyright (c) 2020 Institut Pasteur
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

 # Metagenedb
 
-[![pipeline status](https://gitlab.pasteur.fr/metagenomics/metagenedb/badges/master/pipeline.svg)](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/master)
-[![coverage report](https://gitlab.pasteur.fr/metagenomics/metagenedb/badges/master/coverage.svg)](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/master)
+[![pipeline status](https://gitlab.pasteur.fr/metagenomics/metagenedb/badges/dev/pipeline.svg)](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/dev)
+[![coverage report](https://gitlab.pasteur.fr/metagenomics/metagenedb/badges/dev/coverage.svg)](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/dev)
 
 Django based project to build genes catalog and tools
 to play with it and contact external services.
@@ -93,6 +93,11 @@ DATABASE_PORT=5432
 port is redirected to `5433` on the `localhost`. This is done in order to not interfere with your local
 postgres if you have one. This means you need to change `DATABASE_HOST` to `localhost` and `DATABASE_POS
 
+### Pre-computed statistics
+
+Some statistics about genes are pre-computed and can be accessed through the `/api/catalog/v1/statistics` endpoint.
+The ID is constructed with the following format: `<statisctics-type>-<gene_source>-<method>-<options>`.
+
 ----
 
 ## Run the application
@@ -127,3 +132,21 @@ For the moment you can:
 > **Note**: You can also execute the scripts locally from a `pipenv shell` for instance. You need to make
 sure that you change the way to log to postgres since the access is different from your machine compared to
 from a container.
+
+-----
+
+## Dev tips
+
+#### Profiling code
+
+```python
+from metagenedb.common.utils.profiling import profile
+
+@profile("/my/file/path")
+def my_function(a, b, c):
+    ...
+```
+
+```bash
+snakeviz /my/file/path
+```
\ No newline at end of file

backend/.flake8

 [flake8]
 exclude =
     */migrations/*
+    */src/bioapi/*

backend/Dockerfile

@@ -5,10 +5,11 @@ ENV PYTHONUNBUFFERED 1
 
 # Install pipenv
 RUN pip install pipenv
+RUN apt update && apt install vim -y
 
 WORKDIR /code
 RUN rm -rf Dockerfile
 
 # Copy Pipfile and install
 COPY . /code/
-RUN pipenv install --system --deploy
+RUN pipenv install --system --deploy
\ No newline at end of file

backend/Pipfile

@@ -4,50 +4,69 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-atomicwrites = "==1.3.0"
-attrs = "==19.1.0"
-coverage = "==4.5.3"
-entrypoints = "==0.3"
-flake8 = "==3.7.7"
-importlib-metadata = "==0.18"
-kiwisolver = "==1.1.0"
-mccabe = "==0.6.1"
-more-itertools = "==7.0.0"
-pluggy = "==0.12.0"
-py = "==1.8.0"
-pycodestyle = "==2.5.0"
-pyflakes = "==2.1.1"
-pyparsing = "==2.4.0"
-pytest = "==4.6.3"
-pytest-cov = "==2.7.1"
-pytest-django = "==3.5.0"
-wcwidth = "==0.1.7"
-zipp = "==0.5.1"
-Cycler = "==0.10.0"
+atomicwrites = "*"
+attrs = "*"
+coverage = "*"
+entrypoints = "*"
+flake8 = "*"
+importlib-metadata = "*"
+kiwisolver = "*"
+mccabe = "*"
+more-itertools = "*"
+pluggy = "*"
+py = "*"
+pycodestyle = "*"
+pyflakes = "*"
+pyparsing = "*"
+pytest = "*"
+pytest-cov = "*"
+pytest-django = "*"
+wcwidth = "*"
+zipp = "*"
+Cycler = "*"
 jupyter = "*"
+factory-boy = "*"
+pytest-factoryboy = "*"
+pylint = "*"
+mock = "*"
+snakeviz = "*"
 
 [packages]
-certifi = "==2019.6.16"
-chardet = "==3.0.4"
-django-cors-headers = "==3.0.2"
-django-environ = "==0.4.5"
-django-extensions = "==2.1.7"
-django-filter = "==2.1.0"
-djangorestframework = "==3.9.4"
-djangorestframework-jwt = "==1.11.0"
-idna = "==2.8"
-numpy = "==1.16.4"
-pandas = "==0.24.2"
-psycopg2 = "==2.8.2"
-python-dateutil = "==2.8.0"
-pytz = "==2019.1"
-requests = "==2.22.0"
-six = "==1.12.0"
-sqlparse = "==0.3.0"
-urllib3 = "==1.25.3"
-Django = "==2.2.1"
-PyJWT = "==1.7.1"
+certifi = "*"
+chardet = "*"
+django-cors-headers = "*"
+django-environ = "*"
+django-extensions = "*"
+django-filter = "*"
+djangorestframework = "*"
+djangorestframework-jwt = "*"
+idna = "*"
+numpy = "*"
+pandas = "*"
+psycopg2 = "*"
+python-dateutil = "*"
+pytz = "*"
+requests = "*"
+six = "*"
+sqlparse = "*"
+urllib3 = "*"
+Django = "*"
+PyJWT = "*"
 metagenedb = {editable = true,path = "."}
+drf-yasg = "*"
+packaging = "*"
+python-slugify = "*"
+marshmallow = "*"
+django-pandas = "*"
+bioapi = {git = "https://github.com/khillion/bioapi.git"}
+django-admin-list-filter-dropdown = "*"
+gunicorn = "*"
+pyfastx = "*"
+celery = "*"
+redis = "*"
+django-health-check = "==3.0.0"
+django-redis = "*"
+kombu = "*"
 
 [requires]
 python_version = "3.7"

backend/metagenedb/__init__.py

+from .celery_app import app as celery_app  # noqa
+
+__all__ = ('celery_app')

backend/metagenedb/api/catalog/admin_urls.py

+from django.urls import path
+
+from metagenedb.api.catalog.views import admin_commands
+
+
+urlpatterns = [
+    path('compute-counts/', admin_commands.ComputeCountsAPI.as_view()),
+    path('compute-gene-length/', admin_commands.ComputeGeneLengthAPI.as_view()),
+    path('compute-taxo-repartition/', admin_commands.ComputeTaxonomyRepartitionAPI.as_view()),
+]

backend/metagenedb/api/catalog/filters/__init__.py

+from .function import EggNOGFilter, FunctionFilter  # noqa
+from .gene import GeneFilter  # noqa
+from .taxonomy import TaxonomyFilter  # noqa

backend/metagenedb/api/catalog/filters/function.py

+from django_filters import rest_framework as filters
+from metagenedb.apps.catalog.models import Function, EggNOG
+
+
+class FunctionFilter(filters.FilterSet):
+
+    class Meta:
+        model = Function
+        fields = ['source']
+
+
+class EggNOGFilter(filters.FilterSet):
+
+    class Meta:
+        model = EggNOG
+        fields = ['version']

backend/metagenedb/api/catalog/filters/gene.py

+from django_filters import rest_framework as filters
+from metagenedb.apps.catalog.models import Function, Gene, Taxonomy
+
+
+class GeneFilter(filters.FilterSet):
+    has_taxonomy = filters.BooleanFilter(
+        field_name='taxonomy', lookup_expr="isnull", exclude=True
+    )
+    has_functions = filters.BooleanFilter(
+        field_name='functions', lookup_expr="isnull", distinct=True, exclude=True
+    )
+    taxonomy_rank = filters.ChoiceFilter(
+        choices=Taxonomy.RANK_CHOICES, field_name='taxonomy__rank'
+    )
+    name = filters.CharFilter(
+        field_name='name', lookup_expr='icontains'
+    )
+    length__gt = filters.NumberFilter(
+        field_name='length', lookup_expr='gt'
+    )
+    length__lt = filters.NumberFilter(
+        field_name='length', lookup_expr='lt'
+    )
+    tax_id = filters.CharFilter(
+        method='filter_annotated_tax'
+    )
+    function = filters.ModelMultipleChoiceFilter(
+        queryset=Function.objects.all(),
+        field_name='functions__function_id',
+        to_field_name='function_id'
+    )
+
+    def filter_annotated_tax(self, queryset, name, value):
+        tax_rank = Taxonomy.objects.get(tax_id=value).rank
+        lookup = f"taxonomy__hierarchy__{tax_rank}__tax_id"
+        return queryset.filter(**{lookup: value})
+
+    class Meta:
+        model = Gene
+        fields = ['length', 'name', 'source']

backend/metagenedb/api/catalog/filters/taxonomy.py

+from django_filters import rest_framework as filters
+from metagenedb.apps.catalog.models import Taxonomy
+
+
+class TaxonomyFilter(filters.FilterSet):
+
+    class Meta:
+        model = Taxonomy
+        fields = ['rank', 'name']

backend/metagenedb/api/catalog/qparams_validators/function.py

+from marshmallow import fields
+from marshmallow.validate import OneOf
+
+from metagenedb.apps.catalog import models
+from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
+
+SELECTED_SOURCE = [i[0] for i in models.Function.SOURCE_CHOICES]
+EGGNOG_VERSIONS = [i[0] for i in models.EggNOG.VERSION_CHOICES]
+
+
+class FunctionQueryParams(PaginatedQueryParams):
+    source = fields.String(validate=OneOf(choices=SELECTED_SOURCE))
+
+
+class KeggQueryParams(PaginatedQueryParams):
+    detailed = fields.Boolean()
+
+
+class EggNOGQueryParams(PaginatedQueryParams):
+    version = fields.String(validate=OneOf(choices=EGGNOG_VERSIONS))

backend/metagenedb/api/catalog/qparams_validators/gene.py

+from marshmallow import fields
+
+from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
+
+
+class GeneQueryParams(PaginatedQueryParams):
+    has_taxonomy = fields.Boolean()
+    has_functions = fields.Boolean()
+    length = fields.Integer()
+    length__gt = fields.Integer()
+    length__lt = fields.Integer()
+    name = fields.String()
+    taxonomy_rank = fields.String()
+    tax_id = fields.Integer()
+    function = fields.String()
+    source = fields.String()
+    fasta = fields.Boolean()

backend/metagenedb/api/catalog/qparams_validators/taxonomy.py

+from marshmallow import fields
+
+from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
+
+
+class TaxonomyQueryParams(PaginatedQueryParams):
+    rank = fields.String()
+    name = fields.String()

backend/metagenedb/api/catalog/urls.py

+from django.urls import path
 from django.conf.urls import url, include
-from rest_framework.routers import DefaultRouter
+from rest_framework.routers import DefaultRouter, DynamicRoute, Route
 
-from metagenedb.api.catalog.views import GeneViewSet
+from metagenedb.api.catalog import views
 
+# from metagenedb.api.catalog.views.celery_test import celery_test_view, test_task_view
 
-api_router = DefaultRouter()
-api_router.register(r'genes', GeneViewSet, base_name='genes')
+
+class CustomRouter(DefaultRouter):
+
+    routes = [
+        # List route.
+        Route(
+            url=r'^{prefix}{trailing_slash}$',
+            mapping={
+                'get': 'list',
+                'put': 'update',
+                'post': 'create'
+            },
+            name='{basename}-list',
+            detail=False,
+            initkwargs={'suffix': 'List'}
+        ),
+        # Dynamically generated list routes. Generated using
+        # @action(detail=False) decorator on methods of the viewset.
+        DynamicRoute(
+            url=r'^{prefix}/{url_path}{trailing_slash}$',
+            name='{basename}-{url_name}',
+            detail=False,
+            initkwargs={}
+        ),
+        # Detail route.
+        Route(
+            url=r'^{prefix}/{lookup}{trailing_slash}$',
+            mapping={
+                'get': 'retrieve',
+                'put': 'update',
+                'patch': 'partial_update',
+                'delete': 'destroy'
+            },
+            name='{basename}-detail',
+            detail=True,
+            initkwargs={'suffix': 'Instance'}
+        ),
+        # Dynamically generated detail routes. Generated using
+        # @action(detail=True) decorator on methods of the viewset.
+        DynamicRoute(
+            url=r'^{prefix}/{lookup}/{url_path}{trailing_slash}$',
+            name='{basename}-{url_name}',
+            detail=True,
+            initkwargs={}
+        ),
+    ]
+
+
+api_router = CustomRouter()
+api_router.register(r'functions', views.FunctionViewSet, basename='functions')
+api_router.register(r'kegg-orthologies', views.KeggOrthologyViewSet, basename='kegg-orthologies')
+api_router.register(r'eggnogs', views.EggNOGViewSet, basename='eggnogs')
+api_router.register(r'genes', views.GeneViewSet, basename='genes')
+api_router.register(r'taxonomy', views.TaxonomyViewSet, basename='taxonomy')
+api_router.register(r'statistics', views.StatisticsViewSet, basename='statistics')
 
 
 urlpatterns = [
-    url(r'v1/', include((api_router.urls, 'v1')))
+    url(r'v1/', include((api_router.urls, 'v1'))),
+    path('admin/', include(('metagenedb.api.catalog.admin_urls', 'admin'))),
+    # path('celery-test/', celery_test_view, name='celery-test'),
+    # path('celery-task-test/', test_task_view, name='celery-task-test'),
 ]

backend/metagenedb/api/catalog/views/__init__.py

+from .function import EggNOGViewSet, KeggOrthologyViewSet, FunctionViewSet  # noqa
 from .gene import GeneViewSet  # noqa
+from .statistics import StatisticsViewSet  # noqa
+from .taxonomy import TaxonomyViewSet  # noqa