Commit 813d2bca authored by Kenzo-Hugo Hillion's avatar Kenzo-Hugo Hillion
Browse files

Merge branch 'dev' into 'master'

Prod

See merge request !59
parents fd935b95 e03dbadd
Pipeline #32507 passed with stages
in 3 minutes and 56 seconds
.DS_Store
__pycache__/
*.egg-info/
.env
*.env
.env_dev
.idea/
.vscode
......@@ -8,6 +9,9 @@ __pycache__/
# Backend static files
backend/public
# Backend debugging folder for logs and profiling
debugging
# Frontend
node_modules/
dist/
......@@ -18,3 +22,4 @@ notebooks/
# Tests files
.coverage
.pytest_cache
......@@ -12,6 +12,7 @@ services:
variables:
POSTGRES_DB: postgres
DOCKER_HOST: tcp://localhost:2375/
DOCKER_API_VERSION: "1.39"
# This folder is cached between builds
# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
......@@ -27,37 +28,66 @@ build_backend:
tags:
- k8s
build_frontend:
build_frontend_dev:
image: docker:latest
stage: build
script:
- sh ci/build/build_frontend.sh
variables:
VUE_APP_TITLE: "[DEV] MetageneDB"
tags:
- k8s
only:
refs:
- dev
test-backend:
image: $CI_REGISTRY_IMAGE/backend:${CI_COMMIT_REF_NAME}
services:
- postgres:latest
- postgres:11.4
- redis:alpine
stage: test
variables:
DATABASE_HOST: localhost
DJANGO_SETTINGS_MODULE: "metagenedb.settings-gitlab-ci"
REDIS_HOST: "localhost"
CACHE_TTL: "0"
script:
- apt-get install -y postgresql-client
- cd backend
- pipenv install --dev --system --deploy
- flake8 --max-line-length 120
- until pg_isready -h ${DATABASE_HOST}; do echo waiting; sleep 2; done;
- pytest --cov .
deploy:
deploy-dev:
stage: deploy
image: registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image
variables:
NAMESPACE: ${CI_PROJECT_NAME}-dev
environment:
name: ${CI_PROJECT_NAME}-dev
url: https://${CI_PROJECT_NAME}-dev.k8s-dev.pasteur.fr
url: https://${CI_PROJECT_NAME}-dev.pasteur.cloud
script:
- sh ci/deploy/deploy.sh
tags:
- k8s
only:
refs:
- dev
deploy-prod:
stage: deploy
image: registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image
variables:
NAMESPACE: ${CI_PROJECT_NAME}
environment:
name: ${CI_PROJECT_NAME}
url: https://${CI_PROJECT_NAME}.pasteur.cloud
script:
- sh ci/deploy/deploy.sh
tags:
- k8s
only:
refs:
- master
MIT License
Copyright (c) 2020 Institut Pasteur
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# Metagenedb
[![pipeline status](https://gitlab.pasteur.fr/metagenomics/metagenedb/badges/master/pipeline.svg)](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/master)
[![coverage report](https://gitlab.pasteur.fr/metagenomics/metagenedb/badges/master/coverage.svg)](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/master)
[![pipeline status](https://gitlab.pasteur.fr/metagenomics/metagenedb/badges/dev/pipeline.svg)](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/dev)
[![coverage report](https://gitlab.pasteur.fr/metagenomics/metagenedb/badges/dev/coverage.svg)](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/dev)
Django based project to build genes catalog and tools
to play with it and contact external services.
......@@ -93,6 +93,11 @@ DATABASE_PORT=5432
port is redirected to `5433` on the `localhost`. This is done in order to not interfere with your local
postgres if you have one. This means you need to change `DATABASE_HOST` to `localhost` and `DATABASE_POS
### Pre-computed statistics
Some statistics about genes are pre-computed and can be accessed through the `/api/catalog/v1/statistics` endpoint.
The ID is constructed with the following format: `<statisctics-type>-<gene_source>-<method>-<options>`.
----
## Run the application
......@@ -127,3 +132,21 @@ For the moment you can:
> **Note**: You can also execute the scripts locally from a `pipenv shell` for instance. You need to make
sure that you change the way to log to postgres since the access is different from your machine compared to
from a container.
-----
## Dev tips
#### Profiling code
```python
from metagenedb.common.utils.profiling import profile
@profile("/my/file/path")
def my_function(a, b, c):
...
```
```bash
snakeviz /my/file/path
```
\ No newline at end of file
[flake8]
exclude =
*/migrations/*
*/src/bioapi/*
......@@ -5,10 +5,11 @@ ENV PYTHONUNBUFFERED 1
# Install pipenv
RUN pip install pipenv
RUN apt update && apt install vim -y
WORKDIR /code
RUN rm -rf Dockerfile
# Copy Pipfile and install
COPY . /code/
RUN pipenv install --system --deploy
RUN pipenv install --system --deploy
\ No newline at end of file
......@@ -4,50 +4,69 @@ url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
atomicwrites = "==1.3.0"
attrs = "==19.1.0"
coverage = "==4.5.3"
entrypoints = "==0.3"
flake8 = "==3.7.7"
importlib-metadata = "==0.18"
kiwisolver = "==1.1.0"
mccabe = "==0.6.1"
more-itertools = "==7.0.0"
pluggy = "==0.12.0"
py = "==1.8.0"
pycodestyle = "==2.5.0"
pyflakes = "==2.1.1"
pyparsing = "==2.4.0"
pytest = "==4.6.3"
pytest-cov = "==2.7.1"
pytest-django = "==3.5.0"
wcwidth = "==0.1.7"
zipp = "==0.5.1"
Cycler = "==0.10.0"
atomicwrites = "*"
attrs = "*"
coverage = "*"
entrypoints = "*"
flake8 = "*"
importlib-metadata = "*"
kiwisolver = "*"
mccabe = "*"
more-itertools = "*"
pluggy = "*"
py = "*"
pycodestyle = "*"
pyflakes = "*"
pyparsing = "*"
pytest = "*"
pytest-cov = "*"
pytest-django = "*"
wcwidth = "*"
zipp = "*"
Cycler = "*"
jupyter = "*"
factory-boy = "*"
pytest-factoryboy = "*"
pylint = "*"
mock = "*"
snakeviz = "*"
[packages]
certifi = "==2019.6.16"
chardet = "==3.0.4"
django-cors-headers = "==3.0.2"
django-environ = "==0.4.5"
django-extensions = "==2.1.7"
django-filter = "==2.1.0"
djangorestframework = "==3.9.4"
djangorestframework-jwt = "==1.11.0"
idna = "==2.8"
numpy = "==1.16.4"
pandas = "==0.24.2"
psycopg2 = "==2.8.2"
python-dateutil = "==2.8.0"
pytz = "==2019.1"
requests = "==2.22.0"
six = "==1.12.0"
sqlparse = "==0.3.0"
urllib3 = "==1.25.3"
Django = "==2.2.1"
PyJWT = "==1.7.1"
certifi = "*"
chardet = "*"
django-cors-headers = "*"
django-environ = "*"
django-extensions = "*"
django-filter = "*"
djangorestframework = "*"
djangorestframework-jwt = "*"
idna = "*"
numpy = "*"
pandas = "*"
psycopg2 = "*"
python-dateutil = "*"
pytz = "*"
requests = "*"
six = "*"
sqlparse = "*"
urllib3 = "*"
Django = "*"
PyJWT = "*"
metagenedb = {editable = true,path = "."}
drf-yasg = "*"
packaging = "*"
python-slugify = "*"
marshmallow = "*"
django-pandas = "*"
bioapi = {git = "https://github.com/khillion/bioapi.git"}
django-admin-list-filter-dropdown = "*"
gunicorn = "*"
pyfastx = "*"
celery = "*"
redis = "*"
django-health-check = "==3.0.0"
django-redis = "*"
kombu = "*"
[requires]
python_version = "3.7"
This diff is collapsed.
from .celery_app import app as celery_app # noqa
__all__ = ('celery_app')
from django.urls import path
from metagenedb.api.catalog.views import admin_commands
urlpatterns = [
path('compute-counts/', admin_commands.ComputeCountsAPI.as_view()),
path('compute-gene-length/', admin_commands.ComputeGeneLengthAPI.as_view()),
path('compute-taxo-repartition/', admin_commands.ComputeTaxonomyRepartitionAPI.as_view()),
]
from .function import EggNOGFilter, FunctionFilter # noqa
from .gene import GeneFilter # noqa
from .taxonomy import TaxonomyFilter # noqa
from django_filters import rest_framework as filters
from metagenedb.apps.catalog.models import Function, EggNOG
class FunctionFilter(filters.FilterSet):
class Meta:
model = Function
fields = ['source']
class EggNOGFilter(filters.FilterSet):
class Meta:
model = EggNOG
fields = ['version']
from django_filters import rest_framework as filters
from metagenedb.apps.catalog.models import Function, Gene, Taxonomy
class GeneFilter(filters.FilterSet):
has_taxonomy = filters.BooleanFilter(
field_name='taxonomy', lookup_expr="isnull", exclude=True
)
has_functions = filters.BooleanFilter(
field_name='functions', lookup_expr="isnull", distinct=True, exclude=True
)
taxonomy_rank = filters.ChoiceFilter(
choices=Taxonomy.RANK_CHOICES, field_name='taxonomy__rank'
)
name = filters.CharFilter(
field_name='name', lookup_expr='icontains'
)
length__gt = filters.NumberFilter(
field_name='length', lookup_expr='gt'
)
length__lt = filters.NumberFilter(
field_name='length', lookup_expr='lt'
)
tax_id = filters.CharFilter(
method='filter_annotated_tax'
)
function = filters.ModelMultipleChoiceFilter(
queryset=Function.objects.all(),
field_name='functions__function_id',
to_field_name='function_id'
)
def filter_annotated_tax(self, queryset, name, value):
tax_rank = Taxonomy.objects.get(tax_id=value).rank
lookup = f"taxonomy__hierarchy__{tax_rank}__tax_id"
return queryset.filter(**{lookup: value})
class Meta:
model = Gene
fields = ['length', 'name', 'source']
from django_filters import rest_framework as filters
from metagenedb.apps.catalog.models import Taxonomy
class TaxonomyFilter(filters.FilterSet):
class Meta:
model = Taxonomy
fields = ['rank', 'name']
from marshmallow import fields
from marshmallow.validate import OneOf
from metagenedb.apps.catalog import models
from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
SELECTED_SOURCE = [i[0] for i in models.Function.SOURCE_CHOICES]
EGGNOG_VERSIONS = [i[0] for i in models.EggNOG.VERSION_CHOICES]
class FunctionQueryParams(PaginatedQueryParams):
source = fields.String(validate=OneOf(choices=SELECTED_SOURCE))
class KeggQueryParams(PaginatedQueryParams):
detailed = fields.Boolean()
class EggNOGQueryParams(PaginatedQueryParams):
version = fields.String(validate=OneOf(choices=EGGNOG_VERSIONS))
from marshmallow import fields
from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
class GeneQueryParams(PaginatedQueryParams):
has_taxonomy = fields.Boolean()
has_functions = fields.Boolean()
length = fields.Integer()
length__gt = fields.Integer()
length__lt = fields.Integer()
name = fields.String()
taxonomy_rank = fields.String()
tax_id = fields.Integer()
function = fields.String()
source = fields.String()
fasta = fields.Boolean()
from marshmallow import fields
from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
class TaxonomyQueryParams(PaginatedQueryParams):
rank = fields.String()
name = fields.String()
from django.urls import path
from django.conf.urls import url, include
from rest_framework.routers import DefaultRouter
from rest_framework.routers import DefaultRouter, DynamicRoute, Route
from metagenedb.api.catalog.views import GeneViewSet
from metagenedb.api.catalog import views
# from metagenedb.api.catalog.views.celery_test import celery_test_view, test_task_view
api_router = DefaultRouter()
api_router.register(r'genes', GeneViewSet, base_name='genes')
class CustomRouter(DefaultRouter):
routes = [
# List route.
Route(
url=r'^{prefix}{trailing_slash}$',
mapping={
'get': 'list',
'put': 'update',
'post': 'create'
},
name='{basename}-list',
detail=False,
initkwargs={'suffix': 'List'}
),
# Dynamically generated list routes. Generated using
# @action(detail=False) decorator on methods of the viewset.
DynamicRoute(
url=r'^{prefix}/{url_path}{trailing_slash}$',
name='{basename}-{url_name}',
detail=False,
initkwargs={}
),
# Detail route.
Route(
url=r'^{prefix}/{lookup}{trailing_slash}$',
mapping={
'get': 'retrieve',
'put': 'update',
'patch': 'partial_update',
'delete': 'destroy'
},
name='{basename}-detail',
detail=True,
initkwargs={'suffix': 'Instance'}
),
# Dynamically generated detail routes. Generated using
# @action(detail=True) decorator on methods of the viewset.
DynamicRoute(
url=r'^{prefix}/{lookup}/{url_path}{trailing_slash}$',
name='{basename}-{url_name}',
detail=True,
initkwargs={}
),
]
api_router = CustomRouter()
api_router.register(r'functions', views.FunctionViewSet, basename='functions')
api_router.register(r'kegg-orthologies', views.KeggOrthologyViewSet, basename='kegg-orthologies')
api_router.register(r'eggnogs', views.EggNOGViewSet, basename='eggnogs')
api_router.register(r'genes', views.GeneViewSet, basename='genes')
api_router.register(r'taxonomy', views.TaxonomyViewSet, basename='taxonomy')
api_router.register(r'statistics', views.StatisticsViewSet, basename='statistics')
urlpatterns = [
url(r'v1/', include((api_router.urls, 'v1')))
url(r'v1/', include((api_router.urls, 'v1'))),
path('admin/', include(('metagenedb.api.catalog.admin_urls', 'admin'))),
# path('celery-test/', celery_test_view, name='celery-test'),
# path('celery-task-test/', test_task_view, name='celery-task-test'),
]
from .function import EggNOGViewSet, KeggOrthologyViewSet, FunctionViewSet # noqa
from .gene import GeneViewSet # noqa
from .statistics import StatisticsViewSet # noqa
from .taxonomy import TaxonomyViewSet # noqa
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment