Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Metagenomics
metagenedb
Commits
813d2bca
Commit
813d2bca
authored
Jun 18, 2020
by
Kenzo-Hugo Hillion
♻
Browse files
Merge branch 'dev' into 'master'
Prod See merge request
!59
parents
fd935b95
e03dbadd
Pipeline
#32507
passed with stages
in 3 minutes and 56 seconds
Changes
252
Pipelines
1
Expand all
Hide whitespace changes
Inline
Side-by-side
.gitignore
View file @
813d2bca
.DS_Store
__pycache__/
*.egg-info/
.env
*
.env
.env_dev
.idea/
.vscode
...
...
@@ -8,6 +9,9 @@ __pycache__/
# Backend static files
backend/public
# Backend debugging folder for logs and profiling
debugging
# Frontend
node_modules/
dist/
...
...
@@ -18,3 +22,4 @@ notebooks/
# Tests files
.coverage
.pytest_cache
.gitlab-ci.yml
View file @
813d2bca
...
...
@@ -12,6 +12,7 @@ services:
variables
:
POSTGRES_DB
:
postgres
DOCKER_HOST
:
tcp://localhost:2375/
DOCKER_API_VERSION
:
"
1.39"
# This folder is cached between builds
# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
...
...
@@ -27,37 +28,66 @@ build_backend:
tags
:
-
k8s
build_frontend
:
build_frontend
_dev
:
image
:
docker:latest
stage
:
build
script
:
-
sh ci/build/build_frontend.sh
variables
:
VUE_APP_TITLE
:
"
[DEV]
MetageneDB"
tags
:
-
k8s
only
:
refs
:
-
dev
test-backend
:
image
:
$CI_REGISTRY_IMAGE/backend:${CI_COMMIT_REF_NAME}
services
:
-
postgres:latest
-
postgres:11.4
-
redis:alpine
stage
:
test
variables
:
DATABASE_HOST
:
localhost
DJANGO_SETTINGS_MODULE
:
"
metagenedb.settings-gitlab-ci"
REDIS_HOST
:
"
localhost"
CACHE_TTL
:
"
0"
script
:
-
apt-get install -y postgresql-client
-
cd backend
-
pipenv install --dev --system --deploy
-
flake8 --max-line-length
120
-
until pg_isready -h ${DATABASE_HOST}; do echo waiting; sleep 2; done;
-
pytest --cov .
deploy
:
deploy
-dev
:
stage
:
deploy
image
:
registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image
variables
:
NAMESPACE
:
${CI_PROJECT_NAME}-dev
environment
:
name
:
${CI_PROJECT_NAME}-dev
url
:
https://${CI_PROJECT_NAME}-dev.k8s-dev.pasteur.fr
url
:
https://${CI_PROJECT_NAME}-dev.pasteur.cloud
script
:
-
sh ci/deploy/deploy.sh
tags
:
-
k8s
only
:
refs
:
-
dev
deploy-prod
:
stage
:
deploy
image
:
registry-gitlab.pasteur.fr/dsi-tools/docker-images:docker_kubernetes_image
variables
:
NAMESPACE
:
${CI_PROJECT_NAME}
environment
:
name
:
${CI_PROJECT_NAME}
url
:
https://${CI_PROJECT_NAME}.pasteur.cloud
script
:
-
sh ci/deploy/deploy.sh
tags
:
-
k8s
only
:
refs
:
-
master
LICENSE
0 → 100644
View file @
813d2bca
MIT License
Copyright (c) 2020 Institut Pasteur
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
README.md
View file @
813d2bca
# Metagenedb
[

](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/
master
)
[

](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/
master
)
[

](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/
dev
)
[

](https://gitlab.pasteur.fr/metagenomics/metagenedb/commits/
dev
)
Django based project to build genes catalog and tools
to play with it and contact external services.
...
...
@@ -93,6 +93,11 @@ DATABASE_PORT=5432
port is redirected to `
5433
` on the `
localhost
`. This is done in order to not interfere with your local
postgres if you have one. This means you need to change `
DATABASE_HOST
` to `
localhost
` and `
DATABASE_POS
### Pre-computed statistics
Some statistics about genes are pre-computed and can be accessed through the
`/api/catalog/v1/statistics`
endpoint.
The ID is constructed with the following format:
`<statisctics-type>-<gene_source>-<method>-<options>`
.
----
## Run the application
...
...
@@ -127,3 +132,21 @@ For the moment you can:
> **Note**: You can also execute the scripts locally from a `pipenv shell` for instance. You need to make
sure that you change the way to log to postgres since the access is different from your machine compared to
from a container.
-----
## Dev tips
#### Profiling code
```
python
from
metagenedb.common.utils.profiling
import
profile
@
profile
(
"/my/file/path"
)
def
my_function
(
a
,
b
,
c
):
...
```
```
bash
snakeviz /my/file/path
```
\ No newline at end of file
backend/.flake8
View file @
813d2bca
[flake8]
exclude =
*/migrations/*
*/src/bioapi/*
backend/Dockerfile
View file @
813d2bca
...
...
@@ -5,10 +5,11 @@ ENV PYTHONUNBUFFERED 1
# Install pipenv
RUN
pip
install
pipenv
RUN
apt update
&&
apt
install
vim
-y
WORKDIR
/code
RUN
rm
-rf
Dockerfile
# Copy Pipfile and install
COPY
. /code/
RUN
pipenv
install
--system
--deploy
RUN
pipenv
install
--system
--deploy
\ No newline at end of file
backend/Pipfile
View file @
813d2bca
...
...
@@ -4,50 +4,69 @@ url = "https://pypi.org/simple"
verify_ssl
=
true
[dev-packages]
atomicwrites
=
"
==1.3.0
"
attrs
=
"
==19.1.0
"
coverage
=
"
==4.5.3
"
entrypoints
=
"
==0.3
"
flake8
=
"
==3.7.7
"
importlib-metadata
=
"
==0.18
"
kiwisolver
=
"
==1.1.0
"
mccabe
=
"
==0.6.1
"
more-itertools
=
"
==7.0.0
"
pluggy
=
"
==0.12.0
"
py
=
"
==1.8.0
"
pycodestyle
=
"
==2.5.0
"
pyflakes
=
"
==2.1.1
"
pyparsing
=
"
==2.4.0
"
pytest
=
"
==4.6.3
"
pytest-cov
=
"
==2.7.1
"
pytest-django
=
"
==3.5.0
"
wcwidth
=
"
==0.1.7
"
zipp
=
"
==0.5.1
"
Cycler
=
"
==0.10.0
"
atomicwrites
=
"
*
"
attrs
=
"
*
"
coverage
=
"
*
"
entrypoints
=
"
*
"
flake8
=
"
*
"
importlib-metadata
=
"
*
"
kiwisolver
=
"
*
"
mccabe
=
"
*
"
more-itertools
=
"
*
"
pluggy
=
"
*
"
py
=
"
*
"
pycodestyle
=
"
*
"
pyflakes
=
"
*
"
pyparsing
=
"
*
"
pytest
=
"
*
"
pytest-cov
=
"
*
"
pytest-django
=
"
*
"
wcwidth
=
"
*
"
zipp
=
"
*
"
Cycler
=
"
*
"
jupyter
=
"*"
factory-boy
=
"*"
pytest-factoryboy
=
"*"
pylint
=
"*"
mock
=
"*"
snakeviz
=
"*"
[packages]
certifi
=
"
==2019.6.16
"
chardet
=
"
==3.0.4
"
django-cors-headers
=
"
==3.0.2
"
django-environ
=
"
==0.4.5
"
django-extensions
=
"
==2.1.7
"
django-filter
=
"
==2.1.0
"
djangorestframework
=
"
==3.9.4
"
djangorestframework-jwt
=
"
==1.11.0
"
idna
=
"
==2.8
"
numpy
=
"
==1.16.4
"
pandas
=
"
==0.24.2
"
psycopg2
=
"
==2.8.2
"
python-dateutil
=
"
==2.8.0
"
pytz
=
"
==2019.1
"
requests
=
"
==2.22.0
"
six
=
"
==1.12.0
"
sqlparse
=
"
==0.3.0
"
urllib3
=
"
==1.25.3
"
Django
=
"
==2.2.1
"
PyJWT
=
"
==1.7.1
"
certifi
=
"
*
"
chardet
=
"
*
"
django-cors-headers
=
"
*
"
django-environ
=
"
*
"
django-extensions
=
"
*
"
django-filter
=
"
*
"
djangorestframework
=
"
*
"
djangorestframework-jwt
=
"
*
"
idna
=
"
*
"
numpy
=
"
*
"
pandas
=
"
*
"
psycopg2
=
"
*
"
python-dateutil
=
"
*
"
pytz
=
"
*
"
requests
=
"
*
"
six
=
"
*
"
sqlparse
=
"
*
"
urllib3
=
"
*
"
Django
=
"
*
"
PyJWT
=
"
*
"
metagenedb
=
{
editable
=
true
,
path
=
"."
}
drf-yasg
=
"*"
packaging
=
"*"
python-slugify
=
"*"
marshmallow
=
"*"
django-pandas
=
"*"
bioapi
=
{
git
=
"https://github.com/khillion/bioapi.git"
}
django-admin-list-filter-dropdown
=
"*"
gunicorn
=
"*"
pyfastx
=
"*"
celery
=
"*"
redis
=
"*"
django-health-check
=
"==3.0.0"
django-redis
=
"*"
kombu
=
"*"
[requires]
python_version
=
"3.7"
backend/Pipfile.lock
View file @
813d2bca
This diff is collapsed.
Click to expand it.
backend/metagenedb/__init__.py
View file @
813d2bca
from
.celery_app
import
app
as
celery_app
# noqa
__all__
=
(
'celery_app'
)
backend/metagenedb/api/catalog/admin_urls.py
0 → 100644
View file @
813d2bca
from
django.urls
import
path
from
metagenedb.api.catalog.views
import
admin_commands
urlpatterns
=
[
path
(
'compute-counts/'
,
admin_commands
.
ComputeCountsAPI
.
as_view
()),
path
(
'compute-gene-length/'
,
admin_commands
.
ComputeGeneLengthAPI
.
as_view
()),
path
(
'compute-taxo-repartition/'
,
admin_commands
.
ComputeTaxonomyRepartitionAPI
.
as_view
()),
]
backend/metagenedb/api/catalog/filters/__init__.py
0 → 100644
View file @
813d2bca
from
.function
import
EggNOGFilter
,
FunctionFilter
# noqa
from
.gene
import
GeneFilter
# noqa
from
.taxonomy
import
TaxonomyFilter
# noqa
backend/metagenedb/api/catalog/filters/function.py
0 → 100644
View file @
813d2bca
from
django_filters
import
rest_framework
as
filters
from
metagenedb.apps.catalog.models
import
Function
,
EggNOG
class
FunctionFilter
(
filters
.
FilterSet
):
class
Meta
:
model
=
Function
fields
=
[
'source'
]
class
EggNOGFilter
(
filters
.
FilterSet
):
class
Meta
:
model
=
EggNOG
fields
=
[
'version'
]
backend/metagenedb/api/catalog/filters/gene.py
0 → 100644
View file @
813d2bca
from
django_filters
import
rest_framework
as
filters
from
metagenedb.apps.catalog.models
import
Function
,
Gene
,
Taxonomy
class
GeneFilter
(
filters
.
FilterSet
):
has_taxonomy
=
filters
.
BooleanFilter
(
field_name
=
'taxonomy'
,
lookup_expr
=
"isnull"
,
exclude
=
True
)
has_functions
=
filters
.
BooleanFilter
(
field_name
=
'functions'
,
lookup_expr
=
"isnull"
,
distinct
=
True
,
exclude
=
True
)
taxonomy_rank
=
filters
.
ChoiceFilter
(
choices
=
Taxonomy
.
RANK_CHOICES
,
field_name
=
'taxonomy__rank'
)
name
=
filters
.
CharFilter
(
field_name
=
'name'
,
lookup_expr
=
'icontains'
)
length__gt
=
filters
.
NumberFilter
(
field_name
=
'length'
,
lookup_expr
=
'gt'
)
length__lt
=
filters
.
NumberFilter
(
field_name
=
'length'
,
lookup_expr
=
'lt'
)
tax_id
=
filters
.
CharFilter
(
method
=
'filter_annotated_tax'
)
function
=
filters
.
ModelMultipleChoiceFilter
(
queryset
=
Function
.
objects
.
all
(),
field_name
=
'functions__function_id'
,
to_field_name
=
'function_id'
)
def
filter_annotated_tax
(
self
,
queryset
,
name
,
value
):
tax_rank
=
Taxonomy
.
objects
.
get
(
tax_id
=
value
).
rank
lookup
=
f
"taxonomy__hierarchy__
{
tax_rank
}
__tax_id"
return
queryset
.
filter
(
**
{
lookup
:
value
})
class
Meta
:
model
=
Gene
fields
=
[
'length'
,
'name'
,
'source'
]
backend/metagenedb/api/catalog/filters/taxonomy.py
0 → 100644
View file @
813d2bca
from
django_filters
import
rest_framework
as
filters
from
metagenedb.apps.catalog.models
import
Taxonomy
class
TaxonomyFilter
(
filters
.
FilterSet
):
class
Meta
:
model
=
Taxonomy
fields
=
[
'rank'
,
'name'
]
backend/metagenedb/api/catalog/qparams_validators/__init__.py
0 → 100644
View file @
813d2bca
backend/metagenedb/api/catalog/qparams_validators/function.py
0 → 100644
View file @
813d2bca
from
marshmallow
import
fields
from
marshmallow.validate
import
OneOf
from
metagenedb.apps.catalog
import
models
from
metagenedb.common.django_default.qparams_validators
import
PaginatedQueryParams
SELECTED_SOURCE
=
[
i
[
0
]
for
i
in
models
.
Function
.
SOURCE_CHOICES
]
EGGNOG_VERSIONS
=
[
i
[
0
]
for
i
in
models
.
EggNOG
.
VERSION_CHOICES
]
class
FunctionQueryParams
(
PaginatedQueryParams
):
source
=
fields
.
String
(
validate
=
OneOf
(
choices
=
SELECTED_SOURCE
))
class
KeggQueryParams
(
PaginatedQueryParams
):
detailed
=
fields
.
Boolean
()
class
EggNOGQueryParams
(
PaginatedQueryParams
):
version
=
fields
.
String
(
validate
=
OneOf
(
choices
=
EGGNOG_VERSIONS
))
backend/metagenedb/api/catalog/qparams_validators/gene.py
0 → 100644
View file @
813d2bca
from
marshmallow
import
fields
from
metagenedb.common.django_default.qparams_validators
import
PaginatedQueryParams
class
GeneQueryParams
(
PaginatedQueryParams
):
has_taxonomy
=
fields
.
Boolean
()
has_functions
=
fields
.
Boolean
()
length
=
fields
.
Integer
()
length__gt
=
fields
.
Integer
()
length__lt
=
fields
.
Integer
()
name
=
fields
.
String
()
taxonomy_rank
=
fields
.
String
()
tax_id
=
fields
.
Integer
()
function
=
fields
.
String
()
source
=
fields
.
String
()
fasta
=
fields
.
Boolean
()
backend/metagenedb/api/catalog/qparams_validators/taxonomy.py
0 → 100644
View file @
813d2bca
from
marshmallow
import
fields
from
metagenedb.common.django_default.qparams_validators
import
PaginatedQueryParams
class
TaxonomyQueryParams
(
PaginatedQueryParams
):
rank
=
fields
.
String
()
name
=
fields
.
String
()
backend/metagenedb/api/catalog/urls.py
View file @
813d2bca
from
django.urls
import
path
from
django.conf.urls
import
url
,
include
from
rest_framework.routers
import
DefaultRouter
from
rest_framework.routers
import
DefaultRouter
,
DynamicRoute
,
Route
from
metagenedb.api.catalog
.views
import
GeneViewSet
from
metagenedb.api.catalog
import
views
# from metagenedb.api.catalog.views.celery_test import celery_test_view, test_task_view
api_router
=
DefaultRouter
()
api_router
.
register
(
r
'genes'
,
GeneViewSet
,
base_name
=
'genes'
)
class
CustomRouter
(
DefaultRouter
):
routes
=
[
# List route.
Route
(
url
=
r
'^{prefix}{trailing_slash}$'
,
mapping
=
{
'get'
:
'list'
,
'put'
:
'update'
,
'post'
:
'create'
},
name
=
'{basename}-list'
,
detail
=
False
,
initkwargs
=
{
'suffix'
:
'List'
}
),
# Dynamically generated list routes. Generated using
# @action(detail=False) decorator on methods of the viewset.
DynamicRoute
(
url
=
r
'^{prefix}/{url_path}{trailing_slash}$'
,
name
=
'{basename}-{url_name}'
,
detail
=
False
,
initkwargs
=
{}
),
# Detail route.
Route
(
url
=
r
'^{prefix}/{lookup}{trailing_slash}$'
,
mapping
=
{
'get'
:
'retrieve'
,
'put'
:
'update'
,
'patch'
:
'partial_update'
,
'delete'
:
'destroy'
},
name
=
'{basename}-detail'
,
detail
=
True
,
initkwargs
=
{
'suffix'
:
'Instance'
}
),
# Dynamically generated detail routes. Generated using
# @action(detail=True) decorator on methods of the viewset.
DynamicRoute
(
url
=
r
'^{prefix}/{lookup}/{url_path}{trailing_slash}$'
,
name
=
'{basename}-{url_name}'
,
detail
=
True
,
initkwargs
=
{}
),
]
api_router
=
CustomRouter
()
api_router
.
register
(
r
'functions'
,
views
.
FunctionViewSet
,
basename
=
'functions'
)
api_router
.
register
(
r
'kegg-orthologies'
,
views
.
KeggOrthologyViewSet
,
basename
=
'kegg-orthologies'
)
api_router
.
register
(
r
'eggnogs'
,
views
.
EggNOGViewSet
,
basename
=
'eggnogs'
)
api_router
.
register
(
r
'genes'
,
views
.
GeneViewSet
,
basename
=
'genes'
)
api_router
.
register
(
r
'taxonomy'
,
views
.
TaxonomyViewSet
,
basename
=
'taxonomy'
)
api_router
.
register
(
r
'statistics'
,
views
.
StatisticsViewSet
,
basename
=
'statistics'
)
urlpatterns
=
[
url
(
r
'v1/'
,
include
((
api_router
.
urls
,
'v1'
)))
url
(
r
'v1/'
,
include
((
api_router
.
urls
,
'v1'
))),
path
(
'admin/'
,
include
((
'metagenedb.api.catalog.admin_urls'
,
'admin'
))),
# path('celery-test/', celery_test_view, name='celery-test'),
# path('celery-task-test/', test_task_view, name='celery-task-test'),
]
backend/metagenedb/api/catalog/views/__init__.py
View file @
813d2bca
from
.function
import
EggNOGViewSet
,
KeggOrthologyViewSet
,
FunctionViewSet
# noqa
from
.gene
import
GeneViewSet
# noqa
from
.statistics
import
StatisticsViewSet
# noqa
from
.taxonomy
import
TaxonomyViewSet
# noqa
Prev
1
2
3
4
5
…
13
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment