Commit c1cf1f13 authored by Kenzo-Hugo Hillion's avatar Kenzo-Hugo Hillion
Browse files

Merge branch 'dev' into 'master'

Update prod with new features

See merge request !64
parents cf322915 8f46bd32
Pipeline #33262 passed with stages
in 3 minutes and 53 seconds
......@@ -15,3 +15,4 @@ class GeneQueryParams(PaginatedQueryParams):
function = fields.String()
source = fields.String()
fasta = fields.Boolean()
csv = fields.Boolean()
import hashlib
from io import StringIO
from django.core.cache import cache
from django.conf import settings
from django.http import HttpResponse
from drf_yasg.utils import swagger_auto_schema
......@@ -14,11 +12,12 @@ from metagenedb.apps.catalog.models import Gene
from metagenedb.api.catalog.filters import GeneFilter
from metagenedb.api.catalog.qparams_validators.gene import GeneQueryParams
from metagenedb.apps.catalog.serializers import GeneSerializer
from metagenedb.common.utils.cache import queryset_count_cached
from .base import BulkViewSet
MAX_FASTA_GENES = settings.MAX_FASTA_GENES
MAX_DOWNLOAD_GENES = settings.MAX_DOWNLOAD_GENES
class GeneViewSet(BulkViewSet):
......@@ -29,27 +28,79 @@ class GeneViewSet(BulkViewSet):
lookup_field = 'gene_id'
def _get_queryset_count(self, queryset):
hash_object = hashlib.md5(str(queryset.query).encode('utf-8'))
redis_key = hash_object.hexdigest()
if redis_key in cache:
return cache.get(redis_key)
else:
return queryset.count()
return queryset_count_cached(queryset)
@property
def too_many_genes_error_response(self):
error_message = f'Too many genes in the query, can obtain download up to {MAX_DOWNLOAD_GENES} genes.'
return Response({'message': error_message}, status=HTTP_500_INTERNAL_SERVER_ERROR)
def _check_too_many_genes(self, queryset):
count = self._get_queryset_count(queryset)
return count >= MAX_DOWNLOAD_GENES
def _build_fasta_response(self):
queryset = self.filter_queryset(self.get_queryset())
count = self._get_queryset_count(queryset)
if count >= MAX_FASTA_GENES:
error_message = f'Too many genes in the query, can obtain only up to {MAX_FASTA_GENES} fasta seq.'
return Response({'message': error_message}, status=HTTP_500_INTERNAL_SERVER_ERROR)
fasta_file = StringIO()
for gene in queryset.iterator():
fasta_file.write(gene.fasta)
# generate the file
response = HttpResponse(fasta_file.getvalue(), content_type='text/fasta')
filename = 'metagenedb_sequences.fasta'
response['Content-Disposition'] = 'attachment; filename=%s' % filename
return response
if self._check_too_many_genes(queryset):
return self.too_many_genes_error_response
with StringIO() as fasta_file:
for gene in queryset.iterator():
fasta_file.write(gene.fasta)
# generate the file
response = HttpResponse(fasta_file.getvalue(), content_type='text/fasta')
filename = 'metagenedb_sequences.fasta'
response['Content-Disposition'] = 'attachment; filename=%s' % filename
return response
def _extract_taxonomy_info(self, gene):
if gene.taxonomy is None:
return ['', '', '']
return [gene.taxonomy.tax_id, gene.taxonomy.name, gene.taxonomy.rank]
def _extract_function_info(self, gene):
if not gene.functions.all():
return ['', '']
function_ids = {
'kegg': [],
'eggnog': []
}
for function in gene.functions.all():
function_ids.get(function.source).append(function.function_id)
return [
';'.join(function_ids['kegg']),
';'.join(function_ids['eggnog'])
]
def _get_metadata_line(self, gene):
"""
Transform gene content to a line for metadata extract
"""
gene_items = [
gene.gene_id, gene.name, gene.source, gene.length,
]
gene_items = gene_items + self._extract_taxonomy_info(gene)
gene_items = gene_items + self._extract_function_info(gene)
return ','.join([str(item) for item in gene_items])
def _build_csv_response(self):
queryset = self.filter_queryset(self.get_queryset())
queryset = queryset.select_related("taxonomy").prefetch_related("functions")
if self._check_too_many_genes(queryset):
return self.too_many_genes_error_response
with StringIO() as csv_file:
# Write header
header = ",".join([
'gene_id', 'gene_name', 'gene_source', 'length', 'tax_id', 'tax_name', 'tax_rank',
'kegg_id', 'eggnog_id',
])
csv_file.write(f"{header}\n")
for gene in queryset:
csv_file.write(f"{self._get_metadata_line(gene)}\n")
# generate the file
response = HttpResponse(csv_file.getvalue(), content_type='text/csv')
filename = 'metagenedb.csv'
response['Content-Disposition'] = 'attachment; filename=%s' % filename
return response
@swagger_auto_schema(
tags=['Genes'],
......@@ -65,6 +116,8 @@ class GeneViewSet(BulkViewSet):
return Response(error_message, status=HTTP_422_UNPROCESSABLE_ENTITY)
if query_params.get('fasta', False) is True:
return self._build_fasta_response()
if query_params.get('csv', False) is True:
return self._build_csv_response()
return super().list(request, *args, **kwargs)
@swagger_auto_schema(
......
......@@ -2,14 +2,78 @@ from django.test import TestCase
from django.urls import reverse
from rest_framework import status
from metagenedb.api.catalog.views import GeneViewSet
from metagenedb.apps.catalog.factory import (
GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory, GeneWithTaxonomyFactory
)
class GeneViewSetMock(GeneViewSet):
"""
Make mock in case complex instantiation occurs since tests here are independant from the class itself
"""
def __init__(self):
pass
class TestGenes(TestCase):
def test_get_genes_no_auth(self):
"""
Unauthenticated users should be able to access genes
@TODO make unaccessible
"""
url = reverse('api:catalog:v1:genes-list')
resp = self.client.get(url)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
def test_get_metadata_line_no_functions(self):
gene = GeneFactory()
expected_items = [
gene.gene_id, gene.name, gene.source, gene.length,
'', '', '', '', ''
]
expected_line = ','.join([str(item) for item in expected_items])
# Make test with method from GeneViewSet
viewset = GeneViewSetMock()
tested_line = viewset._get_metadata_line(gene)
self.assertEqual(tested_line, expected_line)
def test_get_metadata_line_with_taxonomy(self):
gene = GeneWithTaxonomyFactory()
expected_items = [
gene.gene_id, gene.name, gene.source, gene.length,
gene.taxonomy.tax_id, gene.taxonomy.name, gene.taxonomy.rank,
'', ''
]
expected_line = ','.join([str(item) for item in expected_items])
# Make test with method from GeneViewSet
viewset = GeneViewSetMock()
tested_line = viewset._get_metadata_line(gene)
self.assertEqual(tested_line, expected_line)
def test_get_metadata_line_with_kegg(self):
gene = GeneWithKeggFactory()
expected_items = [
gene.gene_id, gene.name, gene.source, gene.length,
'', '', '',
gene.functions.all()[0].function_id, ''
]
expected_line = ','.join([str(item) for item in expected_items])
# Make test with method from GeneViewSet
viewset = GeneViewSetMock()
tested_line = viewset._get_metadata_line(gene)
self.assertEqual(tested_line, expected_line)
def test_get_metadata_line_with_eggnog(self):
gene = GeneWithEggNOGFactory()
expected_items = [
gene.gene_id, gene.name, gene.source, gene.length,
'', '', '',
'', gene.functions.all()[0].function_id,
]
expected_line = ','.join([str(item) for item in expected_items])
# Make test with method from GeneViewSet
viewset = GeneViewSetMock()
tested_line = viewset._get_metadata_line(gene)
self.assertEqual(tested_line, expected_line)
from .function import EggNOGFactory, FunctionFactory, KeggOrthologyFactory # noqa
from .gene import GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory # noqa
from .gene import GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory, GeneWithTaxonomyFactory # noqa
from .taxonomy import TaxonomyFactory # noqa
......@@ -11,7 +11,7 @@ from .taxonomy import TaxonomyFactory
faker = Factory.create()
SELECTED_SOURCE = [i[0] for i in models.Function.SOURCE_CHOICES]
GENE_SOURCES = [i[0] for i in models.Gene.SOURCE_CHOICES]
class GeneFactory(DjangoModelFactory):
......@@ -21,6 +21,10 @@ class GeneFactory(DjangoModelFactory):
gene_id = FuzzyLowerText(prefix='gene-', length=15)
name = fuzzy.FuzzyText(prefix='name-', length=15)
length = fuzzy.FuzzyInteger(200, 10000)
source = fuzzy.FuzzyChoice(GENE_SOURCES)
class GeneWithTaxonomyFactory(GeneFactory):
taxonomy = SubFactory(TaxonomyFactory)
......
......@@ -2,7 +2,7 @@ from rest_framework.test import APITestCase
from metagenedb.common.utils.color_generator import generate_color_code
from metagenedb.apps.catalog.factory import (
GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory, TaxonomyFactory
GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory, GeneWithTaxonomyFactory, TaxonomyFactory
)
from .statistics import GeneStatistics, GeneLengthDistribution
......@@ -116,7 +116,7 @@ class TestCounts(BaseTestGeneStatistics):
They all have a taxonomy
"""
cls.genes_no_function = GeneFactory.create_batch(5)
cls.genes_no_function = GeneWithTaxonomyFactory.create_batch(5)
cls.keggs = GeneWithKeggFactory.create_batch(5)
cls.eggnogs = GeneWithEggNOGFactory.create_batch(10)
......@@ -124,7 +124,7 @@ class TestCounts(BaseTestGeneStatistics):
self.assertEqual(self.gene_stats.count_all(), 20)
def test_has_taxonomy(self):
self.assertEqual(self.gene_stats.count_has_taxonomy(), 20)
self.assertEqual(self.gene_stats.count_has_taxonomy(), 5)
def test_count_has_function(self):
self.assertEqual(self.gene_stats.count_has_function(), 15)
......@@ -139,7 +139,7 @@ class TestCounts(BaseTestGeneStatistics):
self.assertEqual(self.gene_stats.count_has_function(source='wrong_source'), 0)
def test_count_has_taxonomy_has_function(self):
self.assertEqual(self.gene_stats.count_has_function_has_taxonomy(), 15)
self.assertEqual(self.gene_stats.count_has_function_has_taxonomy(), 0)
class TestCountWindows(APITestCase):
......
import hashlib
import inspect
from django.conf import settings
from django.core.cache import cache
from django.core.cache.backends.base import DEFAULT_TIMEOUT
from django.core.paginator import Paginator
from django.utils.functional import cached_property
from django.utils.inspect import method_has_no_args
CACHE_TTL = getattr(settings, 'CACHE_TTL', DEFAULT_TIMEOUT)
from metagenedb.common.utils.cache import queryset_count_cached
class CachedCountPaginator(Paginator):
......@@ -17,16 +9,4 @@ class CachedCountPaginator(Paginator):
@cached_property
def count(self):
"""Return the total number of objects, across all pages."""
# Create has from SQL query for REDIS cache
hash_object = hashlib.md5(str(self.object_list.query).encode('utf-8'))
redis_key = hash_object.hexdigest()
if redis_key in cache:
return cache.get(redis_key)
else:
c = getattr(self.object_list, 'count', None)
if callable(c) and not inspect.isbuiltin(c) and method_has_no_args(c):
count = c()
else:
count = len(self.object_list)
cache.set(redis_key, count, timeout=CACHE_TTL)
return count
return queryset_count_cached(self.object_list)
from .count import queryset_count_cached # noqa
import hashlib
import inspect
from django.conf import settings
from django.core.cache import cache
from django.core.cache.backends.base import DEFAULT_TIMEOUT
from django.utils.inspect import method_has_no_args
CACHE_TTL = getattr(settings, 'CACHE_TTL', DEFAULT_TIMEOUT)
def queryset_count_cached(queryset):
# Create has from SQL query for REDIS cache
hash_object = hashlib.md5(str(queryset.query).encode('utf-8'))
redis_key = hash_object.hexdigest()
if redis_key in cache:
return cache.get(redis_key)
else:
c = getattr(queryset, 'count', None)
if callable(c) and not inspect.isbuiltin(c) and method_has_no_args(c):
count = c()
else:
count = len(queryset)
cache.set(redis_key, count, timeout=CACHE_TTL)
return count
......@@ -168,6 +168,7 @@ LOGGING = {
'disable_existing_loggers': False,
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
},
},
......@@ -176,8 +177,8 @@ LOGGING = {
'handlers': ['console'],
'level': env.str('DJANGO_LOG_LEVEL', 'INFO'),
},
'metagenedb': {
'level': env.str('DJANGO_LOG_LEVEL', 'INFO'),
},
'django.db.backends': {
'level': env.str('DB_LOG_LEVEL', 'INFO'),
}
},
}
......@@ -9,4 +9,4 @@ env = environ.Env()
environ.Env.read_env(root('.env')) # reading .env file
# Maximum number of FASTA genes able to retrieve through API
MAX_FASTA_GENES = env.str('MAX_FASTA_GENES', default=100000)
MAX_DOWNLOAD_GENES = env.str('MAX_DOWNLOAD_GENES', default=100000)
<template>
<v-flex xs6 sm4 lg2>
<v-card :color="count.class" dark class="elevation-2">
<v-card-title primary-title>
<div v-if="count.text">
<div class="blue-grey--text text--lighten-5 subheading font-weight-thin mb-1">{{ count.title }}</div>
<div class="display-1 font-weight-medium">{{ count.text }}</div>
</div>
<div v-else class="text-xs-center">
<v-progress-circular
indeterminate
color="secondary"
></v-progress-circular>
</div>
</v-card-title>
</v-card>
</v-flex>
</template>
<script>
export default {
props: {
count: {
type: Object,
required: true,
},
},
};
</script>
<v-flex xs6 sm4 lg2>
<v-card :color="color" dark class="elevation-2">
<v-card-title primary-title>
<div v-if="text">
<div class="blue-grey--text text--lighten-5 subheading font-weight-thin mb-1">{{ title }}</div>
<div class="display-1 font-weight-medium">{{ text }}</div>
</div>
<div v-else class="align-xs-center">
<v-progress-circular
indeterminate
color="secondary"
></v-progress-circular>
</div>
</v-card-title>
</v-card>
</v-flex>
\ No newline at end of file
export default {
props: {
title: {
type: String,
required: true,
},
text: String,
color: {
type: String,
required: true,
},
},
};
\ No newline at end of file
<template src="./countcard.html" lang="html"></template>
<script src="./countcard.js" lang="hs"></script>
\ No newline at end of file
......@@ -63,11 +63,16 @@ export default {
},
buildEggNogDetails(response) {
this.eggnogVersion = response.data.version
var eggnogIdItem = {
title: 'ID',
content: response.data.function_id,
};
if (this.eggnogId.startsWith('COG')) {
eggnogIdItem.url = `https://ftp.ncbi.nih.gov/pub/COG/COG2014/static/byCOG/${eggnogIdItem.content}.html`;
eggnogIdItem.url_label = 'Open in NCBI'
}
this.eggnogDetails = [
{
title: 'ID',
content: response.data.function_id,
},
eggnogIdItem,
{
title: 'Name',
content: response.data.name,
......
<v-card class="pa-2">
<div class="card-body">
<div class="text-xs-center" v-if="!requestDone && noGraph">
<v-progress-circular
indeterminate
color="secondary"
></v-progress-circular>
</div>
<v-alert
v-else-if="requestDone && noGraph"
:value="requestDone && noGraph"
type="warning"
class="text-xs-center"
>
Could not retrieve data from the server.
</v-alert>
<div>
<canvas :id="chartId"></canvas>
</div>
<v-layout
class="mx-2"
justify-space-between
row
wrap
v-if="!noGraph"
>
<v-flex md2>
<v-switch
color="secondary"
v-model="hideLegend"
label="Hide legend"
@change="updateChartOptions">
</v-switch>
</v-flex>
<v-flex sm6 md4 v-if="hideLegend === false">
<v-radio-group
row
v-model="legendPosition"
@change="updateChartOptions"
>
<template v-slot:label>
<div>Position:</div>
</template>
<v-radio
color="secondary"
v-for="n in ['top', 'right']"
:key="n"
:label="`${n}`"
:value="n"
></v-radio>
</v-radio-group>
</v-flex>
<v-flex sm12 md6>
<v-select
v-model="hideLabels"
:items="this.labels"
attach
chips
:label="hideLabelsLabel"
multiple
clearable
@change="updateChartData"
></v-select>
</v-flex>
</v-layout>
</div>
</v-card>
\ No newline at end of file
<template>
<v-card class="pa-2">
<div class="card-body">
<div class="text-xs-center" v-if="noGraph">
<v-progress-circular
indeterminate
color="secondary"
></v-progress-circular>
</div>
<div>
<canvas :id="chartId"></canvas>
</div>
<v-layout
class="mx-2"
justify-space-between
row
wrap
>
<v-flex md2>
<v-switch
color="secondary"
v-model="hideLegend"
label="Hide legend"
@change="updateChartOptions">
</v-switch>
</v-flex>
<v-flex sm6 md4 v-if="hideLegend === false">
<v-radio-group
row
v-model="legendPosition"
@change="updateChartOptions"
>
<template v-slot:label>
<div>Position:</div>
</template>
<v-radio
color="secondary"
v-for="n in ['top', 'right']"
:key="n"
:label="`${n}`"
:value="n"
></v-radio>
</v-radio-group>
</v-flex>
<v-flex sm12 md6>
<v-select
v-model="hideLabels"
:items="this.labels"
attach
chips
:label="hideLabelsLabel"
multiple
clearable
@change="updateChartData"
></v-select>
</v-flex>
</v-layout>
</div>
</v-card>
</template>
<script>
import Chart from 'chart.js';