Merge branch '126-extract-table-csv' into 'dev'

Extract table from gene list

Closes #126

See merge request !60

backend/metagenedb/api/catalog/qparams_validators/gene.py

@@ -15,3 +15,4 @@ class GeneQueryParams(PaginatedQueryParams):
     function = fields.String()
     source = fields.String()
     fasta = fields.Boolean()
+    csv = fields.Boolean()

backend/metagenedb/api/catalog/views/gene.py

-import hashlib
 from io import StringIO
 
-from django.core.cache import cache
 from django.conf import settings
 from django.http import HttpResponse
 from drf_yasg.utils import swagger_auto_schema
@@ -14,11 +12,12 @@ from metagenedb.apps.catalog.models import Gene
 from metagenedb.api.catalog.filters import GeneFilter
 from metagenedb.api.catalog.qparams_validators.gene import GeneQueryParams
 from metagenedb.apps.catalog.serializers import GeneSerializer
+from metagenedb.common.utils.cache import queryset_count_cached
 
 from .base import BulkViewSet
 
 
-MAX_FASTA_GENES = settings.MAX_FASTA_GENES
+MAX_DOWNLOAD_GENES = settings.MAX_DOWNLOAD_GENES
 
 
 class GeneViewSet(BulkViewSet):
@@ -29,27 +28,79 @@ class GeneViewSet(BulkViewSet):
     lookup_field = 'gene_id'
 
     def _get_queryset_count(self, queryset):
-        hash_object = hashlib.md5(str(queryset.query).encode('utf-8'))
-        redis_key = hash_object.hexdigest()
-        if redis_key in cache:
-            return cache.get(redis_key)
-        else:
-            return queryset.count()
+        return queryset_count_cached(queryset)
+
+    @property
+    def too_many_genes_error_response(self):
+        error_message = f'Too many genes in the query, can obtain download up to {MAX_DOWNLOAD_GENES} genes.'
+        return Response({'message': error_message}, status=HTTP_500_INTERNAL_SERVER_ERROR)
+
+    def _check_too_many_genes(self, queryset):
+        count = self._get_queryset_count(queryset)
+        return count >= MAX_DOWNLOAD_GENES
 
     def _build_fasta_response(self):
         queryset = self.filter_queryset(self.get_queryset())
-        count = self._get_queryset_count(queryset)
-        if count >= MAX_FASTA_GENES:
-            error_message = f'Too many genes in the query, can obtain only up to {MAX_FASTA_GENES} fasta seq.'
-            return Response({'message': error_message}, status=HTTP_500_INTERNAL_SERVER_ERROR)
-        fasta_file = StringIO()
-        for gene in queryset.iterator():
-            fasta_file.write(gene.fasta)
-        # generate the file
-        response = HttpResponse(fasta_file.getvalue(), content_type='text/fasta')
-        filename = 'metagenedb_sequences.fasta'
-        response['Content-Disposition'] = 'attachment; filename=%s' % filename
-        return response
+        if self._check_too_many_genes(queryset):
+            return self.too_many_genes_error_response
+        with StringIO() as fasta_file:
+            for gene in queryset.iterator():
+                fasta_file.write(gene.fasta)
+            # generate the file
+            response = HttpResponse(fasta_file.getvalue(), content_type='text/fasta')
+            filename = 'metagenedb_sequences.fasta'
+            response['Content-Disposition'] = 'attachment; filename=%s' % filename
+            return response
+
+    def _extract_taxonomy_info(self, gene):
+        if gene.taxonomy is None:
+            return ['', '', '']
+        return [gene.taxonomy.tax_id, gene.taxonomy.name, gene.taxonomy.rank]
+
+    def _extract_function_info(self, gene):
+        if not gene.functions.all():
+            return ['', '']
+        function_ids = {
+            'kegg': [],
+            'eggnog': []
+        }
+        for function in gene.functions.all():
+            function_ids.get(function.source).append(function.function_id)
+        return [
+            ';'.join(function_ids['kegg']),
+            ';'.join(function_ids['eggnog'])
+        ]
+
+    def _get_metadata_line(self, gene):
+        """
+        Transform gene content to a line for metadata extract
+        """
+        gene_items = [
+            gene.gene_id, gene.name, gene.source, gene.length,
+        ]
+        gene_items = gene_items + self._extract_taxonomy_info(gene)
+        gene_items = gene_items + self._extract_function_info(gene)
+        return ','.join([str(item) for item in gene_items])
+
+    def _build_csv_response(self):
+        queryset = self.filter_queryset(self.get_queryset())
+        queryset = queryset.select_related("taxonomy").prefetch_related("functions")
+        if self._check_too_many_genes(queryset):
+            return self.too_many_genes_error_response
+        with StringIO() as csv_file:
+            # Write header
+            header = ",".join([
+                'gene_id', 'gene_name', 'gene_source', 'length', 'tax_id', 'tax_name', 'tax_rank',
+                'kegg_id', 'eggnog_id',
+            ])
+            csv_file.write(f"{header}\n")
+            for gene in queryset:
+                csv_file.write(f"{self._get_metadata_line(gene)}\n")
+            # generate the file
+            response = HttpResponse(csv_file.getvalue(), content_type='text/csv')
+            filename = 'metagenedb.csv'
+            response['Content-Disposition'] = 'attachment; filename=%s' % filename
+            return response
 
     @swagger_auto_schema(
         tags=['Genes'],
@@ -65,6 +116,8 @@ class GeneViewSet(BulkViewSet):
             return Response(error_message, status=HTTP_422_UNPROCESSABLE_ENTITY)
         if query_params.get('fasta', False) is True:
             return self._build_fasta_response()
+        if query_params.get('csv', False) is True:
+            return self._build_csv_response()
         return super().list(request, *args, **kwargs)
 
     @swagger_auto_schema(

backend/metagenedb/api/catalog/views/test_gene.py

@@ -2,14 +2,78 @@ from django.test import TestCase
 from django.urls import reverse
 from rest_framework import status
 
+from metagenedb.api.catalog.views import GeneViewSet
+from metagenedb.apps.catalog.factory import (
+    GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory, GeneWithTaxonomyFactory
+)
+
+
+class GeneViewSetMock(GeneViewSet):
+    """
+    Make mock in case complex instantiation occurs since tests here are independant from the class itself
+    """
+
+    def __init__(self):
+        pass
+
 
 class TestGenes(TestCase):
 
     def test_get_genes_no_auth(self):
         """
         Unauthenticated users should be able to access genes
-        @TODO make unaccessible
         """
         url = reverse('api:catalog:v1:genes-list')
         resp = self.client.get(url)
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
+
+    def test_get_metadata_line_no_functions(self):
+        gene = GeneFactory()
+        expected_items = [
+            gene.gene_id, gene.name, gene.source, gene.length,
+            '', '', '', '', ''
+        ]
+        expected_line = ','.join([str(item) for item in expected_items])
+        # Make test with method from GeneViewSet
+        viewset = GeneViewSetMock()
+        tested_line = viewset._get_metadata_line(gene)
+        self.assertEqual(tested_line, expected_line)
+
+    def test_get_metadata_line_with_taxonomy(self):
+        gene = GeneWithTaxonomyFactory()
+        expected_items = [
+            gene.gene_id, gene.name, gene.source, gene.length,
+            gene.taxonomy.tax_id, gene.taxonomy.name, gene.taxonomy.rank,
+            '', ''
+        ]
+        expected_line = ','.join([str(item) for item in expected_items])
+        # Make test with method from GeneViewSet
+        viewset = GeneViewSetMock()
+        tested_line = viewset._get_metadata_line(gene)
+        self.assertEqual(tested_line, expected_line)
+
+    def test_get_metadata_line_with_kegg(self):
+        gene = GeneWithKeggFactory()
+        expected_items = [
+            gene.gene_id, gene.name, gene.source, gene.length,
+            '', '', '',
+            gene.functions.all()[0].function_id, ''
+        ]
+        expected_line = ','.join([str(item) for item in expected_items])
+        # Make test with method from GeneViewSet
+        viewset = GeneViewSetMock()
+        tested_line = viewset._get_metadata_line(gene)
+        self.assertEqual(tested_line, expected_line)
+
+    def test_get_metadata_line_with_eggnog(self):
+        gene = GeneWithEggNOGFactory()
+        expected_items = [
+            gene.gene_id, gene.name, gene.source, gene.length,
+            '', '', '',
+            '', gene.functions.all()[0].function_id,
+        ]
+        expected_line = ','.join([str(item) for item in expected_items])
+        # Make test with method from GeneViewSet
+        viewset = GeneViewSetMock()
+        tested_line = viewset._get_metadata_line(gene)
+        self.assertEqual(tested_line, expected_line)

backend/metagenedb/apps/catalog/factory/__init__.py

 from .function import EggNOGFactory, FunctionFactory, KeggOrthologyFactory  # noqa
-from .gene import GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory  # noqa
+from .gene import GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory, GeneWithTaxonomyFactory  # noqa
 from .taxonomy import TaxonomyFactory  # noqa

backend/metagenedb/apps/catalog/factory/gene.py

@@ -11,7 +11,7 @@ from .taxonomy import TaxonomyFactory
 
 faker = Factory.create()
 
-SELECTED_SOURCE = [i[0] for i in models.Function.SOURCE_CHOICES]
+GENE_SOURCES = [i[0] for i in models.Gene.SOURCE_CHOICES]
 
 
 class GeneFactory(DjangoModelFactory):
@@ -21,6 +21,10 @@ class GeneFactory(DjangoModelFactory):
     gene_id = FuzzyLowerText(prefix='gene-', length=15)
     name = fuzzy.FuzzyText(prefix='name-', length=15)
     length = fuzzy.FuzzyInteger(200, 10000)
+    source = fuzzy.FuzzyChoice(GENE_SOURCES)
+
+
+class GeneWithTaxonomyFactory(GeneFactory):
     taxonomy = SubFactory(TaxonomyFactory)
 
 

backend/metagenedb/apps/catalog/operations/test_statistics.py

@@ -2,7 +2,7 @@ from rest_framework.test import APITestCase
 
 from metagenedb.common.utils.color_generator import generate_color_code
 from metagenedb.apps.catalog.factory import (
-    GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory, TaxonomyFactory
+    GeneFactory, GeneWithEggNOGFactory, GeneWithKeggFactory, GeneWithTaxonomyFactory, TaxonomyFactory
 )
 
 from .statistics import GeneStatistics, GeneLengthDistribution
@@ -116,7 +116,7 @@ class TestCounts(BaseTestGeneStatistics):
 
         They all have a taxonomy
         """
-        cls.genes_no_function = GeneFactory.create_batch(5)
+        cls.genes_no_function = GeneWithTaxonomyFactory.create_batch(5)
         cls.keggs = GeneWithKeggFactory.create_batch(5)
         cls.eggnogs = GeneWithEggNOGFactory.create_batch(10)
 
@@ -124,7 +124,7 @@ class TestCounts(BaseTestGeneStatistics):
         self.assertEqual(self.gene_stats.count_all(), 20)
 
     def test_has_taxonomy(self):
-        self.assertEqual(self.gene_stats.count_has_taxonomy(), 20)
+        self.assertEqual(self.gene_stats.count_has_taxonomy(), 5)
 
     def test_count_has_function(self):
         self.assertEqual(self.gene_stats.count_has_function(), 15)
@@ -139,7 +139,7 @@ class TestCounts(BaseTestGeneStatistics):
         self.assertEqual(self.gene_stats.count_has_function(source='wrong_source'), 0)
 
     def test_count_has_taxonomy_has_function(self):
-        self.assertEqual(self.gene_stats.count_has_function_has_taxonomy(), 15)
+        self.assertEqual(self.gene_stats.count_has_function_has_taxonomy(), 0)
 
 
 class TestCountWindows(APITestCase):

backend/metagenedb/common/django_default/paginator.py

-import hashlib
-import inspect
-
-from django.conf import settings
-from django.core.cache import cache
-from django.core.cache.backends.base import DEFAULT_TIMEOUT
 from django.core.paginator import Paginator
 from django.utils.functional import cached_property
-from django.utils.inspect import method_has_no_args
-
 
-CACHE_TTL = getattr(settings, 'CACHE_TTL', DEFAULT_TIMEOUT)
+from metagenedb.common.utils.cache import queryset_count_cached
 
 
 class CachedCountPaginator(Paginator):
@@ -17,16 +9,4 @@ class CachedCountPaginator(Paginator):
     @cached_property
     def count(self):
         """Return the total number of objects, across all pages."""
-        # Create has from SQL query for REDIS cache
-        hash_object = hashlib.md5(str(self.object_list.query).encode('utf-8'))
-        redis_key = hash_object.hexdigest()
-        if redis_key in cache:
-            return cache.get(redis_key)
-        else:
-            c = getattr(self.object_list, 'count', None)
-            if callable(c) and not inspect.isbuiltin(c) and method_has_no_args(c):
-                count = c()
-            else:
-                count = len(self.object_list)
-            cache.set(redis_key, count, timeout=CACHE_TTL)
-            return count
+        return queryset_count_cached(self.object_list)

backend/metagenedb/common/utils/cache/__init__.py

+from .count import queryset_count_cached  # noqa

backend/metagenedb/common/utils/cache/count.py

+import hashlib
+import inspect
+
+from django.conf import settings
+from django.core.cache import cache
+from django.core.cache.backends.base import DEFAULT_TIMEOUT
+from django.utils.inspect import method_has_no_args
+
+
+CACHE_TTL = getattr(settings, 'CACHE_TTL', DEFAULT_TIMEOUT)
+
+
+def queryset_count_cached(queryset):
+    # Create has from SQL query for REDIS cache
+    hash_object = hashlib.md5(str(queryset.query).encode('utf-8'))
+    redis_key = hash_object.hexdigest()
+    if redis_key in cache:
+        return cache.get(redis_key)
+    else:
+        c = getattr(queryset, 'count', None)
+        if callable(c) and not inspect.isbuiltin(c) and method_has_no_args(c):
+            count = c()
+        else:
+            count = len(queryset)
+        cache.set(redis_key, count, timeout=CACHE_TTL)
+        return count

backend/metagenedb/settings/django.py

@@ -168,6 +168,7 @@ LOGGING = {
     'disable_existing_loggers': False,
     'handlers': {
         'console': {
+            'level': 'DEBUG',
             'class': 'logging.StreamHandler',
         },
     },
@@ -176,8 +177,8 @@ LOGGING = {
             'handlers': ['console'],
             'level': env.str('DJANGO_LOG_LEVEL', 'INFO'),
         },
-        'metagenedb': {
-            'level': env.str('DJANGO_LOG_LEVEL', 'INFO'),
-        },
+        'django.db.backends': {
+            'level': env.str('DB_LOG_LEVEL', 'INFO'),
+        }
     },
 }

backend/metagenedb/settings/metagenedb.py

@@ -9,4 +9,4 @@ env = environ.Env()
 environ.Env.read_env(root('.env'))  # reading .env file
 
 # Maximum number of FASTA genes able to retrieve through API
-MAX_FASTA_GENES = env.str('MAX_FASTA_GENES', default=100000)
+MAX_DOWNLOAD_GENES = env.str('MAX_DOWNLOAD_GENES', default=100000)

frontend/src/views/genes/genes.html

@@ -241,11 +241,12 @@
             <template v-slot:activator="{ on }">
               <v-btn
                 fab
-                dark
                 small
+                :loading="loadingCSVDownload"
+                :disabled="loadingCSVDownload"
                 color="primary lighten-3"
+                @click="downloadMetadataCSV"
                 v-on="on"
-                disabled
               >
                 .CSV
               </v-btn>

frontend/src/views/genes/genes.js

@@ -31,7 +31,8 @@ export default {
       loadTable: true,
       requestDone: false,
       // Download FASTA loading
-      downloadReady: true,
+      downloadFastaReady: true,
+      downloadCSVReady: true,
       fab: false,
     };
   },
@@ -110,12 +111,18 @@ export default {
       return numberPages;
     },
     loadingFastaDownload() {
-      return !this.downloadReady;
+      return !this.downloadFastaReady;
+    },
+    loadingCSVDownload() {
+      return !this.downloadCSVReady;
     },
     loadingDownloads() {
       if (this.loadingFastaDownload) {
         return true;
       }
+      else if (this.loadingCSVDownload) {
+        return true;
+      }
       return false;
     },
     showDownloads() {
@@ -196,17 +203,18 @@ export default {
           this.requestDone = true;
         });
     },
-    forceFileDownload(response){
+    forceFileDownload(response, fileName){
       const url = window.URL.createObjectURL(new Blob([response.data]))
       const link = document.createElement('a')
       link.href = url
-      link.setAttribute('download', 'metagenedb_sequences.fasta') //or any other extension
+      link.setAttribute('download', fileName) //or any other extension
       document.body.appendChild(link)
       link.click()
     },
     downloadFasta() {
-      this.downloadReady = false;
-      var qParams = this.qParams;
+      this.downloadFastaReady = false;
+      var qParams = {};
+      Object.assign(qParams, this.qParams);
       qParams.fasta = "true";
       delete qParams['page_size']
       delete qParams['page']
@@ -217,8 +225,30 @@ export default {
         },
       })
         .then((response) => {
+          this.downloadFastaReady = true;
+          this.forceFileDownload(response, 'metagenedb_sequences.fasta');
+        })
+        .catch((error) => {
+          console.error(error);
           this.downloadReady = true;
-          this.forceFileDownload(response);
+        });
+    },
+    downloadMetadataCSV() {
+      this.downloadCSVReady = false;
+      var qParams = {};
+      Object.assign(qParams, this.qParams);
+      qParams.csv = "true";
+      delete qParams['page_size']
+      delete qParams['page']
+      axios.get('/api/catalog/v1/genes', {
+        params: qParams,
+        headers: {
+          Accept: 'application/json',
+        },
+      })
+        .then((response) => {
+          this.downloadCSVReady = true;
+          this.forceFileDownload(response, 'metagenedb_metadata.csv');
         })
         .catch((error) => {
           console.error(error);