Merge branch '99-fasta-from-gene-list' into 'dev'

Obtain fasta for a given query

Closes #99

See merge request !58

backend/metagenedb/api/catalog/qparams_validators/gene.py

@@ -14,3 +14,4 @@ class GeneQueryParams(PaginatedQueryParams):
     tax_id = fields.Integer()
     function = fields.String()
     source = fields.String()
+    fasta = fields.Boolean()

backend/metagenedb/api/catalog/views/gene.py

-from drf_yasg.utils import swagger_auto_schema
+import hashlib
+from io import StringIO
 
+from django.core.cache import cache
+from django.conf import settings
+from django.http import HttpResponse
+from drf_yasg.utils import swagger_auto_schema
+from marshmallow.exceptions import ValidationError
+from rest_framework.response import Response
+from rest_framework.status import (
+    HTTP_422_UNPROCESSABLE_ENTITY, HTTP_500_INTERNAL_SERVER_ERROR
+)
 from metagenedb.apps.catalog.models import Gene
 from metagenedb.api.catalog.filters import GeneFilter
 from metagenedb.api.catalog.qparams_validators.gene import GeneQueryParams
@@ -8,6 +18,9 @@ from metagenedb.apps.catalog.serializers import GeneSerializer
 from .base import BulkViewSet
 
 
+MAX_FASTA_GENES = settings.MAX_FASTA_GENES
+
+
 class GeneViewSet(BulkViewSet):
     queryset = Gene.objects.select_related('taxonomy').prefetch_related('functions')
     serializer_class = GeneSerializer
@@ -15,11 +28,44 @@ class GeneViewSet(BulkViewSet):
     query_params_parser = GeneQueryParams
     lookup_field = 'gene_id'
 
+    def _get_queryset_count(self, queryset):
+        hash_object = hashlib.md5(str(queryset.query).encode('utf-8'))
+        redis_key = hash_object.hexdigest()
+        if redis_key in cache:
+            return cache.get(redis_key)
+        else:
+            return queryset.count()
+
+    def _build_fasta_response(self):
+        queryset = self.filter_queryset(self.get_queryset())
+        count = self._get_queryset_count(queryset)
+        if count >= MAX_FASTA_GENES:
+            error_message = f'Too many genes in the query, can obtain only up to {MAX_FASTA_GENES} fasta seq.'
+            return Response({'message': error_message}, status=HTTP_500_INTERNAL_SERVER_ERROR)
+        fasta_file = StringIO()
+        for gene in queryset.iterator():
+            fasta_file.write(gene.fasta)
+        # generate the file
+        response = HttpResponse(fasta_file.getvalue(), content_type='text/fasta')
+        filename = 'metagenedb_sequences.fasta'
+        response['Content-Disposition'] = 'attachment; filename=%s' % filename
+        return response
+
     @swagger_auto_schema(
         tags=['Genes'],
     )
-    def list(self, *args, **kwargs):
-        return super().list(*args, **kwargs)
+    def list(self, request, *args, **kwargs):
+        try:
+            query_params = self.query_params_parser().load(request.query_params)
+        except ValidationError as validation_error:
+            error_message = validation_error.normalized_messages()
+            error_message.update({
+                'allowed_query_params': ', '.join(self.query_params_parser().declared_fields.keys())
+            })
+            return Response(error_message, status=HTTP_422_UNPROCESSABLE_ENTITY)
+        if query_params.get('fasta', False) is True:
+            return self._build_fasta_response()
+        return super().list(request, *args, **kwargs)
 
     @swagger_auto_schema(
         tags=['Genes'],

backend/metagenedb/apps/catalog/models/gene.py

@@ -28,6 +28,10 @@ class Gene(models.Model):
     def __str__(self):
         return self.gene_id
 
+    @property
+    def fasta(self):
+        return f">{self.gene_id}\n{self.sequence}\n"
+
     class Meta:
         ordering = ['-gene_id']
 

backend/metagenedb/settings/__init__.py

-from .django import *  # noqa
 from .celery import *  # noqa
+from .django import *  # noqa
+from .metagenedb import *  # noqa

backend/metagenedb/settings/metagenedb.py

+"""
+Settings specific to metageneDB app and independant of Django
+"""
+import environ
+
+
+root = environ.Path(__file__) - 3  # get root of the project
+env = environ.Env()
+environ.Env.read_env(root('.env'))  # reading .env file
+
+# Maximum number of FASTA genes able to retrieve through API
+MAX_FASTA_GENES = env.str('MAX_FASTA_GENES', default=100000)

ci/kubernetes/backend.yaml

@@ -49,8 +49,6 @@ spec:
           value: '6379'
         - name: PORT
           value: "8000"
-        - name: DEBUG
-          value: "True"
         ports:
         - containerPort: 8000
         resources:

docker-compose.yaml

@@ -21,7 +21,7 @@ services:
   postgresql:
     container_name: postgresql
     image: postgres:11.4-alpine
-    shm_size: '2gb'
+    shm_size: '8gb'
     ports:
       - "5433:5432"
     volumes:

frontend/src/views/genes/genes.html

@@ -17,7 +17,7 @@
       ></v-text-field>
       </v-toolbar>
       <!-- Filters -->
-      <v-expansion-panel>
+      <v-expansion-panel v-model="showFilters" expand>
       <v-expansion-panel-content class="grey lighten-2">
         <template v-slot:header>
         <div>Show Filters</div>
@@ -189,47 +189,112 @@
       </v-expansion-panel-content>
       </v-expansion-panel>
       <!-- Table -->
-      <v-list v-if="!loadTable">
-        <v-data-table
-          :rows-per-page-items="rowsPerPageItems"
-          :pagination.sync="pagination"
-          :headers="headers"
-          :items="genes"
-          hide-actions
+
+      <v-card flat id="tablegenes" v-if="!loadTable">
+        <v-speed-dial
+          v-model="fab"
+          top
+          :right="downloadRight"
+          :left="downloadLeft"
+          :direction="downloadOpen"
+          transition="scale-transition"
+          v-if="showDownloads"
         >
-          <template v-slot:items="props">
-          <td>
-            <router-link :to="/gene-detail/ + props.item.gene_id">{{ props.item.gene_id }}</router-link>
-          </td>
-          <td>{{ props.item.name }}</td>
-          <td class="text-xs">{{ props.item.length }}</td>
-          <td class="text-xs">
-            {{ props.item.taxonomy }}
-            <v-btn :href="props.item.tax_url" icon flat small target="_blank" class="taxonomy--text mt-1 ml-0" v-if="props.item.taxonomy">
-            <v-icon small>
-              open_in_new
-            </v-icon>
-            </v-btn>
-          </td>
-          <td class="text-xs">
-            <template v-for="(kegg_item, kegg_index) in props.item.keggs">
-              {{ kegg_item.kegg_id }}
-              <v-btn :href="kegg_item.kegg_url" icon flat small target="_blank" class="kegg--text mt-1 ml-0" v-if="kegg_item.kegg_id">
+          <template v-slot:activator>
+            <v-tooltip bottom>
+              <template v-slot:activator="{ on }">
+                <v-btn
+                  :loading="loadingDownloads"
+                  v-model="fab"
+                  color="secondary"
+                  fab
+                  small
+                  class="pt-2"
+                  v-on="on"
+                >
+                  <v-icon>fas fa-download</v-icon>
+                  <v-icon>close</v-icon>
+                </v-btn>
+              </template>
+              <span>Downloads</span>
+            </v-tooltip>
+          </template>
+          <!-- FASTA file -->
+          <v-tooltip bottom>
+            <template v-slot:activator="{ on }">
+              <v-btn
+                fab
+                small
+                :loading="loadingFastaDownload"
+                :disabled="loadingFastaDownload"
+                color="primary lighten-3"
+                @click="downloadFasta"
+                v-on="on"
+              >
+                .FA
+              </v-btn>
+            </template>
+            <span>Download sequences (.fasta)</span>
+          </v-tooltip>
+          <!-- CSV -->
+          <v-tooltip bottom>
+            <template v-slot:activator="{ on }">
+              <v-btn
+                fab
+                dark
+                small
+                color="primary lighten-3"
+                v-on="on"
+                disabled
+              >
+                .CSV
+              </v-btn>
+            </template>
+            <span>Download metadata (.csv)</span>
+          </v-tooltip>
+        </v-speed-dial>
+        <v-list>
+          <v-data-table
+            :rows-per-page-items="rowsPerPageItems"
+            :pagination.sync="pagination"
+            :headers="headers"
+            :items="genes"
+            hide-actions
+          >
+            <template v-slot:items="props">
+            <td>
+              <router-link :to="/gene-detail/ + props.item.gene_id">{{ props.item.gene_id }}</router-link>
+            </td>
+            <td>{{ props.item.name }}</td>
+            <td class="text-xs">{{ props.item.length }}</td>
+            <td class="text-xs">
+              {{ props.item.taxonomy }}
+              <v-btn :href="props.item.tax_url" icon flat small target="_blank" class="taxonomy--text mt-1 ml-0" v-if="props.item.taxonomy">
               <v-icon small>
                 open_in_new
               </v-icon>
               </v-btn>
+            </td>
+            <td class="text-xs">
+              <template v-for="(kegg_item, kegg_index) in props.item.keggs">
+                {{ kegg_item.kegg_id }}
+                <v-btn :href="kegg_item.kegg_url" icon flat small target="_blank" class="kegg--text mt-1 ml-0" v-if="kegg_item.kegg_id">
+                <v-icon small>
+                  open_in_new
+                </v-icon>
+                </v-btn>
+              </template>
+            </td>
+            <td class="text-xs">
+              <template v-for="(eggnog_item, eggnog_index) in props.item.eggnogs">
+                {{ eggnog_item.eggnog_id }}
+              </template>
+            </td>
+            <td class="text-xs">{{ props.item.source }}</td>
             </template>
-          </td>
-          <td class="text-xs">
-            <template v-for="(eggnog_item, eggnog_index) in props.item.eggnogs">
-              {{ eggnog_item.eggnog_id }}
-            </template>
-          </td>
-          <td class="text-xs">{{ props.item.source }}</td>
-          </template>
-        </v-data-table>
-      </v-list>
+          </v-data-table>
+        </v-list>
+      </v-card>
       <!-- Loading bar -->
       <v-card class="text-xs-center" v-if="!requestDone">
         <v-progress-linear

frontend/src/views/genes/genes.js

@@ -15,6 +15,7 @@ export default {
       maxNumberPages: 50000,
       tooManyPages: false,
       // Filters
+      showFilters: [false],
       // - Gene info
       geneSource: 'all',
       searchGeneName: null,
@@ -29,6 +30,9 @@ export default {
       // Information about request
       loadTable: true,
       requestDone: false,
+      // Download FASTA loading
+      downloadReady: true,
+      fab: false,
     };
   },
   computed: {
@@ -104,6 +108,33 @@ export default {
         return numberPages + 1;
       }
       return numberPages;
+    },
+    loadingFastaDownload() {
+      return !this.downloadReady;
+    },
+    loadingDownloads() {
+      if (this.loadingFastaDownload) {
+        return true;
+      }
+      return false;
+    },
+    showDownloads() {
+      if (this.count != 0 && this.count <= 100000) {
+        return true;
+      }
+      return false;
+    },
+    downloadRight() {
+      return !this.showFilters[0];
+    },
+    downloadLeft() {
+      return this.showFilters[0];
+    },
+    downloadOpen() {
+      if (this.showFilters[0]) {
+        return 'right';
+      }
+      return 'left';
     }
   },
   mounted() {
@@ -165,6 +196,35 @@ export default {
           this.requestDone = true;
         });
     },
+    forceFileDownload(response){
+      const url = window.URL.createObjectURL(new Blob([response.data]))
+      const link = document.createElement('a')
+      link.href = url
+      link.setAttribute('download', 'metagenedb_sequences.fasta') //or any other extension
+      document.body.appendChild(link)
+      link.click()
+    },
+    downloadFasta() {
+      this.downloadReady = false;
+      var qParams = this.qParams;
+      qParams.fasta = "true";
+      delete qParams['page_size']
+      delete qParams['page']
+      axios.get('/api/catalog/v1/genes', {
+        params: qParams,
+        headers: {
+          Accept: 'application/json',
+        },
+      })
+        .then((response) => {
+          this.downloadReady = true;
+          this.forceFileDownload(response);
+        })
+        .catch((error) => {
+          console.error(error);
+          this.downloadReady = true;
+        });
+    },
     emptyGeneInformationFilter() {
       this.geneSource = 'all';
       this.filterGeneLength = false;

frontend/src/views/genes/genes.scss

+/* This is for documentation purposes and will not be needed in your application */
+#tablegenes .v-speed-dial {
+  position: absolute;
+  margin-top: -45px;
+  margin-right: 50px;
+}
+
+#tablegenes .v-btn--floating {
+  position: relative;
+}
\ No newline at end of file

frontend/src/views/genes/genes.vue

 <template src="./genes.html" lang="html"></template>
-<script src="./genes.js" lang="js"></script>
\ No newline at end of file
+<script src="./genes.js" lang="js"></script>
+<style src="./genes.scss" lang="scss" scoped></style>
\ No newline at end of file