Merge branch '57-update-igc-script' into 'dev'

Deal with functions in importing IGC genes

Closes #57

See merge request !15

backend/metagenedb/apps/catalog/serializers/gene.py

@@ -41,7 +41,6 @@ class GeneListSerializer(BulkListSerializer):
         GeneFunction.objects.bulk_create(
             self._generate_gene_function_mapping(values, genes)
         )
-        print("hi")
 
     def create(self, validated_data):
         instances = super().create(validated_data)

backend/scripts/populate_db/import_igc_data.py

@@ -6,6 +6,7 @@ import sys
 from itertools import islice
 
 from bioapi import MetageneDBCatalogGeneAPI, MetageneDBCatalogTaxonomyAPI
+from requests.exceptions import HTTPError
 from slugify import slugify
 
 from metagenedb.common.utils.parsers import IGCLineParser
@@ -67,8 +68,6 @@ class ImportIGCGenes(object):
             gene_dict.update(
                 {'taxonomy': resp_dict['results'][0]['tax_id']}
             )
-        else:
-            gene_dict.update({'taxonomy': None})
         return gene_dict
 
     def _parse_gene(self, raw_line, selected_keys=SELECTED_KEYS):
@@ -83,10 +82,10 @@ class ImportIGCGenes(object):
     def _clean_gene(self, gene_dict):
         gene_dict['gene_name'] = gene_dict['gene_id']
         gene_dict['gene_id'] = slugify(gene_dict['gene_id'])
-        gene_dict['functions'] = [{'function_id': gene_dict.pop('kegg_ko')}]
+        gene_dict['functions'] = [gene_dict.pop('kegg_ko')]
         if self.skip_tax:
             gene_dict.pop('taxonomy')
-        if self.skip_functions:
+        if self.skip_functions or 'unknown' in gene_dict['functions']:
             gene_dict.pop('functions')
         return gene_dict
 
@@ -97,9 +96,12 @@ class ImportIGCGenes(object):
                 if not chunk_genes:
                     break
                 genes = [self._clean_gene(self._select_taxonomy(self._parse_gene(i))) for i in chunk_genes]
-                response = self.metagenedb_gene_api.put(genes)
-                self.created_genes += response.get('created').get('count')
-                self.updated_genes += response.get('updated').get('count')
+                try:
+                    response = self.metagenedb_gene_api.put(genes)
+                    self.created_genes += response.get('created').get('count')
+                    self.updated_genes += response.get('updated').get('count')
+                except HTTPError as http_error:
+                    logging.warning("%s: %s; %s", http_error, http_error.response.json(), genes)
                 self.processed_genes += len(chunk_genes)
                 logger.info("%s Genes processed so far...", self.processed_genes)
         logger.info("[DONE] %s/%s Genes created.", self.created_genes, self.total_genes)

backend/scripts/populate_db/test_import_igc_data.py

@@ -85,7 +85,7 @@ class TestCleanGene(TestCase):
             'gene_name': 'gene.01',
             'length': 135,
             'taxonomy': 'Taxo',
-            'functions': [{'function_id': 'K00001'}]
+            'functions': ['K00001']
         }
         test_gene_dict = self.import_igc_genes._clean_gene(self.gene_dict)
         self.assertDictEqual(test_gene_dict, expected_gene_dict)
@@ -96,7 +96,7 @@ class TestCleanGene(TestCase):
             'gene_id': 'gene-01',
             'gene_name': 'gene.01',
             'length': 135,
-            'functions': [{'function_id': 'K00001'}]
+            'functions': ['K00001']
         }
         test_gene_dict = self.import_igc_genes._clean_gene(self.gene_dict)
         self.assertDictEqual(test_gene_dict, expected_gene_dict)
@@ -112,6 +112,20 @@ class TestCleanGene(TestCase):
         test_gene_dict = self.import_igc_genes._clean_gene(self.gene_dict)
         self.assertDictEqual(test_gene_dict, expected_gene_dict)
 
+    def test_unknown_kegg_ko(self):
+        gene_dict = {
+            'gene_id': 'gene.01',
+            'length': 135,
+            'kegg_ko': 'unknown'
+        }
+        expected_gene_dict = {
+            'gene_id': 'gene-01',
+            'gene_name': 'gene.01',
+            'length': 135
+        }
+        test_gene_dict = self.import_igc_genes._clean_gene(gene_dict)
+        self.assertDictEqual(test_gene_dict, expected_gene_dict)
+
 
 class TestSelectTaxonomy(APITestCase):
 
@@ -179,8 +193,7 @@ class TestSelectTaxonomy(APITestCase):
         }
         expected_dict = {
             'gene_id': 'gene',
-            'length': 135,
-            'taxonomy': None
+            'length': 135
         }
         tested_dict = self.import_igc_genes._select_taxonomy(gene_dict)
         self.assertDictEqual(tested_dict, expected_dict)

frontend/src/views/Genes.vue

@@ -25,7 +25,7 @@
                 <td>{{ props.item.gene_name }}</td>
                 <td class="text-xs">{{ props.item.length }}</td>
                 <td class="text-xs"></td>
-                <td class="text-xs">{{ props.item.functions[0] }}</td>
+                <td class="text-xs">{{ props.item.functions }}</td>
               </template>
             </v-data-table>
           </v-list>
@@ -68,7 +68,7 @@ export default {
   },
   methods: {
     getGenes() {
-      axios.get('/api/catalog/v1/functions/', {
+      axios.get('/api/catalog/v1/genes/', {
         headers: {
           Accept: 'application/json',
         },