Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Metagenomics
metagenedb
Commits
da536d6d
Commit
da536d6d
authored
Oct 30, 2019
by
Kenzo-Hugo Hillion
♻
Browse files
Merge branch '57-update-igc-script' into 'dev'
Deal with functions in importing IGC genes Closes #57 See merge request
!15
parents
7f90bf00
8f59511f
Pipeline
#17243
passed with stages
in 2 minutes and 30 seconds
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
backend/metagenedb/apps/catalog/serializers/gene.py
View file @
da536d6d
...
...
@@ -41,7 +41,6 @@ class GeneListSerializer(BulkListSerializer):
GeneFunction
.
objects
.
bulk_create
(
self
.
_generate_gene_function_mapping
(
values
,
genes
)
)
print
(
"hi"
)
def
create
(
self
,
validated_data
):
instances
=
super
().
create
(
validated_data
)
...
...
backend/scripts/populate_db/import_igc_data.py
View file @
da536d6d
...
...
@@ -6,6 +6,7 @@ import sys
from
itertools
import
islice
from
bioapi
import
MetageneDBCatalogGeneAPI
,
MetageneDBCatalogTaxonomyAPI
from
requests.exceptions
import
HTTPError
from
slugify
import
slugify
from
metagenedb.common.utils.parsers
import
IGCLineParser
...
...
@@ -67,8 +68,6 @@ class ImportIGCGenes(object):
gene_dict
.
update
(
{
'taxonomy'
:
resp_dict
[
'results'
][
0
][
'tax_id'
]}
)
else
:
gene_dict
.
update
({
'taxonomy'
:
None
})
return
gene_dict
def
_parse_gene
(
self
,
raw_line
,
selected_keys
=
SELECTED_KEYS
):
...
...
@@ -83,10 +82,10 @@ class ImportIGCGenes(object):
def
_clean_gene
(
self
,
gene_dict
):
gene_dict
[
'gene_name'
]
=
gene_dict
[
'gene_id'
]
gene_dict
[
'gene_id'
]
=
slugify
(
gene_dict
[
'gene_id'
])
gene_dict
[
'functions'
]
=
[
{
'function_id'
:
gene_dict
.
pop
(
'kegg_ko'
)
}
]
gene_dict
[
'functions'
]
=
[
gene_dict
.
pop
(
'kegg_ko'
)]
if
self
.
skip_tax
:
gene_dict
.
pop
(
'taxonomy'
)
if
self
.
skip_functions
:
if
self
.
skip_functions
or
'unknown'
in
gene_dict
[
'functions'
]
:
gene_dict
.
pop
(
'functions'
)
return
gene_dict
...
...
@@ -97,9 +96,12 @@ class ImportIGCGenes(object):
if
not
chunk_genes
:
break
genes
=
[
self
.
_clean_gene
(
self
.
_select_taxonomy
(
self
.
_parse_gene
(
i
)))
for
i
in
chunk_genes
]
response
=
self
.
metagenedb_gene_api
.
put
(
genes
)
self
.
created_genes
+=
response
.
get
(
'created'
).
get
(
'count'
)
self
.
updated_genes
+=
response
.
get
(
'updated'
).
get
(
'count'
)
try
:
response
=
self
.
metagenedb_gene_api
.
put
(
genes
)
self
.
created_genes
+=
response
.
get
(
'created'
).
get
(
'count'
)
self
.
updated_genes
+=
response
.
get
(
'updated'
).
get
(
'count'
)
except
HTTPError
as
http_error
:
logging
.
warning
(
"%s: %s; %s"
,
http_error
,
http_error
.
response
.
json
(),
genes
)
self
.
processed_genes
+=
len
(
chunk_genes
)
logger
.
info
(
"%s Genes processed so far..."
,
self
.
processed_genes
)
logger
.
info
(
"[DONE] %s/%s Genes created."
,
self
.
created_genes
,
self
.
total_genes
)
...
...
backend/scripts/populate_db/test_import_igc_data.py
View file @
da536d6d
...
...
@@ -85,7 +85,7 @@ class TestCleanGene(TestCase):
'gene_name'
:
'gene.01'
,
'length'
:
135
,
'taxonomy'
:
'Taxo'
,
'functions'
:
[
{
'function_id'
:
'K00001'
}
]
'functions'
:
[
'K00001'
]
}
test_gene_dict
=
self
.
import_igc_genes
.
_clean_gene
(
self
.
gene_dict
)
self
.
assertDictEqual
(
test_gene_dict
,
expected_gene_dict
)
...
...
@@ -96,7 +96,7 @@ class TestCleanGene(TestCase):
'gene_id'
:
'gene-01'
,
'gene_name'
:
'gene.01'
,
'length'
:
135
,
'functions'
:
[
{
'function_id'
:
'K00001'
}
]
'functions'
:
[
'K00001'
]
}
test_gene_dict
=
self
.
import_igc_genes
.
_clean_gene
(
self
.
gene_dict
)
self
.
assertDictEqual
(
test_gene_dict
,
expected_gene_dict
)
...
...
@@ -112,6 +112,20 @@ class TestCleanGene(TestCase):
test_gene_dict
=
self
.
import_igc_genes
.
_clean_gene
(
self
.
gene_dict
)
self
.
assertDictEqual
(
test_gene_dict
,
expected_gene_dict
)
def
test_unknown_kegg_ko
(
self
):
gene_dict
=
{
'gene_id'
:
'gene.01'
,
'length'
:
135
,
'kegg_ko'
:
'unknown'
}
expected_gene_dict
=
{
'gene_id'
:
'gene-01'
,
'gene_name'
:
'gene.01'
,
'length'
:
135
}
test_gene_dict
=
self
.
import_igc_genes
.
_clean_gene
(
gene_dict
)
self
.
assertDictEqual
(
test_gene_dict
,
expected_gene_dict
)
class
TestSelectTaxonomy
(
APITestCase
):
...
...
@@ -179,8 +193,7 @@ class TestSelectTaxonomy(APITestCase):
}
expected_dict
=
{
'gene_id'
:
'gene'
,
'length'
:
135
,
'taxonomy'
:
None
'length'
:
135
}
tested_dict
=
self
.
import_igc_genes
.
_select_taxonomy
(
gene_dict
)
self
.
assertDictEqual
(
tested_dict
,
expected_dict
)
frontend/src/views/Genes.vue
View file @
da536d6d
...
...
@@ -25,7 +25,7 @@
<td>
{{
props
.
item
.
gene_name
}}
</td>
<td
class=
"text-xs"
>
{{
props
.
item
.
length
}}
</td>
<td
class=
"text-xs"
></td>
<td
class=
"text-xs"
>
{{
props
.
item
.
functions
[
0
]
}}
</td>
<td
class=
"text-xs"
>
{{
props
.
item
.
functions
}}
</td>
</
template
>
</v-data-table>
</v-list>
...
...
@@ -68,7 +68,7 @@ export default {
},
methods
:
{
getGenes
()
{
axios
.
get
(
'
/api/catalog/v1/
function
s/
'
,
{
axios
.
get
(
'
/api/catalog/v1/
gene
s/
'
,
{
headers
:
{
Accept
:
'
application/json
'
,
},
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment