Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Metagenomics
metagenedb
Commits
2e85bc4f
Commit
2e85bc4f
authored
Aug 26, 2019
by
Kenzo-Hugo Hillion
♻
Browse files
Use API to create genes
parent
a77ffe7b
Changes
3
Hide whitespace changes
Inline
Side-by-side
backend/metagenedb/common/utils/api/baseapi.py
View file @
2e85bc4f
...
...
@@ -9,9 +9,9 @@ _LOGGER = logging.getLogger(__name__)
class
LoggedSession
(
requests
.
Session
):
def
request
(
self
,
method
,
url
,
**
kwargs
):
_LOGGER
.
info
(
f
"
{
method
.
upper
()
}
{
url
}
"
)
_LOGGER
.
debug
(
f
"
{
method
.
upper
()
}
{
url
}
"
)
response
=
super
().
request
(
method
,
url
,
**
kwargs
)
_LOGGER
.
info
(
f
"STATUS CODE:
{
response
.
status_code
}
"
)
_LOGGER
.
debug
(
f
"STATUS CODE:
{
response
.
status_code
}
"
)
return
response
...
...
backend/scripts/populate_db/import_igc_data.py
View file @
2e85bc4f
...
...
@@ -74,6 +74,7 @@ class ImportIGCGenes(object):
self
.
skip_functions
=
skip_functions
def
_clean_gene
(
self
,
gene_dict
):
gene_dict
[
'gene_name'
]
=
gene_dict
[
'gene_id'
]
gene_dict
[
'gene_id'
]
=
slugify
(
gene_dict
[
'gene_id'
])
if
self
.
skip_tax
:
gene_dict
.
pop
(
'taxonomy'
)
...
...
@@ -85,7 +86,7 @@ class ImportIGCGenes(object):
clean_gene_dict
=
self
.
_clean_gene
(
gene_dict
)
try
:
gene_id
=
clean_gene_dict
[
'gene_id'
]
self
.
metagenedb_gene_api
.
get
(
gene_id
)
# Try to get obj to check if it exists
gene_obj
=
self
.
metagenedb_gene_api
.
get
(
gene_id
)
# Try to get obj to check if it exists
self
.
metagenedb_gene_api
.
put
(
gene_id
,
clean_gene_dict
)
except
HTTPError
:
self
.
metagenedb_gene_api
.
post
(
clean_gene_dict
)
...
...
@@ -96,8 +97,8 @@ class ImportIGCGenes(object):
gene_dict_with_taxo
=
select_taxonomy
(
gene_dict
)
try
:
self
.
_upsert_gene
(
gene_dict_with_taxo
)
except
Validation
Error
as
e
:
_LOGGER
.
warning
(
f
"
{
e
.
__dict__
}
for gene_id:
{
gene_dict
.
get
(
'gene_id'
)
}
. Insertion skipped."
)
except
HTTP
Error
as
e
:
_LOGGER
.
warning
(
f
"
{
e
.
response
.
json
()
}
for gene_id:
{
gene_dict
.
get
(
'gene_id'
)
}
. Insertion skipped."
)
def
load_annotation_file_to_db_in_chunks
(
self
,
chunk_size
=
100000
):
processed_genes
=
0
...
...
@@ -119,7 +120,7 @@ def parse_arguments():
parser
=
argparse
.
ArgumentParser
(
description
=
'Populate database from a given IGC annotation file.'
)
# Common arguments for analysis and annotations
parser
.
add_argument
(
'annotation'
,
help
=
'IGC annotation file'
)
parser
.
add_argument
(
'url'
,
help
=
'base URL of the instance.'
,
default
=
'http://localhost/'
)
parser
.
add_argument
(
'
--
url'
,
help
=
'base URL of the instance.'
,
default
=
'http://localhost/'
)
parser
.
add_argument
(
'--skip_taxonomy'
,
action
=
'store_true'
,
help
=
'Skip taxonomy information from genes.'
)
parser
.
add_argument
(
'--skip_functions'
,
action
=
'store_true'
,
help
=
'Skip functions information from genes.'
)
...
...
@@ -131,7 +132,9 @@ def parse_arguments():
def
run
():
args
=
parse_arguments
()
load_annotation_file_to_db_in_chunks
(
args
.
annotation
,
args
.
url
)
import_igc_genes
=
ImportIGCGenes
(
args
.
annotation
,
args
.
url
,
skip_tax
=
args
.
skip_taxonomy
,
skip_functions
=
args
.
skip_functions
)
import_igc_genes
.
load_annotation_file_to_db_in_chunks
()
if
__name__
==
"__main__"
:
...
...
backend/scripts/populate_db/test_import_igc_data.py
View file @
2e85bc4f
...
...
@@ -141,6 +141,40 @@ class TestUpsertGene(APITestCase):
self
.
assertEqual
(
self
.
api_mock
.
get
(
'test-gene01'
)[
'length'
],
356
)
class
TestCleanGene
(
TestCase
):
def
setUp
(
self
):
self
.
import_igc_genes
=
ImportIGCGenes
(
'test'
,
'test'
)
self
.
gene_dict
=
{
'gene_id'
:
'gene.01'
,
'length'
:
135
,
'taxonomy'
:
'Taxo'
,
'kegg_ko'
:
'Genus'
}
def
test_clean_gene
(
self
):
expected_gene_dict
=
{
'gene_id'
:
'gene-01'
,
'gene_name'
:
'gene.01'
,
'length'
:
135
,
'taxonomy'
:
'Taxo'
,
'kegg_ko'
:
'Genus'
}
test_gene_dict
=
self
.
import_igc_genes
.
_clean_gene
(
self
.
gene_dict
)
self
.
assertDictEqual
(
test_gene_dict
,
expected_gene_dict
)
def
test_clean_gene_skip_taxo
(
self
):
self
.
import_igc_genes
.
skip_tax
=
True
expected_gene_dict
=
{
'gene_id'
:
'gene-01'
,
'gene_name'
:
'gene.01'
,
'length'
:
135
,
'kegg_ko'
:
'Genus'
}
test_gene_dict
=
self
.
import_igc_genes
.
_clean_gene
(
self
.
gene_dict
)
self
.
assertDictEqual
(
test_gene_dict
,
expected_gene_dict
)
@
pytest
.
mark
.
django_db
class
TestSelectTaxonomy
(
TestCase
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment