Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Metagenomics
metagenedb
Commits
7d853553
Commit
7d853553
authored
Nov 19, 2019
by
Kenzo-Hugo Hillion
♻
Browse files
add vizualization dev tool for profiling and options for script
parent
4a09070c
Changes
3
Hide whitespace changes
Inline
Side-by-side
backend/Pipfile
View file @
7d853553
...
...
@@ -29,6 +29,7 @@ factory-boy = "*"
pytest-factoryboy
=
"*"
pylint
=
"*"
mock
=
"*"
snakeviz
=
"*"
[packages]
certifi
=
"*"
...
...
backend/Pipfile.lock
View file @
7d853553
{
"_meta"
:
{
"hash"
:
{
"sha256"
:
"
615d076688d5e77a4beecac7c374bd91fb8d8190f6cc17992bd7dd52b525207d
"
"sha256"
:
"
f81cb1460e5f4b94a712ee3af4bd701917a32360529825962b4e146f0efbc9a4
"
},
"pipfile-spec"
:
6
,
"requires"
:
{
...
...
@@ -978,6 +978,14 @@
"index"
:
"pypi"
,
"version"
:
"==1.13.0"
},
"snakeviz"
:
{
"hashes"
:
[
"sha256:5e30f144edb17d875b46cb5f82bd3e67fb5018e534ecc1a94e092ef3ce932c25"
,
"sha256:80acc9c204aeb1e089f209a4c79bb5940dc40b6536a5184c1778a3f448634885"
],
"index"
:
"pypi"
,
"version"
:
"==2.0.1"
},
"terminado"
:
{
"hashes"
:
[
"sha256:4804a774f802306a7d9af7322193c5390f1da0abb429e082a10ef1d46e6fb2c2"
,
...
...
backend/scripts/populate_db/import_igc_data.py
View file @
7d853553
...
...
@@ -11,7 +11,7 @@ from slugify import slugify
from
metagenedb.common.utils.parsers
import
IGCLineParser
logging
.
basicConfig
()
logging
.
basicConfig
(
format
=
'[%(asctime)s] %(levelname)s:%(name)s:%(message)s'
)
logger
=
logging
.
getLogger
()
...
...
@@ -88,7 +88,7 @@ class ImportIGCGenes(object):
gene_dict
.
pop
(
'functions'
)
return
gene_dict
def
load_annotation_file_to_db_in_chunks
(
self
,
chunk_size
=
1000
):
def
load_annotation_file_to_db_in_chunks
(
self
,
chunk_size
=
1000
,
test
=
False
):
with
open
(
self
.
annotation_file
,
'r'
)
as
file
:
while
True
:
chunk_genes
=
list
(
islice
(
file
,
chunk_size
))
...
...
@@ -104,6 +104,8 @@ class ImportIGCGenes(object):
self
.
skipped_genes
+=
len
(
genes
)
self
.
processed_genes
+=
len
(
chunk_genes
)
logger
.
info
(
"%s Genes processed so far..."
,
self
.
processed_genes
)
if
test
is
True
:
break
logger
.
info
(
"[DONE] %s/%s Genes created."
,
self
.
created_genes
,
self
.
total_genes
)
logger
.
info
(
"[DONE] %s/%s Genes updated."
,
self
.
updated_genes
,
self
.
total_genes
)
logger
.
info
(
"[DONE] %s/%s Genes skipped."
,
self
.
skipped_genes
,
self
.
total_genes
)
...
...
@@ -117,8 +119,10 @@ def parse_arguments():
# Common arguments for analysis and annotations
parser
.
add_argument
(
'annotation'
,
help
=
'IGC annotation file'
)
parser
.
add_argument
(
'--url'
,
help
=
'base URL of the instance.'
,
default
=
'http://localhost/'
)
parser
.
add_argument
(
'--chunk_size'
,
type
=
int
,
default
=
1000
,
help
=
'How many genes to handle and create in the same time.'
)
parser
.
add_argument
(
'--skip_taxonomy'
,
action
=
'store_true'
,
help
=
'Skip taxonomy information from genes.'
)
parser
.
add_argument
(
'--skip_functions'
,
action
=
'store_true'
,
help
=
'Skip functions information from genes.'
)
parser
.
add_argument
(
'--test'
,
action
=
'store_true'
,
help
=
'Run only on first chunk.'
)
parser
.
add_argument
(
'-v'
,
'--verbose'
,
action
=
'store_true'
)
try
:
...
...
@@ -133,7 +137,7 @@ def run():
logger
.
setLevel
(
logging
.
INFO
)
import_igc_genes
=
ImportIGCGenes
(
args
.
annotation
,
args
.
url
,
skip_tax
=
args
.
skip_taxonomy
,
skip_functions
=
args
.
skip_functions
)
import_igc_genes
.
load_annotation_file_to_db_in_chunks
()
import_igc_genes
.
load_annotation_file_to_db_in_chunks
(
chunk_size
=
args
.
chunk_size
,
test
=
args
.
test
)
if
__name__
==
"__main__"
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment