Commit 7d853553 authored by Kenzo-Hugo Hillion's avatar Kenzo-Hugo Hillion
Browse files

add vizualization dev tool for profiling and options for script

parent 4a09070c
......@@ -29,6 +29,7 @@ factory-boy = "*"
pytest-factoryboy = "*"
pylint = "*"
mock = "*"
snakeviz = "*"
[packages]
certifi = "*"
......
{
"_meta": {
"hash": {
"sha256": "615d076688d5e77a4beecac7c374bd91fb8d8190f6cc17992bd7dd52b525207d"
"sha256": "f81cb1460e5f4b94a712ee3af4bd701917a32360529825962b4e146f0efbc9a4"
},
"pipfile-spec": 6,
"requires": {
......@@ -978,6 +978,14 @@
"index": "pypi",
"version": "==1.13.0"
},
"snakeviz": {
"hashes": [
"sha256:5e30f144edb17d875b46cb5f82bd3e67fb5018e534ecc1a94e092ef3ce932c25",
"sha256:80acc9c204aeb1e089f209a4c79bb5940dc40b6536a5184c1778a3f448634885"
],
"index": "pypi",
"version": "==2.0.1"
},
"terminado": {
"hashes": [
"sha256:4804a774f802306a7d9af7322193c5390f1da0abb429e082a10ef1d46e6fb2c2",
......
......@@ -11,7 +11,7 @@ from slugify import slugify
from metagenedb.common.utils.parsers import IGCLineParser
logging.basicConfig()
logging.basicConfig(format='[%(asctime)s] %(levelname)s:%(name)s:%(message)s')
logger = logging.getLogger()
......@@ -88,7 +88,7 @@ class ImportIGCGenes(object):
gene_dict.pop('functions')
return gene_dict
def load_annotation_file_to_db_in_chunks(self, chunk_size=1000):
def load_annotation_file_to_db_in_chunks(self, chunk_size=1000, test=False):
with open(self.annotation_file, 'r') as file:
while True:
chunk_genes = list(islice(file, chunk_size))
......@@ -104,6 +104,8 @@ class ImportIGCGenes(object):
self.skipped_genes += len(genes)
self.processed_genes += len(chunk_genes)
logger.info("%s Genes processed so far...", self.processed_genes)
if test is True:
break
logger.info("[DONE] %s/%s Genes created.", self.created_genes, self.total_genes)
logger.info("[DONE] %s/%s Genes updated.", self.updated_genes, self.total_genes)
logger.info("[DONE] %s/%s Genes skipped.", self.skipped_genes, self.total_genes)
......@@ -117,8 +119,10 @@ def parse_arguments():
# Common arguments for analysis and annotations
parser.add_argument('annotation', help='IGC annotation file')
parser.add_argument('--url', help='base URL of the instance.', default='http://localhost/')
parser.add_argument('--chunk_size', type=int, default=1000, help='How many genes to handle and create in the same time.')
parser.add_argument('--skip_taxonomy', action='store_true', help='Skip taxonomy information from genes.')
parser.add_argument('--skip_functions', action='store_true', help='Skip functions information from genes.')
parser.add_argument('--test', action='store_true', help='Run only on first chunk.')
parser.add_argument('-v', '--verbose', action='store_true')
try:
......@@ -133,7 +137,7 @@ def run():
logger.setLevel(logging.INFO)
import_igc_genes = ImportIGCGenes(args.annotation, args.url,
skip_tax=args.skip_taxonomy, skip_functions=args.skip_functions)
import_igc_genes.load_annotation_file_to_db_in_chunks()
import_igc_genes.load_annotation_file_to_db_in_chunks(chunk_size=args.chunk_size, test=args.test)
if __name__ == "__main__":
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment