Skip to content
Snippets Groups Projects
Commit 7fab71d0 authored by Kenzo-Hugo Hillion's avatar Kenzo-Hugo Hillion :recycle:
Browse files

Fix source when loading KEGG using API

parent 8328827b
No related branches found
No related tags found
2 merge requests!59Prod,!14Resolve "Update scripts to use new `PUT` and `POST` to populate database"
Pipeline #14617 passed with stages
in 2 minutes and 16 seconds
.DS_Store
__pycache__/
*.egg-info/
.env
......
......@@ -18,6 +18,8 @@ KEGG_KO_LIST_API = "http://rest.kegg.jp/list/ko"
class ImportKEGGKO(object):
METAGENEDB_FUNCTION_API = MetageneDBCatalogFunctionAPI
ORM_SOURCE_KEY = 'source'
KEGG_SOURCE = 'kegg'
def __init__(self, url, kegg_ko_list_api=KEGG_KO_LIST_API):
self.kegg_ko_list_api = kegg_ko_list_api
......@@ -25,7 +27,7 @@ class ImportKEGGKO(object):
self.processed_kegg = 0
self.created_kegg = 0
self.updated_kegg = 0
self.skipped_kegg = 0
# self.skipped_kegg = 0
def load_all_kegg_ko(self, chunk_size=1000):
all_ko_response = requests.get(self.kegg_ko_list_api)
......@@ -34,6 +36,8 @@ class ImportKEGGKO(object):
self.total_kegg_nb = len(all_ko)
for chunk in generate_chunks(all_ko, chunk_size):
ko_chunk = [KEGGLineParser.ko_list(i) for i in chunk]
for i in ko_chunk:
i.update({self.ORM_SOURCE_KEY: self.KEGG_SOURCE})
response = self.metagenedb_function_api.put(ko_chunk)
self.created_kegg += response.get('created').get('count')
self.updated_kegg += response.get('updated').get('count')
......@@ -42,7 +46,7 @@ class ImportKEGGKO(object):
time.sleep(1)
logger.info("[DONE] %s/%s KEGG KO created.", self.created_kegg, self.total_kegg_nb)
logger.info("[DONE] %s/%s KEGG KO updated.", self.updated_kegg, self.total_kegg_nb)
logger.info("[DONE] %s/%s KEGG KO skipped.", self.skipped_kegg, self.total_kegg_nb)
# logger.info("[DONE] %s/%s KEGG KO skipped.", self.skipped_kegg, self.total_kegg_nb)
def parse_arguments():
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment