Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Metagenomics
metagenedb
Commits
9201ea5e
Commit
9201ea5e
authored
Nov 05, 2019
by
Kenzo-Hugo Hillion
♻
Browse files
Add script to build hierarchy from backend
parent
266e125e
Pipeline
#17617
passed with stages
in 2 minutes and 46 seconds
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
backend/metagenedb/apps/catalog/management/__init__.py
0 → 100644
View file @
9201ea5e
backend/metagenedb/apps/catalog/management/commands/__init__.py
0 → 100644
View file @
9201ea5e
backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py
0 → 100644
View file @
9201ea5e
import
logging
from
django.core.management.base
import
BaseCommand
from
metagenedb.apps.catalog.models
import
Taxonomy
logging
.
basicConfig
(
format
=
'[%(asctime)s] %(levelname)s:%(name)s:%(message)s'
)
logger
=
logging
.
getLogger
(
__name__
)
SELECT_RELATED_PARENT
=
"parent{}"
.
format
(
"__parent"
*
40
)
class
HierarchyBuilder
:
def
__init__
(
self
,
queryset
):
self
.
queryset
=
queryset
self
.
total_tax
=
queryset
.
count
()
self
.
processed_tax
=
0
self
.
hierarchy_built
=
0
self
.
hierarchy_failed
=
0
def
build_all
(
self
,
chunk_size
=
8000
):
logger
.
info
(
"Building all hierarchy for all %s taxonomy items..."
,
self
.
total_tax
)
for
taxonomy
in
self
.
queryset
.
iterator
(
chunk_size
=
chunk_size
):
try
:
hierarchy
=
taxonomy
.
parental_hierarchy
# noqa
self
.
hierarchy_built
+=
1
except
Exception
:
self
.
hierarchy_failed
+=
1
self
.
processed_tax
+=
1
if
self
.
processed_tax
%
10000
==
0
:
logger
.
info
(
"%s/%s Taxonomy processed so far..."
,
self
.
processed_tax
,
self
.
total_tax
)
break
logger
.
info
(
"[DONE] %s/%s Hierarchy built."
,
self
.
hierarchy_built
,
self
.
total_tax
)
logger
.
info
(
"[DONE] %s/%s Hierarchy build skipped."
,
self
.
hierarchy_failed
,
self
.
total_tax
)
class
Command
(
BaseCommand
):
help
=
'Build hierarchy for taxonomy entries.'
def
set_logger_level
(
self
,
verbosity
):
if
verbosity
>
2
:
logger
.
setLevel
(
logging
.
DEBUG
)
elif
verbosity
>
1
:
logger
.
setLevel
(
logging
.
INFO
)
def
get_queryset
(
self
):
return
Taxonomy
.
objects
.
select_related
(
SELECT_RELATED_PARENT
,
"superkingdom"
,
"kingdom"
,
"phylum"
,
"class_rank"
,
"order"
,
"family"
,
"genus"
,
"species"
).
all
()
def
handle
(
self
,
*
args
,
**
options
):
self
.
set_logger_level
(
int
(
options
[
'verbosity'
]))
hierarchy_builder
=
HierarchyBuilder
(
self
.
get_queryset
())
hierarchy_builder
.
build_all
()
ci/kubernetes/postgresql.yaml
View file @
9201ea5e
...
...
@@ -54,10 +54,10 @@ spec:
name
:
postgresql
resources
:
requests
:
memory
:
"
256
Mi"
memory
:
"
512
Mi"
cpu
:
"
100m"
limits
:
memory
:
"
512
Mi"
memory
:
"
1024
Mi"
cpu
:
"
500m"
volumeMounts
:
-
name
:
postgresql
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment