Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Metagenomics
metagenedb
Commits
e59f15d7
Commit
e59f15d7
authored
Nov 05, 2019
by
Kenzo-Hugo Hillion
♻
Browse files
add backend ressources and test option for build hierarchy
parent
f90f67d0
Pipeline
#17622
passed with stages
in 2 minutes and 27 seconds
Changes
2
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py
View file @
e59f15d7
...
...
@@ -7,7 +7,7 @@ from metagenedb.apps.catalog.models import Taxonomy
logging
.
basicConfig
(
format
=
'[%(asctime)s] %(levelname)s:%(name)s:%(message)s'
)
logger
=
logging
.
getLogger
(
__name__
)
SELECT_RELATED_PARENT
=
"parent{}"
.
format
(
"__parent"
*
40
)
SELECT_RELATED_PARENT
=
"parent{}"
.
format
(
"__parent"
*
15
)
class
HierarchyBuilder
:
...
...
@@ -19,7 +19,7 @@ class HierarchyBuilder:
self
.
hierarchy_built
=
0
self
.
hierarchy_failed
=
0
def
build_all
(
self
,
chunk_size
=
8000
):
def
build_all
(
self
,
chunk_size
=
8000
,
test
=
False
):
logger
.
info
(
"Building all hierarchy for all %s taxonomy items..."
,
self
.
total_tax
)
for
taxonomy
in
self
.
queryset
.
iterator
(
chunk_size
=
chunk_size
):
try
:
...
...
@@ -29,6 +29,8 @@ class HierarchyBuilder:
self
.
hierarchy_failed
+=
1
self
.
processed_tax
+=
1
if
self
.
processed_tax
%
10000
==
0
:
if
test
is
True
:
break
logger
.
info
(
"%s/%s Taxonomy processed so far..."
,
self
.
processed_tax
,
self
.
total_tax
)
logger
.
info
(
"[DONE] %s/%s Hierarchy built."
,
self
.
hierarchy_built
,
self
.
total_tax
)
logger
.
info
(
"[DONE] %s/%s Hierarchy build skipped."
,
self
.
hierarchy_failed
,
self
.
total_tax
)
...
...
@@ -37,6 +39,9 @@ class HierarchyBuilder:
class
Command
(
BaseCommand
):
help
=
'Build hierarchy for taxonomy entries.'
def
add_arguments
(
self
,
parser
):
parser
.
add_argument
(
'--test'
,
action
=
'store_true'
,
help
=
'Run only on first 10000 entries.'
)
def
set_logger_level
(
self
,
verbosity
):
if
verbosity
>
2
:
logger
.
setLevel
(
logging
.
DEBUG
)
...
...
@@ -52,4 +57,4 @@ class Command(BaseCommand):
def
handle
(
self
,
*
args
,
**
options
):
self
.
set_logger_level
(
int
(
options
[
'verbosity'
]))
hierarchy_builder
=
HierarchyBuilder
(
self
.
get_queryset
())
hierarchy_builder
.
build_all
()
hierarchy_builder
.
build_all
(
test
=
options
[
'test'
]
)
ci/kubernetes/backend.yaml
View file @
e59f15d7
...
...
@@ -51,11 +51,11 @@ spec:
-
containerPort
:
8000
resources
:
requests
:
memory
:
"
256
Mi"
memory
:
"
512
Mi"
cpu
:
"
250m"
limits
:
memory
:
"
512
Mi"
cpu
:
"
5
00m"
memory
:
"
4096
Mi"
cpu
:
"
10
00m"
imagePullSecrets
:
-
name
:
registry-gitlab
volumes
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment