From e59f15d759e7b9dfdeef28a759987305103b9c65 Mon Sep 17 00:00:00 2001
From: Kenzo-Hugo Hillion <kenzo-hugo.hillion1@pasteur.fr>
Date: Tue, 5 Nov 2019 13:05:05 +0100
Subject: [PATCH] add backend ressources and test option for build hierarchy
---
.../catalog/management/commands/build_hierarchy.py | 11 ++++++++---
ci/kubernetes/backend.yaml | 6 +++---
2 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py b/backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py
index f8c122e..217b2d0 100644
--- a/backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py
+++ b/backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py
@@ -7,7 +7,7 @@ from metagenedb.apps.catalog.models import Taxonomy
logging.basicConfig(format='[%(asctime)s] %(levelname)s:%(name)s:%(message)s')
logger = logging.getLogger(__name__)
-SELECT_RELATED_PARENT = "parent{}".format("__parent" * 40)
+SELECT_RELATED_PARENT = "parent{}".format("__parent" * 15)
class HierarchyBuilder:
@@ -19,7 +19,7 @@ class HierarchyBuilder:
self.hierarchy_built = 0
self.hierarchy_failed = 0
- def build_all(self, chunk_size=8000):
+ def build_all(self, chunk_size=8000, test=False):
logger.info("Building all hierarchy for all %s taxonomy items...", self.total_tax)
for taxonomy in self.queryset.iterator(chunk_size=chunk_size):
try:
@@ -29,6 +29,8 @@ class HierarchyBuilder:
self.hierarchy_failed += 1
self.processed_tax += 1
if self.processed_tax % 10000 == 0:
+ if test is True:
+ break
logger.info("%s/%s Taxonomy processed so far...", self.processed_tax, self.total_tax)
logger.info("[DONE] %s/%s Hierarchy built.", self.hierarchy_built, self.total_tax)
logger.info("[DONE] %s/%s Hierarchy build skipped.", self.hierarchy_failed, self.total_tax)
@@ -37,6 +39,9 @@ class HierarchyBuilder:
class Command(BaseCommand):
help = 'Build hierarchy for taxonomy entries.'
+ def add_arguments(self, parser):
+ parser.add_argument('--test', action='store_true', help='Run only on first 10000 entries.')
+
def set_logger_level(self, verbosity):
if verbosity > 2:
logger.setLevel(logging.DEBUG)
@@ -52,4 +57,4 @@ class Command(BaseCommand):
def handle(self, *args, **options):
self.set_logger_level(int(options['verbosity']))
hierarchy_builder = HierarchyBuilder(self.get_queryset())
- hierarchy_builder.build_all()
+ hierarchy_builder.build_all(test=options['test'])
diff --git a/ci/kubernetes/backend.yaml b/ci/kubernetes/backend.yaml
index 0ea9eba..474f730 100644
--- a/ci/kubernetes/backend.yaml
+++ b/ci/kubernetes/backend.yaml
@@ -51,11 +51,11 @@ spec:
- containerPort: 8000
resources:
requests:
- memory: "256Mi"
+ memory: "512Mi"
cpu: "250m"
limits:
- memory: "512Mi"
- cpu: "500m"
+ memory: "4096Mi"
+ cpu: "1000m"
imagePullSecrets:
- name: registry-gitlab
volumes:
--
GitLab