Commit e59f15d7 authored by Kenzo-Hugo Hillion's avatar Kenzo-Hugo Hillion
Browse files

add backend ressources and test option for build hierarchy

parent f90f67d0
Pipeline #17622 passed with stages
in 2 minutes and 27 seconds
......@@ -7,7 +7,7 @@ from metagenedb.apps.catalog.models import Taxonomy
logging.basicConfig(format='[%(asctime)s] %(levelname)s:%(name)s:%(message)s')
logger = logging.getLogger(__name__)
SELECT_RELATED_PARENT = "parent{}".format("__parent" * 40)
SELECT_RELATED_PARENT = "parent{}".format("__parent" * 15)
class HierarchyBuilder:
......@@ -19,7 +19,7 @@ class HierarchyBuilder:
self.hierarchy_built = 0
self.hierarchy_failed = 0
def build_all(self, chunk_size=8000):
def build_all(self, chunk_size=8000, test=False):
logger.info("Building all hierarchy for all %s taxonomy items...", self.total_tax)
for taxonomy in self.queryset.iterator(chunk_size=chunk_size):
try:
......@@ -29,6 +29,8 @@ class HierarchyBuilder:
self.hierarchy_failed += 1
self.processed_tax += 1
if self.processed_tax % 10000 == 0:
if test is True:
break
logger.info("%s/%s Taxonomy processed so far...", self.processed_tax, self.total_tax)
logger.info("[DONE] %s/%s Hierarchy built.", self.hierarchy_built, self.total_tax)
logger.info("[DONE] %s/%s Hierarchy build skipped.", self.hierarchy_failed, self.total_tax)
......@@ -37,6 +39,9 @@ class HierarchyBuilder:
class Command(BaseCommand):
help = 'Build hierarchy for taxonomy entries.'
def add_arguments(self, parser):
parser.add_argument('--test', action='store_true', help='Run only on first 10000 entries.')
def set_logger_level(self, verbosity):
if verbosity > 2:
logger.setLevel(logging.DEBUG)
......@@ -52,4 +57,4 @@ class Command(BaseCommand):
def handle(self, *args, **options):
self.set_logger_level(int(options['verbosity']))
hierarchy_builder = HierarchyBuilder(self.get_queryset())
hierarchy_builder.build_all()
hierarchy_builder.build_all(test=options['test'])
......@@ -51,11 +51,11 @@ spec:
- containerPort: 8000
resources:
requests:
memory: "256Mi"
memory: "512Mi"
cpu: "250m"
limits:
memory: "512Mi"
cpu: "500m"
memory: "4096Mi"
cpu: "1000m"
imagePullSecrets:
- name: registry-gitlab
volumes:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment