diff --git a/backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py b/backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py index f8c122eba0e224f0c0163dca165b0a9da74b1a1a..217b2d0755de1fd96c4e0a250e83f3251ca55e4f 100644 --- a/backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py +++ b/backend/metagenedb/apps/catalog/management/commands/build_hierarchy.py @@ -7,7 +7,7 @@ from metagenedb.apps.catalog.models import Taxonomy logging.basicConfig(format='[%(asctime)s] %(levelname)s:%(name)s:%(message)s') logger = logging.getLogger(__name__) -SELECT_RELATED_PARENT = "parent{}".format("__parent" * 40) +SELECT_RELATED_PARENT = "parent{}".format("__parent" * 15) class HierarchyBuilder: @@ -19,7 +19,7 @@ class HierarchyBuilder: self.hierarchy_built = 0 self.hierarchy_failed = 0 - def build_all(self, chunk_size=8000): + def build_all(self, chunk_size=8000, test=False): logger.info("Building all hierarchy for all %s taxonomy items...", self.total_tax) for taxonomy in self.queryset.iterator(chunk_size=chunk_size): try: @@ -29,6 +29,8 @@ class HierarchyBuilder: self.hierarchy_failed += 1 self.processed_tax += 1 if self.processed_tax % 10000 == 0: + if test is True: + break logger.info("%s/%s Taxonomy processed so far...", self.processed_tax, self.total_tax) logger.info("[DONE] %s/%s Hierarchy built.", self.hierarchy_built, self.total_tax) logger.info("[DONE] %s/%s Hierarchy build skipped.", self.hierarchy_failed, self.total_tax) @@ -37,6 +39,9 @@ class HierarchyBuilder: class Command(BaseCommand): help = 'Build hierarchy for taxonomy entries.' + def add_arguments(self, parser): + parser.add_argument('--test', action='store_true', help='Run only on first 10000 entries.') + def set_logger_level(self, verbosity): if verbosity > 2: logger.setLevel(logging.DEBUG) @@ -52,4 +57,4 @@ class Command(BaseCommand): def handle(self, *args, **options): self.set_logger_level(int(options['verbosity'])) hierarchy_builder = HierarchyBuilder(self.get_queryset()) - hierarchy_builder.build_all() + hierarchy_builder.build_all(test=options['test']) diff --git a/ci/kubernetes/backend.yaml b/ci/kubernetes/backend.yaml index 0ea9eba34cdd40ea0b9f3e05486f34e6991e3ce4..474f7307cc9f579d6219cdd7b8472b0b584a2019 100644 --- a/ci/kubernetes/backend.yaml +++ b/ci/kubernetes/backend.yaml @@ -51,11 +51,11 @@ spec: - containerPort: 8000 resources: requests: - memory: "256Mi" + memory: "512Mi" cpu: "250m" limits: - memory: "512Mi" - cpu: "500m" + memory: "4096Mi" + cpu: "1000m" imagePullSecrets: - name: registry-gitlab volumes: