Commit 83746eaa authored by Kenzo-Hugo Hillion's avatar Kenzo-Hugo Hillion
Browse files

Add page_size and isolate default classes for django settings

parent 5635b146
Pipeline #18620 passed with stages
in 2 minutes and 17 seconds
from marshmallow import fields
from .base import PaginatedQueryParams
from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
class FunctionQueryParams(PaginatedQueryParams):
......
from marshmallow import fields
from .base import PaginatedQueryParams
from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
class TaxonomyQueryParams(PaginatedQueryParams):
......
......@@ -3,7 +3,7 @@ from rest_framework import status
from rest_framework.response import Response
from rest_framework.viewsets import ModelViewSet
from metagenedb.api.catalog.qparams_validators.base import PaginatedQueryParams
from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
class BulkViewSet(ModelViewSet):
......
from rest_framework.pagination import PageNumberPagination
class DefaultPageNumberPagination(PageNumberPagination):
page_size_query_param = 'page_size'
max_page_size = 5000
......@@ -3,3 +3,4 @@ from marshmallow import Schema, fields
class PaginatedQueryParams(Schema):
page = fields.Integer()
page_size = fields.Integer()
......@@ -105,7 +105,7 @@ REST_FRAMEWORK = {
'rest_framework_jwt.authentication.JSONWebTokenAuthentication',
'rest_framework.authentication.SessionAuthentication',
),
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
'DEFAULT_PAGINATION_CLASS': 'metagenedb.common.django_default.pagination.DefaultPageNumberPagination',
'PAGE_SIZE': 100,
'DEFAULT_FILTER_BACKENDS': (
'django_filters.rest_framework.DjangoFilterBackend',
......
......@@ -34,12 +34,18 @@ class ImportIGCGenes(object):
self.skip_tax = skip_tax
self.skip_functions = skip_functions
def _build_taxo_mapping(self, rank):
def _build_taxo_mapping(self, rank, page_size=1000):
logger.info("Building local mapping for %s level...", rank)
counter = 1
next_page = None
mapping = {}
while counter == 1 or next_page is not None:
current_page = self.metagenedb_taxonomy_api.get_all(params={'page': counter, 'rank': rank})
params = {
'page': counter,
'page_size': page_size,
'rank': rank,
}
current_page = self.metagenedb_taxonomy_api.get_all(params=params)
next_page = current_page['next']
mapping.update({
value['name']: value['tax_id'] for value in current_page['results']
......@@ -47,9 +53,9 @@ class ImportIGCGenes(object):
counter += 1
return mapping
def build_mapping(self):
self.phylum_mapping = self._build_taxo_mapping("phylum")
self.genus_mapping = self._build_taxo_mapping("genus")
def build_mapping(self, page_size=1000):
self.phylum_mapping = self._build_taxo_mapping("phylum", page_size=page_size)
self.genus_mapping = self._build_taxo_mapping("genus", page_size=page_size)
def _reset_counters(self):
self.processed_genes = 0
......@@ -107,7 +113,8 @@ class ImportIGCGenes(object):
def load_annotation_file_to_db_in_chunks(self, chunk_size=1000, test=False):
# Build mapping for different phylum and genus
self.build_mapping()
if not self.skip_tax:
self.build_mapping()
with open(self.annotation_file, 'r') as file:
while True:
chunk_genes = list(islice(file, chunk_size))
......
......@@ -237,6 +237,6 @@ class TestBuildTaxoMapping(APITestCase):
expected_phylum_dict = {
item.name: item.tax_id for item in self.phylum_items
}
self.import_igc_genes.build_mapping()
self.import_igc_genes.build_mapping(page_size=100)
self.assertDictEqual(self.import_igc_genes.genus_mapping, expected_genus_dict)
self.assertDictEqual(self.import_igc_genes.phylum_mapping, expected_phylum_dict)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment