cleaning up backend and rm useless code

backend/metagenedb/api/accounts/tests.py

@@ -12,7 +12,7 @@ class TestAccounts(APITestCase):
     def test_obtain_jwt(self):
 
         # create an inactive user
-        url = reverse('auth:api-jwt-auth')
+        url = reverse('api:auth:api-jwt-auth')
         u = User.objects.create_user(username='user', email='user@foo.com', password='pass')
         u.is_active = False
         u.save()
@@ -39,7 +39,7 @@ class TestAccounts(APITestCase):
         payload = jwt_payload_handler(user)
         token = jwt_encode_handler(payload)
 
-        verify_url = reverse('auth:api-jwt-verify')
+        verify_url = reverse('api:auth:api-jwt-verify')
         credentials = {
             'token': token
         }

backend/metagenedb/api/catalog/urls.py

-from django.urls import re_path
-
 from django.conf.urls import url, include
 from rest_framework.routers import DefaultRouter
 
-from metagenedb.api.catalog.views import GeneViewSet, gene_length
+from metagenedb.api.catalog.views import GeneViewSet
 
 
 api_router = DefaultRouter()
@@ -11,6 +9,5 @@ api_router.register(r'genes', GeneViewSet, base_name='genes')
 
 
 urlpatterns = [
-    re_path(r'^gene_length$', gene_length, name='gene_length'),
-    url(r'v1/', include((api_router.urls, 'v1'), namespace='v1'))
+    url(r'v1/', include((api_router.urls, 'v1')))
 ]

backend/metagenedb/api/catalog/views/__init__.py

 from .gene import GeneViewSet  # noqa
-from .statistics import gene_length  # noqa
 
 __all__ = ['GeneViewSet']

backend/metagenedb/api/catalog/views/gene.py

+import pandas as pd
 from rest_framework.viewsets import GenericViewSet
 from rest_framework import mixins
+from rest_framework.decorators import action
+from rest_framework.response import Response
 
+from metagenedb.common.utils.df_operations import get_mask
 from metagenedb.apps.catalog.models import Gene
 from metagenedb.apps.catalog.serializers import GeneSerializer
 
@@ -8,9 +12,34 @@ from metagenedb.apps.catalog.serializers import GeneSerializer
 class GeneViewSet(mixins.ListModelMixin,
                   mixins.RetrieveModelMixin,
                   GenericViewSet):
-    """
-    The following endpoints are fully provided by mixins:
-    * List gene
-    """
     queryset = Gene.objects.all()
     serializer_class = GeneSerializer
+    GENE_LENGTH_COL = 'gene_length'
+
+    def _count_windows(self, df, window_size=10000, window_col=GENE_LENGTH_COL):
+        """
+        Count how many line of the df belong to each windows defined by the window_size for the window_col
+        :param df:
+        :param window_col: column concerned by the window
+        :param window_size: size of the window
+        :return: {'data': COUNTS_BY_WINDOW, 'labels': START-END}
+        """
+        all_ranges = [(i, i + window_size) for i in range(0, df[window_col].max(), window_size)]
+        data = []
+        labels = []
+        for rg in all_ranges:
+            labels.append(f"{rg[0]}-{rg[1]-1}")
+            data.append(df[get_mask(df, rg, window_col)].count()['gene_length'])
+        return {
+            'counts': data,
+            'labels': labels
+        }
+
+    @action(methods=['get'], detail=False)
+    def gene_length(self, request, window_size=10000):
+        if 'window_size' in request.query_params:
+            window_size = int(request.query_params.get('window_size'))
+        df = pd.DataFrame(list(self.queryset.values(self.GENE_LENGTH_COL)))
+        return Response(
+            {'results': self._count_windows(df, window_size)}
+        )

backend/metagenedb/api/catalog/views/statistics.py

-import pandas as pd
-from rest_framework.decorators import api_view
-from rest_framework.response import Response
-
-from metagenedb.apps.catalog.models import Gene
-
-
-GENE_LENGTH_COL = 'gene_length'
-
-
-def _get_mask(df, rg, col_name):
-    """
-    rg is a range, e.g. (10-20)
-    """
-    return (df[col_name] >= rg[0]) & (df[col_name] < rg[1])
-
-
-def _count_windows(df, window_col, window_size=10000):
-    """
-    Count how many line of the df belong to each windows defined by the window_size for the window_col
-    :param df:
-    :param window_col: column concerned by the window
-    :param window_size: size of the window
-    :return: {'data': COUNTS_BY_WINDOW, 'labels': START-END}
-    """
-    all_ranges = [(i, i + window_size) for i in range(0, df[window_col].max(), window_size)]
-    data = []
-    labels = []
-    for rg in all_ranges:
-        labels.append(f"{rg[0]}-{rg[1]-1}")
-        data.append(df[_get_mask(df, rg, window_col)].count()['gene_length'])
-    return {
-        'counts': data,
-        'labels': labels
-    }
-
-
-@api_view(['GET'])
-def gene_length(request, window_size=10000):
-    if 'window_size' in request.query_params:
-        window_size = int(request.query_params.get('window_size'))
-    df = pd.DataFrame(list(Gene.objects.all().values(GENE_LENGTH_COL)))
-    return Response({
-        'data': _count_windows(df, GENE_LENGTH_COL, window_size)
-    })

backend/metagenedb/api/catalog/views/test_gene.py

@@ -2,19 +2,21 @@ from django.contrib.auth.models import User
 from django.test import TestCase
 from django.urls import reverse
 
+import pandas as pd
 from rest_framework import status
 from rest_framework_jwt.settings import api_settings
 
+from metagenedb.api.catalog.views.gene import GeneViewSet
+
 
 class TestGenes(TestCase):
-    """Post Tests"""
 
     def test_get_genes_no_auth(self):
         """
         Unauthenticated users should be able to access genes
         @TODO make unaccessible
         """
-        url = reverse('catalog:v1:genes-list')
+        url = reverse('api:catalog:v1:genes-list')
         resp = self.client.get(url)
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
 
@@ -32,6 +34,25 @@ class TestGenes(TestCase):
         payload = jwt_payload_handler(user)
         token = jwt_encode_handler(payload)
 
-        url = reverse('catalog:v1:genes-list')
+        url = reverse('api:catalog:v1:genes-list')
         resp = self.client.get(url, format='json', HTTP_AUTHORIZATION=f"JWT {token}")
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
+
+
+class TestCountWindows(TestCase):
+
+    def setUp(self):
+        self.window_col = "gene_length"
+        self.df = pd.DataFrame(
+            [22, 29, 35],
+            columns=[self.window_col]
+        )
+
+    def test_simple_count_window10(self):
+        expected_dict = {
+            'labels': ['0-9', '10-19', '20-29', '30-39'],
+            'counts': [0, 0, 2, 1]
+        }
+        geneviewset = GeneViewSet()
+        test_dict = geneviewset._count_windows(self.df, 10, window_col=self.window_col)
+        self.assertDictEqual(test_dict, expected_dict)

backend/metagenedb/api/catalog/views/test_statistics.py

-import pandas as pd
-from unittest import TestCase
-
-from .statistics import _count_windows
-
-
-class TestCountWindows(TestCase):
-
-    def setUp(self):
-        self.window_col = "gene_length"
-        self.df = pd.DataFrame(
-            [22, 29, 35],
-            columns=[self.window_col]
-        )
-
-    def test_simple_count_window10(self):
-        expected_dict = {
-            'labels': ['0-9', '10-19', '20-29', '30-39'],
-            'counts': [0, 0, 2, 1]
-        }
-        test_dict = _count_windows(self.df, self.window_col, 10)
-        self.assertDictEqual(test_dict, expected_dict)

backend/metagenedb/api/urls.py

@@ -2,6 +2,6 @@ from django.urls import include, path
 
 
 urlpatterns = [
-    path('auth/', include(('metagenedb.api.accounts.urls', 'auth'), namespace='auth')),
-    path('catalog/', include(('metagenedb.api.catalog.urls', 'catalog'), namespace='catalog'))
+    path('auth/', include(('metagenedb.api.accounts.urls', 'auth'))),
+    path('catalog/', include(('metagenedb.api.catalog.urls', 'catalog')))
 ]

backend/metagenedb/common/utils/df_operations.py

+def get_mask(df, range, col_name):
+    """
+    Get mask for a dataframe for a given range on a column defined by col_name
+    Lower is included but not upper:
+      e.g (4,8): 4, 5, 6 and 7 are selected
+    """
+    return (df[col_name] >= range[0]) & (df[col_name] < range[1])

backend/metagenedb/common/utils/test_df_operations.py

+from unittest import TestCase
+
+import pandas as pd
+from pandas.util.testing import assert_series_equal
+
+from metagenedb.common.utils.df_operations import get_mask
+
+
+class TestGetMask(TestCase):
+
+    def setUp(self):
+        self.df = pd.DataFrame(
+            [
+                ['breizh', 6],
+                ['loquemo', 7],
+                ['sea', 3],
+            ],
+            columns=['name', 'length']
+        )
+
+    def test_get_mask(self):
+        range = (5, 10)
+        expected_series = pd.Series(
+            [True, True, False],
+            name='length'
+        )
+        test_mask = get_mask(self.df, range, 'length')
+        assert_series_equal(test_mask, expected_series)
+
+    def test_get_mask_lower_equal(self):
+        range = (3, 5)
+        expected_series = pd.Series(
+            [False, False, True],
+            name='length'
+        )
+        test_mask = get_mask(self.df, range, 'length')
+        assert_series_equal(test_mask, expected_series)
+
+    def test_get_mask_upper_equal(self):
+        range = (4, 6)
+        expected_series = pd.Series(
+            [False, False, False],
+            name='length'
+        )
+        test_mask = get_mask(self.df, range, 'length')
+        assert_series_equal(test_mask, expected_series)

backend/metagenedb/utils/test_parsers.py → backend/metagenedb/common/utils/test_parsers.py

 from unittest import TestCase
 
-from metagenedb.utils.parsers import KEGGLineParser, NCBITaxonomyLineParser
+from metagenedb.common.utils.parsers import KEGGLineParser, NCBITaxonomyLineParser
 
 
 class TestKEGGLineParser(TestCase):

backend/metagenedb/urls.py

@@ -18,6 +18,6 @@ from django.urls import include, path
 
 
 urlpatterns = [
-    path('api/', include('metagenedb.api.urls')),
+    path('api/', include(('metagenedb.api.urls', 'api'))),
     path('admin/', admin.site.urls),
 ]

backend/metagenedb/utils/dict_operations.py

-import logging
-
-logging.basicConfig(level=logging.INFO)
-_LOGGER = logging.getLogger(__name__)
-
-
-def extract_dict(source_dict, keys, keep_original_source=False):
-    """
-    Extract a dict from a given dict based on a given set of keys
-    """
-    extracted_dict = {}
-    for key in keys:
-        try:
-            extracted_dict[key] = source_dict[key]
-            if not keep_original_source:
-                del source_dict[key]
-        except KeyError:
-            _LOGGER.warning(f"[{key}] is not found in the source dict, extraction skipped for this key.")
-    return extracted_dict

backend/metagenedb/utils/test_dict_operations.py

-from unittest import TestCase
-
-from metagenedb.utils.dict_operations import extract_dict
-
-
-class TestExtractDict(TestCase):
-
-    def test_extract_dict(self):
-        source_dict = {'a': 1, 'b': 2}
-        extract_keys = ['b']
-        expected_source_dict = {'a': 1}
-        expected_extract_dict = {'b': 2}
-        test_extract_dict = extract_dict(source_dict, extract_keys)
-        self.assertDictEqual(source_dict, expected_source_dict)
-        self.assertDictEqual(test_extract_dict, expected_extract_dict)
-
-    def test_extract_dict_wrong_key(self):
-        source_dict = {'a': 1, 'b': 2}
-        extract_keys = ['c']
-        expected_source_dict = {'a': 1, 'b': 2}
-        expected_extract_dict = {}
-        test_extract_dict = extract_dict(source_dict, extract_keys)
-        self.assertDictEqual(source_dict, expected_source_dict)
-        self.assertDictEqual(test_extract_dict, expected_extract_dict)
-
-    def test_extract_dict_original_source(self):
-        source_dict = {'a': 1, 'b': 2}
-        extract_keys = ['b']
-        expected_source_dict = {'a': 1, 'b': 2}
-        expected_extract_dict = {'b': 2}
-        test_extract_dict = extract_dict(source_dict, extract_keys, keep_original_source=True)
-        self.assertDictEqual(source_dict, expected_source_dict)
-        self.assertDictEqual(test_extract_dict, expected_extract_dict)

backend/scripts/populate_db/import_ncbi_taxonomy.py

@@ -6,7 +6,7 @@ import sys
 
 import django
 
-from metagenedb.utils.parsers import NCBITaxonomyLineParser
+from metagenedb.common.utils.parsers import NCBITaxonomyLineParser
 
 # Before model import, we need to called django.setup() to Load apps
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "metagenedb.settings")

backend/scripts/populate_db/load_kegg_ko.py

@@ -8,7 +8,7 @@ import sys
 import django
 from django.core.exceptions import ValidationError
 
-from metagenedb.utils.parsers import KEGGLineParser
+from metagenedb.common.utils.parsers import KEGGLineParser
 
 # Before model import, we need to called django.setup() to Load apps
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "metagenedb.settings")

frontend/src/views/Stats.vue

@@ -42,7 +42,7 @@ export default {
   },
   methods: {
     fetchGeneLength(geneLengthWindowSize) {
-      fetch('/api/catalog/gene_length?window_size='.concat(geneLengthWindowSize), {
+      fetch('/api/catalog/v1/genes/gene_length?window_size='.concat(geneLengthWindowSize), {
         method: 'GET',
         headers: {
           Accept: 'application/json',
@@ -51,7 +51,7 @@ export default {
         .then((response) => {
           if (response.ok) {
             response.json().then((json) => {
-              this.geneLengthData = json.data;
+              this.geneLengthData = json.results;
             });
           }
         });