Merge branch '45-permissions-auth' into 'dev'

Use permissions and auth

Closes #45

See merge request !24

backend/.flake8

 [flake8]
 exclude =
     */migrations/*
+    */src/bioapi/*

backend/Pipfile

@@ -56,9 +56,9 @@ metagenedb = {editable = true,path = "."}
 drf-yasg = "*"
 packaging = "*"
 python-slugify = "*"
-master = {git = "https://github.com/khillion/bioapi.git"}
 marshmallow = "*"
 django-pandas = "*"
+bioapi = {git = "https://github.com/khillion/bioapi.git"}
 
 [requires]
 python_version = "3.7"

backend/Pipfile.lock

 {
     "_meta": {
         "hash": {
-            "sha256": "4be3394e3c4abe5fc7b75328ef912eaba09e15365322b7493e256f2def2ff013"
+            "sha256": "c448dca73004a15b2c0d65f4ff1e2b2da1aac4d1439a0be2adf97531783af899"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -16,13 +16,24 @@
         ]
     },
     "default": {
+        "asgiref": {
+            "hashes": [
+                "sha256:7e06d934a7718bf3975acbf87780ba678957b87c7adc056f13b6215d610695a0",
+                "sha256:ea448f92fc35a0ef4b1508f53a04c4670255a3f33d22a81c8fc9c872036adbe5"
+            ],
+            "version": "==3.2.3"
+        },
+        "bioapi": {
+            "git": "https://github.com/khillion/bioapi.git",
+            "ref": "8aa8a5e446887a91ada4c6cc63ea06ef2e52123d"
+        },
         "certifi": {
             "hashes": [
-                "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
-                "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
             ],
             "index": "pypi",
-            "version": "==2019.9.11"
+            "version": "==2019.11.28"
         },
         "chardet": {
             "hashes": [
@@ -48,11 +59,11 @@
         },
         "django": {
             "hashes": [
-                "sha256:16040e1288c6c9f68c6da2fe75ebde83c0a158f6f5d54f4c5177b0c1478c5b86",
-                "sha256:89c2007ca4fa5b351a51a279eccff298520783b713bf28efb89dfb81c80ea49b"
+                "sha256:6f857bd4e574442ba35a7172f1397b303167dae964cf18e53db5e85fe248d000",
+                "sha256:d98c9b6e5eed147bc51f47c014ff6826bd1ab50b166956776ee13db5a58804ae"
             ],
             "index": "pypi",
-            "version": "==2.2.7"
+            "version": "==3.0"
         },
         "django-cors-headers": {
             "hashes": [
@@ -180,15 +191,11 @@
         },
         "marshmallow": {
             "hashes": [
-                "sha256:1a358beb89c2b4d5555272065a9533591a3eb02f1b854f3c4002d88d8f2a1ddb",
-                "sha256:eb97c42c5928b5720812c9268865fe863d4807bc1a8b48ddd7d5c9e1779a6af0"
+                "sha256:0ba81b6da4ae69eb229b74b3c741ff13fe04fb899824377b1aff5aaa1a9fd46e",
+                "sha256:3e53dd9e9358977a3929e45cdbe4a671f9eff53a7d6a23f33ed3eab8c1890d8f"
             ],
             "index": "pypi",
-            "version": "==3.2.2"
-        },
-        "master": {
-            "git": "https://github.com/khillion/bioapi.git",
-            "ref": "c01af30f9b60428c0d8d2aeae04e5adf1fadba36"
+            "version": "==3.3.0"
         },
         "metagenedb": {
             "editable": true,
@@ -514,10 +521,10 @@
         },
         "faker": {
             "hashes": [
-                "sha256:48c03580720e0b46538d528b1296e4e5b24a809dcaf33a7dddec719489a9edb8",
-                "sha256:6327c665c0d8721280b3036d9c9e851c60092bc1f30c8394cc433f8723e2bda5"
+                "sha256:202ad3b2ec16ae7c51c02904fb838831f8d2899e61bf18db1e91a5a582feab11",
+                "sha256:92c84a10bec81217d9cb554ee12b3838c8986ce0b5d45f72f769da22e4bb5432"
             ],
-            "version": "==2.0.4"
+            "version": "==3.0.0"
         },
         "flake8": {
             "hashes": [
@@ -529,11 +536,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26",
-                "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"
+                "sha256:3a8b2dfd0a2c6a3636e7c016a7e54ae04b997d30e69d5eacdca7a6c2221a1402",
+                "sha256:41e688146d000891f32b1669e8573c57e39e5060e7f5f647aa617cd9a9568278"
             ],
             "index": "pypi",
-            "version": "==0.23"
+            "version": "==1.2.0"
         },
         "inflection": {
             "hashes": [
@@ -550,11 +557,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:dfd303b270b7b5232b3d08bd30ec6fd685d8a58cabd54055e3d69d8f029f7280",
-                "sha256:ed7ebe1cba899c1c3ccad6f7f1c2d2369464cc77dba8eebc65e2043e19cda995"
+                "sha256:c66c7e27239855828a764b1e8fc72c24a6f4498a2637572094a78c5551fb9d51",
+                "sha256:f186b01b36609e0c5d0de27c7ef8e80c990c70478f8c880863004b3489a9030e"
             ],
             "markers": "python_version >= '3.3'",
-            "version": "==7.9.0"
+            "version": "==7.10.1"
         },
         "ipython-genutils": {
             "hashes": [
@@ -755,11 +762,11 @@
         },
         "more-itertools": {
             "hashes": [
-                "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
-                "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
+                "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d",
+                "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564"
             ],
             "index": "pypi",
-            "version": "==7.2.0"
+            "version": "==8.0.2"
         },
         "nbconvert": {
             "hashes": [
@@ -874,10 +881,10 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:83ec6c6133ca6b529b7ff5aa826328fd14b5bb02a58c37f4f06384e96a0f94ab",
-                "sha256:b7949de3d396836085fea596998b135a22610bbcc4f2abfe9e448e44cbc58388"
+                "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b",
+                "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe"
             ],
-            "version": "==2.5.1"
+            "version": "==2.5.2"
         },
         "pylint": {
             "hashes": [

backend/metagenedb/api/catalog/views/bulk_viewset.py

@@ -3,11 +3,13 @@ from rest_framework import status
 from rest_framework.response import Response
 from rest_framework.viewsets import ModelViewSet
 
+from metagenedb.common.django_default.permissions import ListAndRetrieveAll
 from metagenedb.common.django_default.qparams_validators import PaginatedQueryParams
 
 
 class BulkViewSet(ModelViewSet):
     query_params_parser = PaginatedQueryParams
+    permission_classes = [ListAndRetrieveAll]
 
     def get_objects(self, instance_ids):
         return self.queryset.in_bulk(instance_ids, field_name=self.lookup_field)

backend/metagenedb/api/catalog/views/gene.py

@@ -5,6 +5,7 @@ from drf_yasg import openapi
 from drf_yasg.utils import swagger_auto_schema
 from marshmallow.exceptions import ValidationError
 from rest_framework.decorators import action
+from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
 from rest_framework.status import HTTP_204_NO_CONTENT, HTTP_422_UNPROCESSABLE_ENTITY
 
@@ -60,6 +61,8 @@ class GeneViewSet(BulkViewSet):
     DEFAULT_LEVEL = 'phylum'
 
     def get_permissions(self):
+        if self.action in ['gene_length', 'taxonomy_counts']:
+            return [AllowAny()]
         return super(self.__class__, self).get_permissions()
 
     def _count_windows(self, queryset, window_size=DEFAULT_WINDOW_SIZE, window_col=GENE_LENGTH_COL,

backend/metagenedb/api/catalog/views/test_bulk_viewset.py

@@ -3,9 +3,35 @@ from rest_framework.test import APITestCase
 
 from metagenedb.apps.catalog.factory import FunctionFactory
 from metagenedb.common.utils.mocks.metagenedb import MetageneDBCatalogFunctionAPIMock
+from metagenedb.common.utils.tests.apitestbase import AdminUserBasedTest
 
 
-class TestOperationsBulkViewSet(APITestCase):
+class TestOperationsBulkViewSetNoCredentials(APITestCase):
+
+    def setUp(self):
+        self.function_api = MetageneDBCatalogFunctionAPIMock(self.client)
+
+    def test_create_function(self):
+        data = {
+            "function_id": 'k_test1',
+            "source": "kegg",
+            "name": "Kegg Test 1"
+        }
+        with self.assertRaises(HTTPError):
+            self.function_api.post(data)
+
+    def test_update_existing_function(self):
+        function = FunctionFactory()
+        data = {
+            "function_id": function.function_id,
+            "source": function.source,
+            "name": "Kegg Test 1"
+        }
+        with self.assertRaises(HTTPError):
+            self.function_api.put(data, function.function_id)
+
+
+class TestOperationsBulkViewSet(AdminUserBasedTest):
     """
     We are testing the different functions through the API directly through the mock redirecting
     requests to the test database.
@@ -14,7 +40,8 @@ class TestOperationsBulkViewSet(APITestCase):
     """
 
     def setUp(self):
-        self.function_api = MetageneDBCatalogFunctionAPIMock(self.client)
+        super().setUp()
+        self.function_api = MetageneDBCatalogFunctionAPIMock(self.client, jwt_token=self.jwt_token)
 
     def test_create_function(self):
         data = {

backend/metagenedb/api/catalog/views/test_gene.py

-from django.contrib.auth.models import User
 from django.test import TestCase
 from django.urls import reverse
 from rest_framework import status
 from rest_framework.test import APITestCase
-from rest_framework_jwt.settings import api_settings
 
 from metagenedb.apps.catalog.factory import GeneFactory, TaxonomyFactory
 from metagenedb.common.utils.mocks.metagenedb import MetageneDBCatalogGeneAPIMock
@@ -20,24 +18,6 @@ class TestGenes(TestCase):
         resp = self.client.get(url)
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
 
-    def test_get_genes_auth(self):
-        """
-        Authenticated users should be able to access genes
-        """
-
-        jwt_payload_handler = api_settings.JWT_PAYLOAD_HANDLER
-        jwt_encode_handler = api_settings.JWT_ENCODE_HANDLER
-
-        user = User.objects.create_user(username='user', email='user@foo.com', password='pass')
-        user.is_active = True
-        user.save()
-        payload = jwt_payload_handler(user)
-        token = jwt_encode_handler(payload)
-
-        url = reverse('api:catalog:v1:genes-list')
-        resp = self.client.get(url, format='json', HTTP_AUTHORIZATION=f"JWT {token}")
-        self.assertEqual(resp.status_code, status.HTTP_200_OK)
-
 
 class TestCountWindowsAPI(APITestCase):
 

backend/metagenedb/common/django_default/permissions.py

+from rest_framework.permissions import BasePermission
+
+
+class ListAndRetrieveAll(BasePermission):
+    """
+    Custom permission to only allow access to lists for admins
+    """
+
+    def has_permission(self, request, view):
+        return view.action in ['list', 'retrieve'] or request.user.is_staff

backend/metagenedb/common/utils/mocks/metagenedb.py

@@ -14,33 +14,43 @@ class MetageneDBAPIMock(MetageneDBCatalogGeneAPI):
     REVERSE_PATH = ''
     BAD_REQUESTS = range(400, 452)
 
-    def __init__(self, client):
+    def __init__(self, client, jwt_token=None):
         self.client = client
         self.reverse_path = ':'.join([self.BASE_REVERSE, self.REVERSE_PATH])
+        self.headers = {}
+        if jwt_token is not None:
+            self.headers.update({
+                'HTTP_AUTHORIZATION': f"JWT {jwt_token}",
+            })
 
     def get_all(self, params=None):
-        response = self.client.get(reverse(f'{self.reverse_path}-list'), params)
+        response = self.client.get(reverse(f'{self.reverse_path}-list'), params, **self.headers)
         if response.status_code in self.BAD_REQUESTS:
             raise HTTPError
         return response.json()
 
     def get(self, entry_id, params=None):
-        response = self.client.get(reverse(f'{self.reverse_path}-detail', kwargs={self.KEY_ID: entry_id}), params)
+        response = self.client.get(reverse(f'{self.reverse_path}-detail', kwargs={self.KEY_ID: entry_id}),
+                                   params, **self.headers)
         if response.status_code in self.BAD_REQUESTS:
             raise HTTPError
         return response.json()
 
     def post(self, data):
-        response = self.client.post(reverse(f'{self.reverse_path}-list'), data, format='json')
+        response = self.client.post(reverse(f'{self.reverse_path}-list'), data, format='json', **self.headers)
         if response.status_code in self.BAD_REQUESTS:
             raise HTTPError
         return response.json()
 
     def put(self, data, entry_id=None):
         if entry_id:
-            return self.client.put(reverse(f'{self.reverse_path}-detail', kwargs={self.KEY_ID: entry_id}),
-                                   data, format='json').json()
-        return self.client.put(reverse(f'{self.reverse_path}-list'), data, format='json').json()
+            response = self.client.put(reverse(f'{self.reverse_path}-detail', kwargs={self.KEY_ID: entry_id}),
+                                       data, format='json', **self.headers)
+        else:
+            response = self.client.put(reverse(f'{self.reverse_path}-list'), data, format='json', **self.headers)
+        if response.status_code in self.BAD_REQUESTS:
+            raise HTTPError
+        return response.json()
 
 
 class MetageneDBCatalogGeneAPIMock(MetageneDBAPIMock):

backend/metagenedb/common/utils/tests/apitestbase.py

+from django.contrib.auth.models import User
+from rest_framework.test import APITestCase
+from rest_framework_jwt.settings import api_settings
+
+
+class AdminUserBasedTest(APITestCase):
+    """
+    We are testing the different functions through the API directly through the mock redirecting
+    requests to the test database.
+
+    The extent is a bit more than a unittest since it is not just involving concerned methods.
+    """
+
+    def setUp(self):
+        jwt_payload_handler = api_settings.JWT_PAYLOAD_HANDLER
+        jwt_encode_handler = api_settings.JWT_ENCODE_HANDLER
+        user = User.objects.create_user(username='user_admin', email='user@admin.com', password='pass')
+        user.is_active = True
+        user.is_staff = True
+        user.save()
+        payload = jwt_payload_handler(user)
+        self.jwt_token = jwt_encode_handler(payload)

backend/metagenedb/settings/django.py

@@ -116,7 +116,7 @@ REST_FRAMEWORK = {
 # JWT Authorization settings
 
 JWT_AUTH = {
-    'JWT_EXPIRATION_DELTA': datetime.timedelta(seconds=3600)
+    'JWT_EXPIRATION_DELTA': datetime.timedelta(days=7)
 }
 
 

backend/scripts/populate_db/import_igc_data.py

@@ -24,12 +24,12 @@ class ImportIGCGenes(object):
     GENUS_COL = 'taxo_genus'
     SELECTED_KEYS = ['gene_id', 'length', 'kegg_ko', PHYLUM_COL, GENUS_COL]
 
-    def __init__(self, annotation_file, url, skip_tax=False, skip_functions=False):
+    def __init__(self, annotation_file, url, jwt_token, skip_tax=False, skip_functions=False):
         self.annotation_file = annotation_file
         self.url = url
-        self.metagenedb_gene_api = self.METAGENEDB_GENE_API(base_url=self.url)
-        self.metagenedb_taxonomy_api = self.METAGENEDB_TAXONOMY_API(base_url=self.url)
-        self.metagenedb_function_api = self.METAGENEDB_FUNCTION_API(base_url=self.url)
+        self.metagenedb_gene_api = self.METAGENEDB_GENE_API(base_url=self.url, jwt_token=jwt_token)
+        self.metagenedb_taxonomy_api = self.METAGENEDB_TAXONOMY_API(base_url=self.url, jwt_token=jwt_token)
+        self.metagenedb_function_api = self.METAGENEDB_FUNCTION_API(base_url=self.url, jwt_token=jwt_token)
         self.total_genes = self._get_number_genes()
         self._reset_counters()
         # Skip some insertion if specified in script options
@@ -177,6 +177,7 @@ def parse_arguments():
     # Common arguments for analysis and annotations
     parser.add_argument('annotation', help='IGC annotation file')
     parser.add_argument('--url', help='base URL of the instance.', default='http://localhost/')
+    parser.add_argument('-t', '--jwt_token', help='your JWT token obtain from web app', required=True)
     parser.add_argument('--chunk_size', type=int, default=1000,
                         help='How many genes to handle and create in the same time.')
     parser.add_argument('--skip_taxonomy', action='store_true', help='Skip taxonomy information from genes.')
@@ -194,7 +195,7 @@ def run():
     args = parse_arguments()
     if args.verbose:
         logger.setLevel(logging.INFO)
-    import_igc_genes = ImportIGCGenes(args.annotation, args.url,
+    import_igc_genes = ImportIGCGenes(args.annotation, args.url, args.jwt_token,
                                       skip_tax=args.skip_taxonomy, skip_functions=args.skip_functions)
     import_igc_genes.load_annotation_file_to_db_in_chunks(chunk_size=args.chunk_size, test=args.test)
 

backend/scripts/populate_db/import_ncbi_taxonomy.py

@@ -18,8 +18,8 @@ class ImportNCBITaxonomy(object):
     METAGENEDB_TAX_API = MetageneDBCatalogTaxonomyAPI
     FOREIGN_KEY_FIELDS = ['parent_tax_id']
 
-    def __init__(self, url, tax_names_file, tax_nodes_file):
-        self.metagenedb_tax_api = self.METAGENEDB_TAX_API(base_url=url)
+    def __init__(self, url, jwt_token, tax_names_file, tax_nodes_file):
+        self.metagenedb_tax_api = self.METAGENEDB_TAX_API(base_url=url, jwt_token=jwt_token)
         self.tax_names_file = tax_names_file
         self.tax_nodes_file = tax_nodes_file
         self.total_tax = self._get_number_nodes()
@@ -104,6 +104,7 @@ def parse_arguments():
     parser.add_argument('--names', help='names.dmp file from ncbi_taxonomy', required=True)
     parser.add_argument('--skip_creation', action='store_true', help='Skip taxonomy creation.')
     parser.add_argument('--url', help='base URL of the instance.', default='http://localhost/')
+    parser.add_argument('-t', '--jwt_token', help='your JWT token obtain from web app', required=True)
     parser.add_argument('-v', '--verbose', action='store_true')
 
     try:
@@ -116,7 +117,7 @@ def run():
     args = parse_arguments()
     if args.verbose:
         logger.setLevel(logging.INFO)
-    import_ncbi_tax = ImportNCBITaxonomy(args.url, args.names, args.nodes)
+    import_ncbi_tax = ImportNCBITaxonomy(args.url, args.jwt_token, args.names, args.nodes)
     taxonomy_names = import_ncbi_tax.import_names()
     if not args.skip_creation:
         import_ncbi_tax.create_taxo_nodes(taxonomy_names)

backend/scripts/populate_db/load_kegg_ko.py

@@ -21,9 +21,9 @@ class ImportKEGGKO(object):
     ORM_SOURCE_KEY = 'source'
     KEGG_SOURCE = 'kegg'
 
-    def __init__(self, url, kegg_ko_list_api=KEGG_KO_LIST_API):
+    def __init__(self, url, jwt_token, kegg_ko_list_api=KEGG_KO_LIST_API):
         self.kegg_ko_list_api = kegg_ko_list_api
-        self.metagenedb_function_api = self.METAGENEDB_FUNCTION_API(base_url=url)
+        self.metagenedb_function_api = self.METAGENEDB_FUNCTION_API(base_url=url, jwt_token=jwt_token)
         self.processed_kegg = 0
         self.created_kegg = 0
         self.updated_kegg = 0
@@ -55,6 +55,7 @@ def parse_arguments():
     """
     parser = argparse.ArgumentParser(description=f'Populate KEGG KO database from {KEGG_KO_LIST_API}.')
     parser.add_argument('--url', help='base URL of the instance.', default='http://localhost/')
+    parser.add_argument('-t', '--jwt_token', help='your JWT token obtain from web app', required=True)
     parser.add_argument('-v', '--verbose', action='store_true')
     try:
         return parser.parse_args()
@@ -66,7 +67,7 @@ def run():
     args = parse_arguments()
     if args.verbose:
         logger.setLevel(logging.INFO)
-    import_kegg_ko = ImportKEGGKO(args.url)
+    import_kegg_ko = ImportKEGGKO(args.url, args.jwt_token)
     import_kegg_ko.load_all_kegg_ko()
 
 

backend/scripts/populate_db/test_import_igc_data.py

@@ -27,7 +27,7 @@ class TestParseGene(TestCase):
             'cohort_assembled'
         ]
         self.raw_line = "\t".join(raw_data)
-        self.import_igc_genes = ImportIGCGenes('test', 'test')
+        self.import_igc_genes = ImportIGCGenes('test', 'test_url', 'test_token')
 
     def test_parse_gene_default_selected_keys(self):
         """
@@ -71,7 +71,7 @@ class TestParseGene(TestCase):
 class TestCleanGene(TestCase):
 
     def setUp(self):
-        self.import_igc_genes = ImportIGCGenes('test', 'test')
+        self.import_igc_genes = ImportIGCGenes('test', 'test_url', 'test_token')
         self.import_igc_genes._select_taxonomy = lambda x: x  # Mock to return same dict
         self.import_igc_genes._clean_functions = lambda x: x
         self.gene_dict = {
@@ -123,7 +123,7 @@ class TestSelectTaxonomy(TestCase):
         self.genus_name = 'Genus1'
         self.phylum_id = 'phylum_1'
         self.phylum_name = 'Phylum1'
-        self.import_igc_genes = ImportIGCGenes('test', 'test')
+        self.import_igc_genes = ImportIGCGenes('test', 'test_url', 'test_token')
         self.import_igc_genes.phylum_mapping = {
             self.phylum_name: self.phylum_id
         }
@@ -227,7 +227,7 @@ class TestBuildTaxoMapping(APITestCase):
         cls.phylum_items = TaxonomyFactory.create_batch(20, rank='phylum')
 
     def setUp(self):
-        self.import_igc_genes = ImportIGCGenes('test', 'test')
+        self.import_igc_genes = ImportIGCGenes('test', 'test_url', 'test_token')
         self.api_mock = MetageneDBCatalogTaxonomyAPIMock(self.client)
         self.import_igc_genes.metagenedb_taxonomy_api = self.api_mock
 
@@ -250,7 +250,7 @@ class TestBuildBuildFunctionCatalog(APITestCase):
         cls.functions = FunctionFactory.create_batch(100)
 
     def setUp(self):
-        self.import_igc_genes = ImportIGCGenes('test', 'test')
+        self.import_igc_genes = ImportIGCGenes('test', 'test_url', 'test_token')
         self.api_mock = MetageneDBCatalogFunctionAPIMock(self.client)
         self.import_igc_genes.metagenedb_function_api = self.api_mock