From 8cfa5c1eb2706a89479ff7c9d27fb61b6af92e59 Mon Sep 17 00:00:00 2001
From: Kenzo-Hugo Hillion <kenzo-hugo.hillion1@pasteur.fr>
Date: Wed, 27 Nov 2019 15:48:13 +0100
Subject: [PATCH] Add profiling tool for performance issues
---
backend/Pipfile | 1 +
backend/Pipfile.lock | 32 ++++++++++++--------
backend/metagenedb/api/catalog/views/gene.py | 14 +++++----
backend/metagenedb/common/utils/profiling.py | 16 ++++++++++
4 files changed, 45 insertions(+), 18 deletions(-)
create mode 100644 backend/metagenedb/common/utils/profiling.py
diff --git a/backend/Pipfile b/backend/Pipfile
index e447a70..182c8ba 100644
--- a/backend/Pipfile
+++ b/backend/Pipfile
@@ -58,6 +58,7 @@ packaging = "*"
python-slugify = "*"
master = {git = "https://github.com/khillion/bioapi.git"}
marshmallow = "*"
+django-pandas = "*"
[requires]
python_version = "3.7"
diff --git a/backend/Pipfile.lock b/backend/Pipfile.lock
index 176351d..46cc1ba 100644
--- a/backend/Pipfile.lock
+++ b/backend/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "5998b6b97448fd635cc1b05787de28ac5ac3344ca6b8055831fca13790fc3f33"
+ "sha256": "4be3394e3c4abe5fc7b75328ef912eaba09e15365322b7493e256f2def2ff013"
},
"pipfile-spec": 6,
"requires": {
@@ -86,6 +86,14 @@
"index": "pypi",
"version": "==2.2.0"
},
+ "django-pandas": {
+ "hashes": [
+ "sha256:738cc03ffb411eef3eb02334d1f5a5d40697099a92ac59eb39629c08a9c2d6fb",
+ "sha256:788f4652012a67d2c5849191b01af58255f7af815ab612bebca019854235a9bc"
+ ],
+ "index": "pypi",
+ "version": "==0.6.1"
+ },
"djangorestframework": {
"hashes": [
"sha256:5488aed8f8df5ec1d70f04b2114abc52ae6729748a176c453313834a9ee179c8",
@@ -812,11 +820,11 @@
},
"pluggy": {
"hashes": [
- "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6",
- "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"
+ "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
+ "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
],
"index": "pypi",
- "version": "==0.13.0"
+ "version": "==0.13.1"
},
"prometheus-client": {
"hashes": [
@@ -866,10 +874,10 @@
},
"pygments": {
"hashes": [
- "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127",
- "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297"
+ "sha256:83ec6c6133ca6b529b7ff5aa826328fd14b5bb02a58c37f4f06384e96a0f94ab",
+ "sha256:b7949de3d396836085fea596998b135a22610bbcc4f2abfe9e448e44cbc58388"
],
- "version": "==2.4.2"
+ "version": "==2.5.1"
},
"pylint": {
"hashes": [
@@ -888,17 +896,17 @@
},
"pyrsistent": {
"hashes": [
- "sha256:eb6545dbeb1aa69ab1fb4809bfbf5a8705e44d92ef8fc7c2361682a47c46c778"
+ "sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b"
],
- "version": "==0.15.5"
+ "version": "==0.15.6"
},
"pytest": {
"hashes": [
- "sha256:8e256fe71eb74e14a4d20a5987bb5e1488f0511ee800680aaedc62b9358714e8",
- "sha256:ff0090819f669aaa0284d0f4aad1a6d9d67a6efdc6dd4eb4ac56b704f890a0d6"
+ "sha256:63344a2e3bce2e4d522fd62b4fdebb647c019f1f9e4ca075debbd13219db4418",
+ "sha256:f67403f33b2b1d25a6756184077394167fe5e2f9d8bdaab30707d19ccec35427"
],
"index": "pypi",
- "version": "==5.2.4"
+ "version": "==5.3.1"
},
"pytest-cov": {
"hashes": [
diff --git a/backend/metagenedb/api/catalog/views/gene.py b/backend/metagenedb/api/catalog/views/gene.py
index 15d63f9..ada0b5e 100644
--- a/backend/metagenedb/api/catalog/views/gene.py
+++ b/backend/metagenedb/api/catalog/views/gene.py
@@ -1,4 +1,5 @@
import pandas as pd
+from django_pandas.io import read_frame
from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema
from marshmallow.exceptions import ValidationError
@@ -7,10 +8,10 @@ from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.status import HTTP_204_NO_CONTENT, HTTP_422_UNPROCESSABLE_ENTITY
-from metagenedb.common.utils.df_operations import get_mask
from metagenedb.apps.catalog.models import Gene
from metagenedb.api.catalog.qparams_validators.gene import GeneLengthQueryParams
from metagenedb.apps.catalog.serializers import GeneSerializer
+from metagenedb.common.utils.df_operations import get_mask
from .bulk_viewset import BulkViewSet
@@ -52,11 +53,12 @@ class GeneViewSet(BulkViewSet):
serializer_class = GeneSerializer
lookup_field = 'gene_id'
GENE_LENGTH_COL = 'length'
+ DEFAULT_WINDOW_SIZE = 10000
def get_permissions(self):
return super(self.__class__, self).get_permissions()
- def _count_windows(self, df, window_size=10000, window_col=GENE_LENGTH_COL):
+ def _count_windows(self, df, window_size=DEFAULT_WINDOW_SIZE, window_col=GENE_LENGTH_COL):
"""
Count how many line of the df belong to each windows defined by the window_size for the window_col
:param df:
@@ -84,7 +86,7 @@ class GeneViewSet(BulkViewSet):
operation_id='Gene length distribution',
)
@action(methods=['get'], detail=False)
- def gene_length(self, request, window_size=10000):
+ def gene_length(self, request):
try:
query_params = GeneLengthQueryParams().load(request.query_params)
except ValidationError as validation_error:
@@ -93,9 +95,9 @@ class GeneViewSet(BulkViewSet):
'allowed_query_params': ', '.join(GeneLengthQueryParams().declared_fields.keys())
})
return Response(error_message, status=HTTP_422_UNPROCESSABLE_ENTITY)
- if 'window_size' in query_params:
- window_size = query_params.get('window_size')
- df = pd.DataFrame(list(self.queryset.values(self.GENE_LENGTH_COL)))
+
+ window_size = query_params.get('window_size', self.DEFAULT_WINDOW_SIZE)
+ df = read_frame(self.queryset.values(self.GENE_LENGTH_COL))
if df.empty:
return Response(
{'results': {}},
diff --git a/backend/metagenedb/common/utils/profiling.py b/backend/metagenedb/common/utils/profiling.py
new file mode 100644
index 0000000..e1c6d3e
--- /dev/null
+++ b/backend/metagenedb/common/utils/profiling.py
@@ -0,0 +1,16 @@
+import cProfile
+import functools
+
+
+def profile(file_path):
+ def decorator_profile(func):
+ @functools.wraps(func)
+ def wrapper_profile(*args, **kwargs):
+ cp = cProfile.Profile()
+ cp.enable()
+ value = func(*args, **kwargs)
+ cp.disable()
+ cp.dump_stats(file_path)
+ return value
+ return wrapper_profile
+ return decorator_profile
--
GitLab