Closes #34 set up computation of stats on celery worker

backend/metagenedb/api/catalog/admin_urls.py

@@ -7,4 +7,5 @@ urlpatterns = [
     path('compute-counts/', admin_commands.ComputeCountsAPI.as_view()),
     path('compute-gene-length/', admin_commands.ComputeGeneLengthAPI.as_view()),
     path('compute-taxo-repartition/', admin_commands.ComputeTaxonomyRepartitionAPI.as_view()),
+    path('compute-taxo-presence/', admin_commands.ComputeTaxonomyPresenceAPI.as_view()),
 ]

backend/metagenedb/api/catalog/views/admin_commands.py

 import logging
 
 from drf_yasg.utils import swagger_auto_schema
-from drf_yasg.openapi import Response
+# from drf_yasg.openapi import Response
 from rest_framework import permissions
-# from rest_framework.response import Response
+from rest_framework.response import Response
 from rest_framework.status import HTTP_500_INTERNAL_SERVER_ERROR
 from rest_framework.views import APIView
 from rest_framework_jwt.authentication import JSONWebTokenAuthentication
 
 from metagenedb.apps.catalog.management.commands.compute_stats import (
-    ComputeCounts, ComputeGeneLength, ComputeTaxonomyRepartition
+    compute_gene_counts, compute_gene_length, compute_taxonomy_presence, compute_taxonomy_repartition
 )
 
 logger = logging.getLogger(__name__)
+SOURCE_CHOICES = ['all', 'virgo', 'igc']
 
 
 class AdminCommandsAPI(APIView):
@@ -21,13 +22,13 @@ class AdminCommandsAPI(APIView):
 
 
 class ComputeStatisticsAPI(AdminCommandsAPI):
-    compute_class = None
+    compute_function = None
     error_message = "Could not compute stats due to server error"
-    success_message = "Stats successfully computed"
+    success_message = "Stats successfully computing on worker...."
 
     def get(self, request, format=None):
         try:
-            self.compute_class().all()
+            self.compute_function.delay(SOURCE_CHOICES)
         except Exception as exception:
             logger.warning(exception)
             return Response({"message": self.error_message},
@@ -36,8 +37,8 @@ class ComputeStatisticsAPI(AdminCommandsAPI):
 
 
 class ComputeCountsAPI(ComputeStatisticsAPI):
-    compute_class = ComputeCounts
-    success_message = "Counts on gene annotation successfully computed"
+    compute_function = compute_gene_counts
+    success_message = "Gene counts successfully computing on worker...."
 
     @swagger_auto_schema(
         operation_description="Compute counts for the different annotations of the genes",
@@ -48,9 +49,22 @@ class ComputeCountsAPI(ComputeStatisticsAPI):
         return super().get(*args, **kwargs)
 
 
+class ComputeGeneLengthAPI(ComputeStatisticsAPI):
+    compute_function = compute_gene_length
+    success_message = "Gene length successfully computing on worker...."
+
+    @swagger_auto_schema(
+        operation_description="Compute gene length distribution",
+        operation_summary="API to compute gene length distribution.",
+        tags=['admin-commands'],
+    )
+    def get(self, *args, **kwargs):
+        return super().get(*args, **kwargs)
+
+
 class ComputeTaxonomyRepartitionAPI(ComputeStatisticsAPI):
-    compute_class = ComputeTaxonomyRepartition
-    success_message = "Different taxonomy repartition successfully computed"
+    compute_function = compute_taxonomy_repartition
+    success_message = "Taxonomy repartition successfully computing on worker...."
 
     @swagger_auto_schema(
         operation_description="Compute repartition of taxonomy among genes",
@@ -61,13 +75,13 @@ class ComputeTaxonomyRepartitionAPI(ComputeStatisticsAPI):
         return super().get(*args, **kwargs)
 
 
-class ComputeGeneLengthAPI(ComputeStatisticsAPI):
-    compute_class = ComputeGeneLength
-    success_message = "Gene length for different windows size successfully computed"
+class ComputeTaxonomyPresenceAPI(ComputeStatisticsAPI):
+    compute_function = compute_taxonomy_presence
+    success_message = "Taxonomy presence successfully computing on worker...."
 
     @swagger_auto_schema(
-        operation_description="Compute gene length distribution",
-        operation_summary="API to compute gene length distribution.",
+        operation_description="Compute presence of taxonomy among genes",
+        operation_summary="API to compute presence of taxonomy.",
         tags=['admin-commands'],
     )
     def get(self, *args, **kwargs):

backend/metagenedb/api/catalog/views/celery_test.py

@@ -13,13 +13,7 @@ def test_task(msg):
     return msg
 
 
-@task(name="poete")
-def mul(x, y):
-    return x * y
-
-
 @api_view()
 def test_task_view(request):
-    msg = test_task.delay("pirooooouettee")
-    muli = mul.delay(2, 4)
-    return Response({"message": f"Tested celery task: {msg.get(timeout=1)} {muli.get(timeout=1)}"})
+    msg = test_task.delay("I am alive")
+    return Response({"message": f"Tested celery task: {msg.get(timeout=60)}"})

backend/metagenedb/apps/catalog/management/commands/compute_stats.py

 import logging
+from typing import List
 
+from celery.decorators import task
 from django.core.exceptions import ValidationError
 from django.core.management.base import BaseCommand
 from slugify import slugify
@@ -30,10 +32,6 @@ class ComputeStatistics:
                 raise(validation_error)
         statistics.save()
 
-    def clean_db(self):
-        logger.info("Deleting all statistics entries...")
-        Statistics.objects.all().delete()
-
 
 class ComputeCounts(ComputeStatistics):
     METHODS = [
@@ -163,6 +161,35 @@ class ComputeTaxonomyPresence(ComputeStatistics):
             self._save_to_db(payload)
 
 
+def clean_db():
+    logger.info("Deleting all statistics entries...")
+    Statistics.objects.all().delete()
+
+
+@task(name="compute_stats.compute_gene_counts")
+def compute_gene_counts(gene_sources: List[str]):
+    for gene_source in gene_sources:
+        ComputeCounts(gene_source).all()
+
+
+@task(name="compute_stats.compute_gene_length")
+def compute_gene_length(gene_sources: List[str]):
+    for gene_source in gene_sources:
+        ComputeGeneLength(gene_source).all()
+
+
+@task(name="compute_stats.compute_taxonomy_repartition")
+def compute_taxonomy_repartition(gene_sources: List[str]):
+    for gene_source in gene_sources:
+        ComputeTaxonomyRepartition(gene_source).all()
+
+
+@task(name="compute_stats.compute_taxonomy_presence")
+def compute_taxonomy_presence(gene_sources: List[str]):
+    for gene_source in gene_sources:
+        ComputeTaxonomyPresence(gene_source).all()
+
+
 class Command(BaseCommand):
     help = "Compute gene catalog statistics."
     STEP_CHOICES = ['clean', 'counts', 'gene-length', 'taxonomy_repartition', 'taxonomy_presence']
@@ -202,14 +229,13 @@ class Command(BaseCommand):
         self.set_logger_level(int(options['verbosity']))
         only_step = self._get_and_validate_only_step(options['only'])
         gene_sources = self._get_and_validate_source(options['source'])
-        for gene_source in gene_sources:
-            if only_step == "clean":
-                ComputeStatistics(gene_source).clean_db()
-            if only_step is None or only_step == "counts":
-                ComputeCounts(gene_source).all()
-            if only_step is None or only_step == "gene-length":
-                ComputeGeneLength(gene_source).all()
-            if only_step is None or only_step == "taxonomy_repartition":
-                ComputeTaxonomyRepartition(gene_source).all()
-            if only_step is None or only_step == "taxonomy_presence":
-                ComputeTaxonomyPresence(gene_source).all()
+        if only_step == "clean":
+            clean_db()
+        if only_step is None or only_step == "counts":
+            compute_gene_counts.delay(gene_sources)
+        if only_step is None or only_step == "gene-length":
+            compute_gene_length.delay(gene_sources)
+        if only_step is None or only_step == "taxonomy_repartition":
+            compute_taxonomy_repartition.delay(gene_sources)
+        if only_step is None or only_step == "taxonomy_presence":
+            compute_taxonomy_presence.delay(gene_sources)