Commit 9891cf32 authored by Kenzo-Hugo Hillion's avatar Kenzo-Hugo Hillion
Browse files

Closes #34 set up computation of stats on celery worker

parent 5333a16a
Pipeline #35301 passed with stages
in 3 minutes and 41 seconds
......@@ -7,4 +7,5 @@ urlpatterns = [
path('compute-counts/', admin_commands.ComputeCountsAPI.as_view()),
path('compute-gene-length/', admin_commands.ComputeGeneLengthAPI.as_view()),
path('compute-taxo-repartition/', admin_commands.ComputeTaxonomyRepartitionAPI.as_view()),
path('compute-taxo-presence/', admin_commands.ComputeTaxonomyPresenceAPI.as_view()),
]
import logging
from drf_yasg.utils import swagger_auto_schema
from drf_yasg.openapi import Response
# from drf_yasg.openapi import Response
from rest_framework import permissions
# from rest_framework.response import Response
from rest_framework.response import Response
from rest_framework.status import HTTP_500_INTERNAL_SERVER_ERROR
from rest_framework.views import APIView
from rest_framework_jwt.authentication import JSONWebTokenAuthentication
from metagenedb.apps.catalog.management.commands.compute_stats import (
ComputeCounts, ComputeGeneLength, ComputeTaxonomyRepartition
compute_gene_counts, compute_gene_length, compute_taxonomy_presence, compute_taxonomy_repartition
)
logger = logging.getLogger(__name__)
SOURCE_CHOICES = ['all', 'virgo', 'igc']
class AdminCommandsAPI(APIView):
......@@ -21,13 +22,13 @@ class AdminCommandsAPI(APIView):
class ComputeStatisticsAPI(AdminCommandsAPI):
compute_class = None
compute_function = None
error_message = "Could not compute stats due to server error"
success_message = "Stats successfully computed"
success_message = "Stats successfully computing on worker...."
def get(self, request, format=None):
try:
self.compute_class().all()
self.compute_function.delay(SOURCE_CHOICES)
except Exception as exception:
logger.warning(exception)
return Response({"message": self.error_message},
......@@ -36,8 +37,8 @@ class ComputeStatisticsAPI(AdminCommandsAPI):
class ComputeCountsAPI(ComputeStatisticsAPI):
compute_class = ComputeCounts
success_message = "Counts on gene annotation successfully computed"
compute_function = compute_gene_counts
success_message = "Gene counts successfully computing on worker...."
@swagger_auto_schema(
operation_description="Compute counts for the different annotations of the genes",
......@@ -48,9 +49,22 @@ class ComputeCountsAPI(ComputeStatisticsAPI):
return super().get(*args, **kwargs)
class ComputeGeneLengthAPI(ComputeStatisticsAPI):
compute_function = compute_gene_length
success_message = "Gene length successfully computing on worker...."
@swagger_auto_schema(
operation_description="Compute gene length distribution",
operation_summary="API to compute gene length distribution.",
tags=['admin-commands'],
)
def get(self, *args, **kwargs):
return super().get(*args, **kwargs)
class ComputeTaxonomyRepartitionAPI(ComputeStatisticsAPI):
compute_class = ComputeTaxonomyRepartition
success_message = "Different taxonomy repartition successfully computed"
compute_function = compute_taxonomy_repartition
success_message = "Taxonomy repartition successfully computing on worker...."
@swagger_auto_schema(
operation_description="Compute repartition of taxonomy among genes",
......@@ -61,13 +75,13 @@ class ComputeTaxonomyRepartitionAPI(ComputeStatisticsAPI):
return super().get(*args, **kwargs)
class ComputeGeneLengthAPI(ComputeStatisticsAPI):
compute_class = ComputeGeneLength
success_message = "Gene length for different windows size successfully computed"
class ComputeTaxonomyPresenceAPI(ComputeStatisticsAPI):
compute_function = compute_taxonomy_presence
success_message = "Taxonomy presence successfully computing on worker...."
@swagger_auto_schema(
operation_description="Compute gene length distribution",
operation_summary="API to compute gene length distribution.",
operation_description="Compute presence of taxonomy among genes",
operation_summary="API to compute presence of taxonomy.",
tags=['admin-commands'],
)
def get(self, *args, **kwargs):
......
......@@ -13,13 +13,7 @@ def test_task(msg):
return msg
@task(name="poete")
def mul(x, y):
return x * y
@api_view()
def test_task_view(request):
msg = test_task.delay("pirooooouettee")
muli = mul.delay(2, 4)
return Response({"message": f"Tested celery task: {msg.get(timeout=1)} {muli.get(timeout=1)}"})
msg = test_task.delay("I am alive")
return Response({"message": f"Tested celery task: {msg.get(timeout=60)}"})
import logging
from typing import List
from celery.decorators import task
from django.core.exceptions import ValidationError
from django.core.management.base import BaseCommand
from slugify import slugify
......@@ -30,10 +32,6 @@ class ComputeStatistics:
raise(validation_error)
statistics.save()
def clean_db(self):
logger.info("Deleting all statistics entries...")
Statistics.objects.all().delete()
class ComputeCounts(ComputeStatistics):
METHODS = [
......@@ -163,6 +161,35 @@ class ComputeTaxonomyPresence(ComputeStatistics):
self._save_to_db(payload)
def clean_db():
logger.info("Deleting all statistics entries...")
Statistics.objects.all().delete()
@task(name="compute_stats.compute_gene_counts")
def compute_gene_counts(gene_sources: List[str]):
for gene_source in gene_sources:
ComputeCounts(gene_source).all()
@task(name="compute_stats.compute_gene_length")
def compute_gene_length(gene_sources: List[str]):
for gene_source in gene_sources:
ComputeGeneLength(gene_source).all()
@task(name="compute_stats.compute_taxonomy_repartition")
def compute_taxonomy_repartition(gene_sources: List[str]):
for gene_source in gene_sources:
ComputeTaxonomyRepartition(gene_source).all()
@task(name="compute_stats.compute_taxonomy_presence")
def compute_taxonomy_presence(gene_sources: List[str]):
for gene_source in gene_sources:
ComputeTaxonomyPresence(gene_source).all()
class Command(BaseCommand):
help = "Compute gene catalog statistics."
STEP_CHOICES = ['clean', 'counts', 'gene-length', 'taxonomy_repartition', 'taxonomy_presence']
......@@ -202,14 +229,13 @@ class Command(BaseCommand):
self.set_logger_level(int(options['verbosity']))
only_step = self._get_and_validate_only_step(options['only'])
gene_sources = self._get_and_validate_source(options['source'])
for gene_source in gene_sources:
if only_step == "clean":
ComputeStatistics(gene_source).clean_db()
if only_step is None or only_step == "counts":
ComputeCounts(gene_source).all()
if only_step is None or only_step == "gene-length":
ComputeGeneLength(gene_source).all()
if only_step is None or only_step == "taxonomy_repartition":
ComputeTaxonomyRepartition(gene_source).all()
if only_step is None or only_step == "taxonomy_presence":
ComputeTaxonomyPresence(gene_source).all()
if only_step == "clean":
clean_db()
if only_step is None or only_step == "counts":
compute_gene_counts.delay(gene_sources)
if only_step is None or only_step == "gene-length":
compute_gene_length.delay(gene_sources)
if only_step is None or only_step == "taxonomy_repartition":
compute_taxonomy_repartition.delay(gene_sources)
if only_step is None or only_step == "taxonomy_presence":
compute_taxonomy_presence.delay(gene_sources)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment