Merge branch '34-celery-redis' into 'dev'

Celery and Redis

See merge request !43

backend/Pipfile

@@ -62,6 +62,10 @@ bioapi = {git = "https://github.com/khillion/bioapi.git"}
 django-admin-list-filter-dropdown = "*"
 gunicorn = "*"
 pyfastx = "*"
+celery = "*"
+redis = "*"
+django-health-check = "==3.0.0"
+django-redis = "*"
 
 [requires]
 python_version = "3.7"

backend/metagenedb/__init__.py

+from .celery_app import app as celery_app  # noqa
+
+__all__ = ('celery_app')

backend/metagenedb/api/catalog/urls.py

@@ -4,6 +4,8 @@ from rest_framework.routers import DefaultRouter, DynamicRoute, Route
 
 from metagenedb.api.catalog import views
 
+from metagenedb.api.catalog.views.celery_test import celery_test_view, test_task_view
+
 
 class CustomRouter(DefaultRouter):
 
@@ -63,5 +65,7 @@ api_router.register(r'statistics', views.StatisticsViewSet, basename='statistics
 
 urlpatterns = [
     url(r'v1/', include((api_router.urls, 'v1'))),
-    path('admin/', include(('metagenedb.api.catalog.admin_urls', 'admin')))
+    path('admin/', include(('metagenedb.api.catalog.admin_urls', 'admin'))),
+    path('celery-test/', celery_test_view, name='celery-test'),
+    path('celery-task-test/', test_task_view, name='celery-task-test')
 ]

backend/metagenedb/api/catalog/views/celery_test.py

+import logging
+
+from celery.decorators import task
+from rest_framework.decorators import api_view
+from rest_framework.response import Response
+
+from metagenedb.celery_app import debug_task
+
+logger = logging.getLogger(__name__)
+
+
+@api_view()
+def celery_test_view(request):
+    debug_task.delay()
+    return Response({"message": "Your task is being processed!"})
+
+
+@task(name="test_celery_task")
+def test_task(msg):
+    logger.info("Test Celery Task")
+    return msg
+
+
+@task(name="poete")
+def mul(x, y):
+    return x * y
+
+
+@api_view()
+def test_task_view(request):
+    msg = test_task.delay("poooouet")
+    muli = mul.delay(2, 4)
+    return Response({"message": f"Tested celery task: {msg.get(timeout=1)} {muli.get(timeout=1)}"})

backend/metagenedb/apps/core/management/commands/watch_celery.py

+"""
+This command allows for celery to be reloaded when project
+code is saved. This command is called in
+`docker-compose.dev.yml` and is only for use in development
+
+https://avilpage.com/2017/05/how-to-auto-reload-celery-workers-in-development.html
+"""
+
+import shlex
+import subprocess
+
+from django.core.management.base import BaseCommand
+from django.utils import autoreload
+
+
+def restart_celery():
+    cmd = 'pkill -9 celery'
+    subprocess.call(shlex.split(cmd))
+    cmd = 'celery worker --app=metagenedb.celery_app:app --loglevel=info'
+    subprocess.call(shlex.split(cmd))
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        print('Starting celery worker with autoreload...')
+        autoreload.run_with_reloader(restart_celery)

backend/metagenedb/celery_app.py

+import os
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'metagenedb.settings')
+app = Celery('metagenedb')
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()
+
+app.conf.beat_schedule = {
+    'display_time-20-seconds': {
+        'task': 'demoapp.tasks.display_time',
+        'schedule': 20.0
+    },
+}
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))

backend/metagenedb/demoapp/tasks.py

+from datetime import datetime
+from celery import shared_task
+
+
+@shared_task
+def display_time():
+    print("The time is %s :" % str(datetime.now()))
+    return True

backend/metagenedb/settings/__init__.py

 from .django import *  # noqa
+from .celery import *  # noqa

backend/metagenedb/settings/celery.py

+import os
+
+# REDIS
+REDIS_URL = "redis://{host}:{port}/1".format(
+    host=os.getenv('REDIS_HOST', 'redis'),
+    port=os.getenv('REDIS_PORT', '6379')
+
+)
+# CELERY
+CELERY_BROKER_URL = REDIS_URL
+CELERY_RESULT_BACKEND = REDIS_URL
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_TASK_SERIALIZER = 'json'
+
+# CACHE
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_URL,
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient"
+        },
+        "KEY_PREFIX": "example"
+    }
+}

backend/metagenedb/settings/django.py

@@ -10,6 +10,7 @@ environ.Env.read_env(root('.env'))  # reading .env file
 
 INSTALLED_APPS = [
     'metagenedb.apps.catalog',
+    'metagenedb.apps.core',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',

docker-compose.yaml

@@ -53,16 +53,63 @@ services:
     depends_on:
       - backend
       - frontend
+      - flower
+      - portainer
     volumes:
       - ./nginx/dev/nginx.conf:/etc/nginx/nginx.conf:ro
       - django-static:/usr/src/app/static
     networks:
       - main
 
+  redis:
+    image: redis:alpine
+    container_name: redis
+    networks:
+      - main
+
+  celery:
+    container_name: celery
+    build:
+      context: ./backend
+    volumes:
+      - ./backend:/code
+    depends_on:
+      - postgresql
+      - redis
+    command: bash -c 'manage.py watch_celery'
+    networks:
+      - main
+
+  flower:
+    image: mher/flower
+    container_name: flower
+    command: flower --url_prefix=flower
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379
+      - FLOWER_PORT=5555
+    ports:
+      - 5555:5555
+    networks:
+      - main
+    depends_on:
+      - celery
+      - redis
+
+  portainer:
+    image: portainer/portainer
+    container_name: portainer
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - portainer-data:/data
+    ports:
+      - 9000:9000
+    networks:
+      - main
 
 volumes:
   postgresql-data:
   django-static:
+  portainer-data:
 
 networks:
   main:

nginx/dev/nginx.conf

@@ -17,6 +17,14 @@ http {
     server frontend:8080;
   }
 
+  upstream flower {
+    server flower:5555;
+  }
+
+  upstream portainer {
+      server portainer:9000;
+  }
+
   server {
     listen 80;
     charset utf-8;
@@ -48,5 +56,29 @@ http {
       proxy_set_header Host $http_host;
     }
 
+    # flower
+    location /flower/ {
+        rewrite ^/flower/(.*)$ /$1 break;
+        proxy_pass http://flower/;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+
+    # portainer
+    location /portainer/ {
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+        proxy_pass http://portainer/;
+    }
+    location /portainer/api/websocket/ {
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_http_version 1.1;
+        proxy_pass http://portainer/api/websocket/;
+    }
+
   }
 }