diff --git a/backend/Pipfile b/backend/Pipfile
index bd9c93b47bdcfa40e58706d9916a10fb471f5b84..a97d56de951efbe75b182852ce933e5721859a5a 100644
--- a/backend/Pipfile
+++ b/backend/Pipfile
@@ -62,6 +62,8 @@ bioapi = {git = "https://github.com/khillion/bioapi.git"}
django-admin-list-filter-dropdown = "*"
gunicorn = "*"
pyfastx = "*"
+celery = "*"
+redis = "*"
[requires]
python_version = "3.7"
diff --git a/backend/Pipfile.lock b/backend/Pipfile.lock
index 8650568a20d8f86afe71e047db85f0898cb983cb..58335afb6e22bbfcd324b4dbc317610f8a5e0f2c 100644
--- a/backend/Pipfile.lock
+++ b/backend/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "1d936f5c462daf6117f9f94b59599097d1aefa89bdf51bc289a953b7196c45bf"
+ "sha256": "9e0bd1084959c97ca9478dc75a8dbf087c1a24e926cf784224e0f2d2431aa99f"
},
"pipfile-spec": 6,
"requires": {
@@ -16,6 +16,13 @@
]
},
"default": {
+ "amqp": {
+ "hashes": [
+ "sha256:6e649ca13a7df3faacdc8bbb280aa9a6602d22fd9d545336077e573a1f4ff3b8",
+ "sha256:77f1aef9410698d20eaeac5b73a87817365f457a507d82edf292e12cbb83b08d"
+ ],
+ "version": "==2.5.2"
+ },
"asgiref": {
"hashes": [
"sha256:7e06d934a7718bf3975acbf87780ba678957b87c7adc056f13b6215d610695a0",
@@ -23,10 +30,25 @@
],
"version": "==3.2.3"
},
+ "billiard": {
+ "hashes": [
+ "sha256:26fd494dc3251f8ce1f5559744f18aeed427fdaf29a75d7baae26752a5d3816f",
+ "sha256:f4e09366653aa3cb3ae8ed16423f9ba1665ff426f087bcdbbed86bf3664fe02c"
+ ],
+ "version": "==3.6.2.0"
+ },
"bioapi": {
"git": "https://github.com/khillion/bioapi.git",
"ref": "699345c6fbd8cc1f25975772d7b6c9a605bf26ff"
},
+ "celery": {
+ "hashes": [
+ "sha256:7c544f37a84a5eadc44cab1aa8c9580dff94636bb81978cdf9bf8012d9ea7d8f",
+ "sha256:d3363bb5df72d74420986a435449f3c3979285941dff57d5d97ecba352a0e3e2"
+ ],
+ "index": "pypi",
+ "version": "==4.4.0"
+ },
"certifi": {
"hashes": [
"sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
@@ -91,11 +113,11 @@
},
"django-extensions": {
"hashes": [
- "sha256:4524eca892d23fa6e93b0620901983b287ff5dc806f1b978d6a98541f06b9471",
- "sha256:936e8e3962024d3c75ea54f4e0248002404ca7ca7fb698430e60b06b5555b4e7"
+ "sha256:1a03c4e8bade575f8c2be6c76456f8a2be3f9b02ab9f47d3535afa9562dc0493",
+ "sha256:2699cc1d6fb4bd393c0b5832fea4bc685f2ace5800b3c9ff222b2080f161ac04"
],
"index": "pypi",
- "version": "==2.2.6"
+ "version": "==2.2.8"
},
"django-filter": {
"hashes": [
@@ -153,6 +175,14 @@
"index": "pypi",
"version": "==2.8"
},
+ "importlib-metadata": {
+ "hashes": [
+ "sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302",
+ "sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
+ ],
+ "markers": "python_version < '3.8'",
+ "version": "==1.5.0"
+ },
"inflection": {
"hashes": [
"sha256:18ea7fb7a7d152853386523def08736aa8c32636b047ade55f7578c4edeb16ca"
@@ -172,6 +202,13 @@
],
"version": "==2.11.1"
},
+ "kombu": {
+ "hashes": [
+ "sha256:2a9e7adff14d046c9996752b2c48b6d9185d0b992106d5160e1a179907a5d4ac",
+ "sha256:67b32ccb6fea030f8799f8fd50dd08e03a4b99464ebc4952d71d8747b1a52ad1"
+ ],
+ "version": "==4.6.7"
+ },
"markupsafe": {
"hashes": [
"sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
@@ -259,23 +296,23 @@
},
"pandas": {
"hashes": [
- "sha256:18bbce2e69855d42397486ee0bb79cb0e4c94af6679fd9392e32ffdb7fcfade0",
- "sha256:35d07389efaf3c478d93725a226941c7fc14714814ba77d6d43b2c9e63ef4af5",
- "sha256:3ea6cc86931f57f18b1240572216f09922d91b19ab8a01cf24734394a3db3bec",
- "sha256:46b0a146e4ba744e350847244767ef297950e9ce02424734b2dd0befd77d9aff",
- "sha256:66c1a49b47c0953dbc6864a6d2578c4c24610f6bb8e4ab165d49b8371aa7745f",
- "sha256:6d5c2d2a3e42100700bac7fe762c17ba0a04d0355feac04bce74a1aa6c8be164",
- "sha256:ab1aa2c50b7c6ba0eccebb146b4d80ed7f5804897b8d54ccddbe49f28c881a94",
- "sha256:ae1ec10e34d22b0f699e38f346381630cae89d5050a2a61315a2be09e3435f99",
- "sha256:b578df33338a09707bfe3e3939c9d46700948133bf829357c3c46795055c9376",
- "sha256:bad77cf498362590ef3a30bc9e769f4fe4399d853861a1ddbefeea8cbf39906c",
- "sha256:c36e4d44d34eaa503776a8fb57ba1305e680e178458c050c2fd8de67604fa209",
- "sha256:d76a8ec22adf0323d362dac8c900b2c66e06eab984ecf04ef072866d8ab6c538",
- "sha256:e8be4f6da608930c0d565240bfbe04fc6f5764d6a9214b02c6231cd5e223591d",
- "sha256:f66c63f357ac31c913f4917f55348ce99c639031567c3284f01dff605da58264"
+ "sha256:23e177d43e4bf68950b0f8788b6a2fef2f478f4ec94883acb627b9264522a98a",
+ "sha256:2530aea4fe46e8df7829c3f05e0a0f821c893885d53cb8ac9b89cc67c143448c",
+ "sha256:303827f0bb40ff610fbada5b12d50014811efcc37aaf6ef03202dc3054bfdda1",
+ "sha256:3b019e3ea9f5d0cfee0efabae2cfd3976874e90bcc3e97b29600e5a9b345ae3d",
+ "sha256:3c07765308f091d81b6735d4f2242bb43c332cc3461cae60543df6b10967fe27",
+ "sha256:5036d4009012a44aa3e50173e482b664c1fae36decd277c49e453463798eca4e",
+ "sha256:6f38969e2325056f9959efbe06c27aa2e94dd35382265ad0703681d993036052",
+ "sha256:74a470d349d52b9d00a2ba192ae1ee22155bb0a300fd1ccb2961006c3fa98ed3",
+ "sha256:7d77034e402165b947f43050a8a415aa3205abfed38d127ea66e57a2b7b5a9e0",
+ "sha256:7f9a509f6f11fa8b9313002ebdf6f690a7aa1dd91efd95d90185371a0d68220e",
+ "sha256:942b5d04762feb0e55b2ad97ce2b254a0ffdd344b56493b04a627266e24f2d82",
+ "sha256:a9fbe41663416bb70ed05f4e16c5f377519c0dc292ba9aa45f5356e37df03a38",
+ "sha256:d10e83866b48c0cdb83281f786564e2a2b51a7ae7b8a950c3442ad3c9e36b48c",
+ "sha256:e2140e1bbf9c46db9936ee70f4be6584d15ff8dc3dfff1da022d71227d53bad3"
],
"index": "pypi",
- "version": "==1.0.0"
+ "version": "==1.0.1"
},
"psycopg2": {
"hashes": [
@@ -369,6 +406,14 @@
"index": "pypi",
"version": "==2019.3"
},
+ "redis": {
+ "hashes": [
+ "sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f",
+ "sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833"
+ ],
+ "index": "pypi",
+ "version": "==3.4.1"
+ },
"requests": {
"hashes": [
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
@@ -446,6 +491,20 @@
],
"index": "pypi",
"version": "==1.25.8"
+ },
+ "vine": {
+ "hashes": [
+ "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
+ "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
+ ],
+ "version": "==1.3.0"
+ },
+ "zipp": {
+ "hashes": [
+ "sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50",
+ "sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e"
+ ],
+ "version": "==2.2.0"
}
},
"develop": {
@@ -454,7 +513,7 @@
"sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0",
"sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"
],
- "markers": "sys_platform == 'darwin'",
+ "markers": "platform_system == 'Darwin'",
"version": "==0.1.0"
},
"astroid": {
@@ -589,7 +648,7 @@
"sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302",
"sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
],
- "index": "pypi",
+ "markers": "python_version < '3.8'",
"version": "==1.5.0"
},
"inflection": {
@@ -680,10 +739,10 @@
},
"jupyter-core": {
"hashes": [
- "sha256:464769f7387d7a62a2403d067f1ddc616655b7f77f5d810c0dd62cb54bfd0fb9",
- "sha256:a183e0ec2e8f6adddf62b0a3fc6a2237e3e0056d381e536d3e7c7ecc3067e244"
+ "sha256:185dfe42800585ca860aa47b5fe0211ee2c33246576d2d664b0b0b8d22aacf3a",
+ "sha256:e91785b8bd7f752711c0c20e5ec6ba0d42178d6321a61396082c55818991caee"
],
- "version": "==4.6.1"
+ "version": "==4.6.2"
},
"kiwisolver": {
"hashes": [
@@ -809,11 +868,11 @@
},
"mock": {
"hashes": [
- "sha256:56358390ebde40c927ec5666626be7d4310a2533ae3aed5c2dd7b55b80687f48",
- "sha256:8fff3fd7c5796ea78ae2847f32e87ad4e111e03fef6e90d03b5efb4882211d78"
+ "sha256:2a572b715f09dd2f0a583d8aeb5bb67d7ed7a8fd31d193cf1227a99c16a67bc3",
+ "sha256:5e48d216809f6f393987ed56920305d8f3c647e6ed35407c1ff2ecb88a9e1151"
],
"index": "pypi",
- "version": "==4.0.0"
+ "version": "==4.0.1"
},
"more-itertools": {
"hashes": [
@@ -1160,11 +1219,10 @@
},
"zipp": {
"hashes": [
- "sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28",
- "sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98"
+ "sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50",
+ "sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e"
],
- "index": "pypi",
- "version": "==2.1.0"
+ "version": "==2.2.0"
}
}
}
diff --git a/backend/metagenedb/api/catalog/urls.py b/backend/metagenedb/api/catalog/urls.py
index 8864400a52a17275575e729a4bb74e7165576a64..704a650a26fdfbc4a98048de7cb1d6b6d7a12e7e 100644
--- a/backend/metagenedb/api/catalog/urls.py
+++ b/backend/metagenedb/api/catalog/urls.py
@@ -4,6 +4,8 @@ from rest_framework.routers import DefaultRouter, DynamicRoute, Route
from metagenedb.api.catalog import views
+from metagenedb.api.catalog.views.celery_test import celery_test_view
+
class CustomRouter(DefaultRouter):
@@ -63,5 +65,6 @@ api_router.register(r'statistics', views.StatisticsViewSet, basename='statistics
urlpatterns = [
url(r'v1/', include((api_router.urls, 'v1'))),
- path('admin/', include(('metagenedb.api.catalog.admin_urls', 'admin')))
+ path('admin/', include(('metagenedb.api.catalog.admin_urls', 'admin'))),
+ path('celery-test/', celery_test_view, name='celery-test')
]
diff --git a/backend/metagenedb/api/catalog/views/celery_test.py b/backend/metagenedb/api/catalog/views/celery_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..07b564d074439ec21786b06952f26afafd306ab9
--- /dev/null
+++ b/backend/metagenedb/api/catalog/views/celery_test.py
@@ -0,0 +1,10 @@
+from metagenedb.celery_app import debug_task
+
+from rest_framework.decorators import api_view
+from rest_framework.response import Response
+
+
+@api_view()
+def celery_test_view(request):
+ debug_task.delay()
+ return Response({"message": "Your task is being processed!"})
diff --git a/backend/metagenedb/apps/core/__init__.py b/backend/metagenedb/apps/core/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/backend/metagenedb/apps/core/management/__init__.py b/backend/metagenedb/apps/core/management/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/backend/metagenedb/apps/core/management/commands/__init__.py b/backend/metagenedb/apps/core/management/commands/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/backend/metagenedb/apps/core/management/commands/watch_celery.py b/backend/metagenedb/apps/core/management/commands/watch_celery.py
new file mode 100644
index 0000000000000000000000000000000000000000..91704388f4114e71ae113020de6b98ccec9ca0ff
--- /dev/null
+++ b/backend/metagenedb/apps/core/management/commands/watch_celery.py
@@ -0,0 +1,27 @@
+"""
+This command allows for celery to be reloaded when project
+code is saved. This command is called in
+`docker-compose.dev.yml` and is only for use in development
+
+https://avilpage.com/2017/05/how-to-auto-reload-celery-workers-in-development.html
+"""
+
+import shlex
+import subprocess
+
+from django.core.management.base import BaseCommand
+from django.utils import autoreload
+
+
+def restart_celery():
+ cmd = 'pkill -9 celery'
+ subprocess.call(shlex.split(cmd))
+ cmd = 'celery worker --app=metagenedb.celery_app:app --loglevel=info'
+ subprocess.call(shlex.split(cmd))
+
+
+class Command(BaseCommand):
+
+ def handle(self, *args, **options):
+ print('Starting celery worker with autoreload...')
+ autoreload.run_with_reloader(restart_celery)
diff --git a/backend/metagenedb/celery_app.py b/backend/metagenedb/celery_app.py
new file mode 100644
index 0000000000000000000000000000000000000000..8e74fb87cae5c0aacb5cbc28348391dd31503951
--- /dev/null
+++ b/backend/metagenedb/celery_app.py
@@ -0,0 +1,15 @@
+import os
+from celery import Celery
+import time
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'metagenedb.settings')
+app = Celery('metagenedb')
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+ print("Doing async task")
+ time.sleep(2)
+ print("Task is done")
diff --git a/backend/metagenedb/settings/__init__.py b/backend/metagenedb/settings/__init__.py
index d74b912b79bc14049cd898fb009af55bb35ece44..0e822e11f55be6c91aab82b5da26e25e414db2e4 100644
--- a/backend/metagenedb/settings/__init__.py
+++ b/backend/metagenedb/settings/__init__.py
@@ -1 +1,2 @@
from .django import * # noqa
+from .celery import * # noqa
diff --git a/backend/metagenedb/settings/celery.py b/backend/metagenedb/settings/celery.py
new file mode 100644
index 0000000000000000000000000000000000000000..28a139d42e146ce40a8888daacf9bdf7badf608c
--- /dev/null
+++ b/backend/metagenedb/settings/celery.py
@@ -0,0 +1,7 @@
+# Celery Configuration
+
+CELERY_BROKER_URL = 'redis://redis:6379'
+CELERY_RESULT_BACKEND = 'redis://redis:6379'
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
diff --git a/backend/metagenedb/settings/django.py b/backend/metagenedb/settings/django.py
index efdcda28f064f58abd25e5aa71e6d355a416a1fa..35e6f33deadbb29b68b980ceb8e505053966d957 100644
--- a/backend/metagenedb/settings/django.py
+++ b/backend/metagenedb/settings/django.py
@@ -10,6 +10,7 @@ environ.Env.read_env(root('.env')) # reading .env file
INSTALLED_APPS = [
'metagenedb.apps.catalog',
+ 'metagenedb.apps.core',
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 1c78a23476ee3ad4879256b3c3da1588012ab46c..8ec1b3828da1e0b8a788e7f69da882eb23816f74 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -59,6 +59,40 @@ services:
networks:
- main
+ redis:
+ image: redis:alpine
+ container_name: redis
+ networks:
+ - main
+
+ celery:
+ container_name: celery
+ build:
+ context: ./backend
+ volumes:
+ - ./backend:/code
+ depends_on:
+ - postgresql
+ - redis
+ command: bash -c 'manage.py watch_celery'
+ networks:
+ - main
+
+ flower:
+ image: mher/flower
+ container_name: flower_dev_vet
+ command: flower --url_prefix=flower
+ environment:
+ - CELERY_BROKER_URL=redis://redis:6379
+ - FLOWER_PORT=5555
+ ports:
+ - 5555:5555
+ networks:
+ - main
+ depends_on:
+ - celery
+ - redis
+
volumes:
postgresql-data:
diff --git a/nginx/dev/nginx.conf b/nginx/dev/nginx.conf
index 112daecc6e15679ee671585e6f97d44cee93cab5..636fb4e2658aa11e3820d0cd6d682d8c988f8b33 100644
--- a/nginx/dev/nginx.conf
+++ b/nginx/dev/nginx.conf
@@ -17,6 +17,10 @@ http {
server frontend:8080;
}
+ upstream flower {
+ server flower:5555;
+ }
+
server {
listen 80;
charset utf-8;
@@ -48,5 +52,16 @@ http {
proxy_set_header Host $http_host;
}
+ # flower
+ location /flower/ {
+ rewrite ^/flower/(.*)$ /$1 break;
+ proxy_pass http://flower/;
+ proxy_set_header Host $host;
+ proxy_redirect off;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ }
+
}
}