Skip to content
GitLab
Commit aaf9cf48 authored by Kenzo-Hugo Hillion's avatar Kenzo-Hugo Hillion ♻
Browse files

Start adding celery to application

parent 67bb84de
Pipeline #24074 passed with stages
in 3 minutes and 2 seconds
Hide whitespace changes
Inline Side-by-side
backend/Pipfile
View file @ aaf9cf48
......@@ -62,6 +62,8 @@ bioapi = {git = "https://github.com/khillion/bioapi.git"}
django-admin-list-filter-dropdown = "*"
gunicorn = "*"
pyfastx = "*"
celery = "*"
redis = "*"
[requires]
python_version = "3.7"
backend/Pipfile.lock
View file @ aaf9cf48
{
"_meta": {
"hash": {
"sha256": "1d936f5c462daf6117f9f94b59599097d1aefa89bdf51bc289a953b7196c45bf"
"sha256": "9e0bd1084959c97ca9478dc75a8dbf087c1a24e926cf784224e0f2d2431aa99f"
},
"pipfile-spec": 6,
"requires": {
......@@ -16,6 +16,13 @@
]
},
"default": {
"amqp": {
"hashes": [
"sha256:6e649ca13a7df3faacdc8bbb280aa9a6602d22fd9d545336077e573a1f4ff3b8",
"sha256:77f1aef9410698d20eaeac5b73a87817365f457a507d82edf292e12cbb83b08d"
],
"version": "==2.5.2"
},
"asgiref": {
"hashes": [
"sha256:7e06d934a7718bf3975acbf87780ba678957b87c7adc056f13b6215d610695a0",
......@@ -23,10 +30,25 @@
],
"version": "==3.2.3"
},
"billiard": {
"hashes": [
"sha256:26fd494dc3251f8ce1f5559744f18aeed427fdaf29a75d7baae26752a5d3816f",
"sha256:f4e09366653aa3cb3ae8ed16423f9ba1665ff426f087bcdbbed86bf3664fe02c"
],
"version": "==3.6.2.0"
},
"bioapi": {
"git": "https://github.com/khillion/bioapi.git",
"ref": "699345c6fbd8cc1f25975772d7b6c9a605bf26ff"
},
"celery": {
"hashes": [
"sha256:7c544f37a84a5eadc44cab1aa8c9580dff94636bb81978cdf9bf8012d9ea7d8f",
"sha256:d3363bb5df72d74420986a435449f3c3979285941dff57d5d97ecba352a0e3e2"
],
"index": "pypi",
"version": "==4.4.0"
},
"certifi": {
"hashes": [
"sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
......@@ -91,11 +113,11 @@
},
"django-extensions": {
"hashes": [
"sha256:4524eca892d23fa6e93b0620901983b287ff5dc806f1b978d6a98541f06b9471",
"sha256:936e8e3962024d3c75ea54f4e0248002404ca7ca7fb698430e60b06b5555b4e7"
"sha256:1a03c4e8bade575f8c2be6c76456f8a2be3f9b02ab9f47d3535afa9562dc0493",
"sha256:2699cc1d6fb4bd393c0b5832fea4bc685f2ace5800b3c9ff222b2080f161ac04"
],
"index": "pypi",
"version": "==2.2.6"
"version": "==2.2.8"
},
"django-filter": {
"hashes": [
......@@ -153,6 +175,14 @@
"index": "pypi",
"version": "==2.8"
},
"importlib-metadata": {
"hashes": [
"sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302",
"sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
],
"markers": "python_version < '3.8'",
"version": "==1.5.0"
},
"inflection": {
"hashes": [
"sha256:18ea7fb7a7d152853386523def08736aa8c32636b047ade55f7578c4edeb16ca"
......@@ -172,6 +202,13 @@
],
"version": "==2.11.1"
},
"kombu": {
"hashes": [
"sha256:2a9e7adff14d046c9996752b2c48b6d9185d0b992106d5160e1a179907a5d4ac",
"sha256:67b32ccb6fea030f8799f8fd50dd08e03a4b99464ebc4952d71d8747b1a52ad1"
],
"version": "==4.6.7"
},
"markupsafe": {
"hashes": [
"sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
......@@ -259,23 +296,23 @@
},
"pandas": {
"hashes": [
"sha256:18bbce2e69855d42397486ee0bb79cb0e4c94af6679fd9392e32ffdb7fcfade0",
"sha256:35d07389efaf3c478d93725a226941c7fc14714814ba77d6d43b2c9e63ef4af5",
"sha256:3ea6cc86931f57f18b1240572216f09922d91b19ab8a01cf24734394a3db3bec",
"sha256:46b0a146e4ba744e350847244767ef297950e9ce02424734b2dd0befd77d9aff",
"sha256:66c1a49b47c0953dbc6864a6d2578c4c24610f6bb8e4ab165d49b8371aa7745f",
"sha256:6d5c2d2a3e42100700bac7fe762c17ba0a04d0355feac04bce74a1aa6c8be164",
"sha256:ab1aa2c50b7c6ba0eccebb146b4d80ed7f5804897b8d54ccddbe49f28c881a94",
"sha256:ae1ec10e34d22b0f699e38f346381630cae89d5050a2a61315a2be09e3435f99",
"sha256:b578df33338a09707bfe3e3939c9d46700948133bf829357c3c46795055c9376",
"sha256:bad77cf498362590ef3a30bc9e769f4fe4399d853861a1ddbefeea8cbf39906c",
"sha256:c36e4d44d34eaa503776a8fb57ba1305e680e178458c050c2fd8de67604fa209",
"sha256:d76a8ec22adf0323d362dac8c900b2c66e06eab984ecf04ef072866d8ab6c538",
"sha256:e8be4f6da608930c0d565240bfbe04fc6f5764d6a9214b02c6231cd5e223591d",
"sha256:f66c63f357ac31c913f4917f55348ce99c639031567c3284f01dff605da58264"
"sha256:23e177d43e4bf68950b0f8788b6a2fef2f478f4ec94883acb627b9264522a98a",
"sha256:2530aea4fe46e8df7829c3f05e0a0f821c893885d53cb8ac9b89cc67c143448c",
"sha256:303827f0bb40ff610fbada5b12d50014811efcc37aaf6ef03202dc3054bfdda1",
"sha256:3b019e3ea9f5d0cfee0efabae2cfd3976874e90bcc3e97b29600e5a9b345ae3d",
"sha256:3c07765308f091d81b6735d4f2242bb43c332cc3461cae60543df6b10967fe27",
"sha256:5036d4009012a44aa3e50173e482b664c1fae36decd277c49e453463798eca4e",
"sha256:6f38969e2325056f9959efbe06c27aa2e94dd35382265ad0703681d993036052",
"sha256:74a470d349d52b9d00a2ba192ae1ee22155bb0a300fd1ccb2961006c3fa98ed3",
"sha256:7d77034e402165b947f43050a8a415aa3205abfed38d127ea66e57a2b7b5a9e0",
"sha256:7f9a509f6f11fa8b9313002ebdf6f690a7aa1dd91efd95d90185371a0d68220e",
"sha256:942b5d04762feb0e55b2ad97ce2b254a0ffdd344b56493b04a627266e24f2d82",
"sha256:a9fbe41663416bb70ed05f4e16c5f377519c0dc292ba9aa45f5356e37df03a38",
"sha256:d10e83866b48c0cdb83281f786564e2a2b51a7ae7b8a950c3442ad3c9e36b48c",
"sha256:e2140e1bbf9c46db9936ee70f4be6584d15ff8dc3dfff1da022d71227d53bad3"
],
"index": "pypi",
"version": "==1.0.0"
"version": "==1.0.1"
},
"psycopg2": {
"hashes": [
......@@ -369,6 +406,14 @@
"index": "pypi",
"version": "==2019.3"
},
"redis": {
"hashes": [
"sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f",
"sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833"
],
"index": "pypi",
"version": "==3.4.1"
},
"requests": {
"hashes": [
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
......@@ -446,6 +491,20 @@
],
"index": "pypi",
"version": "==1.25.8"
},
"vine": {
"hashes": [
"sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
"sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
],
"version": "==1.3.0"
},
"zipp": {
"hashes": [
"sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50",
"sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e"
],
"version": "==2.2.0"
}
},
"develop": {
......@@ -454,7 +513,7 @@
"sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0",
"sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"
],
"markers": "sys_platform == 'darwin'",
"markers": "platform_system == 'Darwin'",
"version": "==0.1.0"
},
"astroid": {
......@@ -589,7 +648,7 @@
"sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302",
"sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
],
"index": "pypi",
"markers": "python_version < '3.8'",
"version": "==1.5.0"
},
"inflection": {
......@@ -680,10 +739,10 @@
},
"jupyter-core": {
"hashes": [
"sha256:464769f7387d7a62a2403d067f1ddc616655b7f77f5d810c0dd62cb54bfd0fb9",
"sha256:a183e0ec2e8f6adddf62b0a3fc6a2237e3e0056d381e536d3e7c7ecc3067e244"
"sha256:185dfe42800585ca860aa47b5fe0211ee2c33246576d2d664b0b0b8d22aacf3a",
"sha256:e91785b8bd7f752711c0c20e5ec6ba0d42178d6321a61396082c55818991caee"
],
"version": "==4.6.1"
"version": "==4.6.2"
},
"kiwisolver": {
"hashes": [
......@@ -809,11 +868,11 @@
},
"mock": {
"hashes": [
"sha256:56358390ebde40c927ec5666626be7d4310a2533ae3aed5c2dd7b55b80687f48",
"sha256:8fff3fd7c5796ea78ae2847f32e87ad4e111e03fef6e90d03b5efb4882211d78"
"sha256:2a572b715f09dd2f0a583d8aeb5bb67d7ed7a8fd31d193cf1227a99c16a67bc3",
"sha256:5e48d216809f6f393987ed56920305d8f3c647e6ed35407c1ff2ecb88a9e1151"
],
"index": "pypi",
"version": "==4.0.0"
"version": "==4.0.1"
},
"more-itertools": {
"hashes": [
......@@ -1160,11 +1219,10 @@
},
"zipp": {
"hashes": [
"sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28",
"sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98"
"sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50",
"sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e"
],
"index": "pypi",
"version": "==2.1.0"
"version": "==2.2.0"
}
}
}
backend/metagenedb/api/catalog/urls.py
View file @ aaf9cf48
......@@ -4,6 +4,8 @@ from rest_framework.routers import DefaultRouter, DynamicRoute, Route
from metagenedb.api.catalog import views
from metagenedb.api.catalog.views.celery_test import celery_test_view
class CustomRouter(DefaultRouter):
......@@ -63,5 +65,6 @@ api_router.register(r'statistics', views.StatisticsViewSet, basename='statistics
urlpatterns = [
url(r'v1/', include((api_router.urls, 'v1'))),
path('admin/', include(('metagenedb.api.catalog.admin_urls', 'admin')))
path('admin/', include(('metagenedb.api.catalog.admin_urls', 'admin'))),
path('celery-test/', celery_test_view, name='celery-test')
]
backend/metagenedb/api/catalog/views/celery_test.py 0 → 100644
View file @ aaf9cf48
from metagenedb.celery_app import debug_task
from rest_framework.decorators import api_view
from rest_framework.response import Response
@api_view()
def celery_test_view(request):
debug_task.delay()
return Response({"message": "Your task is being processed!"})
backend/metagenedb/apps/core/management/commands/watch_celery.py 0 → 100644
View file @ aaf9cf48
"""
This command allows for celery to be reloaded when project
code is saved. This command is called in
`docker-compose.dev.yml` and is only for use in development
https://avilpage.com/2017/05/how-to-auto-reload-celery-workers-in-development.html
"""
import shlex
import subprocess
from django.core.management.base import BaseCommand
from django.utils import autoreload
def restart_celery():
cmd = 'pkill -9 celery'
subprocess.call(shlex.split(cmd))
cmd = 'celery worker --app=metagenedb.celery_app:app --loglevel=info'
subprocess.call(shlex.split(cmd))
class Command(BaseCommand):
def handle(self, *args, **options):
print('Starting celery worker with autoreload...')
autoreload.run_with_reloader(restart_celery)
backend/metagenedb/celery_app.py 0 → 100644
View file @ aaf9cf48
import os
from celery import Celery
import time
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'metagenedb.settings')
app = Celery('metagenedb')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
@app.task(bind=True)
def debug_task(self):
print("Doing async task")
time.sleep(2)
print("Task is done")
backend/metagenedb/settings/__init__.py
View file @ aaf9cf48
from .django import * # noqa
from .celery import * # noqa
backend/metagenedb/settings/celery.py 0 → 100644
View file @ aaf9cf48
# Celery Configuration
CELERY_BROKER_URL = 'redis://redis:6379'
CELERY_RESULT_BACKEND = 'redis://redis:6379'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
backend/metagenedb/settings/django.py
View file @ aaf9cf48
......@@ -10,6 +10,7 @@ environ.Env.read_env(root('.env')) # reading .env file
INSTALLED_APPS = [
'metagenedb.apps.catalog',
'metagenedb.apps.core',
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
......
docker-compose.yaml
View file @ aaf9cf48
......@@ -59,6 +59,40 @@ services:
networks:
- main
redis:
image: redis:alpine
container_name: redis
networks:
- main
celery:
container_name: celery
build:
context: ./backend
volumes:
- ./backend:/code
depends_on:
- postgresql
- redis
command: bash -c 'manage.py watch_celery'
networks:
- main
flower:
image: mher/flower
container_name: flower_dev_vet
command: flower --url_prefix=flower
environment:
- CELERY_BROKER_URL=redis://redis:6379
- FLOWER_PORT=5555
ports:
- 5555:5555
networks:
- main
depends_on:
- celery
- redis
volumes:
postgresql-data:
......
nginx/dev/nginx.conf
View file @ aaf9cf48
......@@ -17,6 +17,10 @@ http {
server frontend:8080;
}
upstream flower {
server flower:5555;
}
server {
listen 80;
charset utf-8;
......@@ -48,5 +52,16 @@ http {
proxy_set_header Host $http_host;
}
# flower
location /flower/ {
rewrite ^/flower/(.*)$ /$1 break;
proxy_pass http://flower/;
proxy_set_header Host $host;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment