Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Metagenomics
metagenedb
Commits
aaf9cf48
Commit
aaf9cf48
authored
Feb 11, 2020
by
Kenzo-Hugo Hillion
♻
Browse files
Start adding celery to application
parent
67bb84de
Pipeline
#24074
passed with stages
in 3 minutes and 2 seconds
Changes
14
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
backend/Pipfile
View file @
aaf9cf48
...
...
@@ -62,6 +62,8 @@ bioapi = {git = "https://github.com/khillion/bioapi.git"}
django-admin-list-filter-dropdown
=
"*"
gunicorn
=
"*"
pyfastx
=
"*"
celery
=
"*"
redis
=
"*"
[requires]
python_version
=
"3.7"
backend/Pipfile.lock
View file @
aaf9cf48
{
"_meta"
:
{
"hash"
:
{
"sha256"
:
"
1d936f5c462daf6117f9f94b59599097d1aefa89bdf51bc289a953b7196c45b
f"
"sha256"
:
"
9e0bd1084959c97ca9478dc75a8dbf087c1a24e926cf784224e0f2d2431aa99
f"
},
"pipfile-spec"
:
6
,
"requires"
:
{
...
...
@@ -16,6 +16,13 @@
]
},
"default"
:
{
"amqp"
:
{
"hashes"
:
[
"sha256:6e649ca13a7df3faacdc8bbb280aa9a6602d22fd9d545336077e573a1f4ff3b8"
,
"sha256:77f1aef9410698d20eaeac5b73a87817365f457a507d82edf292e12cbb83b08d"
],
"version"
:
"==2.5.2"
},
"asgiref"
:
{
"hashes"
:
[
"sha256:7e06d934a7718bf3975acbf87780ba678957b87c7adc056f13b6215d610695a0"
,
...
...
@@ -23,10 +30,25 @@
],
"version"
:
"==3.2.3"
},
"billiard"
:
{
"hashes"
:
[
"sha256:26fd494dc3251f8ce1f5559744f18aeed427fdaf29a75d7baae26752a5d3816f"
,
"sha256:f4e09366653aa3cb3ae8ed16423f9ba1665ff426f087bcdbbed86bf3664fe02c"
],
"version"
:
"==3.6.2.0"
},
"bioapi"
:
{
"git"
:
"https://github.com/khillion/bioapi.git"
,
"ref"
:
"699345c6fbd8cc1f25975772d7b6c9a605bf26ff"
},
"celery"
:
{
"hashes"
:
[
"sha256:7c544f37a84a5eadc44cab1aa8c9580dff94636bb81978cdf9bf8012d9ea7d8f"
,
"sha256:d3363bb5df72d74420986a435449f3c3979285941dff57d5d97ecba352a0e3e2"
],
"index"
:
"pypi"
,
"version"
:
"==4.4.0"
},
"certifi"
:
{
"hashes"
:
[
"sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3"
,
...
...
@@ -91,11 +113,11 @@
},
"django-extensions"
:
{
"hashes"
:
[
"sha256:
4524eca892d23fa6e93b0620901983b287ff5dc806f1b978d6a98541f06b9471
"
,
"sha256:
936e8e3962024d3c75ea54f4e0248002404ca7ca7fb698430e60b06b5555b4e7
"
"sha256:
1a03c4e8bade575f8c2be6c76456f8a2be3f9b02ab9f47d3535afa9562dc0493
"
,
"sha256:
2699cc1d6fb4bd393c0b5832fea4bc685f2ace5800b3c9ff222b2080f161ac04
"
],
"index"
:
"pypi"
,
"version"
:
"==2.2.
6
"
"version"
:
"==2.2.
8
"
},
"django-filter"
:
{
"hashes"
:
[
...
...
@@ -153,6 +175,14 @@
"index"
:
"pypi"
,
"version"
:
"==2.8"
},
"importlib-metadata"
:
{
"hashes"
:
[
"sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302"
,
"sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
],
"markers"
:
"python_version < '3.8'"
,
"version"
:
"==1.5.0"
},
"inflection"
:
{
"hashes"
:
[
"sha256:18ea7fb7a7d152853386523def08736aa8c32636b047ade55f7578c4edeb16ca"
...
...
@@ -172,6 +202,13 @@
],
"version"
:
"==2.11.1"
},
"kombu"
:
{
"hashes"
:
[
"sha256:2a9e7adff14d046c9996752b2c48b6d9185d0b992106d5160e1a179907a5d4ac"
,
"sha256:67b32ccb6fea030f8799f8fd50dd08e03a4b99464ebc4952d71d8747b1a52ad1"
],
"version"
:
"==4.6.7"
},
"markupsafe"
:
{
"hashes"
:
[
"sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473"
,
...
...
@@ -259,23 +296,23 @@
},
"pandas"
:
{
"hashes"
:
[
"sha256:
18bbce2e69855d42397486ee0bb79cb0e4c94af6679fd9392e32ffdb7fcfade0
"
,
"sha256:
35d07389efaf3c478d93725a226941c7fc14714814ba77d6d43b2c9e63ef4af5
"
,
"sha256:3
ea6cc86931f57f18b1240572216f09922d91b19ab8a01cf24734394a3db3bec
"
,
"sha256:
46b0a146e4ba744e350847244767ef297950e9ce02424734b2dd0befd77d9aff
"
,
"sha256:
66c1a49b47c0953dbc6864a6d2578c4c24610f6bb8e4ab165d49b8371aa7745f
"
,
"sha256:
6d5c2d2a3e42100700bac7fe762c17ba0a04d0355feac04bce74a1aa6c8be164
"
,
"sha256:
ab1aa2c50b7c6ba0eccebb146b4d80ed7f5804897b8d54ccddbe49f28c881a94
"
,
"sha256:
ae1ec10e34d22b0f699e38f346381630cae89d5050a2a61315a2be09e3435f99
"
,
"sha256:
b578df33338a09707bfe3e3939c9d46700948133bf829357c3c46795055c9376
"
,
"sha256:
bad77cf498362590ef3a30bc9e769f4fe4399d853861a1ddbefeea8cbf39906c
"
,
"sha256:
c36e4d44d34eaa503776a8fb57ba1305e680e178458c050c2fd8de67604fa209
"
,
"sha256:
d76a8ec22adf0323d362dac8c900b2c66e06eab984ecf04ef072866d8ab6c5
38"
,
"sha256:
e8be4f6da608930c0d565240bfbe04fc6f5764d6a9214b02c6231cd5e223591d
"
,
"sha256:
f66c63f357ac31c913f4917f55348ce99c639031567c3284f01dff605da58264
"
"sha256:
23e177d43e4bf68950b0f8788b6a2fef2f478f4ec94883acb627b9264522a98a
"
,
"sha256:
2530aea4fe46e8df7829c3f05e0a0f821c893885d53cb8ac9b89cc67c143448c
"
,
"sha256:3
03827f0bb40ff610fbada5b12d50014811efcc37aaf6ef03202dc3054bfdda1
"
,
"sha256:
3b019e3ea9f5d0cfee0efabae2cfd3976874e90bcc3e97b29600e5a9b345ae3d
"
,
"sha256:
3c07765308f091d81b6735d4f2242bb43c332cc3461cae60543df6b10967fe27
"
,
"sha256:
5036d4009012a44aa3e50173e482b664c1fae36decd277c49e453463798eca4e
"
,
"sha256:
6f38969e2325056f9959efbe06c27aa2e94dd35382265ad0703681d993036052
"
,
"sha256:
74a470d349d52b9d00a2ba192ae1ee22155bb0a300fd1ccb2961006c3fa98ed3
"
,
"sha256:
7d77034e402165b947f43050a8a415aa3205abfed38d127ea66e57a2b7b5a9e0
"
,
"sha256:
7f9a509f6f11fa8b9313002ebdf6f690a7aa1dd91efd95d90185371a0d68220e
"
,
"sha256:
942b5d04762feb0e55b2ad97ce2b254a0ffdd344b56493b04a627266e24f2d82
"
,
"sha256:
a9fbe41663416bb70ed05f4e16c5f377519c0dc292ba9aa45f5356e37df03a
38"
,
"sha256:
d10e83866b48c0cdb83281f786564e2a2b51a7ae7b8a950c3442ad3c9e36b48c
"
,
"sha256:
e2140e1bbf9c46db9936ee70f4be6584d15ff8dc3dfff1da022d71227d53bad3
"
],
"index"
:
"pypi"
,
"version"
:
"==1.0.
0
"
"version"
:
"==1.0.
1
"
},
"psycopg2"
:
{
"hashes"
:
[
...
...
@@ -369,6 +406,14 @@
"index"
:
"pypi"
,
"version"
:
"==2019.3"
},
"redis"
:
{
"hashes"
:
[
"sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f"
,
"sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833"
],
"index"
:
"pypi"
,
"version"
:
"==3.4.1"
},
"requests"
:
{
"hashes"
:
[
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4"
,
...
...
@@ -446,6 +491,20 @@
],
"index"
:
"pypi"
,
"version"
:
"==1.25.8"
},
"vine"
:
{
"hashes"
:
[
"sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87"
,
"sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
],
"version"
:
"==1.3.0"
},
"zipp"
:
{
"hashes"
:
[
"sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50"
,
"sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e"
],
"version"
:
"==2.2.0"
}
},
"develop"
:
{
...
...
@@ -454,7 +513,7 @@
"sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0"
,
"sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"
],
"markers"
:
"
sys_
platform == '
d
arwin'"
,
"markers"
:
"platform
_system
== '
D
arwin'"
,
"version"
:
"==0.1.0"
},
"astroid"
:
{
...
...
@@ -589,7 +648,7 @@
"sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302"
,
"sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
],
"
index"
:
"pypi
"
,
"
markers"
:
"python_version < '3.8'
"
,
"version"
:
"==1.5.0"
},
"inflection"
:
{
...
...
@@ -680,10 +739,10 @@
},
"jupyter-core"
:
{
"hashes"
:
[
"sha256:
464769f7387d7a62a2403d067f1ddc616655b7f77f5d810c0dd62cb54bfd0fb9
"
,
"sha256:
a183e0ec2e8f6adddf62b0a3fc6a2237e3e0056d381e536d3e7c7ecc3067e244
"
"sha256:
185dfe42800585ca860aa47b5fe0211ee2c33246576d2d664b0b0b8d22aacf3a
"
,
"sha256:
e91785b8bd7f752711c0c20e5ec6ba0d42178d6321a61396082c55818991caee
"
],
"version"
:
"==4.6.
1
"
"version"
:
"==4.6.
2
"
},
"kiwisolver"
:
{
"hashes"
:
[
...
...
@@ -809,11 +868,11 @@
},
"mock"
:
{
"hashes"
:
[
"sha256:
56358390ebde40c927ec5666626be7d4310a2533ae3aed5c2dd7b55b80687f48
"
,
"sha256:
8fff3fd7c5796ea78ae2847f32e87ad4e111e03fef6e90d03b5efb4882211d78
"
"sha256:
2a572b715f09dd2f0a583d8aeb5bb67d7ed7a8fd31d193cf1227a99c16a67bc3
"
,
"sha256:
5e48d216809f6f393987ed56920305d8f3c647e6ed35407c1ff2ecb88a9e1151
"
],
"index"
:
"pypi"
,
"version"
:
"==4.0.
0
"
"version"
:
"==4.0.
1
"
},
"more-itertools"
:
{
"hashes"
:
[
...
...
@@ -1160,11 +1219,10 @@
},
"zipp"
:
{
"hashes"
:
[
"sha256:
ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28
"
,
"sha256:
feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98
"
"sha256:
5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50
"
,
"sha256:
d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e
"
],
"index"
:
"pypi"
,
"version"
:
"==2.1.0"
"version"
:
"==2.2.0"
}
}
}
backend/metagenedb/api/catalog/urls.py
View file @
aaf9cf48
...
...
@@ -4,6 +4,8 @@ from rest_framework.routers import DefaultRouter, DynamicRoute, Route
from
metagenedb.api.catalog
import
views
from
metagenedb.api.catalog.views.celery_test
import
celery_test_view
class
CustomRouter
(
DefaultRouter
):
...
...
@@ -63,5 +65,6 @@ api_router.register(r'statistics', views.StatisticsViewSet, basename='statistics
urlpatterns
=
[
url
(
r
'v1/'
,
include
((
api_router
.
urls
,
'v1'
))),
path
(
'admin/'
,
include
((
'metagenedb.api.catalog.admin_urls'
,
'admin'
)))
path
(
'admin/'
,
include
((
'metagenedb.api.catalog.admin_urls'
,
'admin'
))),
path
(
'celery-test/'
,
celery_test_view
,
name
=
'celery-test'
)
]
backend/metagenedb/api/catalog/views/celery_test.py
0 → 100644
View file @
aaf9cf48
from
metagenedb.celery_app
import
debug_task
from
rest_framework.decorators
import
api_view
from
rest_framework.response
import
Response
@
api_view
()
def
celery_test_view
(
request
):
debug_task
.
delay
()
return
Response
({
"message"
:
"Your task is being processed!"
})
backend/metagenedb/apps/core/__init__.py
0 → 100644
View file @
aaf9cf48
backend/metagenedb/apps/core/management/__init__.py
0 → 100644
View file @
aaf9cf48
backend/metagenedb/apps/core/management/commands/__init__.py
0 → 100644
View file @
aaf9cf48
backend/metagenedb/apps/core/management/commands/watch_celery.py
0 → 100644
View file @
aaf9cf48
"""
This command allows for celery to be reloaded when project
code is saved. This command is called in
`docker-compose.dev.yml` and is only for use in development
https://avilpage.com/2017/05/how-to-auto-reload-celery-workers-in-development.html
"""
import
shlex
import
subprocess
from
django.core.management.base
import
BaseCommand
from
django.utils
import
autoreload
def
restart_celery
():
cmd
=
'pkill -9 celery'
subprocess
.
call
(
shlex
.
split
(
cmd
))
cmd
=
'celery worker --app=metagenedb.celery_app:app --loglevel=info'
subprocess
.
call
(
shlex
.
split
(
cmd
))
class
Command
(
BaseCommand
):
def
handle
(
self
,
*
args
,
**
options
):
print
(
'Starting celery worker with autoreload...'
)
autoreload
.
run_with_reloader
(
restart_celery
)
backend/metagenedb/celery_app.py
0 → 100644
View file @
aaf9cf48
import
os
from
celery
import
Celery
import
time
os
.
environ
.
setdefault
(
'DJANGO_SETTINGS_MODULE'
,
'metagenedb.settings'
)
app
=
Celery
(
'metagenedb'
)
app
.
config_from_object
(
'django.conf:settings'
,
namespace
=
'CELERY'
)
app
.
autodiscover_tasks
()
@
app
.
task
(
bind
=
True
)
def
debug_task
(
self
):
print
(
"Doing async task"
)
time
.
sleep
(
2
)
print
(
"Task is done"
)
backend/metagenedb/settings/__init__.py
View file @
aaf9cf48
from
.django
import
*
# noqa
from
.celery
import
*
# noqa
backend/metagenedb/settings/celery.py
0 → 100644
View file @
aaf9cf48
# Celery Configuration
CELERY_BROKER_URL
=
'redis://redis:6379'
CELERY_RESULT_BACKEND
=
'redis://redis:6379'
CELERY_ACCEPT_CONTENT
=
[
'application/json'
]
CELERY_TASK_SERIALIZER
=
'json'
CELERY_RESULT_SERIALIZER
=
'json'
backend/metagenedb/settings/django.py
View file @
aaf9cf48
...
...
@@ -10,6 +10,7 @@ environ.Env.read_env(root('.env')) # reading .env file
INSTALLED_APPS
=
[
'metagenedb.apps.catalog'
,
'metagenedb.apps.core'
,
'django.contrib.admin'
,
'django.contrib.auth'
,
'django.contrib.contenttypes'
,
...
...
docker-compose.yaml
View file @
aaf9cf48
...
...
@@ -59,6 +59,40 @@ services:
networks
:
-
main
redis
:
image
:
redis:alpine
container_name
:
redis
networks
:
-
main
celery
:
container_name
:
celery
build
:
context
:
./backend
volumes
:
-
./backend:/code
depends_on
:
-
postgresql
-
redis
command
:
bash -c 'manage.py watch_celery'
networks
:
-
main
flower
:
image
:
mher/flower
container_name
:
flower_dev_vet
command
:
flower --url_prefix=flower
environment
:
-
CELERY_BROKER_URL=redis://redis:6379
-
FLOWER_PORT=5555
ports
:
-
5555:5555
networks
:
-
main
depends_on
:
-
celery
-
redis
volumes
:
postgresql-data
:
...
...
nginx/dev/nginx.conf
View file @
aaf9cf48
...
...
@@ -17,6 +17,10 @@ http {
server
frontend
:
8080
;
}
upstream
flower
{
server
flower
:
5555
;
}
server
{
listen
80
;
charset
utf-8
;
...
...
@@ -48,5 +52,16 @@ http {
proxy_set_header
Host
$http_host
;
}
# flower
location
/flower/
{
rewrite
^/flower/(.*)
$
/
$1
break
;
proxy_pass
http://flower/
;
proxy_set_header
Host
$host
;
proxy_redirect
off
;
proxy_http_version
1
.1
;
proxy_set_header
Upgrade
$http_upgrade
;
proxy_set_header
Connection
"upgrade"
;
}
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment