Commit 2b60f851 authored by Bryan  BRANCOTTE's avatar Bryan BRANCOTTE

Merge branch 'xsendfile_in_place_of_media_url' into 'master'

Use /code/persistent_volume and XSendFile in place of MEDIA_ROOT

See merge request !61
parents f591d19c a47e7c80
......@@ -18,7 +18,6 @@ ErrorLog "|/usr/bin/rotatelogs /code/persistent_volume/apache.%Y-%m-%d-%H_%M_%S.
WSGIProcessGroup ${PROJECT_NAME}
Alias ${STATIC_URL}/ /code/.static/
Alias ${MEDIA_URL}/ /code/.media/
Alias /favicon.ico /code/.static/favicon.ico
Alias /robots.txt /code/.static/robots.txt
Redirect permanent "/apple-touch-icon-precomposed.png" "${STATIC_URL}/favicon-256.png"
......@@ -35,12 +34,8 @@ ErrorLog "|/usr/bin/rotatelogs /code/persistent_volume/apache.%Y-%m-%d-%H_%M_%S.
Require all granted
</Directory>
<Directory /code/.media>
Require all granted
</Directory>
XSendFile On
XSendFilePath /code/persistent_volume/data_source_backup/
XSendFilePath /code/persistent_volume/
<Location />
ExpiresActive On
......
......@@ -9,10 +9,6 @@ if [ "$1" == "test" ]; then
pip install --no-cache-dir -r requirements-dev.txt
cp viralhostrange/settings.example.ini viralhostrange/settings.ini || exit 2
MEDIA_ROOT_DIR=$(python manage.py shell -c "from django.conf import settings; print(settings.MEDIA_ROOT)" | grep -v django.db.backends)
mkdir -p $MEDIA_ROOT_DIR
chmod 777 $MEDIA_ROOT_DIR
python manage.py collectstatic --noinput
python manage.py compilemessages || exit 6
coverage run --source='.' manage.py test || exit 3
......@@ -83,15 +79,6 @@ python manage.py migrate || exit 4
# msg_warning "csscompressor missing, passed"
#fi
MEDIA_ROOT_DIR=$(python manage.py shell -c "from django.conf import settings; print(settings.MEDIA_ROOT)" | grep -v django.db.backends)
msg_info "Creating media root at $MEDIA_ROOT_DIR"
if [ "$MEDIA_ROOT_DIR" != "" ]; then
mkdir -p $MEDIA_ROOT_DIR
chmod 777 $MEDIA_ROOT_DIR
else
msg_warning "settings.MEDIA_ROOT missing, passed"
fi
#sudo /usr/sbin/service cron start
#msg_info "Adding cron task to dump db every day"
#cat <(crontab -l) <(echo "") <(echo "* * * * * find $MEDIA_ROOT_DIR/ -type f -name 'Template*.xlsx' -mtime +1 -exec rm {} \; >> /code/persistent_volume/django-crontab.log 2>> /code/persistent_volume/django-crontab.err") | crontab -
......
......@@ -161,7 +161,7 @@ USE_TZ = True
STATIC_ROOT = os.path.join(BASE_DIR, '.static')
STATIC_URL = config('STATIC_URL', default='/static') + '/'
MEDIA_ROOT = os.path.join(BASE_DIR, '.media')
MEDIA_ROOT = os.path.join(BASE_DIR, 'persistent_volume', '.media')
MEDIA_URL = config('MEDIA_URL', default='/media') + '/'
################################################################################
......
......@@ -41,5 +41,5 @@ if not settings.DEBUG:
url(r'^static/(?P<path>.*)$', serve, {'document_root': settings.STATIC_ROOT})
)
urlpatterns.append(
url(r'^media/(?P<path>.*)$', serve, {'document_root': settings.MEDIA_ROOT})
url(r'^media/(?P<path>.*)$', views.xsendfile_serve)
)
......@@ -699,6 +699,18 @@ def get_backup_file_path(log_entry, test_if_exists=False):
return backup_file_path
def get_user_media_root(user):
if user is None or not user.is_authenticated:
file_path = os.path.join(settings.MEDIA_ROOT, "anonymous")
else:
file_path = os.path.join(settings.MEDIA_ROOT, str(user.pk))
try:
os.makedirs(file_path)
except OSError as e:
assert e.errno == errno.EEXIST, e
return file_path
class DataSourceAlteredData(Enum):
UNKNOWN = 0
ALL = 0
......
......@@ -1221,7 +1221,8 @@ class LiveInputVirusHostForm(forms.Form):
widget=forms.Select,
)
def __init__(self, data=None, *args, **kwargs):
def __init__(self, data=None, user=None, *args, **kwargs):
self.user = user
super().__init__(data=data, *args, **kwargs)
self.fields["virus"].widget.attrs["placeholder"] = mark_safe(_("virus_placeholder_in_form"))
self.fields["host"].widget.attrs["placeholder"] = _("host_placeholder_in_form")
......@@ -1310,6 +1311,8 @@ class LiveInputVirusHostForm(forms.Form):
# self.add_error(key, "eee")
def get_template_files(self):
media_root = business_process.get_user_media_root(self.user)
col_header = [explicit_item(n, **ids) for n, ids in self.get_as_list('host')]
row_header = [explicit_item(n, **ids) for n, ids in self.get_as_list('virus')]
......@@ -1329,7 +1332,7 @@ class LiveInputVirusHostForm(forms.Form):
time=timezone.now().strftime('%Hh%Mm%Ss'),
seed="".join([random.choice(string.ascii_letters) for i in range(4)])
)
file_path = os.path.join(settings.MEDIA_ROOT, filename + ".xlsx")
file_path = os.path.join(media_root, filename + ".xlsx")
with pd.ExcelWriter(file_path) as writer:
df_data = pd.DataFrame(
data=[["<response>"] * len(col_header)] * len(row_header),
......@@ -1350,7 +1353,7 @@ class LiveInputVirusHostForm(forms.Form):
data = [data] * 4 + [['...'] * len(data), ]
else:
data = [data, ] * len(row_header)
file_path = os.path.join(settings.MEDIA_ROOT, filename + ".sample.xlsx")
file_path = os.path.join(media_root, filename + ".sample.xlsx")
with pd.ExcelWriter(file_path) as writer:
df_data = pd.DataFrame(
data=data,
......@@ -1361,7 +1364,7 @@ class LiveInputVirusHostForm(forms.Form):
os.chmod(file_path, 0o777)
return dict(
file_url=settings.MEDIA_URL + filename + ".xlsx",
sample_path=os.path.join(settings.MEDIA_ROOT, filename + ".sample.xlsx"),
sample_path=os.path.join(media_root, filename + ".sample.xlsx"),
)
......
......@@ -3,6 +3,7 @@ import logging
import os
from django.contrib.auth import get_user_model
from django.contrib.auth.models import AnonymousUser
from django.db.models import Q
from django.test import TestCase
from django.utils import timezone
......@@ -23,6 +24,18 @@ class DataFileReaderTestCase(TestCase):
username="root",
)
def test_get_media_root(self):
user = get_user_model().objects.create(
username="user",
)
paths = set()
users = [self.owner, user, AnonymousUser()]
for u in users:
paths.add(business_process.get_user_media_root(u))
self.assertEqual(len(paths), len(users))
paths.add(business_process.get_user_media_root(None))
self.assertEqual(len(paths), len(users))
def listdir(self):
for f in os.listdir(self.test_data):
if f.endswith(".xlsx") and not f.endswith(".err.xlsx"):
......
......@@ -491,7 +491,7 @@ class DataSourceCreateViewTestCase(ViewTestCase):
self.assertEqual(response_3.status_code, 302)
response_3 = self.client.post(target_3, form_data_3, follow=True)
old_in_media = set(os.listdir(settings.MEDIA_ROOT))
old_in_media = set(os.listdir(business_process.get_user_media_root(self.user)))
target_4 = response_3.redirect_chain[0][0]
step_name_4 = target_4.split("/")[-2]
......@@ -508,12 +508,12 @@ class DataSourceCreateViewTestCase(ViewTestCase):
self.assertEqual(response_4.status_code, 302)
response_4 = self.client.post(target_4, form_data_4, follow=True)
new_in_media = set(os.listdir(settings.MEDIA_ROOT)) - old_in_media
new_in_media = set(os.listdir(business_process.get_user_media_root(self.user))) - old_in_media
self.assertEqual(len(new_in_media), 2)
complet_file = new_in_media.pop()
if complet_file.endswith(".sample.xlsx"):
complet_file = new_in_media.pop()
vhrs = business_process.parse_file(os.path.join(settings.MEDIA_ROOT, complet_file))
vhrs = business_process.parse_file(os.path.join(business_process.get_user_media_root(self.user), complet_file))
new_viruses = set()
new_hosts = set()
for vhr in vhrs:
......@@ -751,7 +751,7 @@ class DataSourceCreateViewTestCase(ViewTestCase):
self.assertEqual(response_3.status_code, 302)
response_3 = self.client.post(target_3, form_data_3, follow=True)
old_in_media = set(os.listdir(settings.MEDIA_ROOT))
old_in_media = set(os.listdir(business_process.get_user_media_root(self.user)))
target_4 = response_3.redirect_chain[0][0]
step_name_4 = target_4.split("/")[-2]
......@@ -768,12 +768,12 @@ class DataSourceCreateViewTestCase(ViewTestCase):
self.assertEqual(response_4.status_code, 302)
response_4 = self.client.post(target_4, form_data_4, follow=True)
new_in_media = set(os.listdir(settings.MEDIA_ROOT)) - old_in_media
new_in_media = set(os.listdir(business_process.get_user_media_root(self.user))) - old_in_media
self.assertEqual(len(new_in_media), 2)
complet_file = new_in_media.pop()
if complet_file.endswith(".sample.xlsx"):
complet_file = new_in_media.pop()
vhrs = business_process.parse_file(os.path.join(settings.MEDIA_ROOT, complet_file))
vhrs = business_process.parse_file(os.path.join(business_process.get_user_media_root(self.user), complet_file))
new_viruses = set()
new_hosts = set()
for vhr in vhrs:
......@@ -919,7 +919,7 @@ class DataSourceCreateViewTestCase(ViewTestCase):
self.assertEqual(response_3.status_code, 302)
response_3 = self.client.post(target_3, form_data_3, follow=True)
old_in_media = set(os.listdir(settings.MEDIA_ROOT))
old_in_media = set(os.listdir(business_process.get_user_media_root(self.user)))
target_4 = response_3.redirect_chain[0][0]
step_name_4 = target_4.split("/")[-2]
......@@ -946,12 +946,12 @@ h3
self.assertEqual(response_4.status_code, 302)
response_4 = self.client.post(target_4, form_data_4, follow=True)
new_in_media = set(os.listdir(settings.MEDIA_ROOT)) - old_in_media
new_in_media = set(os.listdir(business_process.get_user_media_root(self.user))) - old_in_media
self.assertEqual(len(new_in_media), 2)
complet_file = new_in_media.pop()
if complet_file.endswith(".sample.xlsx"):
complet_file = new_in_media.pop()
vhrs = business_process.parse_file(os.path.join(settings.MEDIA_ROOT, complet_file))
vhrs = business_process.parse_file(os.path.join(business_process.get_user_media_root(self.user), complet_file))
new_viruses = set()
new_hosts = set()
for vhr in vhrs:
......
import itertools
import logging
import os
import shutil
from tempfile import NamedTemporaryFile
from urllib import parse
import pandas as pd
from basetheme_bootstrap.user_preferences import get_user_preferences_for_user
from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.auth.models import AnonymousUser
from django.core.files.uploadedfile import SimpleUploadedFile
from django.db.models import Q, F
from django.http import QueryDict
from django.test import TestCase
from django.http import QueryDict, Http404, HttpResponseNotModified
from django.test import TestCase, RequestFactory, override_settings
from django.urls import reverse
from django.utils import timezone
from live_settings import live_settings
from viralhostrangedb import models, business_process
from viralhostrangedb import models, business_process, views
logger = logging.getLogger(__name__)
......@@ -1531,3 +1534,102 @@ class ProbeTestCase(TestCase):
def test_is_pk(self):
self.assertEqual(200, self.client.get(reverse('viralhostrangedb:probe_ready')).status_code)
self.assertEqual(200, self.client.get(reverse('viralhostrangedb:probe_alive')).status_code)
@override_settings(MEDIA_ROOT=os.path.join(
settings.BASE_DIR,
'persistent_volume',
'.media_for_test',
))
class XSendFileMediaViewTestCase(TestCase):
def tearDown(self) -> None:
if '.media_for_test' in settings.MEDIA_ROOT:
shutil.rmtree(settings.MEDIA_ROOT)
def setUp(self) -> None:
self.owner = get_user_model().objects.create(
username="root",
)
self.user = get_user_model().objects.create(
username="user",
)
self.rf = RequestFactory()
def build_request(self, user, path, **extra):
request = self.rf.get(path=settings.MEDIA_URL + '/' + path, **extra)
request.user = user
return request
def test_toto(self):
path = "my_file.txt"
# request = RequestFactory().get(settings.MEDIA_URL + '/' + path, user=self.user)
# request.user = self.user
# check that missing files ends up in 404
self.assertRaises(
Http404, views.xsendfile_serve,
request=self.build_request(self.user, "my_file.txt"),
path='my_file.txt',
)
# check that dirs cannot be listed
self.assertRaises(
Http404, views.xsendfile_serve,
request=self.build_request(self.user, ""),
path='',
)
# check that sub dirs cannot be listed
self.assertRaises(
Http404, views.xsendfile_serve,
request=self.build_request(self.user, "toto/"),
path='toto/',
)
os.makedirs('/'.join([settings.MEDIA_ROOT, str(self.user.pk), 'toto']))
self.assertRaises(
Http404, views.xsendfile_serve,
request=self.build_request(self.user, "toto/"),
path='toto/',
)
# response = views.xsendfile_serve(request=request, path=path)
# response = views.xsendfile_serve(request=request, path=path)
with open(business_process.get_user_media_root(self.user) + '/' + path + ".old", 'w') as f:
f.write("rr")
for u in [self.user, self.owner, AnonymousUser()]:
with open(business_process.get_user_media_root(u) + '/' + path, 'w') as f:
f.write(";".join([u.username, str(type(u).__name__)]))
# ensure that "my_file.txt" is different for each user
for u in [self.user, self.owner, AnonymousUser()]:
# test that path returned is the file of the current user u, and not anyone else
response = views.xsendfile_serve(request=self.build_request(u, path), path=path)
with open(response['X-Sendfile'], 'r') as f:
self.assertEqual(f.readline(), ";".join([u.username, str(type(u).__name__)]))
# test that my_file.txt.old is not visible to other user
if u is not self.user:
self.assertRaises(
Http404, views.xsendfile_serve,
request=self.build_request(u, path + ".old"),
path=path + ".old",
)
# put file change in the past
ts = (timezone.now() - timezone.timedelta(weeks=4)).timestamp()
os.utime(business_process.get_user_media_root(self.user) + '/' + path, (ts, ts))
# access it
response = views.xsendfile_serve(request=self.build_request(self.user, path), path=path)
# access it agai with it lst modification time
request = self.build_request(self.user, path, HTTP_IF_MODIFIED_SINCE=response["Last-Modified"])
response = views.xsendfile_serve(request=request, path=path)
# it should return a "not modified" response
self.assertIsInstance(response, HttpResponseNotModified)
# edit the file
with open(business_process.get_user_media_root(self.user) + '/' + path, 'w') as f:
f.write("changed !")
response = views.xsendfile_serve(request=request, path=path)
# it should NOT return a "not modified" response anymore
self.assertNotIsInstance(response, HttpResponseNotModified)
import itertools
import json
import logging
import mimetypes
import os
import pathlib
import posixpath
import re
import tempfile
......@@ -23,17 +26,19 @@ from django.core.mail import EmailMultiAlternatives
from django.core.signing import SignatureExpired, BadSignature
from django.db import transaction
from django.db.models import Q, Count, When, IntegerField, Case, Value
from django.http import HttpResponseRedirect, HttpResponse, HttpResponseBadRequest, Http404
from django.http import HttpResponseRedirect, HttpResponse, HttpResponseBadRequest, Http404, HttpResponseNotModified
from django.shortcuts import render, get_object_or_404, redirect
from django.urls import reverse, reverse_lazy
from django.utils import timezone
from django.utils.datastructures import MultiValueDictKeyError
from django.utils.decorators import method_decorator
from django.utils.html import strip_tags, escape
from django.utils.http import http_date
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _, ugettext
from django.views.decorators.cache import cache_page
from django.views.generic import ListView, DetailView, UpdateView, DeleteView
from django.views.static import was_modified_since
from formtools.wizard import views as wizard_views
from live_settings import live_settings
......@@ -1573,6 +1578,8 @@ class DataSourceWizard(wizard_views.NamedUrlSessionWizardView):
kwargs = super().get_form_kwargs(step=step)
if step == "Information":
kwargs.update({"owner": self.request.user})
elif step == "LiveInput":
kwargs.update({"user": self.request.user})
elif step == "Description":
kwargs.update({"required": self.storage.get_step_data("Information").get('Information-public', False)})
if self.storage.get_step_data("Information").get('Information-public', False):
......@@ -1874,3 +1881,33 @@ def is_alive(request):
except OperationalError:
pass
return HttpResponse(status=500)
def xsendfile_serve(request, path):
document_root = business_process.get_user_media_root(request.user)
# From django.view.static.serve, if evolution is needed please see serve function first as most of the following is
# copy pasted
path = posixpath.normpath(path).lstrip('/')
fullpath = pathlib.Path(os.path.join(document_root, path))
if fullpath.is_dir():
# Not allowed at all
raise Http404(_("Directory indexes are not allowed here."))
if not fullpath.exists():
raise Http404(_('"%(path)s" does not exist') % {'path': fullpath})
# Respect the If-Modified-Since header.
statobj = fullpath.stat()
if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'),
statobj.st_mtime, statobj.st_size):
return HttpResponseNotModified()
response = HttpResponse()
content_type, encoding = mimetypes.guess_type(str(fullpath))
response['Content-Type'] = content_type or 'application/octet-stream'
# content_type = content_type or 'application/octet-stream'
# response = FileResponse(fullpath.open('rb'), content_type=content_type)
# The only difference from django.view.static.serve
response['X-Sendfile'] = str(fullpath).encode('utf-8')
response["Last-Modified"] = http_date(statobj.st_mtime)
if encoding:
response["Content-Encoding"] = encoding
return response
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment