Adding export function on virus and host detail page, closing #78

src/viralhostrange/locale/en/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2019-09-30 17:21+0200\n"
+"POT-Creation-Date: 2019-10-01 15:29+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -961,6 +961,15 @@ msgstr ""
 msgid "ViralHostRange browsed data on "
 msgstr ""
 
+#, python-format
+msgid "%(kind)s \"%(name)s\""
+msgstr ""
+
+#. Translators: it is the beginning of a file name
+#, python-format
+msgid "ViralHostRange data of %(host_or_virus_name)s on %(date)s"
+msgstr ""
+
 msgid "Next"
 msgstr ""
 

src/viralhostrange/test_data/download_responses_host_virus/C1_infection_ratio.csv

+Unnamed: 0,"Host ""C1""",Unnamed: 2,Unnamed: 3,Unnamed: 4,Unnamed: 5,Unnamed: 6
+,,,,,,
+,Infection ratio,four_reponse_simple mapped,private user mapped,public user mapped,three_reponse_simple mapped,three_reponse_simple_2 mapped
+Infection ratio,,0,0,0,0,0
+r1,0,0,1,0,0,1
+r2,0,0,0,0,0,0
+r3,0,0,0,0,0,0
+,,,,,,
+,,,,,,
+,Legend,,,,,
+0,No lysis,,,,,
+1,Weak,,,,,
+2,Lysis,,,,,

src/viralhostrange/test_data/download_responses_host_virus/C1_infection_ratio_weak.csv

+Unnamed: 0,"Host ""C1""",Unnamed: 2,Unnamed: 3,Unnamed: 4,Unnamed: 5,Unnamed: 6
+,,,,,,
+,Infection ratio,four_reponse_simple mapped,private user mapped,public user mapped,three_reponse_simple mapped,three_reponse_simple_2 mapped
+Infection ratio,,0,33.33333333333334,0,0,33.33333333333334
+r1,40,0,1,0,0,1
+r2,0,0,0,0,0,0
+r3,0,0,0,0,0,0
+,,,,,,
+,,,,,,
+,Legend,,,,,
+0,No lysis,,,,,
+1,Weak,,,,,
+2,Lysis,,,,,
+,,,,,,
+,,,,,,
+Infection ratio settings,,,,,,
+Consider all positive response as a infection,,,,,,

src/viralhostrange/test_data/download_responses_host_virus/C1_toto.csv

+Unnamed: 0,"Host ""C1""",Unnamed: 2,Unnamed: 3,Unnamed: 4
+,,,,
+,four_reponse_simple mapped,public user mapped,three_reponse_simple mapped,three_reponse_simple_2 mapped
+r1,0,0,0,1
+r2,0,0,0,0
+r3,0,0,0,0
+,,,,
+,,,,
+,Legend,,,
+0,No lysis,,,
+1,Weak,,,
+2,Lysis,,,

src/viralhostrange/test_data/download_responses_host_virus/C1_user.csv

+Unnamed: 0,"Host ""C1""",Unnamed: 2,Unnamed: 3,Unnamed: 4,Unnamed: 5
+,,,,,
+,four_reponse_simple mapped,private user mapped,public user mapped,three_reponse_simple mapped,three_reponse_simple_2 mapped
+r1,0,1,0,0,1
+r2,0,0,0,0,0
+r3,0,0,0,0,0
+,,,,,
+,,,,,
+,Legend,,,,
+0,No lysis,,,,
+1,Weak,,,,
+2,Lysis,,,,

src/viralhostrange/test_data/download_responses_host_virus/C4_infection_ratio.csv

+Unnamed: 0,"Host ""C4""",Unnamed: 2,Unnamed: 3,Unnamed: 4,Unnamed: 5,Unnamed: 6
+,,,,,,
+,Infection ratio,four_reponse_simple mapped,private user mapped,public user mapped,three_reponse_simple mapped,three_reponse_simple_2 mapped
+Infection ratio,,33.33333333333334,0,33.33333333333334,33.33333333333334,0
+r1,60,2,1,2,2,1
+r2,0,0,0,0,0,0
+r3,0,0,1,0,0,1
+,,,,,,
+,,,,,,
+,Legend,,,,,
+0,No lysis,,,,,
+1,Weak,,,,,
+2,Lysis,,,,,

src/viralhostrange/test_data/download_responses_host_virus/r1_user.csv

+Unnamed: 0,"Virus ""r1""",Unnamed: 2,Unnamed: 3,Unnamed: 4,Unnamed: 5,Unnamed: 6,Unnamed: 7,Unnamed: 8
+,,,,,,,,
+,four_reponse_simple mapped,four_reponse_simple not mapped,private user,private user mapped,public user,public user mapped,three_reponse_simple mapped,three_reponse_simple_2 mapped
+C1,0,,,1,,0,0,1
+C2,1,,,1,,0,1,1
+C3,0,,,0,,0,0,0
+C4,2,,,1,,2,2,1
+,,,,,,,,
+,,,,,,,,
+,Legend,,,,,,,
+0,No lysis,,,,,,,
+1,Weak,,,,,,,
+2,Lysis,,,,,,,

src/viralhostrange/viralhostrangedb/templates/viralhostrangedb/vh_detail.html

@@ -36,12 +36,12 @@ title="{% trans 'Open in the NCBI'%}">
                     <input type="text" value="Hello World" id="clipboard_input" style="display:none;"/>
                     {%trans "Copy link into clipboard" %}<!-- {%trans "(recommended)" %}-->
                 </button>
-<!--                <button class="dropdown-item" type="button"-->
-<!--                        onclick="$('form').submit()"-->
-<!--                >-->
-<!--                    <i class="fa fa-download"></i>-->
-<!--                    {%trans "Download displayed data in a spreadsheet" %}-->
-<!--                </button>-->
+                <button class="dropdown-item" type="button"
+                        onclick="$('#form_filter').submit()"
+                >
+                    <i class="fa fa-download"></i>
+                    {%trans "Download displayed data in a spreadsheet" %}
+                </button>
             </div>
         </div>
     </div>
@@ -76,7 +76,7 @@ var detailed_kind="{{detailed_kind}}";
 {% endblock %}
 
 {% block content%}
-<form method="GET" id="form_filter" class="mb-4 col-12" action="/404.html">{% csrf_token %}
+<form method="GET" id="form_filter" class="mb-4 col-12" action="{{download_url}}">{% csrf_token %}
     <div class="row">
         <div class="col-12 text-right">
             <a href="#advanced_holder" class="btn btn-outline-secondary btn-default float-left collapsed" data-toggle="collapse">

src/viralhostrange/viralhostrangedb/tests/test_views.py

@@ -333,7 +333,7 @@ class ViewTestCase(TestCase):
         self.four_reponse_simple_not_mapped = models.DataSource.objects.create(
             owner=self.user,
             public=True,
-            name="four_reponse_simple mapped",
+            name="four_reponse_simple not mapped",
             raw_name="ee",
             kind="FILE",
         )
@@ -2064,7 +2064,7 @@ class HostDetailViewTestCase(ViewTestCase):
         self.assertEqual(response.status_code, 200)
 
 
-class DownloadBrowsedDataTestCase(ViewTestCase):
+class DownloadXXXDataTestCase(ViewTestCase):
 
     def actual_test(
             self,
@@ -2075,10 +2075,12 @@ class DownloadBrowsedDataTestCase(ViewTestCase):
             host=None,
             virus_infection_ratio=False,
             host_infection_ratio=False,
+            infection_ratio=False,
             weak_infection=False,
             agreed_infection=False,
+            url=None,
     ):
-        url = reverse('viralhostrangedb:download_responses') + "?"
+        url = (url or self.url) + "?"
         if ds:
             url += "&".join(["ds=%i" % pk for pk in ds.values_list('pk', flat=True)])
         if virus:
@@ -2089,10 +2091,13 @@ class DownloadBrowsedDataTestCase(ViewTestCase):
             url += "&virus_infection_ratio=true"
         if host_infection_ratio:
             url += "&host_infection_ratio=true"
+        if infection_ratio:
+            url += "&infection_ratio=true"
         if weak_infection:
             url += "&weak_infection=true"
         if agreed_infection:
             url += "&agreed_infection=true"
+        self.client.logout()
         if user:
             self.client.force_login(user)
         response = self.client.get(url)
@@ -2105,7 +2110,7 @@ class DownloadBrowsedDataTestCase(ViewTestCase):
                     self.assertTrue(response.status_code == 200)
                     data_xls = pd.read_excel(f.name, index_col=None)
                     data_xls.to_csv(fc.name, encoding='utf-8', index=False)
-                    filename = os.path.join(self.test_data, "download_responses", filename)
+                    filename = os.path.join(self.test_data, self.file_location, filename)
                     with open(filename, 'r') as expected:
                         computed_content = fc.read().decode("UTF-8")
                         expected_content = expected.read()
@@ -2125,8 +2130,20 @@ class DownloadBrowsedDataTestCase(ViewTestCase):
         except Exception as e:
             print(response.status_code)
             print(url)
+            try:
+                with NamedTemporaryFile(suffix=".xlsx", delete=False) as f:
+                    f.write(response.content)
+                    f.close()
+                print(f.name)
+            except Exception:
+                pass
             raise e
 
+
+class DownloadBrowsedDataTestCase(DownloadXXXDataTestCase):
+    url = reverse('viralhostrangedb:download_responses')
+    file_location = "download_responses"
+
     def test_private_protected_works(self):
         self.actual_test(
             ds=models.DataSource.objects.filter(pk=self.private_data_source_of_user_mapped.pk),
@@ -2268,6 +2285,89 @@ class DownloadBrowsedDataTestCase(ViewTestCase):
         )
 
 
+class DownloadHostDataTestCase(DownloadXXXDataTestCase):
+    file_location = "download_responses_host_virus"
+
+    def setUp(self):
+        super().setUp()
+        self.four_reponse_simple_not_mapped.delete()
+        self.private_data_source_of_user.delete()
+        self.private_data_source_of_toto.delete()
+        self.public_data_source_of_user.delete()
+
+    def test_private_protected_works(self):
+        self.actual_test(
+            url=reverse('viralhostrangedb:host-download', args=[
+                self.private_data_source_of_toto_with_no_virus_or_host_shared.host_set.first().pk]),
+        )
+        self.actual_test(
+            user=self.user,
+            url=reverse('viralhostrangedb:host-download', args=[
+                self.private_data_source_of_toto_with_no_virus_or_host_shared.host_set.first().pk]),
+        )
+
+    def test_works_1(self):
+        self.actual_test(
+            url=reverse('viralhostrangedb:host-download', args=[self.C1.pk]),
+            user=self.user,
+            filename="C1_user.csv",
+        )
+
+    def test_works_2(self):
+        self.actual_test(
+            url=reverse('viralhostrangedb:host-download', args=[self.C1.pk]),
+            user=self.toto,
+            filename="C1_toto.csv",
+        )
+
+    def test_with_infection_ratio_where_only_weak(self):
+        self.actual_test(
+            url=reverse('viralhostrangedb:host-download', args=[self.C1.pk]),
+            user=self.user,
+            filename="C1_infection_ratio.csv",
+            infection_ratio=True,
+        )
+
+    def test_with_infection_ratio_with_lysis(self):
+        self.actual_test(
+            url=reverse('viralhostrangedb:host-download', args=[self.C4.pk]),
+            user=self.user,
+            filename="C4_infection_ratio.csv",
+            infection_ratio=True,
+        )
+
+    def test_with_infection_ratio_where_only_weak_and_weak_infection(self):
+        self.actual_test(
+            url=reverse('viralhostrangedb:host-download', args=[self.C1.pk]),
+            user=self.user,
+            filename="C1_infection_ratio_weak.csv",
+            infection_ratio=True,
+            weak_infection=True,
+        )
+
+
+class DownloadVirusDataTestCase(DownloadXXXDataTestCase):
+    file_location = "download_responses_host_virus"
+
+    def test_private_protected_works(self):
+        self.actual_test(
+            url=reverse('viralhostrangedb:virus-download', args=[
+                self.private_data_source_of_toto_with_no_virus_or_host_shared.virus_set.first().pk]),
+        )
+        self.actual_test(
+            user=self.user,
+            url=reverse('viralhostrangedb:virus-download', args=[
+                self.private_data_source_of_toto_with_no_virus_or_host_shared.virus_set.first().pk]),
+        )
+
+    def test_works_1(self):
+        self.actual_test(
+            url=reverse('viralhostrangedb:virus-download', args=[self.r1.pk]),
+            user=self.user,
+            filename="r1_user.csv",
+        )
+
+
 class ResponseUpdateTestCase(ViewTestCase):
 
     def test_works(self):

src/viralhostrange/viralhostrangedb/urls.py

@@ -23,7 +23,6 @@ urlpatterns = [
     url(r'^data-source/(?P<pk>\d+)/download/$', views.data_source_download, name='data-source-download'),
     url(r'^data-source/(?P<pk>\d+)/update/$', views.data_source_data_update, name='data-source-data-update'),
     url(r'^data-source/(?P<pk>\d+)/edit/$', views.DataSourceUpdateView.as_view(), name='data-source-update'),
-    # url(r'^data-source/create/$', views.DataSourceCreateView.as_view(), name='data-source-create'),
     url(r'^data-source/(?P<pk>\d+)/virus/update/$', views.data_source_virus_update, name='data-source-virus-update'),
     url(r'^data-source/(?P<pk>\d+)/host/update/$', views.data_source_host_update, name='data-source-host-update'),
     url(r'^data-source/(?P<pk>\d+)/virus/delete/$', views.data_source_virus_delete, name='data-source-virus-delete'),
@@ -40,10 +39,15 @@ urlpatterns = [
     url(r'^data-source-contribution/(?P<step>.+)/$', data_source_wizard, name='data-source-create-step'),
     url(r'^data-source-contribution/$', data_source_wizard, name='data-source-create'),
     url(r'^data-source/(?P<pk>\d+)/contact/$', views.contact_data_source_owner, name='data-source-contact'),
+
     url(r'^virus/$', views.VirusListView.as_view(), name='virus-list'),
     url(r'^virus/(?P<pk>\d+)/$', views.VirusDetailView.as_view(), name='virus-detail'),
+    url(r'^virus/(?P<pk>\d+)/download/$', views.virus_download, name='virus-download'),
+
     url(r'^host/$', views.HostListView.as_view(), name='host-list'),
     url(r'^host/(?P<pk>\d+)/$', views.HostDetailView.as_view(), name='host-detail'),
+    url(r'^host/(?P<pk>\d+)/download/$', views.host_download, name='host-download'),
+
     url(r'^response/ds-(?P<ds_pk>\d+)-virus-(?P<virus_pk>\d+)-host-(?P<host_pk>\d+)/update/$',
         views.response_update, name='response-update'),
     url(r'^search/$', views.search, name='search'),

src/viralhostrange/viralhostrangedb/views.py

 import itertools
 import json
 import os
+import re
 import tempfile
 
 import pandas as pd
@@ -255,6 +256,7 @@ class VirusDetailView(mixins.OnlyPublicOrGrantedOrStaffOrOwnedRelatedMixin, Deta
         context["focus_is_on_first_level_of_data"] = True
         context["get_detail_api_url"] = reverse("viralhostrangedb-api:host-detail", args=["000000"])
         context["get_detail_url"] = reverse("viralhostrangedb:host-detail", args=["000000"])
+        context["download_url"] = reverse("viralhostrangedb:virus-download", args=[self.object.pk])
         context["schema"] = models.GlobalViralHostResponseValue.objects_mappable().order_by("value")
         # context["object_update"] = reverse("viralhostrangedb:virus-update", args=[self.object.pk])
         context["form"] = forms.VirusHostDetailViewForm(
@@ -282,6 +284,7 @@ class HostDetailView(mixins.OnlyPublicOrGrantedOrStaffOrOwnedRelatedMixin, Detai
         context["focus_is_on_first_level_of_data"] = False
         context["get_detail_api_url"] = reverse("viralhostrangedb-api:virus-detail", args=["000000"])
         context["get_detail_url"] = reverse("viralhostrangedb:virus-detail", args=["000000"])
+        context["download_url"] = reverse("viralhostrangedb:host-download", args=[self.object.pk])
         context["schema"] = models.GlobalViralHostResponseValue.objects_mappable().order_by("value")
         # context["object_update"] = reverse("viralhostrangedb:host-update", args=[self.object.pk])
         context["form"] = forms.VirusHostDetailViewForm(
@@ -840,6 +843,148 @@ def data_source_download(request, pk):
             return response
 
 
+def virus_download(request, pk):
+    instance = get_object_or_404(mixins.only_public_or_granted_or_staff_or_owned_queryset_filter(
+        self=None,
+        request=request,
+        queryset=models.Virus.objects,
+        path_to_data_source="data_source__",
+    ), pk=pk)
+    return host_or_virus_download(request=request, is_host=False, instance=instance)
+
+
+def host_download(request, pk):
+    instance = get_object_or_404(mixins.only_public_or_granted_or_staff_or_owned_queryset_filter(
+        self=None,
+        request=request,
+        queryset=models.Host.objects,
+        path_to_data_source="data_source__",
+    ), pk=pk)
+    return host_or_virus_download(request=request, is_host=True, instance=instance)
+
+
+def host_or_virus_download(request, is_host, instance):
+    request.GET = request.GET.copy()
+    request.GET['format'] = 'json'
+    form = forms.VirusHostDetailViewForm(
+        user=request.user,
+        data=request.GET if request.method == 'GET' else None,
+    )
+    if not form.is_valid():
+        return HttpResponseBadRequest(content=str(form.errors))
+    request.GET['host' if is_host else 'virus'] = str(instance.pk)
+
+    # Get the data
+    response = views_api.CompleteResponseViewSet.as_view()(request)
+    complete_responses = json.loads(response.rendered_content.decode('utf-8'))
+    data_sources = dict(mixins.only_public_or_granted_or_owned_queryset_filter(
+        self=None,
+        request=request,
+        queryset=instance.data_source
+    ).values_list("pk", "name"))
+
+    # get if the data will be shifted because of the infection ratio
+    infection_ratio_shift = 1 if form.cleaned_data["infection_ratio"] else 0
+
+    col_header = list(data_sources.values())
+    row_header = []
+    data = []
+
+    if infection_ratio_shift == 1:
+        col_header.insert(0, _("Infection ratio"))
+        row_header.insert(0, _("Infection ratio"))
+        data.append([""] * len(col_header))
+
+    # get at which position each col is
+    col_pos = dict([(pk, i) for i, pk in enumerate(
+        data_sources.keys(),
+        start=infection_ratio_shift,
+    )])
+    row_pos = dict()
+
+    row = None
+    for virus_pk, hosts_responses in complete_responses.items():
+        if is_host:
+            row_header.append(models.Virus.objects.get(pk=virus_pk).explicit_name)
+            row = [""] * len(col_header)
+            data.append(row)
+        for host_pk, ds_reponses in hosts_responses.items():
+            if not is_host:
+                row_header.append(models.Host.objects.get(pk=host_pk).explicit_name)
+                row = [""] * len(col_header)
+                data.append(row)
+
+            for ds_pk, response in ds_reponses.items():
+                row[col_pos[int(ds_pk)]] = response
+
+    # Get the virus infection ratio if selected
+    if infection_ratio_shift == 1:
+        response = views_api.VirusInfectionRatioViewSet. \
+            as_view(data_source_aggregated=False)(request=request, slug="host" if is_host else "virus")
+        infection_ratio = json.loads(response.rendered_content.decode('utf-8'))
+
+        virus_infection_ratio = infection_ratio["virus" if is_host else "host"]
+        actual_total = max(v["total"] for v in virus_infection_ratio.values())
+        for pk, value in virus_infection_ratio.items():
+            row_pos[int(pk)] = 1 + len(row_pos)
+            data[len(row_pos)][0] = value['ratio'] * value['total'] * 100 / actual_total
+
+        ds_infection_ratio = infection_ratio["data_source"]
+        actual_total = max(v["total"] for v in ds_infection_ratio.values())
+        for pk, value in ds_infection_ratio.items():
+            data[0][col_pos[int(pk)]] = value['ratio'] * value['total'] * 100 / actual_total
+
+    # Prepare the legend
+    mapping = models.GlobalViralHostResponseValue.objects_mappable().order_by('value')
+    legend_name = []
+    legend_value = []
+    legend = []
+    for m in models.GlobalViralHostResponseValue.objects_mappable().order_by('value'):
+        legend_name.append([m.name])
+        legend_value.append(m.value)
+        legend.append([m.name, m.value])
+    df_legend = pd.DataFrame(legend_name, columns=[str(_("Legend"))], index=legend_value)
+
+    # Prepare the setting help of the infection ratio
+    options = []
+    if form.cleaned_data["weak_infection"]:
+        options.append([form.fields["weak_infection"].label, ])
+    df_options = pd.DataFrame(options, columns=[str(_("Infection ratio settings"))], index=None)
+
+    # Prepare the title
+    title = _("%(kind)s \"%(name)s\"") % dict(
+        kind=_("Host") if is_host else _("Virus"),
+        name=instance.explicit_name,
+    )
+    df_title = pd.DataFrame([], columns=[str(title)], index=None)
+
+    with NamedTemporaryFile(suffix=".xlsx") as f:
+        # write in a temp file
+        with pd.ExcelWriter(f.name) as writer:
+            df_title.to_excel(writer)
+            pd.DataFrame(data,
+                         index=row_header,
+                         columns=col_header,
+                         ) \
+                .to_excel(writer, startcol=0, startrow=2)
+            df_legend.to_excel(writer, startcol=0, startrow=len(row_header) + 3 + 2)
+            if len(options) > 0:
+                df_options.to_excel(writer, startcol=0, startrow=len(row_header) + 3 + len(legend_name) + 3 + 2,
+                                    index=None)
+
+        # then read the temp file and send it to the client
+        with open(f.name, "rb") as excel:
+            response = HttpResponse(excel.read(),
+                                    content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
+            # Translators: it is the beginning of a file name
+            raw_filename = _("ViralHostRange data of %(host_or_virus_name)s on %(date)s") % dict(
+                host_or_virus_name=re.sub(r'[^\w\d-]', '_', instance.explicit_name),
+                date=timezone.now().strftime('%Y%m%d %Hh%M.%S') + ".xlsx"
+            )
+            response['Content-Disposition'] = 'attachment; filename="%s"' % raw_filename
+            return response
+
+
 @method_decorator(login_required, name='dispatch')
 class DataSourceWizard(wizard_views.NamedUrlSessionWizardView):
     def get_upload_or_live_input(self):