views_api.py

import itertools
import math
import re
import time
from functools import reduce
from typing import List
from urllib.parse import urlencode

from django.contrib.auth.models import AnonymousUser
from django.db import connection
from django.db.models import Avg, Q, Count, Func, When, IntegerField, Case, Value, Max, Min, CharField, FloatField, \
    BooleanField
from django.db.models.functions import Concat, Coalesce, Replace, Length, Upper, Cast
from django.db.models.lookups import Transform
from django.http import JsonResponse, HttpResponseBadRequest
from django.urls import reverse
from rest_framework import views as drf_views, viewsets
from rest_framework.filters import BaseFilterBackend
from rest_framework.response import Response

from viralhostrangedb import models, mixins, serializers, forms, business_process


class MyAPIView(drf_views.APIView):
    queryset = None
    filter_backends = []

    def get_queryset(self):
        queryset = self.queryset
        return queryset

    def filter_queryset(self, queryset):
        """
        Given a queryset, filter it with whichever filter backend is in use.
        """
        for backend in list(self.filter_backends):
            queryset = backend().filter_queryset(self.request, queryset, self)
        return queryset

    def get(self, request, *args, **kwargs):
        raise NotImplementedError()


def to_int_array(input_list: List):
    for a in input_list:
        try:
            yield int(a)
        except ValueError:
            pass


def filter_ds(queryset, path_to_data_source, qd, **kwargs):
    if not "ds" in qd:
        return queryset
    getlist = reduce(lambda x, y: x + y, [l.split(',') for l in qd.getlist("ds")])
    return queryset.filter(**{path_to_data_source + "pk__in": to_int_array(getlist)})


def filter_owner(queryset, path_to_data_source, qd, **kwargs):
    if not "owner" in qd:
        return queryset
    getlist = reduce(lambda x, y: x + y, [l.split(',') for l in qd.getlist("owner")])
    return queryset.filter(**{path_to_data_source + "owner__pk__in": to_int_array(getlist)})


def filter_host(queryset, path_to_host, qd, **kwargs):
    if not "host" in qd:
        return queryset
    getlist = reduce(lambda x, y: x + y, [l.split(',') for l in qd.getlist("host")])
    return queryset.filter(**{path_to_host + "pk__in": to_int_array(getlist)})


def filter_virus(queryset, path_to_virus, qd, **kwargs):
    if "virus" not in qd:
        return queryset
    getlist = reduce(lambda x, y: x + y, [l.split(',') for l in qd.getlist("virus")])
    return queryset.filter(**{path_to_virus + "pk__in": to_int_array(getlist)})


def filter_only_published_data(queryset, path_to_data_source, qd, **kwargs):
    if "only_published_data" not in qd:
        return queryset
    return queryset.filter(**{path_to_data_source + "publication_url__isnull": False})


def filter_only_virus_ncbi_id(queryset, path_to_virus, qd, **kwargs):
    if "only_virus_ncbi_id" not in qd:
        return queryset
    return queryset.filter(**{path_to_virus + "is_ncbi_identifier_value": True})


def filter_only_host_ncbi_id(queryset, path_to_host, qd, **kwargs):
    if "only_host_ncbi_id" not in qd:
        return queryset
    return queryset.filter(**{path_to_host + "is_ncbi_identifier_value": True})


def filter_life_domain(queryset, path_to_data_source, qd, **kwargs):
    if "life_domain" not in qd or len(qd["life_domain"]) == 0:
        return queryset
    return queryset.filter(**{path_to_data_source + "life_domain": qd["life_domain"]})


class MetaFilterResponseBackend(BaseFilterBackend):
    paths = dict(
        path_to_data_source="data_source__",
        path_to_virus="virus__",
        path_to_host="host__",
    )

    def filter_queryset(self, request, queryset, view):
        filter_kwargs = dict(qd=request.GET)
        filter_kwargs.update(self.paths)
        queryset = filter_ds(queryset=queryset, **filter_kwargs)
        queryset = filter_owner(queryset=queryset, **filter_kwargs)
        queryset = filter_host(queryset=queryset, **filter_kwargs)
        queryset = filter_virus(queryset=queryset, **filter_kwargs)
        queryset = filter_only_published_data(queryset=queryset, **filter_kwargs)
        queryset = filter_only_virus_ncbi_id(queryset=queryset, **filter_kwargs)
        queryset = filter_only_host_ncbi_id(queryset=queryset, **filter_kwargs)
        queryset = filter_life_domain(queryset=queryset, **filter_kwargs)
        return queryset


class FromGetParamsFilterResponseBackend(MetaFilterResponseBackend):
    paths = dict(
        path_to_data_source="data_source__",
        path_to_virus="virus__",
        path_to_host="host__",
    )

    def filter_queryset(self, request, queryset, view):
        if "allow_overflow" not in request.GET \
                and "ds" not in request.GET \
                and "host" not in request.GET \
                and "virus" not in request.GET:
            return queryset.none()
        return super().filter_queryset(request, queryset, view)


class FromGetParamsFilterVirusBackend(MetaFilterResponseBackend):
    paths = dict(
        path_to_data_source="data_source__",
        path_to_virus="",
        path_to_host="responseindatasource__host__",
    )


class FromGetParamsFilterHostBackend(MetaFilterResponseBackend):
    paths = dict(
        path_to_data_source="data_source__",
        path_to_virus="responseindatasource__virus__",
        path_to_host="",
    )


class FromGetParamsFilterDataSourceBackend(MetaFilterResponseBackend):
    paths = dict(
        path_to_data_source="",
        path_to_virus="virus__",
        path_to_host="host__",
    )


class OnlyPublicOrGrantedOrOwnedFilterBackend(BaseFilterBackend):
    def filter_queryset(self, request, queryset, view):
        return mixins.only_public_or_granted_or_owned_queryset_filter(
            self,
            request=request,
            queryset=queryset,
            path_to_data_source="data_source__",
        )


class OnlyMappedResponseFilterBackend(BaseFilterBackend):
    def filter_queryset(self, request, queryset, view):
        return queryset.filter(~Q(response__pk=models.GlobalViralHostResponseValue.get_not_mapped_yet_pk()))


class Round(Func):
    function = 'ROUND'
    arity = 2
    arg_joiner = '::numeric, '

    def __init__(self, *expressions,  **extra):
        super().__init__(*expressions, output_field=FloatField(), **extra)

    def as_sqlite(self, compiler, connection, **extra_context):
        return super().as_sqlite(compiler, connection, arg_joiner=", ", **extra_context)


class AggregatedResponseViewSet(MyAPIView):
    queryset = models.ViralHostResponseValueInDataSource.objects
    filter_backends = [OnlyMappedResponseFilterBackend, FromGetParamsFilterResponseBackend,
                       OnlyPublicOrGrantedOrOwnedFilterBackend]

    def get(self, request):
        queryset = self.filter_queryset(self.get_queryset())
        queryset = queryset.values("virus__pk", "host__pk") \
            .annotate(val=Round(Avg('response__value'), 2)) \
            .annotate(diff=Count('response__value', distinct=True)) \
            .order_by('virus__pk')

        table = {}
        hosts = None
        last_virus = None
        for entry in queryset:
            virus_pk = entry['virus__pk']
            if last_virus != virus_pk:
                hosts = {}
                last_virus = virus_pk
                table[virus_pk] = hosts
            hosts[entry['host__pk']] = dict(val=entry['val'], diff=entry['diff'])

        return Response(table)


class CompleteResponseViewSet(MyAPIView):
    queryset = models.ViralHostResponseValueInDataSource.objects
    filter_backends = [OnlyMappedResponseFilterBackend, FromGetParamsFilterResponseBackend,
                       OnlyPublicOrGrantedOrOwnedFilterBackend]

    def get(self, request):
        queryset = self.filter_queryset(self.get_queryset())
        queryset = queryset.values("virus__pk", "host__pk", "data_source__pk", "response__value") \
            .order_by('virus__pk', 'host__pk', 'data_source__pk', )

        table = {}
        hosts = None
        virus = None
        response = None
        last_virus = None
        last_host = None
        for entry in queryset:
            virus_pk = entry['virus__pk']
            if last_virus != virus_pk:
                hosts = {}
                last_virus = virus_pk
                table[virus_pk] = hosts
                last_host = None
            host_pk = entry['host__pk']
            if last_host != host_pk:
                response = {}
                last_host = host_pk
                hosts[host_pk] = response
            response[entry['data_source__pk']] = entry['response__value']

        return Response(table)


class PKFromGetParamsFilterBackend(BaseFilterBackend):
    def filter_queryset(self, request, queryset, view):
        if "pks" in request.GET:
            queryset = queryset.filter(pk__in=request.GET["pks"].split(','))
        if "pk" in request.GET:
            queryset = queryset.filter(pk__in=request.GET["pk"].split(','))
        if "ids" in request.GET:
            queryset = queryset.filter(pk__in=request.GET["ids"].split(','))
        if "id" in request.GET:
            queryset = queryset.filter(pk__in=request.GET["id"].split(','))
        return queryset


class VirusViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = models.Virus.objects
    serializer_class = serializers.VirusSerializer
    filter_backends = [
        OnlyPublicOrGrantedOrOwnedFilterBackend,
        PKFromGetParamsFilterBackend,
        FromGetParamsFilterVirusBackend,
    ]


class HostViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = models.Host.objects
    serializer_class = serializers.HostSerializer
    filter_backends = [
        OnlyPublicOrGrantedOrOwnedFilterBackend,
        PKFromGetParamsFilterBackend,
        FromGetParamsFilterHostBackend,
    ]


class OnlyPublicOrGrantedOrOwnedDataSourceFilterBackend(BaseFilterBackend):
    def filter_queryset(self, request, queryset, view):
        return mixins.only_public_or_granted_or_owned_queryset_filter(
            self,
            request=request,
            queryset=queryset,
        )


class DataSourceViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = models.DataSource.objects
    serializer_class = serializers.DataSourceSerializer
    filter_backends = [
        OnlyPublicOrGrantedOrOwnedDataSourceFilterBackend,
        PKFromGetParamsFilterBackend,
        FromGetParamsFilterDataSourceBackend,
    ]


class VirusInfectionRatioViewSet(MyAPIView):
    data_source_aggregated = True
    queryset = models.ViralHostResponseValueInDataSource.objects
    filter_backends = [OnlyMappedResponseFilterBackend, FromGetParamsFilterResponseBackend,
                       OnlyPublicOrGrantedOrOwnedFilterBackend]

    def get(self, request, slug=None, slug_pk=None, *args, **kwargs):
        queryset = self.get_queryset()

        other_slug = None
        if slug == 'virus':
            other_slug = 'host'
        elif slug == 'host':
            other_slug = 'virus'
        else:
            return HttpResponseBadRequest("unknown slug '%s'" % slug)

        # filter only for the current slug
        if slug_pk != None:
            if slug == "virus":
                queryset = queryset.filter(virus__pk=int(str(slug_pk)))
            elif slug == "host":
                queryset = queryset.filter(host__pk=int(str(slug_pk)))
            # As the FromGetParamsFilterResponseBackend prevent to return too much data when no GET parameter is provided we
            # explicitly indicate that even if there is no GET parameter, we should not return an empty queryset.
            request.GET = request.GET.copy()
            request.GET.update({"allow_overflow": True})

        # filter with_backend
        queryset = self.filter_queryset(queryset)

        # picking the minimum global response value to consider that there is an infection
        if "weak_infection" in request.GET:
            # an infection is everything except the lower (i.e: No Lysis)
            min_level = models.GlobalViralHostResponseValue.objects_mappable().aggregate(v=Min("value"))['v']
            min_level = models.GlobalViralHostResponseValue.objects_mappable().filter(value__gt=min_level) \
                .aggregate(v=Min("value"))['v']
        else:
            # an infection is only the higher (i.e: Lysis)
            min_level = models.GlobalViralHostResponseValue.objects_mappable().aggregate(v=Max("value"))['v']

        # annotating the query such as infection is now 0/1
        queryset = queryset.annotate(infection=Case(
            When(response__value__gte=min_level, then=1),
            default=Value(0),
            output_field=IntegerField(),
        ))

        # determine how the response should be aggregated, do we enforce consensus or not
        if "agreed_infection" in request.GET:
            infection_aggregated = Min('infection')
        else:
            infection_aggregated = Max('infection')

        # aggregate per host/virus the infections
        if self.data_source_aggregated:
            queryset = queryset.values('host__pk', 'virus__pk')
        else:
            queryset = queryset.values('host__pk', 'virus__pk', 'data_source__pk')
        queryset = queryset.annotate(infection_aggregated=infection_aggregated)

        # Store infection in a dict where key in the pk, and value the count of infection and no infection
        infection_aggregated_counters = {}
        infection_aggregated_counters_on_data_source = None if self.data_source_aggregated else {}

        if self.data_source_aggregated:
            # we will get either virus__pk or host__pk, so suffixing the slug
            aggregation_key = slug + "__pk"
        else:
            # we will get the aggregation over the other slug: for a host, we get for each virus.
            aggregation_key = other_slug + "__pk"

        # run the query and fetch results
        for o in queryset:
            # infection is either yes(1) or no(0)
            infection_aggregated_counter = infection_aggregated_counters.setdefault(o[aggregation_key], [0, 0])
            # count infection or not an infection
            infection_aggregated_counter[o['infection_aggregated']] += 1
            if infection_aggregated_counters_on_data_source is not None:
                infection_aggregated_counter = infection_aggregated_counters_on_data_source \
                    .setdefault(o['data_source__pk'], [0, 0])
                # count infection or not an infection
                infection_aggregated_counter[o['infection_aggregated']] += 1

        # build the final ratio
        self.aggregate_infection_counters(infection_aggregated_counters)

        if self.data_source_aggregated:
            # return the ratios as a json
            return Response(infection_aggregated_counters)

        self.aggregate_infection_counters(infection_aggregated_counters_on_data_source)

        return Response({
            other_slug: infection_aggregated_counters,
            "data_source": infection_aggregated_counters_on_data_source
        })

    def aggregate_infection_counters(self, infection_aggregated_counters):
        for k, infection_aggregated_counter in infection_aggregated_counters.items():
            s = sum(infection_aggregated_counter)
            # compute the ratio as it is what we want, keep the total in case third part service need it
            infection_aggregated_counters[k] = dict(
                ratio=infection_aggregated_counter[1] / s,
                total=s,
            )


def remplace_greek_letters_name_by_themselves(searched_text):
    p = getattr(remplace_greek_letters_name_by_themselves, "__pattern_instance", None)
    if p is None:
        letters = {
            'alpha': 'α',
            'beta': 'β',
            'gamma': 'γ',
            'delta': 'δ',
            'epsilon': 'ε',
            'zeta': 'ζ',
            'eta': 'η',
            'theta': 'θ',
            'iota': 'ι',
            'kappa': 'κ',
            'lambda': 'λ',
            'mu': 'μ',
            'nu': 'ν',
            'xi': 'ξ',
            'omicron': 'ο',
            'pi': 'π',
            'rho': 'ρ',
            'sigma': 'σ',
            'tau': 'τ',
            'upsilon': 'υ',
            'phi': 'φ',
            'chi': 'χ',
            'psi': 'ψ',
            'omega': 'ω',
        }

        def flagged_replacer():
            def actual_replacer(match):
                match = match.group().lower()
                actual_replacer.has_match = True
                return match.replace(match.strip(), letters[match.strip()])

            actual_replacer.has_match = False

            return actual_replacer

        pattern_instance = re.compile(
            flags=re.IGNORECASE,
            pattern=r'(alpha)|'
                    r'(beta)|'
                    r'(gamma)|'
                    r'(delta)|'
                    r'(epsilon)|'
                    r'(zeta)|'
                    r'(eta)|'
                    r'(theta)|'
                    r'(iota)|'
                    r'(kappa)|'
                    r'(lambda)|'
                    r'(mu)|'
                    r'(nu)|'
                    r'(xi)|'
                    r'(omicron)|'
                    r'(pi)|'
                    r'(rho)|'
                    r'(sigma)|'
                    r'(tau)|'
                    r'(upsilon)|'
                    r'(phi)|'
                    r'(chi)|'
                    r'(psi)|'
                    r'(omega)',
        )
        remplace_greek_letters_name_by_themselves.__pattern_instance = pattern_instance
        remplace_greek_letters_name_by_themselves.__greek_letters_replacer_factory = flagged_replacer
    else:
        pattern_instance = remplace_greek_letters_name_by_themselves.__pattern_instance

    replacer = remplace_greek_letters_name_by_themselves.__greek_letters_replacer_factory()

    return pattern_instance.sub(replacer, searched_text), replacer.has_match


class Unaccent(Transform):
    function = 'UNACCENT'
    lookup_name = 'unaccent'


def search(request):
    form = forms.SearchForm(data=request.POST or request.GET, user=request.user)
    if not form.is_valid():
        return JsonResponse(dict(err="Invalid query", detail=form.errors, app_error_code=1, searched_text=""))

    sorting = form.cleaned_data["sorting"] or "TFIDFnCOV"
    use_tf = sorting[:2] == "TF"
    use_idf = sorting[:5] == "TFIDF"
    use_cov = sorting == "TFIDFnCOV"

    # get the raw searched string
    searched_text = form.cleaned_data["search"]

    # replace all space by a %20 : <i>bla "foo bar" bli<i> become <i>bla foo%20bar bli<i>
    for m in [m.group() for m in re.finditer('"[^"]+"', searched_text)]:
        searched_text = searched_text.replace(m, m.replace("\"", "%20").replace(" ", "%20"))

    # split the search text by space
    raw_searched_texts = [x for x in searched_text.split(" ") if len(x) > 0]

    # find greek letters
    greek_searched_text, has_greek = remplace_greek_letters_name_by_themselves(searched_text)
    if has_greek:
        # if found, add also them, again split by space
        raw_searched_texts += [x if len(x) > 1 else "%20" + x + "%20"
                               for x in greek_searched_text.split(" ")
                               if len(x.strip()) > 0]

    # extract mandatory token
    mandatory = [x[1:] for x in raw_searched_texts if x[0] == '+']
    # extract mandatory token
    banned = [x[1:] for x in raw_searched_texts if x[0] == '-']

    # remove leading "+" and  words starting with "-"
    raw_searched_texts = [x[1:] if x[0] == '+' else x for x in raw_searched_texts if x[0] != '-']

    # as splitting by space is done, so does picking mandatory and banned, we replace back %20 with blank space
    searched_texts = extract_word_and_target_pairs(raw_searched_texts)
    mandatory = extract_word_and_target_pairs(mandatory)
    banned = extract_word_and_target_pairs(banned)
    if len(searched_texts) == 0:
        if len(extract_word_and_target_pairs(raw_searched_texts, True)) > 0:
            return JsonResponse(dict(
                err="Single letter query are not permitted, put it between double quote to override",
                app_error_code=2,
                searched_text=searched_text,
            ))
        return JsonResponse(dict(err="Invalid query", app_error_code=1, searched_text=""))

    # search options
    sample_size = form.cleaned_data["sample_size"]
    kind = form.cleaned_data["kind"] or 'all'
    search_key = "__unaccent__icontains" if connection.vendor == "postgresql" else "__icontains"
    ui_help = form.cleaned_data["ui_help"]
    str_max_length = form.cleaned_data["str_max_length"]

    if sample_size == 0:
        sample_size = 7 if kind == "all" else 15

    qs_virus = models.Virus.objects.annotate(
        her_identifier_s=Coalesce(Cast('her_identifier', CharField()), Value('')),
        tax_id_value_s=Coalesce('tax_id_value', Value('')),
        has_identifier=Case(
            When(
                Q(is_ncbi_identifier_value=True)
                | Q(her_identifier__isnull=False)
                | Q(tax_id_value__isnull=False),
                then=1,
            ),
            default=0,
            output_field=IntegerField(),
        )
    )
    qs_virus = filter_and_sort_qs(
        searched_texts=searched_texts,
        banned=banned,
        mandatory=mandatory,
        search_key=search_key,
        fields_to_search_in=["name", "identifier", "her_identifier_s", "tax_id_value_s"],
        path_to_data_source="data_source__",
        queryset=qs_virus,
        request=request,
        use_idf=use_idf,
        use_tf=use_tf,
        use_cov=use_cov,
        cls_filter_backend=FromGetParamsFilterVirusBackend,
        additional_ordering=["-has_identifier", ],
    )
    # print("@@@@@@@@@@@@@" + searched_text)
    # for o in qs_virus:
    #     print(o.pk, o.name, o.identifier, o.her_identifier, o.her_identifier_s, o.relevance)

    qs_host = models.Host.objects.annotate(
        tax_id_value_s=Coalesce('tax_id_value', Value('')),
        has_identifier=Case(
            When(
                Q(is_ncbi_identifier_value=True)
                | Q(tax_id_value__isnull=False),
                then=1,
            ),
            default=0,
            output_field=IntegerField(),
        )
    )
    qs_host = filter_and_sort_qs(
        searched_texts=searched_texts,
        banned=banned,
        mandatory=mandatory,
        search_key=search_key,
        fields_to_search_in=["name", "identifier", "tax_id_value_s"],
        path_to_data_source="data_source__",
        queryset=qs_host,
        request=request,
        use_idf=use_idf,
        use_tf=use_tf,
        use_cov=use_cov,
        cls_filter_backend=FromGetParamsFilterVirusBackend,
        additional_ordering=["-has_identifier", ],
    )
    # print("%%%%%%%%%%%%%" + searched_text)
    # for o in qs_host:
    #     print(o.pk, o.name, o.identifier, o.relevance)

    # annotating the qs so we have the provider ie provider_* if defined, owner otherwise
    qs_data_source = models.DataSource.objects \
        .annotate(publication_url_or_empty=Coalesce('publication_url', Value(''))) \
        .annotate(owner_str=Concat('owner__first_name', Value(' '), 'owner__last_name')) \
        .annotate(provider_raw=Concat('provider_first_name', Value(' '), 'provider_last_name')) \
        .annotate(provider_str=Case(
        When(provider_raw=' ', then=None),
        default='provider_raw',
        output_field=CharField(),
    )) \
        .annotate(provider=Coalesce('provider_str', 'owner_str')) \
        .annotate(has_publication_url=Case(When(publication_url__isnull=True, then=True),
                                           default=False,
                                           output_field=BooleanField(),
                                           ))

    qs_data_source = filter_and_sort_qs(
        searched_texts=searched_texts,
        banned=banned,
        mandatory=mandatory,
        search_key=search_key,
        fields_to_search_in=["name", "description", "provider", "publication_url_or_empty"],
        path_to_data_source="",
        queryset=qs_data_source,
        request=request,
        use_idf=use_idf,
        use_tf=use_tf,
        use_cov=use_cov,
        cls_filter_backend=FromGetParamsFilterDataSourceBackend,
        additional_ordering=["has_publication_url"],
    )
    # print("#############" + searched_text)
    # for o in qs_data_source:
    #     print(o.pk, o.name, o.relevance)

    responses = dict(
        query=dict(
            searched_text=searched_text,
            mandatory_searched_texts=[x[1] for x in mandatory],
            banned_searched_texts=[x[1] for x in banned],
            alternative_searched_texts=[x[1] for x in searched_texts],
            # mandatory_searched_texts_and_target=mandatory,
            # banned_searched_texts_and_target=banned,
            # alternative_searched_texts_and_target=searched_texts,
            sample_size=sample_size,
            sorting=sorting,
            kind=kind,
        )
    )
    if ui_help:
        responses["ui_help"] = dict()

    get_param = dict(form.cleaned_data)
    get_param["owner"] = list(get_param["owner"].values_list("pk", flat=True))
    get_param["sample_size"] = -1
    get_param.pop("ui_help")
    get_param.pop("kind")
    for k, v in list(get_param.items()):
        if not v:
            get_param.pop(k)
    get_param = urlencode(get_param, doseq=True)

    if kind in ['virus', 'all']:
        if ui_help:
            responses["ui_help"]["virus"] = models.Virus._meta.verbose_name.title()
        responses["virus"] = dict(
            count=qs_virus.count(),
            sample=serializers.VirusSerializerWithURL(
                qs_virus[0:sample_size] if sample_size > 0 else qs_virus, many=True).data,
            subset_url="%s?kind=virus&%s" % (reverse("viralhostrangedb:search"), get_param)
        )
    if kind in ['host', 'all']:
        if ui_help:
            responses["ui_help"]["host"] = models.Host._meta.verbose_name.title()
        responses["host"] = dict(
            count=qs_host.count(),
            sample=serializers.HostSerializerWithURL(
                qs_host[0:sample_size] if sample_size > 0 else qs_host, many=True).data,
            subset_url="%s?kind=host&%s" % (reverse("viralhostrangedb:search"), get_param)
        )
    if kind in ['data_source', 'all']:
        if ui_help:
            responses["ui_help"]["data_source"] = models.DataSource._meta.verbose_name.title()
        responses["data_source"] = dict(
            count=qs_data_source.count(),
            sample=serializers.DataSourceSerializerForSearch(
                instance=qs_data_source[0:sample_size] if sample_size > 0 else qs_data_source,
                many=True,
                str_max_length=str_max_length,
            ).data,
            subset_url="%s?kind=data_source&%s" % (reverse("viralhostrangedb:search"), get_param)
        )
    return JsonResponse(responses)


def extract_word_and_target_pairs(texts, allow_single_letter=False):
    def clean_up(x):
        return x.replace("%20", " ").strip()

    pairs = []
    for s in texts:
        if s.upper().startswith("ID:"):
            targets = ["identifier", "her_identifier_s"]
            word = s[3:]
        elif s.upper().startswith("NCBI:"):
            targets = ["identifier"]
            word = s[5:]
        elif s.upper().startswith("HER:"):
            targets = ["her_identifier_s"]
            word = s[4:]
        elif s.upper().startswith("TAX:"):
            targets = ["tax_id_value"]
            word = s[4:]
        elif s.upper().startswith("NAME:"):
            targets = ["name"]
            word = s[5:]
        elif s.upper().startswith("DESC:"):
            targets = ["description"]
            word = s[5:]
        elif s.upper().startswith("OWNER:"):
            targets = ["provider"]
            word = s[6:]
        elif s.upper().startswith("PROVIDER:"):
            targets = ["provider"]
            word = s[9:]
        elif s.upper().startswith("PROV:"):
            targets = ["provider"]
            word = s[5:]
        else:
            targets = None
            word = s
        if len(word) > 1 or allow_single_letter and len(word) == 1:
            pairs.append((targets, clean_up(word)))
    return pairs


def filter_and_sort_qs(
        searched_texts,
        banned,
        mandatory,
        search_key,
        fields_to_search_in,
        path_to_data_source,
        queryset,
        request,
        use_idf,
        use_tf,
        use_cov,
        cls_filter_backend,
        additional_ordering=None,
):
    additional_ordering = additional_ordering or []
    queryset.query.clear_ordering(force_empty=True)
    if len(mandatory) == 0:
        # filtering data source to get entries matching one of the search key
        # if mandatory is not empty this step is useless as all match must have all keyword in mandatory, and the other
        # can be, but also can not be present. We will use the searched_texts only for sorting
        qs = []
        for s, f in itertools.product(searched_texts, fields_to_search_in):
            if s[0] is None or f in s[0]:
                qs.append(Q(**{f + search_key: s[1]}))
        queryset = queryset.filter(reduce(lambda x, y: x | y, qs, Q(pk=None)))
    for word in mandatory:
        qs = []
        for f in fields_to_search_in:
            if word[0] is None or f in word[0]:
                qs.append(Q(**{f + search_key: word[1]}))
        queryset = queryset.filter(reduce(lambda x, y: x | y, qs, Q(pk=None)))
    for word, f in itertools.product(banned, fields_to_search_in):
        if word[0] is None or f in word[0]:
            queryset = queryset.exclude(**{f + search_key: word[1]})
    if use_idf:
        idf_ds = compute_idf(fields_to_search_in, queryset, search_key, searched_texts, mandatory)
    else:
        idf_ds = {}
    if use_tf:
        # annotating each row with the number of time one of the search key have been found in one of the fields
        queryset = queryset.annotate(relevance=get_hit_count(
            fields=fields_to_search_in,
            searched_texts=searched_texts,
            idf=idf_ds,
            use_cov=use_cov,
        ))
        additional_ordering.insert(0, "-relevance")
    queryset = cls_filter_backend().filter_queryset(request, queryset, None)
    queryset = mixins.only_public_or_granted_or_owned_queryset_filter(
        None,
        request=request,
        queryset=queryset,
        path_to_data_source=path_to_data_source,
    )
    queryset = queryset.order_by(*additional_ordering)
    # if queryset.model == models.Virus:
    #     print("-------", ";".join([str(a) + ":" + str(b) for a, b in searched_texts]))
    #     for o in queryset:
    #         print(o.pk, o.name, o.identifier, o.her_identifier, o.tax_id_value, o.relevance)
    return queryset


def compute_idf(fields_names, qs, search_key, searched_texts, mandatory):
    fields_names = set(fields_names)
    idf = {}
    number_of_ds = qs.count()
    mandatory_words = [x[1] for x in mandatory]
    for target_and_word in searched_texts:
        s = target_and_word[1]
        if s in mandatory_words:
            continue
        try:
            hits = []
            if target_and_word[0] is None:
                fields_for_this_s = fields_names
            else:
                fields_for_this_s = fields_names.intersection(set(target_and_word[0]))
            for f in fields_for_this_s:
                hits.append(Q(**{f + search_key: s}))
            idf[s] = math.log(1 + number_of_ds / qs.filter(
                reduce(lambda x, y: x | y, hits, Q(pk=None))
            ).count())
        except ZeroDivisionError:
            idf[s] = 0
    return idf


if connection.vendor == "postgresql":
    def transformer(x):
        return Unaccent(Upper(x))
else:
    def transformer(x):
        return Upper(x)

__default_idf = math.log(2)


def get_hit_count(fields, searched_texts, idf=None, default_idf=__default_idf, use_cov=False):
    # To count how many time we find s in f, we compare the length of f, with the length of f where each occurrences
    # of s have been removed/replaced-by-empty-string (LR). To get the number of hit we then do (len(f)-LR)/len(s)
    # Note that we don't use f and s directly, we first put it in uppercase, and if possible without accent
    if idf is None:
        idf = dict()
    fields = set(fields)
    tf_idf_s = []
    for target_and_word in searched_texts:
        s = target_and_word[1]
        len_s = len(s)
        nb_hits = []
        if target_and_word[0] is None:
            fields_for_this_s = fields
        else:
            fields_for_this_s = fields.intersection(set(target_and_word[0]))
        for f in fields_for_this_s:
            f_without_s = Length(f) - Length(Replace(transformer(f), transformer(Value(s)), Value('')))
            nb_hit = Cast(f_without_s, FloatField()) / Value(len_s)
            if use_cov:
                nb_hit = Cast(nb_hit + Cast(f_without_s, FloatField()) / (Length(f) + Value(0.00000001)), FloatField())
            nb_hits.append(nb_hit)
        if len(nb_hits) > 0:
            tf_idf_s.append(
                Value(idf.get(s, default_idf)) * (reduce(lambda x, y: x + y, nb_hits))
            )
    return reduce(lambda x, y: x + y, tf_idf_s, Value(0.0, output_field=FloatField()))


def fetch_identifier_status(request):
    start = time.time()
    business_process.fetch_identifier_status(
        queryset=business_process.get_identifier_status_unknown(models.Virus.objects),
    )
    business_process.fetch_identifier_status(
        queryset=business_process.get_identifier_status_unknown(models.Host.objects),
    )
    end = time.time()
    return JsonResponse(dict(duration=end - start))


def get_statistics(request):
    if request.GET.get("public", "FALSE").upper() == "TRUE":
        user = AnonymousUser()
    else:
        user = request.user
    return JsonResponse(
        dict(
            virus_count=mixins.only_public_or_granted_or_owned_queryset_filter(
                self=None,
                request=None,
                user=user,
                queryset=models.Virus.objects,
                path_to_data_source="data_source__",
            ).count(),
            host_count=mixins.only_public_or_granted_or_owned_queryset_filter(
                self=None,
                request=None,
                user=user,
                queryset=models.Host.objects,
                path_to_data_source="data_source__",
            ).count(),
            response_count=mixins.only_public_or_granted_or_owned_queryset_filter(
                self=None,
                request=None,
                user=user,
                queryset=models.ViralHostResponseValueInDataSource.objects,
                path_to_data_source="data_source__",
            ).count(),
            data_source_count=mixins.only_public_or_granted_or_owned_queryset_filter(
                self=None,
                request=None,
                user=user,
                queryset=models.DataSource.objects,
                path_to_data_source="",
            ).count(),
        )
    )