Commit e0cd6c6c authored by Hervé  MENAGER's avatar Hervé MENAGER
Browse files

add style checks and black formatting checks to CI

basically, this adds:
- a CI step to run flake8, _including_ the black plugin to check
  whether or not files need black reformatting
- add flake8 and black to dependencies
- reformat files and correct style so that current code passes
parent 666c419a
stages:
- test
- deploy
test-style:
image: python:3.6
stage: test
script:
- cd ippisite
- pip install -r requirements-dev.txt
- flake8 --config=.flake8
test-centos7-p34:
image: centos:centos7
stage: test
......@@ -56,4 +63,4 @@ deploy-webserver:
- whoami
- ansible-playbook -vvv -i ./hosts_deploy deploy.yaml --extra-vars "deploy_user_name=ippidb repo_api_token=JZS-4cH7bWkFkHa2rAVf marvinjs_apikey=$MARVINJS_APIKEY galaxy_base_url=$GALAXY_BASE_URL galaxy_apikey=$GALAXY_APIKEY galaxy_compoundproperties_workflowid=$GALAXY_COMPOUNDPROPERTIES_WORKFLOWID secret_key=$SECRET_KEY dbname=$DBNAME dbuser=$DBUSER dbpassword=$DBPASSWORD dbhost=$DBHOST dbport=$DBPORT"
only:
- master
\ No newline at end of file
- master
[flake8]
exclude = .git,*migrations*
max-line-length = 88
ignore = E501, W503
exclude = */migrations/*.py, manage.py, docs/source/conf.py
"""
iPPI-DB django admin module
iPPI-DB django admin module
"""
from django.apps import apps
from django.contrib import admin
from django.contrib import messages
from django.contrib.admin.options import get_content_type_for_model
from django.contrib.auth import get_user_model
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.utils.html import format_html
from django.utils.translation import ugettext, ugettext_lazy
from .models import *
from .models import (
Bibliography,
Protein,
Taxonomy,
Domain,
reverse,
ProteinDomainBoundComplex,
ProteinDomainPartnerComplex,
Symmetry,
Compound,
TestActivityDescription,
Ppi,
ProteinDomainComplex,
Contribution,
)
class ViewOnSiteModelAdmin(admin.ModelAdmin):
class Media:
css = {
'all': ('https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css',)
"all": (
"https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css",
)
}
def __init__(self, model, admin_site):
if callable(getattr(model, "get_absolute_url", None)) and callable(self.view_on_site_in_list):
self.list_display += ('view_on_site_in_list',)
if callable(getattr(model, "get_absolute_url", None)) and callable(
self.view_on_site_in_list
):
self.list_display += ("view_on_site_in_list",)
super().__init__(model, admin_site)
def view_on_site_in_list(self, obj):
return format_html(
'<center><a href="' + reverse('admin:view_on_site', kwargs={
'content_type_id': get_content_type_for_model(obj).pk,
'object_id': obj.pk
}) + '"><i class="fa fa-external-link"></i></a><center>')
'<center><a href="'
+ reverse(
"admin:view_on_site",
kwargs={
"content_type_id": get_content_type_for_model(obj).pk,
"object_id": obj.pk,
},
)
+ '"><i class="fa fa-external-link"></i></a><center>'
)
view_on_site_in_list.short_description = format_html('<center>' + ugettext('View on site') + '<center>')
view_on_site_in_list.short_description = format_html(
"<center>" + ugettext("View on site") + "<center>"
)
@admin.register(Bibliography)
class BibliographyAdmin(ViewOnSiteModelAdmin):
list_display = ('authors_list', 'title',
'journal_name', 'biblio_year', 'id_source')
list_display = ("authors_list", "title", "journal_name", "biblio_year", "id_source")
@admin.register(Protein)
class ProteinAdmin(admin.ModelAdmin):
list_display = ('uniprot_id', 'recommended_name_long')
filter_horizontal = ('molecular_functions', 'domains', )
list_display = ("uniprot_id", "recommended_name_long")
filter_horizontal = ("molecular_functions", "domains")
search_fields = (
'uniprot_id',
'recommended_name_long',
'short_name',
'gene_name',
'entry_name',
"uniprot_id",
"recommended_name_long",
"short_name",
"gene_name",
"entry_name",
)
list_filter = ('organism',)
list_filter = ("organism",)
@admin.register(Taxonomy)
class TaxonomyAdmin(admin.ModelAdmin):
list_display = ('taxonomy_id', 'name')
list_display = ("taxonomy_id", "name")
@admin.register(Domain)
class DomainAdmin(admin.ModelAdmin):
list_display = ('pfam_acc', 'pfam_id', 'pfam_description', 'domain_family')
list_display = ("pfam_acc", "pfam_id", "pfam_description", "domain_family")
@admin.register(ProteinDomainBoundComplex)
class ProteinDomainBoundComplexAdmin(ViewOnSiteModelAdmin):
list_display = ('protein', 'domain', 'ppc_copy_nb', 'ppp_copy_nb_per_p')
list_display_links = (
'protein', 'domain', 'ppc_copy_nb', 'ppp_copy_nb_per_p')
list_display = ("protein", "domain", "ppc_copy_nb", "ppp_copy_nb_per_p")
list_display_links = ("protein", "domain", "ppc_copy_nb", "ppp_copy_nb_per_p")
@admin.register(ProteinDomainPartnerComplex)
class ProteinDomainPartnerComplexAdmin(ViewOnSiteModelAdmin):
list_display = ('protein', 'domain', 'ppc_copy_nb')
list_display_links = ('protein', 'domain', 'ppc_copy_nb')
list_display = ("protein", "domain", "ppc_copy_nb")
list_display_links = ("protein", "domain", "ppc_copy_nb")
@admin.register(Symmetry)
class Symmetry(admin.ModelAdmin):
list_display = ('code', 'description')
list_display = ("code", "description")
@admin.register(Compound)
class Compound(ViewOnSiteModelAdmin):
list_display = ('id', 'iupac_name', 'common_name', 'canonical_smile')
search_fields = ('id', 'iupac_name', 'common_name', 'canonical_smile')
list_display = ("id", "iupac_name", "common_name", "canonical_smile")
search_fields = ("id", "iupac_name", "common_name", "canonical_smile")
@admin.register(TestActivityDescription)
class TextActivityDescription(ViewOnSiteModelAdmin):
list_display = ('test_name', 'test_type', 'test_modulation_type')
list_display = ("test_name", "test_type", "test_modulation_type")
@admin.register(Ppi)
class PpiModelAdmin(ViewOnSiteModelAdmin):
filter_horizontal = ('diseases',)
list_display = ('pdb_id', 'name', 'symmetry', 'family')
list_filter = ('diseases',)
search_fields = ('pdb_id', 'name', 'symmetry', 'family__name', 'diseases__name')
filter_horizontal = ("diseases",)
list_display = ("pdb_id", "name", "symmetry", "family")
list_filter = ("diseases",)
search_fields = ("pdb_id", "name", "symmetry", "family__name", "diseases__name")
@admin.register(ProteinDomainComplex)
class ProteinDomainComplexAdmin(ViewOnSiteModelAdmin):
search_fields = ('id', 'protein__id', 'domain__id', 'protein__uniprot_id', 'domain__pfam_acc',)
list_display = ('id', '__str__', 'protein_uniprot_id', 'domain_pfam_acc',)
search_fields = (
"id",
"protein__id",
"domain__id",
"protein__uniprot_id",
"domain__pfam_acc",
)
list_display = ("id", "__str__", "protein_uniprot_id", "domain_pfam_acc")
def protein_uniprot_id(self, o):
return o.protein.uniprot_id
protein_uniprot_id.short_description = 'protein'
protein_uniprot_id.admin_order_field = 'protein__uniprot_id'
protein_uniprot_id.short_description = "protein"
protein_uniprot_id.admin_order_field = "protein__uniprot_id"
def domain_pfam_acc(self, o):
return o.domain.pfam_acc
domain_pfam_acc.short_description = 'Domain'
domain_pfam_acc.admin_order_field = 'domain__pfam_acc'
domain_pfam_acc.short_description = "Domain"
domain_pfam_acc.admin_order_field = "domain__pfam_acc"
for model in apps.get_app_config('ippidb').models.values():
for model in apps.get_app_config("ippidb").models.values():
try:
admin.site.register(model, ViewOnSiteModelAdmin)
except admin.sites.AlreadyRegistered as are:
except admin.sites.AlreadyRegistered:
continue
admin.site.site_header = 'iPPI-DB website administration'
admin.site.site_header = "iPPI-DB website administration"
admin.site.site_title = admin.site.site_header
admin.site.index_header = 'iPPI-DB administration'
admin.site.index_header = "iPPI-DB administration"
admin.site.index_title = admin.site.index_header
......@@ -138,8 +170,7 @@ admin.site.index_title = admin.site.index_header
def grant_contribution_permission(modeladmin, request, queryset, revoke=False):
content_type = ContentType.objects.get_for_model(Contribution)
permission = Permission.objects.get(
codename='add_contribution',
content_type=content_type,
codename="add_contribution", content_type=content_type
)
for o in queryset:
if revoke:
......@@ -149,7 +180,7 @@ def grant_contribution_permission(modeladmin, request, queryset, revoke=False):
pointless = queryset.filter(Q(Q(is_superuser=True) | Q(is_staff=True))).count()
count = queryset.count()
msg = 'Permission %s for %i user(s). '
msg = "Permission %s for %i user(s). "
if revoke:
msg = msg % ("revoked", count - pointless)
......@@ -164,7 +195,8 @@ def grant_contribution_permission(modeladmin, request, queryset, revoke=False):
if pointless:
modeladmin.message_user(
request,
"Note that %i users are staff/superuser and thus always have the permission to contribute" % pointless,
"Note that %i users are staff/superuser and thus always have the permission to contribute"
% pointless,
messages.WARNING,
)
......@@ -184,17 +216,21 @@ revoke_contribution_permission.short_description = ugettext_lazy(
class MyUserAdmin(UserAdmin):
actions = [
grant_contribution_permission,
revoke_contribution_permission,
]
list_display = ('username', 'email', 'first_name', 'last_name', 'is_staff', 'can_contribute')
actions = [grant_contribution_permission, revoke_contribution_permission]
list_display = (
"username",
"email",
"first_name",
"last_name",
"is_staff",
"can_contribute",
)
def can_contribute(self, object):
return object.has_perm('ippidb.add_contribution')
return object.has_perm("ippidb.add_contribution")
can_contribute.boolean = True
admin.site.unregister(get_user_model())
admin.site.register(get_user_model(), MyUserAdmin)
\ No newline at end of file
admin.site.register(get_user_model(), MyUserAdmin)
......@@ -6,4 +6,4 @@ from django.apps import AppConfig
class IppidbConfig(AppConfig):
name = 'ippidb'
name = "ippidb"
This diff is collapsed.
......@@ -2,20 +2,19 @@
iPPI-DB-Galaxy communication module
"""
from bioblend.galaxy import GalaxyInstance
from bioblend.galaxy.tools.inputs import inputs
workflow_id = 'dad6103ff71ca4fe'
galaxy_url = 'https://galaxy-dev.web.pasteur.fr'
api_key = '21c2ce387688b1a785040762f7c9c331'
workflow_id = "dad6103ff71ca4fe"
galaxy_url = "https://galaxy-dev.web.pasteur.fr"
api_key = "21c2ce387688b1a785040762f7c9c331"
def run_workflow_and_get_results(input_file):
gi = GalaxyInstance(galaxy_url, key=api_key)
gi.verify = False
history_id = gi.histories.create_history("ippidb_history")['id']
dataset_id = gi.tools.upload_file(
input_file, history_id)['outputs'][0]['id']
inputs = {'0': {'id': dataset_id, 'src': 'hda'}}
history_id = gi.histories.create_history("ippidb_history")["id"]
dataset_id = gi.tools.upload_file(input_file, history_id)["outputs"][0]["id"]
inputs = {"0": {"id": dataset_id, "src": "hda"}}
workflow_run = gi.workflows.invoke_workflow(
workflow_id, inputs=inputs, history_id=history_id)
workflow_id, inputs=inputs, history_id=history_id
)
print(workflow_run)
......@@ -17,4 +17,3 @@ class Command(BaseCommand):
"Successfully generated compound properties cache for %s compounds" % n
)
)
import argparse
from datetime import datetime
import decimal
from itertools import islice
import json
import os
import re
import time
import tempfile
......@@ -18,17 +16,16 @@ import requests
from ippidb.models import Compound
from ippidb.utils import smi2sdf
# disable insecure HTTP request warnings (used by bioblend)
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# configuration FIXME!!!
BASE_URL = settings.GALAXY_BASE_URL
KEY = settings.GALAXY_APIKEY
WORKFLOW_ID = settings.GALAXY_COMPOUNDPROPERTIES_WORKFLOWID
# disable insecure HTTP request warnings (used by bioblend)
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
class GalaxyCompoundPropertiesRunner(object):
def __init__(self, galaxy_instance):
......@@ -77,7 +74,6 @@ class GalaxyCompoundPropertiesRunner(object):
WORKFLOW_ID, inputs=dataset_map, history_id=history_id
)
workflow_job_id = workflow_job["id"]
workflow_id = workflow_job["workflow_id"]
while workflow_job["state"] not in ["ok", "scheduled"]:
time.sleep(2)
workflow_job = self.galaxy_instance.workflows.show_invocation(
......@@ -122,15 +118,19 @@ def idrange_type(s, pat=re.compile(r"^(\d+)-(\d+)$")):
)
return (int(m.groups()[0]), int(m.groups()[1]))
def dec(decimal_places):
def func(number):
return round(float(number), decimal_places)
return func
def chunks(data, size=10):
it = iter(data)
for i in range(0, len(data), size):
yield {k:data[k] for k in islice(it, size)}
yield {k: data[k] for k in islice(it, size)}
class Command(BaseCommand):
......@@ -170,39 +170,42 @@ class Command(BaseCommand):
compounds = []
pc_properties = {}
already_done_ids = []
if options['json'] is not None:
pc_properties_dict = json.load(open(options['json'].name,'r'))
ids = [int(key) for key, item in pc_properties_dict.items() if 'IUPAC' not in item]
already_done_ids = [int(key) for key, item in pc_properties_dict.items() if 'IUPAC' in item]
if options['all'] is True:
ids = Compound.objects.all().values('id')
elif options['ids']:
ids = Compound.objects.filter(
id__in=options['ids']).values('id')
elif options['idrange']:
if options["json"] is not None:
pc_properties_dict = json.load(open(options["json"].name, "r"))
ids = [
int(key)
for key, item in pc_properties_dict.items()
if "IUPAC" not in item
]
already_done_ids = [
int(key) for key, item in pc_properties_dict.items() if "IUPAC" in item
]
if options["all"] is True:
ids = Compound.objects.all().values("id")
elif options["ids"]:
ids = Compound.objects.filter(id__in=options["ids"]).values("id")
elif options["idrange"]:
ids = Compound.objects.filter(
id__gte=options['idrange'][0], id__lte=options['idrange'][1]
).values('id')
id__gte=options["idrange"][0], id__lte=options["idrange"][1]
).values("id")
else:
compounds = Compound.objects.filter(
iupac_name__isnull=True
).values('id')
ids = [row['id'] for row in ids]
ids = list(set(ids)-set(already_done_ids))
compounds = Compound.objects.filter(iupac_name__isnull=True).values("id")
ids = [row["id"] for row in ids]
ids = list(set(ids) - set(already_done_ids))
compounds = Compound.objects.filter(id__in=ids)
for c in compounds:
smiles_dict[c.id] = c.canonical_smile
# create or reuse existing JSON file to save new results
if options['json']:
json_file = options['json'].name
if options["json"]:
json_file = options["json"].name
else:
json_fh = tempfile.NamedTemporaryFile(mode="w", delete=False)
json.dump({c.id:{} for c in compounds},json_fh)
json.dump({c.id: {} for c in compounds}, json_fh)
json_file = json_fh.name
json_fh.close()
self.stderr.write(self.style.SUCCESS(f"Compound properties file: {json_file}"))
if len(compounds)>0:
self.stderr.write(f'Now processing {len(compounds)} compounds')
if len(compounds) > 0:
self.stderr.write(f"Now processing {len(compounds)} compounds")
# set up Galaxy computation environment
gi = GalaxyInstance(url=BASE_URL, key=KEY, verify=False)
gi.nocache = True
......@@ -214,16 +217,26 @@ class Command(BaseCommand):
fh = tempfile.NamedTemporaryFile(mode="w", delete=False)
fh.write(sdf_string)
fh.close()
self.stderr.write(self.style.SUCCESS(f"Galaxy input SDF file for compounds {smiles_dict_chunk.keys()}: {fh.name}"))
self.stderr.write(
self.style.SUCCESS(
f"Galaxy input SDF file for compounds {smiles_dict_chunk.keys()}: {fh.name}"
)
)
# run computations on Galaxy
pc_properties = runner.compute_properties_for_sdf_file(fh.name)
new_pc_properties_dict = {compound['Name']: compound for compound in pc_properties}
pc_properties_dict = json.load(open(json_file,'r'))
new_pc_properties_dict = {
compound["Name"]: compound for compound in pc_properties
}
pc_properties_dict = json.load(open(json_file, "r"))
pc_properties_dict.update(new_pc_properties_dict)
fh = open(json_file,'w')
fh = open(json_file, "w")
json.dump(pc_properties_dict, fh, indent=4)
fh.close()
self.stderr.write(self.style.SUCCESS(f"Properties added for compounds {smiles_dict_chunk.keys()} in JSON file: {json_file}"))
self.stderr.write(
self.style.SUCCESS(
f"Properties added for compounds {smiles_dict_chunk.keys()} in JSON file: {json_file}"
)
)
# report and update database
property_mapping = {
"CanonicalSmile": ("canonical_smile", str),
......@@ -295,7 +308,7 @@ class Command(BaseCommand):
except ValueError as ve:
self.stderr.write(
self.style.ERROR(
f"Error setting property {ippidb_prop} to {item[galaxy_prop]} in compound {compound.id}"
f"Error setting property {ippidb_prop} to {item[galaxy_prop]} in compound {compound.id} \ndetails:{ve}"
)
)
properties_list.append(ippidb_dict)
......
......@@ -231,9 +231,6 @@ class RefCompoundBiblioImportTask(IppiDBMySQLImportTask):
depends_on = [BibliographyImportTask, CompoundImportTask]
def open_data_source(self):
self.rows = cursor.fetchall()
def migrate_row(self, row):
c = Compound.objects.get(canonical_smile=row[1])
b = Bibliography.objects.get(id_source=row[2])
......@@ -253,7 +250,7 @@ class PpiImportTask(IppiDBMySQLImportTask):
description = "PPIs import"
outer_sql = """
select distinct protein.NumUniprot, domain.PfamNumAccession, complexe.NbCopy,
select distinct protein.NumUniprot, domain.PfamNumAccession, complexe.NbCopy,
cmpdAction.IDComplexeBound, bindingSiteEvidence.CodePDB,
'part1', ppi.IDPPI, disease.Disease, complexe.IDComplexe, ppi.Family
from bindingSite inner join ppi
......@@ -296,8 +293,6 @@ class PpiImportTask(IppiDBMySQLImportTask):
depends_on = [ProteinsImportTask, DomainImportTask, CompoundImportTask]
def migrate_row(self, row):
cursor_aux = self.get_cursor()
cursor_aux2 = self.get_cursor()
# create or retrieve Ppi object
if row[5] == "part1":
ppi = Ppi()
......
......@@ -2,7 +2,6 @@ import json
import io
import base64
import itertools
import math
from django.core.management import BaseCommand
from django.forms.models import model_to_dict
......@@ -12,16 +11,15 @@ import numpy as np
import pandas as pd
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from scipy.spatial import distance
from ippidb.models import Compound, PcaBiplotData
def plot_circle():
theta = np.linspace(0, 2*np.pi, 100)
theta = np.linspace(0, 2 * np.pi, 100)
r = np.sqrt(1.0)
x1 = r*np.cos(theta)
x2 = r*np.sin(theta)
x1 = r * np.cos(theta)
x2 = r * np.sin(theta)
return x1, x2
......@@ -60,62 +58,69 @@ class Command(BaseCommand):
values_list.append(values)
df = pd.DataFrame(values_list)
# drop compounds with undefined property values
df.dropna(how='any', inplace=True)
df.dropna(how="any", inplace=True)
# prepare correlation circle figure
plt.switch_backend("Agg")
fig_, ax = plt.subplots(figsize=(6, 6))
x1, x2 = plot_circle()
plt.plot(x1, x2)
ax.set_aspect(1)
ax.yaxis.set_label_coords(-0.1,0.5)
ax.xaxis.set_label_coords(0.5,-0.08)
ax.yaxis.set_label_coords(-0.1, 0.5)
ax.xaxis.set_label_coords(0.5, -0.08)
# do not process the data unless there are compounds in the dataframe
if len(df.index)>0:
if len(df.index) > 0:
x = df.loc[:, features].values
y = df.loc[:, ["family"]].values
x = StandardScaler().fit_transform(x)
n=x.shape[0] # retrieve number of individuals
p=x.shape[1] # retrieve number of variables
n = x.shape[0] # retrieve number of individuals
p = x.shape[1] # retrieve number of variables
pca = PCA(n_components=p)
principal_components = pca.fit_transform(x)
# compute correlation circle
variance_ratio = pd.Series(pca.explained_variance_ratio_)
coef = np.transpose(pca.components_)