Skip to content
GitLab
Commit 7b0877c2 authored by Bryan BRANCOTTE's avatar Bryan BRANCOTTE
Browse files

Merge branch 'master' into wizard_form

parents 016460ab 586f0113
Pipeline #10174 passed with stage
in 11 minutes and 39 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 7b0877c2
......@@ -13,6 +13,7 @@ test-centos7-p34:
- yum install -y wget
- yum update -y
#- yum install -y desktop-file-utils inchi-devel libxml2-devel swig wxGTK-devel ImageMagick perl-devel perl-generators
- yum install -y git
- yum install -y swig # openbabel pip dep
- yum install -y wxBase wxGTK inchi #openbabel rpm deps
- wget https://gitlab.pasteur.fr/hub/openbabel4ippidb/raw/master/rpmbuild/RPMS/x86_64/openbabel-libs-2.4.1-15.PR1829.py3.el7.x86_64.rpm
......
ippisite/ippidb/management/commands/import_v1_data.py
View file @ 7b0877c2
......@@ -2,12 +2,8 @@ import glob
import re
import copy
from django.utils import timezone
from django.db import transaction
from django.core.management import BaseCommand, CommandError
from django_diu.import_command import ImportCommand, ImportTask, ListImportTask, MysqlImportTask, MyConverter
import mysql.connector
import requests_cache
import click
from ippidb.models import Bibliography, Protein, Taxonomy, MolecularFunction, \
Domain, ProteinDomainBoundComplex, ProteinDomainPartnerComplex, Symmetry, Ppi, PpiComplex, Disease, \
......@@ -16,142 +12,6 @@ from ippidb.models import Bibliography, Protein, Taxonomy, MolecularFunction, \
CompoundCytotoxicityResult, CompoundPKResult, PpiFamily
class MyConverter(mysql.connector.conversion.MySQLConverter):
def row_to_python(self, row, fields):
row = super(MyConverter, self).row_to_python(row, fields)
def to_unicode(col):
if type(col) == bytearray:
return col.decode('utf-8')
return col
return[to_unicode(col) for col in row]
class ImportTask(object):
description = "Abstract import task"
option = ""
target_classes = {}
main_class = None
depends_on = []
def __init__(self, command, traceback=False, stop_on_fail=False, progress_bar=False):
self.out_stream = command.stdout
self.err_stream = command.stderr
self.style = command.style
self.traceback = traceback
self.stop_on_fail = stop_on_fail
self.progress_bar = progress_bar
self.done = False
def _flush_target_models(self):
for target_class in self.target_classes:
self.out_stream.write(
'Flushing {target_class} models...'.format(target_class=target_class.__name__))
target_class.objects.all().delete()
self.out_stream.write(self.style.SUCCESS(
'Successfully flushed {target_class} models!'.format(target_class=target_class.__name__)))
def check_final_count(self):
if self.main_class:
count = self.main_class.objects.count()
if count==self.source_count:
self.out_stream.write(self.style.SUCCESS(
'rows count ok for model {}: expected {}, counted {}'.format(self.main_class.__name__, self.source_count, count)))
else:
message = 'rows count not ok for model {}: expected {}, counted {}'.format(
self.main_class.__name__, self.source_count, count)
if self.stop_on_fail:
raise CommandError(message)
else:
self.out_stream.write(self.style.ERROR(message))
def migrate_row(self,row):
raise NotImplementedError()
def open_data_source(self):
raise NotImplementedError()
def post_process(self):
pass
def _process_rows(self, rows):
for row in rows:
try:
new_object = None
with transaction.atomic():
new_object = self.migrate_row(row)
except Exception as e:
if self.traceback:
import traceback
self.err_stream.write(self.style.NOTICE(traceback.format_exc()))
if self.stop_on_fail:
raise CommandError(
'Failed inserting {}'.format(new_object))
else:
self.out_stream.write(self.style.ERROR(
'Failed inserting {}'.format(new_object)))
else:
if self.progress_bar == False:
self.out_stream.write(self.style.SUCCESS(
'Successfully inserted {}'.format(new_object)))
def count_source(self):
self.source_count = len(self.rows)
def _run_import_loop(self):
self.open_data_source()
self.count_source()
if self.progress_bar is True:
with click.progressbar(self.rows,
label='Importing ' + self.description + ' (' + str(self.source_count) + ' rows to process).') as rows_list:
self._process_rows(rows_list)
else:
self._process_rows(self.rows)
self.post_process()
self.check_final_count()
self.done = True
def check(self):
self.open_data_source()
self.count_source()
self.check_final_count()
def run(self):
self._flush_target_models()
self._run_import_loop()
class MysqlImportTask(ImportTask):
description = "Abstract MySQL import task"
outer_sql = ""
def set_mysql_conn(self, conn):
self.conn = conn
def get_cursor(self):
return self.conn.cursor()
def open_data_source(self):
cursor = self.get_cursor()
cursor.execute(self.outer_sql)
self.rows = cursor.fetchall()
class ListImportTask(ImportTask):
description = "Abstract Python list import task"
DATA = []
def open_data_source(self):
self.rows = self.DATA
class SymmetriesImportTask(ListImportTask):
description = "Symmetries import"
......@@ -181,8 +41,13 @@ class SymmetriesImportTask(ListImportTask):
symmetry.save()
return symmetry
class IppiDBMySQLImportTask(MysqlImportTask):
class ProteinsImportTask(MysqlImportTask):
def __init__(self, command, **kwargs):
super().__init__(command, **kwargs)
self.conn = command.ippidb_source_conn
class ProteinsImportTask(IppiDBMySQLImportTask):
description = "Proteins import"
......@@ -202,7 +67,7 @@ class ProteinsImportTask(MysqlImportTask):
return p
class BibliographyImportTask(MysqlImportTask):
class BibliographyImportTask(IppiDBMySQLImportTask):
description = "Bibliography references import"
......@@ -233,7 +98,7 @@ class BibliographyImportTask(MysqlImportTask):
return b
class DomainImportTask(MysqlImportTask):
class DomainImportTask(IppiDBMySQLImportTask):
description = "Domains import"
......@@ -254,7 +119,7 @@ class DomainImportTask(MysqlImportTask):
return d
class CompoundImportTask(MysqlImportTask):
class CompoundImportTask(IppiDBMySQLImportTask):
description = "Compound import"
......@@ -327,7 +192,7 @@ class CompoundImportTask(MysqlImportTask):
return compound
class RefCompoundBiblioImportTask(MysqlImportTask):
class RefCompoundBiblioImportTask(IppiDBMySQLImportTask):
description = "RefCompoundBiblio import"
......@@ -359,7 +224,7 @@ class RefCompoundBiblioImportTask(MysqlImportTask):
return r
class PpiImportTask(MysqlImportTask):
class PpiImportTask(IppiDBMySQLImportTask):
description = "PPIs import"
......@@ -574,7 +439,7 @@ class AdditionalCasImportTask(ListImportTask):
return ca
class TestActivityDescriptionImportTask(MysqlImportTask):
class TestActivityDescriptionImportTask(IppiDBMySQLImportTask):
description = "TestActivityDescription import"
......@@ -616,7 +481,7 @@ class TestActivityDescriptionImportTask(MysqlImportTask):
tad.save()
return tad
class CompoundActivityResultImportTask(MysqlImportTask):
class CompoundActivityResultImportTask(IppiDBMySQLImportTask):
description = "CompoundActivityResult import"
......@@ -646,7 +511,7 @@ class CompoundActivityResultImportTask(MysqlImportTask):
return car
class TestCytotoxDescriptionImportTask(MysqlImportTask):
class TestCytotoxDescriptionImportTask(IppiDBMySQLImportTask):
description = "TestCytotoxDescription import"
......@@ -679,7 +544,7 @@ class TestCytotoxDescriptionImportTask(MysqlImportTask):
tcd.save()
return tcd
class CompoundCytotoxicityResultImportTask(MysqlImportTask):
class CompoundCytotoxicityResultImportTask(IppiDBMySQLImportTask):
description = "CompoundCytotoxicityResult import"
......@@ -706,7 +571,7 @@ class CompoundCytotoxicityResultImportTask(MysqlImportTask):
return ccr
class TestPKDescriptionImportTask(MysqlImportTask):
class TestPKDescriptionImportTask(IppiDBMySQLImportTask):
description = "TestPKDescription import"
......@@ -750,7 +615,7 @@ class TestPKDescriptionImportTask(MysqlImportTask):
return tpd
class CompoundPKResultImportTask(MysqlImportTask):
class CompoundPKResultImportTask(IppiDBMySQLImportTask):
description = "CompoundPKResult import"
......@@ -784,7 +649,7 @@ class CompoundPKResultImportTask(MysqlImportTask):
cpr.save()
return cpr
class Command(BaseCommand):
class Command(ImportCommand):
help = "Import iPPI-DB data from the MySQL database"
task_classes = [ProteinsImportTask,
......@@ -803,93 +668,6 @@ class Command(BaseCommand):
CompoundPKResultImportTask
]
def add_arguments(self, parser):
task_choices = [task_class.option for task_class in self.task_classes]
task_choices.append('all')
task_help = 'Import task to be run.\n '
task_help += ',\n \n'.join([task_class.option + ': ' + task_class.description for task_class in self.task_classes])
task_help += ',\n \nall: import everything.'
parser.add_argument(
'task',
type=str,
choices=task_choices,
help=task_help,
)
parser.add_argument(
'--all',
action='store_true',
dest='all',
default=False,
help='import everything',
)
parser.add_argument(
'--wscache',
action='store_true',
dest='wscache',
default=False,
help='use web services cache',
)
parser.add_argument(
'--errortb',
action='store_true',
dest='errortb',
default=False,
help='show tracebacks on errors',
)
parser.add_argument(
'--stoponfail',
action='store_true',
dest='stoponfail',
default=False,
help='stop on first error',
)
parser.add_argument(
'--progressbar',
action='store_true',
dest='progress_bar',
default=False,
help='show progressbar instead of logging inserted entries',
)
parser.add_argument(
'--check',
action='store_true',
dest='check',
default=False,
help='only check instead of running import',
)
def handle(self, *args, **options):
conn = mysql.connector.connect(
converter_class=MyConverter, host="localhost", user="root", password="ippidb", database="ippidb")
if options.get('wscache'):
requests_cache.install_cache('ws_cache')
task_option = options.get('task')
# map task names to task classes
option_to_task = {task_class.option: task_class for task_class in self.task_classes}
# map task names to the list of task names that depend upon them
dependency_to_task = {task_class.option: [] for task_class in self.task_classes}
for task_class in self.task_classes:
for task_depency_class in task_class.depends_on:
dependency_to_task[task_depency_class.option].append(task_class.option)
dependency_to_task['all'] = option_to_task.keys()
def append_dependencies(option):
dependencies = set([option])
for depending_option in dependency_to_task.get(option,[]):
dependencies.update(append_dependencies(depending_option))
return dependencies
if task_option:
# list of tasks to run because they depend on initial option,
# directly or not
dependencies = append_dependencies(task_option)
print(dependencies)
dependencies_classes = {task_class for opt, task_class in option_to_task.items() if opt in dependencies}
conn = mysql.connector.connect(
converter_class=MyConverter, host="localhost", user="root", password="ippidb", database="ippidb")
while len(dependencies_classes)>0:
for task_class in copy.copy(dependencies_classes):
if not(set(task_class.depends_on) & dependencies_classes):
task = task_class(self, options['errortb'], options['stoponfail'], options['progress_bar'])
if hasattr(task, 'set_mysql_conn'):
task.set_mysql_conn(conn)
task.run()
dependencies_classes.remove(task_class)
self.ippidb_source_conn = mysql.connector.connect(
converter_class=MyConverter, host="localhost", user="root", password="ippidb", database="ippidb")
\ No newline at end of file
ippisite/requirements-core.txt
View file @ 7b0877c2
......@@ -18,5 +18,6 @@ scipy
matplotlib==2.2.3 #Matplotlib 3.0+ does not support Python 2.x, 3.0, 3.1, 3.2, 3.3, or 3.4.
# postgres driver
psycopg2
git+https://gitlab.pasteur.fr/hmenager/django-diu.git#egg=django_diu
# openbabel is not installed from pip for now
#openbabel
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment