diff --git a/jass/__main__.py b/jass/__main__.py index 743b290525e685bba88ef410b2283158cebfc795..69d18b64501dc05ee6a6fd4c023f393b49d48813 100644 --- a/jass/__main__.py +++ b/jass/__main__.py @@ -4,7 +4,7 @@ import os import argparse -from jass.server import get_jass_app +from jass.server import jass_app from jass.config import config from jass.models.phenotype import get_available_phenotypes @@ -62,8 +62,7 @@ def absolute_path_of_the_file(fileName, output_file = False): def serve(args): - app = get_jass_app() - app.run(host=config["HOST"], port=config["PORT"]) + jass_app.flask_app.run(host=config["HOST"], port=config["PORT"]) def w_list_phenotypes(args): diff --git a/jass/controllers/__init__.py b/jass/controllers/__init__.py index 0e94abd262a9ad33a4968618f3ba933bf97c6740..38c03398ba0b04686bea26fed970650be5a16167 100644 --- a/jass/controllers/__init__.py +++ b/jass/controllers/__init__.py @@ -7,5 +7,4 @@ Submodules .. autosummary:: :toctree: _autosummary - default_controller """ diff --git a/jass/controllers/default_controller.py b/jass/controllers/default_controller.py deleted file mode 100644 index f447e63262de6c024fabcc61ff85c18c1d35abae..0000000000000000000000000000000000000000 --- a/jass/controllers/default_controller.py +++ /dev/null @@ -1,172 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -default_controller ensures the connection between the web interface and the Python JASS-analysis module -""" - -from jass.config import config -from jass.models.project import Project, create_project -from jass.models.phenotype import Phenotype, get_available_phenotypes -import connexion -from flask import send_file, abort -from six import iteritems -import os -from typing import List, Dict - -PHENOTYPES = get_available_phenotypes( - os.path.join(config["DATA_DIR"], "initTable.hdf5") -) # FIXME part of the config - - -def phenotypes_get(): - """ - phenotypes_get - Gets the list of available phenotypes - - :rtype: List[Phenotype] - """ - return PHENOTYPES - - -def projects_post(phenotypeID): - """ - projects_post - Create a new project from a selection of phenotypes - :param phenotypeID: IDs of the phenotypes selected for the project - :type phenotypeID: List[str] - - :rtype: str - """ - return create_project(phenotypeID, PHENOTYPES) - - -def projects_project_id_csv_status_get(projectID): - """ - projects_project_id_csv_status_get - Retrieve the generation status of the genome full csv file - :param projectID: project ID - :type projectID: str - - :rtype: str - """ - return Project(id=projectID).get_csv_file_generation() - - -def projects_project_id_summary_statistics(projectID): - """ - projects_project_id_summary_statistics - Retrieve project summary statistics - """ - return Project(id=projectID).get_project_summary_statistics() - - -def projects_project_id_genome_get(projectID, threshold=None): - """ - projects_project_id_genome_get - Retrieve genome data for a given project - :param projectID: project ID - :type projectID: str - - :rtype: str - """ - return Project(id=projectID).get_project_genomedata() - - -def projects_project_id_global_manhattan_plot_get(projectID): - """ - projects_project_id_global_manhattan_plot_get - Gets the global Manhattan plot stored in the Project folder to display it on the Web interface - """ - try: - return send_file( - Project(id=projectID).get_global_manhattan_plot_path(), mimetype="image/png" - ) - except FileNotFoundError: - status = Project(id=projectID).status - if status == Project.DOES_NOT_EXIST: - abort(404) - elif status["global_manhattan"] == Project.CREATING: - return ( - "Plot is not ready yet", - 202, - {"Content-Type": "text/plain; charset=utf-8"}, - ) - else: - abort(500) - - -def projects_project_id_quadrant_plot_get(projectID): - """ - projects_project_id_quadrant_plot_get - Gets the quadrant plot stored in the Project folder to display it on the Web interface - """ - try: - return send_file( - Project(id=projectID).get_quadrant_plot_path(), mimetype="image/png" - ) - except FileNotFoundError: - status = Project(id=projectID).status - if status == Project.DOES_NOT_EXIST: - abort(404) - elif status["quadrant_plot_status"] == Project.CREATING: - return ( - "Plot is not ready yet", - 202, - {"Content-Type": "text/plain; charset=utf-8"}, - ) - else: - abort(500) - - -def projects_project_id_genome_full_get(projectID): - """ - projects_project_id_genome_full_get - Downloads the file genome_full.csv stored in the Project folder - """ - try: - return send_file( - Project(id=projectID).get_csv_path(), - mimetype="text/csv", - as_attachment=True, - attachment_filename="genome_full.csv" - ) - except FileNotFoundError: - status = Project(id=projectID).status - if status == Project.DOES_NOT_EXIST: - abort(404) - elif status["worktable"] == Project.CREATING: - return ( - "CSV is not ready yet", - 202, - {"Content-Type": "text/plain; charset=utf-8"}, - ) - else: - abort(500) - - -def projects_project_id_local_manhattan_data_get(projectID, chromosome, region): - """ - projects_project_id_local_manhattan_data_get - Return the SumStatTab dataframe of the Project for a given chromosome and region for the Manhattan plot - """ - return Project(id=projectID).get_project_local_manhattan_data(chromosome, region) - - -def projects_project_id_local_heatmap_data_get(projectID, chromosome, region): - """ - projects_project_id_local_heatmap_data_get - Return the SumStatTab dataframe of the Project for a given chromosome and region for the Heatmap plot - """ - return Project(id=projectID).get_project_local_heatmap_data(chromosome, region) - - -def projects_project_idget(projectID): - """ - projects_project_idget - Retrieve a project definition - :param projectID: project ID - :type projectID: str - - :rtype: Phenotype - """ - return Project(id=projectID) diff --git a/jass/encoder.py b/jass/encoder.py deleted file mode 100644 index 175502a4df6e905113b701e38214b296e82af577..0000000000000000000000000000000000000000 --- a/jass/encoder.py +++ /dev/null @@ -1,25 +0,0 @@ -from connexion.apps.flask_app import FlaskJSONEncoder - -# from connexion.decorators import produces -from six import iteritems -from jass.models.base_model_ import Model -from pandas import isnull - - -class JSONEncoder(FlaskJSONEncoder): - include_nulls = False - - def default(self, o): - if isinstance(o, Model): - dikt = {} - for attr, _ in iteritems(o.swagger_types): - value = getattr(o, attr) - if not (isinstance(value, list)) and (value is None or isnull(value)): - if not self.include_nulls: - continue - else: - return None - attr = o.attribute_map[attr] - dikt[attr] = value - return dikt - return produces.JSONEncoder.default(self, o) diff --git a/jass/models/project.py b/jass/models/project.py index 86d84fd67007687a84e205385ae732fdc8a05f81..438c4d8f807d3ee442099bc93f9b70cea5f86b8b 100644 --- a/jass/models/project.py +++ b/jass/models/project.py @@ -1,404 +1,404 @@ -# -*- coding: utf-8 -*- -""" -compute joint statistics and generate plots for a given set of phenotypes -""" -from __future__ import absolute_import -from typing import List, Dict -import os, sys -import shutil -import hashlib -import traceback - -from celery import Celery - -from jass.models.base_model_ import Model -from jass.util import deserialize_model -from jass.models.phenotype import Phenotype -from jass.models.worktable import ( - create_worktable_file, - get_worktable_summary, - get_worktable_genomedata, - get_worktable_local_manhattan_data, - get_worktable_local_heatmap_data, - create_genome_full_csv -) -from jass.models.plots import create_global_plot, create_quadrant_plot -from jass.config import config - -app = Celery("tasks", broker="pyamqp://guest@localhost//") - - -class Project(Model): - - DOES_NOT_EXIST = "DOES_NOT_EXIST" - - CREATING = "CREATING" - - READY = "READY" - - ERROR = "ERROR" - - def __init__(self, id: str = None, phenotypes: List[Phenotype] = None): - """ - Project - a project (list of phenotypes) - - :param id: project ID. - :type id: str - """ - self.swagger_types = {"id": str, - "status": str, - "phenotypes": List[Phenotype], - "progress": str} - - self.attribute_map = { - "id": "id", - "status": "status", - "phenotypes": "phenotypes", - "progress": "progress", - } - - self._id = id - self._phenotypes = phenotypes - if self._id is None: - self._id = self.get_id() - - @classmethod - def from_dict(cls, dikt) -> "Project": - """ - Returns the dict as a model - - :param dikt: A dict. - :type: dict - :return: The Project. - :rtype: Project - """ - return deserialize_model(dikt, cls) - - @property - def id(self) -> str: - """ - Gets the id of this Project. - - :return: The id of this Project. - :rtype: str - """ - return self._id - - @id.setter - def id(self, id: str): - """ - Lists the id of this Project. - - :param id: The id of this Project. - :type id: str - """ - - self._id = id - - @property - def phenotypes(self) -> List[Phenotype]: - """ - Gets the phenotypes list for this project. - - :return: The phenotypes. - :rtype: str - """ - return self._phenotypes - - @phenotypes.setter - def cohort(self, phenotypes: List[Phenotype]): - """ - Lists the phenotypes list for this project. - - :param phenotypes: The phenotypes. - :type phenotypes: str - """ - - self._phenotypes = phenotypes - - def get_folder_path(self): - """ - get_folder_path - Gets the path of the folder where the project data are stored - """ - return os.path.join(config["DATA_DIR"], "project_{}".format(self.id)) - - def get_worktable_path(self): - """ - get_worktable_path - Gets the path of the file workTable.hdf5 - """ - return os.path.join(self.get_folder_path(), "workTable.hdf5") - - def get_csv_path(self): - """ - get_csv_path - Gets the path of the file genome_full.csv - """ - return os.path.join(self.get_folder_path(), "workTable.csv") - - def get_progress_path(self): - """ - get_progress_path - Gets the path of the file containing the current progress percentage of \ - the analysis performed within the project - """ - return os.path.join(self.get_folder_path(), "JASS_progress.txt") - - def get_csv_lock_path(self): - """ - get_csv_lock_path - Gets the path of the lock set-on when the csv file is not available yet - """ - return os.path.join(self.get_folder_path(), "the_lock.txt") - - def get_project_summary_statistics(self): - return get_worktable_summary(self.get_worktable_path()) - - def get_project_genomedata(self): - return get_worktable_genomedata(self.get_worktable_path()) - - def get_project_local_manhattan_data(self, chromosome: str, region: str): - return get_worktable_local_manhattan_data( - self.get_worktable_path(), chromosome, region - ) - - def get_project_local_heatmap_data(self, chromosome: str, region: str): - return get_worktable_local_heatmap_data( - self.get_worktable_path(), chromosome, region - ) - - def get_id(self): - m = hashlib.md5() - for phenotype_id in [phenotype.id for phenotype in self._phenotypes]: - m.update(str(phenotype_id).encode("utf-8")) - return m.hexdigest() - - def get_global_manhattan_plot_path(self): - return os.path.join(self.get_folder_path(), "Manhattan_Plot_Omnibus.png") - - def get_quadrant_plot_path(self): - return os.path.join(self.get_folder_path(), "Quadrant_Plot_Omnibus.png") - - @property - def status(self): - """ - status - Gets the status of the project - """ - if not os.path.exists(self.get_folder_path()): - return Project.DOES_NOT_EXIST - else: - worktable_status = get_file_status(self.get_worktable_path()) - global_manhattan_status = get_file_status( - self.get_global_manhattan_plot_path() - ) - quadrant_plot_status = get_file_status(self.get_quadrant_plot_path()) - return { - # WARNING: project status is hacked so that everything is ready - # only once the final step has completed. - # This avoids the apparent "corrupted hdf5" file situation - # "worktable": worktable_status, - # "global_manhattan": global_manhattan_status, - "worktable": quadrant_plot_status, - "global_manhattan": quadrant_plot_status, - "quadrant_plot_status": quadrant_plot_status, - } - - @property - def progress(self): - """ - progress - Gets the percentage of completion of the phenotype analysis - """ - JASS_progress = 0 - progress_path = self.get_progress_path() - if os.path.exists(progress_path): - file_progress = open(progress_path, "r") - JASS_progress = file_progress.read() - file_progress.close() - return JASS_progress - - def get_csv_file_generation(self): - """ - csv_file_generation - Gets the status of the genome_full csv file generation - """ - the_lock_path = self.get_csv_lock_path() - csv_file = self.get_csv_path() - csv_file_status = Project.CREATING - if (not os.path.isfile(the_lock_path)): - if(os.path.isfile(csv_file)): - csv_file_status = Project.READY - else : - csv_file_status = Project.ERROR - print("csv_file_generation:csv_file_status={}".format(csv_file_status)) - return csv_file_status - -def get_file_building_tb_path(file_path): - return file_path + ".log" - - -def get_file_status(file_path): - if os.path.exists(file_path): - return Project.READY - elif os.path.exists(get_file_building_tb_path(file_path)): - return Project.ERROR - else: - return Project.CREATING - - -@app.task -def create_project_global_plot(worktable_path, plot_path): - try: - create_global_plot(worktable_path, plot_path) - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - log_path = get_file_building_tb_path(plot_path) - log_fh = open(log_path, "w") - traceback.print_exception(exc_type, exc_value, exc_traceback, file=log_fh) - log_fh.close() - - -@app.task -def create_project_quadrant_plot(worktable_path, plot_path): - try: - create_quadrant_plot(worktable_path, plot_path) - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - log_path = get_file_building_tb_path(plot_path) - log_fh = open(log_path, "w") - traceback.print_exception(exc_type, exc_value, exc_traceback, file=log_fh) - log_fh.close() - - -@app.task -def create_project_csv_file(worktable_path, csv_file, Nchunk): - try: - create_genome_full_csv(worktable_path, csv_file, Nchunk=Nchunk) - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - log_path = get_file_building_tb_path(plot_path) - log_fh = open(log_path, "w") - traceback.print_exception(exc_type, exc_value, exc_traceback, file=log_fh) - log_fh.close() - - -@app.task -def create_project_data( - phenotype_ids, - init_table_path, - worktable_path, - remove_nan = False, - stat = "jass.models.stats:omnibus_stat", - csv_file = None, - chunk_size = 50, - significance_treshold = 5*10**-8, - post_filtering = True, - delayed_gen_csv_file = False, - chromosome = None, - start = None, - end = None, - custom_loadings = None, - global_plot_path = None, - quadrant_plot_path = None - ): - - try: - Nchunk = create_worktable_file( - phenotype_ids = phenotype_ids, - init_file_path = init_table_path, - project_hdf_path = worktable_path, - remove_nan = remove_nan, - stat = stat, - optim_na = True, - csv_file = csv_file, - chunk_size = chunk_size, - significance_treshold = significance_treshold, - post_filtering = post_filtering, - delayed_gen_csv_file = delayed_gen_csv_file, - chromosome = chromosome, - pos_Start = start, - pos_End = end, - custom_loadings = custom_loadings - ) - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - log_path = get_file_building_tb_path(worktable_path) - log_fh = open(log_path, "w") - traceback.print_exception(exc_type, exc_value, exc_traceback, file=log_fh) - log_fh.close() - return - - if (global_plot_path is not None): - create_project_global_plot.delay(worktable_path, global_plot_path) - if (quadrant_plot_path is not None): - create_project_quadrant_plot.delay(worktable_path, quadrant_plot_path) - if (delayed_gen_csv_file and (csv_file is not None)): - create_project_csv_file.delay(worktable_path, csv_file, Nchunk=Nchunk) - - -def create_project(phenotype_ids: List[str], available_phenotypes: List[Phenotype]): - available_phenotype_ids = [phenotype.id for phenotype in available_phenotypes] - unavailable_requested_ids = set(phenotype_ids).difference( - set(available_phenotype_ids) - ) - if len(unavailable_requested_ids) > 0: - raise Exception() # FIXME with a nice exception - phenotypes = [ - phenotype for phenotype in available_phenotypes if phenotype.id in phenotype_ids - ] - project = Project(phenotypes=phenotypes) - folder_path = project.get_folder_path() - # if project does not exist - if project.status == Project.DOES_NOT_EXIST: - os.makedirs(folder_path) - create_project_data.delay( - phenotype_ids=phenotype_ids, - init_table_path=os.path.join(config["DATA_DIR"], "initTable.hdf5"), - worktable_path=project.get_worktable_path(), - global_plot_path=project.get_global_manhattan_plot_path(), - quadrant_plot_path=project.get_quadrant_plot_path(), - csv_file=project.get_csv_path(), - delayed_gen_csv_file=True - ) - return project - - -def create_project_local( - phenotype_ids: List[str], - available_phenotypes: List[Phenotype], - ip: str, - chromosome: str, - start: str = None, - end: str = None - ): - available_phenotype_ids = [phenotype.id for phenotype in available_phenotypes] - unavailable_requested_ids = set(phenotype_ids).difference( - set(available_phenotype_ids) - ) - if len(unavailable_requested_ids) > 0: - raise Exception() # FIXME with a nice exception - phenotypes = [ - phenotype for phenotype in available_phenotypes if phenotype.id in phenotype_ids - ] - ip = ip.replace('.', '_') - id_project = "local_{}".format(ip) - project = Project(phenotypes=phenotypes, id=id_project) - folder_path = project.get_folder_path() - # If the folder exists, it is deleted with the files it contains - if os.path.exists(folder_path): - shutil.rmtree(folder_path) - # The folder is created - os.makedirs(folder_path) - create_project_data.delay( - phenotype_ids=phenotype_ids, - init_table_path=os.path.join(config["DATA_DIR"], "initTable.hdf5"), - worktable_path=project.get_worktable_path(), - csv_file=project.get_csv_path(), - chromosome=chromosome, - start=start, - end=end - ) - return project - \ No newline at end of file +# -*- coding: utf-8 -*- +""" +compute joint statistics and generate plots for a given set of phenotypes +""" +from __future__ import absolute_import +from typing import List, Dict +import os, sys +import shutil +import hashlib +import traceback + +from celery import Celery + +from jass.models.base_model_ import Model +from jass.util import deserialize_model +from jass.models.phenotype import Phenotype +from jass.models.worktable import ( + create_worktable_file, + get_worktable_summary, + get_worktable_genomedata, + get_worktable_local_manhattan_data, + get_worktable_local_heatmap_data, + create_genome_full_csv +) +from jass.models.plots import create_global_plot, create_quadrant_plot +from jass.config import config + +app = Celery("tasks", broker="pyamqp://guest@localhost//") + + +class Project(Model): + + DOES_NOT_EXIST = "DOES_NOT_EXIST" + + CREATING = "CREATING" + + READY = "READY" + + ERROR = "ERROR" + + def __init__(self, id: str = None, phenotypes: List[Phenotype] = None): + """ + Project - a project (list of phenotypes) + + :param id: project ID. + :type id: str + """ + self.swagger_types = {"id": str, + "status": str, + "phenotypes": List[Phenotype], + "progress": str} + + self.attribute_map = { + "id": "id", + "status": "status", + "phenotypes": "phenotypes", + "progress": "progress", + } + + self._id = id + self._phenotypes = phenotypes + if self._id is None: + self._id = self.get_id() + + @classmethod + def from_dict(cls, dikt) -> "Project": + """ + Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The Project. + :rtype: Project + """ + return deserialize_model(dikt, cls) + + @property + def id(self) -> str: + """ + Gets the id of this Project. + + :return: The id of this Project. + :rtype: str + """ + return self._id + + @id.setter + def id(self, id: str): + """ + Lists the id of this Project. + + :param id: The id of this Project. + :type id: str + """ + + self._id = id + + @property + def phenotypes(self) -> List[Phenotype]: + """ + Gets the phenotypes list for this project. + + :return: The phenotypes. + :rtype: str + """ + return self._phenotypes + + @phenotypes.setter + def cohort(self, phenotypes: List[Phenotype]): + """ + Lists the phenotypes list for this project. + + :param phenotypes: The phenotypes. + :type phenotypes: str + """ + + self._phenotypes = phenotypes + + def get_folder_path(self): + """ + get_folder_path + Gets the path of the folder where the project data are stored + """ + return os.path.join(config["DATA_DIR"], "project_{}".format(self.id)) + + def get_worktable_path(self): + """ + get_worktable_path + Gets the path of the file workTable.hdf5 + """ + return os.path.join(self.get_folder_path(), "workTable.hdf5") + + def get_csv_path(self): + """ + get_csv_path + Gets the path of the file genome_full.csv + """ + return os.path.join(self.get_folder_path(), "workTable.csv") + + def get_progress_path(self): + """ + get_progress_path + Gets the path of the file containing the current progress percentage of \ + the analysis performed within the project + """ + return os.path.join(self.get_folder_path(), "JASS_progress.txt") + + def get_csv_lock_path(self): + """ + get_csv_lock_path + Gets the path of the lock set-on when the csv file is not available yet + """ + return os.path.join(self.get_folder_path(), "the_lock.txt") + + def get_project_summary_statistics(self): + return get_worktable_summary(self.get_worktable_path()) + + def get_project_genomedata(self): + return get_worktable_genomedata(self.get_worktable_path()) + + def get_project_local_manhattan_data(self, chromosome: str, region: str): + return get_worktable_local_manhattan_data( + self.get_worktable_path(), chromosome, region + ) + + def get_project_local_heatmap_data(self, chromosome: str, region: str): + return get_worktable_local_heatmap_data( + self.get_worktable_path(), chromosome, region + ) + + def get_id(self): + m = hashlib.md5() + for phenotype_id in [phenotype.id for phenotype in self._phenotypes]: + m.update(str(phenotype_id).encode("utf-8")) + return m.hexdigest() + + def get_global_manhattan_plot_path(self): + return os.path.join(self.get_folder_path(), "Manhattan_Plot_Omnibus.png") + + def get_quadrant_plot_path(self): + return os.path.join(self.get_folder_path(), "Quadrant_Plot_Omnibus.png") + + @property + def status(self): + """ + status + Gets the status of the project + """ + if not os.path.exists(self.get_folder_path()): + return Project.DOES_NOT_EXIST + else: + worktable_status = get_file_status(self.get_worktable_path()) + global_manhattan_status = get_file_status( + self.get_global_manhattan_plot_path() + ) + quadrant_plot_status = get_file_status(self.get_quadrant_plot_path()) + return { + # WARNING: project status is hacked so that everything is ready + # only once the final step has completed. + # This avoids the apparent "corrupted hdf5" file situation + # "worktable": worktable_status, + # "global_manhattan": global_manhattan_status, + "worktable": quadrant_plot_status, + "global_manhattan": quadrant_plot_status, + "quadrant_plot_status": quadrant_plot_status, + } + + @property + def progress(self): + """ + progress + Gets the percentage of completion of the phenotype analysis + """ + JASS_progress = 0 + progress_path = self.get_progress_path() + if os.path.exists(progress_path): + file_progress = open(progress_path, "r") + JASS_progress = file_progress.read() + file_progress.close() + return JASS_progress + + def get_csv_file_generation(self): + """ + csv_file_generation + Gets the status of the genome_full csv file generation + """ + the_lock_path = self.get_csv_lock_path() + csv_file = self.get_csv_path() + csv_file_status = Project.CREATING + if (not os.path.isfile(the_lock_path)): + if(os.path.isfile(csv_file)): + csv_file_status = Project.READY + else : + csv_file_status = Project.ERROR + print("csv_file_generation:csv_file_status={}".format(csv_file_status)) + return csv_file_status + +def get_file_building_tb_path(file_path): + return file_path + ".log" + + +def get_file_status(file_path): + if os.path.exists(file_path): + return Project.READY + elif os.path.exists(get_file_building_tb_path(file_path)): + return Project.ERROR + else: + return Project.CREATING + + +@app.task +def create_project_global_plot(worktable_path, plot_path): + try: + create_global_plot(worktable_path, plot_path) + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + log_path = get_file_building_tb_path(plot_path) + log_fh = open(log_path, "w") + traceback.print_exception(exc_type, exc_value, exc_traceback, file=log_fh) + log_fh.close() + + +@app.task +def create_project_quadrant_plot(worktable_path, plot_path): + try: + create_quadrant_plot(worktable_path, plot_path) + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + log_path = get_file_building_tb_path(plot_path) + log_fh = open(log_path, "w") + traceback.print_exception(exc_type, exc_value, exc_traceback, file=log_fh) + log_fh.close() + + +@app.task +def create_project_csv_file(worktable_path, csv_file, Nchunk): + try: + create_genome_full_csv(worktable_path, csv_file, Nchunk=Nchunk) + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + log_path = get_file_building_tb_path(plot_path) + log_fh = open(log_path, "w") + traceback.print_exception(exc_type, exc_value, exc_traceback, file=log_fh) + log_fh.close() + + +@app.task +def create_project_data( + phenotype_ids, + init_table_path, + worktable_path, + remove_nan = False, + stat = "jass.models.stats:omnibus_stat", + csv_file = None, + chunk_size = 50, + significance_treshold = 5*10**-8, + post_filtering = True, + delayed_gen_csv_file = False, + chromosome = None, + start = None, + end = None, + custom_loadings = None, + global_plot_path = None, + quadrant_plot_path = None + ): + + try: + Nchunk = create_worktable_file( + phenotype_ids = phenotype_ids, + init_file_path = init_table_path, + project_hdf_path = worktable_path, + remove_nan = remove_nan, + stat = stat, + optim_na = True, + csv_file = csv_file, + chunk_size = chunk_size, + significance_treshold = significance_treshold, + post_filtering = post_filtering, + delayed_gen_csv_file = delayed_gen_csv_file, + chromosome = chromosome, + pos_Start = start, + pos_End = end, + custom_loadings = custom_loadings + ) + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + log_path = get_file_building_tb_path(worktable_path) + log_fh = open(log_path, "w") + traceback.print_exception(exc_type, exc_value, exc_traceback, file=log_fh) + log_fh.close() + return + + if (global_plot_path is not None): + create_project_global_plot.delay(worktable_path, global_plot_path) + if (quadrant_plot_path is not None): + create_project_quadrant_plot.delay(worktable_path, quadrant_plot_path) + if (delayed_gen_csv_file and (csv_file is not None)): + create_project_csv_file.delay(worktable_path, csv_file, Nchunk=Nchunk) + + +def create_project(phenotype_ids: List[str], available_phenotypes: List[Phenotype]): + available_phenotype_ids = [phenotype.id for phenotype in available_phenotypes] + unavailable_requested_ids = set(phenotype_ids).difference( + set(available_phenotype_ids) + ) + if len(unavailable_requested_ids) > 0: + raise Exception() # FIXME with a nice exception + phenotypes = [ + phenotype for phenotype in available_phenotypes if phenotype.id in phenotype_ids + ] + project = Project(phenotypes=phenotypes) + folder_path = project.get_folder_path() + # if project does not exist + if project.status == Project.DOES_NOT_EXIST: + os.makedirs(folder_path) + create_project_data.delay( + phenotype_ids=phenotype_ids, + init_table_path=os.path.join(config["DATA_DIR"], "initTable.hdf5"), + worktable_path=project.get_worktable_path(), + global_plot_path=project.get_global_manhattan_plot_path(), + quadrant_plot_path=project.get_quadrant_plot_path(), + csv_file=project.get_csv_path(), + delayed_gen_csv_file=True + ) + return project + + +def create_project_local( + phenotype_ids: List[str], + available_phenotypes: List[Phenotype], + ip: str, + chromosome: str, + start: str = None, + end: str = None + ): + available_phenotype_ids = [phenotype.id for phenotype in available_phenotypes] + unavailable_requested_ids = set(phenotype_ids).difference( + set(available_phenotype_ids) + ) + if len(unavailable_requested_ids) > 0: + raise Exception() # FIXME with a nice exception + phenotypes = [ + phenotype for phenotype in available_phenotypes if phenotype.id in phenotype_ids + ] + ip = ip.replace('.', '_') + id_project = "local_{}".format(ip) + project = Project(phenotypes=phenotypes, id=id_project) + folder_path = project.get_folder_path() + # If the folder exists, it is deleted with the files it contains + if os.path.exists(folder_path): + shutil.rmtree(folder_path) + # The folder is created + os.makedirs(folder_path) + create_project_data.delay( + phenotype_ids=phenotype_ids, + init_table_path=os.path.join(config["DATA_DIR"], "initTable.hdf5"), + worktable_path=project.get_worktable_path(), + csv_file=project.get_csv_path(), + chromosome=chromosome, + start=start, + end=end + ) + return project + diff --git a/jass/server.py b/jass/server.py index a6900afbf9c4b68ca8d5413578ebb403a5d5031b..9fd5eff2a383770af390d6569f3aa27331d79ea5 100644 --- a/jass/server.py +++ b/jass/server.py @@ -1,30 +1,307 @@ #!/usr/bin/env python3 -""" -Module that creates the flask app used to run JASS as a web server -""" -import connexion -import flask +import os -from .encoder import JSONEncoder -from .config import config +from flask import Flask, redirect, send_file +from flask.views import MethodView +import marshmallow as ma +from flask_smorest import Api, Blueprint, abort +from webargs.flaskparser import FlaskParser +from jass.config import config +from jass.models.phenotype import get_available_phenotypes +from jass.models.project import create_project, Project -class JassFlaskApp(connexion.FlaskApp): + +class PhenotypeSchema(ma.Schema): + id = ma.fields.String() + consortium = ma.fields.String() + outcome = ma.fields.String() + full_name = ma.fields.String() + typ = ma.fields.String() + ref = ma.fields.String() + ref_link = ma.fields.String() + data_link = ma.fields.String() + data_path = ma.fields.String() + + +class PhenotypeIdsListSchema(ma.Schema): + class Meta: + unknown = ma.EXCLUDE + + phenotypeID = ma.fields.List(ma.fields.String()) + + +class ProjectStatusSchema(ma.Schema): + STATUS_VALUES = ["DOES_NOT_EXIST", "CREATING", "READY", "ERROR"] + global_manhattan = ma.fields.String(validate=ma.validate.OneOf(STATUS_VALUES)) + quadrant_plot_status = ma.fields.String(validate=ma.validate.OneOf(STATUS_VALUES)) + worktable = ma.fields.String(validate=ma.validate.OneOf(STATUS_VALUES)) + + +class ProjectSchema(ma.Schema): + id = ma.fields.String() + status = ma.fields.Nested(ProjectStatusSchema) + phenotypes = ma.fields.List(ma.fields.String()) + progress = ma.fields.String() + + +blp_phenotypes = Blueprint( + "phenotypes", + "phenotypes", + url_prefix="/phenotypes", + description="Operations on phenotypes", +) + +blp_projects = Blueprint( + "projects", "projects", url_prefix="/projects", description="Operations on projects" +) + +def get_phenotypes(): + return get_available_phenotypes( + os.path.join(config["DATA_DIR"], "initTable.hdf5") + ) + + +@blp_phenotypes.route("/") +class PhenotypesMethodView(MethodView): + @blp_phenotypes.response(200, PhenotypeSchema(many=True)) + def get(self): + """List phenotypes""" + return get_phenotypes() + + +@blp_projects.route("/") +class ProjectCreateMethodView(MethodView): + @blp_projects.arguments(PhenotypeIdsListSchema(), location="form") + @blp_projects.response(200, ProjectSchema()) + def post(self, phenotype_ids): + """List projects""" + phenotype_ids = [ + phenotype_id + for ids_with_commas in phenotype_ids["phenotypeID"] + for phenotype_id in ids_with_commas.split(",") + ] + phenotypes = list(filter(lambda d: d.id in phenotype_ids, get_phenotypes())) + return create_project([p.id for p in phenotypes], get_phenotypes()) + + +@blp_projects.route("/<project_id>") +class ProjectDetailMethodView(MethodView): + @blp_projects.response(200, ProjectSchema()) + def get(self, project_id): + return Project(id=project_id) + + +@blp_projects.route("/<project_id>/csv_status") +class ProjectCSVStatusMethodView(MethodView): + + def get(self, project_id): + return Project(id=project_id).get_csv_file_generation() + + +@blp_projects.route("/<project_id>/summary") +class ProjectSummaryMethodView(MethodView): + @blp_projects.response(200, ProjectSchema()) + def get(self, project_id): + return Project(id=project_id).get_project_summary_statistics() + + +@blp_projects.route("/<project_id>/genome") +class ProjectGenomeMethodView(MethodView): + + # @blp_projects.response(200, headers={"Content-Type": "text/csv"}) + def get(self, project_id): + try: + return ( + Project(id=project_id).get_project_genomedata(), + 200, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + except FileNotFoundError: + status = Project(id=project_id).status + if status == Project.DOES_NOT_EXIST: + return ( + f"project {project_id} does not exist", + 404, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + abort(404) + elif status["worktable"] == Project.CREATING: + return ( + "data not ready yet", + 202, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + else: + abort(500) + + +@blp_projects.route("/<project_id>/genome_full") +class ProjectGenomeFullMethodView(MethodView): + + def get(self, project_id): + try: + return send_file( + Project(id=project_id).get_csv_path(), + mimetype="text/csv", + as_attachment=True, + attachment_filename="genome_full.csv", + ) + except FileNotFoundError: + status = Project(id=project_id).status + if status == Project.DOES_NOT_EXIST: + return ( + f"project {project_id} does not exist", + 404, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + elif status["worktable"] == Project.CREATING: + return ( + "data not ready yet", + 202, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + else: + abort(500) + + +@blp_projects.route("/<project_id>/globalmanhattan") +class ProjectGlobalManhattanMethodView(MethodView): + + def get(self, project_id): + try: + return send_file( + Project(id=project_id).get_global_manhattan_plot_path(), + mimetype="image/png", + ) + except FileNotFoundError: + status = Project(id=project_id).status + if status == Project.DOES_NOT_EXIST: + return ( + f"project {project_id} does not exist", + 404, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + elif status["global_manhattan"] == Project.CREATING: + return ( + "data not ready yet", + 202, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + else: + abort(500) + + +@blp_projects.route("/<project_id>/quadrant") +class ProjectQuadrantMethodView(MethodView): + + def get(self, project_id): + try: + return send_file( + Project(id=project_id).get_quadrant_plot_path(), + mimetype="image/png", + ) + except FileNotFoundError: + status = Project(id=project_id).status + if status == Project.DOES_NOT_EXIST: + return ( + f"project {project_id} does not exist", + 404, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + elif status["quadrant_plot_status"] == Project.CREATING: + return ( + "data not ready yet", + 202, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + else: + abort(500) + + +@blp_projects.route("/<project_id>/manhattan/<chromosome>/<region>") +class ProjectLocalManhattanMethodView(MethodView): + + def get(self, project_id, chromosome, region): + try: + return ( + Project(id=project_id).get_project_local_manhattan_data( + chromosome, region + ), + 200, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + except FileNotFoundError: + status = Project(id=project_id).status + if status == Project.DOES_NOT_EXIST: + return ( + f"project {project_id} does not exist", + 404, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + elif status["worktable"] == Project.CREATING: + return ( + "data not ready yet", + 202, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + else: + abort(500) + + +@blp_projects.route("/<project_id>/heatmap/<chromosome>/<region>") +class ProjectLocalHeatMapMethodView(MethodView): + + def get(self, project_id, chromosome, region): + try: + return ( + Project(id=project_id).get_project_local_heatmap_data( + chromosome, region + ), + 200, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + except FileNotFoundError: + status = Project(id=project_id).status + if status == Project.DOES_NOT_EXIST: + return ( + f"project {project_id} does not exist", + 404, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + elif status["worktable"] == Project.CREATING: + return ( + "data not ready yet", + 202, + {"Content-Type": "text/plain; charset=utf-8"}, + ) + else: + abort(500) + + +class JassApp(Flask): """ - JassFlaskApp subclasses connexion's FlaskApp only to customize the static url path + JassApp builds the JASS Flask application """ + def __init__(self): + self.flask_app = Flask(__name__, static_url_path="", static_folder="static") + self.flask_app.config["API_TITLE"] = "JASS API" + self.flask_app.config["API_VERSION"] = "v2.0" + self.flask_app.config["OPENAPI_VERSION"] = "3.0.2" + self.flask_app.route("/")(self.redirect_to_index) + self.api = Api(self.flask_app) + def create_app(self): - app = flask.Flask(self.import_name, static_url_path="", static_folder="static") - app.json_encoder = JSONEncoder - app.route("/")(self.redirect_to_index) - return app + return self.flask_app def redirect_to_index(self): - return flask.redirect("index.html") + return redirect("index.html") + + def register_api_blueprint(self, blp): + self.api.register_blueprint(blp, url_prefix=f"/api/{blp.url_prefix}") -def get_jass_app(): - app = JassFlaskApp(__name__, specification_dir="./swagger/") - app.add_api("swagger.yaml", arguments={"title": "JASS"}, base_path="/api") - return app +jass_app = JassApp() +jass_app.register_api_blueprint(blp_phenotypes) +jass_app.register_api_blueprint(blp_projects) diff --git a/jass/swagger/swagger.yaml b/jass/swagger/swagger.yaml index c47461d30df983b6ba9c213ded0e1ab49e17d23a..cf0b67d7e705109209ed409afbedbb53b81b6885 100644 --- a/jass/swagger/swagger.yaml +++ b/jass/swagger/swagger.yaml @@ -35,7 +35,6 @@ paths: "ref": "Okada et al. 2014" "ref_link": "https://www.ncbi.nlm.nih.gov/pubmed/24390342" "type": "Immunity" - x-openapi-router-controller: jass.controllers.default_controller /projects: post: description: | @@ -117,7 +116,6 @@ paths: "worktable": "READY" progress": "progress": "100" - x-openapi-router-controller: jass.controllers.default_controller "/projects/{projectID}": get: description: | @@ -146,7 +144,6 @@ paths: "worktable": "READY" progress": "progress": "100" - x-openapi-router-controller: jass.controllers.default_controller "/projects/{projectID}/summary": get: description: Retrieve summary statistics for a given project @@ -174,8 +171,6 @@ paths: "NoJOSTSignif": "NoPhenoSignif": 1470 "PhenoSignif": 14 - x-openapi-router-controller: jass.controllers.default_controller - "/projects/{projectID}/csv_status": get: description: | @@ -200,8 +195,6 @@ paths: title: csv_file_generation example: | READY - x-openapi-router-controller: jass.controllers.default_controller - "/projects/{projectID}/genome": get: description: | @@ -234,8 +227,6 @@ paths: Region5,6580614.0,rs2986741,chr1,6548774,G,A,0.0013472918321710914,0.0011119999999999993,None,-3.260540717377886,2.726551316504396 Region6,8306267.0,rs79412885,chr1,9241839,A,G,2.0889091093474285e-13,8.106999999999937e-14,Both,7.46857160133221,-1.2003588580308502 Region7,10086091.5,rs113829298,chr1,10061038,T,C,4.3158209846991565e-05,6.135999999999996e-06,None,-4.5216481219798474,0.5100734569685951 - x-openapi-router-controller: jass.controllers.default_controller - "/projects/{projectID}/genome_full": get: description: | @@ -275,7 +266,6 @@ paths: 4841,1,1,rs10907190,1773772,A,G,951595.0,0.12979175667585227,0.07999999999999982,0.15999999999999964,,-0.938281041511616,1.7506860712521708 4842,1,1,rs10907193,1805391,A,G,951595.0,0.09562672355608258,0.06299999999999988,0.12599999999999975,,-1.0405165049626888,1.8591914944718688 4843,1,1,rs10907194,1712230,T,C,951595.0,0.2669995168398967,0.16000000000000425,0.3200000000000085,,-0.7600913211933399,1.4050715603096189 - x-openapi-router-controller: jass.controllers.default_controller "/projects/{projectID}/globalmanhattan": get: description: | @@ -300,7 +290,6 @@ paths: title: Global manhattan plot example: externalValue: 'globalmanhattan_example.png' - x-openapi-router-controller: jass.controllers.default_controller "/projects/{projectID}/quadrant": get: description: | @@ -325,7 +314,6 @@ paths: title: Quadrant plot example: externalValue: 'quadrant_example.png' - x-openapi-router-controller: jass.controllers.default_controller "/projects/{projectID}/manhattan/{chromosome}/{region}": get: description: Retrieve local manhattan data @@ -380,7 +368,6 @@ paths: Region1,chr1,725196,rs377099097,0.594983644175122 Region1,chr1,725389,rs375619475,0.7032290172253173 Region1,chr1,727841,rs116587930,0.9078685880041112 - x-openapi-router-controller: jass.controllers.default_controller "/projects/{projectID}/heatmap/{chromosome}/{region}": get: description: Retrieve local heatmap data @@ -420,7 +407,6 @@ paths: ID,rs545945172,rs371628865,rs61769339,rs539032812,rs12238997,rs189800799 z_IHEC_MONOP,-0.3623372836601329,-0.429856541533544,-0.8457360635272954,-0.9809852811227732,-0.6936527568935886,0.4382385293216385 z_RA_RA,,,,,, - x-openapi-router-controller: jass.controllers.default_controller components: schemas: Phenotype: diff --git a/jass/test/__init__.py b/jass/test/__init__.py index 86f967d82f02120c4f61393f690934c58972ad34..cbe532b806217bf04a1fbaba907cf68c5a85fef4 100644 --- a/jass/test/__init__.py +++ b/jass/test/__init__.py @@ -1,12 +1,19 @@ import unittest -from ..encoder import JSONEncoder import logging import os, shutil, tempfile import unittest -import connexion import flask_testing + +# replace delay() method with a mock +# which freezes calls in unit tests +from celery.app.task import Task +from unittest.mock import MagicMock +Task.delay = MagicMock() + + +from jass.server import jass_app from jass.models.inittable import create_inittable_file @@ -20,16 +27,16 @@ class JassTestCase(unittest.TestCase): class JassFlaskTestCase(JassTestCase, flask_testing.TestCase): def create_app(self): - logging.getLogger("connexion.operation").setLevel("ERROR") from jass.config import config - self.test_dir = tempfile.mkdtemp() config["DATA_DIR"] = self.test_dir shutil.copy(self.get_file_path_fn("initTable.hdf5"), self.test_dir) - app = connexion.App(__name__, specification_dir="../swagger/") - app.app.json_encoder = JSONEncoder - app.add_api("swagger.yaml") - return app.app + + self.jass_app = jass_app + application = self.jass_app.create_app() + application.config["TESTING"] = True + self.testing_client = application.test_client() + return application def tearDown(self): shutil.rmtree(self.test_dir) diff --git a/jass/test/test_default_controller.py b/jass/test/test_server.py similarity index 52% rename from jass/test/test_default_controller.py rename to jass/test/test_server.py index 839f48375abe67b5f4acc4fd5e9623ca0790f032..da1b77160fc8e09a8bc18f1eaddcb5976c2624bf 100644 --- a/jass/test/test_default_controller.py +++ b/jass/test/test_server.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import os, shutil, tempfile from six import BytesIO -from flask import json +from flask import json, url_for from jass.config import config from . import JassFlaskTestCase @@ -17,11 +17,16 @@ class TestDefaultController(JassFlaskTestCase): def test_phenotypes_get(self): """ - Test case for phenotypes_get + Test case retrieving available phenotypes + """ + response = self.testing_client.open("/api/phenotypes/", method="GET") + self.assert200(response, "Response body is : " + response.data.decode("utf-8")) - + def test_create_project(self): + """ + Test case for creating a project """ - response = self.client.open("/phenotypes", method="GET") + response = self.testing_client.open("/api/projects/", method="POST",data={"phenotypeID":"z_IHEC_MONOP,z_RA_RA"}) self.assert200(response, "Response body is : " + response.data.decode("utf-8")) diff --git a/requirements.txt b/requirements.txt index 25400d93c0388519cea98149d04e2cacbade82a3..f6fc215883a802969f2cf7bcf01d9f501d150a46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/hmenager/connexion.git@master#egg=connexion[swagger-ui] +flask-smorest aiohttp python_dateutil setuptools diff --git a/setup.py b/setup.py index 19675e52aa5d51210c488a360066df4de0043581..67e274e329c3082dabf0228d1e82f7e5dd93eea1 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ README = os.path.join(SETUP_DIR, 'README.md') readme = open(README).read() REQUIRES = [ - "connexion[swagger-ui] @ git+https://github.com/hmenager/connexion.git@master#egg=connexion[swagger-ui]", + "flask-smorest", "aiohttp", "python_dateutil", "setuptools",