Commit d1490469 authored by Hervé  MENAGER's avatar Hervé MENAGER
Browse files

Merge branch 'dev-flasksmorest' into 'dev'

change default handling of missing value : the server and the command line...

See merge request !33
parents d7444b0b 29adcf61
......@@ -2,8 +2,9 @@ FROM centos:7
RUN yum install -y epel-release wget gcc https://repo.ius.io/ius-release-el7.rpm
RUN yum update -y
RUN yum install -y python35u python35u-libs python35u-devel python35u-pip openssl-devel libffi-devel
RUN pip3.5 install ansible
RUN yum install -y python36u python36u-libs python36u-devel python36u-pip openssl-devel libffi-devel
RUN pip3.6 install --upgrade pip
RUN pip3.6 install ansible
RUN yum install -y python-pip
RUN python -m pip install --upgrade pip
COPY . /code
......
......@@ -17,4 +17,4 @@ Submodules
"""
import os
from jass.tasks import celery
\ No newline at end of file
from jass.tasks import celery
......@@ -5,59 +5,66 @@ import os
import sys
import argparse
from jass.server import get_jass_app
from jass.server import jass_app
from jass.config import config
from jass.models.phenotype import get_available_phenotypes
from jass.models.inittable import create_inittable_file, add_gene_annotation
from jass.models.worktable import create_worktable_file
from jass.models.plots import ( create_global_plot,
create_quadrant_plot,
create_local_plot,
create_qq_plot)
from jass.models.plots import (
create_global_plot,
create_quadrant_plot,
create_local_plot,
create_qq_plot,
)
def absolute_path_of_the_file(fileName, output_file = False):
def absolute_path_of_the_file(fileName, output_file=False):
"""
Builds the absolute path of the file : fileName
This makes the execution of JASS functions more robust and flexible
"""
# Build an absolute path if possible
absoluteFilePath = os.path.abspath(fileName)
# Test if the file name is a pattern
is_a_pattern = (os.path.basename(fileName).find("*") > -1)
is_a_pattern = os.path.basename(fileName).find("*") > -1
if (is_a_pattern or output_file) :
if is_a_pattern or output_file:
# Test if the directory path exist
Directory_path_exist = os.path.exists(os.path.dirname(absoluteFilePath))
if (Directory_path_exist == False):
if Directory_path_exist == False:
# Test the path using the Jass data directory
absoluteFilePath = os.path.normpath(os.path.join(config["DATA_DIR"], fileName))
absoluteFilePath = os.path.normpath(
os.path.join(config["DATA_DIR"], fileName)
)
Directory_path_exist = os.path.exists(os.path.dirname(absoluteFilePath))
if (Directory_path_exist == False):
if Directory_path_exist == False:
Message = "The directory of the file {} does not exist".format(fileName)
raise NameError(Message)
else:
# Test if the file path exist
File_path_exist = os.path.exists(absoluteFilePath)
if (File_path_exist == False):
if File_path_exist == False:
# Test the path using the Jass data directory
absoluteFilePath = os.path.normpath(os.path.join(config["DATA_DIR"], fileName))
absoluteFilePath = os.path.normpath(
os.path.join(config["DATA_DIR"], fileName)
)
File_path_exist = os.path.exists(absoluteFilePath)
if (File_path_exist == False):
if File_path_exist == False:
Message = "The file {} does not exist".format(fileName)
raise NameError(Message)
# Test if it is realy a file
Is_a_file = os.path.isfile(absoluteFilePath)
if (not Is_a_file) :
if not Is_a_file:
Message = "{} is not a file".format(fileName)
raise NameError(Message)
......@@ -65,8 +72,7 @@ def absolute_path_of_the_file(fileName, output_file = False):
def serve(args):
app = get_jass_app()
app.run(host=config["HOST"], port=config["PORT"])
jass_app.flask_app.run(host=config["HOST"], port=config["PORT"])
def w_list_phenotypes(args):
......@@ -77,12 +83,12 @@ def w_list_phenotypes(args):
def compute_worktable(args):
csv_file_path = args.csv_file_path
if (csv_file_path is not None):
if csv_file_path is not None:
csv_file_path = absolute_path_of_the_file(csv_file_path, True)
init_table_path = absolute_path_of_the_file(args.init_table_path)
worktable_path = absolute_path_of_the_file(args.worktable_path, True)
selected_phenotypes = args.phenotypes
remove_nan = (args.remove_nans)
remove_nan = args.remove_nans
significance_treshold = float(args.significance_treshold)
post_filtering = bool(args.post_filtering)
custom_loadings = args.custom_loadings
......@@ -91,9 +97,9 @@ def compute_worktable(args):
pos_End = args.end_position
if args.omnibus:
strategy = 'jass.models.stats:omnibus_stat'
strategy = "jass.models.stats:omnibus_stat"
elif args.sumz:
strategy = 'jass.models.stats:sumz_stat'
strategy = "jass.models.stats:sumz_stat"
elif args.fisher_test:
strategy = "jass.models.stats:fisher_test"
elif args.meta_analysis:
......@@ -102,21 +108,21 @@ def compute_worktable(args):
strategy = args.strategy
create_worktable_file(
phenotype_ids = selected_phenotypes,
init_file_path = init_table_path,
project_hdf_path = worktable_path,
remove_nan = remove_nan,
stat = strategy,
optim_na = True,
csv_file = csv_file_path,
chunk_size = int(args.chunk_size),
significance_treshold = significance_treshold,
post_filtering = post_filtering,
delayed_gen_csv_file = False,
chromosome = chromosome,
pos_Start = pos_Start,
pos_End = pos_End,
custom_loadings = custom_loadings
phenotype_ids=selected_phenotypes,
init_file_path=init_table_path,
project_hdf_path=worktable_path,
remove_nan=remove_nan,
stat=strategy,
optim_na=True,
csv_file=csv_file_path,
chunk_size=int(args.chunk_size),
significance_treshold=significance_treshold,
post_filtering=post_filtering,
delayed_gen_csv_file=False,
chromosome=chromosome,
pos_Start=pos_Start,
pos_End=pos_End,
custom_loadings=custom_loadings,
)
......@@ -126,28 +132,25 @@ def w_create_worktable(args):
def w_create_project_data(args):
compute_worktable(args)
worktable_path = absolute_path_of_the_file(args.worktable_path, True)
manhattan_plot_path = args.manhattan_plot_path
if (manhattan_plot_path is not None):
if manhattan_plot_path is not None:
manhattan_plot_path = absolute_path_of_the_file(manhattan_plot_path, True)
create_global_plot(worktable_path, manhattan_plot_path)
quadrant_plot_path = args.quadrant_plot_path
if (quadrant_plot_path is not None):
quadrant_plot_path = absolute_path_of_the_file(quadrant_plot_path, True)
create_quadrant_plot(worktable_path,
quadrant_plot_path,
significance_treshold = float(args.significance_treshold))
if quadrant_plot_path is not None:
quadrant_plot_path = absolute_path_of_the_file(quadrant_plot_path, True)
create_quadrant_plot(
worktable_path,
quadrant_plot_path,
significance_treshold=float(args.significance_treshold),
)
zoom_plot_path = args.zoom_plot_path
if (zoom_plot_path is not None):
if zoom_plot_path is not None:
zoom_plot_path = absolute_path_of_the_file(zoom_plot_path, True)
create_local_plot(worktable_path, zoom_plot_path)
qq_plot_path = args.qq_plot_path
if (qq_plot_path is not None):
if qq_plot_path is not None:
qq_plot_path = absolute_path_of_the_file(qq_plot_path, True)
create_qq_plot(worktable_path, qq_plot_path)
......@@ -155,7 +158,9 @@ def w_create_project_data(args):
def w_create_inittable(args):
input_data_path = absolute_path_of_the_file(args.input_data_path)
init_covariance_path = absolute_path_of_the_file(args.init_covariance_path)
init_genetic_covariance_path = absolute_path_of_the_file(args.init_genetic_covariance_path)
init_genetic_covariance_path = absolute_path_of_the_file(
args.init_genetic_covariance_path
)
regions_map_path = absolute_path_of_the_file(args.regions_map_path)
description_file_path = absolute_path_of_the_file(args.description_file_path)
init_table_path = absolute_path_of_the_file(args.init_table_path, True)
......@@ -166,7 +171,7 @@ def w_create_inittable(args):
description_file_path,
init_table_path,
init_covariance_path,
init_genetic_covariance_path
init_genetic_covariance_path,
)
......@@ -180,9 +185,9 @@ def w_plot_quadrant(args):
worktable_path = absolute_path_of_the_file(args.worktable_path)
plot_path = absolute_path_of_the_file(args.plot_path)
significance_treshold = float(args.significance_treshold)
create_quadrant_plot(worktable_path,
plot_path,
significance_treshold=significance_treshold)
create_quadrant_plot(
worktable_path, plot_path, significance_treshold=significance_treshold
)
def w_gene_annotation(args):
......@@ -190,10 +195,9 @@ def w_gene_annotation(args):
initTable_path = absolute_path_of_the_file(args.init_table_path, True)
df_gene_csv_path = absolute_path_of_the_file(args.gene_csv_path, True)
df_exon_csv_path = absolute_path_of_the_file(args.exon_csv_path, True)
add_gene_annotation(gene_data_path,
initTable_path,
df_gene_csv_path,
df_exon_csv_path)
add_gene_annotation(
gene_data_path, initTable_path, df_gene_csv_path, df_exon_csv_path
)
def get_parser():
......@@ -256,7 +260,7 @@ def get_parser():
)
parser_create_pd.add_argument(
"--significance-treshold",
default=5*10**-8,
default=5 * 10 ** -8,
help="The treshold at which a p-value is considered significant",
)
parser_create_pd.add_argument(
......@@ -279,27 +283,25 @@ def get_parser():
)
parser_create_pd.add_argument(
"--csv-file-path",
required=False,
help="path to the results file in csv format"
"--csv-file-path", required=False, help="path to the results file in csv format"
)
parser_create_pd.add_argument(
"--chromosome-number",
required=False,
help="option used only for local analysis: chromosome number studied"
help="option used only for local analysis: chromosome number studied",
)
parser_create_pd.add_argument(
"--start-position",
required=False,
help="option used only for local analysis: start position of the region studied"
help="option used only for local analysis: start position of the region studied",
)
parser_create_pd.add_argument(
"--end-position",
required=False,
help="option used only for local analysis: end position of the region studied"
help="option used only for local analysis: end position of the region studied",
)
strategies = parser_create_pd.add_mutually_exclusive_group()
......@@ -344,7 +346,7 @@ def get_parser():
parser_create_it.add_argument(
"--init-genetic-covariance-path",
default=None,
help = "path to the genetic covariance file to import",
help="path to the genetic covariance file to import",
)
parser_create_it.set_defaults(func=w_create_inittable)
# ------- create-worktable -------
......@@ -366,13 +368,13 @@ def get_parser():
)
parser_create_wt.add_argument(
"--significance-treshold",
default=5*10**-8,
help="threshold at which a p-value is considered significant"
)
default=5 * 10 ** -8,
help="threshold at which a p-value is considered significant",
)
parser_create_wt.add_argument(
"--post-filtering",
default=True,
help="If a filtering to remove outlier will be applied (in this case the result of SNPs considered aberant will not appear in the worktable)"
help="If a filtering to remove outlier will be applied (in this case the result of SNPs considered aberant will not appear in the worktable)",
)
parser_create_wt.add_argument(
......@@ -382,9 +384,7 @@ def get_parser():
)
parser_create_wt.add_argument(
"--csv-file-path",
required=False,
help="path to the results file in csv format"
"--csv-file-path", required=False, help="path to the results file in csv format"
)
parser_create_wt.add_argument(
......@@ -399,19 +399,19 @@ def get_parser():
parser_create_wt.add_argument(
"--chromosome-number",
required=False,
help="option used only for local analysis: chromosome number studied"
help="option used only for local analysis: chromosome number studied",
)
parser_create_wt.add_argument(
"--start-position",
required=False,
help="option used only for local analysis: start position of the region studied"
help="option used only for local analysis: start position of the region studied",
)
parser_create_wt.add_argument(
"--end-position",
required=False,
help="option used only for local analysis: end position of the region studied"
help="option used only for local analysis: end position of the region studied",
)
strategies = parser_create_wt.add_mutually_exclusive_group()
......@@ -435,9 +435,7 @@ def get_parser():
help="path to the worktable file containing the data",
)
parser_create_mp.add_argument(
"--plot-path",
required=True,
help="path to the manhattan plot file to generate"
"--plot-path", required=True, help="path to the manhattan plot file to generate"
)
parser_create_mp.set_defaults(func=w_plot_manhattan)
......@@ -452,21 +450,20 @@ def get_parser():
help="path to the worktable file containing the data",
)
parser_create_mp.add_argument(
"--plot-path",
required=True,
help="path to the quadrant plot file to generate"
"--plot-path", required=True, help="path to the quadrant plot file to generate"
)
parser_create_mp.add_argument(
"--significance-treshold",
default=5*10**-8,
help="threshold at which a p-value is considered significant"
default=5 * 10 ** -8,
help="threshold at which a p-value is considered significant",
)
parser_create_mp.set_defaults(func=w_plot_quadrant)
# ------- add-gene-annotation -------
parser_create_mp = subparsers.add_parser(
"add-gene-annotation", help="add information about genes ansd exons to the inittable"
"add-gene-annotation",
help="add information about genes ansd exons to the inittable",
)
parser_create_mp.add_argument(
"--gene-data-path",
......@@ -476,17 +473,13 @@ def get_parser():
parser_create_mp.add_argument(
"--init-table-path",
required=True,
help="path to the initial table file to update"
help="path to the initial table file to update",
)
parser_create_mp.add_argument(
"--gene-csv-path",
required=False,
help="path to the file df_gene.csv"
"--gene-csv-path", required=False, help="path to the file df_gene.csv"
)
parser_create_mp.add_argument(
"--exon-csv-path",
required=False,
help="path to the file df_exon.csv"
"--exon-csv-path", required=False, help="path to the file df_exon.csv"
)
parser_create_mp.set_defaults(func=w_gene_annotation)
......@@ -494,16 +487,16 @@ def get_parser():
def main():
print("", file = sys.stderr)
print(" ** ******* ******* *******", file = sys.stderr)
print(" ** ** ** ** **", file = sys.stderr)
print(" ** ** ** ** **", file = sys.stderr)
print(" ** ** ** ****** ******", file = sys.stderr)
print(" ** *********** ** **", file = sys.stderr)
print(" ** ** ** ** ** **", file = sys.stderr)
print(" ******* ** ** ******* *******", file = sys.stderr)
print("", file = sys.stderr)
print("", file = sys.stderr)
print("", file=sys.stderr)
print(" ** ******* ******* *******", file=sys.stderr)
print(" ** ** ** ** **", file=sys.stderr)
print(" ** ** ** ** **", file=sys.stderr)
print(" ** ** ** ****** ******", file=sys.stderr)
print(" ** *********** ** **", file=sys.stderr)
print(" ** ** ** ** ** **", file=sys.stderr)
print(" ******* ** ** ******* *******", file=sys.stderr)
print("", file=sys.stderr)
print("", file=sys.stderr)
parser = get_parser()
args = parser.parse_args()
args.func(args)
......
import os
## Broker settings.
broker_url = os.getenv('JASS_RABBITMQ_URL','amqp://guest:guest@localhost:5672//')
broker_url = os.getenv("JASS_RABBITMQ_URL", "amqp://guest:guest@localhost:5672//")
## Broker settings.
#result_backend = os.getenv('JASS_RABBITMQ_URL','amqp://guest2:guest@localhost:5672//')
result_backend='rpc://'
# result_backend = os.getenv('JASS_RABBITMQ_URL','amqp://guest2:guest@localhost:5672//')
result_backend = "rpc://"
# List of modules to import when the Celery worker starts.
#imports = ('myapp.tasks',)
# imports = ('myapp.tasks',)
## Using the database to store task state and results.
#result_backend = 'db+sqlite:///results.db'
#task_annotations = {'tasks.add': {'rate_limit': '10/s'}}
# result_backend = 'db+sqlite:///results.db'
# task_annotations = {'tasks.add': {'rate_limit': '10/s'}}
"""
the list of controller functions called by the JASS REST web services
Submodules
==========
.. autosummary::
:toctree: _autosummary
default_controller
"""
# -*- coding: utf-8 -*-
"""
default_controller ensures the connection between the web interface and the Python JASS-analysis module
"""
import os
from typing import List, Dict
import connexion
from flask import send_file, abort
from six import iteritems
from jass.config import config
from jass.models.project import Project
from jass.models.phenotype import Phenotype, get_available_phenotypes
from jass.tasks import create_project
PHENOTYPES = get_available_phenotypes(
os.path.join(config["DATA_DIR"], "initTable.hdf5")
) # FIXME part of the config
def phenotypes_get():
"""
phenotypes_get
Gets the list of available phenotypes
:rtype: List[Phenotype]
"""
return PHENOTYPES
def projects_post(phenotypeID):
"""
projects_post
Create a new project from a selection of phenotypes
:param phenotypeID: IDs of the phenotypes selected for the project
:type phenotypeID: List[str]
:rtype: str
"""
return create_project(phenotypeID, PHENOTYPES)
def local_project_post(phenotypeID, chromosome, start, end):
"""
local_project_post
Create a new local project from a chromosome number, start and end positions
and a selection of phenotypes
:param param: IDs of the phenotypes selected for the project
:type phenotypeID: List[str]
:rtype: str
"""
return create_project(phenotypeID, PHENOTYPES, chromosome, start, end)
def projects_project_id_csv_status_get(projectID):
"""
projects_project_id_csv_status_get
Retrieve the generation status of the genome full csv file
:param projectID: project ID
:type projectID: str
:rtype: str
"""
return Project(id=projectID).get_csv_file_generation()
def projects_project_id_summary_statistics(projectID):
"""
projects_project_id_summary_statistics
Retrieve project summary statistics
"""
return Project(id=projectID).get_project_summary_statistics()
def projects_project_id_genome_get(projectID, threshold=None):
"""
projects_project_id_genome_get
Retrieve genome data for a given project
:param projectID: project ID
:type projectID: str
:rtype: str
"""
return Project(id=projectID).get_project_genomedata()
def projects_project_id_global_manhattan_plot_get(projectID):
"""
projects_project_id_global_manhattan_plot_get
Gets the global Manhattan plot stored in the Project folder to display it on the Web interface
"""
try:
return send_file(
Project(id=projectID).get_global_manhattan_plot_path(), mimetype="image/png"
)
except FileNotFoundError:
status = Project(id=projectID).status
if status == Project.DOES_NOT_EXIST:
abort(404)
elif status["global_manhattan"] == Project.CREATING:
return (
"Plot is not ready yet",
202,
{"Content-Type": "text/plain; charset=utf-8"},
)
else:
abort(500)
def projects_project_id_zoom_plot_get(projectID):
"""
projects_project_id_zoom_plot_get
Gets the zoom plot stored in the local Project folder to display it on the Web interface
"""
try:
return send_file(
Project(id=projectID).get_zoom_plot_path(), mimetype="image/png"
)
except FileNotFoundError:
status = Project(id=projectID).status
if status == Project.DOES_NOT_EXIST:
abort(404)
elif status["zoom_plot"] == Project.CREATING:
return (
"Plot is not ready yet",
202,
{"Content-Type": "text/plain; charset=utf-8"},
)
else:
abort(500)
def projects_project_id_quadrant_plot_get(projectID):
"""
projects_project_id_quadrant_plot_get
Gets the quadrant plot stored in the Project folder to display it on the Web interface
"""
try:
return send_file(
Project(id=projectID).get_quadrant_plot_path(), mimetype="image/png"
)
except FileNotFoundError:
status = Project(id=projectID).status