diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index db82122c4ff78813e512a432cb338595ad9d27d5..9800abf0ca5584d0084698915c1c8c2c901435cf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,7 +22,7 @@ test-python:
'3.8',
# '3.9',
'3.10',
- '3.11',
+# '3.11',
]
diff --git a/chart/values.yaml b/chart/values.yaml
index eb92f13adcfe9883b787bb3cad7ec3ce20fe7b09..f70af91d3c9dd17f628f887956c18a92b94e8951 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -82,7 +82,7 @@ backCronTasks:
periodicity: "6 6 * * *"
projects:
- size: 20Gi
+ size: 19Gi
minSizeToKeepFree: 10240 # size is in MB
storageClassName: isilon
diff --git a/client/pages/index.vue b/client/pages/index.vue
index 2001b7e24d2c8de6810bf8fafd642eb007d780ef..7f6077b103d7683e63102c0c8bc6517e898cfece 100644
--- a/client/pages/index.vue
+++ b/client/pages/index.vue
@@ -57,7 +57,11 @@ export default {
},
async asyncData({ $axios, params, $auth }) {
try {
- const initmeta = await $axios.$get('/initmeta')
+ const data = {
+ initTableName: "initTable.hdf5"
+ // initTableName: "initTableTest1.hdf5"
+ };
+ const initmeta = await $axios.$post('/initmeta', data)
return {
initmeta,
}
diff --git a/client/pages/phenotypes.vue b/client/pages/phenotypes.vue
index fbf1383304e1379070032d1d263545b681b5de26..16f0475c1966db59c8b27eacc2b795d595756f9a 100644
--- a/client/pages/phenotypes.vue
+++ b/client/pages/phenotypes.vue
@@ -45,7 +45,11 @@ Please select the traits you want to analyze jointly with the Omnibus test. A re
<script>
export default {
async asyncData({ $axios, params, $auth }) {
- const phenotypes = await $axios.$get('/phenotypes')
+ const data = {
+ initTableName: "initTable.hdf5"
+ // initTableName: "initTableTest1.hdf5"
+ };
+ const phenotypes = await $axios.$post('/phenotypes', data)
return {
phenotypes,
}
@@ -83,7 +87,12 @@ export default {
},
runAnalysis(){
const { selectedRows, $axios } = this;
- const phenotypeIds = {phenotypeID: selectedRows.map(function(phenotype){return phenotype.id})};
+ const phenotypeIds = {
+ phenotypeID: selectedRows.map(function(phenotype){return phenotype.id}),
+ initTableName: "initTable.hdf5"
+ // phenotypeID: ["z_MAGIC_FAST-GLUCOSE", "z_MAGIC_FAST-INSULIN"],
+ // initTableName: "initTableTest.hdf5"
+ };
console.log("REQUEST POST")
console.log(phenotypeIds)
diff --git a/jass/models/inittable.py b/jass/models/inittable.py
index 4852e11059d2ddf4e7bbfe53f4e9f865c2790e6d..dc7fc85e38a4e740f7d3a9c3fbb17cb46e024461 100644
--- a/jass/models/inittable.py
+++ b/jass/models/inittable.py
@@ -28,9 +28,16 @@ class InitMeta(object):
def get_inittable_meta(file_name):
init_store = HDFStore(file_name, mode='r')
nb_snps = init_store.get_storer("SumStatTab").nrows
+ name=f"Name of {file_name.split('/')[-1]}"
+ desc=f"Description {file_name.split('/')[-1]}"
init_store.close()
nb_phenotypes = read_hdf(file_name, "PhenoList").shape[0]
- return {"nb_snps":int(nb_snps), "nb_phenotypes":int(nb_phenotypes)}
+ return dict(
+ nb_snps=int(nb_snps),
+ nb_phenotypes=int(nb_phenotypes),
+ name=name,
+ desc=desc,
+ )
def get_gwasname(file_name):
return "_".join(os.path.basename(file_name).split("_")[0:3])
diff --git a/jass/models/phenotype.py b/jass/models/phenotype.py
index 885cf05428ea0be79212961083ba0a110e962357..b19bb5d7beeca5c02de5e683764ad4ced3cabe78 100644
--- a/jass/models/phenotype.py
+++ b/jass/models/phenotype.py
@@ -1,8 +1,12 @@
# coding: utf-8
-from typing import List
+from typing import List, Optional
+import os
import pandas
-from pydantic import BaseModel
+import re
+from pydantic import BaseModel, validator
+
+from jass.config import config
class Phenotype(BaseModel):
@@ -72,5 +76,20 @@ def get_available_phenotypes(init_file_path: str):
return phenotypes
-class PhenotypeIdList(BaseModel):
+_initTableNamePattern = re.compile("^([A-Z]*[a-z]*-*\.?[0-9]*)+$")
+
+
+class InitTableNameModel(BaseModel):
+ initTableName: Optional[str] = "initTable.hdf5"
+
+ @validator("initTableName")
+ def validate_path_injection(cls, value):
+ if not _initTableNamePattern.match(value):
+ raise ValueError(f"Prohibited char, only \"{_initTableNamePattern.pattern}\" allowed.")
+ return value
+
+ def get_init_table_path(self):
+ return os.path.join(config["DATA_DIR"], self.initTableName)
+
+class PhenotypeIdList(InitTableNameModel, BaseModel):
phenotypeID: List[str] = []
diff --git a/jass/server.py b/jass/server.py
index 2912409ee55d497e8b0c0acd2b89cf536f96d966..12473e650e94998a41555413ead91334840ce3c1 100644
--- a/jass/server.py
+++ b/jass/server.py
@@ -12,7 +12,7 @@ from tables import HDF5ExtError
from jass import util
from jass.config import config
-from jass.models.phenotype import Phenotype, get_available_phenotypes, PhenotypeIdList
+from jass.models.phenotype import Phenotype, get_available_phenotypes, PhenotypeIdList, InitTableNameModel
from jass.models.inittable import get_inittable_meta
from jass.models.project import GlobalProject, load_project as project__load_project
from jass.tasks import create_project, run_project_analysis_if_needed, get_queue_status
@@ -52,22 +52,33 @@ async def read_index():
return RedirectResponse(url="/webui/")
-@app.get("/api/phenotypes", response_model=List[Phenotype])
-def phenotypes_list():
+@app.get("/api/tables", response_model=List[str])
+def inittable_list():
+ """List initTables"""
+ for filename in os.listdir(config["DATA_DIR"]):
+ if filename.endswith(".hdf5") and "worktable" not in filename:
+ yield filename
+
+
+@app.post("/api/phenotypes", response_model=List[Phenotype])
+def phenotypes_list(init_table_name: InitTableNameModel):
"""List phenotypes"""
- return get_available_phenotypes(os.path.join(config["DATA_DIR"], "initTable.hdf5"))
+ try:
+ return get_available_phenotypes(init_table_name.get_init_table_path())
+ except FileNotFoundError as e: # initTable does not exists
+ raise HTTPException(status_code=404, detail=str(e))
-@app.get("/api/initmeta")
-def inittable_meta():
+@app.post("/api/initmeta")
+def inittable_meta(init_table_name: InitTableNameModel):
"""Number of phenotype and SNPs"""
- return get_inittable_meta(os.path.join(config["DATA_DIR"], "initTable.hdf5"))
+ return get_inittable_meta(init_table_name.get_init_table_path())
@app.post("/api/projects", response_model=GlobalProject)
def project_create(phenotype_id_list: PhenotypeIdList):
return create_project(
phenotype_id_list.phenotypeID,
- get_available_phenotypes(os.path.join(config["DATA_DIR"], "initTable.hdf5")),
+ init_table_name=phenotype_id_list.initTableName,
)
diff --git a/jass/tasks.py b/jass/tasks.py
index 25c96082385add317e0be42d8211e9c305e07ea0..82e9b9d9c014f7468726cfb268e376099ff9b3cb 100644
--- a/jass/tasks.py
+++ b/jass/tasks.py
@@ -10,7 +10,7 @@ from flask import Flask
import jass.models.project
from jass.models.project import GlobalProject, Project, ensure_space_in_project_dir
-from jass.models.phenotype import Phenotype
+from jass.models.phenotype import Phenotype, get_available_phenotypes
from jass.config import config
@@ -155,22 +155,22 @@ def run_project_analysis_if_needed(project):
def create_project(
phenotype_ids: List[str],
- available_phenotypes: List[Phenotype],
chromosome: str = None,
start: str = None,
end: str = None,
+ init_table_name: str = None,
):
-
- available_phenotype_ids = [phenotype.id for phenotype in available_phenotypes]
+ init_table_path = os.path.join(config["DATA_DIR"], init_table_name)
+ available_phenotypes=get_available_phenotypes(init_table_path)
unavailable_requested_ids = set(phenotype_ids).difference(
- set(available_phenotype_ids)
+ set(phenotype.id for phenotype in available_phenotypes)
)
if len(unavailable_requested_ids) > 0:
raise HTTPException(status_code=404, detail=f"Phenotype IDs not found: {','.join(unavailable_requested_ids)}")
phenotypes = [
phenotype for phenotype in available_phenotypes if phenotype.id in phenotype_ids
]
- project = GlobalProject(phenotypes=phenotypes)
+ project = GlobalProject(phenotypes=phenotypes, init_table_path=init_table_path)
project.create(fail_if_exists=False)
ensure_space_in_project_dir(except_project_id=project.id)
diff --git a/jass/test/__init__.py b/jass/test/__init__.py
index 25cecd228521526e63f108c1b71fb40649abfcaf..7f6ab04bc94ab97ebf0e1746bbab91735d233504 100644
--- a/jass/test/__init__.py
+++ b/jass/test/__init__.py
@@ -30,6 +30,10 @@ class JassWebClientTestCase(JassTestCase):
self.test_dir = tempfile.mkdtemp()
config["DATA_DIR"] = self.test_dir
shutil.copy(self.get_file_path_fn("initTable.hdf5"), self.test_dir)
+ try:
+ shutil.copy(self.get_file_path_fn("initTableTest1.hdf5"), self.test_dir)
+ except FileNotFoundError:
+ pass
from jass.server import app
self.testing_client = TestClient(app)
diff --git a/jass/test/data_real/initTableTest1.hdf5 b/jass/test/data_real/initTableTest1.hdf5
new file mode 100644
index 0000000000000000000000000000000000000000..d49423695dca6a4ffa014e0aaf0ec45f02ffb68c
Binary files /dev/null and b/jass/test/data_real/initTableTest1.hdf5 differ
diff --git a/jass/test/test_server.py b/jass/test/test_server.py
index 4bf6b023063b356a51b0ae240ee1e82a36c0f5f1..024202f4c4d35470b0a22118e7300871197b47e1 100644
--- a/jass/test/test_server.py
+++ b/jass/test/test_server.py
@@ -13,28 +13,68 @@ from . import JassWebClientTestCase
class TestDefaultController(JassWebClientTestCase):
"""DefaultController integration test stubs"""
- test_folder = "data_test1"
+ test_folder = "data_real"
- def test_phenotypes_get(self):
+ def test_phenotypes_post(self):
"""
Test case retrieving available phenotypes
"""
- response = self.testing_client.get("/api/phenotypes")
+ response = self.testing_client.post("/api/phenotypes", json={})
self.assert200(response, "Response body is : " + response.content.decode("utf-8"))
+ response = self.testing_client.post(
+ "/api/phenotypes",
+ json={"initTableName": "initTable.hdf5"},
+ )
+ self.assert200(response, "Response body is : " + response.content.decode("utf-8"))
+ json_response_main = json.loads(response.content.decode("utf-8"))
+ phenotypes_main = set(p["id"] for p in json_response_main)
+
+ response = self.testing_client.post(
+ "/api/phenotypes",
+ json={"initTableName": "initTableTest1.hdf5"},
+ )
+ self.assert200(response, "Response body is : " + response.content.decode("utf-8"))
+ json_response_t1 = json.loads(response.content.decode("utf-8"))
+ phenotypes_t1 = set(p["id"] for p in json_response_t1)
+
+ self.assertNotEqual(json_response_t1, json_response_main)
+ self.assertNotEqual(phenotypes_main, phenotypes_t1)
+ self.assertEqual(phenotypes_main.intersection(phenotypes_t1), set())
+
+ response = self.testing_client.post(
+ "/api/phenotypes",
+ json={"initTableName": "initTableMissing.hdf5"},
+ )
+ self.assertEqual(response.status_code, 404, response.content.decode("utf-8"))
+
def test_create_project(self):
"""
Test case for creating a project
"""
response = self.testing_client.post(
"/api/projects",
- json={"phenotypeID": ["z_DISNEY_POCAHONT"]},
+ json={"phenotypeID": ["z_MAGIC_FAST-GLUCOSE"]},
)
self.assert200(response, "Response body is : " + response.content.decode("utf-8"))
def test_initmeta(self):
- response = self.testing_client.get("/api/initmeta")
+ response = self.testing_client.post("/api/initmeta", json={})
+ self.assert200(response, "Response body is : " + response.content.decode("utf-8"))
+ respMain = json.loads(response.content.decode("utf-8"))
+
+ response = self.testing_client.post("/api/initmeta", json={"initTableName": "initTableTest1.hdf5"})
+ self.assert200(response, "Response body is : " + response.content.decode("utf-8"))
+ respT1 = json.loads(response.content.decode("utf-8"))
+
+ self.assertNotEqual(respT1, respMain)
+ self.assertSetEqual(set(respMain.keys()), {'nb_phenotypes', 'nb_snps', 'name', 'desc'})
+
+ def test_get_tables(self):
+ response = self.testing_client.get("/api/tables")
self.assert200(response, "Response body is : " + response.content.decode("utf-8"))
+ resp = json.loads(response.content.decode("utf-8"))
+ self.assertSetEqual({"initTable.hdf5", "initTableTest1.hdf5"}, set(resp))
if __name__ == "__main__":
diff --git a/test_docker_compose.sh b/test_docker_compose.sh
index 63e88f45f3bd0198e44c12b03da1eb6aba8f25f6..065e9633f738282b499bd295a8db824ec8c6efd2 100755
--- a/test_docker_compose.sh
+++ b/test_docker_compose.sh
@@ -25,16 +25,40 @@ curl -I http://0.0.0.0:3001/phenotypes/ \
if [ "$(cat log/02-*.log | grep "200 OK" | wc -l)" == "0" ]; then exit 02; fi
+echo "Test fetching phenotypes through API"
+curl --location --request POST http://0.0.0.0:8080/api/phenotypes/ \
+--header 'Content-Type: application/json' \
+--data-raw '{"initTableName":"initTableTest1.hdf5"}' \
+ -o log/11-post-phenotypes.json \
+ 1> log/11-post-phenotypes.log \
+ 2> log/11-post-phenotypes.err
+
+if [ "$(cat log/11-*.json | grep "Internal Server Error" | wc -l)" == "1" ]; then exit 11; fi
+
+
+echo "Test fetching phenotypes through API"
+curl --location --request POST http://0.0.0.0:8080/api/phenotypes/ \
+--header 'Content-Type: application/json' \
+--data-raw '{"initTableName":"initTable.hdf5"}' \
+ -o log/12-post-phenotypes.json \
+ 1> log/12-post-phenotypes.log \
+ 2> log/12-post-phenotypes.err
+
+if [ "$(cat log/12-*.json | grep "Internal Server Error" | wc -l)" == "1" ]; then exit 12; fi
+
+if [ "$(cat log/11-*.json)" == "$(cat log/12-*.json)" ]; then echo "should not be equal"; exit 12; fi
+
+
echo "Testing project creation"
curl --location --request POST 'http://0.0.0.0:8080/api/projects' \
--header 'Accept: text/html' \
--header 'Content-Type: application/json' \
--data-raw '{"phenotypeID": ["z_MAGIC_FAST-GLUCOSE", "z_MAGIC_FAST-INSULIN"]}' \
- -o log/03-post-phenotypes.json \
- 1> log/03-post-phenotypes.log \
- 2> log/03-post-phenotypes.err
+ -o log/03-post-project.json \
+ 1> log/03-post-project.log \
+ 2> log/03-post-project.err
-PROJECT_ID=$(jq .id log/03-post-phenotypes.json | sed 's/\"//g')
+PROJECT_ID=$(jq .id log/03-post-project.json | sed 's/\"//g')
if [ ! -d "projects/project_$PROJECT_ID" ]; then
exit 03
fi