integrate new tests

Merge branch 'rework-tests' into predict_gain

jass/test/data_test2/z_FIAT_CINQUECENTO_chr2.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr2_reg03_07	1	A	G	97.001
-rs_chr2_reg03_08	2	A	G	98.001
-rs_chr2_reg03_09	3	A	G	99.001
-rs_chr2_reg04_10	4	A	G	100.001
-rs_chr2_reg04_11	5	A	G	101.001
-rs_chr2_reg04_12	6	A	G	102.001

jass/test/data_test2/z_FIAT_CINQUECENTO_chr3.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr3_reg05_13	1	A	G	103.001
-rs_chr3_reg05_14	2	A	G	104.001
-rs_chr3_reg05_15	3	A	G	105.001

jass/test/data_test2/z_FIAT_CINQUECENTO_chr4.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr4_reg07_19	1	A	G	109.001
-rs_chr4_reg07_20	2	A	G	110.001
-rs_chr4_reg07_21	3	A	G	111.001
-rs_chr4_reg08_22	4	A	G	112.001
-rs_chr4_reg08_23	5	A	G	113.001
-rs_chr4_reg08_24	6	A	G	114.001

jass/test/data_test2/z_FIAT_CINQUECENTO_chr5.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr5_reg09_25	1	A	G	115.001
-rs_chr5_reg09_26	2	A	G	116.001
-rs_chr5_reg09_27	3	A	G	117.001

jass/test/data_test2/z_MERCO_SMART_chr1.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr1_reg01_01	1	A	G	121.001
-rs_chr1_reg01_02	2	A	G	122.001
-rs_chr1_reg01_03	3	A	G	123.001
-rs_chr1_reg02_04	4	A	G	124.001
-rs_chr1_reg02_05	5	A	G	125.001
-rs_chr1_reg02_06	6	A	G	126.001

jass/test/data_test2/z_MERCO_SMART_chr2.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr2_reg03_07	1	A	G	127.001
-rs_chr2_reg03_08	2	A	G	128.001
-rs_chr2_reg03_09	3	A	G	129.001
-rs_chr2_reg04_10	4	A	G	130.001
-rs_chr2_reg04_11	5	A	G	131.001
-rs_chr2_reg04_12	6	A	G	132.001

jass/test/data_test2/z_MERCO_SMART_chr3.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr3_reg05_13	1	A	G	133.001
-rs_chr3_reg05_14	2	A	G	134.001
-rs_chr3_reg05_15	3	A	G	135.001
-rs_chr3_reg06_16	4	A	G	136.001
-rs_chr3_reg06_17	5	A	G	137.001
-rs_chr3_reg06_18	6	A	G	138.001

jass/test/data_test2/z_MERCO_SMART_chr4.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr4_reg08_22	4	A	G	142.001
-rs_chr4_reg08_23	5	A	G	143.001
-rs_chr4_reg08_24	6	A	G	144.001

jass/test/data_test2/z_MERCO_SMART_chr5.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr5_reg09_25	1	A	G	145.001
-rs_chr5_reg09_26	2	A	G	146.001
-rs_chr5_reg09_27	3	A	G	147.001

jass/test/data_test2/z_TATA_TATANANO_chr1.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr1_reg01_01	1	A	G	151.001
-rs_chr1_reg01_02	2	A	G	152.001
-rs_chr1_reg01_03	3	A	G	153.001
-rs_chr1_reg02_04	4	A	G	154.001
-rs_chr1_reg02_05	5	A	G	155.001
-rs_chr1_reg02_06	6	A	G	156.001

jass/test/data_test2/z_TATA_TATANANO_chr2.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr2_reg03_07	1	A	G	157.001
-rs_chr2_reg03_08	2	A	G	158.001
-rs_chr2_reg03_09	3	A	G	159.001
-rs_chr2_reg04_10	4	A	G	160.001
-rs_chr2_reg04_11	5	A	G	161.001
-rs_chr2_reg04_12	6	A	G	162.001

jass/test/data_test2/z_TATA_TATANANO_chr3.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr3_reg05_13	1	A	G	163.001
-rs_chr3_reg05_14	2	A	G	164.001
-rs_chr3_reg05_15	3	A	G	165.001
-rs_chr3_reg06_16	4	A	G	166.001
-rs_chr3_reg06_17	5	A	G	167.001
-rs_chr3_reg06_18	6	A	G	168.001

jass/test/data_test2/z_TATA_TATANANO_chr4.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr4_reg07_19	1	A	G	169.001
-rs_chr4_reg07_20	2	A	G	170.001
-rs_chr4_reg07_21	3	A	G	171.001

jass/test/data_test2/z_TATA_TATANANO_chr5.txt

-rsid	position	refAllele	altAllele	Z
-rs_chr5_reg09_25	1	A	G	175.001
-rs_chr5_reg09_26	2	A	G	176.001
-rs_chr5_reg09_27	3	A	G	177.001

jass/test/test_init_table.py

@@ -19,10 +19,13 @@ class TestInitTable(object):
         # Create a temporary directory
         self.test_dir = tempfile.mkdtemp()
         input_data_path = self.get_file_path_fn("*chr*.txt")
-        init_covariance_path = self.get_file_path_fn("COV.csv")
+        if self.computed_cov:
+            init_covariance_path = None
+        else:
+            init_covariance_path = self.get_file_path_fn("COV.csv")
         regions_map_path = self.get_file_path_fn("regions.txt")
         description_file_path = self.get_file_path_fn("summary.csv")
-        self.expected_hdf_path = self.get_file_path_fn("initTable.hdf5")
+        self.expected_hdf_path = self.get_file_path_fn(f"initTable{'-computed-cov' if self.computed_cov else ''}.hdf5")
         init_table_path = os.path.join(self.test_dir, "test_init_table.hdf5")
         create_inittable_file(
             input_data_path,
@@ -66,15 +69,15 @@ class TestInitTable(object):
         assert_frame_equal(self.expected_cov, self.result_cov, check_like=True)
 
 
-params = [("nonans", "data_test1"), ("withnans", "data_test2")]
+params = [("nonans", "data_real", True), ("withnans", "data_real", False)]
 
-for name, param in params:
+for name, param, computed_cov in params:
     cls_name = "TestInitTable_%s" % (name,)
     print(cls_name)
     globals()[cls_name] = type(
         cls_name,
         (TestInitTable, JassTestCase),
-        {"test_folder": param, "__test__": True},
+        {"test_folder": param, "computed_cov":computed_cov, "__test__": True},
     )
 
 if __name__ == "__main__":

jass/test/test_plots.py

@@ -2,24 +2,21 @@
 
 from __future__ import absolute_import
 import os, shutil, tempfile
+from pathlib import Path
 
-from pandas import read_hdf
-from pandas.testing import assert_frame_equal
-
-from jass.models.plots import create_global_plot
+from jass.models import plots
 
 from . import JassTestCase
 
 
 class TestPlots(JassTestCase):
 
-    test_folder = "data_test1"
+    test_folder = "data_real"
 
     def setUp(self):
         # Create a temporary directory
-        self.test_dir = tempfile.mkdtemp()
-        self.worktable_hdf_path = self.get_file_path_fn("worktable.hdf5")
-        self.global_plot_path = os.path.join(self.test_dir, "global_manhattan.png")
+        self.test_dir = Path(tempfile.mkdtemp())
+        self.worktable_hdf_path = self.get_file_path_fn("worktable-withnans.hdf5")
 
     def tearDown(self):
         # Remove the directory after the test
@@ -27,10 +24,16 @@ class TestPlots(JassTestCase):
         pass
 
     def test_create_global_plot(self):
-        """
-        Compare result and expected SumStatJostTab
-        """
-        create_global_plot(self.worktable_hdf_path, self.global_plot_path)
+        plots.create_global_plot(self.worktable_hdf_path, self.test_dir / "global_plot.png")
+
+    def test_create_qq_plot(self):
+        plots.create_qq_plot(self.worktable_hdf_path, self.test_dir / "qq_plot.png")
+
+    def test_create_quadrant_plot(self):
+        plots.create_quadrant_plot(self.worktable_hdf_path, self.test_dir / "quadrant_plot.png")
+
+    def test_create_qq_plot_by_GWAS(self):
+        plots.create_qq_plot_by_GWAS(self.worktable_hdf_path, self.test_dir)
 
 
 if __name__ == "__main__":

jass/test/test_worktable.py

@@ -21,7 +21,7 @@ class TestWorkTable(object):
         cls.test_dir = tempfile.mkdtemp()
         init_file_path = cls.get_file_path_fn("initTable.hdf5")
         cls.result_hdf_path = os.path.join(cls.test_dir, "test_worktable.hdf5")
-        cls.expected_hdf_path = cls.get_file_path_fn("worktable.hdf5")
+        cls.expected_hdf_path = cls.get_file_path_fn(f"worktable{'-nonans' if cls.remove_nan else '-withnans'}.hdf5")
         # with coerce_pickle_protocol(4):
         create_worktable_file(
             cls.phenotypes_sel, init_file_path, cls.result_hdf_path, cls.remove_nan
@@ -84,21 +84,16 @@ class TestWorkTable(object):
         )
 
 
-phenotypes_disney = ["z_DISNEY_POCAHONT", "z_DISNEY_RATATOUY"]
-phenotypes_cars = [
-    "z_BMW_ISETTA",
-    "z_BMW_MINI",
-    "z_FIAT_CINQCENT",
-    "z_FIAT_CINQUECENTO",
-    "z_MERCO_SMART",
-    "z_TATA_TATANANO",
+phenotypes_real = [
+    "z_MAGIC_FAST-GLUCOSE",
+    "z_MAGIC_FAST-INSULIN",
+    "z_MAGIC_GLUCOSE-TOLERANCE",
+    "z_MAGIC_HBA1C",
 ]
 
 params = [
-    ("nonans", "data_test1", phenotypes_disney, True),
-    ("withnans", "data_test2", phenotypes_cars, False),
-    # ("nonans", "data_test1_37", phenotypes_disney, True),
-    # ("withnans", "data_test2_37", phenotypes_cars, False),
+    ("nonans", "data_real", phenotypes_real, True),
+    ("withnans", "data_real", phenotypes_real, False),
 ]
 
 for name, param, phenotypes_sel, remove_nan in params:

jass/test/update_test_hdf5_files.sh

@@ -25,7 +25,13 @@ for DATA_DIR in $DATA_DIRS; do
   echo "Creating inittable"
   jass create-inittable --input-data-path "./${DATA_DIR}/z*.txt" --init-covariance-path "./${DATA_DIR}/COV.csv" --init-genetic-covariance-path ${GEN_COV} --regions-map-path "./${DATA_DIR}/regions.txt" --description-file-path "./${DATA_DIR}/summary.csv" --init-table-metadata-path "./${DATA_DIR}/metadata.txt" --init-table-path "./${DATA_DIR}/initTable.hdf5"
 
+  echo "Creating inittable without pre-computed covariance matrix"
+  jass create-inittable --input-data-path "./${DATA_DIR}/z*.txt"                                                --init-genetic-covariance-path ${GEN_COV} --regions-map-path "./${DATA_DIR}/regions.txt" --description-file-path "./${DATA_DIR}/summary.csv" --init-table-metadata-path "./${DATA_DIR}/metadata.txt" --init-table-path "./${DATA_DIR}/initTable-computed-cov.hdf5"
+
+  echo "Creating worktable with Nan"
+  jass create-project-data --init-table-path "${DATA_DIR}/initTable.hdf5" --phenotype ${TRAITS} --worktable-path ./${DATA_DIR}/worktable-withnans.hdf5
+
   echo "Creating worktable"
-  jass create-project-data --init-table-path "${DATA_DIR}/initTable.hdf5" --phenotype ${TRAITS} --worktable-path ./${DATA_DIR}/worktable.hdf5
+  jass create-project-data --init-table-path "${DATA_DIR}/initTable.hdf5" --phenotype ${TRAITS} --worktable-path ./${DATA_DIR}/worktable-nonans.hdf5 --remove-nans
 
 done
\ No newline at end of file