add filter imputation quality

92a461de · hjulienne · 8f85b369 · 92a461de · 92a461de · 92a461de
Commit 92a461de authored Nov 29, 2019 by hjulienne
31 changed files
--- a/doc/_build/doctrees/_autosummary/jass_preprocessing.doctree
+++ b/doc/_build/doctrees/_autosummary/jass_preprocessing.doctree
--- a/doc/_build/doctrees/_autosummary/modules.doctree
+++ b/doc/_build/doctrees/_autosummary/modules.doctree
--- a/doc/_build/doctrees/environment.pickle
+++ b/doc/_build/doctrees/environment.pickle
--- a/doc/_build/doctrees/index.doctree
+++ b/doc/_build/doctrees/index.doctree
--- a/doc/_build/html/.buildinfo
+++ b/doc/_build/html/.buildinfo
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 7890896c7cf0d3614ade393a5c103246
+config: f0cd103aab646a141229409df671f02e
 tags: 645f666f9bcd5a90fca523b33c5a78b7
--- a/doc/_build/html/_autosummary/jass_preprocessing.html
+++ b/doc/_build/html/_autosummary/jass_preprocessing.html
--- a/doc/_build/html/_autosummary/modules.html
+++ b/doc/_build/html/_autosummary/modules.html
@@ -17,32 +17,34 @@
  

  
-
-  
+  <script type="text/javascript" src="../_static/js/modernizr.min.js"></script>
  
    
+      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../_static/jquery.js"></script>
+        <script type="text/javascript" src="../_static/underscore.js"></script>
+        <script type="text/javascript" src="../_static/doctools.js"></script>
+        <script type="text/javascript" src="../_static/language_data.js"></script>
+    
+    <script type="text/javascript" src="../_static/js/theme.js"></script>

-  
+    

+  
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" /> 
-
-  
-  <script src="../_static/js/modernizr.min.js"></script>
-
 </head>

 <body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
-
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search">
+        <div class="wy-side-nav-search" >
          

          
@@ -154,9 +156,7 @@
 <li class="toctree-l2"><a class="reference internal" href="jass_preprocessing.html#module-jass_preprocessing.map_gwas">jass_preprocessing.map_gwas module</a></li>
 <li class="toctree-l2"><a class="reference internal" href="jass_preprocessing.html#module-jass_preprocessing.map_reference">jass_preprocessing.map_reference module</a></li>
 <li class="toctree-l2"><a class="reference internal" href="jass_preprocessing.html#module-jass_preprocessing.save_output">jass_preprocessing.save_output module</a></li>
-<li class="toctree-l2"><a class="reference internal" href="jass_preprocessing.html#module-jass_preprocessing">Module contents</a><ul class="simple">
-</ul>
-</li>
+<li class="toctree-l2"><a class="reference internal" href="jass_preprocessing.html#module-jass_preprocessing">Module contents</a></li>
 </ul>
 </li>
 </ul>
@@ -191,25 +191,16 @@
  


-  
-
-    
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
-        <script type="text/javascript" src="../_static/jquery.js"></script>
-        <script type="text/javascript" src="../_static/underscore.js"></script>
-        <script type="text/javascript" src="../_static/doctools.js"></script>
-    
-
-  
-
-  <script type="text/javascript" src="../_static/js/theme.js"></script>
-
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
-  </script> 
+  </script>
+
+  
+  
+    
+   

 </body>
 </html>
\ No newline at end of file
--- a/doc/_build/html/_modules/index.html
+++ b/doc/_build/html/_modules/index.html
@@ -17,32 +17,34 @@
  

  
-
-  
+  <script type="text/javascript" src="../_static/js/modernizr.min.js"></script>
  
    
+      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../_static/jquery.js"></script>
+        <script type="text/javascript" src="../_static/underscore.js"></script>
+        <script type="text/javascript" src="../_static/doctools.js"></script>
+        <script type="text/javascript" src="../_static/language_data.js"></script>
+    
+    <script type="text/javascript" src="../_static/js/theme.js"></script>

-  
+    

+  
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" /> 
-
-  
-  <script src="../_static/js/modernizr.min.js"></script>
-
 </head>

 <body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
-
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search">
+        <div class="wy-side-nav-search" >
          

          
@@ -171,25 +173,16 @@
  


-  
-
-    
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
-        <script type="text/javascript" src="../_static/jquery.js"></script>
-        <script type="text/javascript" src="../_static/underscore.js"></script>
-        <script type="text/javascript" src="../_static/doctools.js"></script>
-    
-
-  
-
-  <script type="text/javascript" src="../_static/js/theme.js"></script>
-
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
-  </script> 
+  </script>
+
+  
+  
+    
+   

 </body>
 </html>
\ No newline at end of file
--- a/doc/_build/html/_modules/jass_preprocessing/compute_score.html
+++ b/doc/_build/html/_modules/jass_preprocessing/compute_score.html
@@ -17,32 +17,34 @@
  

  
-
-  
+  <script type="text/javascript" src="../../_static/js/modernizr.min.js"></script>
  
    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../_static/language_data.js"></script>
+    
+    <script type="text/javascript" src="../../_static/js/theme.js"></script>

-  
+    

+  
  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" /> 
-
-  
-  <script src="../../_static/js/modernizr.min.js"></script>
-
 </head>

 <body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
-
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search">
+        <div class="wy-side-nav-search" >
          

          
@@ -147,7 +149,7 @@
 <span class="kn">import</span> <span class="nn">seaborn</span> <span class="k">as</span> <span class="nn">sns</span>
 <span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>

-<span class="n">perSS</span> <span class="o">=</span> <span class="mf">0.7</span>
+

 <div class="viewcode-block" id="compute_z_score"><a class="viewcode-back" href="../../_autosummary/jass_preprocessing.html#jass_preprocessing.compute_score.compute_z_score">[docs]</a><span class="k">def</span> <span class="nf">compute_z_score</span><span class="p">(</span><span class="n">mgwas</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
@@ -168,7 +170,7 @@

    <span class="k">return</span> <span class="n">mgwas</span></div>

-<div class="viewcode-block" id="compute_sample_size"><a class="viewcode-back" href="../../_autosummary/jass_preprocessing.html#jass_preprocessing.compute_score.compute_sample_size">[docs]</a><span class="k">def</span> <span class="nf">compute_sample_size</span><span class="p">(</span><span class="n">mgwas</span><span class="p">,</span> <span class="n">diagnostic_folder</span><span class="p">,</span> <span class="n">trait</span><span class="p">):</span>
+<div class="viewcode-block" id="compute_sample_size"><a class="viewcode-back" href="../../_autosummary/jass_preprocessing.html#jass_preprocessing.compute_score.compute_sample_size">[docs]</a><span class="k">def</span> <span class="nf">compute_sample_size</span><span class="p">(</span><span class="n">mgwas</span><span class="p">,</span> <span class="n">diagnostic_folder</span><span class="p">,</span> <span class="n">trait</span><span class="p">,</span> <span class="n">perSS</span> <span class="o">=</span> <span class="mf">0.7</span><span class="p">):</span>

    <span class="k">if</span> <span class="s1">&#39;n&#39;</span> <span class="ow">in</span> <span class="n">mgwas</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
        <span class="n">myN</span> <span class="o">=</span> <span class="n">mgwas</span><span class="o">.</span><span class="n">n</span>
@@ -192,7 +194,7 @@
        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Some snp had an infinite sample size&quot;</span><span class="p">)</span>

    <span class="n">myW_thres</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">percentile</span><span class="p">(</span><span class="n">myN</span><span class="o">.</span><span class="n">dropna</span><span class="p">(),</span> <span class="mi">90</span><span class="p">)</span>
-    <span class="n">ss_thres</span> <span class="o">=</span> <span class="n">perSS</span> <span class="o">*</span> <span class="n">myW_thres</span>
+    <span class="n">ss_thres</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">perSS</span><span class="p">)</span> <span class="o">*</span> <span class="n">myW_thres</span>
    <span class="n">mgwas</span><span class="p">[</span><span class="s2">&quot;computed_N&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">myN</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">clf</span><span class="p">()</span>
    <span class="n">p1</span> <span class="o">=</span> <span class="n">sns</span><span class="o">.</span><span class="n">distplot</span><span class="p">(</span><span class="n">mgwas</span><span class="o">.</span><span class="n">computed_N</span><span class="p">[</span><span class="o">~</span><span class="n">mgwas</span><span class="o">.</span><span class="n">computed_N</span><span class="o">.</span><span class="n">isna</span><span class="p">()])</span>
@@ -235,25 +237,16 @@
  


-  
-
-    
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
-        <script type="text/javascript" src="../../_static/jquery.js"></script>
-        <script type="text/javascript" src="../../_static/underscore.js"></script>
-        <script type="text/javascript" src="../../_static/doctools.js"></script>
-    
-
-  
-
-  <script type="text/javascript" src="../../_static/js/theme.js"></script>
-
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
-  </script> 
+  </script>
+
+  
+  
+    
+   

 </body>
 </html>
\ No newline at end of file
--- a/doc/_build/html/_modules/jass_preprocessing/dna_utils.html
+++ b/doc/_build/html/_modules/jass_preprocessing/dna_utils.html
@@ -17,32 +17,34 @@
  

  
-
-  
+  <script type="text/javascript" src="../../_static/js/modernizr.min.js"></script>
  
    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../_static/language_data.js"></script>
+    
+    <script type="text/javascript" src="../../_static/js/theme.js"></script>

-  
+    

+  
  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" /> 
-
-  
-  <script src="../../_static/js/modernizr.min.js"></script>
-
 </head>

 <body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
-
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search">
+        <div class="wy-side-nav-search" >
          

          
@@ -183,25 +185,16 @@
  


-  
-
-    
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
-        <script type="text/javascript" src="../../_static/jquery.js"></script>
-        <script type="text/javascript" src="../../_static/underscore.js"></script>
-        <script type="text/javascript" src="../../_static/doctools.js"></script>
-    
-
-  
-
-  <script type="text/javascript" src="../../_static/js/theme.js"></script>
-
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
-  </script> 
+  </script>
+
+  
+  
+    
+   

 </body>
 </html>
\ No newline at end of file
--- a/doc/_build/html/_modules/jass_preprocessing/map_gwas.html
+++ b/doc/_build/html/_modules/jass_preprocessing/map_gwas.html
@@ -17,32 +17,34 @@
  

  
-
-  
+  <script type="text/javascript" src="../../_static/js/modernizr.min.js"></script>
  
    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../_static/language_data.js"></script>
+    
+    <script type="text/javascript" src="../../_static/js/theme.js"></script>

-  
+    

+  
  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" /> 
-
-  
-  <script src="../../_static/js/modernizr.min.js"></script>
-
 </head>

 <body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
-
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search">
+        <div class="wy-side-nav-search" >
          

          
@@ -151,6 +153,9 @@
 <span class="kn">import</span> <span class="nn">sys</span>
 <span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
 <span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
+<span class="kn">import</span> <span class="nn">gzip</span>
+<span class="kn">import</span> <span class="nn">re</span>
+

 <div class="viewcode-block" id="walkfs"><a class="viewcode-back" href="../../_autosummary/jass_preprocessing.html#jass_preprocessing.map_gwas.walkfs">[docs]</a><span class="k">def</span> <span class="nf">walkfs</span><span class="p">(</span><span class="n">startdir</span><span class="p">,</span> <span class="n">findfile</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
@@ -214,46 +219,48 @@
    <span class="k">return</span> <span class="n">df</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">def_missing</span><span class="p">,</span> <span class="n">nan_vec</span><span class="p">)</span></div>


-<div class="viewcode-block" id="map_columns_position"><a class="viewcode-back" href="../../_autosummary/jass_preprocessing.html#jass_preprocessing.map_gwas.map_columns_position">[docs]</a><span class="k">def</span> <span class="nf">map_columns_position</span><span class="p">(</span><span class="n">gwas_internal_link</span><span class="p">,</span>  <span class="n">GWAS_labels</span><span class="p">):</span>
+<div class="viewcode-block" id="map_columns_position"><a class="viewcode-back" href="../../_autosummary/jass_preprocessing.html#jass_preprocessing.map_gwas.map_columns_position">[docs]</a><span class="k">def</span> <span class="nf">map_columns_position</span><span class="p">(</span><span class="n">gwas_internal_link</span><span class="p">,</span>  <span class="n">column_dict</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Find column position for each specific Gwas</span>

 <span class="sd">    Args:</span>
 <span class="sd">        gwas_internal_link (str): filename of the GWAS data (with path)</span>
-<span class="sd">        GWAS_labels (str): filename of the csv information file</span>
+<span class="sd">        GWAS_labels (pd.DataFrame): corresponding row of the information file</span>

 <span class="sd">    Return:</span>
 <span class="sd">        pandas Series with column position and column names as index</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">column_dict</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">GWAS_labels</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">na_values</span><span class="o">=</span><span class="s1">&#39;na&#39;</span><span class="p">,</span> <span class="n">index_col</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
-
+    <span class="nb">print</span><span class="p">(</span><span class="n">gwas_internal_link</span><span class="p">)</span>
    <span class="n">gwas_file</span> <span class="o">=</span> <span class="n">gwas_internal_link</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
-
-    <span class="n">my_labels</span> <span class="o">=</span> <span class="n">column_dict</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">gwas_file</span><span class="p">]</span>
-
    <span class="c1">#Our standart labels:</span>
-    <span class="n">reference_label</span> <span class="o">=</span> <span class="n">column_dict</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
-    <span class="c1"># labels in the GWAS files</span>
-    <span class="n">target_lab</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Index</span><span class="p">(</span><span class="n">my_labels</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tolist</span><span class="p">())</span>

-    <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">gwas_internal_link</span><span class="p">)</span>
+    <span class="n">reference_label</span> <span class="o">=</span> <span class="n">column_dict</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">reference_label</span><span class="p">)</span>
+    <span class="c1"># labels in the GWAS files</span>
+    <span class="n">target_lab</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Index</span><span class="p">(</span><span class="n">column_dict</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tolist</span><span class="p">())</span>
+    <span class="n">is_gzipped</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;.gz$&quot;</span><span class="p">,</span> <span class="n">gwas_internal_link</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">is_gzipped</span><span class="p">:</span>
+        <span class="n">f</span> <span class="o">=</span> <span class="n">gzip</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">gwas_internal_link</span><span class="p">)</span>
+        <span class="n">line</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
+        <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">gwas_internal_link</span><span class="p">)</span>
+        <span class="n">line</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
    <span class="n">count_line</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="n">line</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
-    <span class="nb">print</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
-    <span class="n">header</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Index</span><span class="p">(</span><span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">())</span>

+    <span class="n">header</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Index</span><span class="p">(</span><span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">())</span>
    <span class="k">def</span> <span class="nf">get_position</span><span class="p">(</span><span class="n">I</span><span class="p">,</span><span class="n">x</span><span class="p">):</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">return</span> <span class="n">I</span><span class="o">.</span><span class="n">get_loc</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span>
-    <span class="n">label_position</span> <span class="o">=</span> <span class="p">[</span><span class="n">get_position</span><span class="p">(</span><span class="n">header</span><span class="p">,</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">target_lab</span><span class="p">]</span>

+    <span class="n">label_position</span> <span class="o">=</span> <span class="p">[</span><span class="n">get_position</span><span class="p">(</span><span class="n">header</span><span class="p">,</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">target_lab</span><span class="p">]</span>

    <span class="n">mapgw</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">label_position</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="n">reference_label</span><span class="p">)</span>
    <span class="n">mapgw</span> <span class="o">=</span> <span class="n">mapgw</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="o">~</span><span class="n">mapgw</span><span class="o">.</span><span class="n">isna</span><span class="p">()]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>
    <span class="n">mapgw</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-    <span class="nb">print</span><span class="p">(</span><span class="n">mapgw</span><span class="p">)</span>
+
    <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
    <span class="k">return</span> <span class="n">mapgw</span></div>

@@ -269,18 +276,28 @@
 <span class="sd">    Return:</span>
 <span class="sd">        a pandas dataframe with missing value all equal to np.nan</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-
+    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Reading file:&quot;</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">gwas_internal_link</span><span class="p">)</span>
+    <span class="n">is_gzipped</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;.gz$&quot;</span><span class="p">,</span> <span class="n">gwas_internal_link</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">is_gzipped</span><span class="p">:</span>
+        <span class="n">compression</span> <span class="o">=</span> <span class="s1">&#39;gzip&#39;</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">compression</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="nb">print</span><span class="p">(</span><span class="n">column_map</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">column_map</span><span class="o">.</span><span class="n">index</span><span class="p">)</span>
    <span class="n">fullGWAS</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">gwas_internal_link</span><span class="p">,</span> <span class="n">delim_whitespace</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-                               <span class="n">usecols</span> <span class="o">=</span> <span class="n">column_map</span><span class="o">.</span><span class="n">values</span><span class="p">,</span> <span class="c1">#column_dict[&#39;label_position&#39;].keys(),</span>
+                               <span class="n">usecols</span> <span class="o">=</span> <span class="n">column_map</span><span class="o">.</span><span class="n">values</span><span class="p">,</span>
+                               <span class="n">compression</span><span class="o">=</span><span class="n">compression</span><span class="p">,</span>
+                                <span class="c1">#column_dict[&#39;label_position&#39;].keys(),</span>
                               <span class="n">names</span><span class="o">=</span> <span class="n">column_map</span><span class="o">.</span><span class="n">index</span><span class="p">,</span>
-                                <span class="n">index_col</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-                                 <span class="n">header</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">na_values</span><span class="o">=</span> <span class="p">[</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="s1">&#39;#N/A&#39;</span><span class="p">,</span> <span class="s1">&#39;#N/A&#39;</span><span class="p">,</span> <span class="s1">&#39;N/A&#39;</span><span class="p">,</span>
-                                                       <span class="s1">&#39;#NA&#39;</span><span class="p">,</span> <span class="s1">&#39;-1.#IND&#39;</span><span class="p">,</span> <span class="s1">&#39;-1.#QNAN&#39;</span><span class="p">,</span>
-                                                 <span class="s1">&#39;-NaN&#39;</span><span class="p">,</span>
-                                                 <span class="s1">&#39;-nan&#39;</span><span class="p">,</span> <span class="s1">&#39;1.#IND&#39;</span><span class="p">,</span> <span class="s1">&#39;1.#QNAN&#39;</span><span class="p">,</span> <span class="s1">&#39;N/A&#39;</span><span class="p">,</span>
+                                 <span class="n">header</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">na_values</span><span class="o">=</span> <span class="p">[</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="s1">&#39;#N/A&#39;</span><span class="p">,</span> <span class="s1">&#39;#N/A&#39;</span><span class="p">,</span> <span class="s1">&#39;N/A&#39;</span><span class="p">,</span><span class="s1">&#39;#NA&#39;</span><span class="p">,</span> <span class="s1">&#39;-1.#IND&#39;</span><span class="p">,</span> <span class="s1">&#39;-1.#QNAN&#39;</span><span class="p">,</span>
+                                                 <span class="s1">&#39;-NaN&#39;</span><span class="p">,</span> <span class="s1">&#39;-nan&#39;</span><span class="p">,</span> <span class="s1">&#39;1.#IND&#39;</span><span class="p">,</span> <span class="s1">&#39;1.#QNAN&#39;</span><span class="p">,</span> <span class="s1">&#39;N/A&#39;</span><span class="p">,</span>
                                                 <span class="s1">&#39;NA&#39;</span><span class="p">,</span> <span class="s1">&#39;NULL&#39;</span><span class="p">,</span> <span class="s1">&#39;NaN&#39;</span><span class="p">,</span>
-                                                 <span class="s1">&#39;nan&#39;</span><span class="p">,</span> <span class="s1">&#39;na&#39;</span><span class="p">,</span> <span class="s1">&#39;.&#39;</span><span class="p">])</span>
-
+                                                 <span class="s1">&#39;nan&#39;</span><span class="p">,</span> <span class="s1">&#39;na&#39;</span><span class="p">,</span> <span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;-&#39;</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;snpid&quot;</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span> <span class="s2">&quot;a1&quot;</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span><span class="s2">&quot;a2&quot;</span><span class="p">:</span><span class="nb">str</span><span class="p">,</span><span class="s2">&quot;freq&quot;</span><span class="p">:</span><span class="nb">float</span><span class="p">,</span> <span class="s2">&quot;z&quot;</span><span class="p">:</span><span class="nb">float</span><span class="p">,</span><span class="s2">&quot;se&quot;</span><span class="p">:</span><span class="nb">float</span><span class="p">,</span> <span class="s2">&quot;pval&quot;</span><span class="p">:</span><span class="nb">float</span><span class="p">})</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">fullGWAS</span><span class="o">.</span><span class="n">head</span><span class="p">())</span>
+    <span class="n">fullGWAS</span><span class="o">.</span><span class="n">set_index</span><span class="p">(</span><span class="s2">&quot;snpid&quot;</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+    <span class="nb">print</span><span class="p">(</span><span class="n">fullGWAS</span><span class="o">.</span><span class="n">head</span><span class="p">())</span>
    <span class="n">fullGWAS</span> <span class="o">=</span> <span class="n">fullGWAS</span><span class="p">[</span><span class="o">~</span><span class="n">fullGWAS</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">duplicated</span><span class="p">(</span><span class="n">keep</span><span class="o">=</span><span class="s1">&#39;first&#39;</span><span class="p">)]</span>
    <span class="c1">#fullGWAS = convert_missing_values(fullGWAS)</span>

@@ -314,25 +331,16 @@
  


-  
-
-    
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
-        <script type="text/javascript" src="../../_static/jquery.js"></script>
-        <script type="text/javascript" src="../../_static/underscore.js"></script>
-        <script type="text/javascript" src="../../_static/doctools.js"></script>
-    
-
-  
-
-  <script type="text/javascript" src="../../_static/js/theme.js"></script>
-
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
-  </script> 
+  </script>
+
+  
+  
+    
+   

 </body>
 </html>
\ No newline at end of file
--- a/doc/_build/html/_modules/jass_preprocessing/map_reference.html
+++ b/doc/_build/html/_modules/jass_preprocessing/map_reference.html
@@ -17,32 +17,34 @@