From 6f017f28a6908954a545afd2753c87dac3e3591f Mon Sep 17 00:00:00 2001
From: Veronique Legrand <vlegrand@pasteur.fr>
Date: Wed, 25 Aug 2021 17:39:09 +0200
Subject: [PATCH] fixed unit and non regression test; changed non regression
 test for the -q option; fully commented the handling of undef files

---
 src/FqAuxBackend.cpp                          |  2 +
 src/FqAuxBackend.h                            |  2 +-
 src/FqBaseBackend.cpp                         | 17 ++--
 src/FqBaseBackend.h                           |  6 +-
 src/FqMainBackend.cpp                         | 16 ++--
 src/FqMainBackend.h                           |  2 +-
 src/fqreader.cpp                              |  6 +-
 src/fqreader.h                                |  2 +-
 src/unit_test_fqreader.cpp                    | 84 ++++++++---------
 .../non_regression/ids_read_kept_q13m300.txt  | 66 ++++++++++++++
 test/rock_mem.sh                              | 89 +++++++++++++------
 11 files changed, 201 insertions(+), 91 deletions(-)
 create mode 100644 test/data/non_regression/ids_read_kept_q13m300.txt

diff --git a/src/FqAuxBackend.cpp b/src/FqAuxBackend.cpp
index 311e5e3..a61b2f8 100644
--- a/src/FqAuxBackend.cpp
+++ b/src/FqAuxBackend.cpp
@@ -142,9 +142,11 @@ void FqAuxBackend::closeFile() {
     }
 }
 
+/*
 void FqAuxBackend::writeToUndefFile() {
     FqBaseBackend::writeToUndefFile(buf_info);
 }
+*/
 
 
 
diff --git a/src/FqAuxBackend.h b/src/FqAuxBackend.h
index 1f0b863..392d503 100644
--- a/src/FqAuxBackend.h
+++ b/src/FqAuxBackend.h
@@ -35,7 +35,7 @@ public:
     void openFile(char * ficname, unsigned char id);
     void closeFile();
     int getNextRead(rinfo *);
-    void writeToUndefFile();
+    //void writeToUndefFile();
     void resetCurFqRecord();
 
 };
diff --git a/src/FqBaseBackend.cpp b/src/FqBaseBackend.cpp
index ebbfbaf..fd55744 100644
--- a/src/FqBaseBackend.cpp
+++ b/src/FqBaseBackend.cpp
@@ -60,10 +60,10 @@ void FqBaseBackend::closeInputFile() {
 void FqBaseBackend::setOutputFile(char * ofilename) {
     o_filename=ofilename;
 }
-
+/*
 void FqBaseBackend::setUndefFile(char * ficname) {
     undef_filename=ficname;
-}
+}*/
 
 
 void FqBaseBackend::openInputFile() {
@@ -83,13 +83,14 @@ void FqBaseBackend::openOutputFile(){
     openFile4Output(o_filename,&o_f_desc);
 }
 
+/*
 void FqBaseBackend::openUndefFile(){
   //std::cout<<qual_thres.min_correct_k_mers_in_read<<endl;
   if (qual_thres.min_correct_k_mers_in_read>=1) {
     if (undef_filename==NULL) throw std::runtime_error("No file currently associated to this backend for undefined reads.");
     openFile4Output(undef_filename,&undef_f_desc);
   }
-}
+}*/
 
 void FqBaseBackend::openFile4Output(char * filename, int * f_desc) {
     mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
@@ -118,16 +119,17 @@ void FqBaseBackend::closeOutputFile() {
     }
 }
 
-void FqBaseBackend::closeUndefFile() {
+/*void FqBaseBackend::closeUndefFile() {
     if (undef_f_desc!=-1) {
         if (close(undef_f_desc)==-1) err(errno,"cannot close file: %s.",undef_filename);
         undef_f_desc=-1;
     }
-}
+}*/
 
+/*
 void FqBaseBackend::writeStrToUndefFile(char * start_in_buf, int len) {
     if (write(undef_f_desc,start_in_buf,len)==-1) err(errno,"cannot write in file: %s.",undef_filename);
-}
+}*/
 
 /*
  * Writes a fq record to the file containing "undefined" reads.
@@ -138,6 +140,7 @@ void FqBaseBackend::writeStrToUndefFile(char * start_in_buf, int len) {
  * So, values for buf_info.pchar and buf_info.cnt may or may not contain the final \n.
  * This is why I use the addCR parameter to add it or not.
  */
+/*
 void FqBaseBackend::writeToUndefFile(const T_buf_info& buf_info,const int& addCR) {
   int l=strlen(cur_fq_record);
     if (l==0) {
@@ -148,7 +151,7 @@ void FqBaseBackend::writeToUndefFile(const T_buf_info& buf_info,const int& addCR
         cur_fq_record[l+addCR]='\0';
         writeStrToUndefFile(cur_fq_record,strlen(cur_fq_record)); // Try that, if there is performance problem due to resolving inheritance, will find a different way
     }
-}
+}*/
 
 
 
diff --git a/src/FqBaseBackend.h b/src/FqBaseBackend.h
index 8abe845..a91a8ac 100644
--- a/src/FqBaseBackend.h
+++ b/src/FqBaseBackend.h
@@ -99,7 +99,7 @@ protected:
     void onIncScore(T_fq_rec_info& rec_info,T_buf_info& buf_info,int& n);
     // void debug_processBuf(int evt,const T_buf_info& buf_info,const unsigned long& rstart_offset);
 
-    friend void processPEFiles(char *, unsigned char,char * , unsigned char,srp *,char *,char *, size_t);
+    friend void processPEFiles(char *, unsigned char,char * , unsigned char,srp *, size_t);
 
     /* for testing only */
     int test_mode;
@@ -146,11 +146,11 @@ public:
     void openOutputFile();
     void writeToOutput(const unsigned long&);
     void closeOutputFile();
-    void setUndefFile(char * ficname);
+    /*void setUndefFile(char * ficname);
     void openUndefFile();
     void writeStrToUndefFile(char * start_in_buf, int len);
     void writeToUndefFile(const T_buf_info& buf_info,const int& addCR=0);
-    void closeUndefFile();
+    void closeUndefFile();*/
 
     static void setQualThreshold(const FasqQualThreshold& a_qual_thres){
         FqBaseBackend::qual_thres.min_correct_k_mers_in_read=a_qual_thres.min_correct_k_mers_in_read;
diff --git a/src/FqMainBackend.cpp b/src/FqMainBackend.cpp
index 2e2ff6f..fe85cff 100644
--- a/src/FqMainBackend.cpp
+++ b/src/FqMainBackend.cpp
@@ -71,8 +71,8 @@ void FqMainBackend::processFile(char * filename,unsigned char f_id) {
     size_t buffer_size;
     (test_mode)?buffer_size=test_bufsize:buffer_size=FqBaseBackend::bufsize;
 
-    this->openUndefFile();
-    if (p_auxFqProcessor!=NULL) p_auxFqProcessor->openUndefFile();
+    //this->openUndefFile();
+    //if (p_auxFqProcessor!=NULL) p_auxFqProcessor->openUndefFile();
 
     int f_single=open(filename,O_RDONLY,mode);
     if (f_single==-1) {
@@ -90,8 +90,8 @@ void FqMainBackend::processFile(char * filename,unsigned char f_id) {
         processBuf(buf_info,f_id,cur_offset);
     }
     close(f_single);
-    this->closeUndefFile();
-    if (p_auxFqProcessor!=NULL) p_auxFqProcessor->closeUndefFile();
+    /*this->closeUndefFile();
+    if (p_auxFqProcessor!=NULL) p_auxFqProcessor->closeUndefFile();*/
 
     free(buf);
 }
@@ -119,10 +119,12 @@ void FqMainBackend::onEndFastqRecord(T_fq_rec_info& rec_info,const T_buf_info& b
      rec_info.nb_nucleotides_in_read_PE2>0?nb_k_mer_PE2=rec_info.nb_nucleotides_in_read_PE2+1-qual_thres.k:nb_k_mer_PE2=0;
      if (!treat_PE_separately || (p_auxFqProcessor==NULL)) {
          nb_k_mer+=nb_k_mer_PE2;
-         (nb_k_mer-rec_info.nb_k_mers_in_error-rec_info.nb_k_mers_in_error_in_PE2>=qual_thres.min_correct_k_mers_in_read)?ref_k_dim.push_back(rp):writeToUndefFile(bufinfo);
+         //(nb_k_mer-rec_info.nb_k_mers_in_error-rec_info.nb_k_mers_in_error_in_PE2>=qual_thres.min_correct_k_mers_in_read)?ref_k_dim.push_back(rp):writeToUndefFile(bufinfo);
+         if (nb_k_mer-rec_info.nb_k_mers_in_error-rec_info.nb_k_mers_in_error_in_PE2>=qual_thres.min_correct_k_mers_in_read) ref_k_dim.push_back(rp);
      } else {
          // ((nb_k_mer_PE2-rec_info.nb_k_mers_in_error_in_PE2>=1) && (nb_k_mer-rec_info.nb_k_mers_in_error>=1))?ref_k_dim.push_back(rp):writeToUndefFile(bufinfo);
-         ((nb_k_mer_PE2-rec_info.nb_k_mers_in_error_in_PE2>=qual_thres.min_correct_k_mers_in_read) && (nb_k_mer-rec_info.nb_k_mers_in_error>=qual_thres.min_correct_k_mers_in_read))?ref_k_dim.push_back(rp):writeToUndefFile(bufinfo);
+         // ((nb_k_mer_PE2-rec_info.nb_k_mers_in_error_in_PE2>=qual_thres.min_correct_k_mers_in_read) && (nb_k_mer-rec_info.nb_k_mers_in_error>=qual_thres.min_correct_k_mers_in_read))?ref_k_dim.push_back(rp):writeToUndefFile(bufinfo);
+    	 if ((nb_k_mer_PE2-rec_info.nb_k_mers_in_error_in_PE2>=qual_thres.min_correct_k_mers_in_read) && (nb_k_mer-rec_info.nb_k_mers_in_error>=qual_thres.min_correct_k_mers_in_read)) ref_k_dim.push_back(rp);
      }
      // empty buffer keeping current record
      cur_fq_record[0]='\0';
@@ -193,10 +195,12 @@ void FqMainBackend::processBuf(T_buf_info& buf_info,unsigned char f_id,unsigned
  * 1- record was all inside current buffer => write it as is.
  * 2- beginning of record was at the end of one buffer and the end of the record was processed in a 2nd buffer; so, record was stored in memory. In that case, write cur_fq_record to undef file.
  */
+/*
 void FqMainBackend::writeToUndefFile(const T_buf_info& buf_info) {
     FqBaseBackend::writeToUndefFile(buf_info,1); // if this is slow due to inheritance resolution, use template method or other solution. This doesn't semm to slow the perfs according to profiling. So I leave it as is; it's clearer than templates.
     if (p_auxFqProcessor!=NULL) {
         p_auxFqProcessor->writeToUndefFile();
     }
 }
+*/
 
diff --git a/src/FqMainBackend.h b/src/FqMainBackend.h
index 37f5f00..aad846b 100644
--- a/src/FqMainBackend.h
+++ b/src/FqMainBackend.h
@@ -39,7 +39,7 @@ class FqMainBackend : public FqBaseBackend {
     srp * p_scoreReadStruct; /* Where we store information about the reads. */
     // char current_id[50]; used only for debug
     // void debug_processBuf(int evt,const T_buf_info&, const unsigned long &);
-    void writeToUndefFile(const T_buf_info&);
+    //void writeToUndefFile(const T_buf_info&);
     void onEndFastqRecord(T_fq_rec_info& rec_info,const T_buf_info& bufinfo);
 
 
diff --git a/src/fqreader.cpp b/src/fqreader.cpp
index 61bab10..80f7fde 100644
--- a/src/fqreader.cpp
+++ b/src/fqreader.cpp
@@ -51,15 +51,15 @@ void processSingleFile(char * fq_s,unsigned char f_id, srp* io_sr) {
 
 /* Processes 1 pair of files containing PE reads.
  * Used for unit testing only.*/
-void processPEFiles(char * fq_1, unsigned char f_id1,char * fq_2, unsigned char f_id2,srp * io_sr ,char * fq_1_test_undef,char * fq_2_test_undef, size_t test_bufsize) {
+void processPEFiles(char * fq_1, unsigned char f_id1,char * fq_2, unsigned char f_id2,srp * io_sr, size_t test_bufsize) {
     FqAuxBackend be_fq2;
     FqMainBackend be_fq1(io_sr);
     if (test_bufsize) {
         be_fq2.setTestMode(test_bufsize);
         be_fq1.setTestMode(test_bufsize);
     }
-    if (fq_1_test_undef!=NULL) be_fq1.setUndefFile(fq_1_test_undef);
-    if (fq_2_test_undef!=NULL) be_fq2.setUndefFile(fq_2_test_undef);
+    //if (fq_1_test_undef!=NULL) be_fq1.setUndefFile(fq_1_test_undef);
+    //if (fq_2_test_undef!=NULL) be_fq2.setUndefFile(fq_2_test_undef);
 
     be_fq2.openFile(fq_2,f_id2);
     be_fq1.setAuxProcessor(&be_fq2);
diff --git a/src/fqreader.h b/src/fqreader.h
index dc8a7d1..9aedb12 100644
--- a/src/fqreader.h
+++ b/src/fqreader.h
@@ -35,6 +35,6 @@ using namespace std;
 
 
 void processSingleFile(char *, unsigned char, srp*);
-void processPEFiles(char * fq_1, unsigned char f_id1,char * gq_2, unsigned char f_id2,srp *io_sr,char * fq_1_test_undef=NULL,char * fq_2_test_undef=NULL,size_t test_bufsize=0);
+void processPEFiles(char * fq_1, unsigned char f_id1,char * gq_2, unsigned char f_id2,srp *io_sr,size_t test_bufsize=0);
 unsigned long processInputFiles(const std::vector<IO_fq_files>& ,const vector<PE_files>& ,FqBaseBackend * [], const FasqQualThreshold&,srp*,const int& PE_process_mode);
 #endif
diff --git a/src/unit_test_fqreader.cpp b/src/unit_test_fqreader.cpp
index 6702ff5..22e1104 100644
--- a/src/unit_test_fqreader.cpp
+++ b/src/unit_test_fqreader.cpp
@@ -102,7 +102,7 @@ void test_processSingleFileWithMQOption() {
     FqMainBackend::setTreatPEMode(0);
     FqMainBackend be_fq=FqMainBackend(&sr);
     
-    be_fq.setUndefFile((char *) "../test/data/unit/test_single.undef.fq");
+    //be_fq.setUndefFile((char *) "../test/data/unit/test_single.undef.fq");
     be_fq.processFile((char *) "../test/data/unit/test_single.fq",f_id);
 
     srp::reverse_iterator rit;
@@ -118,14 +118,14 @@ void test_processSingleFileWithMQOption() {
         }
     }
     assert(cnt_read==6);
-    assert(remove((char *) "../test/data/unit/test_single.undef.fq")==0);
+    //assert(remove((char *) "../test/data/unit/test_single.undef.fq")==0);
     sr.clear();
 
     cnt_read=0;
     qual_thres.min_correct_k_mers_in_read=130;
     FqBaseBackend::setQualThreshold(qual_thres);
     FqMainBackend be_fq2=FqMainBackend(&sr);
-    be_fq2.setUndefFile((char *) "../test/data/unit/test_single2.undef.fq");
+    //be_fq2.setUndefFile((char *) "../test/data/unit/test_single2.undef.fq");
     be_fq2.processFile((char *) "../test/data/unit/test_single.fq",f_id);
 
     for (rit=sr.rbegin(); rit!=sr.rend(); ++rit) { //process map in reverse order (by decreasing scores).
@@ -139,7 +139,7 @@ void test_processSingleFileWithMQOption() {
         }
     }
     assert(cnt_read==3);
-    assert(remove((char *) "../test/data/unit/test_single2.undef.fq")==0);
+    //assert(remove((char *) "../test/data/unit/test_single2.undef.fq")==0);
     sr.clear();
 
     cnt_read=0;
@@ -148,7 +148,7 @@ void test_processSingleFileWithMQOption() {
     FqBaseBackend::setQualThreshold(qual_thres);
 
     FqMainBackend be_fq3=FqMainBackend(&sr);
-    be_fq3.setUndefFile((char *) "../test/data/unit/test_single3.undef.fq");
+    //be_fq3.setUndefFile((char *) "../test/data/unit/test_single3.undef.fq");
     be_fq3.processFile((char *) "../test/data/unit/test_single.fq",f_id);
 
     for (rit=sr.rbegin(); rit!=sr.rend(); ++rit) { //process map in reverse order (by decreasing scores).
@@ -159,7 +159,7 @@ void test_processSingleFileWithMQOption() {
         }
     }
     assert(cnt_read==0);
-    assert(remove((char *) "../test/data/unit/test_single3.undef.fq")==0);
+    //assert(remove((char *) "../test/data/unit/test_single3.undef.fq")==0);
 }
 
 /*
@@ -288,14 +288,14 @@ void aux_testPEFilesMQ(FasqQualThreshold qual_thres,int nb_expected_reads) {
     srp sr;
     char * fq_1_test=(char *) "../test/data/unit/09-4607_S43_R1.fastq";
     char * fq_2_test=(char *) "../test/data/unit/09-4607_S43_R2.fastq";
-    char * fq_1_test_undef=(char *) "../test/data/unit/09-4607_S43_R1.undef.fastq";
-    char * fq_2_test_undef=(char *) "../test/data/unit/09-4607_S43_R2.undef.fastq";
+    //char * fq_1_test_undef=(char *) "../test/data/unit/09-4607_S43_R1.undef.fastq";
+    //char * fq_2_test_undef=(char *) "../test/data/unit/09-4607_S43_R2.undef.fastq";
 
     unsigned char f_id1=1;
     unsigned char f_id2=2;
     FqMainBackend::setTreatPEMode(0);
     FqBaseBackend::setQualThreshold(qual_thres);
-    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,fq_1_test_undef,fq_2_test_undef);
+    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr);
 
     srp::reverse_iterator rit;
     i_dim::iterator it_offs;
@@ -310,9 +310,9 @@ void aux_testPEFilesMQ(FasqQualThreshold qual_thres,int nb_expected_reads) {
             }
         }
     }
-    assert(cnt_read==nb_expected_reads);
+   /* assert(cnt_read==nb_expected_reads);
     assert(remove((char *) "../test/data/unit/09-4607_S43_R1.undef.fastq")==0);
-    assert(remove((char *) "../test/data/unit/09-4607_S43_R2.undef.fastq")==0);
+    assert(remove((char *) "../test/data/unit/09-4607_S43_R2.undef.fastq")==0);*/
 }
 
 void test_processPEFilesWithMQOptions() {
@@ -403,7 +403,7 @@ void test_processPE_not_as_single() {
 
     FqBaseBackend::setQualThreshold(qual_thres);
     FqMainBackend::setTreatPEMode(1);
-    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,fq_1_test_undef,fq_2_test_undef,1000);
+    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,1000);
 
     srp::reverse_iterator rit;
     i_dim::iterator it_offs;
@@ -420,11 +420,11 @@ void test_processPE_not_as_single() {
     }
     assert(cnt_read==1);
 
-    assert(compareFilesLileByLine(fq_1_test_undef,fq_1_expected_undef)==0);
+    /*assert(compareFilesLileByLine(fq_1_test_undef,fq_1_expected_undef)==0);
     assert(compareFilesLileByLine(fq_2_test_undef,fq_2_expected_undef)==0);
 
     assert(remove((char *) "../test/data/unit/test_PE1_PE_not_as_single.undef.fastq")==0);
-    assert(remove((char *) "../test/data/unit/test_PE2_PE_not_as_single.undef.fastq")==0);
+    assert(remove((char *) "../test/data/unit/test_PE2_PE_not_as_single.undef.fastq")==0);*/
 
 }
 
@@ -447,7 +447,7 @@ void test_processPE_not_as_single2() {
 
     FqBaseBackend::setQualThreshold(qual_thres);
     FqMainBackend::setTreatPEMode(1);
-    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,fq_1_test_undef,fq_2_test_undef,1000);
+    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,1000);
 
     srp::reverse_iterator rit;
     i_dim::iterator it_offs;
@@ -464,11 +464,11 @@ void test_processPE_not_as_single2() {
     }
     assert(cnt_read==0);
 
-    assert(compareFilesLileByLine(fq_1_test_undef,fq_1_test)==0);
+    /*assert(compareFilesLileByLine(fq_1_test_undef,fq_1_test)==0);
     assert(compareFilesLileByLine(fq_2_test_undef,fq_2_test)==0);
 
     assert(remove((char *) "../test/data/unit/test_PE1_PE_not_as_single_pathological.undef.fastq")==0);
-    assert(remove((char *) "../test/data/unit/test_PE2_PE_not_as_single_pathological.undef.fastq")==0);
+    assert(remove((char *) "../test/data/unit/test_PE2_PE_not_as_single_pathological.undef.fastq")==0);*/
 
 }
 
@@ -490,7 +490,7 @@ void test_processPE_not_as_singleWithMQ() {
 
     FqBaseBackend::setQualThreshold(qual_thres);
     FqMainBackend::setTreatPEMode(1);
-    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,fq_1_test_undef,fq_2_test_undef,1000);
+    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,1000);
 
     srp::reverse_iterator rit;
     i_dim::iterator it_offs;
@@ -506,11 +506,11 @@ void test_processPE_not_as_singleWithMQ() {
         }
     }
     assert(cnt_read==1);
-    struct stat fileStat;
+    /*struct stat fileStat;
     stat(fq_1_test_undef, &fileStat);
     assert(fileStat.st_size==0);
     stat(fq_2_test_undef, &fileStat);
-    assert(fileStat.st_size==0);
+    assert(fileStat.st_size==0);*/
     sr.clear();
 
     qual_thres.k=30;
@@ -519,7 +519,7 @@ void test_processPE_not_as_singleWithMQ() {
 
     FqBaseBackend::setQualThreshold(qual_thres);
     FqMainBackend::setTreatPEMode(1);
-    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,fq_1_test_undef,fq_2_test_undef,1000);
+    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,1000);
 
 
 
@@ -537,8 +537,8 @@ void test_processPE_not_as_singleWithMQ() {
 
     sr.clear();
 
-    assert(compareFilesLileByLine(fq_1_test_undef,fq_1_test)==0);
-    assert(compareFilesLileByLine(fq_2_test_undef,fq_2_test)==0);
+    /*assert(compareFilesLileByLine(fq_1_test_undef,fq_1_test)==0);
+    assert(compareFilesLileByLine(fq_2_test_undef,fq_2_test)==0);*/
 
     qual_thres.k=30;
     qual_thres.min_correct_k_mers_in_read=13;
@@ -546,7 +546,7 @@ void test_processPE_not_as_singleWithMQ() {
 
     FqBaseBackend::setQualThreshold(qual_thres);
     FqMainBackend::setTreatPEMode(1);
-    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,fq_1_test_undef,fq_2_test_undef,1000);
+    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,1000);
 
 
     cnt_read=0;
@@ -563,14 +563,14 @@ void test_processPE_not_as_singleWithMQ() {
     sr.clear();
 
 
-    stat(fq_1_test_undef, &fileStat);
+    /*stat(fq_1_test_undef, &fileStat);
     assert(fileStat.st_size==0);
     stat(fq_2_test_undef, &fileStat);
     assert(fileStat.st_size==0);
 
 
     assert(remove((char *) fq_1_test_undef)==0);
-    assert(remove((char *) fq_2_test_undef)==0);
+    assert(remove((char *) fq_2_test_undef)==0);*/
 
 }
 
@@ -647,14 +647,14 @@ AAAAAEEAEEEEEEEEE6EE/EEEEEEEEAEEAEEEEEEEEEEEEEEAEEEA/A/EEEAEEEEEE/EE</EAEEEEEE/E
     FqBaseBackend::setQualThreshold(qual_thres);
     FqMainBackend::setTreatPEMode(0);
     FqMainBackend be(&sr);
-    be.setUndefFile((char *) "../test/data/unit/test_processBuf.undef.fq");
+    //be.setUndefFile((char *) "../test/data/unit/test_processBuf.undef.fq");
     buf_info.buf=buf;
     buf_info.pchar=buf;
     buf_info.cnt=0;
     buf_info.real_bufsize=strlen(buf);
-    be.openUndefFile();
+    //be.openUndefFile();
     be.processBuf(buf_info,f_id1,348);
-    be.closeUndefFile();
+    //be.closeUndefFile();
     // check that read is rejected
     for (rit=sr.rbegin(); rit!=sr.rend(); ++rit) { //process map in reverse order (by decreasing scores).
        for (it_offs=rit->second.begin();it_offs!=rit->second.end();it_offs++) {
@@ -664,7 +664,7 @@ AAAAAEEAEEEEEEEEE6EE/EEEEEEEEAEEAEEEEEEEEEEEEEEAEEEA/A/EEEAEEEEEE/EE</EAEEEEEE/E
        }
     }
     assert(cnt_read==0);
-    std::ifstream t("../test/data/unit/test_processBuf.undef.fq");
+    /*std::ifstream t("../test/data/unit/test_processBuf.undef.fq");
     std::string rej_str;
   
     t.seekg(0, std::ios::end);
@@ -675,7 +675,7 @@ AAAAAEEAEEEEEEEEE6EE/EEEEEEEEAEEAEEEEEEEEEEEEEEAEEEA/A/EEEAEEEEEE/EE</EAEEEEEE/E
                std::istreambuf_iterator<char>());
     assert(strcmp(rej_str.c_str(),buf)==0);
 
-    assert(remove((char *) "../test/data/unit/test_processBuf.undef.fq")==0);
+    assert(remove((char *) "../test/data/unit/test_processBuf.undef.fq")==0);*/
 }
 
 void test_processBufPE() {
@@ -707,10 +707,10 @@ AAAAAEEEEEEEEEEAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAEEEAEEEEEEEEEEEEEEAEEEEEEEEEEEE
     FqMainBackend::setTreatPEMode(0);
 
     FqMainBackend be(&sr);
-    be.setUndefFile((char *) "../test/data/unit/test_processBuf_PE1.undef.fq");
+    //be.setUndefFile((char *) "../test/data/unit/test_processBuf_PE1.undef.fq");
 
     FqAuxBackend be2;
-    be2.setUndefFile((char *) "../test/data/unit/test_processBuf_PE2.undef.fq");
+    //be2.setUndefFile((char *) "../test/data/unit/test_processBuf_PE2.undef.fq");
 
 
     buf_info.buf=buf_PE1;
@@ -726,11 +726,11 @@ AAAAAEEEEEEEEEEAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAEEEAEEEEEEEEEEEEEEAEEEEEEEEEEEE
 
     be.setAuxProcessor(&be2);
 
-    be.openUndefFile();
-    be2.openUndefFile();
+    //be.openUndefFile();
+    //be2.openUndefFile();
     be.processBuf(buf_info,f_id1,348);
-    be.closeUndefFile();
-    be2.closeUndefFile();
+    //be.closeUndefFile();
+    //be2.closeUndefFile();
 
     // check that read is selected
     for (rit=sr.rbegin(); rit!=sr.rend(); ++rit) { //process map in reverse order (by decreasing scores).
@@ -742,7 +742,7 @@ AAAAAEEEEEEEEEEAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAEEEAEEEEEEEEEEEEEEAEEEEEEEEEEEE
     }
     assert(cnt_read==1);
 
-    struct stat fileStat;
+    /*struct stat fileStat;
     stat("../test/data/unit/test_processBuf_PE1.undef.fq", &fileStat);
     assert(fileStat.st_size==0);
 
@@ -750,7 +750,7 @@ AAAAAEEEEEEEEEEAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAEEEAEEEEEEEEEEEEEEAEEEEEEEEEEEE
     assert(fileStat.st_size==0);
 
     assert(remove((char *) "../test/data/unit/test_processBuf_PE1.undef.fq")==0);
-    assert(remove((char *) "../test/data/unit/test_processBuf_PE2.undef.fq")==0);
+    assert(remove((char *) "../test/data/unit/test_processBuf_PE2.undef.fq")==0);*/
 }
 
 
@@ -774,7 +774,7 @@ void Aux_MimicBigPEFilesWithMQOptions(const FasqQualThreshold& qual_thres,const
     FqBaseBackend::setQualThreshold(qual_thres);
     FqMainBackend::setTreatPEMode(0);
 
-    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,fq_1_test_undef,fq_2_test_undef,bufsize);
+    processPEFiles(fq_1_test, f_id1,fq_2_test, f_id2,&sr,bufsize);
 
     srp::reverse_iterator rit;
     i_dim::iterator it_offs;
@@ -790,7 +790,7 @@ void Aux_MimicBigPEFilesWithMQOptions(const FasqQualThreshold& qual_thres,const
         }
     }
     assert(cnt_read==nb_expected_reads);
-    if (nb_expected_reads==6) {
+    /*if (nb_expected_reads==6) {
         assert(compareFilesLileByLine(fq_1_test_undef,fq_1_expected_undef_100)==0);
         assert(compareFilesLileByLine(fq_2_test_undef,fq_2_expected_undef_100)==0);
     } else {
@@ -798,7 +798,7 @@ void Aux_MimicBigPEFilesWithMQOptions(const FasqQualThreshold& qual_thres,const
         assert(compareFilesLileByLine(fq_2_test_undef,fq_2_expected_undef_180)==0);
     }
     assert(remove((char *) "../test/data/unit/09-4607_S43_R1.undef.fastq")==0);
-    assert(remove((char *) "../test/data/unit/09-4607_S43_R2.undef.fastq")==0);
+    assert(remove((char *) "../test/data/unit/09-4607_S43_R2.undef.fastq")==0);*/
 }
 
 
diff --git a/test/data/non_regression/ids_read_kept_q13m300.txt b/test/data/non_regression/ids_read_kept_q13m300.txt
new file mode 100644
index 0000000..e80d257
--- /dev/null
+++ b/test/data/non_regression/ids_read_kept_q13m300.txt
@@ -0,0 +1,66 @@
+@SRR1222430.1
+@SRR1222430.10
+@SRR1222430.13
+@SRR1222430.16
+@SRR1222430.17
+@SRR1222430.18
+@SRR1222430.19
+@SRR1222430.21
+@SRR1222430.22
+@SRR1222430.23
+@SRR1222430.25
+@SRR1222430.27
+@SRR1222430.28
+@SRR1222430.29
+@SRR1222430.3
+@SRR1222430.30
+@SRR1222430.31
+@SRR1222430.32
+@SRR1222430.33
+@SRR1222430.35
+@SRR1222430.36
+@SRR1222430.38
+@SRR1222430.39
+@SRR1222430.42
+@SRR1222430.43
+@SRR1222430.44
+@SRR1222430.45
+@SRR1222430.47
+@SRR1222430.5
+@SRR1222430.50
+@SRR1222430.51
+@SRR1222430.53
+@SRR1222430.54
+@SRR1222430.55
+@SRR1222430.56
+@SRR1222430.57
+@SRR1222430.58
+@SRR1222430.59
+@SRR1222430.6
+@SRR1222430.61
+@SRR1222430.63
+@SRR1222430.64
+@SRR1222430.67
+@SRR1222430.68
+@SRR1222430.69
+@SRR1222430.7
+@SRR1222430.70
+@SRR1222430.71
+@SRR1222430.72
+@SRR1222430.73
+@SRR1222430.75
+@SRR1222430.77
+@SRR1222430.8
+@SRR1222430.80
+@SRR1222430.81
+@SRR1222430.83
+@SRR1222430.85
+@SRR1222430.86
+@SRR1222430.87
+@SRR1222430.88
+@SRR1222430.9
+@SRR1222430.90
+@SRR1222430.91
+@SRR1222430.92
+@SRR1222430.93
+@SRR1222430.95
diff --git a/test/rock_mem.sh b/test/rock_mem.sh
index 11fcf54..7f3fcb0 100755
--- a/test/rock_mem.sh
+++ b/test/rock_mem.sh
@@ -85,10 +85,10 @@ rm -f "klebsiella_100_1.rock.fq" || exit 117
 rm -f "klebsiella_100_2.rock.fq" || exit 118
 rm -f "test_single.rock.fq"|| exit 119
 rm -f "test_single2.rock.fq"|| exit 120
-rm -f "klebsiella_100_1.undefined.fq" || exit 121
-rm -f "klebsiella_100_2.undefined.fq" || exit 122
-rm -f "test_single.undefined.fq" || exit 123
-rm -f "test_single2.undefined.fq" || exit 124
+#rm -f "klebsiella_100_1.undefined.fq" || exit 121
+#rm -f "klebsiella_100_2.undefined.fq" || exit 122
+#rm -f "test_single.undefined.fq" || exit 123
+#rm -f "test_single2.undefined.fq" || exit 124
 
 
 
@@ -105,10 +105,10 @@ rm -f "klebsiella_100_1.rock.fq" || exit 127
 rm -f "klebsiella_100_2.rock.fq" || exit 128
 rm -f "test_single.rock.fq"|| exit 129
 rm -f "test_single2.rock.fq"|| exit 130
-rm -f "klebsiella_100_1.undefined.fq" || exit 131
-rm -f "klebsiella_100_2.undefined.fq" || exit 132
-rm -f "test_single.undefined.fq" || exit 133
-rm -f "test_single2.undefined.fq" || exit 134
+#rm -f "klebsiella_100_1.undefined.fq" || exit 131
+#rm -f "klebsiella_100_2.undefined.fq" || exit 132
+#rm -f "test_single.undefined.fq" || exit 133
+#rm -f "test_single2.undefined.fq" || exit 134
 
 echo " "
 echo "##################################################################################"
@@ -158,8 +158,9 @@ echo "##########################################################################
 echo "testing ROCK with a quality score threshold for nucleotides and minimum number of valid k-mer to keep a read."
 
 mkdir tmp
+echo "../src/rock -C 100 -k 30 -c 1 -l 2 -o ${srcdir}/data/iofiles.args/output_files_noNQ_Thres.txt  ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq"
 ../src/rock -C 100 -k 30 -c 1 -l 2 -o ${srcdir}/data/iofiles.args/output_files_noNQ_Thres.txt  ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 150
-
+echo "../src/rock -C 100 -k 30 -c 1 -l 2 -q 2 -m 5 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_very_low.txt  ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq"
 ../src/rock -C 100 -k 30 -c 1 -l 2 -q 2 -m 5 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_very_low.txt  ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 151
 
 ret1=`diff tmp/klebsiella_100_1_very_low_qual_thres.fq tmp/klebsiella_100_1_no_qual_thres.fq|wc -l`
@@ -168,36 +169,70 @@ ret2=`diff tmp/klebsiella_100_2_very_low_qual_thres.fq tmp/klebsiella_100_2_no_q
 test $ret2 -eq 0 || exit 153
 
 
-[ -f "tmp/klebsiella_100_1.undefined.fq" ] || exit 154
-[ -f "tmp/klebsiella_100_2.undefined.fq" ] || exit 155
+#[ -f "tmp/klebsiella_100_1.undefined.fq" ] || exit 154
+#[ -f "tmp/klebsiella_100_2.undefined.fq" ] || exit 155
 
-ret1=`cat tmp/klebsiella_100_1.undefined.fq|wc -l`
-ret2=`cat tmp/klebsiella_100_2.undefined.fq|wc -l`
+#ret1=`cat tmp/klebsiella_100_1.undefined.fq|wc -l`
+#ret2=`cat tmp/klebsiella_100_2.undefined.fq|wc -l`
 
-test $ret1 -eq 0 || exit 156
-test $ret1 -eq 0 || exit 157
+#test $ret1 -eq 0 || exit 156
+#test $ret1 -eq 0 || exit 157
 
+# All reads should be rejected.
+echo "rock -k 30 -C 100 -c 1 -l 2 -q 13 -m 500 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq"
 ../src/rock -k 30 -C 100 -c 1 -l 2 -q 13 -m 500 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 158
+[ -f "tmp/klebsiella_100_1_13_qual_thres.fq" ] || exit 159
+[ -f "tmp/klebsiella_100_2_13_qual_thres.fq" ] || exit 160
+
+echo "both files are here"
+
+ret1=`cat tmp/klebsiella_100_1_13_qual_thres.fq|wc -l`
+ret2=`cat tmp/klebsiella_100_2_13_qual_thres.fq|wc -l`
 
-[ -f "tmp/klebsiella_100_1.undefined.fq" ] || exit 159
-[ -f "tmp/klebsiella_100_2.undefined.fq" ] || exit 160
+echo "ret1="$ret1
+echo "ret2="$ret2
 
-ret1=`cat tmp/klebsiella_100_1.undefined.fq|wc -l`
-ret2=`cat tmp/klebsiella_100_2.undefined.fq|wc -l`
+test $ret1 -eq 0 || exit 161
+test $ret2 -eq 0 || exit 162
 
-test $ret1 -eq 400 || exit 161
-test $ret1 -eq 400 || exit 162
+#[ -f "tmp/klebsiella_100_1.undefined.fq" ] || exit 159
+#[ -f "tmp/klebsiella_100_2.undefined.fq" ] || exit 160
 
+#ret1=`cat tmp/klebsiella_100_1.undefined.fq|wc -l`
+#ret2=`cat tmp/klebsiella_100_2.undefined.fq|wc -l`
+
+#test $ret1 -eq 400 || exit 161
+#test $ret1 -eq 400 || exit 162
+
+echo "rock -k 30 -C 100 -c 1 -l 2 -q 13 -m 300 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq"
 ../src/rock -k 30 -C 100 -c 1 -l 2 -q 13 -m 300 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 163
 
-[ -f "tmp/klebsiella_100_1.undefined.fq" ] || exit 164
-[ -f "tmp/klebsiella_100_2.undefined.fq" ] || exit 165
+[ -f "tmp/klebsiella_100_1_13_qual_thres.fq" ] || exit 164
+[ -f "tmp/klebsiella_100_2_13_qual_thres.fq" ] || exit 165
+
+ret1=`cat tmp/klebsiella_100_1_13_qual_thres.fq|wc -l`
+ret2=`cat tmp/klebsiella_100_2_13_qual_thres.fq|wc -l`
+
+test $ret1 -eq 264 || exit 166
+test $ret2 -eq 264 || exit 167
+
+# check that reads that are kept are always the same.
+lst1=`cat tmp/klebsiella_100_1_13_qual_thres.fq|grep @S|cut -d ' ' -f 1|sort`
+lst2=`cat tmp/klebsiella_100_2_13_qual_thres.fq|grep @S|cut -d ' ' -f 1|sort`
+
+lst_ref=`cat ${srcdir}/data/non_regression/ids_read_kept_q13m300.txt`
+
+test "$lst1" == "$lst2" || exit 1671
+test "$lst_ref" == "$lst1" || exit 1672
+
+#[ -f "tmp/klebsiella_100_1.undefined.fq" ] || exit 164
+#[ -f "tmp/klebsiella_100_2.undefined.fq" ] || exit 165
 
-ret1=`cat tmp/klebsiella_100_1.undefined.fq|wc -l`
-ret2=`cat tmp/klebsiella_100_2.undefined.fq|wc -l`
+#ret1=`cat tmp/klebsiella_100_1.undefined.fq|wc -l`
+#ret2=`cat tmp/klebsiella_100_2.undefined.fq|wc -l`
 
-test $ret1 -eq 136 || exit 166
-test $ret1 -eq 136 || exit 167
+#test $ret1 -eq 136 || exit 166
+#test $ret1 -eq 136 || exit 167
 
 echo " "
 echo "***##################################################################################"
-- 
GitLab