diff --git a/.gitignore b/.gitignore
index 6f03a8b8fc2b57ce345349900fcb6e49107a2f24..760227bd53241aad8c58b0d987ab49771e3673fb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,7 +6,7 @@ Makefile
.deps
rock-*.tar.gz
rock
-unit_test_*
+unit_test_*.o
.settings/*
.autom4te.cache/*
*.xcodeproj
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e5dfa6dbe78d07351716fb712e34dc637f26f61e..ce0de39d568c71f0cbb7261dbeab49e662ac8954 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,12 +12,13 @@ makefile_generation_and_compilation_plus_run_nrt_tests_cluster:
- echo $PATH
- ls /usr/bin
- yum update -y && yum install -y perl perl-Thread-Queue.noarch && /usr/bin/perl --version
+ - aclocal
- autoconf -i
- automake
- ./configure
- make; cd src
- ./unit_test_fqreader
- - ./unit_test_main_utils
+ - ./unit_test_math_utils
- ./unit_test_fqwriter
- ./unit_test_cms
- ./unit_test_read_utils
diff --git a/Makefile.in b/Makefile.in
index 05d2fbfac7387b2d0fd8fc7f93a986bd6910151a..5f3bb84e3eda11bc555445b625c208ebd8292335 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1,7 +1,7 @@
-# Makefile.in generated by automake 1.15 from Makefile.am.
+# Makefile.in generated by automake 1.16.3 from Makefile.am.
# @configure_input@
-# Copyright (C) 1994-2014 Free Software Foundation, Inc.
+# Copyright (C) 1994-2020 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -134,7 +134,7 @@ am__recursive_targets = \
$(RECURSIVE_CLEAN_TARGETS) \
$(am__extra_recursive_targets)
AM_RECURSIVE_TARGETS = $(am__recursive_targets:-recursive=) TAGS CTAGS \
- cscope distdir dist dist-all distcheck
+ cscope distdir distdir-am dist dist-all distcheck
am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
# Read a list of newline-separated strings from the standard input,
# and print each of them once, without duplicates. Input order is
@@ -197,6 +197,8 @@ am__relativize = \
DIST_ARCHIVES = $(distdir).tar.gz
GZIP_ENV = --best
DIST_TARGETS = dist-gzip
+# Exists only to be overridden by the user if desired.
+AM_DISTCHECK_DVI_TARGET = dvi
distuninstallcheck_listfiles = find . -type f -print
am__distuninstallcheck_listfiles = $(distuninstallcheck_listfiles) \
| sed 's|^\./|$(prefix)/|' | grep -v '$(infodir)/dir$$'
@@ -324,8 +326,8 @@ Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
echo ' $(SHELL) ./config.status'; \
$(SHELL) ./config.status;; \
*) \
- echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe)'; \
- cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe);; \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__maybe_remake_depfiles)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__maybe_remake_depfiles);; \
esac;
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
@@ -443,7 +445,10 @@ distclean-tags:
-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
-rm -f cscope.out cscope.in.out cscope.po.out cscope.files
-distdir: $(DISTFILES)
+distdir: $(BUILT_SOURCES)
+ $(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
$(am__remove_distdir)
test -d "$(distdir)" || mkdir "$(distdir)"
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
@@ -508,7 +513,7 @@ distdir: $(DISTFILES)
! -type d ! -perm -444 -exec $(install_sh) -c -m a+r {} {} \; \
|| chmod -R a+r "$(distdir)"
dist-gzip: distdir
- tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).tar.gz
+ tardir=$(distdir) && $(am__tar) | eval GZIP= gzip $(GZIP_ENV) -c >$(distdir).tar.gz
$(am__post_remove_distdir)
dist-bzip2: distdir
@@ -523,6 +528,10 @@ dist-xz: distdir
tardir=$(distdir) && $(am__tar) | XZ_OPT=$${XZ_OPT--e} xz -c >$(distdir).tar.xz
$(am__post_remove_distdir)
+dist-zstd: distdir
+ tardir=$(distdir) && $(am__tar) | zstd -c $${ZSTD_CLEVEL-$${ZSTD_OPT--19}} >$(distdir).tar.zst
+ $(am__post_remove_distdir)
+
dist-tarZ: distdir
@echo WARNING: "Support for distribution archives compressed with" \
"legacy program 'compress' is deprecated." >&2
@@ -534,7 +543,7 @@ dist-shar: distdir
@echo WARNING: "Support for shar distribution archives is" \
"deprecated." >&2
@echo WARNING: "It will be removed altogether in Automake 2.0" >&2
- shar $(distdir) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).shar.gz
+ shar $(distdir) | eval GZIP= gzip $(GZIP_ENV) -c >$(distdir).shar.gz
$(am__post_remove_distdir)
dist-zip: distdir
@@ -552,7 +561,7 @@ dist dist-all:
distcheck: dist
case '$(DIST_ARCHIVES)' in \
*.tar.gz*) \
- GZIP=$(GZIP_ENV) gzip -dc $(distdir).tar.gz | $(am__untar) ;;\
+ eval GZIP= gzip $(GZIP_ENV) -dc $(distdir).tar.gz | $(am__untar) ;;\
*.tar.bz2*) \
bzip2 -dc $(distdir).tar.bz2 | $(am__untar) ;;\
*.tar.lz*) \
@@ -562,9 +571,11 @@ distcheck: dist
*.tar.Z*) \
uncompress -c $(distdir).tar.Z | $(am__untar) ;;\
*.shar.gz*) \
- GZIP=$(GZIP_ENV) gzip -dc $(distdir).shar.gz | unshar ;;\
+ eval GZIP= gzip $(GZIP_ENV) -dc $(distdir).shar.gz | unshar ;;\
*.zip*) \
unzip $(distdir).zip ;;\
+ *.tar.zst*) \
+ zstd -dc $(distdir).tar.zst | $(am__untar) ;;\
esac
chmod -R a-w $(distdir)
chmod u+w $(distdir)
@@ -580,7 +591,7 @@ distcheck: dist
$(DISTCHECK_CONFIGURE_FLAGS) \
--srcdir=../.. --prefix="$$dc_install_base" \
&& $(MAKE) $(AM_MAKEFLAGS) \
- && $(MAKE) $(AM_MAKEFLAGS) dvi \
+ && $(MAKE) $(AM_MAKEFLAGS) $(AM_DISTCHECK_DVI_TARGET) \
&& $(MAKE) $(AM_MAKEFLAGS) check \
&& $(MAKE) $(AM_MAKEFLAGS) install \
&& $(MAKE) $(AM_MAKEFLAGS) installcheck \
@@ -741,7 +752,7 @@ uninstall-am:
am--refresh check check-am clean clean-cscope clean-generic \
cscope cscopelist-am ctags ctags-am dist dist-all dist-bzip2 \
dist-gzip dist-lzip dist-shar dist-tarZ dist-xz dist-zip \
- distcheck distclean distclean-generic distclean-tags \
+ dist-zstd distcheck distclean distclean-generic distclean-tags \
distcleancheck distdir distuninstallcheck dvi dvi-am html \
html-am info info-am install install-am install-data \
install-data-am install-dvi install-dvi-am install-exec \
diff --git a/NEWS b/NEWS
index 7d12cce10a6ed496d6a84a228b4b51fc4f414b33..ae6b4469b61f802b8ad39ffc81a42506d7f0223c 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,27 @@
+New in version 1.9.2
+ * Now using sysinfo to determine the amount of RAM available under a linux system.
+ * refactored code.
+ * New usage function
+ * new verbose mode.
+
+New in version 1.9.2
+ * bugfix: when the expected number of distinct kmers was provided by the user with the -n option, it was converted to int instead of unsigned long
+ which led to incorrect values when calculating lamdba.
+ * new formula for computing the approximation of the number of distinct kmers in the CMS.
+ * Updated Usage.
+
+New in version 1.9.1
+ * added option -f to allow the users to specify what maximum value of collision probability they want in the CMS
+
+New in version 1.9
+ * Removed option -p; processing PE as single (formerly -p 0) giving the best assembly results.
+ * Removed -g option (unuseful)
+ * changed collision probability limit to determine lamda from 0.1 to 0.05
+ * changed default value for k-mer size from 30 to 25.
+ * changed default value for minimum threshold (-c option). Set it to 0 (no minimum threshold) instead of 5.
+ * changed default value for minimum quality score for nucleotides. Set it to 0 (no minimum threshold instead of 10.
+
+
New in version 1.8
* Bugfix: The "process PE separately option" was only taken into account when calculating the median coverage.
During original parsing for determining which reads are "undefined", PE were considered as single so the -m option for PE was not useful.
diff --git a/aclocal.m4 b/aclocal.m4
index bf79d078cb0afb5a0ca6509ef121bb351a1bb92b..d26c5a1b9447ae7844b91e3f0157ecc2b0c4b9be 100644
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -1,6 +1,6 @@
-# generated automatically by aclocal 1.15 -*- Autoconf -*-
+# generated automatically by aclocal 1.16.3 -*- Autoconf -*-
-# Copyright (C) 1996-2014 Free Software Foundation, Inc.
+# Copyright (C) 1996-2020 Free Software Foundation, Inc.
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -20,7 +20,7 @@ You have another version of autoconf. It may work, but is not guaranteed to.
If you have problems, you may need to regenerate the build system entirely.
To do so, use the procedure documented by the package, typically 'autoreconf'.])])
-# Copyright (C) 2002-2014 Free Software Foundation, Inc.
+# Copyright (C) 2002-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -32,10 +32,10 @@ To do so, use the procedure documented by the package, typically 'autoreconf'.])
# generated from the m4 files accompanying Automake X.Y.
# (This private macro should not be called outside this file.)
AC_DEFUN([AM_AUTOMAKE_VERSION],
-[am__api_version='1.15'
+[am__api_version='1.16'
dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to
dnl require some minimum version. Point them to the right macro.
-m4_if([$1], [1.15], [],
+m4_if([$1], [1.16.3], [],
[AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl
])
@@ -51,14 +51,14 @@ m4_define([_AM_AUTOCONF_VERSION], [])
# Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced.
# This function is AC_REQUIREd by AM_INIT_AUTOMAKE.
AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION],
-[AM_AUTOMAKE_VERSION([1.15])dnl
+[AM_AUTOMAKE_VERSION([1.16.3])dnl
m4_ifndef([AC_AUTOCONF_VERSION],
[m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
_AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))])
# AM_AUX_DIR_EXPAND -*- Autoconf -*-
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -110,7 +110,7 @@ am_aux_dir=`cd "$ac_aux_dir" && pwd`
# AM_CONDITIONAL -*- Autoconf -*-
-# Copyright (C) 1997-2014 Free Software Foundation, Inc.
+# Copyright (C) 1997-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -141,7 +141,7 @@ AC_CONFIG_COMMANDS_PRE(
Usually this means the macro was only invoked conditionally.]])
fi])])
-# Copyright (C) 1999-2014 Free Software Foundation, Inc.
+# Copyright (C) 1999-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -332,13 +332,12 @@ _AM_SUBST_NOTMAKE([am__nodep])dnl
# Generate code to set up dependency tracking. -*- Autoconf -*-
-# Copyright (C) 1999-2014 Free Software Foundation, Inc.
+# Copyright (C) 1999-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
-
# _AM_OUTPUT_DEPENDENCY_COMMANDS
# ------------------------------
AC_DEFUN([_AM_OUTPUT_DEPENDENCY_COMMANDS],
@@ -346,49 +345,43 @@ AC_DEFUN([_AM_OUTPUT_DEPENDENCY_COMMANDS],
# Older Autoconf quotes --file arguments for eval, but not when files
# are listed without --file. Let's play safe and only enable the eval
# if we detect the quoting.
- case $CONFIG_FILES in
- *\'*) eval set x "$CONFIG_FILES" ;;
- *) set x $CONFIG_FILES ;;
- esac
+ # TODO: see whether this extra hack can be removed once we start
+ # requiring Autoconf 2.70 or later.
+ AS_CASE([$CONFIG_FILES],
+ [*\'*], [eval set x "$CONFIG_FILES"],
+ [*], [set x $CONFIG_FILES])
shift
- for mf
+ # Used to flag and report bootstrapping failures.
+ am_rc=0
+ for am_mf
do
# Strip MF so we end up with the name of the file.
- mf=`echo "$mf" | sed -e 's/:.*$//'`
- # Check whether this is an Automake generated Makefile or not.
- # We used to match only the files named 'Makefile.in', but
- # some people rename them; so instead we look at the file content.
- # Grep'ing the first line is not enough: some people post-process
- # each Makefile.in and add a new line on top of each file to say so.
- # Grep'ing the whole file is not good either: AIX grep has a line
+ am_mf=`AS_ECHO(["$am_mf"]) | sed -e 's/:.*$//'`
+ # Check whether this is an Automake generated Makefile which includes
+ # dependency-tracking related rules and includes.
+ # Grep'ing the whole file directly is not great: AIX grep has a line
# limit of 2048, but all sed's we know have understand at least 4000.
- if sed -n 's,^#.*generated by automake.*,X,p' "$mf" | grep X >/dev/null 2>&1; then
- dirpart=`AS_DIRNAME("$mf")`
- else
- continue
- fi
- # Extract the definition of DEPDIR, am__include, and am__quote
- # from the Makefile without running 'make'.
- DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"`
- test -z "$DEPDIR" && continue
- am__include=`sed -n 's/^am__include = //p' < "$mf"`
- test -z "$am__include" && continue
- am__quote=`sed -n 's/^am__quote = //p' < "$mf"`
- # Find all dependency output files, they are included files with
- # $(DEPDIR) in their names. We invoke sed twice because it is the
- # simplest approach to changing $(DEPDIR) to its actual value in the
- # expansion.
- for file in `sed -n "
- s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \
- sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g'`; do
- # Make sure the directory exists.
- test -f "$dirpart/$file" && continue
- fdir=`AS_DIRNAME(["$file"])`
- AS_MKDIR_P([$dirpart/$fdir])
- # echo "creating $dirpart/$file"
- echo '# dummy' > "$dirpart/$file"
- done
+ sed -n 's,^am--depfiles:.*,X,p' "$am_mf" | grep X >/dev/null 2>&1 \
+ || continue
+ am_dirpart=`AS_DIRNAME(["$am_mf"])`
+ am_filepart=`AS_BASENAME(["$am_mf"])`
+ AM_RUN_LOG([cd "$am_dirpart" \
+ && sed -e '/# am--include-marker/d' "$am_filepart" \
+ | $MAKE -f - am--depfiles]) || am_rc=$?
done
+ if test $am_rc -ne 0; then
+ AC_MSG_FAILURE([Something went wrong bootstrapping makefile fragments
+ for automatic dependency tracking. If GNU make was not used, consider
+ re-running the configure script with MAKE="gmake" (or whatever is
+ necessary). You can also try re-running configure with the
+ '--disable-dependency-tracking' option to at least be able to build
+ the package (albeit without support for automatic dependency tracking).])
+ fi
+ AS_UNSET([am_dirpart])
+ AS_UNSET([am_filepart])
+ AS_UNSET([am_mf])
+ AS_UNSET([am_rc])
+ rm -f conftest-deps.mk
}
])# _AM_OUTPUT_DEPENDENCY_COMMANDS
@@ -397,18 +390,17 @@ AC_DEFUN([_AM_OUTPUT_DEPENDENCY_COMMANDS],
# -----------------------------
# This macro should only be invoked once -- use via AC_REQUIRE.
#
-# This code is only required when automatic dependency tracking
-# is enabled. FIXME. This creates each '.P' file that we will
-# need in order to bootstrap the dependency handling code.
+# This code is only required when automatic dependency tracking is enabled.
+# This creates each '.Po' and '.Plo' makefile fragment that we'll need in
+# order to bootstrap the dependency handling code.
AC_DEFUN([AM_OUTPUT_DEPENDENCY_COMMANDS],
[AC_CONFIG_COMMANDS([depfiles],
[test x"$AMDEP_TRUE" != x"" || _AM_OUTPUT_DEPENDENCY_COMMANDS],
- [AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir"])
-])
+ [AMDEP_TRUE="$AMDEP_TRUE" MAKE="${MAKE-make}"])])
# Do all the work for Automake. -*- Autoconf -*-
-# Copyright (C) 1996-2014 Free Software Foundation, Inc.
+# Copyright (C) 1996-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -495,8 +487,8 @@ AC_REQUIRE([AM_PROG_INSTALL_STRIP])dnl
AC_REQUIRE([AC_PROG_MKDIR_P])dnl
# For better backward compatibility. To be removed once Automake 1.9.x
# dies out for good. For more background, see:
-# <http://lists.gnu.org/archive/html/automake/2012-07/msg00001.html>
-# <http://lists.gnu.org/archive/html/automake/2012-07/msg00014.html>
+# <https://lists.gnu.org/archive/html/automake/2012-07/msg00001.html>
+# <https://lists.gnu.org/archive/html/automake/2012-07/msg00014.html>
AC_SUBST([mkdir_p], ['$(MKDIR_P)'])
# We need awk for the "check" target (and possibly the TAP driver). The
# system "awk" is bad on some platforms.
@@ -563,7 +555,7 @@ END
Aborting the configuration process, to ensure you take notice of the issue.
You can download and install GNU coreutils to get an 'rm' implementation
-that behaves properly: <http://www.gnu.org/software/coreutils/>.
+that behaves properly: <https://www.gnu.org/software/coreutils/>.
If you want to complete the configuration process using your problematic
'rm' anyway, export the environment variable ACCEPT_INFERIOR_RM_PROGRAM
@@ -605,7 +597,7 @@ for _am_header in $config_headers :; do
done
echo "timestamp for $_am_arg" >`AS_DIRNAME(["$_am_arg"])`/stamp-h[]$_am_stamp_count])
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -626,7 +618,7 @@ if test x"${install_sh+set}" != xset; then
fi
AC_SUBST([install_sh])])
-# Copyright (C) 2003-2014 Free Software Foundation, Inc.
+# Copyright (C) 2003-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -647,7 +639,7 @@ AC_SUBST([am__leading_dot])])
# Check to see how 'make' treats includes. -*- Autoconf -*-
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -655,49 +647,42 @@ AC_SUBST([am__leading_dot])])
# AM_MAKE_INCLUDE()
# -----------------
-# Check to see how make treats includes.
+# Check whether make has an 'include' directive that can support all
+# the idioms we need for our automatic dependency tracking code.
AC_DEFUN([AM_MAKE_INCLUDE],
-[am_make=${MAKE-make}
-cat > confinc << 'END'
+[AC_MSG_CHECKING([whether ${MAKE-make} supports the include directive])
+cat > confinc.mk << 'END'
am__doit:
- @echo this is the am__doit target
+ @echo this is the am__doit target >confinc.out
.PHONY: am__doit
END
-# If we don't find an include directive, just comment out the code.
-AC_MSG_CHECKING([for style of include used by $am_make])
am__include="#"
am__quote=
-_am_result=none
-# First try GNU make style include.
-echo "include confinc" > confmf
-# Ignore all kinds of additional output from 'make'.
-case `$am_make -s -f confmf 2> /dev/null` in #(
-*the\ am__doit\ target*)
- am__include=include
- am__quote=
- _am_result=GNU
- ;;
-esac
-# Now try BSD make style include.
-if test "$am__include" = "#"; then
- echo '.include "confinc"' > confmf
- case `$am_make -s -f confmf 2> /dev/null` in #(
- *the\ am__doit\ target*)
- am__include=.include
- am__quote="\""
- _am_result=BSD
- ;;
- esac
-fi
-AC_SUBST([am__include])
-AC_SUBST([am__quote])
-AC_MSG_RESULT([$_am_result])
-rm -f confinc confmf
-])
+# BSD make does it like this.
+echo '.include "confinc.mk" # ignored' > confmf.BSD
+# Other make implementations (GNU, Solaris 10, AIX) do it like this.
+echo 'include confinc.mk # ignored' > confmf.GNU
+_am_result=no
+for s in GNU BSD; do
+ AM_RUN_LOG([${MAKE-make} -f confmf.$s && cat confinc.out])
+ AS_CASE([$?:`cat confinc.out 2>/dev/null`],
+ ['0:this is the am__doit target'],
+ [AS_CASE([$s],
+ [BSD], [am__include='.include' am__quote='"'],
+ [am__include='include' am__quote=''])])
+ if test "$am__include" != "#"; then
+ _am_result="yes ($s style)"
+ break
+ fi
+done
+rm -f confinc.* confmf.*
+AC_MSG_RESULT([${_am_result}])
+AC_SUBST([am__include])])
+AC_SUBST([am__quote])])
# Fake the existence of programs that GNU maintainers use. -*- Autoconf -*-
-# Copyright (C) 1997-2014 Free Software Foundation, Inc.
+# Copyright (C) 1997-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -718,12 +703,7 @@ AC_DEFUN([AM_MISSING_HAS_RUN],
[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl
AC_REQUIRE_AUX_FILE([missing])dnl
if test x"${MISSING+set}" != xset; then
- case $am_aux_dir in
- *\ * | *\ *)
- MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;;
- *)
- MISSING="\${SHELL} $am_aux_dir/missing" ;;
- esac
+ MISSING="\${SHELL} '$am_aux_dir/missing'"
fi
# Use eval to expand $SHELL
if eval "$MISSING --is-lightweight"; then
@@ -736,7 +716,7 @@ fi
# Helper functions for option handling. -*- Autoconf -*-
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -765,9 +745,26 @@ AC_DEFUN([_AM_SET_OPTIONS],
AC_DEFUN([_AM_IF_OPTION],
[m4_ifset(_AM_MANGLE_OPTION([$1]), [$2], [$3])])
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_RUN_LOG(COMMAND)
+# -------------------
+# Run COMMAND, save the exit status in ac_status, and log it.
+# (This has been adapted from Autoconf's _AC_RUN_LOG macro.)
+AC_DEFUN([AM_RUN_LOG],
+[{ echo "$as_me:$LINENO: $1" >&AS_MESSAGE_LOG_FD
+ ($1) >&AS_MESSAGE_LOG_FD 2>&AS_MESSAGE_LOG_FD
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD
+ (exit $ac_status); }])
+
# Check to make sure that the build environment is sane. -*- Autoconf -*-
-# Copyright (C) 1996-2014 Free Software Foundation, Inc.
+# Copyright (C) 1996-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -848,7 +845,7 @@ AC_CONFIG_COMMANDS_PRE(
rm -f conftest.file
])
-# Copyright (C) 2009-2014 Free Software Foundation, Inc.
+# Copyright (C) 2009-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -908,7 +905,7 @@ AC_SUBST([AM_BACKSLASH])dnl
_AM_SUBST_NOTMAKE([AM_BACKSLASH])dnl
])
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -936,7 +933,7 @@ fi
INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s"
AC_SUBST([INSTALL_STRIP_PROGRAM])])
-# Copyright (C) 2006-2014 Free Software Foundation, Inc.
+# Copyright (C) 2006-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -955,7 +952,7 @@ AC_DEFUN([AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE($@)])
# Check how to create a tarball. -*- Autoconf -*-
-# Copyright (C) 2004-2014 Free Software Foundation, Inc.
+# Copyright (C) 2004-2020 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
diff --git a/configure b/configure
index 9f0a058331a00f4056a61abfa5afada0cedb5815..b461cb9240113ded0b3366bdfce8d18f0994fd6a 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for rock 1.8.
+# Generated by GNU Autoconf 2.69 for rock 1.9.2.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -576,8 +576,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='rock'
PACKAGE_TARNAME='rock'
-PACKAGE_VERSION='1.8'
-PACKAGE_STRING='rock 1.8'
+PACKAGE_VERSION='1.9.2'
+PACKAGE_STRING='rock 1.9.2'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
@@ -594,7 +594,6 @@ am__nodep
AMDEPBACKSLASH
AMDEP_FALSE
AMDEP_TRUE
-am__quote
am__include
DEPDIR
OBJEXT
@@ -680,7 +679,8 @@ PACKAGE_VERSION
PACKAGE_TARNAME
PACKAGE_NAME
PATH_SEPARATOR
-SHELL'
+SHELL
+am__quote'
ac_subst_files=''
ac_user_opts='
enable_option_checking
@@ -1236,7 +1236,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures rock 1.8 to adapt to many kinds of systems.
+\`configure' configures rock 1.9.2 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1307,7 +1307,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of rock 1.8:";;
+ short | recursive ) echo "Configuration of rock 1.9.2:";;
esac
cat <<\_ACEOF
@@ -1397,7 +1397,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-rock configure 1.8
+rock configure 1.9.2
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1452,7 +1452,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by rock $as_me 1.8, which was
+It was created by rock $as_me 1.9.2, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@@ -1942,7 +1942,7 @@ test -n "$target_alias" &&
NONENONEs,x,x, &&
program_prefix=${target_alias}-
-am__api_version='1.15'
+am__api_version='1.16'
# Find a good install program. We prefer a C program (faster),
# so one script is as good as another. But avoid the broken or
@@ -2118,12 +2118,7 @@ program_transform_name=`$as_echo "$program_transform_name" | sed "$ac_script"`
am_aux_dir=`cd "$ac_aux_dir" && pwd`
if test x"${MISSING+set}" != xset; then
- case $am_aux_dir in
- *\ * | *\ *)
- MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;;
- *)
- MISSING="\${SHELL} $am_aux_dir/missing" ;;
- esac
+ MISSING="\${SHELL} '$am_aux_dir/missing'"
fi
# Use eval to expand $SHELL
if eval "$MISSING --is-lightweight"; then
@@ -2428,7 +2423,7 @@ fi
# Define the identity of the package.
PACKAGE='rock'
- VERSION='1.8'
+ VERSION='1.9.2'
cat >>confdefs.h <<_ACEOF
@@ -2458,8 +2453,8 @@ MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"}
# For better backward compatibility. To be removed once Automake 1.9.x
# dies out for good. For more background, see:
-# <http://lists.gnu.org/archive/html/automake/2012-07/msg00001.html>
-# <http://lists.gnu.org/archive/html/automake/2012-07/msg00014.html>
+# <https://lists.gnu.org/archive/html/automake/2012-07/msg00001.html>
+# <https://lists.gnu.org/archive/html/automake/2012-07/msg00014.html>
mkdir_p='$(MKDIR_P)'
# We need awk for the "check" target (and possibly the TAP driver). The
@@ -2510,7 +2505,7 @@ END
Aborting the configuration process, to ensure you take notice of the issue.
You can download and install GNU coreutils to get an 'rm' implementation
-that behaves properly: <http://www.gnu.org/software/coreutils/>.
+that behaves properly: <https://www.gnu.org/software/coreutils/>.
If you want to complete the configuration process using your problematic
'rm' anyway, export the environment variable ACCEPT_INFERIOR_RM_PROGRAM
@@ -3033,45 +3028,45 @@ DEPDIR="${am__leading_dot}deps"
ac_config_commands="$ac_config_commands depfiles"
-
-am_make=${MAKE-make}
-cat > confinc << 'END'
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${MAKE-make} supports the include directive" >&5
+$as_echo_n "checking whether ${MAKE-make} supports the include directive... " >&6; }
+cat > confinc.mk << 'END'
am__doit:
- @echo this is the am__doit target
+ @echo this is the am__doit target >confinc.out
.PHONY: am__doit
END
-# If we don't find an include directive, just comment out the code.
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for style of include used by $am_make" >&5
-$as_echo_n "checking for style of include used by $am_make... " >&6; }
am__include="#"
am__quote=
-_am_result=none
-# First try GNU make style include.
-echo "include confinc" > confmf
-# Ignore all kinds of additional output from 'make'.
-case `$am_make -s -f confmf 2> /dev/null` in #(
-*the\ am__doit\ target*)
- am__include=include
- am__quote=
- _am_result=GNU
- ;;
-esac
-# Now try BSD make style include.
-if test "$am__include" = "#"; then
- echo '.include "confinc"' > confmf
- case `$am_make -s -f confmf 2> /dev/null` in #(
- *the\ am__doit\ target*)
- am__include=.include
- am__quote="\""
- _am_result=BSD
+# BSD make does it like this.
+echo '.include "confinc.mk" # ignored' > confmf.BSD
+# Other make implementations (GNU, Solaris 10, AIX) do it like this.
+echo 'include confinc.mk # ignored' > confmf.GNU
+_am_result=no
+for s in GNU BSD; do
+ { echo "$as_me:$LINENO: ${MAKE-make} -f confmf.$s && cat confinc.out" >&5
+ (${MAKE-make} -f confmf.$s && cat confinc.out) >&5 2>&5
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); }
+ case $?:`cat confinc.out 2>/dev/null` in #(
+ '0:this is the am__doit target') :
+ case $s in #(
+ BSD) :
+ am__include='.include' am__quote='"' ;; #(
+ *) :
+ am__include='include' am__quote='' ;;
+esac ;; #(
+ *) :
;;
- esac
-fi
-
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $_am_result" >&5
-$as_echo "$_am_result" >&6; }
-rm -f confinc confmf
+esac
+ if test "$am__include" != "#"; then
+ _am_result="yes ($s style)"
+ break
+ fi
+done
+rm -f confinc.* confmf.*
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: ${_am_result}" >&5
+$as_echo "${_am_result}" >&6; }
# Check whether --enable-dependency-tracking was given.
if test "${enable_dependency_tracking+set}" = set; then :
@@ -3921,7 +3916,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by rock $as_me 1.8, which was
+This file was extended by rock $as_me 1.9.2, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -3978,7 +3973,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
-rock config.status 1.8
+rock config.status 1.9.2
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
@@ -4086,7 +4081,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
#
# INIT-COMMANDS
#
-AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir"
+AMDEP_TRUE="$AMDEP_TRUE" MAKE="${MAKE-make}"
_ACEOF
@@ -4534,29 +4529,35 @@ $as_echo "$as_me: executing $ac_file commands" >&6;}
# Older Autoconf quotes --file arguments for eval, but not when files
# are listed without --file. Let's play safe and only enable the eval
# if we detect the quoting.
- case $CONFIG_FILES in
- *\'*) eval set x "$CONFIG_FILES" ;;
- *) set x $CONFIG_FILES ;;
- esac
+ # TODO: see whether this extra hack can be removed once we start
+ # requiring Autoconf 2.70 or later.
+ case $CONFIG_FILES in #(
+ *\'*) :
+ eval set x "$CONFIG_FILES" ;; #(
+ *) :
+ set x $CONFIG_FILES ;; #(
+ *) :
+ ;;
+esac
shift
- for mf
+ # Used to flag and report bootstrapping failures.
+ am_rc=0
+ for am_mf
do
# Strip MF so we end up with the name of the file.
- mf=`echo "$mf" | sed -e 's/:.*$//'`
- # Check whether this is an Automake generated Makefile or not.
- # We used to match only the files named 'Makefile.in', but
- # some people rename them; so instead we look at the file content.
- # Grep'ing the first line is not enough: some people post-process
- # each Makefile.in and add a new line on top of each file to say so.
- # Grep'ing the whole file is not good either: AIX grep has a line
+ am_mf=`$as_echo "$am_mf" | sed -e 's/:.*$//'`
+ # Check whether this is an Automake generated Makefile which includes
+ # dependency-tracking related rules and includes.
+ # Grep'ing the whole file directly is not great: AIX grep has a line
# limit of 2048, but all sed's we know have understand at least 4000.
- if sed -n 's,^#.*generated by automake.*,X,p' "$mf" | grep X >/dev/null 2>&1; then
- dirpart=`$as_dirname -- "$mf" ||
-$as_expr X"$mf" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
- X"$mf" : 'X\(//\)[^/]' \| \
- X"$mf" : 'X\(//\)$' \| \
- X"$mf" : 'X\(/\)' \| . 2>/dev/null ||
-$as_echo X"$mf" |
+ sed -n 's,^am--depfiles:.*,X,p' "$am_mf" | grep X >/dev/null 2>&1 \
+ || continue
+ am_dirpart=`$as_dirname -- "$am_mf" ||
+$as_expr X"$am_mf" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$am_mf" : 'X\(//\)[^/]' \| \
+ X"$am_mf" : 'X\(//\)$' \| \
+ X"$am_mf" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X"$am_mf" |
sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
s//\1/
q
@@ -4574,53 +4575,50 @@ $as_echo X"$mf" |
q
}
s/.*/./; q'`
- else
- continue
- fi
- # Extract the definition of DEPDIR, am__include, and am__quote
- # from the Makefile without running 'make'.
- DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"`
- test -z "$DEPDIR" && continue
- am__include=`sed -n 's/^am__include = //p' < "$mf"`
- test -z "$am__include" && continue
- am__quote=`sed -n 's/^am__quote = //p' < "$mf"`
- # Find all dependency output files, they are included files with
- # $(DEPDIR) in their names. We invoke sed twice because it is the
- # simplest approach to changing $(DEPDIR) to its actual value in the
- # expansion.
- for file in `sed -n "
- s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \
- sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g'`; do
- # Make sure the directory exists.
- test -f "$dirpart/$file" && continue
- fdir=`$as_dirname -- "$file" ||
-$as_expr X"$file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
- X"$file" : 'X\(//\)[^/]' \| \
- X"$file" : 'X\(//\)$' \| \
- X"$file" : 'X\(/\)' \| . 2>/dev/null ||
-$as_echo X"$file" |
- sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
- s//\1/
- q
- }
- /^X\(\/\/\)[^/].*/{
+ am_filepart=`$as_basename -- "$am_mf" ||
+$as_expr X/"$am_mf" : '.*/\([^/][^/]*\)/*$' \| \
+ X"$am_mf" : 'X\(//\)$' \| \
+ X"$am_mf" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X/"$am_mf" |
+ sed '/^.*\/\([^/][^/]*\)\/*$/{
s//\1/
q
}
- /^X\(\/\/\)$/{
+ /^X\/\(\/\/\)$/{
s//\1/
q
}
- /^X\(\/\).*/{
+ /^X\/\(\/\).*/{
s//\1/
q
}
s/.*/./; q'`
- as_dir=$dirpart/$fdir; as_fn_mkdir_p
- # echo "creating $dirpart/$file"
- echo '# dummy' > "$dirpart/$file"
- done
+ { echo "$as_me:$LINENO: cd "$am_dirpart" \
+ && sed -e '/# am--include-marker/d' "$am_filepart" \
+ | $MAKE -f - am--depfiles" >&5
+ (cd "$am_dirpart" \
+ && sed -e '/# am--include-marker/d' "$am_filepart" \
+ | $MAKE -f - am--depfiles) >&5 2>&5
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); } || am_rc=$?
done
+ if test $am_rc -ne 0; then
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "Something went wrong bootstrapping makefile fragments
+ for automatic dependency tracking. If GNU make was not used, consider
+ re-running the configure script with MAKE=\"gmake\" (or whatever is
+ necessary). You can also try re-running configure with the
+ '--disable-dependency-tracking' option to at least be able to build
+ the package (albeit without support for automatic dependency tracking).
+See \`config.log' for more details" "$LINENO" 5; }
+ fi
+ { am_dirpart=; unset am_dirpart;}
+ { am_filepart=; unset am_filepart;}
+ { am_mf=; unset am_mf;}
+ { am_rc=; unset am_rc;}
+ rm -f conftest-deps.mk
}
;;
diff --git a/configure.ac b/configure.ac
index 9c00933d9de0274bab9283d6fd1e85ce3c94d2b3..44554820549dd83acfec0cf0d02a145d60ea9e88 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,7 +1,7 @@
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ(2.59)
-AC_INIT(rock, 1.8)
+AC_INIT(rock, 1.9.3)
AC_CANONICAL_SYSTEM
diff --git a/doc/Makefile.in b/doc/Makefile.in
index 13f362c4625dc4010341cb676d27f2f283d88841..7a8b464c7b82138ca915b3882560afb88e9d6088 100644
--- a/doc/Makefile.in
+++ b/doc/Makefile.in
@@ -1,7 +1,7 @@
-# Makefile.in generated by automake 1.15 from Makefile.am.
+# Makefile.in generated by automake 1.16.3 from Makefile.am.
# @configure_input@
-# Copyright (C) 1994-2014 Free Software Foundation, Inc.
+# Copyright (C) 1994-2020 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -275,8 +275,8 @@ Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
*config.status*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
*) \
- echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
- cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
esac;
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
@@ -335,7 +335,10 @@ ctags CTAGS:
cscope cscopelist:
-distdir: $(DISTFILES)
+distdir: $(BUILT_SOURCES)
+ $(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
list='$(DISTFILES)'; \
diff --git a/doc/rock.pod b/doc/rock.pod
index 3e5c84305ca754fa4da48c931881699b44551ee7..ac7d24f9ce38b7f7f44d8aeccf1af19c9e9dda26 100644
--- a/doc/rock.pod
+++ b/doc/rock.pod
@@ -11,7 +11,7 @@
=over 4
-=item B<rock> [B<-h>] [B<-i> F<file>] [B<-o> F<file>] [B<-k> F<k_mer_size>] [B<-p> F<filter_PE_mode>] [B<-q> F<nucl_qual_score_threshold>] [B<-C> F<kappa>] [B<-c> F<kappa_prime>] [B<-l> F<lambda>] [B<-n> F<nb_distinct_k_mer>] [B<-m> F<min_valid_k_mer_per_read>] [B<-g> F<CMS size in GB>] [Args]
+=item B<rock> [B<-h>] [B<-i> F<file>] [B<-o> F<file>] [B<-k> F<k_mer_size>] [B<-q> F<nucl_qual_score_threshold>] [B<-C> F<kappa>] [B<-c> F<kappa_prime>] [B<-l> F<lambda>] [B<-n> F<nb_distinct_k_mer>] [B<-m> F<min_valid_k_mer_per_read>] [Args]
=back
@@ -30,35 +30,23 @@ For more information on F<file> format, see the description section.
=item -o F<file>
List of output fastq file names is in F<file>.
Note that names of output fastq files must be in the same order as names of input fastq.
-Result of filtering 1rst fastq file will go in the first first fastq file mentionned in F<file>
+Result of filtering 1rst fastq file will go in the first fastq file mentionned in F<file>
For more information on F<file> format, see the description section.
=item -k
-Specify wanted k-mer size. Default is 30; maximum is 32.
+Specify wanted k-mer size. Default is 25; maximum is 32.
=item -q
Specify minimum threshold for nucleotides quality score. Nucleotides that have a score below that threshold will be considered as errors (just like N nucleotides).
-Default is 10.
+Default is 0.
=item -m
-To be used together with -q.
+Can be used together with -q.
Specify minimum number of correct k-mers required to keep a read for CMS filter.
Indicate only a integer in version 1.4 and later.
-By default, PE reads are processed independantly (ie kept in the CMS if at least PE1 or PE2 has coverage<=C and >=c).
-As -m defaults to 1,it means that as there is at least 1 valid k-mer in each part of the PE it is kept.
-
-=item -p
-
-Use this option to tell ROCK how you want the paired end reads to be processed.
-Argument can take value 0,1 or 2.
-0 means that PE are processed as single.
-1 means that PE are processed separately and that the read will be removed by the low filter if both parts of it have a median coverage below the given threshold (strict filter).
-2 means that PE are processed separately and that the read will be removed by the low filter if one part of it has a median coverage below the given threshold (lenient filter).
-Default value is 1.
-
=item -C
@@ -66,27 +54,23 @@ Specify maximum coverage wanted. Default is 70. Maximum is 65535.
=item -c
-Specify lower threshold for coverage. Default is 5 (no lower threshold).
+Specify lower threshold for coverage. Default is 0 (no lower threshold).
+
+=item -f
+
+Specify maximum collision probability in the CMS (default is 0.05).
+To be used with -n so that ROCK can compute the appropriate number of arrays in the CMS.
=item -l
Specify lambda (number of arrays wanted in the count min sketch filter).
-Default is minimum between computed l and 4.
-l is the biggest value such as l<B/(b*MAX_INT) where:
- b is 1 or 2 depending on kappa value,
- B is the RAM quantity on the machine.
+Default is minimum 4.
=item -n
Indicate the number of distinct k-mers in input fastq files.
This is useful to compute a more appropriate value than the default one for lambda if you have not specified it with -l.
-=item -g
-
-Wanted size for the CMS (in gigabytes).
-Default is to use the minimum between
-1) all the machine's memory minus a constant for storing input fastq files indexes.
-2) a default value of 4.
=item -v
@@ -107,18 +91,16 @@ Its aim is to filter fastq files on different criteria:
3rd median coverage of the read: if more than half of the k-mers in the read have occurred less than kappa_prime times; then the read is removed.
This is very useful for removing reads from "polluted" samples.
-
B<ROCK> is highly parameterizable via its options.
It is possible to refine filtering by adding more criteria:
1) nucleotide score threshold: k-mers containing at least 1 nucleotide below specified threshold will not be taken into account.
2) minimum number of correct k-mers : let X be this number. Reads containing less than X k-mers without nucleotides below threshold will not be processed.
They will be put in files named with .undefined extension.
- 3) For pair end reads, it is possible to indicate if you want them to be processed as single. By default they are processed separately.
+ 3) Pair end reads are processed as single.
It is possible to specify the size of the count min sketch by:
1) indicating the number of arrays that you want in it (via the -l option),
2) indicating the number of different k-mers in your input files (via the -n option). Thus B<ROCK> will compute the best lambda value by minimizing the probability of collision and taking into account the amount of RAM on your system.
- 3) indicating (in GigaBytes) the size you want it to have. B<ROCK> will compute the number of arrays in it depending on kappa value.
B<ROCK> supports 2 implementations for the count min sketch component. It chooses the implementation to use depending on kappa value.
Indeed for small kappa values (ie <255) a single Byte can be used for storing k-mer occurrences counters.
@@ -142,28 +124,26 @@ Reads for median coverage < 20 (kappa_prime) will be removed.
Rock will filter the 3 files in input with default parameters:
high filter=70,
-low filter=5
-nucleotide quality score threshold=10
+low filter=0
+nucleotide quality score threshold=0
minimum number of correct k-mer in a read=1
-pair end reads are processed separately
-lambda=min(4,possible value according to RAM amount on the execution machine).
+lambda=4.
-And it will produce 6 output files in the execution directory.
+And it will produce 3 output files in the execution directory.
These files will be named:
my_PE1.rock.fq,myPE2.rock.fq
my_single.rock.fq
-my_PE1.undefined.fq,myPE2.undefined.fq
-my_single.undefined.fq
+
=back
=head1 NOTE ON MEMORY CONSUMPTION.
-count min sketch size is MAX_INT*lambda or MAX_INT*lambda*2 if kappa>255.
+count min sketch size is MAX_UINT*lambda or MAX_UINT*lambda*2 if kappa>255.
Besides count min sketch, memory is also needed to store information related to reads in the input fastq files.
13 Bytes per fastq record are needed plus some memory for the containers to work and for the read/write buffers.
-To give you an idea, processing approximately 1.8 billion of reads in 2 fastq files (tot 500 GB data on disk) took 28 GB of memory plus 8 GB for the CMS (lambda=4, byte implementation).
+To give you an idea, processing approximately 1.8 billion of reads in 2 fastq files (tot 500 GB data on disk) took 28 GB of memory plus 16 GB for the CMS (lambda=4, byte implementation).
diff --git a/src/CountMinSketch.hpp b/src/CountMinSketch.hpp
index 3d82b0b0d88696eb4e92d97306a5fba71d97870e..b552a286802d4d38fe297a773ab1da4b53f86550 100644
--- a/src/CountMinSketch.hpp
+++ b/src/CountMinSketch.hpp
@@ -12,73 +12,23 @@
#include <stdlib.h>
#include <string.h>
#include <limits.h>
+#include <vector>
+#include <cmath>
#include "rock_commons.h"
-
#define BAD_TYPE -1
-// Store the non mersenne prime numbers for modulo hashing in this array.
- const int Pi_js[500]={2147469629, 2147469637, 2147469659, 2147469679, 2147469703, 2147469781, 2147469817, 2147469823, 2147469829, 2147469881,\
- 2147469917, 2147469943, 2147469949, 2147469983, 2147470007, 2147470019, 2147470027, 2147470043, 2147470057, 2147470067,\
- 2147470081, 2147470111, 2147470123, 2147470139, 2147470147, 2147470177, 2147470183, 2147470211, 2147470229, 2147470249,\
- 2147470313, 2147470327, 2147470333, 2147470361, 2147470427, 2147470453, 2147470511, 2147470513, 2147470529, 2147470531,\
- 2147470553, 2147470579, 2147470597, 2147470603, 2147470627, 2147470643, 2147470673, 2147470679, 2147470723, 2147470727,\
- 2147470733, 2147470751, 2147470769, 2147470771,2147483059, 2147483069, 2147483077, 2147483123, 2147483137, 2147483171,\
- 2147473897, 2147473921, 2147473963, 2147474009, 2147474027, 2147474029, 2147474071, 2147474093, 2147474113, 2147474123,\
- 2147474149, 2147474159, 2147474201, 2147474213, 2147474239, 2147474279, 2147474359, 2147474383, 2147474393, 2147474477,\
- 2147474479, 2147474491, 2147474513, 2147474519, 2147474531, 2147474551, 2147474597, 2147474627, 2147474657, 2147474711,\
- 2147474717, 2147474789, 2147474803, 2147474807, 2147474809, 2147474831, 2147474837, 2147474843, 2147474851, 2147474881,\
- 2147474887, 2147474891, 2147474921, 2147474929, 2147474947, 2147474951, 2147474963, 2147475047, 2147475061, 2147475103,\
- 2147475107, 2147475149, 2147475179, 2147475181, 2147475193, 2147475203, 2147475221, 2147475229, 2147475233, 2147475251,\
- 2147475257, 2147475269, 2147475277, 2147475331, 2147475347, 2147475349, 2147475367, 2147475373, 2147475397, 2147475401,\
- 2147475413, 2147475439, 2147475481, 2147475487, 2147475497, 2147475503, 2147475509, 2147475521, 2147475541, 2147475553,\
- 2147475559, 2147475563, 2147475587, 2147475593, 2147475601, 2147475641, 2147475653, 2147475691, 2147475713, 2147475721,\
- 2147475739, 2147475787, 2147475791, 2147475797, 2147475829, 2147475851, 2147475859, 2147475871, 2147475899, 2147475929,\
- 2147475971, 2147475973, 2147475977, 2147475997, 2147476031, 2147476073, 2147476087, 2147476109, 2147476127, 2147476139,\
- 2147476141, 2147476169, 2147476183, 2147476211, 2147476249, 2147476291, 2147476321, 2147476327, 2147476367, 2147476381,\
- 2147476399, 2147476417, 2147476517, 2147476519, 2147476543, 2147476607, 2147476619, 2147476649, 2147476663, 2147476687,\
- 2147476693, 2147476699, 2147476739, 2147476741, 2147476763, 2147476769, 2147476777, 2147476789, 2147476819, 2147476823,\
- 2147476841, 2147476871, 2147476897, 2147476927, 2147476931, 2147476937, 2147476943, 2147476951, 2147476963, 2147476979,\
- 2147477021, 2147477029, 2147477063, 2147477093, 2147477107, 2147477113, 2147477159, 2147477191, 2147477201, 2147477203,\
- 2147477207, 2147477209, 2147477237, 2147477249, 2147477273, 2147477323, 2147477393, 2147477399, 2147477419, 2147477443,\
- 2147477467, 2147477473, 2147477503, 2147477513, 2147477531, 2147477533, 2147477599, 2147477627, 2147477681, 2147477687,\
- 2147477699, 2147477701, 2147477737, 2147477807, 2147477809, 2147477833, 2147477851, 2147477861, 2147477873, 2147477879,\
- 2147477881, 2147477933, 2147477953, 2147477989, 2147478013, 2147478017, 2147478049, 2147478079, 2147478083, 2147483179,\
- 2147478089, 2147478127, 2147478133, 2147478149, 2147478253, 2147478259, 2147478293, 2147478299, 2147478331, 2147478349,\
- 2147478373, 2147478461, 2147478481, 2147478491, 2147478497, 2147478503, 2147478517, 2147478521, 2147478563, 2147478569,\
- 2147478581, 2147478601, 2147478611, 2147478647, 2147478649, 2147478653, 2147478659, 2147478661, 2147478673, 2147478701,\
- 2147478703, 2147478719, 2147478721, 2147478727, 2147478731, 2147478733, 2147478763, 2147478791, 2147478821, 2147478859,\
- 2147478863, 2147478889, 2147478899, 2147478911, 2147478919, 2147478937, 2147478959, 2147478961, 2147478967, 2147478997,\
- 2147479013, 2147479031, 2147479057, 2147479063, 2147479079, 2147479091, 2147479097, 2147479121, 2147479129, 2147479133,\
- 2147479171, 2147479189, 2147479231, 2147479259, 2147479273, 2147479307, 2147479339, 2147479349, 2147479361, 2147479381,\
- 2147479403, 2147479421, 2147479447, 2147479489, 2147479507, 2147479513, 2147479517, 2147479531, 2147479547, 2147479549,\
- 2147479573, 2147479589, 2147479601, 2147479619, 2147479637, 2147479643, 2147479657, 2147479681, 2147479751, 2147479753,\
- 2147479757, 2147479781, 2147479787, 2147479819, 2147479823, 2147479879, 2147479891, 2147479897, 2147479907, 2147479937,\
- 2147479991, 2147480009, 2147480011, 2147480039, 2147480161, 2147480197, 2147480207, 2147480219, 2147480227, 2147480297,\
- 2147480299, 2147480311, 2147480327, 2147480369, 2147480429, 2147480437, 2147480459, 2147480471, 2147480507, 2147480519,\
- 2147480527, 2147480551, 2147480591, 2147480611, 2147480623, 2147480641, 2147480651, 2147480677, 2147480683, 2147480707,\
- 2147480723, 2147480743, 2147480747, 2147480791, 2147480837, 2147480843, 2147480849, 2147480893, 2147480897, 2147480899,\
- 2147480921, 2147480927, 2147480941, 2147480957, 2147480969, 2147480971, 2147480989, 2147481019, 2147481031, 2147481053,\
- 2147481071, 2147481139, 2147481143, 2147481151, 2147481173, 2147481179, 2147481199, 2147481209, 2147481247, 2147481263,\
- 2147481269, 2147481283, 2147481311, 2147481317, 2147481337, 2147481353, 2147481359, 2147481367, 2147481373, 2147481487,\
- 2147481491, 2147481499, 2147481509, 2147481529, 2147481563, 2147481571, 2147481629, 2147481673, 2147481793, 2147481797,\
- 2147481811, 2147481827, 2147481863, 2147481883, 2147481893, 2147481899, 2147481901, 2147481907, 2147481937, 2147481949,\
- 2147481967, 2147481997, 2147482021, 2147482063, 2147482081, 2147482091, 2147482093, 2147482121, 2147482223, 2147482231,\
- 2147482237, 2147482273, 2147482291, 2147482327, 2147482343, 2147482349, 2147482361, 2147482367, 2147482409, 2147482417,\
- 2147482481, 2147482501, 2147482507, 2147482577, 2147482583, 2147482591, 2147482621, 2147482661, 2147482663, 2147482681,\
- 2147482693, 2147482697, 2147482739, 2147482763, 2147482801, 2147482811, 2147482817, 2147482819, 2147482859, 2147482867,\
- 2147482873, 2147482877, 2147482921, 2147482937, 2147482943, 2147482949, 2147482951, 2147483029, 2147483033, 2147483053};
-
typedef struct {
int lambda;
int kappa;
int kappa_prime;
- int filter_size; // max amount of RAM wanted for the CMS.
+ int max_filter_size; // max amount of RAM wanted for the CMS.
int filter_PE_separately; // indicates whether PE reads must be treated as single by the cms. Indeed it may appear that 1 end contains useful k-mer but that other end contains k-mer such that "global" median is below threshold.
// In this case, read is rejected by the CMS (default behavior). We want to do an experimentation to see if keeping such reads wold lead to better results in assembling.
// Values can be 0 (process PE as single),1 (process PE separately with strict implementation for kappa_prime) or 2 (process PE separately with lenient implementation for kappa prime).
+ float wanted_max_collision_proba;
} CMSparams;
template <typename T>
@@ -111,37 +61,18 @@ private:
T ** cms_lambda_array;
T mask;
+ std::vector<unsigned long> zeroes; //! Number of unset buckets per hash.
- inline int hash64to32(const unsigned long& w ,const int& j) {
- return w % Pi_js[j];
- }
-
- // bit shift version of hash function to start. Keep it just in case. Not used because it is very slow.
- /* int hash64to32bs(unsigned long w,int j) {
- unsigned long h_tmp;
- unsigned long h=~w;
- h+=w<<18;
- h_tmp=h>>31;
- h_tmp&=mask1;
- h=h^h_tmp;
- h*=j;
- h_tmp=h>>11;
- h_tmp&=mask2;
- h ^=h_tmp;
- h+=h<<6;
- h_tmp=h>>22;
- h_tmp&=mask3;
- h^=h_tmp;
- return (int) (h& 2147483647);
- }*/
-
+ inline unsigned int hash64to32(const unsigned long& w ,const int& j) {
+ return w % Pi_js[j];
+ }
inline int isRCovBelowThres(const T_read_numericValues& read_val, const int& threshold);
inline int isRCovBelowThresPartial(const readNumericValues& read_val, const int& threshold, const int& start=0,const int& stop=0);
- void init(int glambda,int gkappa,int gkappa_prime,int filter_PE_separately=1);
+ void init(int glambda,int gkappa,int gkappa_prime,int filter_PE_separately=0);
// for unit tests.
friend void test_CMS(int lambda,int kappa,int kappa_prime);
@@ -165,18 +96,24 @@ public:
free(cms_lambda_array);
}
+ //! This method must be called after getNbDistinctKMers because getNbDistinctKmers sets the counters.
+ std::vector<unsigned long> getUnsetBuckets() {
+ return zeroes;
+ }
+
inline void addKMer(const unsigned long& val1) {
- int h,j;
- T cnt;
- j=lambda;
- while(--j>=0) {
- h=hash64to32(val1,j);
- cnt=cms_lambda_array[j] [h];
- cnt++;
- cms_lambda_array[j] [h]=(cnt & mask);
- }
- }
+ int j;
+ unsigned int h;
+ T cnt;
+ j=lambda;
+ while(--j>=0) {
+ h=hash64to32(val1,j);
+ cnt=cms_lambda_array[j] [h];
+ cnt++;
+ cms_lambda_array[j] [h]=(cnt & mask);
+ }
+ }
// keep that just for unit testing purpose
T getEstimatedNbOcc(const unsigned long& val);
@@ -198,14 +135,15 @@ public:
/*
* Determines whether median of k-mer coverage is below threshold or not.
*/
-#include <stdio.h>
+//#include <stdio.h>
// this is a few seconds faster
template<typename T> inline int CountMinSketch<T>::isRCovBelowThres(const T_read_numericValues& read_val, const int& threshold) {
int PE1_below_thres;
int PE2_below_thres=0;
int a1=0,a2=0;
int b1,b2=0;
- int j,h;
+ int j;
+ unsigned int h;
T min;
int num=read_val.single_or_PE_val.size();
@@ -312,8 +250,8 @@ template<typename T> void CountMinSketch<T>::init(int glambda,int gkappa,int gka
mask=get_mask<T>::value;
cms_lambda_array= (T**) malloc(lambda*sizeof(T*));
for (j=0;j<lambda;j++) {
- cms_lambda_array[j]=(T *) malloc(sizeof(T)*INT_MAX);
- memset(cms_lambda_array[j],0,INT_MAX);
+ cms_lambda_array[j]=(T *) malloc(sizeof(T)*k_arr_cms_size);
+ memset(cms_lambda_array[j],0,k_arr_cms_size);
}
}
@@ -355,22 +293,35 @@ template<typename T> int CountMinSketch<T>::isBeneathMinKappa(const T_read_numer
return res;
}
+
+
/*
- * Go through the arrays of the CMS and returns the number of distinct k_mers (biggest nbr of non-zero counters amongst all arrays)
+ * Go through the arrays of the CMS and returns an estimation of the number of distinct k_mers (new formula from Alexis)
*/
template<typename T> unsigned long CountMinSketch<T>::getNbDistinctKMers() {
- int j,h;
- unsigned long min=INT_MAX;
+ int j;
+ unsigned long max=0;
+ unsigned int h;
+ unsigned long n;
+ unsigned long m;
for (j=lambda-1;j>=0;--j) {
- unsigned long nb_k_mers=0;
- for (h=INT_MAX-1;h>=0;--h) {
- (cms_lambda_array[j] [h]>0)? nb_k_mers+=1: nb_k_mers;
+ m=Pi_js[j];
+ unsigned long z=0; // number of zeroes in a CMS array.
+ for (h=Pi_js[j]-1;h>0;--h) { // Have to process the case of h=0 separately otherwise as h is now an unsigned int, it is always >=0 which causes an infinite loop.
+ (cms_lambda_array[j] [h]==0)? z+=1: z;
}
- (nb_k_mers<min) ?min=nb_k_mers: min;
+ (cms_lambda_array[j] [0]==0)? z+=1: z;
+ zeroes.push_back(z);
+ double lnz=log(z);
+ double lnm=log(m);
+ double lnm1=log(m-1);
+ double deno=lnm1-lnm;
+ double nume=lnz-lnm;
+ n=nume/deno;
+ (n>max)?max=n:max;
}
- return min;
+ return max;
}
-
#endif /* COUNTMINSKETCH_HPP_ */
diff --git a/src/Filter.hpp b/src/Filter.hpp
index 9b2eaf5e54d877f52b43e26096f4d9487e0825cf..a1797a2e4db2a2140bb20a17c5b1f4e087761f8d 100644
--- a/src/Filter.hpp
+++ b/src/Filter.hpp
@@ -7,12 +7,14 @@
#ifndef FILTER_HPP_
#define FILTER_HPP_
#include <errno.h>
+#include <err.h>
#include <sys/time.h>
#include <sys/resource.h>
#include "CountMinSketch.hpp"
#include "ReadProcessor.h"
#include "read_utils.h"
+#include "ROCKparams.h"
static const unsigned short ushortmask=65535;
@@ -29,9 +31,10 @@ class Filter {
ShortCountMinSketch * pShortCMS;
FasqQualThreshold qual_thres;
- long nb_bytes_before_fill_CMS,nb_bytes_after_fill_CMS;
+ // long nb_bytes_before_fill_CMS,nb_bytes_after_fill_CMS;
+ //long nb_bytes_CMS; //! Number of bytes taken by the underlying CMS.
- void getRSS(int before_fill=0);
+ void getRSS();
template <typename T> void underlyingfillCMS(FqBaseBackend* map_id_backend[],int nb_be, int k, srp* io_sr, CountMinSketch<T>* cms);
@@ -42,18 +45,20 @@ public:
Filter(const CMSparams& parms,const FasqQualThreshold& the_qual_thres) {
pByteCMS=NULL;
pShortCMS=NULL;
- getRSS();
if (parms.kappa<ubytemask) pByteCMS=new ByteCountMinSketch(parms);
else pShortCMS=new ShortCountMinSketch(parms);
qual_thres=the_qual_thres;
-
- nb_bytes_after_fill_CMS=0;
+ //getRSS();
+ //nb_bytes_CMS=0;
+ //nb_bytes_after_fill_CMS=0;
}
void fillCMS(FqBaseBackend* map_id_backend[],int nb_be,int k,srp* io_sr);
void lowFilterCMS(FqBaseBackend* map_id_backend[],int nb_be,int k, srp* io_sr);
- unsigned long getApproxNbDistinctKMers();
+ unsigned long getApproxNbDistinctKMers() const ;
unsigned long getSize();
+ unsigned int getNbBitsForCounters() const; // returns the size of the dada type used for the counters of the CMS.
+ std::vector<unsigned long> getUnsetBuckets() const;
};
template <typename T> void Filter::underlyingfillCMS(FqBaseBackend* map_id_backend[],int nb_be, int k, srp* io_sr, CountMinSketch<T>* cms) {
@@ -138,7 +143,7 @@ template <typename T> void Filter::underlyinglowFilterCMS(FqBaseBackend* map_id_
void Filter::fillCMS(FqBaseBackend* map_id_backend[],int nb_be,int k,srp* io_sr) {
if (pByteCMS!=NULL) underlyingfillCMS<unsigned char>(map_id_backend,nb_be, k, io_sr,pByteCMS);
else underlyingfillCMS<unsigned short>(map_id_backend,nb_be, k, io_sr,pShortCMS);
- getRSS(1);
+ //getRSS();
}
void Filter::lowFilterCMS(FqBaseBackend* map_id_backend[],int nb_be,int k, srp* io_sr) {
@@ -146,23 +151,33 @@ void Filter::lowFilterCMS(FqBaseBackend* map_id_backend[],int nb_be,int k, srp*
else underlyinglowFilterCMS<unsigned short>(map_id_backend,nb_be, k, io_sr,pShortCMS);
}
-unsigned long Filter::getApproxNbDistinctKMers() {
+unsigned long Filter::getApproxNbDistinctKMers() const {
if (pByteCMS!=NULL) return pByteCMS->getNbDistinctKMers();
else return pShortCMS->getNbDistinctKMers();
}
-void Filter::getRSS(int before_fill) {
+std::vector<unsigned long> Filter::getUnsetBuckets() const {
+ if (pByteCMS!=NULL) return pByteCMS->getUnsetBuckets();
+ else return pShortCMS->getUnsetBuckets();
+}
+
+/*
+void Filter::getRSS() {
struct rusage usage;
int res2=getrusage(RUSAGE_SELF,&usage);
if (res2==-1) err(errno,"cannot get resource usage.");
- if (before_fill==0) nb_bytes_before_fill_CMS=usage.ru_maxrss;
- else nb_bytes_after_fill_CMS=usage.ru_maxrss;
+ nb_bytes_CMS=usage.ru_maxrss;
}
unsigned long Filter::getSize() {
- unsigned long cms_size=nb_bytes_after_fill_CMS-nb_bytes_before_fill_CMS;
+ unsigned long cms_size=nb_bytes_CMS;
// cms_size/=8; ru_maxrss seems to be already in Bytes.
return cms_size;
+}*/
+
+unsigned int Filter::getNbBitsForCounters() const {
+ if (pByteCMS!=NULL) return sizeof(unsigned char)*8;
+ else return sizeof(unsigned short)*8;
}
#endif
diff --git a/src/FqBaseBackend.h b/src/FqBaseBackend.h
index 1cd30fef43fb860e5e103611f960828017a9a43d..4dd4e6461ae713fb2546540616bfe76bb0579485 100644
--- a/src/FqBaseBackend.h
+++ b/src/FqBaseBackend.h
@@ -5,6 +5,8 @@
* Author: vlegrand
*/
+
+
#ifndef FQBASEBACKEND_H_
#define FQBASEBACKEND_H_
diff --git a/src/FqMainBackend.cpp b/src/FqMainBackend.cpp
index 4125c7e193891291914dcde94d4eb02f98a25104..e779cd8f45399b282f04ee29673960b0f0eadef1 100644
--- a/src/FqMainBackend.cpp
+++ b/src/FqMainBackend.cpp
@@ -38,6 +38,7 @@ using namespace std;
FqMainBackend::FqMainBackend(srp * io_sr):FqBaseBackend() {
p_auxFqProcessor=NULL;
p_scoreReadStruct=io_sr;
+ cnt_k_mers=0;
}
@@ -62,10 +63,6 @@ void FqMainBackend::processFile(char * filename,unsigned char f_id) {
err(errno,"cannot open file: %s.",filename);
}
- fp=fdopen(f_single,"r");
- if (fp==NULL) {
- err(errno,"cannot open file: %s.",filename);
- }
buf=(char *) malloc(buffer_size*sizeof(char));
if (buf==NULL) {
err(errno,"cannot allocate memory: %lu bytes.",buffer_size);
@@ -114,6 +111,7 @@ void FqMainBackend::onEndFastqRecord(T_fq_rec_info& rec_info,const T_buf_info& b
// empty buffer keeping current record
cur_fq_record[0]='\0';
if (p_auxFqProcessor!=NULL) p_auxFqProcessor->resetCurFqRecord();
+ cnt_k_mers+=nb_k_mer;
}
diff --git a/src/FqMainBackend.h b/src/FqMainBackend.h
index 656ab4bb210533bdcf5d52c848591276482f944c..73770438fbf89088b19a0333782d53fe55dbf82a 100644
--- a/src/FqMainBackend.h
+++ b/src/FqMainBackend.h
@@ -19,10 +19,11 @@
class FqMainBackend : public FqBaseBackend {
// static int treat_PE_as_single;
static int treat_PE_separately;
+ unsigned long cnt_k_mers; //! counter for the total number of k-mers found in a single file or in a pair of PE files.
+ //! This includes k-mers in error.
FqAuxBackend * p_auxFqProcessor; /* Another fastq processor component is necessary for handling the case of PE reads.*/
srp * p_scoreReadStruct; /* Where we store information about the reads. */
// char current_id[50]; used only for debug
-
// void debug_processBuf(int evt,const T_buf_info&, const unsigned long &);
void writeToUndefFile(const T_buf_info&);
void onEndFastqRecord(T_fq_rec_info& rec_info,const T_buf_info& bufinfo);
@@ -44,6 +45,10 @@ public:
static void setTreatPEMode(const int& treat_PE){
FqMainBackend::treat_PE_separately=treat_PE;
}
+
+ unsigned long getCntKMers() {
+ return cnt_k_mers;
+ }
};
#endif /* FQMAINBACKEND_H_ */
diff --git a/src/Makefile.am b/src/Makefile.am
index 9a87fb480901152739dc7409b5ee6e0be55360da..362bef115255dac3e46d19fa3b05d2b01a0b9347 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -3,7 +3,7 @@ LINTDEFS = $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS)
AM_CPPFLAGS = -DDATADIR=\"$(pkgdatadir)\"
bin_PROGRAMS=rock
-noinst_PROGRAMS = unit_test_fqreader unit_test_read_utils unit_test_cms unit_test_fqwriter unit_test_main_utils
+noinst_PROGRAMS = unit_test_fqreader unit_test_read_utils unit_test_cms unit_test_fqwriter unit_test_math_utils
## noinst_PROGRAMS =
noinst_LIBRARIES = librock.a
@@ -24,10 +24,10 @@ unit_test_cms_LDADD=librock.a
unit_test_fqwriter_SOURCES=unit_test_fqwriter.cpp
unit_test_fqwriter_LDADD=librock.a
-unit_test_main_utils_SOURCES=unit_test_main_utils.cpp
-unit_test_main_utils_LDADD=librock.a
+unit_test_math_utils_SOURCES=unit_test_math_utils.cpp
+unit_test_math_utils_LDADD=librock.a
librock_a_SOURCES = $(SRC)
-SRC = fqreader.cpp FqBaseBackend.cpp FqAuxBackend.cpp FqMainBackend.cpp read_utils.cpp ReadProcessor.cpp fqwriter.cpp main_utils.cpp ROCKparams.cpp unit_tests_tools.cpp
-HDR = srp.h fqreader.h FqConstants.h FqBaseBackend.h FqAuxBackend.h FqMainBackend.h ReadProcessor.h rock_commons.h read_utils.h CountMinSketch.hpp Filter.hpp fqwriter.h main_utils.h ROCKparams.h unit_tests_tools.h
+SRC = math_utils.cpp low_level_utils.cpp fqreader.cpp FqBaseBackend.cpp FqAuxBackend.cpp FqMainBackend.cpp read_utils.cpp ReadProcessor.cpp fqwriter.cpp ROCKparams.cpp unit_tests_tools.cpp
+HDR = math_utils.h low_level_utils.h srp.h fqreader.h FqConstants.h FqBaseBackend.h FqAuxBackend.h FqMainBackend.h ReadProcessor.h rock_commons.h read_utils.h CountMinSketch.hpp Filter.hpp fqwriter.h ROCKparams.h unit_tests_tools.h
diff --git a/src/Makefile.in b/src/Makefile.in
index cc233db332b3e107f589abc2dd3c1ae50a6eca7e..6eb87c3d511facc312efa2af50a48dccc8637ed7 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -1,7 +1,7 @@
-# Makefile.in generated by automake 1.15 from Makefile.am.
+# Makefile.in generated by automake 1.16.3 from Makefile.am.
# @configure_input@
-# Copyright (C) 1994-2014 Free Software Foundation, Inc.
+# Copyright (C) 1994-2020 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -94,7 +94,7 @@ target_triplet = @target@
bin_PROGRAMS = rock$(EXEEXT)
noinst_PROGRAMS = unit_test_fqreader$(EXEEXT) \
unit_test_read_utils$(EXEEXT) unit_test_cms$(EXEEXT) \
- unit_test_fqwriter$(EXEEXT) unit_test_main_utils$(EXEEXT)
+ unit_test_fqwriter$(EXEEXT) unit_test_math_utils$(EXEEXT)
subdir = src
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/configure.ac
@@ -105,6 +105,8 @@ DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \
mkinstalldirs = $(install_sh) -d
CONFIG_CLEAN_FILES =
CONFIG_CLEAN_VPATH_FILES =
+am__installdirs = "$(DESTDIR)$(bindir)"
+PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS)
LIBRARIES = $(noinst_LIBRARIES)
AR = ar
ARFLAGS = cru
@@ -114,15 +116,14 @@ am__v_AR_0 = @echo " AR " $@;
am__v_AR_1 =
librock_a_AR = $(AR) $(ARFLAGS)
librock_a_LIBADD =
-am__objects_1 = fqreader.$(OBJEXT) FqBaseBackend.$(OBJEXT) \
+am__objects_1 = math_utils.$(OBJEXT) low_level_utils.$(OBJEXT) \
+ fqreader.$(OBJEXT) FqBaseBackend.$(OBJEXT) \
FqAuxBackend.$(OBJEXT) FqMainBackend.$(OBJEXT) \
read_utils.$(OBJEXT) ReadProcessor.$(OBJEXT) \
- fqwriter.$(OBJEXT) main_utils.$(OBJEXT) ROCKparams.$(OBJEXT) \
+ fqwriter.$(OBJEXT) ROCKparams.$(OBJEXT) \
unit_tests_tools.$(OBJEXT)
am_librock_a_OBJECTS = $(am__objects_1)
librock_a_OBJECTS = $(am_librock_a_OBJECTS)
-am__installdirs = "$(DESTDIR)$(bindir)"
-PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS)
am_rock_OBJECTS = rock.$(OBJEXT)
rock_OBJECTS = $(am_rock_OBJECTS)
rock_DEPENDENCIES = librock.a
@@ -135,9 +136,9 @@ unit_test_fqreader_DEPENDENCIES = librock.a
am_unit_test_fqwriter_OBJECTS = unit_test_fqwriter.$(OBJEXT)
unit_test_fqwriter_OBJECTS = $(am_unit_test_fqwriter_OBJECTS)
unit_test_fqwriter_DEPENDENCIES = librock.a
-am_unit_test_main_utils_OBJECTS = unit_test_main_utils.$(OBJEXT)
-unit_test_main_utils_OBJECTS = $(am_unit_test_main_utils_OBJECTS)
-unit_test_main_utils_DEPENDENCIES = librock.a
+am_unit_test_math_utils_OBJECTS = unit_test_math_utils.$(OBJEXT)
+unit_test_math_utils_OBJECTS = $(am_unit_test_math_utils_OBJECTS)
+unit_test_math_utils_DEPENDENCIES = librock.a
am_unit_test_read_utils_OBJECTS = unit_test_read_utils.$(OBJEXT)
unit_test_read_utils_OBJECTS = $(am_unit_test_read_utils_OBJECTS)
unit_test_read_utils_DEPENDENCIES = librock.a
@@ -155,7 +156,18 @@ am__v_at_0 = @
am__v_at_1 =
DEFAULT_INCLUDES = -I.@am__isrc@
depcomp = $(SHELL) $(top_srcdir)/depcomp
-am__depfiles_maybe = depfiles
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/FqAuxBackend.Po \
+ ./$(DEPDIR)/FqBaseBackend.Po ./$(DEPDIR)/FqMainBackend.Po \
+ ./$(DEPDIR)/ROCKparams.Po ./$(DEPDIR)/ReadProcessor.Po \
+ ./$(DEPDIR)/fqreader.Po ./$(DEPDIR)/fqwriter.Po \
+ ./$(DEPDIR)/low_level_utils.Po ./$(DEPDIR)/math_utils.Po \
+ ./$(DEPDIR)/read_utils.Po ./$(DEPDIR)/rock.Po \
+ ./$(DEPDIR)/unit_test_cms.Po ./$(DEPDIR)/unit_test_fqreader.Po \
+ ./$(DEPDIR)/unit_test_fqwriter.Po \
+ ./$(DEPDIR)/unit_test_math_utils.Po \
+ ./$(DEPDIR)/unit_test_read_utils.Po \
+ ./$(DEPDIR)/unit_tests_tools.Po
am__mv = mv -f
CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
@@ -172,11 +184,11 @@ am__v_CXXLD_0 = @echo " CXXLD " $@;
am__v_CXXLD_1 =
SOURCES = $(librock_a_SOURCES) $(rock_SOURCES) \
$(unit_test_cms_SOURCES) $(unit_test_fqreader_SOURCES) \
- $(unit_test_fqwriter_SOURCES) $(unit_test_main_utils_SOURCES) \
+ $(unit_test_fqwriter_SOURCES) $(unit_test_math_utils_SOURCES) \
$(unit_test_read_utils_SOURCES)
DIST_SOURCES = $(librock_a_SOURCES) $(rock_SOURCES) \
$(unit_test_cms_SOURCES) $(unit_test_fqreader_SOURCES) \
- $(unit_test_fqwriter_SOURCES) $(unit_test_main_utils_SOURCES) \
+ $(unit_test_fqwriter_SOURCES) $(unit_test_math_utils_SOURCES) \
$(unit_test_read_utils_SOURCES)
am__can_run_installinfo = \
case $$AM_UPDATE_INFO_DIR in \
@@ -317,11 +329,11 @@ unit_test_cms_SOURCES = unit_test_cms.cpp
unit_test_cms_LDADD = librock.a
unit_test_fqwriter_SOURCES = unit_test_fqwriter.cpp
unit_test_fqwriter_LDADD = librock.a
-unit_test_main_utils_SOURCES = unit_test_main_utils.cpp
-unit_test_main_utils_LDADD = librock.a
+unit_test_math_utils_SOURCES = unit_test_math_utils.cpp
+unit_test_math_utils_LDADD = librock.a
librock_a_SOURCES = $(SRC)
-SRC = fqreader.cpp FqBaseBackend.cpp FqAuxBackend.cpp FqMainBackend.cpp read_utils.cpp ReadProcessor.cpp fqwriter.cpp main_utils.cpp ROCKparams.cpp unit_tests_tools.cpp
-HDR = srp.h fqreader.h FqConstants.h FqBaseBackend.h FqAuxBackend.h FqMainBackend.h ReadProcessor.h rock_commons.h read_utils.h CountMinSketch.hpp Filter.hpp fqwriter.h main_utils.h ROCKparams.h unit_tests_tools.h
+SRC = math_utils.cpp low_level_utils.cpp fqreader.cpp FqBaseBackend.cpp FqAuxBackend.cpp FqMainBackend.cpp read_utils.cpp ReadProcessor.cpp fqwriter.cpp ROCKparams.cpp unit_tests_tools.cpp
+HDR = math_utils.h low_level_utils.h srp.h fqreader.h FqConstants.h FqBaseBackend.h FqAuxBackend.h FqMainBackend.h ReadProcessor.h rock_commons.h read_utils.h CountMinSketch.hpp Filter.hpp fqwriter.h ROCKparams.h unit_tests_tools.h
all: all-am
.SUFFIXES:
@@ -343,8 +355,8 @@ Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
*config.status*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
*) \
- echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
- cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
esac;
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
@@ -355,14 +367,6 @@ $(top_srcdir)/configure: $(am__configure_deps)
$(ACLOCAL_M4): $(am__aclocal_m4_deps)
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
$(am__aclocal_m4_deps):
-
-clean-noinstLIBRARIES:
- -test -z "$(noinst_LIBRARIES)" || rm -f $(noinst_LIBRARIES)
-
-librock.a: $(librock_a_OBJECTS) $(librock_a_DEPENDENCIES) $(EXTRA_librock_a_DEPENDENCIES)
- $(AM_V_at)-rm -f librock.a
- $(AM_V_AR)$(librock_a_AR) librock.a $(librock_a_OBJECTS) $(librock_a_LIBADD)
- $(AM_V_at)$(RANLIB) librock.a
install-binPROGRAMS: $(bin_PROGRAMS)
@$(NORMAL_INSTALL)
@list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \
@@ -409,6 +413,14 @@ clean-binPROGRAMS:
clean-noinstPROGRAMS:
-test -z "$(noinst_PROGRAMS)" || rm -f $(noinst_PROGRAMS)
+clean-noinstLIBRARIES:
+ -test -z "$(noinst_LIBRARIES)" || rm -f $(noinst_LIBRARIES)
+
+librock.a: $(librock_a_OBJECTS) $(librock_a_DEPENDENCIES) $(EXTRA_librock_a_DEPENDENCIES)
+ $(AM_V_at)-rm -f librock.a
+ $(AM_V_AR)$(librock_a_AR) librock.a $(librock_a_OBJECTS) $(librock_a_LIBADD)
+ $(AM_V_at)$(RANLIB) librock.a
+
rock$(EXEEXT): $(rock_OBJECTS) $(rock_DEPENDENCIES) $(EXTRA_rock_DEPENDENCIES)
@rm -f rock$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(rock_OBJECTS) $(rock_LDADD) $(LIBS)
@@ -425,9 +437,9 @@ unit_test_fqwriter$(EXEEXT): $(unit_test_fqwriter_OBJECTS) $(unit_test_fqwriter_
@rm -f unit_test_fqwriter$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(unit_test_fqwriter_OBJECTS) $(unit_test_fqwriter_LDADD) $(LIBS)
-unit_test_main_utils$(EXEEXT): $(unit_test_main_utils_OBJECTS) $(unit_test_main_utils_DEPENDENCIES) $(EXTRA_unit_test_main_utils_DEPENDENCIES)
- @rm -f unit_test_main_utils$(EXEEXT)
- $(AM_V_CXXLD)$(CXXLINK) $(unit_test_main_utils_OBJECTS) $(unit_test_main_utils_LDADD) $(LIBS)
+unit_test_math_utils$(EXEEXT): $(unit_test_math_utils_OBJECTS) $(unit_test_math_utils_DEPENDENCIES) $(EXTRA_unit_test_math_utils_DEPENDENCIES)
+ @rm -f unit_test_math_utils$(EXEEXT)
+ $(AM_V_CXXLD)$(CXXLINK) $(unit_test_math_utils_OBJECTS) $(unit_test_math_utils_LDADD) $(LIBS)
unit_test_read_utils$(EXEEXT): $(unit_test_read_utils_OBJECTS) $(unit_test_read_utils_DEPENDENCIES) $(EXTRA_unit_test_read_utils_DEPENDENCIES)
@rm -f unit_test_read_utils$(EXEEXT)
@@ -439,22 +451,29 @@ mostlyclean-compile:
distclean-compile:
-rm -f *.tab.c
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FqAuxBackend.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FqBaseBackend.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FqMainBackend.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ROCKparams.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ReadProcessor.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fqreader.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fqwriter.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main_utils.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/read_utils.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rock.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_cms.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_fqreader.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_fqwriter.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_main_utils.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_read_utils.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_tests_tools.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FqAuxBackend.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FqBaseBackend.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FqMainBackend.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ROCKparams.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ReadProcessor.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fqreader.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fqwriter.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/low_level_utils.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/math_utils.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/read_utils.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rock.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_cms.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_fqreader.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_fqwriter.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_math_utils.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test_read_utils.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_tests_tools.Po@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+ @$(MKDIR_P) $(@D)
+ @echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
.cpp.o:
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
@@ -522,7 +541,10 @@ cscopelist-am: $(am__tagged_files)
distclean-tags:
-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
-distdir: $(DISTFILES)
+distdir: $(BUILT_SOURCES)
+ $(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
list='$(DISTFILES)'; \
@@ -554,7 +576,7 @@ distdir: $(DISTFILES)
done
check-am: all-am
check: check-am
-all-am: Makefile $(LIBRARIES) $(PROGRAMS) $(HEADERS)
+all-am: Makefile $(PROGRAMS) $(LIBRARIES) $(HEADERS)
installdirs:
for dir in "$(DESTDIR)$(bindir)"; do \
test -z "$$dir" || $(MKDIR_P) "$$dir"; \
@@ -595,7 +617,23 @@ clean-am: clean-binPROGRAMS clean-generic clean-noinstLIBRARIES \
clean-noinstPROGRAMS mostlyclean-am
distclean: distclean-am
- -rm -rf ./$(DEPDIR)
+ -rm -f ./$(DEPDIR)/FqAuxBackend.Po
+ -rm -f ./$(DEPDIR)/FqBaseBackend.Po
+ -rm -f ./$(DEPDIR)/FqMainBackend.Po
+ -rm -f ./$(DEPDIR)/ROCKparams.Po
+ -rm -f ./$(DEPDIR)/ReadProcessor.Po
+ -rm -f ./$(DEPDIR)/fqreader.Po
+ -rm -f ./$(DEPDIR)/fqwriter.Po
+ -rm -f ./$(DEPDIR)/low_level_utils.Po
+ -rm -f ./$(DEPDIR)/math_utils.Po
+ -rm -f ./$(DEPDIR)/read_utils.Po
+ -rm -f ./$(DEPDIR)/rock.Po
+ -rm -f ./$(DEPDIR)/unit_test_cms.Po
+ -rm -f ./$(DEPDIR)/unit_test_fqreader.Po
+ -rm -f ./$(DEPDIR)/unit_test_fqwriter.Po
+ -rm -f ./$(DEPDIR)/unit_test_math_utils.Po
+ -rm -f ./$(DEPDIR)/unit_test_read_utils.Po
+ -rm -f ./$(DEPDIR)/unit_tests_tools.Po
-rm -f Makefile
distclean-am: clean-am distclean-compile distclean-generic \
distclean-tags
@@ -641,7 +679,23 @@ install-ps-am:
installcheck-am:
maintainer-clean: maintainer-clean-am
- -rm -rf ./$(DEPDIR)
+ -rm -f ./$(DEPDIR)/FqAuxBackend.Po
+ -rm -f ./$(DEPDIR)/FqBaseBackend.Po
+ -rm -f ./$(DEPDIR)/FqMainBackend.Po
+ -rm -f ./$(DEPDIR)/ROCKparams.Po
+ -rm -f ./$(DEPDIR)/ReadProcessor.Po
+ -rm -f ./$(DEPDIR)/fqreader.Po
+ -rm -f ./$(DEPDIR)/fqwriter.Po
+ -rm -f ./$(DEPDIR)/low_level_utils.Po
+ -rm -f ./$(DEPDIR)/math_utils.Po
+ -rm -f ./$(DEPDIR)/read_utils.Po
+ -rm -f ./$(DEPDIR)/rock.Po
+ -rm -f ./$(DEPDIR)/unit_test_cms.Po
+ -rm -f ./$(DEPDIR)/unit_test_fqreader.Po
+ -rm -f ./$(DEPDIR)/unit_test_fqwriter.Po
+ -rm -f ./$(DEPDIR)/unit_test_math_utils.Po
+ -rm -f ./$(DEPDIR)/unit_test_read_utils.Po
+ -rm -f ./$(DEPDIR)/unit_tests_tools.Po
-rm -f Makefile
maintainer-clean-am: distclean-am maintainer-clean-generic
@@ -661,7 +715,7 @@ uninstall-am: uninstall-binPROGRAMS
.MAKE: install-am install-strip
-.PHONY: CTAGS GTAGS TAGS all all-am check check-am clean \
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
clean-binPROGRAMS clean-generic clean-noinstLIBRARIES \
clean-noinstPROGRAMS cscopelist-am ctags ctags-am distclean \
distclean-compile distclean-generic distclean-tags distdir dvi \
diff --git a/src/ROCKparams.cpp b/src/ROCKparams.cpp
old mode 100644
new mode 100755
index 65f5c8db998c48d75049314240c65cbaf9a035e5..86547ee31422641eb54e1eb46816e8b2ea31e991
--- a/src/ROCKparams.cpp
+++ b/src/ROCKparams.cpp
@@ -12,24 +12,63 @@
#include <vector>
#include <cmath>
#include "FqConstants.h"
+#include "math_utils.h"
#include "ROCKparams.h"
using namespace std;
const int ROCKparams::output_ext;
const int ROCKparams::undef_ext;
+
+/*
void ROCKparams::computeLambda() {
unsigned long tmp=parms.filter_size;
tmp*=1073741824; // this is in fact 1024*1024*1024.
- parms.lambda=tmp/INT_MAX;
+ parms.lambda=tmp/UINT_MAX;
if (parms.kappa>get_mask<unsigned char>::value) parms.lambda=parms.lambda/sizeof(unsigned short);
-}
-
+}*/
+/*
int ROCKparams::getfilterPEMode() {
return parms.filter_PE_separately;
+}*/
+
+void usage(int status) {
+cout<<"ROCK v@@@ Copyright (C) 2016-2021 Institut Pasteur"<<endl;
+cout<<endl;
+cout<<"Reducing Over-Covering K-mers within FASTQ file(s)"<<endl;
+cout<<endl;
+cout<<"USAGE: rock [options] [files]"<<endl;
+cout<<endl;
+cout<<"OPTIONS:"<<endl;
+cout<<" -i <file> file containing the name(s) of the input FASTQ file(s) to"<<endl;
+cout<<" process; single-end: one file name per line; paired-end:"<<endl;
+cout<<" two file names per line separated by a comma; up to 15"<<endl;
+cout<<" FASTQ file names can be specified; of note, input file"<<endl;
+cout<<" name(s) can also be specified as program argument(s)"<<endl;
+cout<<" -o <file> file containing the name(s) of the output FASTQ file(s);"<<endl;
+cout<<" FASTQ file name(s) should be structured in the same way as"<<endl;
+cout<<" the file specified in option -i."<<endl;
+cout<<" -k <int> k-mer length (default 25)"<<endl;
+cout<<" -c <int> lower-bound k-mer coverage depth threshold (default: 0)"<<endl;
+cout<<" -C <int> upper-bound k-mer coverage depth threshold (default: 70)"<<endl;
+cout<<" -l <int> number of hashing function(s) (default: 4)"<<endl;
+cout<<" -n <int> expected total number of distinct k-mers within the input"<<endl;
+cout<<" read sequences; not compatible with option -l."<<endl;
+cout<<" -f <float> maximum expected false positive probability when computing"<<endl;
+cout<<" the optimal number of hashing functions from the number of"<<endl;
+cout<<" distinct k-mers specified with option -n (default: 0.05)."<<endl;
+cout<<" -q <int> sets as valid only k-mers made up of nucleotides with"<<endl;
+cout<<" Phred score (+33 offset) above this cutoff (default: 0)"<<endl;
+cout<<" -m <int> minimum number of valid k-mer(s) to consider a read; all"<<endl;
+cout<<" non-considered reads are written into output file(s) with"<<endl;
+cout<<" extension undefined.fastq (default: 1)"<<endl;
+cout<<" -v verbose mode"<<endl;
+cout<<" -h prints this message and exit"<<endl;
+exit(status);
}
+
void ROCKparams::processMainArgs(int optind, const int argc, char ** argv,std::vector<string>& v_input_lines) {
if (optind==argc) return; // no arguments
while (optind<argc) { // otherwise, only arguments expected are names of files to filter.
@@ -40,8 +79,28 @@ void ROCKparams::processMainArgs(int optind, const int argc, char ** argv,std::v
}
+
+/*
+ * Minimize lambda for a given number of k-mers.
+ * We want p<=0.01,
+ * so choose smallest lambda so that ccdf(kappa_prime,nb_k_mer,array_size) exp lambda<=0.01.
+ * Smallest is either low filter either high filter if there is no low filter.
+ */
+int ROCKparams::getBestLambdaForN(const unsigned long& nb_k_mers,const int& smallest_kappa,const unsigned int& m,const float& max_collision_proba) {
+ float lnp=log(max_collision_proba);
+ float ccdfr=ccdf(smallest_kappa,nb_k_mers,m);
+ float lnccdf=log(ccdfr);
+ float tmp=lnp/lnccdf;
+ int min_l=ceil(tmp);
+ if (min_l==0) min_l=1;
+ return min_l;
+}
+
+
+
+
void ROCKparams::optArgConsistency(const string& input_file,const string& output_file,
- const int& g,CMSparams& parms,const int& nb_k_mers,
+ const int& g,CMSparams& parms,const unsigned long& nb_k_mers,
const std::vector<string>& v_input_lines) {
if (input_file.empty() && v_input_lines.empty()) {
std::cout<<"You must provide filename via -i or arguments to indicate ROCK what to filter." <<std::endl;
@@ -59,53 +118,40 @@ void ROCKparams::optArgConsistency(const string& input_file,const string& output
cout<<"-l and -n options are mutually exclusive."<<endl;
usage(EXIT_FAILURE);
}
- float avail_mem=getNodePhysMemory()-defaultGRPMAXSize;
- // cout<<avail_mem;
- if (g>avail_mem) {
- std::cout<<"This machine only has "<<getNodePhysMemory()<<" Gigabytes of RAM. This is not enough to run ROCK with a CMS of size: "<<g<<endl;
- exit(EXIT_FAILURE);
- } else if (g!=0) {
- parms.filter_size=g;
- computeLambda();
- }
if (nb_k_mers) { // user indicated number of k-mers; use it to minimize parms.lambda
- parms.lambda=getBestLambdaForN(nb_k_mers,parms.lambda);
+ int smallest_kappa;
+ if (parms.kappa_prime==0) smallest_kappa=parms.kappa;
+ else smallest_kappa=parms.kappa_prime;
+ parms.lambda=getBestLambdaForN(nb_k_mers,smallest_kappa,Pi_js[0],parms.wanted_max_collision_proba);
}
if (parms.kappa_prime>=parms.kappa) {
cout<<"ERROR lower filter is higher than high filter!"<<endl;
- exit(EXIT_FAILURE);
+ usage(EXIT_FAILURE);
}
if (qual_thres.nucl_score_threshold<0) {
cout<<"ERROR nucleotide score threshold must be positive."<<endl;
- exit(EXIT_FAILURE);
+ usage(EXIT_FAILURE);
}
if (qual_thres.min_correct_k_mers_in_read<1) {
cout<<"minimum number of correct k-mers in read must be a positive integer."<<endl;
- exit(EXIT_FAILURE);
+ usage(EXIT_FAILURE);
}
if (parms.lambda==0) {
- // This happens when the user doesn't specify lambda nor g nor nb_k_mers.
- computeLambda();
- parms.lambda=min(parms.lambda,proposedLambda);
- if (parms.lambda==0) {
- cout<<"Not enough RAM on the machine. Total RAM in GB must be bigger than INT_MAX to allow at least 1 array in the CMS."<<endl;
- exit(EXIT_FAILURE);
- }
+ // This happens when the user doesn't specify lambda nor nb_k_mers.
+ parms.lambda=k_proposed_lamda;
}
#ifdef DEBUG
- cout<<"parms.lambda="<<parms.lambda<<" INT_MAX="<<INT_MAX<<endl;
+ cout<<"parms.lambda="<<parms.lambda<<" UINT_MAX="<<UINT_MAX<<endl;
#endif
- unsigned long cms_size=INT_MAX;
+ cms_size=k_arr_cms_size;
cms_size*=parms.lambda;
- //cout<<"minimum cms size:"<<cms_size<<endl;
if (parms.kappa>get_mask<unsigned char>::value) cms_size=cms_size*sizeof(unsigned short);
- //cout<<"cms_size in B="<<cms_size<<endl;
cms_size=ceil(cms_size/1024.0/1024/1024); // convert to gigabytes.
- //cout<<"cms_size in GB="<<cms_size<<endl;
if (cms_size>getNodePhysMemory()-defaultGRPMAXSize) {
cout<<"Not enough RAM on the machine to run rock with a CMS of size:"<<cms_size<<" GB."<<endl;
- exit(EXIT_FAILURE);
+ cout<<" Maybe you should think of increasing the value for the low filter (-c option)"<<endl;
+ usage(EXIT_FAILURE);
}
}
@@ -268,18 +314,10 @@ void ROCKparams::initFromMainOptsArgs(int argc,char ** argv) {
int i,q,m;
std::vector<string> v_input_lines;
std::vector<string> v_output_lines;
- static int PE_separately=1;
-/*
- static struct option long_options[] =
- {
- {"separate_PE", no_argument, &PE_as_single,0},
- {0,0,0,0}
- };*/
+ // static int PE_separately=1;
+ float proba=k_max_collision_proba;
- //int option_index=0;
-
- // while((i = getopt_long(argc, argv, "i:o:l:k:c:C:g:n:vq:m:",long_options,&option_index)) != -1) {
- while((i = getopt(argc, argv, "i:o:l:k:c:C:g:n:vq:m:p:")) != -1) {
+ while((i = getopt(argc, argv, "i:o:l:k:c:C:n:vhq:m:f:")) != -1) {
switch(i) {
case 0:
break;
@@ -287,6 +325,13 @@ void ROCKparams::initFromMainOptsArgs(int argc,char ** argv) {
input_file=optarg;break;
case 'c':
parms.kappa_prime=atoi(optarg);break;
+ case 'f':
+ proba=atof(optarg);
+ if ((proba<=0.0) or (proba>=1.0)) {
+ cout<<"maximum for collision probability in the CMS must be > 0.0 and <1.0."<<endl;
+ usage(EXIT_FAILURE);
+ }
+ parms.wanted_max_collision_proba=proba;break;
case 'h':
usage(EXIT_SUCCESS); break;
case 'o':
@@ -311,27 +356,15 @@ void ROCKparams::initFromMainOptsArgs(int argc,char ** argv) {
cout<<"Bad value for k. Must choose k so that 0<k<=32."<<endl;
usage(EXIT_FAILURE);
}
+ qual_thres.k=k;
break;
case 'n':
- nb_k_mers=atoi(optarg);break; // number of distinct k-mers
- case 'g':
- // cout<<optarg<<endl;
- g=atoi(optarg);
- break;
+ char * t;
+ //cout<<optarg<<endl;
+ nb_k_mers=strtoul(optarg,&t,10);break; // number of distinct k-mers
case 'v':
verbose_mode=1;
break;
- case 'p':
- if (strlen(optarg)>1) {
- cout<<"value for -p option must be 0, 1 or 2";
- usage(EXIT_FAILURE);
- }
- PE_separately=atoi(optarg);
- if (PE_separately<0 or PE_separately>2) {
- cout<<"value for -p option must be 0, 1 or 2";
- usage(EXIT_FAILURE);
- }
- break;
case 'q':
q=atoi(optarg);
if (q<0) {
@@ -346,22 +379,19 @@ void ROCKparams::initFromMainOptsArgs(int argc,char ** argv) {
cout<<"minimum number of valid k-mer for keeping a read must be an integer >=1"<<endl;
usage(EXIT_FAILURE);
}
-
qual_thres.min_correct_k_mers_in_read=atoi(optarg);
break;
default:
usage(EXIT_FAILURE); break; }
}
- /*if (!PE_as_single && qual_thres.min_correct_k_mers_in_read!=1) {
- cout<<"Incompatible options: when PE are processed independently, minimum number of correct k-mers in each end is 1."<<endl;
- usage(EXIT_FAILURE);
- }*/
- parms.filter_PE_separately=PE_separately;
- //cout<<PE_as_single<<endl;
processMainArgs(optind, argc, argv,v_input_lines);
optArgConsistency(input_file,output_file,g,parms,nb_k_mers,v_input_lines);
if (nb_k_mers) {
- expected_collision_proba=getCollisionProba(nb_k_mers,parms.lambda);
+ int smallest_kappa;
+ if (parms.kappa_prime==0) smallest_kappa=parms.kappa;
+ else smallest_kappa=parms.kappa_prime;
+ //float ROCKparams::getCollisionProba(const int& smallest_kappa,const unsigned long& nb_k_mers,const unsigned int& arr_cms_size,const int& lambda) const
+ expected_collision_proba=getCollisionProba(smallest_kappa,nb_k_mers,Pi_js[0],parms.lambda);
}
if ((v_input_lines.empty() || v_output_lines.empty()) && (loadInOutFileArgs(input_file,output_file,v_input_lines,v_output_lines)==EXIT_FAILURE)) throw EXIT_FAILURE;
if (processInOutFileArgs(v_input_lines,v_output_lines,single_files,v_PE_files,f_id)!=EXIT_SUCCESS) throw EXIT_FAILURE;
diff --git a/src/ROCKparams.h b/src/ROCKparams.h
old mode 100644
new mode 100755
index f726a9f895513849b501bf3afee5e6ecf6e5dc94..a8213843f1ce938b217cd87bd2c1cc5cd0ef39ad
--- a/src/ROCKparams.h
+++ b/src/ROCKparams.h
@@ -14,16 +14,22 @@
#include <vector>
#include <getopt.h>
#include <iostream>
+#include "low_level_utils.h"
#include "FqConstants.h"
+#include "rock_commons.h"
#include "CountMinSketch.hpp"
-#include "main_utils.h"
#define defaultGRPMAXSize 1 //(in gigabytes) Default maximum size of srp structure but of course this size depends of the number of reads we are processing.
// 1 GB is enough to store approx 82 millions of reads (must take into account memory needed for stl containers).
// This constant is used only if you use rock's default behavior (use 90% of the machine's RAM).
// If you specify a lambda, it is possible to use more memory for storing more reads and less memory for the CMS.
-#define proposedLambda 4
+#define k_sep_pair_end ','
+#define k_ext '.'
+#define path_sep '/'
+#define k_proposed_lamda 4
+
+#define k_err_parms_inconsistency 1
@@ -37,6 +43,7 @@ typedef struct {
using namespace std;
+
class ROCKparams{
static const int output_ext=1;
static const int undef_ext=2;
@@ -44,7 +51,7 @@ class ROCKparams{
CMSparams parms;
FasqQualThreshold qual_thres;
int g; // max RAM to use for the CMS if specified by the user.
- int nb_k_mers; // expected number of k-mers in input data if specified by the user.
+ unsigned long nb_k_mers; // expected number of k-mers in input data if specified by the user.
int k; // size of the k-mers
int verbose_mode;
std::string input_file,output_file;
@@ -53,10 +60,11 @@ class ROCKparams{
vector<PE_files> v_PE_files;
int f_id;
unsigned long cms_size;
+ //unsigned long total_machine_RAM;
float expected_collision_proba;
- void computeLambda();
+ //void computeLambda();
void processMainArgs(int optind, int argc, char ** argv,std::vector<string>& v_input_lines);
int loadInOutFileArgs(const std::string& input_file,const std::string& output_file,std::vector<string>& v_input_lines,std::vector<string>& v_output_lines);
int loadFileArgs(const std::string& afile,std::vector<string>& v_lines);
@@ -71,41 +79,49 @@ class ROCKparams{
* Also set some CMS parameters depending on value of other options.
*/
void optArgConsistency(const string& input_file,const string& output_file,
- const int& g,CMSparams& parms,const int& nb_k_mers,
+ const int& g,CMSparams& parms,const unsigned long& nb_k_mers,
const std::vector<string>& v_input_lines);
+ int getBestLambdaForN(const unsigned long& nb_k_mers,const int& smallest_kappa,const unsigned int& m,const float& max_collision_proba);
+
friend void test_processIOFileArgs();
+ friend void test_getBestLambdaForN();
+
public:
FasqQualThreshold getQualThreshold();
- int getfilterPEMode();
+ //int getfilterPEMode();
ROCKparams() {
f_id=0; // to generate id of input/output fastq files. +1=total number of files.
g=0;
- k=30;
+ k=25;
nb_k_mers=0;
parms.kappa=70;
- parms.kappa_prime=5;
+ parms.kappa_prime=0;
parms.lambda=0;
- parms.filter_PE_separately=1;
+ parms.filter_PE_separately=0;
+ parms.wanted_max_collision_proba=k_max_collision_proba;
qual_thres.min_correct_k_mers_in_read=1;
- qual_thres.nucl_score_threshold=10+k_phred_32;
+ qual_thres.nucl_score_threshold=k_phred_32;
qual_thres.k=k;
verbose_mode=0;
cms_size=0;
- expected_collision_proba=0.0;
+ expected_collision_proba=0.0; //! collision probability that is computed at the beginning of ROCK from the expected number of distinct k-mers provided by the user.
//min_correct_k_mers_in_read=0;
- parms.filter_size=getNodePhysMemory()/100.0*90-defaultGRPMAXSize; // Prefer not to use all the machine's memory
- if (parms.filter_size==0) throw EXIT_FAILURE;
+ //total_machine_RAM=getNodePhysMemory();
+ parms.max_filter_size=getNodePhysMemory()/100.0*90-defaultGRPMAXSize; // Prefer not to use all the machine's memory
+ if (parms.max_filter_size==0) throw EXIT_FAILURE;
}
void initFromMainOptsArgs(int argc,char ** argv);
- CMSparams getCMSparams() {
+
+
+ CMSparams getCMSparams() const {
return parms;
}
@@ -118,48 +134,44 @@ public:
return f_id;
}
- std::vector<IO_fq_files> get_single_files() {
+ std::vector<IO_fq_files> get_single_files() const {
return single_files;
}
- vector<PE_files> get_PE_files() {
+ vector<PE_files> get_PE_files() const {
return v_PE_files;
}
- int get_k() {
+ int get_k() const {
return k;
}
- void printVerboseInfo() {
- std::vector<IO_fq_files>::iterator it_s;
- vector<PE_files>::iterator it_pe;
- cout<<"Input files: "<<endl;
- cout<<"single:"<<endl;
- for (it_s=single_files.begin();it_s!=single_files.end();it_s++) {
- cout<<" "<<it_s->in_fq_file<<endl;
- }
- cout<<"pair-end:"<<endl;
- for (it_pe=v_PE_files.begin();it_pe!=v_PE_files.end();it_pe++) {
- cout<<" "<<it_pe->PE1.in_fq_file<<" "<<it_pe->PE2.in_fq_file<<endl;
- }
- cout<<endl;
- cout<<"k="<<k<<endl;
- cout<<"kappa="<<parms.kappa<<endl;
- cout<<"kappa prime="<<parms.kappa_prime;
- cout<<endl;
- cout<<"CMS size="<<cms_size<<" Bytes for "<<parms.lambda<<" arrays."<<endl;
- cout<<endl;
- if (nb_k_mers!=0) cout<<"expected probability of collision was: "<<expected_collision_proba<<endl;
- }
-
- void setFilterSize(unsigned long fsize) {
+ // cms_size is computed when ROCK starts.
+ /*void setFilterSize(unsigned long fsize) {
cms_size=fsize;
+ }*/
+
+ unsigned long getFilterSize() const {
+ /*unsigned long cms_size_in_GB=cms_size/1024;
+ cms_size_in_GB/=1024;
+ cms_size_in_GB/=1024;
+ return cms_size_in_GB;*/
+ return cms_size;
}
+
int is_verbose() {
return verbose_mode;
}
+ unsigned long get_expected_nbKmers() const {
+ return nb_k_mers;
+ }
+
+ float get_expected_collision_proba() const {
+ return expected_collision_proba;
+ }
+
};
diff --git a/src/fqreader.cpp b/src/fqreader.cpp
index d918e68fbfa4b385775db92a8f358168a3a557fe..6a79cac52e9abff170aac790cdb004301f3a5238 100644
--- a/src/fqreader.cpp
+++ b/src/fqreader.cpp
@@ -56,8 +56,12 @@ void processPEFiles(char * fq_1, unsigned char f_id1,char * fq_2, unsigned char
* pointers to Fqbackend objects.
* This function ALLOCATES MEMORY with new.
*/
-void processInputFiles(const std::vector<IO_fq_files>& single_files,const vector<PE_files>& v_PE_files,FqBaseBackend * array_be[],const FasqQualThreshold& a_qual_thres, srp* io_sr,const int& PE_process_mode) {
- unsigned char f_id=1;
+unsigned long processInputFiles(const std::vector<IO_fq_files>& single_files,\
+ const vector<PE_files>& v_PE_files,FqBaseBackend * array_be[],\
+ const FasqQualThreshold& a_qual_thres, srp* io_sr,\
+ const int& PE_process_mode) {
+ unsigned long tot_nb_kmers=0;
+ unsigned char f_id=1;
FqBaseBackend::setQualThreshold(a_qual_thres);
FqMainBackend::setTreatPEMode(PE_process_mode);
std::vector<IO_fq_files>::const_iterator it_single;
@@ -68,6 +72,7 @@ void processInputFiles(const std::vector<IO_fq_files>& single_files,const vector
be_fq->setOutputFile((char *) it_single->out_fq_file.c_str());
array_be[f_id-1]=be_fq;
f_id+=1;
+ tot_nb_kmers+=be_fq->getCntKMers();
}
vector<PE_files>::const_iterator it_pe;
for (it_pe=v_PE_files.begin();it_pe!=v_PE_files.end();it_pe++) {
@@ -87,7 +92,9 @@ void processInputFiles(const std::vector<IO_fq_files>& single_files,const vector
array_be[f_id1-1]=be_fq1;
array_be[f_id-1]=be_fq2;
f_id++;
+ tot_nb_kmers+=be_fq1->getCntKMers();
}
+ return tot_nb_kmers;
}
diff --git a/src/fqreader.h b/src/fqreader.h
index 17ce828ee3a317be0c6688c298d10834e5e39c44..79badaae6b25eed118d2716092c02e3a62486015 100644
--- a/src/fqreader.h
+++ b/src/fqreader.h
@@ -19,5 +19,5 @@ using namespace std;
void processSingleFile(char *, unsigned char, srp*);
void processPEFiles(char * fq_1, unsigned char f_id1,char * gq_2, unsigned char f_id2,srp *io_sr,char * fq_1_test_undef=NULL,char * fq_2_test_undef=NULL,size_t test_bufsize=0);
-void processInputFiles(const std::vector<IO_fq_files>& ,const vector<PE_files>& ,FqBaseBackend * [], const FasqQualThreshold&,srp*,const int& PE_process_mode);
+unsigned long processInputFiles(const std::vector<IO_fq_files>& ,const vector<PE_files>& ,FqBaseBackend * [], const FasqQualThreshold&,srp*,const int& PE_process_mode);
#endif
diff --git a/src/fqwriter.cpp b/src/fqwriter.cpp
index 6ca2278364de6327eb68a5dc15e213a458802ced..eabb289fe52ac27db5231b94a4c9ca0266b8b89f 100644
--- a/src/fqwriter.cpp
+++ b/src/fqwriter.cpp
@@ -19,7 +19,7 @@ using namespace std;
// TODO this looks a little bit like getDnaStr in read_utils.cpp. Some refactoring could be useful. Think about
// classes to encapsulate that without decreasing perfs.
-void writeFilteredFastq(FqBaseBackend* map_id_backend[],int nb_be, const srp& io_sr){
+void writeFilteredFastq(FqBaseBackend* map_id_backend[],int nb_be,T_read_counters& rc, const srp& io_sr){
srp::const_reverse_iterator it;
i_dim::const_iterator it_offs;
k_dim::const_iterator it_struct;
@@ -30,6 +30,8 @@ void writeFilteredFastq(FqBaseBackend* map_id_backend[],int nb_be, const srp& io
// 1rst step, open files before fetching/writing reads from/to them
int i;
+ rc.nb_input_reads=0;
+ rc.nb_output_reads=0;
for (i=0;i<nb_be;i++){
map_id_backend[i]->openInputFile();
map_id_backend[i]->openOutputFile();
@@ -39,7 +41,9 @@ void writeFilteredFastq(FqBaseBackend* map_id_backend[],int nb_be, const srp& io
for (it_offs=it->second.begin();it_offs!=it->second.end();it_offs++) {
unsigned long j=it_offs->first;
for (it_struct=it_offs->second.begin();it_struct!=it_offs->second.end();it_struct++) {
+ rc.nb_input_reads++;
if (it_struct->fileid) {
+ rc.nb_output_reads++;
fid_stored=it_struct->fileid;
f_id1=fid_stored >>4;
f_id2=fid_stored &mask_fid;
diff --git a/src/fqwriter.h b/src/fqwriter.h
index 97d90b44bc98e75818d57d7052b588a492de7cc4..59514b63dc5854c8fbf07f81262825372cfb8700 100644
--- a/src/fqwriter.h
+++ b/src/fqwriter.h
@@ -8,7 +8,7 @@
#ifndef FQWRITER_H_
#define FQWRITER_H_
-void writeFilteredFastq(FqBaseBackend* map_id_backend[],int nb_be, const srp& io_sr);
+void writeFilteredFastq(FqBaseBackend* map_id_backend[],int nb_be, T_read_counters& rc, const srp& io_sr);
diff --git a/src/low_level_utils.cpp b/src/low_level_utils.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..dc11bec6cdeaf169c155382a9ed24a53c37035da
--- /dev/null
+++ b/src/low_level_utils.cpp
@@ -0,0 +1,74 @@
+/*
+ * low_level_utils.cpp
+ *
+ * Created on: 7 mai 2021
+ * Author: vlegrand
+ */
+
+#include <string>
+#include <iostream>
+#include <sys/stat.h>
+#include <sys/sysctl.h>
+#if !defined(_SC_PHYS_PAGES) and !__APPLE__
+ #include <sys/sysinfo.h>
+#endif
+#include "low_level_utils.h"
+
+using namespace std;
+
+/*
+ * Given a filename (including full path), determine if parent directory exists.
+ * If is doesn't, display error message and exit.
+ */
+std::string checkDirExists(const string o_file) {
+ string parent_dir=".";
+ std::size_t o_found = o_file.find_last_of("/");
+ if (o_found!=string::npos) {
+ parent_dir=o_file.substr(0,o_found);
+ // cout<<parent_dir<<endl;
+ struct stat info;
+ if (stat(parent_dir.c_str(),&info)!=0) {
+ cout<<"parent directory for output files: "<<parent_dir<<" doesn't exist."<<endl;
+ exit(EXIT_FAILURE);
+ }
+ }
+ return parent_dir;
+}
+
+/*
+ * Use this function to get the amount of RAM in GB on the machine (used or not).
+ * Aim is to make sure that ROCK can run on the current machine with the requirements that the user gave.
+ */
+unsigned long getNodePhysMemory() {
+ uint64_t mem = 0;
+
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+ mem = sysconf(_SC_PHYS_PAGES);
+ mem *= sysconf(_SC_PAGESIZE);
+ mem /= 1 * 1024 * 1024 * 1024;
+#elif __APPLE__
+ size_t len = sizeof(mem);
+ int ret=sysctlbyname("hw.memsize", (void*) &mem, &len, NULL,0);
+ if (ret==-1) {
+ cout<<"Couldn't determine how much RAM is usable on your system."<<errno<<endl;
+ mem=0;
+ }
+ mem /= 1 * 1024 * 1024 * 1024;
+#elif !defined(_SC_PHYS_PAGES) and !__APPLE__
+ struct sysinfo si;
+ int ret=sysinfo(&si);
+ if (ret==-1) {
+ cout<<"Couldn't determine how much RAM is usable on your system."<<errno<<endl;
+ mem=0;
+ }
+ mem=si.totalram;
+ mem /= 1 * 1024 * 1024 * 1024;
+ //cout<<"total RAM="<<mem<<endl;
+//cout<<"ROCK has not been tested on this operating system, please contact author."<<endl;
+#endif
+
+ return (unsigned long)mem;
+}
+
+
+
diff --git a/src/low_level_utils.h b/src/low_level_utils.h
new file mode 100644
index 0000000000000000000000000000000000000000..081c045e964d8061b795ab588091097fccc10378
--- /dev/null
+++ b/src/low_level_utils.h
@@ -0,0 +1,22 @@
+/*
+ * low_level_utils.h
+ *
+ * Created on: 7 mai 2021
+ * Author: vlegrand
+ * Low level utilities
+ */
+
+#ifndef SRC_LOW_LEVEL_UTILS_H_
+#define SRC_LOW_LEVEL_UTILS_H_
+
+#include <string>
+
+using namespace std;
+
+string checkDirExists(const string o_file);
+unsigned long getNodePhysMemory();
+
+
+
+
+#endif /* SRC_LOW_LEVEL_UTILS_H_ */
diff --git a/src/main_utils.cpp b/src/main_utils.cpp
deleted file mode 100644
index d5da68ad0197fba66b79332d1f9f1fa44c7d087a..0000000000000000000000000000000000000000
--- a/src/main_utils.cpp
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * main_utils.cpp
- *
- * Created on: May 9, 2016
- * Author: vlegrand
- */
-#include <stdint.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/sysctl.h>
-#include <sys/stat.h>
-#include <limits.h>
-#include <string.h>
-#include <iostream>
-#include <fstream>
-#include <cmath>
-#include "main_utils.h"
-
-using namespace std;
-
-/*
- * Use this function to get the amount of RAM in GB on the machine (used or not).
- * Aim is to make sure that ROCK can run on the current machine with the requirements that the user gave.
- */
-unsigned long getNodePhysMemory() {
- uint64_t mem = 0;
-
-#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
- mem = sysconf(_SC_PHYS_PAGES);
- mem *= sysconf(_SC_PAGESIZE);
- mem /= 1 * 1024 * 1024 * 1024;
-#elif __APPLE__
- size_t len = sizeof(mem);
- int ret=sysctlbyname("hw.memsize", (void*) &mem, &len, NULL,0);
- if (ret==-1) {
- cout<<"Couldn't determine how much RAM is usable on your system."<<errno<<endl;
- mem=0;
- }
- mem /= 1 * 1024 * 1024 * 1024;
-#else
- cout<<"ROCK has not been tested on this operating system, please contact author."<<endl;
-#endif
-
- return (unsigned long)mem;
-}
-
-
-/*
- * Given a filename (including full path), determine if parent directory exists.
- * If is doesn't, display error message and exit.
- */
-std::string checkDirExists(const string o_file) {
- string parent_dir=".";
- std::size_t o_found = o_file.find_last_of("/");
- if (o_found!=string::npos) {
- parent_dir=o_file.substr(0,o_found);
- // cout<<parent_dir<<endl;
- struct stat info;
- if (stat(parent_dir.c_str(),&info)!=0) {
- cout<<"parent directory for output files: "<<parent_dir<<" doesn't exist."<<endl;
- exit(EXIT_FAILURE);
- }
- }
- return parent_dir;
-}
-
-
-/*
- * Minimize lambda for a given number of k-mers.
- * We want p<=0.01,
- * so choose smallest lambda so that [1-(1-1/m)exp n] exp lambda<=0.01
- */
-int getBestLambdaForN(const unsigned long& nb_k_mers,int lambda_max) {
- int lambda=2;
- int min_lambda=lambda;
- if (lambda_max==0) { // no upper bound specified by the user via the -g option
- lambda_max=500;
- }
- long double tmp=1.0/INT_MAX;
- tmp=1.0-tmp;
- tmp=pow(tmp,nb_k_mers);
- tmp=1.0-tmp;
- while (lambda<=lambda_max) {
- min_lambda=lambda;
- long double p=pow(tmp,lambda);
- if (p<=0.01) break;
- lambda+=1;
- }
- return min_lambda;
-}
-
-/*
- * Compute collision probability
- */
-float getCollisionProba(const unsigned long& nb_k_mers,const int& lambda) {
- long double tmp=1.0/INT_MAX;
- tmp=1.0-tmp;
- tmp=pow(tmp,nb_k_mers);
- tmp=1.0-tmp;
- long double p=pow(tmp,lambda);
- if (p<0.0001) p=0;
- // rounding
- p=p*1000+0.5;
- return floor(p)/1000;
- //return p;
-}
-
-void usage(int status) {
- cout<<"usage: rock [options] [args]"<<endl<<endl;
- cout<<" -i <file> .... file name containing the list of the FASTQ files to process."<<endl;
- cout<<" -o <file> .... file name containing the list of the output FASTQ file names."<<endl;
- cout<<" -h .... Print this message and exit."<<endl;
- cout<<" -k <int> .... k-mer size. (default 30)."<<endl;
- cout<<" -c <int> .... lower coverage threshold (default: 5)."<<endl;
- cout<<" -C <int> .... upper coverage threshold (default: 70; max: 65535)."<<endl;
- cout<<" -l <int> .... size of the count min sketch (default: at most 4, depending on the available RAM)"<<endl;
- cout<<" -n <int> .... (expected) number of distinct k-mers within the processed reads."<<endl;
- cout<<" -g <int> .... maximum RAM to use (in Gb) for the CMS."<<endl;
- cout<<" -p <int> .... treat PE as single, separately with strict -c filter, separately with lenient -c fileter. Default is 1."<<endl;
- cout<<" -q <int> .... minimum quality threshold for nucleotides. Default is 10."<<endl;
- cout<<" -m <int> .... minimum of valid k-mer threshold for reads. Default is 1."<<endl;
- cout<<" -v .... verbose"<<endl;
- exit(status); }
-
diff --git a/src/main_utils.h b/src/main_utils.h
deleted file mode 100644
index 080e584c9a2089639c762a9480f4376682c31563..0000000000000000000000000000000000000000
--- a/src/main_utils.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * main_utils.h
- *
- * Created on: May 9, 2016
- * Author: vlegrand
- */
-
-#ifndef MAIN_UTILS_H_
-#define MAIN_UTILS_H_
-
-#define k_max_input_files 15
-#define k_sep_pair_end ','
-#define k_ext '.'
-#define path_sep '/'
-
-#include <string>
-#include <vector>
-#include "rock_commons.h"
-
-
-
-
-unsigned long getNodePhysMemory();
-std::string checkDirExists(const std::string o_file);
-/*int loadInOutFileArgs(const std::string& input_file,const std::string& output_file,std::vector<std::string>& v_input_lines,std::vector<std::string>& v_output_lines);
-int processInOutFileArgs(const std::vector<std::string>& v_input_lines,std::vector<std::string>& v_output_lines,std::vector<IO_fq_files>& single_files,std::vector<PE_files>& v_PE_files,int& f_id);*/
-int getBestLambdaForN(const unsigned long& nb_k_mers,int lambda_max);
-float getCollisionProba(const unsigned long& nb_k_mers,const int& lambda);
-void usage(int status);
-
-
-#endif /* MAIN_UTILS_H_ */
diff --git a/src/math_utils.cpp b/src/math_utils.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6a952f4381123e1ab47dda0b71a966092da7138e
--- /dev/null
+++ b/src/math_utils.cpp
@@ -0,0 +1,74 @@
+/*
+ * math_utils.cpp
+ *
+ * Created on: 7 mai 2021
+ * Author: vlegrand
+ */
+#include <cmath>
+
+
+/*
+ * logarithm of the gamma function;
+ * from here: https://www.cec.uchile.cl/cinetica/pcordero/MC_libros/NumericalRecipesinC.pdf
+ */
+double gammln(const unsigned long& xx) {
+ double x,y,tmp,ser;
+
+ static double cof[6]={76.18009172947146,-86.50532032941677,\
+ 24.01409824083091,-1.231739572450155,\
+ 0.001208650973866179,-0.000005395239384953};
+ int j;
+ y=x=xx;
+ tmp=x+5.5;
+ tmp -=(x+0.5)*log(tmp);
+ ser=1.000000000190015;
+ for (j=0;j<=5;j++) ser+=cof[j]/++y;
+ return -tmp+log(2.5066282746310005*ser/x);
+}
+
+
+/*
+ * here x is the smallest value of the counter for a given k-mer in the CMS.
+ * m is the size of a CMS array (AC: in fact, we use the biggest prime number as an approximation of the array's size).
+ */
+double pmf(const float&x,const unsigned long& nb_k_mers,const unsigned int& m) {
+ double res;
+ double lnn=log(nb_k_mers);
+ double lnm=log(m);
+ double diff=lnn-lnm;
+ double p=x*diff;
+ diff=double(nb_k_mers)/m;
+ //double truc=exp(diff);
+ double tmp=p-diff;
+ double g=gammln(x+1);
+ tmp -=g;
+ res=exp(tmp);
+ return res;
+}
+
+double ccdf(const int& smallest_kappa,const unsigned long& nb_k_mers,const unsigned int& m) {
+ double res;
+ double s=0.0;
+ for (int i=0;i<=smallest_kappa;i++) {
+ s+=pmf(i,nb_k_mers,m);
+ }
+ res=1-s;
+ if (res<0) res=0.00000001; // It happens because pmf if an approximation of the probability distribution.
+ // So it happens that ccdf is negative but very close to 0... As we need a non null value for the next calculation,
+ // return a very small constant.
+ if (res>=0.999) res=0.999; // Need to do that because we compule ln(ccdf) to determine lamda and we need it to be non null otherwise we'd get a division by 0.
+ return res;
+}
+
+
+float getCollisionProba(const int& smallest_kappa,const unsigned long& nb_k_mers,const unsigned int& arr_cms_size,const int& lambda) {
+ float ccfdr=ccdf(smallest_kappa,nb_k_mers,arr_cms_size);
+ float fpp=pow(ccfdr,lambda);
+ return fpp;
+}
+
+
+
+
+
+
diff --git a/src/math_utils.h b/src/math_utils.h
new file mode 100644
index 0000000000000000000000000000000000000000..9c9d045c8b67b5b072b57e5192abd38dc2b2754f
--- /dev/null
+++ b/src/math_utils.h
@@ -0,0 +1,20 @@
+/*
+ * math_utils.h
+ *
+ * Created on: 7 mai 2021
+ * Author: vlegrand
+ * Gather here utilities methods for computing the collision probability
+ */
+
+#ifndef SRC_MATH_UTILS_H_
+#define SRC_MATH_UTILS_H_
+
+
+
+double gammln(const unsigned long& xx);
+double pmf(const float&x,const unsigned long& nb_k_mers,const unsigned int& m);
+double ccdf(const int& smallest_kappa,const unsigned long& nb_k_mers,const unsigned int& m);
+float getCollisionProba(const int& smallest_kappa,const unsigned long& nb_k_mers,const unsigned int& arr_cms_size,const int& lambda);
+
+
+#endif /* SRC_MATH_UTILS_H_ */
diff --git a/src/rock b/src/rock
deleted file mode 100755
index 0e813008a14de1d9c6a298f91678bba4a6e6f788..0000000000000000000000000000000000000000
Binary files a/src/rock and /dev/null differ
diff --git a/src/rock.cpp b/src/rock.cpp
old mode 100644
new mode 100755
index 240950011f520897265cd176a800475d60feea0e..5f889c7652240954b1299805352db5553d0dbd55
--- a/src/rock.cpp
+++ b/src/rock.cpp
@@ -13,11 +13,12 @@
#include <string.h>
#include <iostream>
+#include <iomanip>
#include <map>
#include <vector>
#include <unistd.h>
-
+#include "math_utils.h"
#include "CountMinSketch.hpp"
#include "rock_commons.h"
#include "srp.h"
@@ -26,7 +27,6 @@
#include "fqreader.h"
#include "fqwriter.h"
#include "Filter.hpp"
-#include "main_utils.h"
#include "ROCKparams.h"
@@ -45,6 +45,75 @@ void printRUsage() {
}
#endif
+// method used for debug only
+void printFastqQualThreshold(const FasqQualThreshold& q) {
+ cout<<"nucl_score_threshold="<<q.nucl_score_threshold<<endl;
+ cout<<"min_correct_k_mers_in_read="<<q.min_correct_k_mers_in_read<<endl;
+ cout<<"k="<<q.k<<endl;
+}
+
+
+// function used for debugging purpose only
+void printCMSparams(const CMSparams& p) {
+ cout<<"CMSparams content:"<<endl;
+ cout<<"lambda="<<p.lambda<<endl;
+ cout<<"kappa="<<p.kappa<<endl;
+ cout<<"kappa_prime="<<p.kappa_prime<<endl;
+ cout<<"filter_size="<<p.max_filter_size<<endl;
+ cout<<"filter_PE_separately="<<p.filter_PE_separately<<endl;
+}
+
+const void printVerboseInfo(const ROCKparams& Rparms,const Filter& filter,const T_read_counters& rc,\
+ const unsigned long& tot_nb_kmers) {
+ cout<<setw(40)<<left<<"total available RAM (Gb)"<<getNodePhysMemory()<<endl;
+ std::vector<IO_fq_files>::iterator it_s;
+ vector<PE_files>::iterator it_pe;
+ vector<PE_files> v_PE_files=Rparms.get_PE_files();
+ vector<IO_fq_files> single_files=Rparms.get_single_files();
+ for (it_pe=v_PE_files.begin();it_pe!=v_PE_files.end();it_pe++) {
+ cout<<setw(40)<<left<<"PE input files"<<it_pe->PE1.in_fq_file<<" "<<it_pe->PE2.in_fq_file<<endl;
+ }
+ for (it_s=single_files.begin();it_s!=single_files.end();it_s++) {
+ cout<<setw(40)<<left<<"SE input file(s)"<<it_s->in_fq_file<<endl;
+ }
+ CMSparams parms=Rparms.getCMSparams();
+ cout<<setw(40)<<left<<"upper-bound coverage depth"<<parms.kappa<<endl;
+ cout<<setw(40)<<left<<"lower-bound coverage depth"<<parms.kappa_prime<<endl;
+ cout<<setw(40)<<left<<"k-mer length"<<Rparms.get_k()<<endl;
+ cout<<setw(40)<<left<<"expected no. distinct k-mers"<<Rparms.get_expected_nbKmers()<<endl;
+ cout<<setw(40)<<left<<"no. hash. function(s)"<<parms.lambda<<endl;
+
+ cout<<setw(40)<<left<<"expected false positive proba."<<setprecision (6)<<fixed<<Rparms.get_expected_collision_proba()<<" (cov. depth > 1)"<<endl;
+
+ for (int i=0;i<parms.lambda;i++) {
+ string tmp="no. buckets for hash. ";
+ tmp+=to_string(i+1);
+ cout<<setw(40)<<left<<tmp<<Pi_js[i]<<endl;
+ }
+ cout<<setw(40)<<left<<"no. bits per bucket"<<filter.getNbBitsForCounters()<<endl;
+ cout<<setw(40)<<left<<"count-min sketch size (Gb)"<<Rparms.getFilterSize()<<endl;
+ cout<<setw(40)<<left<<"no. input reads"<<rc.nb_input_reads<<endl;
+ cout<<setw(40)<<left<<"no. input k-mers"<<tot_nb_kmers<<endl;
+ unsigned long approx_nbDiffKm=filter.getApproxNbDistinctKMers();
+ std::vector<unsigned long> zeroes=filter.getUnsetBuckets();
+ for (int i=0;i<=parms.lambda-1;i++){
+ string tmp="no. unset buckets for hash. ";
+ tmp+=to_string(i+1);
+ cout<<setw(40)<<left<<tmp<<zeroes[parms.lambda-1-i]<<endl;
+ }
+ cout<<setw(40)<<left<<"approx. no. distinct k-mers"<<approx_nbDiffKm<<endl;
+ int smallest_kappa=parms.kappa;
+ if (parms.kappa_prime>0) smallest_kappa=parms.kappa_prime;
+ float p =getCollisionProba(smallest_kappa,approx_nbDiffKm,Pi_js[0],parms.lambda);
+ cout<<setw(40)<<left<<"approx. false positive proba."<<setprecision (6)<<p<<endl;
+ cout<<setw(40)<<left<<"no. selected reads"<<rc.nb_output_reads<<endl;
+ for (it_pe=v_PE_files.begin();it_pe!=v_PE_files.end();it_pe++) {
+ cout<<setw(40)<<left<<"PE output files"<<it_pe->PE1.out_fq_file<<" "<<it_pe->PE2.out_fq_file<<" "<<it_pe->PE1.undef_fq_file<<" "<<it_pe->PE2.undef_fq_file<<endl;
+ }
+ for (it_s=single_files.begin();it_s!=single_files.end();it_s++) {
+ cout<<setw(40)<<left<<"SE output files"<<it_s->out_fq_file<<" "<<it_s->undef_fq_file<<endl;
+ }
+}
@@ -54,11 +123,14 @@ int main(int argc,char * argv[]) {
printRUsage();
#endif
srp sr;
+ T_read_counters rc;
ROCKparams main_parms;
main_parms.initFromMainOptsArgs(argc,argv);
int f_id=main_parms.get_f_id();
CMSparams parms=main_parms.getCMSparams();
+ //printCMSparams(parms);
FasqQualThreshold qual_thres=main_parms.getQualThresholds();
+ //printFastqQualThreshold(qual_thres);
std::vector<IO_fq_files> single_files=main_parms.get_single_files();
vector<PE_files> v_PE_files=main_parms.get_PE_files();
@@ -69,7 +141,7 @@ int main(int argc,char * argv[]) {
cout<<"processed input args; going to start reading fastq files"<<endl;
printRUsage();
#endif
- processInputFiles(single_files,v_PE_files,map_id_backend,main_parms.getQualThreshold(),&sr,main_parms.getfilterPEMode());
+ unsigned long tot_nb_kmers=processInputFiles(single_files,v_PE_files,map_id_backend,main_parms.getQualThreshold(),&sr,0); // Removing -p option, set default mode to 0 (PE processed as single).
#ifdef BENCHMARK
cout<<"finished loading fastq file into sr structure"<<endl;
cout<<"size of srp structure="<<sizeof(sr)<<endl;
@@ -95,7 +167,7 @@ int main(int argc,char * argv[]) {
cout<<"Now going to remove read with cov<kappa_prime"<<endl;
#endif
- main_parms.setFilterSize(the_filter.getSize());
+ // main_parms.setFilterSize(the_filter.getSize()); // no more used max_rss is not precise enough
// Now, remove reads that are beneath kappa_prime
if (parms.kappa_prime) the_filter.lowFilterCMS(map_id_backend,f_id,k,&sr);
@@ -105,7 +177,7 @@ int main(int argc,char * argv[]) {
printRUsage();
cout<<"Should now write output files"<<endl;
#endif
- writeFilteredFastq(map_id_backend,f_id,sr);
+ writeFilteredFastq(map_id_backend,f_id,rc,sr);
#ifdef BENCHMARK
cout<<"finished writing filtered reads"<<endl;
printRUsage();
@@ -121,12 +193,14 @@ int main(int argc,char * argv[]) {
cout<<"going to compute collision probability"<<endl;
printRUsage();
#endif
-
- float p =getCollisionProba(approx_nb_k_mers,parms.lambda);
+ /*
+ int smallest_kappa=parms.kappa;
+ if (parms.kappa_prime>0) smallest_kappa=parms.kappa_prime;
+ float p =getCollisionProba(smallest_kappa,approx_nb_k_mers,Pi_js[0],parms.lambda);
cout<<"estimated probability of collision:"<<p<<endl;
cout<<"estimated number of distinct k-mers:"<<approx_nb_k_mers<<endl;
-
- if (main_parms.is_verbose()) main_parms.printVerboseInfo();
+ */
+ if (main_parms.is_verbose()) printVerboseInfo(main_parms,the_filter,rc,tot_nb_kmers);
#ifdef BENCHMARK
cout<<"finished,going to exit "<<endl;
diff --git a/src/rock_commons.h b/src/rock_commons.h
index 72ecb5aad6d5d63b006a3ab73dc3e7f6a7e1de2e..09533f6035b9aa29cd49f00297ca2a681a183959 100644
--- a/src/rock_commons.h
+++ b/src/rock_commons.h
@@ -1,5 +1,5 @@
/*
- * rock_types2.h
+ * rock_commons.h
*
* Created on: Jan 20, 2016
* Author: vlegrand
@@ -23,8 +23,24 @@
* (fqreader, read processing, CMS filling, result writing) and to the main program.
*
*/
-
+#define k_max_collision_proba 0.05
#define k_max_input_files 15
+#define k_max_array_in_cms 50 // Theorical maximum of arrays in the count min sketch.
+#define k_arr_cms_size UINT_MAX // size of 1 CMS array
+
+// Store the prime numbers for modulo hashing in this array.
+/* Use the 50 biggest prime numbers < 2^32. They have the propriety that their binary representation 9i)
+ * contains at least 13 zeroes and (ii) at least 11 occurrences of '10'.
+ * This disparity in the binary representation of a primary number guaranties that all bits of a numerator x
+ * will be well affected during the x%p operation.
+*/
+const unsigned int Pi_js[k_max_array_in_cms]={4283781797,4283781461,4283738773,4283738453,4280985941,4280898901,4275393833,4275393173,\
+ 4275382933,4274694821,4273296553,4273285717,4273253029,4273121621,4273034581,4272772181,\
+ 4272771749,4272769621,4272761509,4272728741,4272630421,4272629077,4272608581,4272608533,\
+ 4272599701,4272598181,4272597673,4272597353,4272597349,4272597157,4272596149,4272596117,\
+ 4272593557,4272592549,4272592277,4272592229,4272592213,4272592021,4272589493,4272575849,\
+ 4272575833,4272575819,4272575701,4272575669,4272515669,4272511657,4272511637,4272510629,\
+ 4272510283,4272510101};
typedef struct {
std::string in_fq_file;
@@ -46,6 +62,11 @@ typedef struct {
// indicate at which index the single_or_PE_val array starts containing values for PE2 k-mers.
}T_read_numericValues;
+typedef struct {
+ unsigned int nb_input_reads;
+ unsigned int nb_output_reads;
+}T_read_counters;
+
diff --git a/src/rock_types.h b/src/rock_types.h
deleted file mode 100644
index 06a79f04123888b714feb1b8e2ad006885fadc40..0000000000000000000000000000000000000000
--- a/src/rock_types.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * rock_types2.h
- *
- * Created on: Jan 20, 2016
- * Author: vlegrand
- * Gather here typedefs and structures useful for the main program and "inter-modules" data exchange.
- * I divided rock into 4 "modules":
- * 1- fqreader (parses fastq and fills 3D array of structures representing all reads).
- * 2- read-utils (utility functions for DNA reads processing)
- * 3- CMS (fill and use).
- * 4- fqwriter (writes fq filtered by CMS).
- */
-
-#ifndef ROCK_TYPES_H_
-#define ROCK_TYPES_H_
-
-#include <map>
-#include "FqBaseBackend.h"
-
-/*typedef struct {
- FqBaseBackend& fq_be;
-
-}fq_io;*/
-
-// typedef std::map<unsigned char,FqBaseBackend&> fq_file_map;
-
-#define k_max_input_files 15
-
-#endif /* ROCK_TYPES_H_ */
diff --git a/src/srp_old.h b/src/srp_old.h
deleted file mode 100644
index 0f7c0661ff99651435125ae2a3f8e0e458f095ae..0000000000000000000000000000000000000000
--- a/src/srp_old.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * srp.h
- *
- * Created on: Dec 8, 2015
- * Author: vlegrand
- *
- * SRP stands for score-read position. Indeed, this structure associates a score with a read's position inthe input fq file.
- */
-
-#ifndef SRP_H_
-#define SRP_H_
-
-typedef struct { /* Here store read offset in file whose id is fileid.*/
- char fileid;
- unsigned long read_a1;
-}rpos;
-
-char ** l_files; /* array of filenames; indexed on fileid */
-
-typedef rpos *** srp;
-
-/*
- * Physically, srp is :
- * An array indexed on read total score (Arr1).
- * Arr1 contains pointers to other other arrays (Arr2).
- * Arr2 is indexed on something that is deduced from offset (read position in a file).
- * Arr2 contains pointers to array of rpos structures.
- */
-
-#endif /* SRP_H_ */
diff --git a/src/unit_test_cms.cpp b/src/unit_test_cms.cpp
index b77fbf459dcd99536766559970762461f7c32cc2..4f0bfbef0ca58be26183ed766b2efa52926b1103 100644
--- a/src/unit_test_cms.cpp
+++ b/src/unit_test_cms.cpp
@@ -11,8 +11,103 @@
#include <climits>
using namespace std;
+#include "ROCKparams.h"
#include "CountMinSketch.hpp"
-#include "main_utils.h"
+
+
+
+void test_getBestLambdaForN() {
+ ROCKparams Rparms;
+ unsigned long nb_k_mer=5000000000;
+ int best=Rparms.getBestLambdaForN(nb_k_mer,2,UINT_MAX,0.05);
+ assert(best==2);
+ nb_k_mer=200000000;
+ best=Rparms.getBestLambdaForN(nb_k_mer,5,UINT_MAX,0.1);
+ assert(best==1);
+ nb_k_mer=600000000;
+ best=Rparms.getBestLambdaForN(nb_k_mer,70,UINT_MAX,0.1);
+ assert(best==1);
+ nb_k_mer=2000000000;
+ best=Rparms.getBestLambdaForN(nb_k_mer,10,UINT_MAX,0.1);
+ assert(best==1);
+ nb_k_mer=10000000000;
+ best=Rparms.getBestLambdaForN(nb_k_mer,5,UINT_MAX,0.1);
+ //cout<<best<<endl;
+ assert(best==1);
+ best=Rparms.getBestLambdaForN(nb_k_mer,1,UINT_MAX,0.05);
+ //cout<<best<<endl;
+ assert(best==8);
+
+ nb_k_mer=50000000000;
+ best=Rparms.getBestLambdaForN(nb_k_mer,5,UINT_MAX,0.1);// this returns 90
+ assert(best==90);
+ best=Rparms.getBestLambdaForN(nb_k_mer,50,UINT_MAX,0.1);
+ assert(best==1);
+
+ best=Rparms.getBestLambdaForN(nb_k_mer,50,UINT_MAX,0.05);
+ assert(best==1);
+ best=Rparms.getBestLambdaForN(nb_k_mer,10,UINT_MAX,0.05);
+ assert(best==7);
+}
+
+void test_processIOFileArgs() {
+ std::vector<IO_fq_files> single_files;
+ vector<PE_files> v_PE_files;
+ std::vector<string> v_input_lines;
+ std::vector<string> v_output_lines;
+ ROCKparams main_parms;
+ int f_id=0;
+ int ret=main_parms.loadInOutFileArgs("../test/data/unit/list_input1.txt","../test/data/unit/list_output1.txt",v_input_lines,v_output_lines);
+ assert(ret==EXIT_SUCCESS);
+ ret=main_parms.processInOutFileArgs(v_input_lines,v_output_lines,single_files,v_PE_files,f_id);
+ assert(ret==EXIT_FAILURE);
+ f_id=0;
+ v_PE_files.clear();
+ single_files.clear();
+ v_input_lines.clear();
+ v_output_lines.clear();
+ ret=main_parms.loadInOutFileArgs("../test/data/unit/list_input2.txt","../test/data/unit/list_output2.txt",v_input_lines,v_output_lines);
+ ret=main_parms.processInOutFileArgs(v_input_lines,v_output_lines,single_files,v_PE_files,f_id);
+ assert(ret==EXIT_FAILURE);
+ f_id=0;
+ v_PE_files.clear();
+ single_files.clear();
+ v_input_lines.clear();
+ v_output_lines.clear();
+ ret=main_parms.loadInOutFileArgs("../test/data/unit/list_input3.txt","../test/data/unit/list_output3.txt",v_input_lines,v_output_lines);
+ ret=main_parms.processInOutFileArgs(v_input_lines,v_output_lines,single_files,v_PE_files,f_id);
+ assert(ret==EXIT_SUCCESS);
+ assert(f_id==14);
+ assert(single_files.size()==2);
+ assert(v_PE_files.size()==6);
+ IO_fq_files s=single_files[0];
+ assert(s.in_fq_file.compare("fifi")==0);
+ assert(s.out_fq_file.compare("ofifi")==0);
+ PE_files s2=v_PE_files[5];
+ assert(s2.PE2.in_fq_file.compare("nono")==0);
+ assert(s2.PE2.out_fq_file.compare("onono")==0);
+
+ v_output_lines.clear();
+ v_input_lines.clear();
+ v_PE_files.clear();
+ single_files.clear();
+ ret=main_parms.loadInOutFileArgs("../test/data/unit/list_input3.txt","",v_input_lines,v_output_lines);
+ ret=main_parms.processInOutFileArgs(v_input_lines,v_output_lines,single_files,v_PE_files,f_id);
+ s=single_files[0];
+ assert(s.out_fq_file.compare("fifi.rock.fq")==0);
+ s2=v_PE_files[5];
+ assert(s2.PE2.out_fq_file.compare("nono.rock.fq")==0);
+
+}
+
+
+void test_ROCKparams() {
+ cout<<"test computation of the best lambda value depending on nb distinct k-mers."<<endl;
+ test_getBestLambdaForN();
+ cout<<"testing processing of IOFile arguments ( following 2 error messages are what is expected; don't worry)."<<endl;
+ test_processIOFileArgs();
+ cout<<"done"<<endl;
+}
void test_CMS(int lambda,int kappa,int kappa_prime) {
@@ -27,10 +122,11 @@ void test_CMS(int lambda,int kappa,int kappa_prime) {
}
// Now, check that our k-mer is present in the CMS.
int min=cms.getEstimatedNbOcc(num);
- std::cout<<"min="<<min<<endl;
+ //std::cout<<"min="<<min<<endl;
assert((min==num) || (min==num-get_mask<unsigned char>::value-1)); // addKmer doesn't check kappa.
unsigned long nb_distinct_k_mers=cms.getNbDistinctKMers();
+ //cout<<nb_distinct_k_mers<<endl;
assert(nb_distinct_k_mers==1);
// mimic a read (as given by part 2 output).
@@ -355,13 +451,15 @@ int main(int argc, char **argv) {
return -1;
}
-
+ cout<<"testing CMS params"<<endl;
+ test_ROCKparams();
cout<<"testing CMS with lambda="<<lambda<<endl;
test_CMS(lambda,kappa,kappa_prime); // Finally using C arrays (maps implied storing hash keys : 4 Bytes per k_mer overhead) but each array is of size INT_MAX...
- cout<<"testing CMS with PE not as single strict kappa prime implementation"<<endl;
- testCMS_with_PE_NotasSingle(lambda,kappa,kappa_prime);
+ //cout<<"testing CMS with PE not as single strict kappa prime implementation"<<endl;
+ // VL comment these tests since AC says That the best option is to treat PE as single.
+ /*testCMS_with_PE_NotasSingle(lambda,kappa,kappa_prime);
cout<<"testing CMS with PE not as single lenient kappa prime implementation"<<endl;
- testCMS_with_PE_NotasSingle2(lambda,kappa,kappa_prime);
+ testCMS_with_PE_NotasSingle2(lambda,kappa,kappa_prime);*/
cout<<"done"<<endl;
// profiling_CMS_fill();
diff --git a/src/unit_test_fqwriter.cpp b/src/unit_test_fqwriter.cpp
index 21ad59c061b642b1a79a70d58ff975b6a2e96129..db0d8bf566b1ab383bcc3cd4b424575753db5c50 100644
--- a/src/unit_test_fqwriter.cpp
+++ b/src/unit_test_fqwriter.cpp
@@ -57,6 +57,7 @@ CCCCCCFFFCCCGGGGGGGGGGHHHHHHHHGHHHHHHHHHHGGGGGGGGHHGHHHHGHGHGHHHHHHHHHFHHHGGGGGG
void test_write_PE() {
srp sr;
+ T_read_counters rc;
srp::reverse_iterator rit;
i_dim::iterator it_offs;
k_dim::iterator it_struct;
@@ -98,7 +99,9 @@ void test_write_PE() {
be_fq1->setOutputFile((char *) "../test/data/unit/klebsiella_PE1_filtered.fq");
be_fq2->setOutputFile((char *) "../test/data/unit/klebsiella_PE2_filtered.fq");
- writeFilteredFastq(map_id_backend,2, sr);
+ writeFilteredFastq(map_id_backend,2,rc,sr);
+ assert(rc.nb_input_reads==3);
+ assert(rc.nb_output_reads==3);
// step 4 : re-read output files and check their content.
mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
diff --git a/src/unit_test_main_utils.cpp b/src/unit_test_main_utils.cpp
deleted file mode 100644
index 5d05fa1646b6a1f2688d27bdf3917b2d6e975508..0000000000000000000000000000000000000000
--- a/src/unit_test_main_utils.cpp
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * test_main_utils.cpp
- *
- * Created on: May 25, 2016
- * Author: vlegrand
- */
-
-#include <iostream>
-#include <cassert>
-#include <string>
-#include <cmath>
-
-
-using namespace std;
-
-#include "rock_commons.h"
-#include "main_utils.h"
-#include "ROCKparams.h"
-
-void test_processIOFileArgs() {
- std::vector<IO_fq_files> single_files;
- vector<PE_files> v_PE_files;
- std::vector<string> v_input_lines;
- std::vector<string> v_output_lines;
- ROCKparams main_parms;
- int f_id=0;
- int ret=main_parms.loadInOutFileArgs("../test/data/unit/list_input1.txt","../test/data/unit/list_output1.txt",v_input_lines,v_output_lines);
- assert(ret==EXIT_SUCCESS);
- ret=main_parms.processInOutFileArgs(v_input_lines,v_output_lines,single_files,v_PE_files,f_id);
- assert(ret==EXIT_FAILURE);
- f_id=0;
- v_PE_files.clear();
- single_files.clear();
- v_input_lines.clear();
- v_output_lines.clear();
- ret=main_parms.loadInOutFileArgs("../test/data/unit/list_input2.txt","../test/data/unit/list_output2.txt",v_input_lines,v_output_lines);
- ret=main_parms.processInOutFileArgs(v_input_lines,v_output_lines,single_files,v_PE_files,f_id);
- assert(ret==EXIT_FAILURE);
- f_id=0;
- v_PE_files.clear();
- single_files.clear();
- v_input_lines.clear();
- v_output_lines.clear();
- ret=main_parms.loadInOutFileArgs("../test/data/unit/list_input3.txt","../test/data/unit/list_output3.txt",v_input_lines,v_output_lines);
- ret=main_parms.processInOutFileArgs(v_input_lines,v_output_lines,single_files,v_PE_files,f_id);
- assert(ret==EXIT_SUCCESS);
- assert(f_id==14);
- assert(single_files.size()==2);
- assert(v_PE_files.size()==6);
- IO_fq_files s=single_files[0];
- assert(s.in_fq_file.compare("fifi")==0);
- assert(s.out_fq_file.compare("ofifi")==0);
- PE_files s2=v_PE_files[5];
- assert(s2.PE2.in_fq_file.compare("nono")==0);
- assert(s2.PE2.out_fq_file.compare("onono")==0);
-
- v_output_lines.clear();
- v_input_lines.clear();
- v_PE_files.clear();
- single_files.clear();
- ret=main_parms.loadInOutFileArgs("../test/data/unit/list_input3.txt","",v_input_lines,v_output_lines);
- ret=main_parms.processInOutFileArgs(v_input_lines,v_output_lines,single_files,v_PE_files,f_id);
- s=single_files[0];
- assert(s.out_fq_file.compare("fifi.rock.fq")==0);
- s2=v_PE_files[5];
- assert(s2.PE2.out_fq_file.compare("nono.rock.fq")==0);
-
-}
-
-
-void test_getBestLambdaForN() {
- int lambda_max=10;
- unsigned long nb_k_mer=200000000;
- int best=getBestLambdaForN(nb_k_mer,lambda_max);
- assert(best==2);
- nb_k_mer=600000000;
- best=getBestLambdaForN(nb_k_mer,lambda_max);
- assert(best==4);
- nb_k_mer=2000000000;
- best=getBestLambdaForN(nb_k_mer,lambda_max);
- // cout<<best<<endl;
- assert(best==10);
- nb_k_mer=10000000000;
- best=getBestLambdaForN(nb_k_mer,lambda_max);
- assert(best==lambda_max);
-}
-
-void test_getCollisionProba() {
- float p =getCollisionProba(1,2);
- // cout<<p<<endl;
- assert(p==0.0);
- p =getCollisionProba(1000000000,2);
- // cout<<p<<endl;
- assert(floor(p*1000+0.5)==139);
- p =getCollisionProba(1000000000,4);
- assert(floor(p*1000+0.5)==19);
- p =getCollisionProba(20000000000,10);
- assert(floor(p*1000+0.5)==999);
-}
-
-
-int main() {
- cout<<"testing main_utils"<<endl;
- cout<<"testing processing of IOFile arguments ( following 2 error messages are what is expected; don't worry)."<<endl;
- test_processIOFileArgs();
- cout<<"test computation of the bast lambda value depending on nb distinct k-mers."<<endl;
- test_getBestLambdaForN();
- cout<<"testing computation of collision probability."<<endl;
- test_getCollisionProba();
- cout<<"done"<<endl;
-}
-
-
diff --git a/src/unit_test_math_utils.cpp b/src/unit_test_math_utils.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..cee4d472638205ead410cbd0d214f3b539ce888a
--- /dev/null
+++ b/src/unit_test_math_utils.cpp
@@ -0,0 +1,86 @@
+/*
+ * unit_test_math_utils.cpp
+ *
+ * Created on: 7 mai 2021
+ * Author: vlegrand
+ */
+#include <iostream>
+#include <cassert>
+#include <climits>
+#include <cmath>
+#include "math_utils.h"
+
+using namespace std;
+
+void test_getCollisionProba() {
+ float p=getCollisionProba(2,5000000000,UINT_MAX,2);
+ assert(round(p*10000)==127);
+ p=getCollisionProba(2,5000000000,UINT_MAX,1);
+ assert(p=0.1128);
+ p =getCollisionProba(1,2,UINT_MAX,3);
+ //cout<<p<<endl;
+ assert(p==0.0);
+ p =getCollisionProba(1,1000000000,UINT_MAX,1);
+ assert(floor(p*1000+0.5)==23);
+ p =getCollisionProba(5,1000000000,UINT_MAX,1);
+ assert(floor(p*1000+0.5)==0);
+ p =getCollisionProba(5,50000000000,UINT_MAX,1);
+ assert(floor(p*1000+0.5)==975);
+}
+
+void test_gammln() {
+ float a;
+ a=gammln(1);
+ assert(exp(a)==1);
+ a=gammln(2);
+ assert(exp(a)==1);
+ a=gammln(3);
+ assert(exp(a)==2);
+ a=gammln(4);
+ assert(exp(a)==6);
+ a=gammln(5);
+ assert(exp(a)==24);
+ a=gammln(6);
+ float tmp=exp(a);
+ float tmp2=tmp*10000;
+ float tmp3=round(tmp2)/10000;
+ //printf("%s \n",tmp3);
+ assert(tmp3==120);
+ /* unsigned long n=5000;
+ unsigned long m=n+1;
+ a=gammln(m);
+ float b=gammln(n);
+ double truc=exp(a-b);
+ assert(truc==n);
+ */
+}
+
+void test_pmf() {
+ unsigned long nb_k_mers=5000000000;
+ float p=pmf(0,nb_k_mers,UINT_MAX);
+ assert(round(p*10000)==3122);
+ p=pmf(1,nb_k_mers,UINT_MAX);
+ assert(round(p*10000)==3634);
+ p=pmf(2,nb_k_mers,UINT_MAX);
+ assert(round(p*10000)==2115);
+}
+
+void test_ccdf() {
+ unsigned long nb_k_mers=5000000000;
+ double res=ccdf(2,nb_k_mers,UINT_MAX);
+ assert(round(res*10000)==1128);
+}
+
+
+int main(int argc, char **argv) {
+ cout<<"testing the gammln function"<<endl;
+ test_gammln();
+ cout<<"testing the pmf function"<<endl;
+ test_pmf();
+ cout<<"testing the ccdf function"<<endl;
+ test_ccdf();
+ cout<<"testing computation of collision probability."<<endl;
+ test_getCollisionProba();
+}
+
+
diff --git a/src/unit_test_read_utils.cpp b/src/unit_test_read_utils.cpp
index da9d830afc0095f1e182353bcaa0a2ef7ed836c9..5b00b7aff765724596fd7574f2c802bc5c3fd93e 100644
--- a/src/unit_test_read_utils.cpp
+++ b/src/unit_test_read_utils.cpp
@@ -46,7 +46,8 @@ void test_getReadSingle() {
char * tmp=(char *) a_seqs.fq_record_buf;
tmp+=a_seqs.start_idx;
memcpy(dna_read,tmp,a_seqs.length);
- // std::cout<<dna_read<<endl;
+ dna_read[a_seqs.length]='\0';
+ std::cout<<dna_read<<endl;
assert(strcmp(dna_read,"TTTTTAGGTGCTACCATAACACCAACTGTTTTCACCATAATTTTAAAATCAAGCATTAGAGACGCTTCTCTAATGTATTGCAAATCTAGTTCTACCATTTGATGAAAATCTAATTCATTTCTTCCACTAACCTGCCATAATCCAGTACAACCTGGTATAACGGTCAAACGCTTTTTATCATAGGAACTGTATTCTCCTACCTCACGTGGCAAAGGAGGTCTTGGACCAACAATTGCCATGTCTCCTTTAACCACATTCCAAAGCTGTGGTA")==0);
diff --git a/test/Makefile.in b/test/Makefile.in
index df8c1656ab9131bdc1cf21823bf9461a4bcfadee..3f6c6ea019b11f5f94e1c5e13eda218618ebe64e 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -1,7 +1,7 @@
-# Makefile.in generated by automake 1.15 from Makefile.am.
+# Makefile.in generated by automake 1.16.3 from Makefile.am.
# @configure_input@
-# Copyright (C) 1994-2014 Free Software Foundation, Inc.
+# Copyright (C) 1994-2020 Free Software Foundation, Inc.
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
@@ -299,6 +299,7 @@ am__set_TESTS_bases = \
bases='$(TEST_LOGS)'; \
bases=`for i in $$bases; do echo $$i; done | sed 's/\.log$$//'`; \
bases=`echo $$bases`
+AM_TESTSUITE_SUMMARY_HEADER = ' for $(PACKAGE_STRING)'
RECHECK_LOGS = $(TEST_LOGS)
AM_RECURSIVE_TARGETS = check recheck
TEST_SUITE_LOG = test-suite.log
@@ -464,8 +465,8 @@ Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
*config.status*) \
cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
*) \
- echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
- cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
esac;
$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
@@ -590,7 +591,7 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
fi; \
echo "$${col}$$br$${std}"; \
- echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}Testsuite summary"$(AM_TESTSUITE_SUMMARY_HEADER)"$${std}"; \
echo "$${col}$$br$${std}"; \
create_testsuite_report --maybe-color; \
echo "$$col$$br$$std"; \
@@ -603,7 +604,7 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
fi; \
$$success || exit 1
-check-TESTS:
+check-TESTS:
@list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
@list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list
@test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
@@ -646,7 +647,10 @@ rock.sh.log: rock.sh
@am__EXEEXT_TRUE@ $(am__common_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS) $(TEST_LOG_DRIVER_FLAGS) -- $(TEST_LOG_COMPILE) \
@am__EXEEXT_TRUE@ "$$tst" $(AM_TESTS_FD_REDIRECT)
-distdir: $(DISTFILES)
+distdir: $(BUILT_SOURCES)
+ $(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
list='$(DISTFILES)'; \
diff --git a/test/rock.sh b/test/rock.sh
index 524608b25262272903a5977c969bd512c211c00c..861642e3a6b70c9dbff7961f476ca5e0a6e27894 100755
--- a/test/rock.sh
+++ b/test/rock.sh
@@ -14,11 +14,13 @@ echo " "
echo "##################################################################################"
echo "checking rock options and error mesages"
#../src/rock|grep "input file name is mandatory" >/dev/null || exit 1
-../src/rock|grep "usage" >/dev/null || exit 2
-../src/rock -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt fic1.fq fic2.fq|grep "It cannot be both" >/dev/null ||exit 3
+../src/rock|grep -i "usage" >/dev/null || exit 2
+../src/rock -l 1 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt fic1.fq fic2.fq|grep "It cannot be both" >/dev/null ||exit 3
#../src/rock -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "usage" >/dev/null ||exit 4
#../src/rock -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt
-../src/rock -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "parent directory for output files:" >/dev/null ||exit 5
+../src/rock -l 1 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "parent directory for output files:" >/dev/null ||exit 5
+../src/rock -l 1 -f 0 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "maximum">/dev/null||exit 51
+../src/rock -l 1 -f 1.1 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "maximum">/dev/null||exit 52
echo " "
echo "##################################################################################"
@@ -55,19 +57,19 @@ mkdir data/fastq.filtered || exit 6
echo " "
echo "##################################################################################"
echo "doing more checks on options and error messages"
-../src/rock -C 500000 -p 0 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "Bad value for kappa" >/dev/null || exit 7
-../src/rock -C 500 -c 600 -p 0 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "ERROR lower filter is higher than high filter" >/dev/null || exit 8
-../src/rock -C 500 -c 400 -k 60 -p 0 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "Bad value for k" >/dev/null || exit 9
-../src/rock -C 500 -c 400 -k 10 -l 0 -p 0 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "Bad value for lambda" >/dev/null || exit 10
-../src/rock -C 500 -c 400 -k 10 -l 500 -p 0 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "Not enough RAM on the machine" >/dev/null || exit 11
-../src/rock -C 500 -c 400 -k 10 -l 12 -g 25 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "options are mutually exclusive" >/dev/null|| exit 12
-../src/rock -C 500 -c 400 -k 10 -g 10000 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "This machine only has" >/dev/null || exit 13
+../src/rock -l 1 -C 500000 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "Bad value for kappa" >/dev/null || exit 7
+../src/rock -l 1 -C 500 -c 600 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "ERROR lower filter is higher than high filter" >/dev/null || exit 8
+../src/rock -l 1 -C 500 -c 400 -k 60 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "Bad value for k" >/dev/null || exit 9
+../src/rock -C 500 -c 400 -k 10 -l 0 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "Bad value for lambda" >/dev/null || exit 10
+../src/rock -C 500 -c 400 -k 10 -l 500 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "Not enough RAM on the machine" >/dev/null || exit 11
+#../src/rock -C 500 -c 400 -k 10 -l 12 -g 25 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "options are mutually exclusive" >/dev/null|| exit 12
+../src/rock -C 500 -c 400 -k 10 -n 50000000000 -c 1 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "think of increasing the value for the low filter" >/dev/null || exit 13
../src/rock -C 500 -c 400 -k 10 -l 12 -n 85000000 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt|grep "options are mutually exclusive">/dev/null || exit 14
-../src/rock -C 500 -c 400 -k 10 -q 3 -m 0 -p 0 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "minimum number of valid k-mer for keeping a read must be an integer">/dev/null || exit 15
-../src/rock -C 500 -c 400 -k 10 -q -1 -m 2 -p 0 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "q must be">/dev/null || exit 16
-../src/rock -C 500 -c 400 -k 10 -q 4 -m 2 -p 10 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "value for -p option must be 0, 1 or 2">/dev/null || exit 17
-../src/rock -C 500 -c 400 -k 10 -q 4 -m 2 -p 1,5 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "value for -p option must be 0, 1 or 2">/dev/null || exit 18
-../src/rock -C 500 -c 400 -k 10 -q 4 -m 2 -p -1 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "value for -p option must be 0, 1 or 2">/dev/null || exit 17
+../src/rock -l 1 -C 500 -c 400 -k 10 -q 3 -m 0 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "minimum number of valid k-mer for keeping a read must be an integer">/dev/null || exit 15
+../src/rock -l 1 -C 500 -c 400 -k 10 -q -1 -m 2 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "q must be">/dev/null || exit 16
+#../src/rock -C 500 -c 400 -k 10 -q 4 -m 2 -p 10 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "value for -p option must be 0, 1 or 2">/dev/null || exit 17
+#../src/rock -C 500 -c 400 -k 10 -q 4 -m 2 -p 1,5 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "value for -p option must be 0, 1 or 2">/dev/null || exit 18
+#../src/rock -C 500 -c 400 -k 10 -q 4 -m 2 -p -1 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "value for -p option must be 0, 1 or 2">/dev/null || exit 17
#../src/rock -C 500 -c 400 -k 10 -q 2 -m 2 -i ${srcdir}/test/data/iofiles.args/extract_klebsiella_long_reads_100.txt|grep "Incompatible options">/dev/null || exit 161
@@ -75,7 +77,7 @@ echo "doing more checks on options and error messages"
../src/unit_test_cms CHECK_MEM
if [ $? = 0 ] ## we have enough memory to run the tests in rock_mem.sh
then
- ${srcdir}/rock_mem.sh ||exit 255
+ ${srcdir}/rock_mem.sh ||exit 255
fi
# unit tests
@@ -98,8 +100,8 @@ echo "unit testing for fqwriter"
echo " "
echo "#################################################################################"
-echo "unit testing for main utils"
-../src/unit_test_main_utils || exit 25
+echo "unit testing for math utils"
+../src/unit_test_math_utils || exit 25
# cleanup
echo " "
diff --git a/test/rock.sh.log b/test/rock.sh.log
deleted file mode 100644
index 49e00a5e08ed10346d46371f1ea1eb7f445983da..0000000000000000000000000000000000000000
--- a/test/rock.sh.log
+++ /dev/null
@@ -1,199 +0,0 @@
-srcdir=.
-
-##################################################################################
-checking rock options and error mesages
-
-##################################################################################
-creating directory for test output files
-copying input data for tests
-checking content of local data directory
-total 16
--rw-r--r-- 1 vlegrand staff 86 Dec 1 2016 extract_klebsiella_long_reads_100.txt
--rw-r--r-- 1 vlegrand staff 112 Dec 1 2016 extract_klebsiella_long_reads_100_filtered.txt
-drwxr-xr-x 10 vlegrand staff 340 Jun 28 13:37 fastq.raw
-drwxr-xr-x 10 vlegrand staff 340 Jun 28 15:10 iofiles.args
-drwxr-xr-x 35 vlegrand staff 1190 Jun 29 10:01 unit
-checking content of local data/fastq.raw directory
-total 448
--rw-r--r-- 1 vlegrand staff 52814 Mar 9 2016 klebsiella_100_1.fq
--rw-r--r-- 1 vlegrand staff 52814 Jun 7 2016 klebsiella_100_1.rock.fq
--rw-r--r-- 1 vlegrand staff 52864 Mar 9 2016 klebsiella_100_2.fq
--rw-r--r-- 1 vlegrand staff 52864 Jun 7 2016 klebsiella_100_2.rock.fq
--rw-r--r--@ 1 vlegrand staff 2667 Jun 27 11:18 klebsiella_5_1_bad_scores.fq
--rw-r--r--@ 1 vlegrand staff 2670 Jun 27 11:18 klebsiella_5_2_bad_scores.fq
--rw-r--r-- 1 vlegrand staff 3195 Jun 28 13:36 klebsiella_6_1_bad_scores.fq
--rw-r--r-- 1 vlegrand staff 3142 Jun 28 13:37 klebsiella_6_2_bad_scores.fq
-checking content of local data/iofiles.args directory
-total 64
--rw-r--r-- 1 vlegrand staff 86 May 9 2016 extract_klebsiella_long_reads_100.txt
--rw-r--r-- 1 vlegrand staff 112 May 9 2016 extract_klebsiella_long_reads_100_filtered.txt
--rw-r--r-- 1 vlegrand staff 70 Jan 6 15:21 output_files_2_PE_separated.txt
--rw-r--r-- 1 vlegrand staff 70 Jun 28 15:10 output_files_2_PE_separated_lenient.txt
--rw-r--r-- 1 vlegrand staff 76 Nov 30 2016 output_files_NQ_Thres_12.txt
--rw-r--r-- 1 vlegrand staff 76 Nov 30 2016 output_files_NQ_Thres_13.txt
--rw-r--r-- 1 vlegrand staff 88 Nov 28 2016 output_files_NQ_Thres_very_low.txt
--rw-r--r-- 1 vlegrand staff 76 Nov 28 2016 output_files_noNQ_Thres.txt
-
-##################################################################################
-doing more checks on options and error messages
-INT_MAX=2147483647
-UINT_MAX=4294967295
-LONG_MAX=9223372036854775807
-ULONG_MAX=18446744073709551615
-ULONG_MAX/UINT_MAX=4294967297
-ULONG_MAX%UINT_MAX=4294967294
-sizeof(short)=2
-sizeof(int)=4
-sizeof(long)=8
-checking that your machine has enough RAM to run test
-machine has: 16 GB of RAM
-testing CMS with lambda=2
-size of the CMS component: 32
-min=200
-testing CMS with PE not as single strict kappa prime implementation
-size of the CMS component: 8
-testing CMS with PE not as single lenient kappa prime implementation
-size of the CMS component: 8
-done
-srcdir=.
-
-##################################################################################
-testing high filter
-
-##################################################################################
-testing low filter
-
-##################################################################################
-testing that input fastq file names can be provided in command line.
-erasing test result files
-
-##################################################################################
-testing verbose mode
-erasing test result files
-
-##################################################################################
-testing rock with a quality score threshold for nucleotides.
-
-##################################################################################
-testing ROCK with a quality score threshold for nucleotides and minimum number of valid k-mer to keep a read.
-
-##################################################################################
-testing ROCK with a quality score threshold for nucleotides and PE processed separately with strict filter
-
-##################################################################################
-testing ROCK with a quality score threshold for nucleotides, PE processed separately (strict filter) and -m option
-
-##################################################################################
-testing ROCK with no quality score threshold for nucleotides, no low filter and PE processed separately (strict filter)
-
-##################################################################################
-testing ROCK with no quality score threshold for nucleotides, no low filter and PE processed separately (lenient filter). Results should be the same as with strict filter
-
-##################################################################################
-testing ROCK with a quality score threshold for nucleotides, PE processed separately (lenient filter) and -m option
- Now use the scrict filter on the same data and check that the only difference is read SRR1222430.6 that should be kept by scrict filter
-
-##################################################################################
-unit testing for cms component
-INT_MAX=2147483647
-UINT_MAX=4294967295
-LONG_MAX=9223372036854775807
-ULONG_MAX=18446744073709551615
-ULONG_MAX/UINT_MAX=4294967297
-ULONG_MAX%UINT_MAX=4294967294
-sizeof(short)=2
-sizeof(int)=4
-sizeof(long)=8
-checking that your machine has enough RAM to run test
-machine has: 16 GB of RAM
-testing CMS with lambda=2
-size of the CMS component: 32
-min=200
-testing CMS with PE not as single strict kappa prime implementation
-size of the CMS component: 8
-testing CMS with PE not as single lenient kappa prime implementation
-size of the CMS component: 8
-done
-
-##################################################################################
-running unit tests
-unit testing for fqreader
-test for single file
-offset_quotient=0
-test for PE files
- fileid1=1 read_a1=0
- fileid2=2 read_a2=0
-349 0 fileid1=1 read_a1=349
- fileid2=2 read_a2=0
-698 0 fileid1=1 read_a1=698
- fileid2=2 read_a2=0
-test the case of records that contain @ character in quality score
-score=35
- fileid1=3 read_a1=0
- fileid2=4 read_a2=0
- fileid1=3 read_a1=3108
- fileid2=4 read_a2=2
- fileid1=3 read_a1=4222
- fileid2=4 read_a2=4
-score=34
- fileid1=3 read_a1=4778
- fileid2=4 read_a2=4
-score=33
- fileid1=3 read_a1=558
- fileid2=4 read_a2=-2
- fileid1=3 read_a1=1114
- fileid2=4 read_a2=0
- fileid1=3 read_a1=3666
- fileid2=4 read_a2=2
-score=29
- fileid1=3 read_a1=2196
- fileid2=4 read_a2=2
-score=27
- fileid1=3 read_a1=2668
- fileid2=4 read_a2=2
-score=26
- fileid1=3 read_a1=1642
- fileid2=4 read_a2=0
-test for both single and PE files
-testing higher level function processInputFiles
-test processing single files with thresholds (nucleotide quality score and minimum number of valid k_mers in read).
-test processing PE files with thresholds (nucleotide quality score and minimum number of valid k_mers in read).
-test processing files with thresholds (nucleotide quality score and minimum number of valid k_mers in read) mimic big files that need several read operations.
-test processing PE files with nucleotide quality threshold and PE not treated as single option.
-test processing PE files with lots of errors in PE2; nucleotide quality threshold and PE not treated as single option.
-test processing PE files with PE not treated as single; quality score threshold and minimum number of valid k-mer.
-done
-
-##################################################################################
-unit testing for read utils
-test getting single reads.
-ACCCAAACTTGCCAGACTTGTGTAGAACGTCCAATATGTATCGGCATCGCTTCCACATGAATGAATCCTTGTTCCACACTTTTTATATGATTCGCATTAATTTCTTGTCCGAAAATCAACTGATTTTTTGCAACATTTTCTCCCGCTCCAAGACTGGCTGCATGTTCTGCAAGCGCAACAGAAACACCACCATGCAAGTAGCCAAAGGGTTGTTTGACCTGATCTGTTATTTCAAGCGCCAGTTCCACTC
-test getting PE reads.
-GGTTCTGTTGGCTCAAACGCCGTCACTTCATTGATCAAAAGCTTATAATGCGTGCCAAAGTCCGCCATCGAGACGACTACGCCTTCCCCTGCTTTCCCGTCAAAAACGAGTCTTGCCGGATCTTCACGGTCTCCCCTCGAAAGCGGCGAAATCTTAGAGGAAGGTGGATATAATGCCGTCACATCGAACTTTGAAGATCTATACGGCATGCAGCAGCTTCCAGGTCTTGCGGTGCAACGTTTAATGGCAGATGGCTACGGTTTTGCGGGGGAAGGAGACTGGAAAACGGCGGCGATCGACCG
-test getting reads with a quality score constraint on nucleotides.
-PE dna read:
-GGNTCTGTTGGCTCAAACGCCNTCACNTCNTTGNTCAAAAGCTTATAATGCGNGCCAAANTCCGCCATCGNGACNNCTACGCCTTCCCCNGCTNTCCCGNCAAANNCNNGTCTTGCCGGANCTTCNCNNNCTCCNNTCGAAAGCGGCGAAATCTTAGNGGAAGGNGGANATAATGCNNTCNCNTCGNACNNTGAANNTNTANNCGGCANNCNGCAGNNTCNAGGNCTTNCNGNGNAACGNTTAANNGCAGATGGCTACGGNTTTGCNGGGGNANGAGACNGGANANCNNNGNCGNTCGNCCN
-test converting k_mers of different size into number
-test converting an entire read to a series of numbers
-testing the case of N nucleotides
-testing higher level function: decomposeReadInKMerNums
-done
-
-##################################################################################
-unit testing for fqwriter
-
-#################################################################################
-unit testing for main utils
-testing main_utils
-testing processing of IOFile arguments ( following 2 error messages are what is expected; don't worry).
-ROCK cannot handle more than 15 input files.
-Inconsistency between input and output files lists!
-ROCK cannot handle more than 15 input files.
-test computation of the bast lambda value depending on nb distinct k-mers.
-testing computation of collision probability.
-done
-
-##################################################################################
-cleaning
-erase_indata=
-PASS rock.sh (exit status: 0)
diff --git a/test/rock.sh.trs b/test/rock.sh.trs
deleted file mode 100644
index 3f0f396c4e40da33ee1725afcd68cba7884199c9..0000000000000000000000000000000000000000
--- a/test/rock.sh.trs
+++ /dev/null
@@ -1,4 +0,0 @@
-:test-result: PASS
-:global-test-result: PASS
-:recheck: no
-:copy-in-global-log: no
diff --git a/test/rock_mem.sh b/test/rock_mem.sh
index d4956a709b561544921272c72143d2ead21d9b0b..11fcf54d4e1d30ad02abad738166543f1d5453a4 100755
--- a/test/rock_mem.sh
+++ b/test/rock_mem.sh
@@ -10,17 +10,13 @@ echo "##########################################################################
echo "testing high filter"
-../src/rock -C 100 -c 0 -l 2 -p 0 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt >/dev/null || exit 101
+../src/rock -C 100 -c 0 -l 2 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt >/dev/null || exit 101
# output files should be the same size, contain the same elements but not in the same order.
nb_PE1=`grep -c "@" ${srcdir}/data/fastq.raw/klebsiella_100_1.fq`
nb_PE2=`grep -c "@" ${srcdir}/data/fastq.raw/klebsiella_100_2.fq`
-#ls -l ${srcdir}/
-#ls -l ${srcdir}/data/
-#ls -l data/fastq.filtered/
-
[ -f "data/fastq.filtered/SRR1222430_1.filtered.fastq" ] || exit 102
[ -f "data/fastq.filtered/SRR1222430_2.filtered.fastq" ] || exit 103
@@ -30,11 +26,33 @@ nb_PE2_filtered=`grep -c "@" data/fastq.filtered/SRR1222430_2.filtered.fastq`
test $nb_PE1 -eq $nb_PE1_filtered || exit 104
test $nb_PE2 -eq $nb_PE2_filtered || exit 105
+echo " "
+echo "##################################################################################"
+echo "testing high filter and -f option"
+
+../src/rock -C 100 -c 0 -l 1 -f 0.05 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt >/dev/null || exit 301
+
+# output files should be the same size, contain the same elements but not in the same order.
+nb_PE1=`grep -c "@" ${srcdir}/data/fastq.raw/klebsiella_100_1.fq`
+nb_PE2=`grep -c "@" ${srcdir}/data/fastq.raw/klebsiella_100_2.fq`
+
+
+
+[ -f "data/fastq.filtered/SRR1222430_1.filtered.fastq" ] || exit 302
+[ -f "data/fastq.filtered/SRR1222430_2.filtered.fastq" ] || exit 303
+
+nb_PE1_filtered=`grep -c "@" data/fastq.filtered/SRR1222430_1.filtered.fastq`
+nb_PE2_filtered=`grep -c "@" data/fastq.filtered/SRR1222430_2.filtered.fastq`
+
+test $nb_PE1 -eq $nb_PE1_filtered || exit 304
+test $nb_PE2 -eq $nb_PE2_filtered || exit 305
+
+
# test low filter.
echo " "
echo "##################################################################################"
echo "testing low filter"
-../src/rock -C 100 -c 99 -l 2 -p 0 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt >/dev/null || exit 106
+../src/rock -C 100 -c 99 -l 2 -i ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100.txt -o ${srcdir}/data/iofiles.args/extract_klebsiella_long_reads_100_filtered.txt >/dev/null || exit 106
[ -f "data/fastq.filtered/SRR1222430_1.filtered.fastq" ] || exit 107
[ -f "data/fastq.filtered/SRR1222430_2.filtered.fastq" ] || exit 108
@@ -49,7 +67,7 @@ test $nb_PE2_filtered -eq 0 || exit 110
echo " "
echo "##################################################################################"
echo "testing that input fastq file names can be provided in command line."
-../src/rock -C 100 -c 1 -l 2 -p 0 ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq ${srcdir}/data/unit/test_single.fq ${srcdir}/data/unit/test_single2.fq >/dev/null || exit 111
+../src/rock -C 100 -c 1 -l 2 ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq ${srcdir}/data/unit/test_single.fq ${srcdir}/data/unit/test_single2.fq >/dev/null || exit 111
[ -f "klebsiella_100_1.rock.fq" ] || exit 112
[ -f "klebsiella_100_2.rock.fq" ] || exit 113
@@ -78,9 +96,9 @@ rm -f "test_single2.undefined.fq" || exit 124
echo " "
echo "##################################################################################"
echo "testing verbose mode"
-../src/rock -C 100 -c 1 -l 2 -p 0 -v ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq ${srcdir}/data/unit/test_single.fq ${srcdir}/data/unit/test_single2.fq|grep "CMS size=" >/dev/null || exit 125
+../src/rock -C 100 -c 1 -l 2 -v ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq ${srcdir}/data/unit/test_single.fq ${srcdir}/data/unit/test_single2.fq|grep "count-min sketch size (Gb)" >/dev/null || exit 125
-../src/rock -C 100 -c 1 -v -p 0 -n 1000 ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq ${srcdir}/data/unit/test_single.fq ${srcdir}/data/unit/test_single2.fq|grep "expected probability of collision was:" >/dev/null || exit 126
+../src/rock -C 100 -c 1 -v -n 1000 ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq ${srcdir}/data/unit/test_single.fq ${srcdir}/data/unit/test_single2.fq|grep "expected false positive proba" >/dev/null || exit 126
echo "erasing test result files"
rm -f "klebsiella_100_1.rock.fq" || exit 127
@@ -108,18 +126,18 @@ expected_diff2="> @SRR1222430.37 37 length=251 \
> A3>333BFA2FF4GBFFDGGCGED?FGEGGHGFFFEEG?AF13@50>///11B13@@1/>//B0?>////<//B/00??@/--:--.;:C000;:0/0009.-9:.00:-.;9/9...-;.--9@--9:////-9-9..////9/;//;9///.9-..--------..99.9.//////;-;--9-.////://9/9.;.-;-99-.//.;////-;?9;...9-9-----9;-.;.;/.-9.;/99=--;"
mkdir tmp
-../src/rock -C 100 -c 1 -l 2 -p 0 -o ${srcdir}/data/iofiles.args/output_files_noNQ_Thres.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 140
+../src/rock -C 100 -k 30 -c 1 -l 2 -o ${srcdir}/data/iofiles.args/output_files_noNQ_Thres.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 140
-../src/rock -C 100 -c 1 -l 2 -q 2 -p 0 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_very_low.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 141
+../src/rock -C 100 -k 30 -c 1 -l 2 -q 2 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_very_low.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 141
ret1=`diff tmp/klebsiella_100_1_very_low_qual_thres.fq tmp/klebsiella_100_1_no_qual_thres.fq|wc -l`
test $ret1 -eq 0 || exit 142
ret2=`diff tmp/klebsiella_100_2_very_low_qual_thres.fq tmp/klebsiella_100_2_no_qual_thres.fq|wc -l`
test $ret2 -eq 0 || exit 143
-../src/rock -C 100 -c 1 -l 2 -q 12 -p 0 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_12.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null ||exit 144
+../src/rock -C 100 -k 30 -c 1 -l 2 -q 12 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_12.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null ||exit 144
-../src/rock -C 100 -c 1 -l 2 -q 13 -p 0 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null ||exit 145
+../src/rock -C 100 -k 30 -c 1 -l 2 -q 13 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null ||exit 145
ret1=`diff tmp/klebsiella_100_1_13_qual_thres.fq tmp/klebsiella_100_1_12_qual_thres.fq|grep -c "@SRR1222430.37"`
test $ret1 -eq 1 || exit 146
@@ -140,9 +158,9 @@ echo "##########################################################################
echo "testing ROCK with a quality score threshold for nucleotides and minimum number of valid k-mer to keep a read."
mkdir tmp
-../src/rock -C 100 -c 1 -l 2 -p 0 -o ${srcdir}/data/iofiles.args/output_files_noNQ_Thres.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 150
+../src/rock -C 100 -k 30 -c 1 -l 2 -o ${srcdir}/data/iofiles.args/output_files_noNQ_Thres.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 150
-../src/rock -C 100 -c 1 -l 2 -q 2 -m 5 -p 0 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_very_low.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 151
+../src/rock -C 100 -k 30 -c 1 -l 2 -q 2 -m 5 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_very_low.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 151
ret1=`diff tmp/klebsiella_100_1_very_low_qual_thres.fq tmp/klebsiella_100_1_no_qual_thres.fq|wc -l`
test $ret1 -eq 0 || exit 152
@@ -159,7 +177,7 @@ ret2=`cat tmp/klebsiella_100_2.undefined.fq|wc -l`
test $ret1 -eq 0 || exit 156
test $ret1 -eq 0 || exit 157
-../src/rock -C 100 -c 1 -l 2 -q 13 -m 500 -p 0 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 158
+../src/rock -k 30 -C 100 -c 1 -l 2 -q 13 -m 500 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 158
[ -f "tmp/klebsiella_100_1.undefined.fq" ] || exit 159
[ -f "tmp/klebsiella_100_2.undefined.fq" ] || exit 160
@@ -170,7 +188,7 @@ ret2=`cat tmp/klebsiella_100_2.undefined.fq|wc -l`
test $ret1 -eq 400 || exit 161
test $ret1 -eq 400 || exit 162
-../src/rock -C 100 -c 1 -l 2 -q 13 -m 300 -p 0 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 163
+../src/rock -k 30 -C 100 -c 1 -l 2 -q 13 -m 300 -o ${srcdir}/data/iofiles.args/output_files_NQ_Thres_13.txt ${srcdir}/data/fastq.raw/klebsiella_100_1.fq,${srcdir}/data/fastq.raw/klebsiella_100_2.fq >/dev/null || exit 163
[ -f "tmp/klebsiella_100_1.undefined.fq" ] || exit 164
[ -f "tmp/klebsiella_100_2.undefined.fq" ] || exit 165
@@ -182,386 +200,386 @@ test $ret1 -eq 136 || exit 166
test $ret1 -eq 136 || exit 167
echo " "
-echo "##################################################################################"
-echo "testing ROCK with a quality score threshold for nucleotides and PE processed separately with strict filter "
-../src/rock -C 100 -c 1 -l 2 -q 2 -p 1 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated.txt ${srcdir}/data/fastq.raw/klebsiella_5_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_5_2_bad_scores.fq >/dev/null || exit 168
-
-[ -f "tmp/klebsiella_5_1_bad_scores.undefined.fq" ] || exit 169
-[ -f "tmp/klebsiella_5_2_bad_scores.undefined.fq" ] || exit 170
-
-ret1=`cat tmp/klebsiella_5_1_bad_scores.undefined.fq|wc -l`
-ret2=`cat tmp/klebsiella_5_2_bad_scores.undefined.fq|wc -l`
-
-test $ret1 -eq 8 || exit 171
-test $ret2 -eq 8 || exit 171
-
-ret1=`grep -c SRR1222430.1 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.1 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 2 || exit 172
-test $ret2 -eq 2 || exit 172
-
-ret1=`grep -c SRR1222430.4 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.4 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 2 || exit 173
-test $ret2 -eq 2 || exit 173
-
-ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 174
-test $ret2 -eq 0 || exit 174
-
-ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 175
-test $ret2 -eq 0 || exit 175
-
-ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 176
-test $ret2 -eq 0 || exit 176
-
-ret1=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
-ret2=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
-
-test $ret1 -eq 8 || exit 177
-test $ret2 -eq 8 || exit 177
-
-ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 178
-test $ret2 -eq 2 || exit 178
-
-ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 0 || exit 179
-test $ret2 -eq 0 || exit 179
-
-ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 180
-test $ret2 -eq 2 || exit 180
-
-rm -f tmp/klebsiella_5_1_bad_scores.undefined.fq
-rm -f tmp/klebsiella_5_2_bad_scores.undefined.fq
-rm -f tmp/klebsiella_5_1_2_qual_thres.fq
-rm -f tmp/klebsiella_5_2_2_qual_thres.fq
-
-
-echo " "
-echo "##################################################################################"
-echo "testing ROCK with a quality score threshold for nucleotides, PE processed separately (strict filter) and -m option"
-../src/rock -C 100 -c 1 -l 2 -q 2 -m 2 -p 1 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated.txt ${srcdir}/data/fastq.raw/klebsiella_5_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_5_2_bad_scores.fq >/dev/null || exit 281
-
-[ -f "tmp/klebsiella_5_1_bad_scores.undefined.fq" ] || exit 182
-[ -f "tmp/klebsiella_5_2_bad_scores.undefined.fq" ] || exit 182
-
-ret1=`cat tmp/klebsiella_5_1_bad_scores.undefined.fq|wc -l`
-ret2=`cat tmp/klebsiella_5_2_bad_scores.undefined.fq|wc -l`
-
-test $ret1 -eq 8 || exit 183
-test $ret2 -eq 8 || exit 183
-
-ret1=`grep -c SRR1222430.1 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.1 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 2 || exit 184
-test $ret2 -eq 2 || exit 184
-
-ret1=`grep -c SRR1222430.4 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.4 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 2 || exit 185
-test $ret2 -eq 2 || exit 185
-
-ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 186
-test $ret2 -eq 0 || exit 186
-
-ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 187
-test $ret2 -eq 0 || exit 187
-
-ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 188
-test $ret2 -eq 0 || exit 188
-
-ret1=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
-ret2=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
-
-test $ret1 -eq 8 || exit 189
-test $ret2 -eq 8 || exit 189
-
-ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 190
-test $ret2 -eq 2 || exit 190
-
-ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 0 || exit 191
-test $ret2 -eq 0 || exit 191
-
-ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 192
-test $ret2 -eq 2 || exit 192
-
-rm -f tmp/klebsiella_5_1_bad_scores.undefined.fq
-rm -f tmp/klebsiella_5_2_bad_scores.undefined.fq
-rm -f tmp/klebsiella_5_1_2_qual_thres.fq
-rm -f tmp/klebsiella_5_2_2_qual_thres.fq
-
-
-
-echo " "
-echo "##################################################################################"
-echo "testing ROCK with no quality score threshold for nucleotides, no low filter and PE processed separately (strict filter)"
-
-../src/rock -C 100 -c 0 -l 2 -q 0 -p 1 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated.txt ${srcdir}/data/fastq.raw/klebsiella_5_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_5_2_bad_scores.fq >/dev/null || exit 193
-
-[ -f "tmp/klebsiella_5_1_bad_scores.undefined.fq" ] || exit 194
-[ -f "tmp/klebsiella_5_2_bad_scores.undefined.fq" ] || exit 194
-
-ret1=`cat tmp/klebsiella_5_1_bad_scores.undefined.fq|wc -l`
-ret2=`cat tmp/klebsiella_5_2_bad_scores.undefined.fq|wc -l`
-
-test $ret1 -eq 0 || exit 195
-test $ret2 -eq 0 || exit 195
-
-ret1=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
-ret2=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
-
-test $ret1 -eq 20 || exit 196
-test $ret2 -eq 20 || exit 196
-
-ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 197
-test $ret2 -eq 2 || exit 197
-
-ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 198
-test $ret2 -eq 2 || exit 198
-
-ret1=`grep -c SRR1222430.4 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.4 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 199
-test $ret2 -eq 2 || exit 199
-
-ret1=`grep -c SRR1222430.1 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.1 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 200
-test $ret2 -eq 2 || exit 200
-
-ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 201
-test $ret2 -eq 2 || exit 201
-
-rm -f tmp/klebsiella_5_1_bad_scores.undefined.fq
-rm -f tmp/klebsiella_5_2_bad_scores.undefined.fq
-rm -f tmp/klebsiella_5_1_2_qual_thres.fq
-rm -f tmp/klebsiella_5_2_2_qual_thres.fq
-
-
-echo " "
-echo "##################################################################################"
-echo "testing ROCK with no quality score threshold for nucleotides, no low filter and PE processed separately (lenient filter). Results should be the same as with strict filter"
-
-../src/rock -C 100 -c 0 -l 2 -q 0 -p 2 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated.txt ${srcdir}/data/fastq.raw/klebsiella_5_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_5_2_bad_scores.fq >/dev/null || exit 202
-
-[ -f "tmp/klebsiella_5_1_bad_scores.undefined.fq" ] || exit 202
-[ -f "tmp/klebsiella_5_2_bad_scores.undefined.fq" ] || exit 202
-
-ret1=`cat tmp/klebsiella_5_1_bad_scores.undefined.fq|wc -l`
-ret2=`cat tmp/klebsiella_5_2_bad_scores.undefined.fq|wc -l`
-
-test $ret1 -eq 0 || exit 203
-test $ret2 -eq 0 || exit 203
-
-ret1=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
-ret2=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
-
-test $ret1 -eq 20 || exit 204
-test $ret2 -eq 20 || exit 204
-
-ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 205
-test $ret2 -eq 2 || exit 205
-
-ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 206
-test $ret2 -eq 2 || exit 206
-
-ret1=`grep -c SRR1222430.4 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.4 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 207
-test $ret2 -eq 2 || exit 207
-
-ret1=`grep -c SRR1222430.1 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.1 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 208
-test $ret2 -eq 2 || exit 208
-
-ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 209
-test $ret2 -eq 2 || exit 209
-
-
-rm -f tmp/klebsiella_5_1_bad_scores.undefined.fq
-rm -f tmp/klebsiella_5_2_bad_scores.undefined.fq
-rm -f tmp/klebsiella_5_1_2_qual_thres.fq
-rm -f tmp/klebsiella_5_2_2_qual_thres.fq
-
-
-echo " "
-echo "##################################################################################"
-echo "testing ROCK with a quality score threshold for nucleotides, PE processed separately (lenient filter) and -m option"
-../src/rock -C 100 -c 1 -l 2 -q 2 -m 2 -p 2 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated_lenient.txt ${srcdir}/data/fastq.raw/klebsiella_6_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_6_2_bad_scores.fq >/dev/null || exit 210
-
-[ -f "tmp/klebsiella_6_1_bad_scores.undefined.fq" ] || exit 211
-[ -f "tmp/klebsiella_6_2_bad_scores.undefined.fq" ] || exit 211
-
-ret1=`cat tmp/klebsiella_6_1_bad_scores.undefined.fq|wc -l`
-ret2=`cat tmp/klebsiella_6_2_bad_scores.undefined.fq|wc -l`
-
-test $ret1 -eq 8 || exit 212
-test $ret2 -eq 8 || exit 212
-
-ret1=`grep -c SRR1222430.1 tmp/klebsiella_6_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.1 tmp/klebsiella_6_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 2 || exit 213
-test $ret2 -eq 2 || exit 213
-
-ret1=`grep -c SRR1222430.4 tmp/klebsiella_6_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.4 tmp/klebsiella_6_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 2 || exit 214
-test $ret2 -eq 2 || exit 214
-
-ret1=`grep -c SRR1222430.2 tmp/klebsiella_6_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.2 tmp/klebsiella_6_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 215
-test $ret2 -eq 0 || exit 215
-
-ret1=`grep -c SRR1222430.3 tmp/klebsiella_6_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.3 tmp/klebsiella_6_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 216
-test $ret2 -eq 0 || exit 216
-
-ret1=`grep -c SRR1222430.5 tmp/klebsiella_6_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.5 tmp/klebsiella_6_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 217
-test $ret2 -eq 0 || exit 217
-
-ret1=`grep -c SRR1222430.6 tmp/klebsiella_6_1_bad_scores.undefined.fq`
-ret2=`grep -c SRR1222430.6 tmp/klebsiella_6_2_bad_scores.undefined.fq`
-
-test $ret1 -eq 0 || exit 218
-test $ret2 -eq 0 || exit 218
-
-ret1=`cat tmp/klebsiella_6_1_2_qual_thres.fq|wc -l`
-ret2=`cat tmp/klebsiella_6_1_2_qual_thres.fq|wc -l`
-
-test $ret1 -eq 8 || exit 219
-test $ret2 -eq 8 || exit 219
-
-ret1=`grep -c SRR1222430.2 tmp/klebsiella_6_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.2 tmp/klebsiella_6_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 220
-test $ret2 -eq 2 || exit 220
-
-ret1=`grep -c SRR1222430.3 tmp/klebsiella_6_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.3 tmp/klebsiella_6_2_2_qual_thres.fq`
-
-test $ret1 -eq 0 || exit 221
-test $ret2 -eq 0 || exit 221
-
-ret1=`grep -c SRR1222430.5 tmp/klebsiella_6_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.5 tmp/klebsiella_6_2_2_qual_thres.fq`
-
-test $ret1 -eq 2 || exit 222
-test $ret2 -eq 2 || exit 222
-
-ret1=`grep -c SRR1222430.6 tmp/klebsiella_6_1_2_qual_thres.fq`
-ret2=`grep -c SRR1222430.6 tmp/klebsiella_6_2_2_qual_thres.fq`
-
-test $ret1 -eq 0 || exit 223
-test $ret2 -eq 0 || exit 223
-
-mv tmp/klebsiella_6_1_bad_scores.undefined.fq tmp/klebsiella_6_1_bad_scores.undefined.fq.lenient
-mv tmp/klebsiella_6_2_bad_scores.undefined.fq tmp/klebsiella_6_2_bad_scores.undefined.fq.lenient
-mv tmp/klebsiella_6_1_2_qual_thres.fq tmp/klebsiella_6_1_2_qual_thres.fq.lenient
-mv tmp/klebsiella_6_2_2_qual_thres.fq tmp/klebsiella_6_2_2_qual_thres.fq.lenient
-
-echo " Now use the scrict filter on the same data and check that the only difference is read SRR1222430.6 that should be kept by scrict filter"
-
-../src/rock -C 100 -c 1 -l 2 -q 2 -m 2 -p 1 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated_lenient.txt ${srcdir}/data/fastq.raw/klebsiella_6_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_6_2_bad_scores.fq >/dev/null || exit 224
-
-ret1=`diff tmp/klebsiella_6_1_bad_scores.undefined.fq tmp/klebsiella_6_1_bad_scores.undefined.fq.lenient|wc -l`
-test $ret1 -eq 0 ||exit 225
-
-ret2=`diff tmp/klebsiella_6_2_bad_scores.undefined.fq tmp/klebsiella_6_2_bad_scores.undefined.fq.lenient|wc -l`
-test $ret2 -eq 0 ||exit 226
-
-ret3=`diff tmp/klebsiella_6_1_2_qual_thres.fq tmp/klebsiella_6_1_2_qual_thres.fq.lenient|grep -c SRR1222430.6`
-test $ret3 -eq 2 ||exit 227
-
-ret4=`diff tmp/klebsiella_6_1_2_qual_thres.fq tmp/klebsiella_6_1_2_qual_thres.fq.lenient|wc -l`
-test $ret4 -eq 5 || exit 228 # put 5 because there are 4 lines for 1 read and 1rts line that looks like: "9,12d8"
-
-ret5=`diff tmp/klebsiella_6_2_2_qual_thres.fq tmp/klebsiella_6_2_2_qual_thres.fq.lenient|grep -c SRR1222430.6`
-test $ret5 -eq 2 ||exit 229
-
-ret6=`diff tmp/klebsiella_6_2_2_qual_thres.fq tmp/klebsiella_6_2_2_qual_thres.fq.lenient|wc -l`
-test $ret6 -eq 5 || exit 230
-
-
-
-rm -f tmp/klebsiella_6_1_bad_scores.undefined.fq
-rm -f tmp/klebsiella_6_2_bad_scores.undefined.fq
-rm -f tmp/klebsiella_6_1_2_qual_thres.fq
-rm -f tmp/klebsiella_6_2_2_qual_thres.fq
-rm -fr tmp/klebsiella_6_1_bad_scores.undefined.fq.lenient
-rm -fr tmp/klebsiella_6_2_bad_scores.undefined.fq.lenient
-rm -fr tmp/klebsiella_6_1_2_qual_thres.fq.lenient
-rm -fr tmp/klebsiella_6_2_2_qual_thres.fq.lenient
+echo "***##################################################################################"
+#echo "testing ROCK with a quality score threshold for nucleotides and PE processed separately with strict filter "
+#../src/rock -C 100 -c 1 -l 2 -q 2 -p 1 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated.txt ${srcdir}/data/fastq.raw/klebsiella_5_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_5_2_bad_scores.fq >/dev/null || exit 168
+
+#[ -f "tmp/klebsiella_5_1_bad_scores.undefined.fq" ] || exit 169
+#[ -f "tmp/klebsiella_5_2_bad_scores.undefined.fq" ] || exit 170
+
+# ret1=`cat tmp/klebsiella_5_1_bad_scores.undefined.fq|wc -l`
+# ret2=`cat tmp/klebsiella_5_2_bad_scores.undefined.fq|wc -l`
+#
+# test $ret1 -eq 8 || exit 171
+# test $ret2 -eq 8 || exit 171
+#
+# ret1=`grep -c SRR1222430.1 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.1 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 2 || exit 172
+# test $ret2 -eq 2 || exit 172
+#
+# ret1=`grep -c SRR1222430.4 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.4 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 2 || exit 173
+# test $ret2 -eq 2 || exit 173
+#
+# ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 174
+# test $ret2 -eq 0 || exit 174
+#
+# ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 175
+# test $ret2 -eq 0 || exit 175
+#
+# ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 176
+# test $ret2 -eq 0 || exit 176
+#
+# ret1=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
+# ret2=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
+#
+# test $ret1 -eq 8 || exit 177
+# test $ret2 -eq 8 || exit 177
+#
+# ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 178
+# test $ret2 -eq 2 || exit 178
+#
+# ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 0 || exit 179
+# test $ret2 -eq 0 || exit 179
+#
+# ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 180
+# test $ret2 -eq 2 || exit 180
+#
+# rm -f tmp/klebsiella_5_1_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_5_2_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_5_1_2_qual_thres.fq
+# rm -f tmp/klebsiella_5_2_2_qual_thres.fq
+
+
+# echo " "
+# echo "##################################################################################"
+# echo "testing ROCK with a quality score threshold for nucleotides, PE processed separately (strict filter) and -m option"
+# ../src/rock -C 100 -c 1 -l 2 -q 2 -m 2 -p 1 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated.txt ${srcdir}/data/fastq.raw/klebsiella_5_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_5_2_bad_scores.fq >/dev/null || exit 281
+#
+# [ -f "tmp/klebsiella_5_1_bad_scores.undefined.fq" ] || exit 182
+# [ -f "tmp/klebsiella_5_2_bad_scores.undefined.fq" ] || exit 182
+#
+# ret1=`cat tmp/klebsiella_5_1_bad_scores.undefined.fq|wc -l`
+# ret2=`cat tmp/klebsiella_5_2_bad_scores.undefined.fq|wc -l`
+#
+# test $ret1 -eq 8 || exit 183
+# test $ret2 -eq 8 || exit 183
+#
+# ret1=`grep -c SRR1222430.1 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.1 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 2 || exit 184
+# test $ret2 -eq 2 || exit 184
+#
+# ret1=`grep -c SRR1222430.4 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.4 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 2 || exit 185
+# test $ret2 -eq 2 || exit 185
+#
+# ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 186
+# test $ret2 -eq 0 || exit 186
+#
+# ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 187
+# test $ret2 -eq 0 || exit 187
+#
+# ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 188
+# test $ret2 -eq 0 || exit 188
+#
+# ret1=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
+# ret2=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
+#
+# test $ret1 -eq 8 || exit 189
+# test $ret2 -eq 8 || exit 189
+#
+# ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 190
+# test $ret2 -eq 2 || exit 190
+#
+# ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 0 || exit 191
+# test $ret2 -eq 0 || exit 191
+#
+# ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 192
+# test $ret2 -eq 2 || exit 192
+#
+# rm -f tmp/klebsiella_5_1_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_5_2_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_5_1_2_qual_thres.fq
+# rm -f tmp/klebsiella_5_2_2_qual_thres.fq
+#
+
+
+# echo " "
+# echo "##################################################################################"
+# echo "testing ROCK with no quality score threshold for nucleotides, no low filter and PE processed separately (strict filter)"
+#
+# ../src/rock -C 100 -c 0 -l 2 -q 0 -p 1 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated.txt ${srcdir}/data/fastq.raw/klebsiella_5_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_5_2_bad_scores.fq >/dev/null || exit 193
+#
+# [ -f "tmp/klebsiella_5_1_bad_scores.undefined.fq" ] || exit 194
+# [ -f "tmp/klebsiella_5_2_bad_scores.undefined.fq" ] || exit 194
+#
+# ret1=`cat tmp/klebsiella_5_1_bad_scores.undefined.fq|wc -l`
+# ret2=`cat tmp/klebsiella_5_2_bad_scores.undefined.fq|wc -l`
+#
+# test $ret1 -eq 0 || exit 195
+# test $ret2 -eq 0 || exit 195
+#
+# ret1=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
+# ret2=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
+#
+# test $ret1 -eq 20 || exit 196
+# test $ret2 -eq 20 || exit 196
+#
+# ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 197
+# test $ret2 -eq 2 || exit 197
+#
+# ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 198
+# test $ret2 -eq 2 || exit 198
+#
+# ret1=`grep -c SRR1222430.4 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.4 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 199
+# test $ret2 -eq 2 || exit 199
+#
+# ret1=`grep -c SRR1222430.1 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.1 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 200
+# test $ret2 -eq 2 || exit 200
+#
+# ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 201
+# test $ret2 -eq 2 || exit 201
+#
+# rm -f tmp/klebsiella_5_1_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_5_2_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_5_1_2_qual_thres.fq
+# rm -f tmp/klebsiella_5_2_2_qual_thres.fq
+
+
+# echo " "
+# echo "##################################################################################"
+# echo "testing ROCK with no quality score threshold for nucleotides, no low filter and PE processed separately (lenient filter). Results should be the same as with strict filter"
+#
+# ../src/rock -C 100 -c 0 -l 2 -q 0 -p 2 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated.txt ${srcdir}/data/fastq.raw/klebsiella_5_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_5_2_bad_scores.fq >/dev/null || exit 202
+#
+# [ -f "tmp/klebsiella_5_1_bad_scores.undefined.fq" ] || exit 202
+# [ -f "tmp/klebsiella_5_2_bad_scores.undefined.fq" ] || exit 202
+#
+# ret1=`cat tmp/klebsiella_5_1_bad_scores.undefined.fq|wc -l`
+# ret2=`cat tmp/klebsiella_5_2_bad_scores.undefined.fq|wc -l`
+#
+# test $ret1 -eq 0 || exit 203
+# test $ret2 -eq 0 || exit 203
+#
+# ret1=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
+# ret2=`cat tmp/klebsiella_5_1_2_qual_thres.fq|wc -l`
+#
+# test $ret1 -eq 20 || exit 204
+# test $ret2 -eq 20 || exit 204
+#
+# ret1=`grep -c SRR1222430.2 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.2 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 205
+# test $ret2 -eq 2 || exit 205
+#
+# ret1=`grep -c SRR1222430.3 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.3 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 206
+# test $ret2 -eq 2 || exit 206
+#
+# ret1=`grep -c SRR1222430.4 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.4 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 207
+# test $ret2 -eq 2 || exit 207
+#
+# ret1=`grep -c SRR1222430.1 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.1 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 208
+# test $ret2 -eq 2 || exit 208
+#
+# ret1=`grep -c SRR1222430.5 tmp/klebsiella_5_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.5 tmp/klebsiella_5_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 209
+# test $ret2 -eq 2 || exit 209
+#
+#
+# rm -f tmp/klebsiella_5_1_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_5_2_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_5_1_2_qual_thres.fq
+# rm -f tmp/klebsiella_5_2_2_qual_thres.fq
+
+
+# echo " "
+# echo "##################################################################################"
+# echo "testing ROCK with a quality score threshold for nucleotides, PE processed separately (lenient filter) and -m option"
+# ../src/rock -C 100 -c 1 -l 2 -q 2 -m 2 -p 2 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated_lenient.txt ${srcdir}/data/fastq.raw/klebsiella_6_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_6_2_bad_scores.fq >/dev/null || exit 210
+#
+# [ -f "tmp/klebsiella_6_1_bad_scores.undefined.fq" ] || exit 211
+# [ -f "tmp/klebsiella_6_2_bad_scores.undefined.fq" ] || exit 211
+#
+# ret1=`cat tmp/klebsiella_6_1_bad_scores.undefined.fq|wc -l`
+# ret2=`cat tmp/klebsiella_6_2_bad_scores.undefined.fq|wc -l`
+#
+# test $ret1 -eq 8 || exit 212
+# test $ret2 -eq 8 || exit 212
+#
+# ret1=`grep -c SRR1222430.1 tmp/klebsiella_6_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.1 tmp/klebsiella_6_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 2 || exit 213
+# test $ret2 -eq 2 || exit 213
+#
+# ret1=`grep -c SRR1222430.4 tmp/klebsiella_6_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.4 tmp/klebsiella_6_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 2 || exit 214
+# test $ret2 -eq 2 || exit 214
+#
+# ret1=`grep -c SRR1222430.2 tmp/klebsiella_6_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.2 tmp/klebsiella_6_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 215
+# test $ret2 -eq 0 || exit 215
+#
+# ret1=`grep -c SRR1222430.3 tmp/klebsiella_6_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.3 tmp/klebsiella_6_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 216
+# test $ret2 -eq 0 || exit 216
+#
+# ret1=`grep -c SRR1222430.5 tmp/klebsiella_6_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.5 tmp/klebsiella_6_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 217
+# test $ret2 -eq 0 || exit 217
+#
+# ret1=`grep -c SRR1222430.6 tmp/klebsiella_6_1_bad_scores.undefined.fq`
+# ret2=`grep -c SRR1222430.6 tmp/klebsiella_6_2_bad_scores.undefined.fq`
+#
+# test $ret1 -eq 0 || exit 218
+# test $ret2 -eq 0 || exit 218
+#
+# ret1=`cat tmp/klebsiella_6_1_2_qual_thres.fq|wc -l`
+# ret2=`cat tmp/klebsiella_6_1_2_qual_thres.fq|wc -l`
+#
+# test $ret1 -eq 8 || exit 219
+# test $ret2 -eq 8 || exit 219
+#
+# ret1=`grep -c SRR1222430.2 tmp/klebsiella_6_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.2 tmp/klebsiella_6_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 220
+# test $ret2 -eq 2 || exit 220
+#
+# ret1=`grep -c SRR1222430.3 tmp/klebsiella_6_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.3 tmp/klebsiella_6_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 0 || exit 221
+# test $ret2 -eq 0 || exit 221
+#
+# ret1=`grep -c SRR1222430.5 tmp/klebsiella_6_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.5 tmp/klebsiella_6_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 2 || exit 222
+# test $ret2 -eq 2 || exit 222
+#
+# ret1=`grep -c SRR1222430.6 tmp/klebsiella_6_1_2_qual_thres.fq`
+# ret2=`grep -c SRR1222430.6 tmp/klebsiella_6_2_2_qual_thres.fq`
+#
+# test $ret1 -eq 0 || exit 223
+# test $ret2 -eq 0 || exit 223
+#
+# mv tmp/klebsiella_6_1_bad_scores.undefined.fq tmp/klebsiella_6_1_bad_scores.undefined.fq.lenient
+# mv tmp/klebsiella_6_2_bad_scores.undefined.fq tmp/klebsiella_6_2_bad_scores.undefined.fq.lenient
+# mv tmp/klebsiella_6_1_2_qual_thres.fq tmp/klebsiella_6_1_2_qual_thres.fq.lenient
+# mv tmp/klebsiella_6_2_2_qual_thres.fq tmp/klebsiella_6_2_2_qual_thres.fq.lenient
+#
+# echo " Now use the scrict filter on the same data and check that the only difference is read SRR1222430.6 that should be kept by scrict filter"
+#
+# ../src/rock -C 100 -c 1 -l 2 -q 2 -m 2 -p 1 -o ${srcdir}/data/iofiles.args/output_files_2_PE_separated_lenient.txt ${srcdir}/data/fastq.raw/klebsiella_6_1_bad_scores.fq,${srcdir}/data/fastq.raw/klebsiella_6_2_bad_scores.fq >/dev/null || exit 224
+#
+# ret1=`diff tmp/klebsiella_6_1_bad_scores.undefined.fq tmp/klebsiella_6_1_bad_scores.undefined.fq.lenient|wc -l`
+# test $ret1 -eq 0 ||exit 225
+#
+# ret2=`diff tmp/klebsiella_6_2_bad_scores.undefined.fq tmp/klebsiella_6_2_bad_scores.undefined.fq.lenient|wc -l`
+# test $ret2 -eq 0 ||exit 226
+#
+# ret3=`diff tmp/klebsiella_6_1_2_qual_thres.fq tmp/klebsiella_6_1_2_qual_thres.fq.lenient|grep -c SRR1222430.6`
+# test $ret3 -eq 2 ||exit 227
+#
+# ret4=`diff tmp/klebsiella_6_1_2_qual_thres.fq tmp/klebsiella_6_1_2_qual_thres.fq.lenient|wc -l`
+# test $ret4 -eq 5 || exit 228 # put 5 because there are 4 lines for 1 read and 1rts line that looks like: "9,12d8"
+#
+# ret5=`diff tmp/klebsiella_6_2_2_qual_thres.fq tmp/klebsiella_6_2_2_qual_thres.fq.lenient|grep -c SRR1222430.6`
+# test $ret5 -eq 2 ||exit 229
+#
+# ret6=`diff tmp/klebsiella_6_2_2_qual_thres.fq tmp/klebsiella_6_2_2_qual_thres.fq.lenient|wc -l`
+# test $ret6 -eq 5 || exit 230
+#
+#
+#
+# rm -f tmp/klebsiella_6_1_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_6_2_bad_scores.undefined.fq
+# rm -f tmp/klebsiella_6_1_2_qual_thres.fq
+# rm -f tmp/klebsiella_6_2_2_qual_thres.fq
+# rm -fr tmp/klebsiella_6_1_bad_scores.undefined.fq.lenient
+# rm -fr tmp/klebsiella_6_2_bad_scores.undefined.fq.lenient
+# rm -fr tmp/klebsiella_6_1_2_qual_thres.fq.lenient
+# rm -fr tmp/klebsiella_6_2_2_qual_thres.fq.lenient
diff --git a/test/test-suite.log b/test/test-suite.log
deleted file mode 100644
index cf857ab5e7119b2f79e598a2e6cd5ba6270964c0..0000000000000000000000000000000000000000
--- a/test/test-suite.log
+++ /dev/null
@@ -1,14 +0,0 @@
-===================================
- rock 1.7: test/test-suite.log
-===================================
-
-# TOTAL: 1
-# PASS: 1
-# SKIP: 0
-# XFAIL: 0
-# FAIL: 0
-# XPASS: 0
-# ERROR: 0
-
-.. contents:: :depth: 2
-