1.0

wgetENAHTS.sh

@@ -134,6 +134,13 @@ EOF
   WGET_STATIC_OPTIONS="--no-check-certificate --retry-connrefused --random-wait --timeout=15 --quiet";     
   WGET="$WGET_BIN $WGET_STATIC_OPTIONS";
 #                                                                                                            #
+# -- gzip -------------------------------------------------------------------------------------------------  #
+#                                                                                                            #
+  GZIP_BIN=gzip;
+  [ ! $(command -v $GZIP_BIN) ] && echoxit "no $GZIP_BIN detected" ;
+  GZIP_STATIC_OPTIONS="--quiet";     
+  GZIP="$GZIP_BIN $GZIP_STATIC_OPTIONS";
+#                                                                                                            #
 ##############################################################################################################
 
 
@@ -203,7 +210,7 @@ fi
 [[ $WAITIME =~ ^[0-9]+$ ]]   || echoxit "incorrect value (option -w): $WAITIME" ;
  [ $WAITIME -eq 0 ]          && WAITIME=$NTHREADS;
  [ $WAITIME -lt 1 ]          && WAITIME=1;
-
+TMPF=$(mktemp -t -p ${TMPDIR:-/tmp});
 
 ##############################################################################################################
 ####                                                                                                      ####
@@ -220,9 +227,10 @@ then
 fi
 [ $N -eq 0 ] && echoxit "no accession found" ;
 
-trap "echo;for a in $(echo $ACCNLIST);do if ls $OUTDIR/\$a*.fastq.gz&>/dev/null;then for f in $OUTDIR/\$a*.fastq.gz;do if ! gzip -t \$f&>/dev/null;then echo removing \$f;rm -f \$f;fi;done;fi;done;exit 1;" SIGINT ;
+trap "rm -f $TMPF;echo;for a in $(echo $ACCNLIST);do if ls $OUTDIR/\$a*.fastq.gz&>/dev/null;then for f in $OUTDIR/\$a*.fastq.gz;do if ! $GZIP -t \$f&>/dev/null;then echo removing \$f;rm -f \$f;fi;done;fi;done;exit 1;" SIGINT ;
 
 C=0;
+DL=0; echo $DL > $TMPF ;
 for ACCN in $ACCNLIST
 do
   let C++ ;
@@ -233,20 +241,20 @@ do
     ls -lho $OUTFQ | while read line ; do echo -e "\033[90m$line\033[0m" ; done      >&2 ;
     continue ;
   fi
-  if [ ${#ACCN} -eq 9 ]
-  then URL="$FTPENA/${ACCN:0:6}/$ACCN/";
-  elif [ ${#ACCN} -eq 10 ]
-  then URL="$FTPENA/${ACCN:0:6}/00${ACCN:9:1}/$ACCN/";
-  else URL="$FTPENA/${ACCN:0:6}/0${ACCN:9:2}/$ACCN/";
+  nc=${#ACCN};
+  if   [ $nc -eq 9 ];  then URL="$FTPENA/${ACCN:0:6}/$ACCN/";
+  elif [ $nc -eq 10 ]; then URL="$FTPENA/${ACCN:0:6}/00${ACCN:9:1}/$ACCN/";
+  else                      URL="$FTPENA/${ACCN:0:6}/0${ACCN:9:2}/$ACCN/";
   fi
   if $WGET_TEST $URL
   then
     if $DWNL
     then
+      let DL++ ; echo $DL > $TMPF ;
       ECHS="[$C/$N] downloading content of $URL";
       ECHE="[$C/$N] download completed for accession $ACCN";
       ECHP="[WARNING] download problem for accession $ACCN";
-      echo "echo $ECHS; while true; do $WGET_DWNL $URL && break; done; if ls $OUTFQ &>/dev/null; then echo $ECHE; else echo $ECHP >&2; fi" ;
+      echo "echo $ECHS; sleep $WAITIME; while true; do $WGET_DWNL $URL && break; done; if ls $OUTFQ &>/dev/null; then echo $ECHE; else echo $ECHP >&2; fi" ;
       sleep $WAITIME ;
     else
       echo -e "[$C/$N]\033[32m existing repository for accession $ACCN: $URL\033[0m" >&2 ;
@@ -259,7 +267,12 @@ do
 done | xargs -P $NTHREADS -I CMD bash -c CMD ;
 
 
-if ! $DWNL ; then exit 0 ; fi
+DL=$(cat $TMPF); rm -f $TMPF ;
+if [ $DL -eq 0 ]; then exit 0 ; fi ## NOTE: no download
+if [ $DL -eq 1 ] 
+then echo "$(chrono) $DL downloaded repository" ;
+else echo "$(chrono) $DL downloaded repositories" ;
+fi
 
 
 ##############################################################################################################
@@ -274,40 +287,41 @@ for ACCN in $ACCNLIST
 do
   let C++ ;
   OUTFQ=$OUTDIR/$ACCN*.fastq.gz; [ "$OUTDIR" == "." ] && OUTFQ=$ACCN*.fastq.gz;
-  if [ ${#ACCN} -eq 9 ]
-  then URL="$FTPENA/${ACCN:0:6}/$ACCN/";
-  elif [ ${#ACCN} -eq 10 ]
-  then URL="$FTPENA/${ACCN:0:6}/00${ACCN:9:1}/$ACCN/";
-  else URL="$FTPENA/${ACCN:0:6}/0${ACCN:9:2}/$ACCN/";
+  nc=${#ACCN};
+  if   [ $nc -eq 9 ];  then URL="$FTPENA/${ACCN:0:6}/$ACCN/";
+  elif [ $nc -eq 10 ]; then URL="$FTPENA/${ACCN:0:6}/00${ACCN:9:1}/$ACCN/";
+  else                      URL="$FTPENA/${ACCN:0:6}/0${ACCN:9:2}/$ACCN/";
   fi
   if $WGET_TEST $URL
   then
-    echo -n "[$C/$N] checking downloaded files for accession $ACCN ... " ;
+    echo -n "[$C/$N] checking downloaded files for accession $ACCN ..." ;
     n1=$(ls $OUTFQ 2>/dev/null | wc -l);
     if [ $n1 -eq 0 ]
     then
       echo -e "\n\033[31m[WARNING] problem with accession $ACCN\033[0m"    >&2 ;
-      echo -n "[$C/$N] downloading again content of $URL ... " ;
+      echo -n "[$C/$N] downloading again content of $URL ..." ;
     else
       for f in $OUTFQ
       do
-        if ! gzip -t $f &>/dev/null
+        if ! $GZIP --test $f 2>/dev/null
         then 
           echo -e "\n\033[31m[WARNING] problem with file $f\033[0m"        >&2 ;
-          rm -f $OUTFQ ;
-          echo -n "[$C/$N] downloading again $f ... " ;
-        fi
+          rm -f $f ;
+          echo -n "[$C/$N] downloading again $f ..." ;
+        else
+	  echo -n "." ;
+	fi
       done
       n1=$(ls $OUTFQ 2>/dev/null | wc -l);
     fi
-    while true; do $WGET_DWNL $URL && break; done ## NOTE: does nothing when all files were downloaded
+    while true ; do $WGET_DWNL $URL && break ; done ; ## NOTE: does nothing when all files were downloaded
     n2=$(ls $OUTFQ 2>/dev/null | wc -l);
     if [ $n2 -eq 0 ]
     then
       echo -e "\n\033[31m[FAIL] unable to download content of $URL\033[0m" >&2 ;
       sleep $WAITIME ;
     else
-      echo "[ok]" ;
+      echo " [ok]" ;
       ls -lho $OUTFQ ;
       NF=$(( $NF + $n2 ));
       if [ $n1 -eq $n2 ]; then sleep 0.5 ; else sleep $WAITIME ; fi
@@ -317,7 +331,11 @@ do
   fi	
 done
 
-echo "$(chrono) $NF downloaded files" ;
+if [ $NF -eq 1 ] 
+then echo "$(chrono) $NF downloaded file" ;
+else echo "$(chrono) $NF downloaded files" ;
+fi
 
 exit 0 ;
 
+