Commit 4258ff42 authored by Alexis  CRISCUOLO's avatar Alexis CRISCUOLO
Browse files

1.0

parent 375df1db
......@@ -134,6 +134,13 @@ EOF
WGET_STATIC_OPTIONS="--no-check-certificate --retry-connrefused --random-wait --timeout=15 --quiet";
WGET="$WGET_BIN $WGET_STATIC_OPTIONS";
# #
# -- gzip ------------------------------------------------------------------------------------------------- #
# #
GZIP_BIN=gzip;
[ ! $(command -v $GZIP_BIN) ] && echoxit "no $GZIP_BIN detected" ;
GZIP_STATIC_OPTIONS="--quiet";
GZIP="$GZIP_BIN $GZIP_STATIC_OPTIONS";
# #
##############################################################################################################
......@@ -203,7 +210,7 @@ fi
[[ $WAITIME =~ ^[0-9]+$ ]] || echoxit "incorrect value (option -w): $WAITIME" ;
[ $WAITIME -eq 0 ] && WAITIME=$NTHREADS;
[ $WAITIME -lt 1 ] && WAITIME=1;
TMPF=$(mktemp -t -p ${TMPDIR:-/tmp});
##############################################################################################################
#### ####
......@@ -220,9 +227,10 @@ then
fi
[ $N -eq 0 ] && echoxit "no accession found" ;
trap "echo;for a in $(echo $ACCNLIST);do if ls $OUTDIR/\$a*.fastq.gz&>/dev/null;then for f in $OUTDIR/\$a*.fastq.gz;do if ! gzip -t \$f&>/dev/null;then echo removing \$f;rm -f \$f;fi;done;fi;done;exit 1;" SIGINT ;
trap "rm -f $TMPF;echo;for a in $(echo $ACCNLIST);do if ls $OUTDIR/\$a*.fastq.gz&>/dev/null;then for f in $OUTDIR/\$a*.fastq.gz;do if ! $GZIP -t \$f&>/dev/null;then echo removing \$f;rm -f \$f;fi;done;fi;done;exit 1;" SIGINT ;
C=0;
DL=0; echo $DL > $TMPF ;
for ACCN in $ACCNLIST
do
let C++ ;
......@@ -233,20 +241,20 @@ do
ls -lho $OUTFQ | while read line ; do echo -e "\033[90m$line\033[0m" ; done >&2 ;
continue ;
fi
if [ ${#ACCN} -eq 9 ]
then URL="$FTPENA/${ACCN:0:6}/$ACCN/";
elif [ ${#ACCN} -eq 10 ]
then URL="$FTPENA/${ACCN:0:6}/00${ACCN:9:1}/$ACCN/";
else URL="$FTPENA/${ACCN:0:6}/0${ACCN:9:2}/$ACCN/";
nc=${#ACCN};
if [ $nc -eq 9 ]; then URL="$FTPENA/${ACCN:0:6}/$ACCN/";
elif [ $nc -eq 10 ]; then URL="$FTPENA/${ACCN:0:6}/00${ACCN:9:1}/$ACCN/";
else URL="$FTPENA/${ACCN:0:6}/0${ACCN:9:2}/$ACCN/";
fi
if $WGET_TEST $URL
then
if $DWNL
then
let DL++ ; echo $DL > $TMPF ;
ECHS="[$C/$N] downloading content of $URL";
ECHE="[$C/$N] download completed for accession $ACCN";
ECHP="[WARNING] download problem for accession $ACCN";
echo "echo $ECHS; while true; do $WGET_DWNL $URL && break; done; if ls $OUTFQ &>/dev/null; then echo $ECHE; else echo $ECHP >&2; fi" ;
echo "echo $ECHS; sleep $WAITIME; while true; do $WGET_DWNL $URL && break; done; if ls $OUTFQ &>/dev/null; then echo $ECHE; else echo $ECHP >&2; fi" ;
sleep $WAITIME ;
else
echo -e "[$C/$N]\033[32m existing repository for accession $ACCN: $URL\033[0m" >&2 ;
......@@ -259,7 +267,12 @@ do
done | xargs -P $NTHREADS -I CMD bash -c CMD ;
if ! $DWNL ; then exit 0 ; fi
DL=$(cat $TMPF); rm -f $TMPF ;
if [ $DL -eq 0 ]; then exit 0 ; fi ## NOTE: no download
if [ $DL -eq 1 ]
then echo "$(chrono) $DL downloaded repository" ;
else echo "$(chrono) $DL downloaded repositories" ;
fi
##############################################################################################################
......@@ -274,40 +287,41 @@ for ACCN in $ACCNLIST
do
let C++ ;
OUTFQ=$OUTDIR/$ACCN*.fastq.gz; [ "$OUTDIR" == "." ] && OUTFQ=$ACCN*.fastq.gz;
if [ ${#ACCN} -eq 9 ]
then URL="$FTPENA/${ACCN:0:6}/$ACCN/";
elif [ ${#ACCN} -eq 10 ]
then URL="$FTPENA/${ACCN:0:6}/00${ACCN:9:1}/$ACCN/";
else URL="$FTPENA/${ACCN:0:6}/0${ACCN:9:2}/$ACCN/";
nc=${#ACCN};
if [ $nc -eq 9 ]; then URL="$FTPENA/${ACCN:0:6}/$ACCN/";
elif [ $nc -eq 10 ]; then URL="$FTPENA/${ACCN:0:6}/00${ACCN:9:1}/$ACCN/";
else URL="$FTPENA/${ACCN:0:6}/0${ACCN:9:2}/$ACCN/";
fi
if $WGET_TEST $URL
then
echo -n "[$C/$N] checking downloaded files for accession $ACCN ... " ;
echo -n "[$C/$N] checking downloaded files for accession $ACCN ..." ;
n1=$(ls $OUTFQ 2>/dev/null | wc -l);
if [ $n1 -eq 0 ]
then
echo -e "\n\033[31m[WARNING] problem with accession $ACCN\033[0m" >&2 ;
echo -n "[$C/$N] downloading again content of $URL ... " ;
echo -n "[$C/$N] downloading again content of $URL ..." ;
else
for f in $OUTFQ
do
if ! gzip -t $f &>/dev/null
if ! $GZIP --test $f 2>/dev/null
then
echo -e "\n\033[31m[WARNING] problem with file $f\033[0m" >&2 ;
rm -f $OUTFQ ;
echo -n "[$C/$N] downloading again $f ... " ;
fi
rm -f $f ;
echo -n "[$C/$N] downloading again $f ..." ;
else
echo -n "." ;
fi
done
n1=$(ls $OUTFQ 2>/dev/null | wc -l);
fi
while true; do $WGET_DWNL $URL && break; done ## NOTE: does nothing when all files were downloaded
while true ; do $WGET_DWNL $URL && break ; done ; ## NOTE: does nothing when all files were downloaded
n2=$(ls $OUTFQ 2>/dev/null | wc -l);
if [ $n2 -eq 0 ]
then
echo -e "\n\033[31m[FAIL] unable to download content of $URL\033[0m" >&2 ;
sleep $WAITIME ;
else
echo "[ok]" ;
echo " [ok]" ;
ls -lho $OUTFQ ;
NF=$(( $NF + $n2 ));
if [ $n1 -eq $n2 ]; then sleep 0.5 ; else sleep $WAITIME ; fi
......@@ -317,7 +331,11 @@ do
fi
done
echo "$(chrono) $NF downloaded files" ;
if [ $NF -eq 1 ]
then echo "$(chrono) $NF downloaded file" ;
else echo "$(chrono) $NF downloaded files" ;
fi
exit 0 ;
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment