Commit bbd3cdb7 authored by Rayan  CHIKHI's avatar Rayan CHIKHI
Browse files

catching even more read dl errors with that fastp robustness fix

parent 513c10bd
......@@ -77,12 +77,14 @@ def process_file(accession, region):
# download reads from accession
os.system('mkdir -p out/')
prefetch_start = datetime.now()
os.system('prefetch '+accession)
# 'is larger than maximum allowed: skipped' solved by:
# https://wiki.rc.hms.harvard.edu/display/O2/Aspera+to+download+NCBI+SRA+data
os.system('prefetch --max-size 35G '+accession)
prefetch_time = datetime.now() - prefetch_start
sdb_log(sdb,accession,'prefetch_time',int(prefetch_time.seconds))
os.system('mkdir -p tmp/')
pfqdump_start = datetime.now()
os.system('/parallel-fastq-dump --split-files --outdir out/ --tmpdir tmp/ --threads 4 --sra-id '+accession)
os.system('/parallel-fastq-dump --skip-technical --split-3 --outdir out/ --tmpdir tmp/ --threads 4 --sra-id '+accession)
pfqdump_time = datetime.now() - pfqdump_start
sdb_log(sdb,accession,'pfqdump_time',int(pfqdump_time.seconds))
......@@ -105,9 +107,17 @@ def process_file(accession, region):
if os.stat(accession+".fastq").st_size == 0:
# fastp output is empty, most likely those reads have dummy quality values. retry.
print("retrying fastp without quality filtering", flush=true)
print("retrying fastp without quality filtering", flush=True)
sdb_log(sdb,accession,'fastp_noqual','True')
fastp(accession,"out/",sdb,"--disable_quality_filtering")
if os.stat(accession+".fastq").st_size == 0:
# fastp output is STILL empty. could be that read1 or read2 are too short. consider both unpaired
print("retrying fastp unpaired mode", flush=True)
os.system('mv out/' + accession + '_1.fastq out/' + accession + '_a.fastq')
os.system('mv out/' + accession + '_2.fastq out/' + accession + '_b.fastq')
sdb_log(sdb,accession,'fastp_unpaired','True')
fastp(accession,"out/",sdb,"--disable_quality_filtering")
if os.stat(accession+".fastq").st_size == 0:
print("fastp produced empty output even without quality filtering", flush=True)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment