tweaks

src/batch_processor.py

@@ -39,8 +39,8 @@ def process_file(accession, region):
         os.system(' '.join(["df", "-h","."]))
 
         # some debug
-        public_sra_files = os.listdir(os.getcwd() + "/public/sra/")
-        print("at start, dir listing of public/sra/", public_sra_files)
+        print("at start, dir listing of /mnt/serratus-data")
+        os.system(' '.join(['ls','-alR','.']))
 
         # download reads from accession
         os.system('mkdir -p out/')
@@ -53,6 +53,7 @@ def process_file(accession, region):
         inputDataFn = accession+".inputdata.txt"
         g = open(inputDataFn,"w")
         for f in files:
+            print("file: " + f + " size: " + str(os.stat("out/"+f).st_size))
             g.write(f + " " + str(os.stat("out/"+f).st_size)+"\n")
         g.close()
 
@@ -60,7 +61,7 @@ def process_file(accession, region):
         # as per https://github.com/ababaian/serratus/blob/master/containers/serratus-dl/run_dl-sra.sh#L26
 
         # run fastp
-        os.system(' '.join(["cat","out/*.fastq","|","/fastp", "--trim_poly_x", "--stdin", "-o", accession + ".fastq"]))
+        os.system(' '.join(["cat","out/*.fastq","|","/fastp", "--thread", "4", "--trim_poly_x", "--stdin", "-o", accession + ".fastq"]))
             
         if os.stat(accession+".fastq").st_size == 0:
             print("fastp produced empty output")

template/template.yaml

@@ -138,7 +138,7 @@ Resources:
         Type: SPOT
         MinvCpus: 0
         DesiredvCpus: 0
-        MaxvCpus: 1000
+        MaxvCpus: 4000
         AllocationStrategy: SPOT_CAPACITY_OPTIMIZED
         InstanceTypes:
         - optimal