robustness

src/batch_processor.py

@@ -48,6 +48,10 @@ def process_file(accession, region):
 
         # run fastp
         os.system(' '.join(["cat","out/*.fastq","|","../fastp", "--trim_poly_x", "--stdin", "-o", accession + ".fastq"]))
+            
+        if os.stat(accession+".fastq").st_size == 0:
+            print("fastp produced empty output")
+            exit(1)
         
         # upload filtered reads to s3
         outputBucket = "serratus-rayan"

template/template.yaml

@@ -110,7 +110,7 @@ Resources:
             - Ref: AWS::Region
             - ".amazonaws.com/serratus-dl-batch-job:latest"
         Vcpus: 2
-        Memory: 1000
+        Memory: 2000
         MountPoints:
           - ContainerPath: /tmp
             SourceVolume: temp_dir