one more attempt at making an EBS volume..

src/Dockerfile

@@ -45,6 +45,6 @@ RUN wget --quiet http://opengene.org/fastp/fastp
 RUN chmod +x fastp 
 
 RUN pwd
-RUN df -h .
+RUN df -T
 RUN ls
 

src/batch_processor.py

@@ -25,11 +25,13 @@ def process_file(accession, region):
 
         # go to /tmp (that's where local storage / nvme is)
         # go to /data instead (EBS)
-        os.chdir("/data")
+        os.chdir("/mnt/serratus-data")
         os.system(' '.join(["pwd"]))
         
         # check free space
-        os.system(' '.join(["df", "-h", "."]))
+        os.system(' '.join(["df", "-T"]))
+        os.system(' '.join(["df", "-h","."]))
+        os.system(' '.join(["lsblk"]))
 
         # download reads from accession
         os.system('mkdir -p out/')

template/template.yaml

@@ -110,15 +110,15 @@ Resources:
             - Ref: AWS::Region
             - ".amazonaws.com/serratus-dl-batch-job:latest"
         Vcpus: 4 
-        Memory: 6000
+        Memory: 8000 # seems that 4000 wasn't enough for parallel-fastq-dump, 4 threads
         MountPoints:
-          - ContainerPath: "/data"
+          - ContainerPath: "/serratus-data"
             ReadOnly: false
-            SourceVolume: data
+            SourceVolume: serratus-data
         Volumes:
-          - Name: data
+          - Name: serratus-data
             Host:
-              SourcePath: "/data"
+              SourcePath: "/serratus-data"
       RetryStrategy:
         Attempts: 1
   RayanSerratusDlBatchProcessingJobQueue:
@@ -136,8 +136,8 @@ Resources:
       Type: MANAGED
       ComputeResources:
         Type: SPOT
-        MinvCpus: 0
-        DesiredvCpus: 0
+        MinvCpus: 4
+        DesiredvCpus: 4
         MaxvCpus: 1000
         AllocationStrategy: SPOT_CAPACITY_OPTIMIZED
         InstanceTypes:
@@ -162,11 +162,11 @@ Resources:
       LaunchTemplateName: "Special-inc-nvme-assembly"
       LaunchTemplateData:
         BlockDeviceMappings:
-        - DeviceName:  '/dev/sdb'  # Amazon ECS-Optimized Amazon Linux 2 ami accepts EDS vol /dev/sd[b-l]
+        - DeviceName:  '/dev/sdh'  # Amazon ECS-Optimized Amazon Linux 2 ami accepts EDS vol /dev/sd[b-l]
           Ebs:
               DeleteOnTermination: true
-              Encrypted: true
-              VolumeSize: 1000 # high because of multitenancy
+              Encrypted: false
+              VolumeSize: 500
               VolumeType: "gp2"
         UserData:
          "Fn::Base64": !Sub |
@@ -175,21 +175,16 @@ Resources:
 
             --==MYBOUNDARY==
             Content-Type: text/x-shellscript; charset="us-ascii"
-            #!/bin/bash          
-            mkfs.ext4  /dev/sdb
-            (
-            echo n # Add a new partition
-            echo p # Primary partition
-            echo 1 # Partition number
-            echo   # First sector (Accept default: 1)
-            echo   # Last sector (Accept default: varies)
-            echo w # Write changes
-            ) | fdisk  /dev/sdb
-            mkfs.ext4  /dev/xvdb1
-            mkdir /data
-            echo " /dev/xvdb1       /data  ext4    defaults        0       2" >> /etc/fstab
+
+            #!/bin/bash -xe  
+            # https://stackoverflow.com/questions/41073906/how-to-attach-and-mount-volumes-to-an-ec2-instance-using-cloudformation 
+            mkfs.ext4  /dev/xvdh
+            mkdir /mnt/serratus-data
+            echo -e "/dev/xvdh\t/mnt/serratus-data\text4\tdefaults\t0\t0" >> /etc/fstab
             mount -a
+            chmod 777 /serratus-data
             service docker restart 
+
             --==MYBOUNDARY==--
 
 Outputs: