renamed repo to batch-unitigs and more serratus-related changes

README.md

@@ -24,7 +24,7 @@ Use the `-assembly` flag to deploy the `fastp+Miniak31` Dockerfile instead of BC
 
 ### Testing
 
-1. AWS S3 bucket - aws-unitigs-<YOUR_ACCOUNT_NUMBER> is created as part of the stack.
+1. AWS S3 bucket - batch-unitigs-<YOUR_ACCOUNT_NUMBER> is created as part of the stack.
 2. Drop a dataset in it. This will trigger the Lambda to trigger the AWS Batch
 3. In AWS Console > Batch, Notice the Job runs and performs the operation based on the pushed container image.
 
@@ -38,7 +38,7 @@ In short:
 
 Which does:
 
-1. AWS Console > S3 bucket - aws-unitigs-<YOUR_ACCOUNT_NUMBER> - Delete the contents of the file
+1. AWS Console > S3 bucket - batch-unitigs-<YOUR_ACCOUNT_NUMBER> - Delete the contents of the file
 3. run the below command to delete the stack.
 
     ```

add_permissions_for_custom_bucket.sh

-aws lambda add-permission --function-name BatchProcessingLambdaInvokeFunction --principal s3.amazonaws.com \
+aws lambda add-permission --function-name RayanUnitigsBatchProcessingLambdaInvokeFunction --principal s3.amazonaws.com \
     --statement-id s3invoke --action "lambda:InvokeFunction" \
     --source-arn arn:aws:s3:::serratus-rayan \
     --source-account $(aws sts get-caller-identity --query Account --output text)

cleanup.sh

 ID=$(aws sts get-caller-identity --query Account --output text)
-aws s3 rm s3://aws-unitigs-$ID --recursive 
+aws s3 rm s3://batch-unitigs-$ID --recursive 
 sleep 5
-aws cloudformation delete-stack --stack-name aws-unitigs
+aws cloudformation delete-stack --stack-name batch-unitigs

spinup.sh

-aws cloudformation create-stack --stack-name aws-unitigs --template-body file://template/template.yaml --capabilities CAPABILITY_NAMED_IAM
+aws cloudformation create-stack --stack-name batch-unitigs --template-body file://template/template.yaml --capabilities CAPABILITY_NAMED_IAM
 
+bash ./add_permissions_for_custom_bucket.sh

src/Dockerfile

@@ -19,12 +19,12 @@ ARG AWS_SECRET_ACCESS_KEY
 #ENV AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
 
 # BCALM install from binaries
-RUN aws s3 cp s3://aws-unitigs-tools/bcalm-binaries-v2.2.3-Linux.tar.gz .
+RUN aws s3 cp s3://serratus-rayan/tools/bcalm-binaries-v2.2.3-Linux.tar.gz .
 RUN tar xf bcalm-binaries-v2.2.3-Linux.tar.gz && rm bcalm-binaries-v2.2.3-Linux.tar.gz
 RUN mv bcalm-binaries-v2.2.3-Linux/bin/bcalm ./
 
 # MFcompress
-RUN aws s3 cp s3://aws-unitigs-tools/MFCompress-linux64-1.01.tgz .
+RUN aws s3 cp s3://serratus-rayan/tools/MFCompress-linux64-1.01.tgz .
 RUN tar xf MFCompress-linux64-1.01.tgz && rm MFCompress-linux64-1.01.tgz
 RUN mv MFCompress-linux64-1.01/MFCompressC ./
 

src/get-aws-profile.sh

@@ -60,6 +60,14 @@ display_usage ()
   echo_stderr "    FOO_SESSION_TOKEN=\$($0 --profile=myprofile --session-token)"
 }
 
+#get profile from environment variable 
+if [[ -z "${AWS_PROFILE}" ]]; then
+    # aws-profile is unset
+    ;
+else
+    PROFILE="${AWS_PROFILE}"
+fi
+
 for i in "$@"
 do
 case $i in

template/template.yaml

@@ -209,21 +209,21 @@ Resources:
             # make /tmp usable by everyone
             chmod 777 /mnt/tmp_ext
 
-            service docker stop  
-            # copy the docker directory to the ephemeral storage
-            rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/ 
-            # set the data directory to the ephemeral storage in the config file of the docker deamon 
-            DOCKER_CFG_FILE=/etc/docker/daemon.json
-            if [ ! -e "${!DOCKER_CFG_FILE}" ]; then
-                # need to create a non empty file for sed to work
-                echo "{" > ${!DOCKER_CFG_FILE}
-            else
-                # replace the last } of the file by a ,
-                sed -i s/}$/,/ ${!DOCKER_CFG_FILE}
-            fi
-            sed -i '$ a "data-root": "/mnt/docker_ext/"' ${!DOCKER_CFG_FILE}
-            sed -i '$ a }' ${!DOCKER_CFG_FILE}
-            service docker start
+            #service docker stop  
+            ## copy the docker directory to the ephemeral storage
+            #rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/ 
+            ## set the data directory to the ephemeral storage in the config file of the docker deamon 
+            #DOCKER_CFG_FILE=/etc/docker/daemon.json
+            #if [ ! -e "${!DOCKER_CFG_FILE}" ]; then
+            #    # need to create a non empty file for sed to work
+            #    echo "{" > ${!DOCKER_CFG_FILE}
+            #else
+            #    # replace the last } of the file by a ,
+            #    sed -i s/}$/,/ ${!DOCKER_CFG_FILE}
+            #fi
+            #sed -i '$ a "data-root": "/mnt/docker_ext/"' ${!DOCKER_CFG_FILE}
+            #sed -i '$ a }' ${!DOCKER_CFG_FILE}
+            #service docker start
 
             --==MYBOUNDARY==--
 
@@ -232,7 +232,8 @@ Resources:
     DependsOn: RayanUnitigsBatchProcessBucketPermission
     Properties: 
       BucketName: 
-          !Sub 'aws-unitigs-${AWS::AccountId}'
+        #!Sub 'aws-unitigs-${AWS::AccountId}' # a newly made bucket
+        !Sub 'serratus-rayan' # serratus' configuration
       NotificationConfiguration:
         LambdaConfigurations:
         - Event: 's3:ObjectCreated:*'
@@ -251,7 +252,9 @@ Resources:
       FunctionName: !Ref RayanUnitigsBatchProcessingLambdaInvokeFunction
       Principal: s3.amazonaws.com
       SourceAccount: !Ref "AWS::AccountId"
-      SourceArn: !Sub "arn:aws:s3:::aws-unitigs-${AWS::AccountId}"
+      #SourceArn: !Sub "arn:aws:s3:::aws-unitigs-${AWS::AccountId}"
+      SourceArn: !Sub "arn:aws:s3:::serratus-rayan"
+
   RayanUnitigsBatchProcessingLambdaInvokeFunction:
     Type: AWS::Lambda::Function
     Properties: