From 4ed88aa2a16b880bef977d63d9085d8833e20d5a Mon Sep 17 00:00:00 2001
From: rayan <rayan.chikhi@pasteur.fr>
Date: Sun, 14 Jun 2020 11:29:00 +0200
Subject: [PATCH] renamed repo to batch-unitigs and more serratus-related
changes
---
README.md | 4 +-
add_permissions_for_custom_bucket.sh | 2 +-
cleanup.sh | 4 +-
.../create_tools_bucket.sh | 0
spinup.sh | 3 +-
src/Dockerfile | 4 +-
src/get-aws-profile.sh | 8 ++++
template/template.yaml | 37 ++++++++++---------
8 files changed, 37 insertions(+), 25 deletions(-)
rename create_tools_bucket.sh => old/create_tools_bucket.sh (100%)
diff --git a/README.md b/README.md
index 2f4358f..b106606 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,7 @@ Use the `-assembly` flag to deploy the `fastp+Miniak31` Dockerfile instead of BC
### Testing
-1. AWS S3 bucket - aws-unitigs-<YOUR_ACCOUNT_NUMBER> is created as part of the stack.
+1. AWS S3 bucket - batch-unitigs-<YOUR_ACCOUNT_NUMBER> is created as part of the stack.
2. Drop a dataset in it. This will trigger the Lambda to trigger the AWS Batch
3. In AWS Console > Batch, Notice the Job runs and performs the operation based on the pushed container image.
@@ -38,7 +38,7 @@ In short:
Which does:
-1. AWS Console > S3 bucket - aws-unitigs-<YOUR_ACCOUNT_NUMBER> - Delete the contents of the file
+1. AWS Console > S3 bucket - batch-unitigs-<YOUR_ACCOUNT_NUMBER> - Delete the contents of the file
3. run the below command to delete the stack.
```
diff --git a/add_permissions_for_custom_bucket.sh b/add_permissions_for_custom_bucket.sh
index d4b3141..f3288ff 100644
--- a/add_permissions_for_custom_bucket.sh
+++ b/add_permissions_for_custom_bucket.sh
@@ -1,4 +1,4 @@
-aws lambda add-permission --function-name BatchProcessingLambdaInvokeFunction --principal s3.amazonaws.com \
+aws lambda add-permission --function-name RayanUnitigsBatchProcessingLambdaInvokeFunction --principal s3.amazonaws.com \
--statement-id s3invoke --action "lambda:InvokeFunction" \
--source-arn arn:aws:s3:::serratus-rayan \
--source-account $(aws sts get-caller-identity --query Account --output text)
diff --git a/cleanup.sh b/cleanup.sh
index c16c128..6a65c06 100644
--- a/cleanup.sh
+++ b/cleanup.sh
@@ -1,4 +1,4 @@
ID=$(aws sts get-caller-identity --query Account --output text)
-aws s3 rm s3://aws-unitigs-$ID --recursive
+aws s3 rm s3://batch-unitigs-$ID --recursive
sleep 5
-aws cloudformation delete-stack --stack-name aws-unitigs
+aws cloudformation delete-stack --stack-name batch-unitigs
diff --git a/create_tools_bucket.sh b/old/create_tools_bucket.sh
similarity index 100%
rename from create_tools_bucket.sh
rename to old/create_tools_bucket.sh
diff --git a/spinup.sh b/spinup.sh
index a62cc06..f414953 100644
--- a/spinup.sh
+++ b/spinup.sh
@@ -1,2 +1,3 @@
-aws cloudformation create-stack --stack-name aws-unitigs --template-body file://template/template.yaml --capabilities CAPABILITY_NAMED_IAM
+aws cloudformation create-stack --stack-name batch-unitigs --template-body file://template/template.yaml --capabilities CAPABILITY_NAMED_IAM
+bash ./add_permissions_for_custom_bucket.sh
diff --git a/src/Dockerfile b/src/Dockerfile
index 35a50a1..45425d9 100644
--- a/src/Dockerfile
+++ b/src/Dockerfile
@@ -19,12 +19,12 @@ ARG AWS_SECRET_ACCESS_KEY
#ENV AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
# BCALM install from binaries
-RUN aws s3 cp s3://aws-unitigs-tools/bcalm-binaries-v2.2.3-Linux.tar.gz .
+RUN aws s3 cp s3://serratus-rayan/tools/bcalm-binaries-v2.2.3-Linux.tar.gz .
RUN tar xf bcalm-binaries-v2.2.3-Linux.tar.gz && rm bcalm-binaries-v2.2.3-Linux.tar.gz
RUN mv bcalm-binaries-v2.2.3-Linux/bin/bcalm ./
# MFcompress
-RUN aws s3 cp s3://aws-unitigs-tools/MFCompress-linux64-1.01.tgz .
+RUN aws s3 cp s3://serratus-rayan/tools/MFCompress-linux64-1.01.tgz .
RUN tar xf MFCompress-linux64-1.01.tgz && rm MFCompress-linux64-1.01.tgz
RUN mv MFCompress-linux64-1.01/MFCompressC ./
diff --git a/src/get-aws-profile.sh b/src/get-aws-profile.sh
index 39b8d2b..d5f5e3d 100644
--- a/src/get-aws-profile.sh
+++ b/src/get-aws-profile.sh
@@ -60,6 +60,14 @@ display_usage ()
echo_stderr " FOO_SESSION_TOKEN=\$($0 --profile=myprofile --session-token)"
}
+#get profile from environment variable
+if [[ -z "${AWS_PROFILE}" ]]; then
+ # aws-profile is unset
+ ;
+else
+ PROFILE="${AWS_PROFILE}"
+fi
+
for i in "$@"
do
case $i in
diff --git a/template/template.yaml b/template/template.yaml
index b93d716..f459b88 100644
--- a/template/template.yaml
+++ b/template/template.yaml
@@ -209,21 +209,21 @@ Resources:
# make /tmp usable by everyone
chmod 777 /mnt/tmp_ext
- service docker stop
- # copy the docker directory to the ephemeral storage
- rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/
- # set the data directory to the ephemeral storage in the config file of the docker deamon
- DOCKER_CFG_FILE=/etc/docker/daemon.json
- if [ ! -e "${!DOCKER_CFG_FILE}" ]; then
- # need to create a non empty file for sed to work
- echo "{" > ${!DOCKER_CFG_FILE}
- else
- # replace the last } of the file by a ,
- sed -i s/}$/,/ ${!DOCKER_CFG_FILE}
- fi
- sed -i '$ a "data-root": "/mnt/docker_ext/"' ${!DOCKER_CFG_FILE}
- sed -i '$ a }' ${!DOCKER_CFG_FILE}
- service docker start
+ #service docker stop
+ ## copy the docker directory to the ephemeral storage
+ #rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/
+ ## set the data directory to the ephemeral storage in the config file of the docker deamon
+ #DOCKER_CFG_FILE=/etc/docker/daemon.json
+ #if [ ! -e "${!DOCKER_CFG_FILE}" ]; then
+ # # need to create a non empty file for sed to work
+ # echo "{" > ${!DOCKER_CFG_FILE}
+ #else
+ # # replace the last } of the file by a ,
+ # sed -i s/}$/,/ ${!DOCKER_CFG_FILE}
+ #fi
+ #sed -i '$ a "data-root": "/mnt/docker_ext/"' ${!DOCKER_CFG_FILE}
+ #sed -i '$ a }' ${!DOCKER_CFG_FILE}
+ #service docker start
--==MYBOUNDARY==--
@@ -232,7 +232,8 @@ Resources:
DependsOn: RayanUnitigsBatchProcessBucketPermission
Properties:
BucketName:
- !Sub 'aws-unitigs-${AWS::AccountId}'
+ #!Sub 'aws-unitigs-${AWS::AccountId}' # a newly made bucket
+ !Sub 'serratus-rayan' # serratus' configuration
NotificationConfiguration:
LambdaConfigurations:
- Event: 's3:ObjectCreated:*'
@@ -251,7 +252,9 @@ Resources:
FunctionName: !Ref RayanUnitigsBatchProcessingLambdaInvokeFunction
Principal: s3.amazonaws.com
SourceAccount: !Ref "AWS::AccountId"
- SourceArn: !Sub "arn:aws:s3:::aws-unitigs-${AWS::AccountId}"
+ #SourceArn: !Sub "arn:aws:s3:::aws-unitigs-${AWS::AccountId}"
+ SourceArn: !Sub "arn:aws:s3:::serratus-rayan"
+
RayanUnitigsBatchProcessingLambdaInvokeFunction:
Type: AWS::Lambda::Function
Properties:
--
GitLab