Commit 4ed88aa2 authored by Rayan  CHIKHI's avatar Rayan CHIKHI
Browse files

renamed repo to batch-unitigs and more serratus-related changes

parent d099ea32
......@@ -24,7 +24,7 @@ Use the `-assembly` flag to deploy the `fastp+Miniak31` Dockerfile instead of BC
### Testing
1. AWS S3 bucket - aws-unitigs-<YOUR_ACCOUNT_NUMBER> is created as part of the stack.
1. AWS S3 bucket - batch-unitigs-<YOUR_ACCOUNT_NUMBER> is created as part of the stack.
2. Drop a dataset in it. This will trigger the Lambda to trigger the AWS Batch
3. In AWS Console > Batch, Notice the Job runs and performs the operation based on the pushed container image.
......@@ -38,7 +38,7 @@ In short:
Which does:
1. AWS Console > S3 bucket - aws-unitigs-<YOUR_ACCOUNT_NUMBER> - Delete the contents of the file
1. AWS Console > S3 bucket - batch-unitigs-<YOUR_ACCOUNT_NUMBER> - Delete the contents of the file
3. run the below command to delete the stack.
```
......
aws lambda add-permission --function-name BatchProcessingLambdaInvokeFunction --principal s3.amazonaws.com \
aws lambda add-permission --function-name RayanUnitigsBatchProcessingLambdaInvokeFunction --principal s3.amazonaws.com \
--statement-id s3invoke --action "lambda:InvokeFunction" \
--source-arn arn:aws:s3:::serratus-rayan \
--source-account $(aws sts get-caller-identity --query Account --output text)
ID=$(aws sts get-caller-identity --query Account --output text)
aws s3 rm s3://aws-unitigs-$ID --recursive
aws s3 rm s3://batch-unitigs-$ID --recursive
sleep 5
aws cloudformation delete-stack --stack-name aws-unitigs
aws cloudformation delete-stack --stack-name batch-unitigs
aws cloudformation create-stack --stack-name aws-unitigs --template-body file://template/template.yaml --capabilities CAPABILITY_NAMED_IAM
aws cloudformation create-stack --stack-name batch-unitigs --template-body file://template/template.yaml --capabilities CAPABILITY_NAMED_IAM
bash ./add_permissions_for_custom_bucket.sh
......@@ -19,12 +19,12 @@ ARG AWS_SECRET_ACCESS_KEY
#ENV AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
# BCALM install from binaries
RUN aws s3 cp s3://aws-unitigs-tools/bcalm-binaries-v2.2.3-Linux.tar.gz .
RUN aws s3 cp s3://serratus-rayan/tools/bcalm-binaries-v2.2.3-Linux.tar.gz .
RUN tar xf bcalm-binaries-v2.2.3-Linux.tar.gz && rm bcalm-binaries-v2.2.3-Linux.tar.gz
RUN mv bcalm-binaries-v2.2.3-Linux/bin/bcalm ./
# MFcompress
RUN aws s3 cp s3://aws-unitigs-tools/MFCompress-linux64-1.01.tgz .
RUN aws s3 cp s3://serratus-rayan/tools/MFCompress-linux64-1.01.tgz .
RUN tar xf MFCompress-linux64-1.01.tgz && rm MFCompress-linux64-1.01.tgz
RUN mv MFCompress-linux64-1.01/MFCompressC ./
......
......@@ -60,6 +60,14 @@ display_usage ()
echo_stderr " FOO_SESSION_TOKEN=\$($0 --profile=myprofile --session-token)"
}
#get profile from environment variable
if [[ -z "${AWS_PROFILE}" ]]; then
# aws-profile is unset
;
else
PROFILE="${AWS_PROFILE}"
fi
for i in "$@"
do
case $i in
......
......@@ -209,21 +209,21 @@ Resources:
# make /tmp usable by everyone
chmod 777 /mnt/tmp_ext
service docker stop
# copy the docker directory to the ephemeral storage
rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/
# set the data directory to the ephemeral storage in the config file of the docker deamon
DOCKER_CFG_FILE=/etc/docker/daemon.json
if [ ! -e "${!DOCKER_CFG_FILE}" ]; then
# need to create a non empty file for sed to work
echo "{" > ${!DOCKER_CFG_FILE}
else
# replace the last } of the file by a ,
sed -i s/}$/,/ ${!DOCKER_CFG_FILE}
fi
sed -i '$ a "data-root": "/mnt/docker_ext/"' ${!DOCKER_CFG_FILE}
sed -i '$ a }' ${!DOCKER_CFG_FILE}
service docker start
#service docker stop
## copy the docker directory to the ephemeral storage
#rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/
## set the data directory to the ephemeral storage in the config file of the docker deamon
#DOCKER_CFG_FILE=/etc/docker/daemon.json
#if [ ! -e "${!DOCKER_CFG_FILE}" ]; then
# # need to create a non empty file for sed to work
# echo "{" > ${!DOCKER_CFG_FILE}
#else
# # replace the last } of the file by a ,
# sed -i s/}$/,/ ${!DOCKER_CFG_FILE}
#fi
#sed -i '$ a "data-root": "/mnt/docker_ext/"' ${!DOCKER_CFG_FILE}
#sed -i '$ a }' ${!DOCKER_CFG_FILE}
#service docker start
--==MYBOUNDARY==--
......@@ -232,7 +232,8 @@ Resources:
DependsOn: RayanUnitigsBatchProcessBucketPermission
Properties:
BucketName:
!Sub 'aws-unitigs-${AWS::AccountId}'
#!Sub 'aws-unitigs-${AWS::AccountId}' # a newly made bucket
!Sub 'serratus-rayan' # serratus' configuration
NotificationConfiguration:
LambdaConfigurations:
- Event: 's3:ObjectCreated:*'
......@@ -251,7 +252,9 @@ Resources:
FunctionName: !Ref RayanUnitigsBatchProcessingLambdaInvokeFunction
Principal: s3.amazonaws.com
SourceAccount: !Ref "AWS::AccountId"
SourceArn: !Sub "arn:aws:s3:::aws-unitigs-${AWS::AccountId}"
#SourceArn: !Sub "arn:aws:s3:::aws-unitigs-${AWS::AccountId}"
SourceArn: !Sub "arn:aws:s3:::serratus-rayan"
RayanUnitigsBatchProcessingLambdaInvokeFunction:
Type: AWS::Lambda::Function
Properties:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment