Commit ce100040 authored by Rayan  CHIKHI's avatar Rayan CHIKHI
Browse files

for testing, batch creation (of unitigs) without S3 bucket event

parent fc0d1a0f
aws lambda add-permission --function-name RayanUnitigsBatchProcessingLambdaInvokeFunction --principal s3.amazonaws.com \
aws lambda add-permission --function-name RayanUnitigsBatchLambdaInvokeFunction --principal s3.amazonaws.com \
--statement-id s3invoke --action "lambda:InvokeFunction" \
--source-arn arn:aws:s3:::serratus-rayan \
--source-account $(aws sts get-caller-identity --query Account --output text)
......@@ -14,6 +14,8 @@ docker build -t $DOCKER_CONTAINER \
.
echo "Authenticating against AWS ECR..."
eval $(aws ecr get-login --no-include-email --region us-east-1)
# create repository (only needed the first time)
aws ecr create-repository --repository-name $DOCKER_CONTAINER
echo "Tagging ${REPO}..."
docker tag $DOCKER_CONTAINER:latest $REPO:$TAG
docker tag $DOCKER_CONTAINER:latest $REPO:latest
......
aws cloudformation create-stack --stack-name batch-unitigs --template-body file://template/template.yaml --capabilities CAPABILITY_NAMED_IAM
#aws cloudformation create-stack --stack-name batch-unitigs --template-body file://template/template.yaml --capabilities CAPABILITY_NAMED_IAM
aws cloudformation create-stack --stack-name batch-unitigs --template-body file://template/template_nobucket_nolambda.yaml --capabilities CAPABILITY_NAMED_IAM
bash ./add_permissions_for_custom_bucket.sh
#sleep 3 # unsure if enough or too much
#bash ./add_permissions_for_custom_bucket.sh
......@@ -232,33 +232,22 @@ Resources:
DependsOn: RayanUnitigsBatchProcessBucketPermission
Properties:
BucketName:
#!Sub 'aws-unitigs-${AWS::AccountId}' # a newly made bucket
!Sub 'serratus-rayan' # serratus' configuration
!Sub 'aws-unitigs-${AWS::AccountId}' # a newly made bucket
#!Sub 'serratus-rayan' # serratus' configuration
NotificationConfiguration:
LambdaConfigurations:
- Event: 's3:ObjectCreated:*'
Function: !GetAtt RayanUnitigsBatchProcessingLambdaInvokeFunction.Arn
Function: !GetAtt RayanUnitigsBatchLambdaInvokeFunction.Arn
Filter:
S3Key:
Rules:
- Name: suffix
Value: '.fastq'
RayanUnitigsBatchProcessBucketPermission:
Type: AWS::Lambda::Permission
Properties:
Action: 'lambda:InvokeFunction'
FunctionName: !Ref RayanUnitigsBatchProcessingLambdaInvokeFunction
Principal: s3.amazonaws.com
SourceAccount: !Ref "AWS::AccountId"
#SourceArn: !Sub "arn:aws:s3:::aws-unitigs-${AWS::AccountId}"
SourceArn: !Sub "arn:aws:s3:::serratus-rayan"
RayanUnitigsBatchProcessingLambdaInvokeFunction:
RayanUnitigsBatchLambdaInvokeFunction:
Type: AWS::Lambda::Function
Properties:
FunctionName: RayanUnitigsBatchProcessingLambdaInvokeFunction
FunctionName: RayanUnitigsBatchLambdaInvokeFunction
Description: Python Function Handler that would be triggered BY s3 events TO the aws batch
Handler: index.lambda_handler
Runtime: python3.6
......@@ -311,6 +300,18 @@ Resources:
print("Job ID is {}.".format(response['jobId']))
return response
RayanUnitigsBatchProcessBucketPermission:
Type: AWS::Lambda::Permission
Properties:
Action: 'lambda:InvokeFunction'
FunctionName: !Ref RayanUnitigsBatchLambdaInvokeFunction
Principal: s3.amazonaws.com
SourceAccount: !Ref "AWS::AccountId"
SourceArn: !Sub "arn:aws:s3:::aws-unitigs-${AWS::AccountId}"
#SourceArn: !Sub "arn:aws:s3:::serratus-rayan"
Outputs:
ComputeEnvironmentArn:
Value:
......@@ -326,4 +327,4 @@ Outputs:
Ref: RayanUnitigsBatchProcessS3Bucket
LambdaName:
Value:
Ref: RayanUnitigsBatchProcessingLambdaInvokeFunction
Ref: RayanUnitigsBatchLambdaInvokeFunction
---
AWSTemplateFormatVersion: '2010-09-09'
Description: 'Orchestrating an Application Process with AWS Batch using CloudFormation'
Resources:
VPC:
Type: AWS::EC2::VPC
Properties:
CidrBlock: 10.0.0.0/16
InternetGateway:
Type: AWS::EC2::InternetGateway
RouteTable:
Type: AWS::EC2::RouteTable
Properties:
VpcId:
Ref: VPC
VPCGatewayAttachment:
Type: AWS::EC2::VPCGatewayAttachment
Properties:
VpcId:
Ref: VPC
InternetGatewayId:
Ref: InternetGateway
SecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: EC2 Security Group for instances launched in the VPC by Batch
VpcId:
Ref: VPC
Subnet:
Type: AWS::EC2::Subnet
Properties:
CidrBlock: 10.0.0.0/24
VpcId:
Ref: VPC
MapPublicIpOnLaunch: 'True'
Route:
Type: AWS::EC2::Route
Properties:
RouteTableId:
Ref: RouteTable
DestinationCidrBlock: 0.0.0.0/0
GatewayId:
Ref: InternetGateway
SubnetRouteTableAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
RouteTableId:
Ref: RouteTable
SubnetId:
Ref: Subnet
BatchServiceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal:
Service: batch.amazonaws.com
Action: sts:AssumeRole
ManagedPolicyArns:
- arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole
IamInstanceProfile:
Type: AWS::IAM::InstanceProfile
Properties:
Roles:
- Ref: EcsInstanceRole
EcsInstanceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2008-10-17'
Statement:
- Sid: ''
Effect: Allow
Principal:
Service: ec2.amazonaws.com
Action: sts:AssumeRole
ManagedPolicyArns:
- arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role
- arn:aws:iam::aws:policy/AmazonS3FullAccess
RayanUnitigsSpotIamFleetRole: # taken from https://github.com/aodn/aws-wps/blob/master/wps-cloudformation-template.yaml
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service: spot.amazonaws.com
Action: sts:AssumeRole
- Effect: Allow
Principal:
Service: spotfleet.amazonaws.com
Action: sts:AssumeRole
ManagedPolicyArns:
- arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole
RayanUnitigsBatchProcessingJobDefinition:
Type: AWS::Batch::JobDefinition
Properties:
Type: container
JobDefinitionName: RayanUnitigsBatchJobDefinition
ContainerProperties:
Image:
Fn::Join:
- ''
- - Ref: AWS::AccountId
- .dkr.ecr.
- Ref: AWS::Region
- ".amazonaws.com/aws-batch-s3-unitigs-job:latest"
Vcpus: 4
Memory: 7000
MountPoints:
- ContainerPath: /tmp
SourceVolume: temp_dir
Volumes:
- Host:
SourcePath: /tmp
Name: temp_dir
RetryStrategy:
Attempts: 1
RayanUnitigsBatchProcessingJobQueue:
Type: AWS::Batch::JobQueue
Properties:
JobQueueName: RayanUnitigsBatchProcessingJobQueue
Priority: 1
ComputeEnvironmentOrder:
- Order: 1
ComputeEnvironment:
Ref: RayanUnitigsComputeEnvironment
RayanUnitigsComputeEnvironment:
Type: AWS::Batch::ComputeEnvironment
Properties:
Type: MANAGED
ComputeResources:
Type: SPOT
MinvCpus: 0
DesiredvCpus: 0
MaxvCpus: 1000
#AllocationStrategy: SPOT_CAPACITY_OPTIMIZED # maybe let's not activate cause i really want a c5d and nothing else
InstanceTypes:
- c5d
#- optimal
BidPercentage: 100
SpotIamFleetRole: !Ref RayanUnitigsSpotIamFleetRole
Subnets:
- Ref: Subnet
SecurityGroupIds:
- Ref: SecurityGroup
InstanceRole:
Ref: IamInstanceProfile
LaunchTemplate:
LaunchTemplateId: !Ref SpecialComputeLaunchTemplate
Version: !GetAtt SpecialComputeLaunchTemplate.LatestVersionNumber
ServiceRole:
Ref: BatchServiceRole
SpecialComputeLaunchTemplate: # https://github.com/vfrank66/awsbatchlaunchtemplate/blob/master/aws-batch-launch-ami.yaml
Type: AWS::EC2::LaunchTemplate
Properties:
LaunchTemplateName: "Special-inc-nvme"
LaunchTemplateData:
BlockDeviceMappings:
- DeviceName: /dev/xvdcz
VirtualName: ephemeral0
UserData:
"Fn::Base64": !Sub |
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="
--==MYBOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash
# from https://forums.aws.amazon.com/message.jspa?messageID=867011
yum install -y rsync
# mount the ephemeral storage
mkfs.ext4 /dev/nvme1n1
mount -t ext4 /dev/nvme1n1 /mnt/
# make temp directory for containers usage
# should be used in the Batch job definition (MountPoints)
mkdir /mnt/tmp_ext
rsync -avPHSX /tmp/ /mnt/tmp_ext/
# modify fstab to mount /tmp on the new storage.
sed -i '$ a /mnt/tmp_ext /tmp none bind 0 0' /etc/fstab
mount -a
# make /tmp usable by everyone
chmod 777 /mnt/tmp_ext
#service docker stop
## copy the docker directory to the ephemeral storage
#rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/
## set the data directory to the ephemeral storage in the config file of the docker deamon
#DOCKER_CFG_FILE=/etc/docker/daemon.json
#if [ ! -e "${!DOCKER_CFG_FILE}" ]; then
# # need to create a non empty file for sed to work
# echo "{" > ${!DOCKER_CFG_FILE}
#else
# # replace the last } of the file by a ,
# sed -i s/}$/,/ ${!DOCKER_CFG_FILE}
#fi
#sed -i '$ a "data-root": "/mnt/docker_ext/"' ${!DOCKER_CFG_FILE}
#sed -i '$ a }' ${!DOCKER_CFG_FILE}
#service docker start
--==MYBOUNDARY==--
Outputs:
ComputeEnvironmentArn:
Value:
Ref: RayanUnitigsComputeEnvironment
BatchProcessingJobQueueArn:
Value:
Ref: RayanUnitigsBatchProcessingJobQueue
BatchProcessingJobDefinitionArn:
Value:
Ref: RayanUnitigsBatchProcessingJobDefinition
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment