template.yaml 6.1 KB
Newer Older
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
1
2
---
AWSTemplateFormatVersion: '2010-09-09'
3
Description: 'Serratus Batch job for downloading reads, trimming, uploading to S3'
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
Resources:
  VPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: 10.0.0.0/16
  InternetGateway:
    Type: AWS::EC2::InternetGateway
  RouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId:
        Ref: VPC
  VPCGatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId:
        Ref: VPC
      InternetGatewayId:
        Ref: InternetGateway
  SecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: EC2 Security Group for instances launched in the VPC by Batch
      VpcId:
        Ref: VPC
  Subnet:
    Type: AWS::EC2::Subnet
    Properties:
      CidrBlock: 10.0.0.0/24
      VpcId:
        Ref: VPC
      MapPublicIpOnLaunch: 'True'
  Route:
    Type: AWS::EC2::Route
    Properties:
      RouteTableId:
        Ref: RouteTable
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId:
        Ref: InternetGateway
  SubnetRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId:
        Ref: RouteTable
      SubnetId:
        Ref: Subnet
  BatchServiceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service: batch.amazonaws.com
          Action: sts:AssumeRole
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole
  IamInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
      - Ref: EcsInstanceRole
  EcsInstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2008-10-17'
        Statement:
        - Sid: ''
          Effect: Allow
          Principal:
            Service: ec2.amazonaws.com
          Action: sts:AssumeRole
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role
      - arn:aws:iam::aws:policy/AmazonS3FullAccess
82
      - arn:aws:iam::aws:policy/AmazonElasticMapReduceFullAccess # for sdb
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

  SpotIamFleetRole: # taken from https://github.com/aodn/aws-wps/blob/master/wps-cloudformation-template.yaml
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service: spot.amazonaws.com
            Action: sts:AssumeRole
          - Effect: Allow
            Principal:
              Service: spotfleet.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole
  RayanSerratusDlBatchProcessingJobDefinition:
    Type: AWS::Batch::JobDefinition
    Properties:
      Type: container
      JobDefinitionName: RayanSerratusDlBatchJobDefinition
      ContainerProperties:
        Image:
          Fn::Join:
          - ''
          - - Ref: AWS::AccountId
            - .dkr.ecr.
            - Ref: AWS::Region
Rayan  CHIKHI's avatar
fix    
Rayan CHIKHI committed
112
            - ".amazonaws.com/serratus-dl-batch-job:latest"
113
        Vcpus: 4 
114
        Memory: 8000 # seems that 4000 wasn't enough for parallel-fastq-dump, 4 threads
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
115
        MountPoints:
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
116
          - ContainerPath: "/mnt/serratus-data"
117
            ReadOnly: false
118
            SourceVolume: serratus-data
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
119
        Volumes:
120
          - Name: serratus-data
121
            Host:
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
122
              SourcePath: "/mnt/serratus-data"
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
      RetryStrategy:
        Attempts: 1
  RayanSerratusDlBatchProcessingJobQueue:
    Type: AWS::Batch::JobQueue
    Properties:
      JobQueueName: RayanSerratusDlBatchProcessingJobQueue
      Priority: 1
      ComputeEnvironmentOrder:
      - Order: 1
        ComputeEnvironment:
          Ref: RayanSerratusDlComputeEnvironment
  RayanSerratusDlComputeEnvironment:
    Type: AWS::Batch::ComputeEnvironment
    Properties:
      Type: MANAGED
      ComputeResources:
        Type: SPOT
140
141
        MinvCpus: 0
        DesiredvCpus: 0
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
142
        MaxvCpus: 10000
143
        AllocationStrategy: SPOT_CAPACITY_OPTIMIZED
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
144
        InstanceTypes:
145
        - optimal
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
        BidPercentage: 100
        SpotIamFleetRole: !Ref SpotIamFleetRole
        Subnets:
        - Ref: Subnet
        SecurityGroupIds:
        - Ref: SecurityGroup
        InstanceRole:
          Ref: IamInstanceProfile
        LaunchTemplate:
          LaunchTemplateId: !Ref SpecialComputeLaunchTemplate
          Version: !GetAtt SpecialComputeLaunchTemplate.LatestVersionNumber
      ServiceRole:
        Ref: BatchServiceRole

  SpecialComputeLaunchTemplate: # https://github.com/vfrank66/awsbatchlaunchtemplate/blob/master/aws-batch-launch-ami.yaml
    Type: AWS::EC2::LaunchTemplate
    Properties:
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
163
      LaunchTemplateName: "Serratus-batch-dl-EBS-launch-template"
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
164
165
      LaunchTemplateData:
        BlockDeviceMappings:
166
        - DeviceName:  '/dev/sdh'  # Amazon ECS-Optimized Amazon Linux 2 ami accepts EDS vol /dev/sd[b-l]
167
168
          Ebs:
              DeleteOnTermination: true
169
              Encrypted: false
Rayan  CHIKHI's avatar
tweaks    
Rayan CHIKHI committed
170
              VolumeSize: 1000 # raised it for the tough dl instances
171
              VolumeType: "gp2"
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
172
173
174
175
176
177
178
        UserData:
         "Fn::Base64": !Sub |
            MIME-Version: 1.0
            Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="

            --==MYBOUNDARY==
            Content-Type: text/x-shellscript; charset="us-ascii"
179
180
181
182
183
184

            #!/bin/bash -xe  
            # https://stackoverflow.com/questions/41073906/how-to-attach-and-mount-volumes-to-an-ec2-instance-using-cloudformation 
            mkfs.ext4  /dev/xvdh
            mkdir /mnt/serratus-data
            echo -e "/dev/xvdh\t/mnt/serratus-data\text4\tdefaults\t0\t0" >> /etc/fstab
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
185
            mount -a
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
186
187
188
            lsblk > /lsblk.txt
            df -T > /dfT.txt
            chmod 777 /mnt/serratus-data
189
            service docker restart 
190

Rayan  CHIKHI's avatar
Rayan CHIKHI committed
191
192
193
194
195
196
197
198
199
200
201
202
            --==MYBOUNDARY==--

Outputs:
  ComputeEnvironmentArn:
    Value:
      Ref: RayanSerratusDlComputeEnvironment
  BatchProcessingJobQueueArn:
    Value:
      Ref: RayanSerratusDlBatchProcessingJobQueue
  BatchProcessingJobDefinitionArn:
    Value:
      Ref: RayanSerratusDlBatchProcessingJobDefinition