template.yaml 6.72 KB
Newer Older
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
---
AWSTemplateFormatVersion: '2010-09-09'
Description: 'Orchestrating an Application Process with AWS Batch using CloudFormation'
Resources:
  VPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: 10.0.0.0/16
  InternetGateway:
    Type: AWS::EC2::InternetGateway
  RouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId:
        Ref: VPC
  VPCGatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId:
        Ref: VPC
      InternetGatewayId:
        Ref: InternetGateway
  SecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: EC2 Security Group for instances launched in the VPC by Batch
      VpcId:
        Ref: VPC
  Subnet:
    Type: AWS::EC2::Subnet
    Properties:
      CidrBlock: 10.0.0.0/24
      VpcId:
        Ref: VPC
      MapPublicIpOnLaunch: 'True'
  Route:
    Type: AWS::EC2::Route
    Properties:
      RouteTableId:
        Ref: RouteTable
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId:
        Ref: InternetGateway
  SubnetRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId:
        Ref: RouteTable
      SubnetId:
        Ref: Subnet
  BatchServiceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service: batch.amazonaws.com
          Action: sts:AssumeRole
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole
  IamInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
      - Ref: EcsInstanceRole
  EcsInstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2008-10-17'
        Statement:
        - Sid: ''
          Effect: Allow
          Principal:
            Service: ec2.amazonaws.com
          Action: sts:AssumeRole
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role
      - arn:aws:iam::aws:policy/AmazonS3FullAccess

  SpotIamFleetRole: # taken from https://github.com/aodn/aws-wps/blob/master/wps-cloudformation-template.yaml
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service: spot.amazonaws.com
            Action: sts:AssumeRole
          - Effect: Allow
            Principal:
              Service: spotfleet.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole
  RayanSerratusDlBatchProcessingJobDefinition:
    Type: AWS::Batch::JobDefinition
    Properties:
      Type: container
      JobDefinitionName: RayanSerratusDlBatchJobDefinition
      ContainerProperties:
        Image:
          Fn::Join:
          - ''
          - - Ref: AWS::AccountId
            - .dkr.ecr.
            - Ref: AWS::Region
Rayan  CHIKHI's avatar
fix    
Rayan CHIKHI committed
111
112
113
            - ".amazonaws.com/serratus-dl-batch-job:latest"
        Vcpus: 2
        Memory: 1000
Rayan  CHIKHI's avatar
Rayan CHIKHI committed
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
        MountPoints:
          - ContainerPath: /tmp
            SourceVolume: temp_dir
        Volumes:
          - Host:
              SourcePath: /tmp
            Name: temp_dir
      RetryStrategy:
        Attempts: 1
  RayanSerratusDlBatchProcessingJobQueue:
    Type: AWS::Batch::JobQueue
    Properties:
      JobQueueName: RayanSerratusDlBatchProcessingJobQueue
      Priority: 1
      ComputeEnvironmentOrder:
      - Order: 1
        ComputeEnvironment:
          Ref: RayanSerratusDlComputeEnvironment
  RayanSerratusDlComputeEnvironment:
    Type: AWS::Batch::ComputeEnvironment
    Properties:
      Type: MANAGED
      ComputeResources:
        Type: SPOT
        MinvCpus: 0
        DesiredvCpus: 0
        MaxvCpus: 4
        #AllocationStrategy: SPOT_CAPACITY_OPTIMIZED # maybe let's not activate cause i really want a c5d and nothing else
        InstanceTypes:
        - c5d
        #- optimal
        BidPercentage: 100
        SpotIamFleetRole: !Ref SpotIamFleetRole
        Subnets:
        - Ref: Subnet
        SecurityGroupIds:
        - Ref: SecurityGroup
        InstanceRole:
          Ref: IamInstanceProfile
        LaunchTemplate:
          LaunchTemplateId: !Ref SpecialComputeLaunchTemplate
          Version: !GetAtt SpecialComputeLaunchTemplate.LatestVersionNumber
      ServiceRole:
        Ref: BatchServiceRole

  SpecialComputeLaunchTemplate: # https://github.com/vfrank66/awsbatchlaunchtemplate/blob/master/aws-batch-launch-ami.yaml
    Type: AWS::EC2::LaunchTemplate
    Properties:
      LaunchTemplateName: "Special-inc-nvme-assembly"
      LaunchTemplateData:
        BlockDeviceMappings:
          - DeviceName: /dev/xvdcz
            VirtualName: ephemeral0
        UserData:
         "Fn::Base64": !Sub |
            MIME-Version: 1.0
            Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="

            --==MYBOUNDARY==
            Content-Type: text/x-shellscript; charset="us-ascii"

            #!/bin/bash
            # from https://forums.aws.amazon.com/message.jspa?messageID=867011
            yum install -y rsync
            # mount the ephemeral storage
            mkfs.ext4 /dev/nvme1n1
            mount -t ext4 /dev/nvme1n1 /mnt/ 

            # make temp directory for containers usage
            # should be used in the Batch job definition (MountPoints)
            mkdir /mnt/tmp_ext 
            rsync -avPHSX /tmp/ /mnt/tmp_ext/ 
            # modify fstab to mount /tmp on the new storage.
            sed -i '$ a /mnt/tmp_ext  /tmp  none bind 0 0' /etc/fstab
            mount -a
            # make /tmp usable by everyone
            chmod 777 /mnt/tmp_ext

            service docker stop  
            # copy the docker directory to the ephemeral storage
            rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/ 
            # set the data directory to the ephemeral storage in the config file of the docker deamon 
            DOCKER_CFG_FILE=/etc/docker/daemon.json
            if [ ! -e "${!DOCKER_CFG_FILE}" ]; then
                # need to create a non empty file for sed to work
                echo "{" > ${!DOCKER_CFG_FILE}
            else
                # replace the last } of the file by a ,
                sed -i s/}$/,/ ${!DOCKER_CFG_FILE}
            fi
            sed -i '$ a "data-root": "/mnt/docker_ext/"' ${!DOCKER_CFG_FILE}
            sed -i '$ a }' ${!DOCKER_CFG_FILE}
            service docker start

            --==MYBOUNDARY==--

Outputs:
  ComputeEnvironmentArn:
    Value:
      Ref: RayanSerratusDlComputeEnvironment
  BatchProcessingJobQueueArn:
    Value:
      Ref: RayanSerratusDlBatchProcessingJobQueue
  BatchProcessingJobDefinitionArn:
    Value:
      Ref: RayanSerratusDlBatchProcessingJobDefinition