Commit 0230a29b authored by Rayan  CHIKHI's avatar Rayan CHIKHI
Browse files

tweaks

parent aea8299d
......@@ -129,12 +129,12 @@ Resources:
- .dkr.ecr.
- Ref: AWS::Region
- ".amazonaws.com/aws-batch-s3-unitigs-job:latest"
Vcpus: 2
Vcpus: 4
Memory: 7000
Command:
- python
- batch_processor.py
MountPoints:
MountPoints:
- ContainerPath: /tmp
SourceVolume: temp_dir
Volumes:
......@@ -178,14 +178,15 @@ Resources:
Version: !GetAtt SpecialComputeLaunchTemplate.LatestVersionNumber
ServiceRole:
Ref: BatchServiceRole
SpecialComputeLaunchTemplate: # https://github.com/vfrank66/awsbatchlaunchtemplate/blob/master/aws-batch-launch-ami.yaml
Type: AWS::EC2::LaunchTemplate
Properties:
LaunchTemplateName: "Special-inc-nvme"
LaunchTemplateData:
BlockDeviceMappings:
- DeviceName: /dev/xvdcz
VirtualName: ephemeral0
BlockDeviceMappings:
- DeviceName: /dev/xvdcz
VirtualName: ephemeral0
UserData:
"Fn::Base64": !Sub |
MIME-Version: 1.0
......@@ -194,39 +195,40 @@ Resources:
--==MYBOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash
# from https://forums.aws.amazon.com/message.jspa?messageID=867011
yum install -y rsync
# mount the ephemeral storage
mkfs.ext4 /dev/nvme1n1
mount -t ext4 /dev/nvme1n1 /mnt/
#!/bin/bash
# from https://forums.aws.amazon.com/message.jspa?messageID=867011
yum install -y rsync
# mount the ephemeral storage
mkfs.ext4 /dev/nvme1n1
mount -t ext4 /dev/nvme1n1 /mnt/
# make temp directory for containers usage
# should be used in the Batch job definition (MountPoints)
mkdir /mnt/tmp_ext
rsync -avPHSX /tmp/ /mnt/tmp_ext/
# modify fstab to mount /tmp on the new storage.
sed -i '$ a /mnt/tmp_ext /tmp none bind 0 0' /etc/fstab
mount -a
# make /tmp usable by everyone
chmod 777 /mnt/tmp_ext
# make temp directory for containers usage
# should be used in the Batch job definition (MountPoints)
mkdir /mnt/tmp_ext
rsync -avPHSX /tmp/ /mnt/tmp_ext/
# modify fstab to mount /tmp on the new storage.
sed -i '$ a /mnt/tmp_ext /tmp none bind 0 0' /etc/fstab
mount -a
# make /tmp usable by everyone
chmod 777 /mnt/tmp_ext
service docker stop
# copy the docker directory to the ephemeral storage
rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/
# set the data directory to the ephemeral storage in the config file of the docker deamon
DOCKER_CFG_FILE=/etc/docker/daemon.json
if [ ! -e "${DOCKER_CFG_FILE}" ]; then
# need to create a non empty file for sed to work
echo "{" > ${DOCKER_CFG_FILE}
else
# replace the last } of the file by a ,
sed -i s/}$/,/ ${DOCKER_CFG_FILE}
fi
sed -i '$ a "data-root": "/mnt/docker_ext/"' ${DOCKER_CFG_FILE}
sed -i '$ a }' ${DOCKER_CFG_FILE}
service docker start
--==MYBOUNDARY==
service docker stop
# copy the docker directory to the ephemeral storage
rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/
# set the data directory to the ephemeral storage in the config file of the docker deamon
DOCKER_CFG_FILE=/etc/docker/daemon.json
if [ ! -e "${!DOCKER_CFG_FILE}" ]; then
# need to create a non empty file for sed to work
echo "{" > ${!DOCKER_CFG_FILE}
else
# replace the last } of the file by a ,
sed -i s/}$/,/ ${!DOCKER_CFG_FILE}
fi
sed -i '$ a "data-root": "/mnt/docker_ext/"' ${!DOCKER_CFG_FILE}
sed -i '$ a }' ${!DOCKER_CFG_FILE}
service docker start
--==MYBOUNDARY==--
BatchProcessS3Bucket:
Type: AWS::S3::Bucket
......@@ -309,28 +311,6 @@ Resources:
print("Job ID is {}.".format(response['jobId']))
return response
BatchProcessRepository:
Type: AWS::ECR::Repository
Properties:
RepositoryName: "aws-batch-s3-unitigs-job"
RepositoryPolicyText:
Version: "2012-10-17"
Statement:
-
Sid: AllowPushPull
Effect: Allow
Principal:
AWS:
- !Sub arn:aws:iam::${AWS::AccountId}:role/${EcsInstanceRole}
Action:
- "ecr:GetDownloadUrlForLayer"
- "ecr:BatchGetImage"
- "ecr:BatchCheckLayerAvailability"
- "ecr:PutImage"
- "ecr:InitiateLayerUpload"
- "ecr:UploadLayerPart"
- "ecr:CompleteLayerUpload"
Outputs:
ComputeEnvironmentArn:
Value:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment