Commit 1a8d34d7 authored by Rayan  CHIKHI's avatar Rayan CHIKHI
Browse files

hail mary to use c5d.* local space

parent 2d0b1402
......@@ -16,6 +16,7 @@ Execute the below commands to spin up the infrastructure cloudformation stack.
```
./deploy-docker.sh
./create_tools_bucket.sh
./spinup.sh
```
......
FROM python
# https://pythonspeed.com/articles/alpine-docker-python/
COPY batch_processor.py /
WORKDIR /tmp
COPY batch_processor.py ./
RUN pip install --upgrade pip && \
pip install boto3 awscli
......@@ -27,6 +29,7 @@ RUN tar xf MFCompress-linux64-1.01.tgz && rm MFCompress-linux64-1.01.tgz
RUN mv MFCompress-linux64-1.01/MFCompressC ./
RUN pwd
RUN df -h .
RUN ls
#CMD ["python", "batch_processor.py"]
......@@ -134,6 +134,13 @@ Resources:
Command:
- python
- batch_processor.py
MountPoints:
- ContainerPath: /tmp
SourceVolume: temp_dir
Volumes:
- Host:
SourcePath: /tmp
Name: temp_dir
RetryStrategy:
Attempts: 1
BatchProcessingJobQueue:
......@@ -166,8 +173,61 @@ Resources:
- Ref: SecurityGroup
InstanceRole:
Ref: IamInstanceProfile
LaunchTemplate:
LaunchTemplateId: !Ref SpecialComputeLaunchTemplate
Version: !GetAtt SpecialComputeLaunchTemplate.LatestVersionNumber
ServiceRole:
Ref: BatchServiceRole
SpecialComputeLaunchTemplate: # https://github.com/vfrank66/awsbatchlaunchtemplate/blob/master/aws-batch-launch-ami.yaml
Type: AWS::EC2::LaunchTemplate
Properties:
LaunchTemplateName: "Special-inc-nvme"
LaunchTemplateData:
BlockDeviceMappings:
- DeviceName: /dev/xvdcz
VirtualName: ephemeral0
UserData:
"Fn::Base64": !Sub |
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="
--==MYBOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash
# from https://forums.aws.amazon.com/message.jspa?messageID=867011
yum install -y rsync
# mount the ephemeral storage
mkfs.ext4 /dev/nvme1n1
mount -t ext4 /dev/nvme1n1 /mnt/
# make temp directory for containers usage
# should be used in the Batch job definition (MountPoints)
mkdir /mnt/tmp_ext
rsync -avPHSX /tmp/ /mnt/tmp_ext/
# modify fstab to mount /tmp on the new storage.
sed -i '$ a /mnt/tmp_ext /tmp none bind 0 0' /etc/fstab
mount -a
# make /tmp usable by everyone
chmod 777 /mnt/tmp_ext
service docker stop
# copy the docker directory to the ephemeral storage
rsync -avPHSX /var/lib/docker/ /mnt/docker_ext/
# set the data directory to the ephemeral storage in the config file of the docker deamon
DOCKER_CFG_FILE=/etc/docker/daemon.json
if [ ! -e "${DOCKER_CFG_FILE}" ]; then
# need to create a non empty file for sed to work
echo "{" > ${DOCKER_CFG_FILE}
else
# replace the last } of the file by a ,
sed -i s/}$/,/ ${DOCKER_CFG_FILE}
fi
sed -i '$ a "data-root": "/mnt/docker_ext/"' ${DOCKER_CFG_FILE}
sed -i '$ a }' ${DOCKER_CFG_FILE}
service docker start
--==MYBOUNDARY==
BatchProcessS3Bucket:
Type: AWS::S3::Bucket
DependsOn: BatchProcessBucketPermission
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment