Commit ebc6cc21 authored by Rayan  CHIKHI's avatar Rayan CHIKHI
Browse files

fix to properly work in /tmp

parent 0230a29b
FROM python
# https://pythonspeed.com/articles/alpine-docker-python/
WORKDIR /tmp
WORKDIR /
COPY batch_processor.py ./
COPY batch_processor.py .
RUN pip install --upgrade pip && \
pip install boto3 awscli
......
......@@ -23,19 +23,25 @@ def process_file(inputBucket, fileName, region, delete_original=False):
prefix = fileName
print("region - " + region)
startTime = datetime.now()
# go to /tmp (important, that's where local storage / nvme is)
os.chdir("/tmp")
# check free space
os.system(' '.join(["df", "-h", "."]))
# download reads from s3
local_file = str(fileName)
s3.download_file(inputBucket, fileName, local_file)
print("downloaded file to",local_file)
# run bcalm
os.system(' '.join(["./bcalm", "-kmer-size", "31", "-in", local_file, "-skip-links"]))
os.system(' '.join(["../bcalm", "-kmer-size", "31", "-in", local_file, "-skip-links"]))
unitigs_filename = '.'.join(local_file.split('.')[:-1])+".unitigs.fa"
# run mfc
os.system(' '.join(["./MFCompressC",unitigs_filename]))
os.system(' '.join(["../MFCompressC",unitigs_filename]))
compressed_unitigs_filename = unitigs_filename + ".mfc"
......
......@@ -131,9 +131,6 @@ Resources:
- ".amazonaws.com/aws-batch-s3-unitigs-job:latest"
Vcpus: 4
Memory: 7000
Command:
- python
- batch_processor.py
MountPoints:
- ContainerPath: /tmp
SourceVolume: temp_dir
......@@ -163,7 +160,7 @@ Resources:
MaxvCpus: 4
#AllocationStrategy: SPOT_CAPACITY_OPTIMIZED # maybe let's not activate cause i really want a c5d and nothing else
InstanceTypes:
- c5d.xlarge
- c5d
#- optimal
BidPercentage: 100
SpotIamFleetRole: !Ref SpotIamFleetRole
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment