Skip to content
Snippets Groups Projects
Commit 74013222 authored by Blaise Li's avatar Blaise Li
Browse files

Build container in 2 stages.

parent 4a829491
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
singularity/Makefile
+ 5
− 2
View file @ 74013222
......@@ -19,5 +19,8 @@ install: run_pipeline run_pipeline.sh workflows_shell.sh
# TODO: Bind home on temporary directory at build time in order to decrease "pollution" risks?
# Should depend on the rest of the repository.
run_pipeline: run_pipeline.def
nohup sh -c "yes | sudo singularity build run_pipeline run_pipeline.def > build.err 2>&1" &
run_pipeline: run_pipeline.def workflows_base.sif
sh -c "yes | sudo singularity build run_pipeline run_pipeline.def | tee build.err 2>&1"
workflows_base.sif: workflows_base.def
sh -c "yes | sudo singularity build workflows_base.sif workflows_base.def | tee build_base.err 2>&1"
singularity/run_pipeline.def
+ 11
− 104
View file @ 74013222
Bootstrap:docker
From:debian:buster-slim
Stage: build-haskell
Bootstrap:localimage
From:workflows_base.sif
%post
apt-get update -y
apt-get install -y curl libtinfo-dev
mkdir -p /root/.local/bin
export PATH="/root/.local/bin:${PATH}"
export PATH="/usr/local/bin:${PATH}"
curl -sSL https://get.haskellstack.org/ | sh
stack upgrade
mkdir -p /usr/local/src
cd /usr/local/src
# Getting the workflows
cd /usr/local/src
git clone https://gitlab+deploy-token-75:sakyTxfe-PxPHDwqsoGm@gitlab.pasteur.fr/bli/bioinfo_utils.git
cd bioinfo_utils
git submodule update --init --remote --merge
# Note that currently only remove-duplicates-from-sorted-fastq
# and trim-t-tail-from-fastq are copied to the next stage
has_haskell_install=$(find . -name install.sh -print | xargs dirname | grep "Haskell")
for dir in ${has_haskell_install}
do
(cd ${dir} && grep "local" install.sh && ./install.sh)
done
Bootstrap:docker
From:python:3.7-buster
%files from build-haskell
/usr/local/bin/remove-duplicates-from-sorted-fastq
/usr/local/bin/trim-t-tail-from-fastq
%post
apt-get -y update
# Useful for interactive work using the container
apt-get -y install vim
# To be able to generate rule graphs of the workflows
apt-get -y install graphviz
# run_pipeline.sh (and maybe other tools) needs bc
apt-get -y install bc rsync
apt-get -y install bedops bedtools bowtie2 cutadapt subread hisat2 parallel samtools
# Some programs are not provided by debian
# mkdir -p /usr/local/src
#####################
# Installing bioawk #
#####################
cd /usr/local/src
git clone https://github.com/lh3/bioawk.git
cd bioawk
apt install -y byacc
make
ln -s /usr/local/src/bioawk/bioawk /usr/local/bin/.
##########################
# Installing fastq-tools #
##########################
cd /usr/local/src
wget --continue http://homes.cs.washington.edu/~dcjones/fastq-tools/fastq-tools-0.8.tar.gz
tar -xzf fastq-tools-0.8.tar.gz
cd fastq-tools-0.8
./configure
make
make install
#########################
# Installing kent utils #
#########################
cd /usr/local/src
#mkdir UCSC-tools
#cd UCSC-tools
rsync -azvP rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/linux.x86_64/bedGraphToBigWig /usr/local/bin/.
# It is possible to test whether the R install is already OK
deb_source="deb http://cran.irsn.fr/bin/linux/debian buster-cran35/"
echo ${deb_source} >> /etc/apt/sources.list.d/cran.list
apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF'
apt-get -y update
apt-get -y install r-base-core r-base-dev
R -e 'install.packages("BiocManager")'
R -e 'BiocManager::install(ask=FALSE)'
R -e 'BiocManager::install(c("docopt", "DESeq2"), ask=FALSE)'
# To use the "local" python, not the system one.
export PATH="/usr/local/bin":$PATH
# To avoid using python things installed in the HOME of root
# (that will be mounted during singularity build)
export PYTHONNOUSERSITE=1
python3.7 -m pip install --upgrade pip
# Useful for experimenting in shell mode
python3.7 -m pip install ipython
# Should be installed before pyBedTools for the latter to have numpy support
python3.7 -m pip install numpy
# Needs to install before custom pybedtools
# python3.7 -m pip install pysam
# To get scipy properly built as requirement:
python3.7 -m pip install pybind11
# Needed to manually cythonize custom pybedtools before installing it
#/usr/bin/env python3 -m pip install Cython
#/usr/bin/env python3 -m pip install --global-option="cythonize" git+https://github.com/blaiseli/pybedtools.git@fix_missing_headers
# Getting the workflows
cd /usr/local/src
#git clone --recurse-submodules https://gitlab+deploy-token-75:sakyTxfe-PxPHDwqsoGm@gitlab.pasteur.fr/bli/bioinfo_utils.git
git clone https://gitlab+deploy-token-75:sakyTxfe-PxPHDwqsoGm@gitlab.pasteur.fr/bli/bioinfo_utils.git
cd bioinfo_utils
pip install -r requirements.txt
#git submodule update --init --remote --merge
#has_requirements=$(find . -name requirements.txt -print | xargs dirname)
## Needed for scipy (https://stackoverflow.com/a/58534155/1878788)
## No, does not solve the issue...
## apt install -y gfortran
#for dir in ${has_requirements}
#do
# (cd ${dir} && pip install -r requirements.txt)
#done
#has_install=$(find . -name install.sh -print | xargs dirname | grep -v "Nim" | grep -v "Haskell")
#for dir in ${has_install}
#do
# (cd ${dir} && ./install.sh)
#done
# Recording program versions
for prog in bedops bedtools bowtie2 cutadapt hisat2 parallel samtools
do
echo "# ${prog}" >> /usr/local/share/doc/program_versions.txt
eval "${prog} --version" >> /usr/local/share/doc/program_versions.txt
done
prog="featureCounts"
echo "# ${prog}" >> /usr/local/share/doc/program_versions.txt
eval "${prog} -v" 2>> /usr/local/share/doc/program_versions.txt
%environment
export LC_ALL=C
......
singularity/workflows_base.def 0 → 100644
+ 107
− 0
View file @ 74013222
Bootstrap:docker
From:debian:buster-slim
Stage: build-haskell
%post
apt-get update -y
apt-get install -y curl libtinfo-dev
mkdir -p /root/.local/bin
export PATH="/root/.local/bin:${PATH}"
export PATH="/usr/local/bin:${PATH}"
curl -sSL https://get.haskellstack.org/ | sh
stack upgrade
mkdir -p /usr/local/src
cd /usr/local/src
# Getting the workflows
cd /usr/local/src
git clone https://gitlab+deploy-token-75:sakyTxfe-PxPHDwqsoGm@gitlab.pasteur.fr/bli/bioinfo_utils.git
cd bioinfo_utils
git submodule update --init --remote --merge
# Note that currently only remove-duplicates-from-sorted-fastq
# and trim-t-tail-from-fastq are copied to the next stage
has_haskell_install=$(find . -name install.sh -print | xargs dirname | grep "Haskell")
for dir in ${has_haskell_install}
do
(cd ${dir} && grep "local" install.sh && ./install.sh)
done
Bootstrap:docker
From:python:3.7-buster
%files from build-haskell
/usr/local/bin/remove-duplicates-from-sorted-fastq
/usr/local/bin/trim-t-tail-from-fastq
%post
apt-get -y update
# Useful for interactive work using the container
apt-get -y install vim
# To be able to generate rule graphs of the workflows
apt-get -y install graphviz
# run_pipeline.sh (and maybe other tools) needs bc
apt-get -y install bc rsync
apt-get -y install bedops bedtools bowtie2 cutadapt subread hisat2 parallel samtools
# Some programs are not provided by debian
# mkdir -p /usr/local/src
#####################
# Installing bioawk #
#####################
cd /usr/local/src
git clone https://github.com/lh3/bioawk.git
cd bioawk
apt install -y byacc
make
ln -s /usr/local/src/bioawk/bioawk /usr/local/bin/.
##########################
# Installing fastq-tools #
##########################
cd /usr/local/src
wget --continue http://homes.cs.washington.edu/~dcjones/fastq-tools/fastq-tools-0.8.tar.gz
tar -xzf fastq-tools-0.8.tar.gz
cd fastq-tools-0.8
./configure
make
make install
#########################
# Installing kent utils #
#########################
cd /usr/local/src
#mkdir UCSC-tools
#cd UCSC-tools
rsync -azvP rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/linux.x86_64/bedGraphToBigWig /usr/local/bin/.
# It is possible to test whether the R install is already OK
deb_source="deb http://cran.irsn.fr/bin/linux/debian buster-cran35/"
echo ${deb_source} >> /etc/apt/sources.list.d/cran.list
apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF'
apt-get -y update
apt-get -y install r-base-core r-base-dev
R -e 'install.packages("BiocManager")'
R -e 'BiocManager::install(ask=FALSE)'
R -e 'BiocManager::install(c("docopt", "DESeq2"), ask=FALSE)'
# To use the "local" python, not the system one.
export PATH="/usr/local/bin":$PATH
# To avoid using python things installed in the HOME of root
# (that will be mounted during singularity build)
export PYTHONNOUSERSITE=1
python3.7 -m pip install --upgrade pip
# Useful for experimenting in shell mode
python3.7 -m pip install ipython
# Should be installed before pyBedTools for the latter to have numpy support
python3.7 -m pip install numpy
# Needs to install before custom pybedtools
# python3.7 -m pip install pysam
# To get scipy properly built as requirement:
python3.7 -m pip install pybind11
%environment
export LC_ALL=C
# To use the "local" python, not the system one.
export PATH="/usr/local/bin":$PATH
# To avoid using python things installed in the HOME of the user
# (that will be mounted during container execution)
export PYTHONNOUSERSITE=1
export PATH=/usr/local/src/bioinfo_utils:"${PATH}"
%runscript
exec "$@"
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment