diff --git a/singularity/Makefile b/singularity/Makefile index 461090b95cfac5d8e15657b9d017e0f8c3e32daa..85d77a67903fce852886c168154370cae90a8b3d 100644 --- a/singularity/Makefile +++ b/singularity/Makefile @@ -19,5 +19,8 @@ install: run_pipeline run_pipeline.sh workflows_shell.sh # TODO: Bind home on temporary directory at build time in order to decrease "pollution" risks? # Should depend on the rest of the repository. -run_pipeline: run_pipeline.def - nohup sh -c "yes | sudo singularity build run_pipeline run_pipeline.def > build.err 2>&1" & +run_pipeline: run_pipeline.def workflows_base.sif + sh -c "yes | sudo singularity build run_pipeline run_pipeline.def | tee build.err 2>&1" + +workflows_base.sif: workflows_base.def + sh -c "yes | sudo singularity build workflows_base.sif workflows_base.def | tee build_base.err 2>&1" diff --git a/singularity/run_pipeline.def b/singularity/run_pipeline.def index 93e9672516a7a0666138f02895dc6cccff63c62a..d7a5ac581eff30f36275f14fda3c71da0b46ae27 100644 --- a/singularity/run_pipeline.def +++ b/singularity/run_pipeline.def @@ -1,121 +1,28 @@ -Bootstrap:docker -From:debian:buster-slim -Stage: build-haskell +Bootstrap:localimage +From:workflows_base.sif %post - apt-get update -y - apt-get install -y curl libtinfo-dev - mkdir -p /root/.local/bin - export PATH="/root/.local/bin:${PATH}" - export PATH="/usr/local/bin:${PATH}" - curl -sSL https://get.haskellstack.org/ | sh - stack upgrade - mkdir -p /usr/local/src - cd /usr/local/src - # Getting the workflows - cd /usr/local/src - git clone https://gitlab+deploy-token-75:sakyTxfe-PxPHDwqsoGm@gitlab.pasteur.fr/bli/bioinfo_utils.git - cd bioinfo_utils - git submodule update --init --remote --merge - # Note that currently only remove-duplicates-from-sorted-fastq - # and trim-t-tail-from-fastq are copied to the next stage - has_haskell_install=$(find . -name install.sh -print | xargs dirname | grep "Haskell") - for dir in ${has_haskell_install} - do - (cd ${dir} && grep "local" install.sh && ./install.sh) - done - -Bootstrap:docker -From:python:3.7-buster - -%files from build-haskell - /usr/local/bin/remove-duplicates-from-sorted-fastq - /usr/local/bin/trim-t-tail-from-fastq - -%post - apt-get -y update - # Useful for interactive work using the container - apt-get -y install vim - # To be able to generate rule graphs of the workflows - apt-get -y install graphviz - # run_pipeline.sh (and maybe other tools) needs bc - apt-get -y install bc rsync - apt-get -y install bedops bedtools bowtie2 cutadapt subread hisat2 parallel samtools - # Some programs are not provided by debian - # mkdir -p /usr/local/src - ##################### - # Installing bioawk # - ##################### - cd /usr/local/src - git clone https://github.com/lh3/bioawk.git - cd bioawk - apt install -y byacc - make - ln -s /usr/local/src/bioawk/bioawk /usr/local/bin/. - ########################## - # Installing fastq-tools # - ########################## - cd /usr/local/src - wget --continue http://homes.cs.washington.edu/~dcjones/fastq-tools/fastq-tools-0.8.tar.gz - tar -xzf fastq-tools-0.8.tar.gz - cd fastq-tools-0.8 - ./configure - make - make install - ######################### - # Installing kent utils # - ######################### - cd /usr/local/src - #mkdir UCSC-tools - #cd UCSC-tools - rsync -azvP rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/linux.x86_64/bedGraphToBigWig /usr/local/bin/. - # It is possible to test whether the R install is already OK - deb_source="deb http://cran.irsn.fr/bin/linux/debian buster-cran35/" - echo ${deb_source} >> /etc/apt/sources.list.d/cran.list - apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF' - apt-get -y update - apt-get -y install r-base-core r-base-dev - R -e 'install.packages("BiocManager")' - R -e 'BiocManager::install(ask=FALSE)' - R -e 'BiocManager::install(c("docopt", "DESeq2"), ask=FALSE)' # To use the "local" python, not the system one. export PATH="/usr/local/bin":$PATH # To avoid using python things installed in the HOME of root # (that will be mounted during singularity build) export PYTHONNOUSERSITE=1 - python3.7 -m pip install --upgrade pip - # Useful for experimenting in shell mode - python3.7 -m pip install ipython - # Should be installed before pyBedTools for the latter to have numpy support - python3.7 -m pip install numpy - # Needs to install before custom pybedtools - # python3.7 -m pip install pysam - # To get scipy properly built as requirement: - python3.7 -m pip install pybind11 - # Needed to manually cythonize custom pybedtools before installing it - #/usr/bin/env python3 -m pip install Cython - #/usr/bin/env python3 -m pip install --global-option="cythonize" git+https://github.com/blaiseli/pybedtools.git@fix_missing_headers # Getting the workflows cd /usr/local/src #git clone --recurse-submodules https://gitlab+deploy-token-75:sakyTxfe-PxPHDwqsoGm@gitlab.pasteur.fr/bli/bioinfo_utils.git git clone https://gitlab+deploy-token-75:sakyTxfe-PxPHDwqsoGm@gitlab.pasteur.fr/bli/bioinfo_utils.git cd bioinfo_utils pip install -r requirements.txt - #git submodule update --init --remote --merge - #has_requirements=$(find . -name requirements.txt -print | xargs dirname) - ## Needed for scipy (https://stackoverflow.com/a/58534155/1878788) - ## No, does not solve the issue... - ## apt install -y gfortran - #for dir in ${has_requirements} - #do - # (cd ${dir} && pip install -r requirements.txt) - #done - #has_install=$(find . -name install.sh -print | xargs dirname | grep -v "Nim" | grep -v "Haskell") - #for dir in ${has_install} - #do - # (cd ${dir} && ./install.sh) - #done + # Recording program versions + for prog in bedops bedtools bowtie2 cutadapt hisat2 parallel samtools + do + echo "# ${prog}" >> /usr/local/share/doc/program_versions.txt + eval "${prog} --version" >> /usr/local/share/doc/program_versions.txt + done + prog="featureCounts" + echo "# ${prog}" >> /usr/local/share/doc/program_versions.txt + eval "${prog} -v" 2>> /usr/local/share/doc/program_versions.txt %environment export LC_ALL=C diff --git a/singularity/workflows_base.def b/singularity/workflows_base.def new file mode 100644 index 0000000000000000000000000000000000000000..c1221f655d7a1b66c996c5f5e06e1e022d745879 --- /dev/null +++ b/singularity/workflows_base.def @@ -0,0 +1,107 @@ +Bootstrap:docker +From:debian:buster-slim +Stage: build-haskell + +%post + apt-get update -y + apt-get install -y curl libtinfo-dev + mkdir -p /root/.local/bin + export PATH="/root/.local/bin:${PATH}" + export PATH="/usr/local/bin:${PATH}" + curl -sSL https://get.haskellstack.org/ | sh + stack upgrade + mkdir -p /usr/local/src + cd /usr/local/src + # Getting the workflows + cd /usr/local/src + git clone https://gitlab+deploy-token-75:sakyTxfe-PxPHDwqsoGm@gitlab.pasteur.fr/bli/bioinfo_utils.git + cd bioinfo_utils + git submodule update --init --remote --merge + # Note that currently only remove-duplicates-from-sorted-fastq + # and trim-t-tail-from-fastq are copied to the next stage + has_haskell_install=$(find . -name install.sh -print | xargs dirname | grep "Haskell") + for dir in ${has_haskell_install} + do + (cd ${dir} && grep "local" install.sh && ./install.sh) + done + +Bootstrap:docker +From:python:3.7-buster + +%files from build-haskell + /usr/local/bin/remove-duplicates-from-sorted-fastq + /usr/local/bin/trim-t-tail-from-fastq + +%post + apt-get -y update + # Useful for interactive work using the container + apt-get -y install vim + # To be able to generate rule graphs of the workflows + apt-get -y install graphviz + # run_pipeline.sh (and maybe other tools) needs bc + apt-get -y install bc rsync + apt-get -y install bedops bedtools bowtie2 cutadapt subread hisat2 parallel samtools + # Some programs are not provided by debian + # mkdir -p /usr/local/src + ##################### + # Installing bioawk # + ##################### + cd /usr/local/src + git clone https://github.com/lh3/bioawk.git + cd bioawk + apt install -y byacc + make + ln -s /usr/local/src/bioawk/bioawk /usr/local/bin/. + ########################## + # Installing fastq-tools # + ########################## + cd /usr/local/src + wget --continue http://homes.cs.washington.edu/~dcjones/fastq-tools/fastq-tools-0.8.tar.gz + tar -xzf fastq-tools-0.8.tar.gz + cd fastq-tools-0.8 + ./configure + make + make install + ######################### + # Installing kent utils # + ######################### + cd /usr/local/src + #mkdir UCSC-tools + #cd UCSC-tools + rsync -azvP rsync://hgdownload.soe.ucsc.edu/genome/admin/exe/linux.x86_64/bedGraphToBigWig /usr/local/bin/. + # It is possible to test whether the R install is already OK + deb_source="deb http://cran.irsn.fr/bin/linux/debian buster-cran35/" + echo ${deb_source} >> /etc/apt/sources.list.d/cran.list + apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF' + apt-get -y update + apt-get -y install r-base-core r-base-dev + R -e 'install.packages("BiocManager")' + R -e 'BiocManager::install(ask=FALSE)' + R -e 'BiocManager::install(c("docopt", "DESeq2"), ask=FALSE)' + + # To use the "local" python, not the system one. + export PATH="/usr/local/bin":$PATH + # To avoid using python things installed in the HOME of root + # (that will be mounted during singularity build) + export PYTHONNOUSERSITE=1 + python3.7 -m pip install --upgrade pip + # Useful for experimenting in shell mode + python3.7 -m pip install ipython + # Should be installed before pyBedTools for the latter to have numpy support + python3.7 -m pip install numpy + # Needs to install before custom pybedtools + # python3.7 -m pip install pysam + # To get scipy properly built as requirement: + python3.7 -m pip install pybind11 + +%environment + export LC_ALL=C + # To use the "local" python, not the system one. + export PATH="/usr/local/bin":$PATH + # To avoid using python things installed in the HOME of the user + # (that will be mounted during container execution) + export PYTHONNOUSERSITE=1 + export PATH=/usr/local/src/bioinfo_utils:"${PATH}" + +%runscript + exec "$@"