Skip to content
Snippets Groups Projects
Commit cf1ce8af authored by François LAURENT's avatar François LAURENT
Browse files

support for podman, upgrade to julia 1.9, LARVATAGGER_DEFAULT_BRANCH renamed... 

support for podman, upgrade to julia 1.9, LARVATAGGER_DEFAULT_BRANCH renamed LARVATAGGER_BRANCH, TAGGINGBACKENDS_BRANCH, --with-backend MaggotUBA/*
parent 425669db
No related branches found
No related tags found
1 merge request!2Support for podman + more control on git branches
Pipeline #104121 canceled
Stage: test
Hide whitespace changes
Inline Side-by-side
recipes/Dockerfile
+ 7
− 25
View file @ cf1ce8af
FROM julia:1.8.5-bullseye AS base
FROM julia:1.9.0-bullseye AS base
ARG PROJECT_DIR=/app
ARG BRANCH=main
......@@ -35,12 +35,12 @@ RUN apt-get update \
FROM prebuild AS backend
ARG PROJECT_DIR=/app
ARG BRANCH=main
ARG TAGGINGBACKENDS_BRANCH=$BRANCH
ARG BACKEND
RUN if [ "$BACKEND" = "MaggotUBA/20221005-1" ]; then \
cd $PROJECT_DIR \
&& julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$BRANCH\")" \
&& julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$TAGGINGBACKENDS_BRANCH\")" \
&& git clone --depth 1 --no-tags --single-branch -b 20221005 https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter MaggotUBA \
&& cd MaggotUBA \
&& rm -rf .git \
......@@ -50,7 +50,7 @@ RUN if [ "$BACKEND" = "MaggotUBA/20221005-1" ]; then \
&& rm -rf ~/.cache; \
elif [ "$BACKEND" = "MaggotUBA/20221228" ]; then \
cd $PROJECT_DIR \
&& julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$BRANCH\")" \
&& julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$TAGGINGBACKENDS_BRANCH\")" \
&& git clone --depth 1 --no-tags --single-branch -b dev https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter MaggotUBA \
&& cd MaggotUBA \
&& rm -rf .git \
......@@ -64,28 +64,10 @@ RUN if [ "$BACKEND" = "MaggotUBA/20221005-1" ]; then \
&& apt-get autoremove -y wget \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf ~/.cache; \
elif [ "$BACKEND" = "MaggotUBA/20230111" ]; then \
cd $PROJECT_DIR \
&& julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$BRANCH\")" \
&& git clone --depth 1 --no-tags --single-branch -b 20230111 https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter MaggotUBA \
&& cd MaggotUBA \
&& rm -rf .git \
&& poetry install --only main \
&& poetry add "pynvml==11.4.1" \
&& rm -rf ~/.cache; \
elif [ "$BACKEND" = "MaggotUBA/20230129" ]; then \
cd $PROJECT_DIR \
&& julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$BRANCH\")" \
&& git clone --depth 1 --no-tags --single-branch -b 20230129 https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter MaggotUBA \
&& cd MaggotUBA \
&& rm -rf .git \
&& poetry install --only main \
&& poetry add "pynvml==11.4.1" \
&& rm -rf ~/.cache; \
elif [ "$BACKEND" = "MaggotUBA/20230311" ]; then \
elif [ "$(echo $BACKEND | cut -d/ -f1)" = "MaggotUBA" ]; then \
cd $PROJECT_DIR \
&& julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$BRANCH\")" \
&& git clone --depth 1 --no-tags --single-branch -b 20230311 https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter MaggotUBA \
&& julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$TAGGINGBACKENDS_BRANCH\")" \
&& git clone --depth 1 --no-tags --single-branch -b $(echo $BACKEND | cut -d/ -f2) https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter MaggotUBA \
&& cd MaggotUBA \
&& rm -rf .git \
&& poetry install --only main \
......
scripts/larvatagger.sh
+ 60
− 28
View file @ cf1ce8af
#!/bin/bash
cmd=$1; shift
if [ -z "$docker" ]; then
docker=docker
fi
for _ in $(seq $#); do
case $1 in
build|open|import|merge|train|predict|-V|--version|-h|--help|--more-help|--upgrade)
cmd=$1
shift
break
;;
*)
# note: if DOCKER_ARGS is externally defined, it must end with an explicit space
DOCKER_ARGS="${DOCKER_ARGS}$1 "
shift
esac
done
if [ -z "$LARVATAGGER_IMAGE" ]; then
if [ "$cmd" = "build" -o -n "$(docker images | grep '^larvatagger ')" ]; then
if [ "$cmd" = "build" -o -n "$($docker images | grep '^larvatagger ')" ]; then
LARVATAGGER_IMAGE=larvatagger
else
LARVATAGGER_IMAGE=flaur/larvatagger
......@@ -36,6 +52,7 @@ elif [ "${1:0:15}" = "--with-backend=" ]; then
DOCKER_ARGS="--build-arg BACKEND=${1:15} $DOCKER_ARGS"; shift
TARGET=backend
elif [ "$1" = "--no-cache" ]; then
echo "Deprecation warning: argument $1 should now be passed before build"
DOCKER_ARGS="--no-cache $DOCKER_ARGS"; shift
else
echo "argument not supported: $1"; shift
......@@ -47,24 +64,34 @@ if [ "$BUILD" == "--dev" ]; then
if ! [[ "$LARVATAGGER_IMAGE" == *:* ]]; then LARVATAGGER_IMAGE="${LARVATAGGER_IMAGE}:dev"; fi
PROJECT_ROOT=$(basename $(pwd))
cd ..
DOCKER_BUILDKIT=1 docker build -t "$LARVATAGGER_IMAGE" -f "$PROJECT_ROOT/recipes/Dockerfile.local" ${DOCKER_ARGS}.
DOCKER_BUILDKIT=1 $docker build -t "$LARVATAGGER_IMAGE" -f "$PROJECT_ROOT/recipes/Dockerfile.local" ${DOCKER_ARGS}.
elif [ "$BUILD" == "--stable" ]; then
if ! [[ "$LARVATAGGER_IMAGE" == *:* ]]; then LARVATAGGER_IMAGE="${LARVATAGGER_IMAGE}:stable"; fi
docker build -t "$LARVATAGGER_IMAGE" -f recipes/Dockerfile ${DOCKER_ARGS}.
$docker build -t "$LARVATAGGER_IMAGE" -f recipes/Dockerfile ${DOCKER_ARGS}.
else
if ! [[ "$LARVATAGGER_IMAGE" == *:* ]]; then LARVATAGGER_IMAGE="${LARVATAGGER_IMAGE}:latest"; fi
if [ -z "$LARVATAGGER_DEFAULT_BRANCH" ]; then LARVATAGGER_DEFAULT_BRANCH=dev; fi
docker build -t "$LARVATAGGER_IMAGE" -f recipes/Dockerfile ${DOCKER_ARGS}--build-arg BRANCH=$LARVATAGGER_DEFAULT_BRANCH .
if [ -z "$LARVATAGGER_BRANCH" ]; then
if [ -z "$LARVATAGGER_DEFAULT_BRANCH" ]; then
LARVATAGGER_BRANCH=dev;
else
echo "Deprecation notice: LARVATAGGER_DEFAULT_BRANCH has been renamed LARVATAGGER_BRANCH"
LARVATAGGER_BRANCH=$LARVATAGGER_DEFAULT_BRANCH
fi
fi
if [ -n "$TAGGINGBACKENDS_BRANCH" ]; then
DOCKER_ARGS="--build-arg TAGGINGBACKENDS_BRANCH=$TAGGINGBACKENDS_BRANCH $DOCKER_ARGS"
fi
$docker build -t "$LARVATAGGER_IMAGE" -f recipes/Dockerfile ${DOCKER_ARGS}--build-arg BRANCH=$LARVATAGGER_BRANCH .
fi
;;
open)
if [ -z "$LARVATAGGER_PORT" -o "$LARVATAGGER_PORT" == "9284" ]; then
DOCKER_ARGS="-p 9284:9284"
DOCKER_ARGS="-p 9284:9284 $DOCKER_ARGS"
TAGGER_ARGS=
else
DOCKER_ARGS="-p $LARVATAGGER_PORT:$LARVATAGGER_PORT"
DOCKER_ARGS="-p $LARVATAGGER_PORT:$LARVATAGGER_PORT $DOCKER_ARGS"
TAGGER_ARGS="--port=$LARVATAGGER_PORT"
fi
......@@ -85,7 +112,7 @@ fi
RUN_ARGS="$RUN_ARGS --mount type=bind,src=\"$(realpath $instance)\",dst=/app/$backend/models/$(basename $instance)"
done
eval "exec docker run $RUN_ARGS -i $DOCKER_ARGS \"$LARVATAGGER_IMAGE\" open \"/data/$file\" $TAGGER_ARGS $@"
eval "exec $docker run $RUN_ARGS -i ${DOCKER_ARGS}\"$LARVATAGGER_IMAGE\" open \"/data/$file\" $TAGGER_ARGS $@"
;;
import | merge)
......@@ -95,7 +122,7 @@ file=$(basename "$1"); shift
RUN_ARGS="$RUN_ARGS -v \"$parentdir\":/data"
eval "docker run $RUN_ARGS \"$LARVATAGGER_IMAGE\" $cmd \"/data/$file\" $@"
eval "$docker run $DOCKER_ARGS$RUN_ARGS \"$LARVATAGGER_IMAGE\" $cmd \"/data/$file\" $@"
;;
train)
......@@ -121,8 +148,10 @@ backend=$2; shift 2
elif [ "${1:0:10}" = "--backend=" ]; then
backend="${1:10}"; shift
elif [ "$1" = "--gpus" -o "$1" == "--cpus" -o "$1" = "-m" -o "$1" = "--memory" ]; then
echo "Deprecation warning: argument $1 should now be passed before train"
RUN_ARGS="$RUN_ARGS $1 $2"; shift 2
elif [ "$1" = "--rm" -o "${1:0:7}" = "--gpus=" -o "${1:0:7}" = "--cpus=" -o "${1:0:9}" = "--memory=" ]; then
echo "Deprecation warning: arguments --rm|--cpus|--gpus|--memory should now be passed before train"
RUN_ARGS="$RUN_ARGS $1"; shift
else
break
......@@ -135,7 +164,7 @@ src=$(cd "$external_models_dir"; pwd -P)
RUN_ARGS="$RUN_ARGS --mount type=bind,src=\"$src\",dst=/app/$backend/models"
fi
DOCKER_RUN="docker run $RUN_ARGS \"$LARVATAGGER_IMAGE\" train \"/app/$backend\" \"/data/$data_repository\" \"$tagger\" $@"
DOCKER_RUN="$docker run $DOCKER_ARGS$RUN_ARGS \"$LARVATAGGER_IMAGE\" train \"/app/$backend\" \"/data/$data_repository\" \"$tagger\" $@"
echo $DOCKER_RUN
eval $DOCKER_RUN
;;
......@@ -168,8 +197,10 @@ backend=$2; shift 2
elif [ "${1:0:10}" = "--backend=" ]; then
backend="${1:10}"; shift
elif [ "$1" = "--gpus" -o "$1" == "--cpus" -o "$1" = "-m" -o "$1" = "--memory" ]; then
echo "Deprecation warning: argument $1 should now be passed before predict"
RUN_ARGS="$RUN_ARGS $1 $2"; shift 2
elif [ "$1" = "--rm" -o "${1:0:7}" = "--gpus=" -o "${1:0:7}" = "--cpus=" -o "${1:0:9}" = "--memory=" ]; then
echo "Deprecation warning: arguments --rm|--cpus|--gpus|--memory should now be passed before predict"
RUN_ARGS="$RUN_ARGS $1"; shift
else
break
......@@ -185,7 +216,7 @@ fi
RUN_ARGS="$RUN_ARGS --mount type=bind,src=\"$(realpath $tagger_path)\",dst=/app/$backend/models/$tagger"
fi
DOCKER_RUN="docker run $RUN_ARGS \"$LARVATAGGER_IMAGE\" predict \"/app/$backend\" \"$tagger\" \"/data/$data_file\" $@"
DOCKER_RUN="$docker run $DOCKER_ARGS$RUN_ARGS \"$LARVATAGGER_IMAGE\" predict \"/app/$backend\" \"$tagger\" \"/data/$data_file\" $@"
echo $DOCKER_RUN
eval $DOCKER_RUN
;;
......@@ -197,35 +228,36 @@ echo 'larvatagger.jl extra arguments must come after larvatagger.sh arguments.'
echo 'Printing larvatagger.jl --help...'
echo
docker run "$LARVATAGGER_IMAGE" --help
$docker run ${DOCKER_ARGS}"$LARVATAGGER_IMAGE" --help
;;
-V|--version)
docker run "$LARVATAGGER_IMAGE" --version
$docker run ${DOCKER_ARGS}"$LARVATAGGER_IMAGE" --version
;;
--update)
if [ -n "$(docker images | grep '^larvatagger latest ')" ]; then
docker rmi -f larvatagger:latest
if [ -n "$($docker images | grep '^larvatagger latest ')" ]; then
$docker rmi ${DOCKER_ARGS}-f larvatagger:latest
fi
docker pull flaur/larvatagger:latest
$docker pull ${DOCKER_ARGS}flaur/larvatagger:latest
;;
*)
echo "usage: $0 build [--stable] [--with-default-backend]"
echo " $0 open <filepath>"
echo " $0 import <filepath> [<outputfilename>] [--id=<runid>] [--framerate=<fps>]"
echo " $0 train <datarepository> <taggername> [--backend <name>]"
echo " [--rm] [--cpus <c>] [--gpus <g>] [--memory <m>]"
echo " $0 predict <datafile> [--backend <name>] [--model-instance <taggername>]"
echo " [--rm] [--cpus <c>] [--gpus <g>] [--memory <m>]"
echo " $0 merge <filepath> [<outputfilename>]"
echo " $0 --more-help"
echo " $0 --version"
echo " $0 --update"
cat << EOT
Usage: $0 build [--stable] [--with-default-backend] [--with-backend <backend>]
$0 open <filepath> [...]
$0 import <filepath> [<outputfilename>] [...]
$0 train <datarepository> <taggername> [--backend <name>] [...]
$0 predict <datafile> [--backend <name>] [--model-instance <taggername>] [...]
$0 merge <filepath> [<outputfilename>] [...]
$0 --more-help
$0 --version
$0 --update
See --more-help for more information about additional options [...] to larvatagger.jl
EOT
;;
esac
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment