From 3abb649bead90e1072b45bc11dcf9ca19191eea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20=20LAURENT?= <francois.laurent@pasteur.fr> Date: Thu, 5 Jun 2025 17:50:04 +0200 Subject: [PATCH] Set of commits to be tagged v0.20.2 --- Manifest-v1.11.toml | 14 ++-- Manifest.toml | 10 +-- Project.toml | 2 +- README.md | 5 +- recipes/Dockerfile | 11 ++- recipes/Dockerfile.local | 6 +- scripts/larvatagger.sh | 26 ++++--- test/rest_server.sh | 150 ++++++++++++++++++++++++++------------- 8 files changed, 140 insertions(+), 84 deletions(-) diff --git a/Manifest-v1.11.toml b/Manifest-v1.11.toml index 0aa18a8..9e70d5f 100644 --- a/Manifest-v1.11.toml +++ b/Manifest-v1.11.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.4" +julia_version = "1.11.5" manifest_format = "2.0" project_hash = "ba336c605739d728e2989784d94690e67b22a25d" @@ -996,12 +996,12 @@ uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.2.0" [[deps.NyxWidgets]] -deps = ["Bonito", "Colors", "Format", "LazyArtifacts", "Observables"] -git-tree-sha1 = "cdc029870ac2ab9522e700f3e2b3209b33389fa4" +deps = ["Base64", "Bonito", "Colors", "Format", "LazyArtifacts", "Observables"] +git-tree-sha1 = "b6f181833b439da5fd08e545777b8adc25256d43" repo-rev = "main" repo-url = "https://gitlab.com/dbc-nyx/NyxWidgets.jl" uuid = "c288fd06-43d3-4b04-8307-797133353e2e" -version = "0.2.2" +version = "0.2.3" [[deps.Observables]] git-tree-sha1 = "7438a59546cf62428fc9d1bc94729146d37a7225" @@ -1057,7 +1057,7 @@ version = "3.2.4+0" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" -version = "0.8.1+4" +version = "0.8.5+0" [[deps.OpenMPI_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] @@ -1158,11 +1158,11 @@ version = "0.3.3" [[deps.PlanarLarvae]] deps = ["DelimitedFiles", "HDF5", "JSON3", "LinearAlgebra", "MAT", "Meshes", "OrderedCollections", "Random", "SHA", "StaticArrays", "Statistics", "StatsBase", "StructTypes"] -git-tree-sha1 = "d964d040e319fe3bd9140e5bf91d648de6acc96f" +git-tree-sha1 = "a234429c8f57accaecf143762733fb95c0e55304" repo-rev = "main" repo-url = "https://gitlab.pasteur.fr/nyx/PlanarLarvae.jl" uuid = "c2615984-ef14-4d40-b148-916c85b43307" -version = "0.16.0" +version = "0.16.1" [[deps.PlotUtils]] deps = ["ColorSchemes", "Colors", "Dates", "PrecompileTools", "Printf", "Random", "Reexport", "StableRNGs", "Statistics"] diff --git a/Manifest.toml b/Manifest.toml index ddca19f..3b1a79b 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -984,12 +984,12 @@ uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.2.0" [[deps.NyxWidgets]] -deps = ["Bonito", "Colors", "Format", "LazyArtifacts", "Observables"] -git-tree-sha1 = "cdc029870ac2ab9522e700f3e2b3209b33389fa4" +deps = ["Base64", "Bonito", "Colors", "Format", "LazyArtifacts", "Observables"] +git-tree-sha1 = "b6f181833b439da5fd08e545777b8adc25256d43" repo-rev = "main" repo-url = "https://gitlab.com/dbc-nyx/NyxWidgets.jl" uuid = "c288fd06-43d3-4b04-8307-797133353e2e" -version = "0.2.2" +version = "0.2.3" [[deps.Observables]] git-tree-sha1 = "7438a59546cf62428fc9d1bc94729146d37a7225" @@ -1142,11 +1142,11 @@ version = "0.3.3" [[deps.PlanarLarvae]] deps = ["DelimitedFiles", "HDF5", "JSON3", "LinearAlgebra", "MAT", "Meshes", "OrderedCollections", "Random", "SHA", "StaticArrays", "Statistics", "StatsBase", "StructTypes"] -git-tree-sha1 = "d964d040e319fe3bd9140e5bf91d648de6acc96f" +git-tree-sha1 = "a234429c8f57accaecf143762733fb95c0e55304" repo-rev = "main" repo-url = "https://gitlab.pasteur.fr/nyx/PlanarLarvae.jl" uuid = "c2615984-ef14-4d40-b148-916c85b43307" -version = "0.16.0" +version = "0.16.1" [[deps.PlotUtils]] deps = ["ColorSchemes", "Colors", "Dates", "PrecompileTools", "Printf", "Random", "Reexport", "StableRNGs", "Statistics"] diff --git a/Project.toml b/Project.toml index 8074769..5680b33 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "LarvaTagger" uuid = "8b3b36f1-dfed-446e-8561-ea19fe966a4d" authors = ["François Laurent", "Institut Pasteur"] -version = "0.20.1" +version = "0.20.2" [deps] Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8" diff --git a/README.md b/README.md index da5807a..5e4feb3 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,10 @@ If the backend directory is created right in the *LarvaTagger* directory, *Larva Otherwise, to let the *larvaeditor* function know about *MaggotUBA* or any other backend, in the *Julia* interpreter, type: ``` -julia> using LarvaTagger; display(larvaeditor("path/to/data/file"; backend_directory="path/to/MaggotUBA/parent/directory")) +julia> using LarvaTagger +julia> using Bonito +julia> app = larvaeditor("path/to/data/file"; backend_directory="path/to/MaggotUBA/parent/directory"); +julia> Server(app, "0.0.0.0", 9284) ``` Similarly, to let *LarvaTagger* know about *MaggotUBA*: diff --git a/recipes/Dockerfile b/recipes/Dockerfile index c4d6989..fe9a87c 100644 --- a/recipes/Dockerfile +++ b/recipes/Dockerfile @@ -1,9 +1,8 @@ FROM julia:1.10.9-bullseye AS base -ARG PROJECT_DIR=/app ARG BRANCH=main -ENV JULIA_PROJECT=${PROJECT_DIR} \ +ENV JULIA_PROJECT=/app \ JULIA_DEPOT_PATH=/usr/local/share/julia RUN apt-get update \ @@ -46,7 +45,7 @@ RUN if [ -z $TAGGINGBACKENDS_BRANCH ]; then \ TAGGINGBACKENDS_BRANCH=dev; \ fi; \ if [ "$BACKEND" = "MaggotUBA/20221005-1" ]; then \ - cd $PROJECT_DIR \ + cd /app \ && julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$TAGGINGBACKENDS_BRANCH\")" \ && git clone --depth 1 --no-tags --single-branch -b 20221005 https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter MaggotUBA \ && cd MaggotUBA \ @@ -56,7 +55,7 @@ RUN if [ -z $TAGGINGBACKENDS_BRANCH ]; then \ && poetry add "pynvml==11.4.1" \ && rm -rf ~/.cache; \ elif [ "$BACKEND" = "MaggotUBA/20221228" ]; then \ - cd $PROJECT_DIR \ + cd /app \ && julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$TAGGINGBACKENDS_BRANCH\")" \ && git clone --depth 1 --no-tags --single-branch -b dev https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter MaggotUBA \ && cd MaggotUBA \ @@ -72,7 +71,7 @@ RUN if [ -z $TAGGINGBACKENDS_BRANCH ]; then \ && rm -rf /var/lib/apt/lists/* \ && rm -rf ~/.cache; \ elif [ "$(echo $BACKEND | cut -d/ -f1)" = "MaggotUBA" ]; then \ - cd $PROJECT_DIR \ + cd /app \ && julia -e "using Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$TAGGINGBACKENDS_BRANCH\")" \ && git clone --depth 1 --no-tags --single-branch -b $(echo $BACKEND | cut -d/ -f2) https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter MaggotUBA \ && cd MaggotUBA \ @@ -83,7 +82,7 @@ RUN if [ -z $TAGGINGBACKENDS_BRANCH ]; then \ && if [ "$(echo $BACKEND | cut -d/ -f2)" = "main" ] || [ "$(echo $BACKEND | cut -d/ -f2)" = "dev" ]; then \ julia -e 'using Pkg; Pkg.add("JSON3")' \ && scripts/make_models.jl default \ - && cd $PROJECT_DIR \ + && cd /app \ && recipes/patch.sh; \ fi \ && rm -rf ~/.cache; \ diff --git a/recipes/Dockerfile.local b/recipes/Dockerfile.local index 34d31fc..d1d7888 100644 --- a/recipes/Dockerfile.local +++ b/recipes/Dockerfile.local @@ -22,11 +22,11 @@ ARG MAGGOTUBA_CORE=./MaggotUBA-core ARG MAGGOTUBA_ADAPTER=./MaggotUBA-adapter COPY $PLANARLARVAE/src /app/PlanarLarvae/src -COPY $PLANARLARVAE/Project.toml $PLANARLARVAE/Manifest.toml* /app/PlanarLarvae/ +COPY $PLANARLARVAE/Project.toml $PLANARLARVAE/Manifest*.toml /app/PlanarLarvae/ COPY $LARVATAGGER/src /app/src COPY $LARVATAGGER/scripts /app/scripts COPY $LARVATAGGER/recipes/patch.sh /app/recipes/ -COPY $LARVATAGGER/Project.toml $LARVATAGGER/Manifest.toml /app/ +COPY $LARVATAGGER/Project.toml $LARVATAGGER/Manifest*.toml /app/ RUN apt-get update \ && apt-get install -y git python3-pip \ @@ -39,7 +39,7 @@ RUN apt-get update \ COPY $TAGGINGBACKENDS/src /app/TaggingBackends/src COPY $TAGGINGBACKENDS/scripts /app/TaggingBackends/scripts -COPY $TAGGINGBACKENDS/Project.toml $TAGGINGBACKENDS/Manifest.toml $TAGGINGBACKENDS/pyproject.toml /app/TaggingBackends/ +COPY $TAGGINGBACKENDS/Project.toml $TAGGINGBACKENDS/Manifest*.toml $TAGGINGBACKENDS/pyproject.toml /app/TaggingBackends/ COPY $MAGGOTUBA_CORE/src /app/MaggotUBA-core/src COPY $MAGGOTUBA_CORE/pyproject.toml /app/MaggotUBA-core/ COPY $MAGGOTUBA_ADAPTER/src /app/MaggotUBA/src diff --git a/scripts/larvatagger.sh b/scripts/larvatagger.sh index ff32aec..b47d1f0 100755 --- a/scripts/larvatagger.sh +++ b/scripts/larvatagger.sh @@ -75,7 +75,7 @@ case "$cmd" in build) if ! [ -f recipes/Dockerfile ]; then -echo "the build command can only be run from the project root directory" +echo "the build command can only be run from the project root directory" >&2 exit 1 fi TARGET=base @@ -95,7 +95,7 @@ elif [ "$1" = "--no-cache" ]; then echo "Deprecation warning: argument $1 should now be passed before build" shift else -echo "argument not supported: $1"; shift +echo "argument not supported: $1" >&2 exit 1 fi done @@ -171,8 +171,10 @@ backend=MaggotUBA while [ -n "$1" -a "$1" = "--external-instance" ]; do instance=$2; shift 2 if ! command -v realpath &>/dev/null; then -echo "realpath: command not found" -echo "on macOS: brew install coreutils" +cat <<EOF >&2 +realpath: command not found +on macOS: brew install coreutils +EOF exit 1 fi RUN_ARGS="$RUN_ARGS --mount type=bind,src=\"$(realpath $instance)\",dst=/app/$backend/models/$(basename $instance)" @@ -197,8 +199,10 @@ backend=MaggotUBA while [ -n "$1" -a "$1" = "--external-instance" ]; do instance=$2; shift 2 if ! command -v realpath &>/dev/null; then -echo "realpath: command not found" -echo "on macOS: brew install coreutils" +cat <<EOF >&2 +realpath: command not found +on macOS: brew install coreutils +EOF exit 1 fi RUN_ARGS="$RUN_ARGS --mount type=bind,src=\"$(realpath $instance)\",dst=/app/$backend/models/$(basename $instance)" @@ -294,7 +298,7 @@ tagger=$2; shift 2 elif [ "${1:0:17}" = "--model-instance=" ]; then tagger="${1:17}"; shift else -echo "argument not supported: $1" +echo "argument not supported: $1" >&2 exit 1 fi if [ -d "$tagger" ]; then @@ -318,8 +322,10 @@ done if [ -n "$tagger_path" ]; then if ! command -v realpath &>/dev/null; then -echo "realpath: command not found" -echo "on macOS: brew install coreutils" +cat <<EOF >&2 +realpath: command not found +on macOS: brew install coreutils +EOF exit 1 fi RUN_ARGS="$RUN_ARGS --mount type=bind,src=\"$(realpath $tagger_path)\",dst=/app/$backend/models/$tagger" @@ -395,7 +401,7 @@ $docker pull ${DOCKER_ARGS}flaur/larvatagger:latest *) -cat << EOT +cat << EOT >&2 Wrapper script to operate larvatagger.jl in a Docker image. Usage: $0 build [--stable] [--with-default-backend] [--with-backend <backend>] diff --git a/test/rest_server.sh b/test/rest_server.sh index d69c186..6cd6d57 100755 --- a/test/rest_server.sh +++ b/test/rest_server.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -m +set -em this_script=$(realpath "${BASH_SOURCE[0]}") larvatagger_jl_project_root=$(dirname "$(dirname "${this_script}")") @@ -24,41 +24,73 @@ if [ -z "`julia +1.10 -v 2>/dev/null`" ]; then fi # run and background the backend server -JULIA_PROJECT="${larvatagger_project_root}/TaggingBackends" \ - julia +1.10 --project="${larvatagger_jl_project_root}" -iq \ - -e "using LarvaTagger.REST.Server; run_backend(\"${larvatagger_project_root}\"; port=${lt_backend_port})" & -lt_backend_pid=$! +launch_server=`mktemp --suffix ".sh"` || exit 1 +cat <<EOF >${launch_server} +#!/bin/sh +JULIA_PROJECT="${larvatagger_project_root}/TaggingBackends" \\ + julia +1.10 --project="${larvatagger_jl_project_root}" -iq \\ + -e "using LarvaTagger.REST.Server; run_backend(\"${larvatagger_project_root}\"; port=${lt_backend_port})" 2>&1 +EOF +#cat ${launch_server} +chmod +x ${launch_server} +${launch_server} | sed '/ signal (15): Terminated$/ q' & +sleep 1 +rm ${launch_server} +lt_backend_ppid=`ps au | grep -e " ${launch_server}\$" | tr -s ' ' | cut -d' ' -f2` +lt_backend_pid=`ps x -o "%p %r %c" | tr -s ' ' | grep " ${lt_backend_ppid} julia\$" | cut -d' ' -f2` # wait for the server to be ready while [ "`curl -s http://localhost:${lt_backend_port}/status`" != "up" ]; do sleep 1 done -echo "status: success" +success() { + echo -e "\e[32m$1: \e[0m\e[32;1msuccess\e[0m" +} +success "status" + +failure() { + echo -e "\e[31m$1: \e[0m\e[31;1mfailure\e[0m" >&2 +} + +abort() { + echo "tests aborted; terminating the backend" >&2 + kill $lt_backend_pid + exit 1 +} # get a session token for a tagging backend token=`curl -sS http://localhost:${lt_backend_port}/get-token/${tagging_backend}/${tagging_model}` if [ "${token:0:3}" = "jl_" ]; then - echo "get-token: success" + success "get-token" else - echo "get-token: failure" - echo "expected: string starting with \"jl_\"" - echo "got: $token" + failure "get-token" + cat <<EOF >&2 +expected: string starting with \"jl_\" +got: $token +EOF + abort fi # send one text file, one binary file -curl -sS -F "f=@${larvatagger_jl_project_root}/README.md" -F "g=@${larvatagger_project_root}/Artefacts/PlanarLarvae/trx_small.tgz" http://localhost:${lt_backend_port}/push-file/${tagging_backend}/${tagging_model}/${token} 1>/dev/null +curl -sS -F "f=@${larvatagger_jl_project_root}/README.md" \ + -F "g=@${larvatagger_project_root}/Artefacts/PlanarLarvae/trx_small.tgz" \ + http://localhost:${lt_backend_port}/push-file/${tagging_backend}/${tagging_model}/${token} 1>/dev/null -generated_files=(`ls "${larvatagger_project_root}/${tagging_backend}/data/raw/${token}"`) +destination_dir="${larvatagger_project_root}/${tagging_backend}/data/raw/${token}" +generated_files=(`ls "$destination_dir"`) expected_files=("README.md" "trx_small.tgz") -if [ ${#generated_files[@]} -eq ${#expected_files[@]} -a \ - "${generated_files[0]}" = "${expected_files[0]}" -a \ - "${generated_files[1]}" = "${expected_files[1]}" ]; then - echo "push-file: success" +if [ ${#generated_files[@]} -eq ${#expected_files[@]} ] && \ + [ "${generated_files[0]}" = "${expected_files[0]}" -a \ + "${generated_files[1]}" = "${expected_files[1]}" ] && \ + `cmp "${destination_dir}/README.md" "${larvatagger_jl_project_root}/README.md" >/dev/null` && \ + `cmp "${destination_dir}/trx_small.tgz" "${larvatagger_project_root}/Artefacts/PlanarLarvae/trx_small.tgz" >/dev/null`; then + success "push-file" else - echo "push-file: failure" + failure "push-file" + abort fi # discover the sent files @@ -67,11 +99,14 @@ discovered_files=`curl -sS http://localhost:${lt_backend_port}/list-files/${tagg expected_files='["README.md", "trx_small.tgz"]' if [ "$discovered_files" = "$expected_files" ]; then - echo "list-files: success" + success "list-files" else - echo "list-files: failure" - echo "expected: $expected_files" - echo "got: $discovered_files" + failure "list-files" + cat <<EOF >&2 +expected: $expected_files +got: $discovered_files +EOF + abort fi # make file in processed data dir @@ -90,13 +125,16 @@ expected_content="{ }" if [ "$retrieved_content" = "$expected_content" ]; then - echo "pull-file: success" + success "pull-file" else - echo "pull-file: failure" - echo "expected file content:" - echo "$expected_content" - echo "got:" - echo "$retrieved_content" + failure "pull-file" +cat <<EOF >&2 +expected file content: +$expected_content +got: +$retrieved_content +EOF + abort fi # request file deletion @@ -104,12 +142,15 @@ curl -sS http://localhost:${lt_backend_port}/reset-data/${tagging_backend}/${tag stored_files=`ls "${larvatagger_project_root}/${tagging_backend}/data/raw/${token}/"` if [ -z "$stored_files" ]; then - echo "reset-data: success" + success "reset-data" else - echo "reset-data: failure" - echo "expected empty list" - echo "got:" - echo "$stored_files" + failure "reset-data" + cat <<EOF >&2 +expected empty list +got: +$stored_files +EOF + abort fi # discover the backend(s) and their models @@ -118,13 +159,16 @@ discovered_taggers=`curl -sS http://localhost:${lt_backend_port}/list-taggers` expected_taggers="[{\"name\":\"MaggotUBA\",\"description\":\"Action classifiers based on MaggotUBA encoders\",\"homepage\":\"https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter\",\"models\":[{\"name\":\"20230311\",\"description\":\"Tagger trained on t15 to emulate JBM's tagger on the 7-behavior classification task\",\"homepage\":\"https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter#20230311-0-and-20230311\"},{\"name\":\"20230311-0\",\"description\":\"Tagger trained on t15 to emulate JBM's tagger on the 12-behavior classification task\",\"homepage\":\"https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter#20230311-0-and-20230311\"}]}]" if [ "$discovered_taggers" = "$expected_taggers" ]; then - echo "list-taggers: success" + success "list-taggers" else - echo "list-taggers: failure" - echo "expected:" - echo "$expected_taggers" - echo "got:" - echo "$discovered_taggers" + failure "list-taggers" +cat <<EOF >&2 +expected: +$expected_taggers +got: +$discovered_taggers +EOF + abort fi cp "${larvatagger_jl_project_root}"/20140805_101522@FCF_attP2_1500062@UAS_TNT_2_0003@t5@p_8_45s1x30s0s#p_8_105s10x2s10s#n#n@100.* "${larvatagger_project_root}/${tagging_backend}/data/raw/${token}/" @@ -138,26 +182,30 @@ curl -sS http://localhost:${lt_backend_port}/predict/${tagging_backend}/${taggin generated_files=(`ls "${larvatagger_project_root}/${tagging_backend}/data/processed/${token}/"`) -if [ ${#generated_files[@]} = 1 -a "${generated_files[0]}" = "predicted.label" ]; then - echo "predict: success" +if [ ${#generated_files[@]} = 1 ] && [ "${generated_files[0]}" = "predicted.label" ]; then + success "predict" else - echo "predict: failure" - echo "expected: predicted.label" - echo "got:" - echo "$generated_files" + failure "predict" + cat <<EOF >&2 +expected: predicted.label +got: +$generated_files +EOF + abort fi # clear session data curl -sS http://localhost:${lt_backend_port}/close/${tagging_backend}/${tagging_model}/${token} 1>/dev/null -if ! [ -d "${larvatagger_project_root}/${tagging_backend}/data/raw/${token}" ] && \ - ! [ -d "${larvatagger_project_root}/${tagging_backend}/data/interim/${token}" ] && \ - ! [ -d "${larvatagger_project_root}/${tagging_backend}/data/processed/${token}" ]; then - echo "close: success" +data_dir="${larvatagger_project_root}/${tagging_backend}/data" +if ! [ -d "${data_dir}/raw/${token}" ] && \ + ! [ -d "${data_dir}/interim/${token}" ] && \ + ! [ -d "${data_dir}/processed/${token}" ]; then + success "close" else - echo "close: failure" + failure "close" + abort fi -echo "tests complete; terminating the backend (a stack trace will follow)" +echo "tests complete; terminating the backend" kill $lt_backend_pid -wait $lt_backend_pid -- GitLab