diff --git a/README.md b/README.md index 9996377e17efce0fab5afc88c784e9a5affc5836..6f7173b55073ecc6c4ce9423d3bc7104009e2881 100644 --- a/README.md +++ b/README.md @@ -111,15 +111,12 @@ curl -sSL "https://gitlab.pasteur.fr/nyx/larvatagger.jl/-/raw/dev/scripts/instal ``` In the latter case, the script may install several extra dependencies, but not all of them. -In particular, Python is required; either 3.8 with `--with-default-backend`, or 3.11 with `--with-backend --experimental`. +In particular, Python is required; either 3.11 with `--with-default-backend`, or 3.8 with `--with-backend --legacy`. If `pyenv` is available, the script will use this tool to install Python. -Otherwise, `python3.8` and `python3.8-venv` may have to be manually installed. +Otherwise, `python3.11` and `python3.11-venv` may have to be manually installed. On WSL, the script will attempt to install `pyenv` and Python (tested with Ubuntu 20.04). -On macOS, the full LarvaTagger suite can be installed only with the `--with-backend --experimental` options: -``` -curl -sSL "https://gitlab.pasteur.fr/nyx/larvatagger.jl/-/raw/dev/scripts/install.sh?ref_type=heads&inline=false" | /bin/bash -s -- --with-backend --experimental -``` +On macOS, the full LarvaTagger suite can be installed only with the default options (`--legacy` is not supported). This script can also uninstall LarvaTagger (if installed with the same script) with: `curl -sSL "https://gitlab.pasteur.fr/nyx/larvatagger.jl/-/raw/dev/scripts/install.sh?ref_type=heads&inline=false" | /bin/bash -s -- --uninstall` which can be useful for example prior to reinstalling after failure. diff --git a/recipes/Dockerfile b/recipes/Dockerfile index 11d0e1c7eed52e319119861f568fd68bbf6351e1..b83d356afb8ef07a4c877bff310958d837632c5c 100644 --- a/recipes/Dockerfile +++ b/recipes/Dockerfile @@ -1,4 +1,4 @@ -FROM julia:1.10.8-bullseye AS base +FROM julia:1.10.9-bullseye AS base ARG PROJECT_DIR=/app ARG BRANCH=main @@ -82,7 +82,8 @@ RUN if [ -z $TAGGINGBACKENDS_BRANCH ]; then \ && poetry add "pynvml==11.4.1" \ && if [ "$(echo $BACKEND | cut -d/ -f2)" = "main" ] || [ "$(echo $BACKEND | cut -d/ -f2)" = "dev" ]; then \ julia -e 'using Pkg; Pkg.add("JSON3")' \ - && scripts/make_models.jl default; \ + && scripts/make_models.jl default \ + && recipes/patch.sh; \ fi \ && rm -rf ~/.cache; \ fi diff --git a/recipes/Dockerfile.local b/recipes/Dockerfile.local index 5d033139a6a0d0f2e682adac54fd69786a886cac..34d31fc60666d0e7bde7d6d2b85459e2b7cf5d4c 100644 --- a/recipes/Dockerfile.local +++ b/recipes/Dockerfile.local @@ -1,30 +1,31 @@ # To be built with scripts/larvatagger.sh build --dev -FROM julia:1.8.2-bullseye +FROM julia:1.10.9-bullseye ENV JULIA_PROJECT=/app/TaggingBackends ENV JULIA_DEPOT_PATH=/usr/local/share/julia ENV POETRY_VIRTUALENVS_PATH=/usr/local/share/poetry # We assume: -# * current directory name is LarvaTagger; contains the LarvaTagger.jl project; -# * PlanarLarvae.jl project is available as sibling directory PlanarLarvae; +# * current directory name is LarvaTagger.jl; contains the LarvaTagger.jl project; +# * PlanarLarvae.jl project is available as sibling directory PlanarLarvae.jl; # * TaggingBackends project is available as sibling directory TaggingBackends; # * MaggotUBA-core project is available as sibling directory MaggotUBA-core; # * MaggotUBA-adapter project is available as sibling directory MaggotUBA-adapter. # Paths are given relative to parent directory, since larvatagger.sh will move 1 level up # prior to calling docker build -ARG PLANARLARVAE=./PlanarLarvae -ARG LARVATAGGER=./LarvaTagger +ARG PLANARLARVAE=./PlanarLarvae.jl +ARG LARVATAGGER=./LarvaTagger.jl ARG TAGGINGBACKENDS=./TaggingBackends ARG MAGGOTUBA_CORE=./MaggotUBA-core ARG MAGGOTUBA_ADAPTER=./MaggotUBA-adapter COPY $PLANARLARVAE/src /app/PlanarLarvae/src -COPY $PLANARLARVAE/Project.toml $PLANARLARVAE/Manifest.toml /app/PlanarLarvae/ +COPY $PLANARLARVAE/Project.toml $PLANARLARVAE/Manifest.toml* /app/PlanarLarvae/ COPY $LARVATAGGER/src /app/src COPY $LARVATAGGER/scripts /app/scripts +COPY $LARVATAGGER/recipes/patch.sh /app/recipes/ COPY $LARVATAGGER/Project.toml $LARVATAGGER/Manifest.toml /app/ RUN apt-get update \ @@ -43,7 +44,8 @@ COPY $MAGGOTUBA_CORE/src /app/MaggotUBA-core/src COPY $MAGGOTUBA_CORE/pyproject.toml /app/MaggotUBA-core/ COPY $MAGGOTUBA_ADAPTER/src /app/MaggotUBA/src COPY $MAGGOTUBA_ADAPTER/pyproject.toml /app/MaggotUBA/ -COPY $MAGGOTUBA_ADAPTER/models/20221005 /app/MaggotUBA/models/20221005 +COPY $MAGGOTUBA_ADAPTER/models/20230311 /app/MaggotUBA/models/20230311 +COPY $MAGGOTUBA_ADAPTER/models/20230311-0 /app/MaggotUBA/models/20230311-0 COPY $MAGGOTUBA_ADAPTER/pretrained_models/default /app/MaggotUBA/pretrained_models/default RUN python3 -m pip install poetry \ @@ -54,6 +56,8 @@ RUN python3 -m pip install poetry \ && cd /app/MaggotUBA \ && poetry add ../MaggotUBA-core \ && poetry add ../TaggingBackends \ - && poetry install + && poetry install \ + && cd /app \ + && recipes/patch.sh ENTRYPOINT ["larvatagger.jl"] diff --git a/recipes/Dockerfile.pasteurjanelia b/recipes/Dockerfile.pasteurjanelia index 64924602d24a6ad3087c08e86a79eeaebc48d865..cdb79e2f85f7880058a68a0d3ca4c537e6df229c 100644 --- a/recipes/Dockerfile.pasteurjanelia +++ b/recipes/Dockerfile.pasteurjanelia @@ -27,5 +27,6 @@ RUN cd $PROJECT_DIR \ && make package \ && rm -rf bin/matlab/2023b/bin/glnxa64/matlab_startup_plugins/matlab_graphics_ui \ && rm -rf bin/matlab/2023b/bin/glnxa64/matlab_startup_plugins/foundation/platform/pf_matlab_integ \ - && rm -rf .git ~/.cache + && rm -rf .git ~/.cache \ + && cd .. && recipes/patch.sh diff --git a/recipes/patch.sh b/recipes/patch.sh new file mode 100755 index 0000000000000000000000000000000000000000..b32d49141ec69574aae8ea173081698f6c14ea92 --- /dev/null +++ b/recipes/patch.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +# patch the taggers in a Docker image so that they include metadata.json files + +if [ -d "MaggotUBA" ]; then + if ! [ -f "MaggotUBA/metadata.json" ]; then + cat <<"EOF" >MaggotUBA/metadata.json +{ + "name": "MaggotUBA", + "homepage": "https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter", + "description": "Action classifiers based on MaggotUBA encoders" +} +EOF + fi + + dir="MaggotUBA/models/20230311" + if [ -d "$dir" ] && ! [ -f "$dir/metadata.json" ]; then + cat <<"EOF" >$dir/metadata.json +{ + "name": "20230311", + "homepage": "https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter#20230311-0-and-20230311", + "description": "Tagger trained on t15 to emulate JBM's tagger on the 7-behavior classification task" +} +EOF + fi + + dir="MaggotUBA/models/20230311-0" + if [ -d "$dir" ] && ! [ -f "$dir/metadata.json" ]; then + cat <<"EOF" >$dir/metadata.json +{ + "name": "20230311-0", + "homepage": "https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter#20230311-0-and-20230311", + "description": "Tagger trained on t15 to emulate JBM's tagger on the 12-behavior classification task" +} +EOF + fi + +fi + +if [ -d "PasteurJanelia" ]; then + if ! [ -f "PasteurJanelia/metadata.json" ]; then + cat <<"EOF" >PasteurJanelia/metadata.json +{ + "name": "PasteurJanelia", + "homepage": "https://gitlab.pasteur.fr/nyx/PasteurJanelia-adapter", + "description": "Action classifiers initially designed by JBM at Janelia" +} +EOF + fi + + dir="PasteurJanelia/models/5layers" + if [ -d "$dir" ] && ! [ -f "$dir/metadata.json" ]; then + cat <<"EOF" >$dir/metadata.json +{ + "name": "5layers", + "homepage": "https://gitlab.pasteur.fr/nyx/PasteurJanelia-adapter", + "description": "JBM's final tagger for use on the t2 and t7 trackers" +} +EOF + fi + +fi diff --git a/scripts/larvatagger.sh b/scripts/larvatagger.sh index 3cf76b9281fb3a4978dd88f2b717148b0639bf20..ff32aec3c3e84f312f7ab2ca329bc6419391c262 100755 --- a/scripts/larvatagger.sh +++ b/scripts/larvatagger.sh @@ -2,7 +2,7 @@ for _ in $(seq $#); do case $1 in - open|import|merge|train|predict|-V|--version|--more-help|reverse-mapping|confusion) + open|import|merge|train|predict|-V|--version|--more-help|reverse-mapping|confusion|backend) cmd=$1 shift break @@ -102,6 +102,15 @@ done if [ -z "$no_cache" ] && [ -z "$cache" ]; then DOCKER_ARGS="--no-cache $DOCKER_ARGS" fi + +if [ "$BUILD" == "--dev" ]; then +if ! [[ "$LARVATAGGER_IMAGE" == *:* ]]; then LARVATAGGER_IMAGE="${LARVATAGGER_IMAGE}:dev"; fi +PROJECT_ROOT=$(basename $(pwd)) +cd .. +echo "DOCKER_BUILDKIT=1 $docker build -t \"$LARVATAGGER_IMAGE\" -f \"$PROJECT_ROOT/recipes/Dockerfile.local\" ${DOCKER_ARGS}." +DOCKER_BUILDKIT=1 $docker build -t "$LARVATAGGER_IMAGE" -f "$PROJECT_ROOT/recipes/Dockerfile.local" ${DOCKER_ARGS}. +else + if [ -z "$target" ]; then DOCKER_ARGS="--target $TARGET $DOCKER_ARGS" fi @@ -111,19 +120,16 @@ else echo "Using environment variable DOCKERFILE= $DOCKERFILE" fi DOCKER_ARGS="-f \"$DOCKERFILE\" $DOCKER_ARGS" -if [ "$BUILD" == "--dev" ]; then -if ! [[ "$LARVATAGGER_IMAGE" == *:* ]]; then LARVATAGGER_IMAGE="${LARVATAGGER_IMAGE}:dev"; fi -PROJECT_ROOT=$(basename $(pwd)) -cd .. -DOCKER_BUILDKIT=1 $docker build -t "$LARVATAGGER_IMAGE" -f "$PROJECT_ROOT/recipes/Dockerfile.local" ${DOCKER_ARGS}. -elif [ "$BUILD" == "--stable" ]; then + +if [ "$BUILD" == "--stable" ]; then if ! [[ "$LARVATAGGER_IMAGE" == *:* ]]; then LARVATAGGER_IMAGE="${LARVATAGGER_IMAGE}:stable"; fi $docker build -t "$LARVATAGGER_IMAGE" ${DOCKER_ARGS}. else + if ! [[ "$LARVATAGGER_IMAGE" == *:* ]]; then LARVATAGGER_IMAGE="${LARVATAGGER_IMAGE}:latest"; fi if [ -z "$LARVATAGGER_BRANCH" ]; then if [ -z "$LARVATAGGER_DEFAULT_BRANCH" ]; then - LARVATAGGER_BRANCH=dev; + LARVATAGGER_BRANCH=dev else echo "Deprecation notice: LARVATAGGER_DEFAULT_BRANCH has been renamed LARVATAGGER_BRANCH" LARVATAGGER_BRANCH=$LARVATAGGER_DEFAULT_BRANCH @@ -141,6 +147,7 @@ DOCKER_BUILD="$docker build -t "$LARVATAGGER_IMAGE" ${DOCKER_ARGS}--build-arg BR echo $DOCKER_BUILD eval $DOCKER_BUILD fi +fi ;; open) @@ -174,6 +181,34 @@ done DOCKER_RUN="exec $docker run $RUN_ARGS -i ${DOCKER_ARGS}\"$LARVATAGGER_IMAGE\" open \"/data/$file\" $TAGGER_ARGS $@" echo $DOCKER_RUN eval $DOCKER_RUN +;; + + backend) + +if [ -z "$LARVATAGGER_PORT" ]; then +LARVATAGGER_PORT=9285 +elif [ "$LARVATAGGER_PORT" != "9285" ]; then +echo "Using environment variable: LARVATAGGER_PORT= $LARVATAGGER_PORT" +fi +DOCKER_ARGS="-p $LARVATAGGER_PORT:$LARVATAGGER_PORT $DOCKER_ARGS" + +# undocumented feature (copied-pasted from open) +backend=MaggotUBA +while [ -n "$1" -a "$1" = "--external-instance" ]; do +instance=$2; shift 2 +if ! command -v realpath &>/dev/null; then +echo "realpath: command not found" +echo "on macOS: brew install coreutils" +exit 1 +fi +RUN_ARGS="$RUN_ARGS --mount type=bind,src=\"$(realpath $instance)\",dst=/app/$backend/models/$(basename $instance)" +done + +RUN_ARGS="$RUN_ARGS --entrypoint=julia" + +DOCKER_RUN="exec $docker run $RUN_ARGS -i ${DOCKER_ARGS}\"$LARVATAGGER_IMAGE\" $@ --project=/app -e 'using LarvaTagger.REST.Server; run_backend(\"/app\"; port=$LARVATAGGER_PORT, host=\"0.0.0.0\")'" +echo $DOCKER_RUN +eval $DOCKER_RUN ;; import | merge) @@ -371,6 +406,7 @@ Usage: $0 build [--stable] [--with-default-backend] [--with-backend <backend>] $0 confusion <datarepository> $0 merge <filepath> [<outputfilename>] [...] $0 reverse-mapping <filepath> <filename> <outputfilename> + $0 backend $0 --more-help $0 --version $0 --update @@ -390,6 +426,8 @@ the second one with unmapped labels. It generates a third label file with demapp from the first file. This is useful when the first file diverges from the second one by some manual editions, on top of label mapping. +The backend command runs a LarvaTagger's REST server that listens to port 9285 per default. + See --more-help for more information about additional arguments for the other commands from larvatagger.jl. EOT diff --git a/src/REST/Client.jl b/src/REST/Client.jl index 1a914834e6b4cebfa3c9246774787165f04d4595..611608e7e9c6cd178ce04b569a8aed3a24d42f67 100644 --- a/src/REST/Client.jl +++ b/src/REST/Client.jl @@ -209,11 +209,24 @@ end listmodels(back::LTBackend) = listmodels(back, Val(false)) -listmodels(back::LTBackend, ::Val{false}) = collect(keys(back.taggers)) +function listmodels(back::LTBackend, ::Val{false}) + [OrderedDict("name" => name, + "description" => get(back.metadata[name], :description, ""), + "homepage" => get(back.metadata[name], :homepage, ""), + ) for name in keys(back.taggers)] +end function listmodels(back::LTBackend, ::Val{true}) map(back.active_tagging_backend) do tagging_backend - isnothing(tagging_backend) ? String[] : collect(keys(back.taggers[tagging_backend])) + models = OrderedDict{String, String}[] + for name in keys(back.taggers[tagging_backend]) + metadata = back.metadata[tagging_backend][:models][name] + push!(models, OrderedDict("name" => name, + "description" => get(metadata, :description, ""), + "homepage" => get(metadata, :homepage, ""), + )) + end + return models end end diff --git a/src/REST/Model.jl b/src/REST/Model.jl index 781ce7ab71a807fd8c48344d98bf8c329139b3ba..9effac9d4dfe4bba08fb3e7e4daad87356538559 100644 --- a/src/REST/Model.jl +++ b/src/REST/Model.jl @@ -187,26 +187,49 @@ function pullfile(lt_backend, backend_dir, model_instance, token, filename) return HTTP.Response(200, header, body) end +function loadmodel(dir, hasinstances=false) + metadata = nothing + for filename in ("metadata", "metadata.json") + if isfile(joinpath(dir, filename)) + metadata = JSON3.read(joinpath(dir, filename)) + break + end + end + name = basename(dir) + T = if hasinstances + Union{AbstractString, Vector{OrderedDict{AbstractString, AbstractString}}} + else + AbstractString + end + model = OrderedDict{AbstractString, T}( + "name" => name, + "description" => "", + "homepage" => "", + ) + if !isnothing(metadata) + for key in keys(model) + key′ = Symbol(key) + if haskey(metadata, key′) + model[key] = metadata[key′] + end + end + end + return model +end + function listtaggers(lt_backend) inventory = Vector{OrderedDict{String, Any}}() backends_dir = lt_backend.root[] - for tagging_backend in readdir(backends_dir) - tagging_backend_path = joinpath(backends_dir, tagging_backend) + for tagging_backend_path in readdir(backends_dir; join=true) Taggers.isbackend(tagging_backend_path) || continue models_dir = joinpath(tagging_backend_path, "models") - models = [OrderedDict( - "name" => dir, - "description" => "", - "homepage" => "", - ) for dir in readdir(models_dir) if isdir(joinpath(models_dir, dir))] - push!(inventory, OrderedDict( - "name" => tagging_backend, - "description" => "", - "homepage" => "", - "models" => models, - )) + models = loadmodel.(readdir(models_dir; join=true)) + isempty(models) && continue + tagging_backend = loadmodel(tagging_backend_path, true) + tagging_backend["models"] = models + push!(inventory, tagging_backend) end - return JSON3.write(inventory) + return JSON3.write(unique(inventory)) end function predict(lt_backend, backend_dir, model_instance, token) diff --git a/src/larvatagger.js b/src/larvatagger.js index d65fb7dfd6428106854c18bced86df6aa57e887a..8437424d0d5ae6ebd80506e8f9a72dae5a256b65 100644 --- a/src/larvatagger.js +++ b/src/larvatagger.js @@ -132,12 +132,17 @@ const LarvaTagger = (function () { selectElement.remove(0); } for (let i = 0; i < selectOptions.length; i++) { - const option = document.createElement('option'); - option.value = selectOptions[i]; - option.text = selectOptions[i]; + let option = document.createElement('option'), + name = selectOptions[i]; + if (typeof name === 'object') { + option.title = name['description']; + name = name['name']; + } + option.value = name; + option.text = name; if (jlObserver !== undefined) { option.ondblclick = () => { - jlObserver.notify(selectOptions[i]); + jlObserver.notify(name); }; } selectElement.append(option); diff --git a/src/wgl.jl b/src/wgl.jl index 6674dc505cfd75b82be44d16a658101addeb99e2..2579cac58ae0ad67fa74829f64a899c5316ebf28 100644 --- a/src/wgl.jl +++ b/src/wgl.jl @@ -1001,17 +1001,25 @@ function backendmenu(controller, location) BackendMenu(backends, models, send2backend, dom_id()) end +function lowermodel(model) + if model isa AbstractString + DOM.option(model; value=model) + else + name = model["name"] + DOM.option(name; value=name, title=model["description"]) + end +end + function lowerdom(bs::BackendMenu) model_instance = get_model_instance(bs.backends) update_model_instance = js"(evt)=>{ $model_instance.notify(evt.srcElement.value); }" - select_dom = DOM.select(DOM.option(model; value=model) for model in bs.models[]; + select_dom = DOM.select(lowermodel.(bs.models[]); onchange=update_model_instance, class=css_button) active_backend = get_active_backend(bs.backends) update_active_backend = js"(evt)=>{ $active_backend.notify(evt.srcElement.value); }" return DOM.div(DOM.label("Backend"), - DOM.select(DOM.option(backend; value=backend) - for backend in get_backend_names(bs.backends); + DOM.select(lowermodel.(get_backend_names(bs.backends)); onchange=update_active_backend, class=css_button), DOM.label("Model instance"), diff --git a/test/rest_server.sh b/test/rest_server.sh index 6de171f82297bbda75c4a929589cd40ced7285f5..d69c186bf862b2826ea1c1e2d620d67fb0c22272 100755 --- a/test/rest_server.sh +++ b/test/rest_server.sh @@ -12,7 +12,7 @@ lt_backend_port=9285 # assumed directory structure: # the MaggotUBA-adapter, TaggingBackends, NyxArtefacts and LarvaTagger.jl projects are siblings in the ${larvatagger_project_root} directory; -# the MaggotUBA-adapter project is available both as directory MaggotUBA-adapter and link MaggotUBA (${tagging_backend}); +# the MaggotUBA-adapter project is available both as directory MaggotUBA (${tagging_backend}); # the MaggotUBA-adapter tagging backend contains two trained models (or model instances): 20230311 and 20230311-0; # the NyxArtefacts project is available as directory Artefacts; # MaggotUBA is installed with JULIA_PROJECT pointing at the TaggingBackends directory @@ -113,32 +113,18 @@ else fi # discover the backend(s) and their models -discovered_backends=`curl -sS http://localhost:${lt_backend_port}/list-backends` - -backend_metadata() { - local desc1= - local link1= - local desc2= - local link2= - local desc3= - local link3= - echo "{\"name\":\"$1\",\"description\":\"$desc1\",\"homepage\":\"$link1\",\"models\":[{\"name\":\"20230311\",\"description\":\"$desc2\",\"homepage\":\"$link2\"},{\"name\":\"20230311-0\",\"description\":\"$desc3\",\"homepage\":\"$link3\"}]}" -} -expected_backends="[`backend_metadata $tagging_backend`,`backend_metadata ${tagging_backend}-adapter`]" +discovered_taggers=`curl -sS http://localhost:${lt_backend_port}/list-taggers` + +expected_taggers="[{\"name\":\"MaggotUBA\",\"description\":\"Action classifiers based on MaggotUBA encoders\",\"homepage\":\"https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter\",\"models\":[{\"name\":\"20230311\",\"description\":\"Tagger trained on t15 to emulate JBM's tagger on the 7-behavior classification task\",\"homepage\":\"https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter#20230311-0-and-20230311\"},{\"name\":\"20230311-0\",\"description\":\"Tagger trained on t15 to emulate JBM's tagger on the 12-behavior classification task\",\"homepage\":\"https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter#20230311-0-and-20230311\"}]}]" -if [ "$discovered_backends" = "$expected_backends" ]; then - echo "list-backends: success" +if [ "$discovered_taggers" = "$expected_taggers" ]; then + echo "list-taggers: success" else - echo "list-backends: failure" + echo "list-taggers: failure" echo "expected:" - echo "$expected_backends" + echo "$expected_taggers" echo "got:" - echo "$discovered_backends" - # further diagnose - model_instances=(`ls "${larvatagger_project_root}/${tagging_backend}/models"`) - if ! [ ${#model_instances[@]} -eq 2 -a "${model_instances[0]}" = "20230311" -a "${model_instances[1]}" = "20230311-0" ]; then - echo "tagging backend is not adequately set up" - fi + echo "$discovered_taggers" fi cp "${larvatagger_jl_project_root}"/20140805_101522@FCF_attP2_1500062@UAS_TNT_2_0003@t5@p_8_45s1x30s0s#p_8_105s10x2s10s#n#n@100.* "${larvatagger_project_root}/${tagging_backend}/data/raw/${token}/"