diff --git a/Project.toml b/Project.toml index 7266aab1169e6fe2ee3fab70fb979e4e1a17d0cf..b2799a788939137295dd376e10fa2b317a81fa04 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "LarvaTagger" uuid = "8b3b36f1-dfed-446e-8561-ea19fe966a4d" authors = ["François Laurent", "Institut Pasteur"] -version = "0.15.1" +version = "0.15.2" [deps] Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" diff --git a/recipes/Dockerfile b/recipes/Dockerfile index a0559be26666226623afcb98151d7a6f8a1fccba..3d6439a18bbb575dc223b85a9624d98df0621c07 100644 --- a/recipes/Dockerfile +++ b/recipes/Dockerfile @@ -1,4 +1,4 @@ -FROM julia:1.9.0-bullseye AS base +FROM julia:1.9.1-bullseye AS base ARG PROJECT_DIR=/app ARG BRANCH=main @@ -12,10 +12,14 @@ RUN apt-get update \ && git clone --depth 1 --no-tags --single-branch -b $BRANCH https://gitlab.pasteur.fr/nyx/larvatagger.jl "$JULIA_PROJECT" \ && rm -rf "$JULIA_PROJECT/.git" \ && julia -e 'using Pkg; try Pkg.rm("TaggingBackends"); catch end; Pkg.instantiate()' \ - && ln -s "$JULIA_PROJECT/scripts/larvatagger.jl" /bin \ + && ln -s "$JULIA_PROJECT/scripts/larvatagger" /bin \ && mkdir -p "$JULIA_DEPOT_PATH/logs" && rm -f "$JULIA_DEPOT_PATH/logs/manifest_usage.toml" && ln -s /dev/null "$JULIA_DEPOT_PATH/logs/manifest_usage.toml" -ENTRYPOINT ["larvatagger.jl"] +#RUN $JULIA_PROJECT/test/precompile.sh --shallow \ +# && mv larvatagger.so /lib/ \ +# && rm -rf $JULIA_PROJECT/test/data + +ENTRYPOINT ["/app/scripts/larvatagger"] #, "-J/lib/larvatagger.so"] diff --git a/recipes/README.md b/recipes/README.md index 28e0fbabc71e1ba8d277c5e999a27b8138fbb8f9..e18e888ed9020dbb2e5aa5efdd2e8e6a47c1d90a 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -163,7 +163,7 @@ docker pull flaur/larvatagger ``` Beware that images that ship with backends are relatively large files (>5GB on disk). -If you are not interested in automatic tagging, use the `flaur/larvatagger:0.15.1-standalone` image instead. +If you are not interested in automatic tagging, use the `flaur/larvatagger:0.15.2-standalone` image instead. ### Upgrading diff --git a/scripts/larvatagger b/scripts/larvatagger new file mode 100755 index 0000000000000000000000000000000000000000..861fc2439238c82f61fda77838fa3a7d3645653c --- /dev/null +++ b/scripts/larvatagger @@ -0,0 +1,136 @@ +#!/bin/bash + +# This script is written in Bash and replaces a Julia script whose name (and extension) +# is kept for backward compatibility. + +# Former scripts/larvatagger.jl and src/cli.jl have been split in distinct scripts and +# source files in an attempt to fix issue https://gitlab.pasteur.fr/nyx/larvatagger.jl/-/issues/61 + +currentdir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") + +if [ "${1:0:2}" = "-J" ]; then +cmd=$2 +else +cmd=$1 +fi + +case $cmd in +open) + shift + $currentdir/larvatagger-gui.jl $@ + ;; + +import|merge|train|predict|--version|-V) + $currentdir/larvatagger-toolkit.jl $@ + ;; + +*) + cat << "EOT" +LarvaTagger + +Usage: + larvatagger open <file-path> [--backends=<path>] [--port=<number>] [--quiet] [--viewer] [--browser] [--manual-label=<label>] + larvatagger import <input-path> [<output-file>] [--id=<id>] [--framerate=<fps>] [--pixelsize=<μm>] [--overrides=<comma-separated-list>] [--default-label=<label>] [--manual-label=<label>] [--decode] [--copy-labels] + larvatagger train <backend-path> <data-path> <model-instance> [--pretrained-model=<instance>] [--labels=<comma-separated-list>] [--sample-size=<N>] [--balancing-strategy=<strategy>] [--class-weights=<csv>] [--manual-label=<label>] [--layers=<N>] [--iterations=<N>] [--seed=<seed>] + larvatagger predict <backend-path> <model-instance> <data-path> [--output=<filename>] [--make-dataset] [--skip-make-dataset] [--data-isolation] + larvatagger merge <input-path> <input-file> [<output-file>] [--manual-label=<label>] [--decode] + larvatagger -V | --version + larvatagger -h | --help + +Options: + -h --help Show this screen. + -V --version Show version. + -q --quiet Do not show instructions. + --id=<id> Run or assay ID, e.g. `date_time`. + --framerate=<fps> Camera frame rate, in frames per second. + --pixelsize=<μm> Camera pixel size, in micrometers. + --backends=<path> Path to backend repository. + --port=<number> Port number the server listens to. + --viewer Disable editing capabilities. + --browser Automatically open a browser tab at the served location. + --make-dataset Perform the make_dataset stage prior to proceeding to predict_model. + --skip-make-dataset Skip the make_dataset stage (default behavior since TaggingBackends==0.10). + --data-isolation Isolate the backend data directories for parallel tagging of multiple data files. + --sample-size=<N> Sample only N track segments from the data repository. + --layers=<N> (MaggotUBA) Number of layers of the classifier. + --iterations=<N> (MaggotUBA) Number of training iterations (can be two integers separated by a comma). + --seed=<seed> Seed for the backend's random number generators. + --decode Do not encode the labels into integer indices. + --copy-labels Replicate discrete behavior data from the input file. + --default-label=<label> Label all untagged data as <label>. + --manual-label=<label> Secondary label for manually labelled data [default: edited]. + --labels=<comma-separated-list> Comma-separated list of behavior tags/labels. + --class-weights=<csv> Comma-separated list of floats. + --pretrained-model=<instance> Name of the pretrained encoder (from `pretrained_models` registry). + --balancing-strategy=<strategy> Any of `auto`, `maggotuba`, `none` [default: auto]. + --overrides=<comma-separated-list> Comma-separated list of key:value pairs. + -o <filename> --output=<filename> Predicted labels filename. + + +Commands: + + open Launch the server-based GUI. + + Backends defined in LarvaTagger project root directory are automatically found. Other + backend locations can be specified with the --backends argument. + + import Generate a label file and store metadata. + + This command is particularly useful to specify missing information such as the camera + framerate for FIMTrack v2 table.csv files. A label file is created with a pointer to the + original csv file, and a metadata section with the provided information. + A second usage applies to label files specifically, and consists in extending the tracks + and timesteps on basis of the associated data dependencies and assigning a default label + to the data points that are not already defined in the label file. + Note: label files should always come as siblings of their data dependencies (located in + the same directory). + + merge Combine the content of label files. + + <input-path> is the path to a manually edited label file whose content is to be augmented + with manual editions from a second (sibling) label file <input-file>. If no output file + is specified, the resulting content is dumped onto the standard output. All label files + must be siblings, i.e. located in the same directory. + If no editions are found in the second input file, all the defined labels are copied from + the second to the first file. + + train Train a tagger. + + <data-path> can be a path to a file or directory. + --class-weights requires --labels to be defined and the specified comma-separated values + should match those given by --labels. + + predict Automatically label tracking data. + + <data-path> can be a path to a file or directory. + <data-path> can also be a .txt file listing data files; one relative path per line. + +EOT + +esac + +exit 0 + +# Equivalent Julia script (slower): + +#!/bin/bash +#= +if [ -z "$(which realpath)" ]; then +macos_realpath=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) +PROJECT_DIR=$(dirname $macos_realpath) +else +PROJECT_DIR=$(dirname $(dirname $(realpath "${BASH_SOURCE[0]}"))) +fi +FLAGS= +if [ "$1" = "--sysimage" -o "$1" = "-J" ]; then FLAGS="--sysimage $2 "; shift 2; fi +if [ "${1:0:2}" = "-J" ]; then FLAGS="$1"; shift; fi +if [ "$1" = "open" -a -n "$2" -a -f "$2" ]; then FLAGS="$FLAGS -iq "; fi +if [ -z "$JULIA" ]; then JULIA=julia; fi +exec $JULIA --project="$PROJECT_DIR" --color=yes --startup-file=no $FLAGS\ + "${BASH_SOURCE[0]}" "$@" +=# + +projectdir = dirname(Base.active_project()) +include(joinpath(projectdir, "src/cli.jl")) # defines `main` + +main(); diff --git a/scripts/larvatagger.bat b/scripts/larvatagger.bat index 9e25800a6ee7695134eb09a2e8a88e3015dd2ea3..d68f80b2609f9e5216e044b8c068d9f753acef66 100644 --- a/scripts/larvatagger.bat +++ b/scripts/larvatagger.bat @@ -1,4 +1,4 @@ if not exist larvatagger.ps1 ( PowerShell.exe -Command "Invoke-WebRequest https://gitlab.pasteur.fr/nyx/larvatagger.jl/-/raw/main/scripts/larvatagger.ps1?inline=false -OutFile larvatagger.ps1" ) -PowerShell.exe -ExecutionPolicy ByPass -File larvatagger.ps1 $* +PowerShell.exe -ExecutionPolicy ByPass -File larvatagger.ps1 %* diff --git a/scripts/larvatagger.jl b/scripts/larvatagger.jl deleted file mode 100755 index fb9352a28da17f9ba0d476bc83053bcd7d1683c1..0000000000000000000000000000000000000000 --- a/scripts/larvatagger.jl +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash - -# This script is written in Bash and replaces a Julia script whose name (and extension) -# is kept for backward compatibility. - -# Former scripts/larvatagger.jl and src/cli.jl have been split in distinct scripts and -# source files in an attempt to fix issue https://gitlab.pasteur.fr/nyx/larvatagger.jl/-/issues/61 - -currentdir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") - -case $1 in -open) - shift - $currentdir/larvatagger-gui.jl $@ - ;; - -import|merge|train|predict|--version|-V) - $currentdir/larvatagger-toolkit.jl $@ - ;; - -*) - cat << "EOT" -LarvaTagger.jl - -Usage: - larvatagger.jl open <file-path> [--backends=<path>] [--port=<number>] [--quiet] [--viewer] [--browser] [--manual-label=<label>] - larvatagger.jl import <input-path> [<output-file>] [--id=<id>] [--framerate=<fps>] [--pixelsize=<μm>] [--overrides=<comma-separated-list>] [--default-label=<label>] [--manual-label=<label>] [--decode] [--copy-labels] - larvatagger.jl train <backend-path> <data-path> <model-instance> [--pretrained-model=<instance>] [--labels=<comma-separated-list>] [--sample-size=<N>] [--balancing-strategy=<strategy>] [--class-weights=<csv>] [--manual-label=<label>] [--layers=<N>] [--iterations=<N>] [--seed=<seed>] - larvatagger.jl predict <backend-path> <model-instance> <data-path> [--output=<filename>] [--make-dataset] [--skip-make-dataset] [--data-isolation] - larvatagger.jl merge <input-path> <input-file> [<output-file>] [--manual-label=<label>] [--decode] - larvatagger.jl -V | --version - larvatagger.jl -h | --help - -Options: - -h --help Show this screen. - -V --version Show version. - -q --quiet Do not show instructions. - --id=<id> Run or assay ID, e.g. `date_time`. - --framerate=<fps> Camera frame rate, in frames per second. - --pixelsize=<μm> Camera pixel size, in micrometers. - --backends=<path> Path to backend repository. - --port=<number> Port number the server listens to. - --viewer Disable editing capabilities. - --browser Automatically open a browser tab at the served location. - --make-dataset Perform the make_dataset stage prior to proceeding to predict_model. - --skip-make-dataset Skip the make_dataset stage (default behavior since TaggingBackends==0.10). - --data-isolation Isolate the backend data directories for parallel tagging of multiple data files. - --sample-size=<N> Sample only N track segments from the data repository. - --layers=<N> (MaggotUBA) Number of layers of the classifier. - --iterations=<N> (MaggotUBA) Number of training iterations (can be two integers separated by a comma). - --seed=<seed> Seed for the backend's random number generators. - --decode Do not encode the labels into integer indices. - --copy-labels Replicate discrete behavior data from the input file. - --default-label=<label> Label all untagged data as <label>. - --manual-label=<label> Secondary label for manually labelled data [default: edited]. - --labels=<comma-separated-list> Comma-separated list of behavior tags/labels. - --class-weights=<csv> Comma-separated list of floats. - --pretrained-model=<instance> Name of the pretrained encoder (from `pretrained_models` registry). - --balancing-strategy=<strategy> Any of `auto`, `maggotuba`, `none` [default: auto]. - --overrides=<comma-separated-list> Comma-separated list of key:value pairs. - -o <filename> --output=<filename> Predicted labels filename. - - -Commands: - - open Launch the server-based GUI. - - Backends defined in LarvaTagger project root directory are automatically found. Other - backend locations can be specified with the --backends argument. - - import Generate a label file and store metadata. - - This command is particularly useful to specify missing information such as the camera - framerate for FIMTrack v2 table.csv files. A label file is created with a pointer to the - original csv file, and a metadata section with the provided information. - A second usage applies to label files specifically, and consists in extending the tracks - and timesteps on basis of the associated data dependencies and assigning a default label - to the data points that are not already defined in the label file. - Note: label files should always come as siblings of their data dependencies (located in - the same directory). - - merge Combine the content of label files. - - <input-path> is the path to a manually edited label file whose content is to be augmented - with manual editions from a second (sibling) label file <input-file>. If no output file - is specified, the resulting content is dumped onto the standard output. All label files - must be siblings, i.e. located in the same directory. - If no editions are found in the second input file, all the defined labels are copied from - the second to the first file. - - train Train a tagger. - - <data-path> can be a path to a file or directory. - --class-weights requires --labels to be defined and the specified comma-separated values - should match those given by --labels. - - predict Automatically label tracking data. - - <data-path> can be a path to a file or directory. - <data-path> can also be a .txt file listing data files; one relative path per line. - -EOT - -esac - -exit 0 - -# Equivalent Julia script (slower): - -#!/bin/bash -#= -if [ -z "$(which realpath)" ]; then -macos_realpath=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) -PROJECT_DIR=$(dirname $macos_realpath) -else -PROJECT_DIR=$(dirname $(dirname $(realpath "${BASH_SOURCE[0]}"))) -fi -FLAGS= -if [ "$1" = "--sysimage" -o "$1" = "-J" ]; then FLAGS="--sysimage $2 "; shift 2; fi -if [ "${1:0:2}" = "-J" ]; then FLAGS="$1"; shift; fi -if [ "$1" = "open" -a -n "$2" -a -f "$2" ]; then FLAGS="$FLAGS -iq "; fi -if [ -z "$JULIA" ]; then JULIA=julia; fi -exec $JULIA --project="$PROJECT_DIR" --color=yes --startup-file=no $FLAGS\ - "${BASH_SOURCE[0]}" "$@" -=# - -projectdir = dirname(Base.active_project()) -include(joinpath(projectdir, "src/cli.jl")) # defines `main` - -main(); diff --git a/scripts/larvatagger.jl b/scripts/larvatagger.jl new file mode 120000 index 0000000000000000000000000000000000000000..ec18b86a7141581e528d5188dd7eb4ad8dfd46c2 --- /dev/null +++ b/scripts/larvatagger.jl @@ -0,0 +1 @@ +larvatagger \ No newline at end of file diff --git a/scripts/larvatagger.sh b/scripts/larvatagger.sh index 5f4b7fbe8308538731dde0bda4ba5287e5b49ded..4ead861d3a9d64fa5b40f47caf058a82cec0da1c 100755 --- a/scripts/larvatagger.sh +++ b/scripts/larvatagger.sh @@ -1,27 +1,46 @@ #!/bin/bash -if [ -z "$docker" ]; then - if docker --version &> /dev/null; then - docker=docker - else - docker=podman - fi -fi - for _ in $(seq $#); do case $1 in - build|open|import|merge|train|predict|-V|--version|-h|--help|--more-help|--upgrade) + open|import|merge|train|predict|-V|--version|--more-help) + cmd=$1 + shift + break + ;; + build|-h|--help|--update) cmd=$1 shift + without_rm=1 break ;; + --no-rm) + without_rm=1 + ;; *) + if [ "$1" = "--rm" ]; then + with_rm=1 + fi # note: if DOCKER_ARGS is externally defined, it must end with an explicit space DOCKER_ARGS="${DOCKER_ARGS}$1 " shift esac done +if [ -z "$with_rm" -a -z "$without_rm" ]; then + echo "Upcoming change: --rm will become default; pass --no-rm to maintain current behavior" +fi + +if [ -z "$docker" ]; then + if docker --version &> /dev/null; then + docker=docker + else + docker=podman + if [ "$cmd" != "build" ]; then + DOCKER_ARGS="${DOCKER_ARGS}--security-opt label=disable " + fi + fi +fi + if [ -z "$LARVATAGGER_IMAGE" ]; then if [ "$cmd" = "build" -o -n "$($docker images | grep '^larvatagger ')" ]; then LARVATAGGER_IMAGE=larvatagger diff --git a/src/files.jl b/src/files.jl index eee1db8d8c78fdeb248b63a9e45047c4d932e897..5401ee2b6a7985f09c7528e1e2cdf6d9c80cb011 100644 --- a/src/files.jl +++ b/src/files.jl @@ -239,18 +239,6 @@ interpolate(s="yyyymmdd_HHMMSS") = Dates.format(Dates.now(), s) function savetofile(controller, file; datafile=nothing, merge=false) dir = cwd(controller) - filepath = joinpath(dir, file) - if merge - tmpfile = tempname(dir; cleanup=false) * ".label" - tmpfile′= basename(tmpfile) - try - savetofile(controller, tmpfile′; datafile=datafile) - merge′(filepath, tmpfile′, getmanualtag(controller), file) - finally - rm(tmpfile; force=true) - end - return - end if '{' in file prefix, remainder = split(file, '{'; limit=2) if '}' in remainder @@ -258,11 +246,25 @@ function savetofile(controller, file; datafile=nothing, merge=false) if length(parts) == 2 infix, suffix = parts else + @warn "Too many braces" file infix, suffix = join(parts[1:end-1], '}'), parts[end] end file = prefix * interpolate(infix) * suffix end end + filepath = joinpath(dir, file) + if merge + tmpfile = tempname(dir; cleanup=false) * ".label" + tmpfile′= basename(tmpfile) + try + savetofile(controller, tmpfile′; datafile=datafile) + @info "Merging file $tmpfile′ into $(file)" + merge′(filepath, tmpfile′, getmanualtag(controller), file) + finally + rm(tmpfile; force=true) + end + return + end dataset = getoutput(controller) @assert length(dataset) == 1 run = first(values(dataset)) @@ -406,19 +408,12 @@ function explicit_editions_needed(controller, editiontag) file = preload(originalfile) if file isa Formats.JSONLabels Formats.load!(file) - attributes = file.run.attributes - if haskey(attributes, :metadata) - metadata = attributes[:metadata] - if haskey(metadata, :software) - software = metadata[:software] - return haskey(software, Datasets.coerce(eltype(keys(software)), :tagger)) # the file was automatically generated - end - end - if haskey(attributes, :secondarylabels) - return editiontag ∈ attributes[:secondarylabels] - end + return haskey(file.run, :labels) + elseif file isa Formats.Trxmat + return true + else + return false end - return false end function getinputfile(controller) @@ -444,29 +439,41 @@ struct OutputFile merge end +OutputFile() = OutputFile( + Observable{Union{Nothing, String}}("{yyyymmdd_HHMMSS}.label"), + Observable{Bool}(false), +) + +function reset!(outputfile::OutputFile) + outputfile.name[] = nothing +end + function getoutputfile(controller) hub = gethub(controller) outputfile = nothing try outputfile = hub[:outputfile] catch - hub[:outputfile] = outputfile = OutputFile( - Observable{Union{Nothing, String}}("{yyyymmdd_HHMMSS}.label"), - Observable{Bool}(false)) + hub[:outputfile] = outputfile = OutputFile() on(outputfile.name) do file + dir = cwd(controller) if isnothing(file) outputfile.name.val = "{yyyymmdd_HHMMSS}.label" - elseif isfile(file) + elseif isfile(joinpath(dir, file)) twooptiondialog(hub, outputfile.merge, "File already exists", "Do you want to save the manual editions only (merge), or entirely overwrite the file?", "Merge", "Overwrite") else savetofile(hub, file) + reset!(outputfile) end end on(outputfile.merge) do merge - isnothing(merge) || savetofile(hub, outputfile.name[]; merge=merge) + if !isnothing(merge) + savetofile(hub, outputfile.name[]; merge=merge) + reset!(outputfile) + end end end return outputfile diff --git a/src/plots.jl b/src/plots.jl index c3123b29decb1b6377178771f6c6b70041d909d9..c5b21a867310f223a32dd1829202ae6012cfda38 100644 --- a/src/plots.jl +++ b/src/plots.jl @@ -474,6 +474,7 @@ end struct DecoratedLarva larva::StatefulLarva activearea + label decorated::AbstractObservable{Bool} end @@ -497,7 +498,8 @@ function DecoratedLarva(larva::StatefulLarva) bl, tr = Meshes.Point2f(l, b), Meshes.Point2f(r, t) br, tl = Meshes.Point2f(r, b), Meshes.Point2f(l, t) activearea = Meshes.Quadrangle(bl, br, tr, tl) - DecoratedLarva(larva, activearea, decorated) + label = string(larva.model.id) + DecoratedLarva(larva, activearea, label, decorated) end function DecoratedLarva(larva::LarvaModel, args...; kwargs...) @@ -524,15 +526,15 @@ function Makie.plot!(plot::LarvaPlot{Tuple{DecoratedLarva}}) #decoratedlarva.primitive_child = plot.plots[1] # decoration - outline = Vector(vertices(decoratedlarva.activearea)) p = Makie.Point2f ∘ coordinates + outline = p.(Vector(vertices(decoratedlarva.activearea))) close!(path) = push!(path, path[1]) - lines!(plot, close!(p.(outline)); + lines!(plot, close!(outline); color=theme[:DecoratedLarva][:hover_color], linewidth=theme[:DecoratedLarva][:hover_linewidth], visible=decorated, ) - # TODO: plot larva ID? + text!(plot, mean(outline); text=decoratedlarva.label, visible=decorated) end struct DecoratedLarvae diff --git a/src/wgl.jl b/src/wgl.jl index af0ca77abc00aa0918ad5df84f56ed4734cd903f..c9cad2c70a4623d433ca57ff2afb8b4ac616f8d9 100644 --- a/src/wgl.jl +++ b/src/wgl.jl @@ -1200,7 +1200,7 @@ function twooptiondialog(controller, answer, title, message, button1, button2) document.getElementById('two-option-dialog-button2').innerHTML = $button2; """) tod.visible[] = true - # TODO: remove previous listener + Observables.clear(tod.answer) on(tod.answer) do ans tod.visible[] = false answer[] = ans