Merge branch 'dev' into 'main'

Set of commits to be tagged v0.20.1

Closes #316

See merge request !25

Project.toml

 name = "LarvaTagger"
 uuid = "8b3b36f1-dfed-446e-8561-ea19fe966a4d"
 authors = ["François Laurent", "Institut Pasteur"]
-version = "0.20"
+version = "0.20.1"
 
 [deps]
 Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8"

recipes/release.sh

@@ -14,7 +14,7 @@ docker=docker LARVATAGGER_IMAGE=flaur/larvatagger:$RELEASE-20230311 scripts/larv
 docker tag flaur/larvatagger:$RELEASE-20230311 flaur/larvatagger:latest
 docker build -t flaur/larvatagger:$RELEASE-bigfat -f recipes/Dockerfile.pasteurjanelia --no-cache .
 
-test/predict_and_retrain.sh
+#test/predict_and_retrain.sh
 
 cat <<EOF
 Next steps are:

scripts/install.sh

@@ -50,7 +50,7 @@ fi
 if [ "$1" = "--uninstall" ]; then
   for pkg in MaggotUBA TaggingBackends; do
     if [ -d "$LARVATAGGER_PATH/$pkg" ]; then
-      env=$(cd "$LARVATAGGER_PATH/$pkg" && poetry env info -p)
+      env=$(cd "$LARVATAGGER_PATH/$pkg" && poetry env info -p 2>/dev/null)
       [ -d "$env" ] && rm -rf "$env"
     fi
   done
@@ -278,9 +278,6 @@ EOF
 }
 
 if [ -n "$WITH_BACKEND" ]; then
-  if [ "`uname`" = "Darwin" ]; then
-    echo "WARNING: the default tagging backend is not supported by macOS"
-  fi
   if ! command -v python$PYTHON_VERSION &>/dev/null; then
     if command -v pyenv &>/dev/null; then
       [ `pyenv versions | grep $PYTHON_VERSION` ] || pyenv install $PYTHON_VERSION
@@ -370,7 +367,7 @@ else
 activate() {
   # pyenv activation is necessary on WSL
   command -v pyenv &>/dev/null && [ -n "`pyenv versions | grep '  $PYTHON_VERSION'`" ] && pyenv local $PYTHON_VERSION
-  poetry env use $PYTHON_VERSION
+  poetry env use $PYTHON_VERSION 2>/dev/null
 }
 
 if [ -d TaggingBackends ]; then

src/REST/Client.jl

@@ -219,6 +219,7 @@ end
 function listmodels(back::LTBackend, ::Val{true})
     map(back.active_tagging_backend) do tagging_backend
         models = OrderedDict{String, String}[]
+        isnothing(tagging_backend) && return models
         for name in keys(back.taggers[tagging_backend])
             metadata = back.metadata[tagging_backend][:models][name]
             push!(models, OrderedDict("name" => name,

src/REST/Model.jl

@@ -14,13 +14,15 @@ struct LTBackend
     root
     tokens
     lock
+    token_expiry
 end
 
 function LTBackend()
     root = Ref{AbstractString}("")
     tokens = Dict{String, Dict{String, Dict{String, Float64}}}()
     lock = ReentrantLock()
-    LTBackend(root, tokens, lock)
+    token_expiry = Ref{Union{Nothing, Real}}(nothing)
+    LTBackend(root, tokens, lock, token_expiry)
 end
 
 Base.lock(f::Function, backend::LTBackend) = lock(f, backend.lock)
@@ -66,6 +68,7 @@ end
 
 function gettoken(lt_backend, backend_dir, model_instance)
     tagger = gettagger(lt_backend, backend_dir, model_instance)
+    resetdata(lt_backend) # perform server-wide maintenance
     return tagger.sandbox
 end
 
@@ -88,6 +91,29 @@ function resetdata(lt_backend, backend_dir, model_instance, token, datadir=nothi
     nothing
 end
 
+function resetdata(lt_backend, min_age)
+    isnothing(min_age) && return
+    @assert min_age isa Real
+    lock(lt_backend) do
+        for (backend_dir, instances) in pairs(lt_backend.tokens)
+            for (model_instance, tokens) in pairs(instances)
+                for (token, created) in pairs(copy(tokens))
+                    age = time() - created
+                    if min_age <= age
+                        @info "resetdata" backend_dir model_instance token age
+                        tagger = gettagger(lt_backend, backend_dir, model_instance, token)
+                        Taggers.resetdata(tagger)
+                        pop!(tokens, token)
+                    end
+                end
+            end
+        end
+    end
+    nothing
+end
+
+resetdata(lt_backend) = resetdata(lt_backend, lt_backend.token_expiry[])
+
 function listfiles(lt_backend, backend_dir, model_instance, token, data_dir)
     tagger = gettagger(lt_backend, backend_dir, model_instance, token)
     dir = Taggers.datadir(tagger, data_dir)

src/REST/Server.jl

@@ -17,8 +17,9 @@ end
 # state
 const lt_backend = LTBackend()
 
-function run_backend(root::AbstractString; kwargs...)
+function run_backend(root::AbstractString, token_expiry=nothing; kwargs...)
     lt_backend.root[] = root
+    lt_backend.token_expiry[] = token_expiry
     run_backend(; kwargs...)
 end
 
@@ -49,6 +50,14 @@ end
 end
 
 
+@get "/reset-data/{min_age}" function (
+        request,
+        min_age::Int,
+    )
+    resetdata(lt_backend, min_age)
+end
+
+
 @get "/reset-data/{backend_dir}/{model_instance}/{token}" function(
         request,
         backend_dir::String,

src/backends.jl

@@ -51,7 +51,11 @@ function getbackends(controller, location=nothing)
     else
         if !isnothing(location) && startswith(location, "http://")
             back = REST.Client.LTBackend(location)
-            REST.Client.connect(back; preselect_tagger=true)
+            try
+                REST.Client.connect(back; preselect_tagger=true)
+            catch
+                @error "Failed to connect to backend"
+            end
             controller[:backends] = back
         else
             backends = Backends(controller, location)

src/cli_open.jl

@@ -54,8 +54,14 @@ function main(args=ARGS; exit_on_error=false)
     infile = parsed_args["<file-path>"]
     if isempty(infile)
         infile = nothing
-    elseif !(startswith(infile, "http://") || isfile(infile))
-        if isdir(infile)
+    elseif !startswith(infile, "http://")
+        if isfile(infile)
+            dataroot = dirname(infile)
+            if !isempty(dataroot)
+                cd(dataroot)
+                infile = basename(infile)
+            end
+        elseif isdir(infile)
             dataroot = infile
             infile = nothing
             cd(dataroot)

src/files.jl

@@ -249,6 +249,25 @@ end
 
 getoutput(controller) = gethub(controller)[:output]
 
+function valid_filename(name)
+    if startswith(name, ".")
+        return false
+    end
+    # adapted from NyxUI.jl (MIT license, same author)
+    windows_extra = "|:*?<>"
+    for c in "/\\'\"`" * windows_extra
+        if c in name
+            return false
+        end
+    end
+    for nonprintable in 0x0:0x31
+        if nonprintable in name
+            return false
+        end
+    end
+    return true
+end
+
 interpolate(s="yyyymmdd_HHMMSS") = Dates.format(Dates.now(), s)
 
 function savetofile(controller, file; datafile=nothing, merge=false)
@@ -474,6 +493,11 @@ function getoutputfile(controller)
             dir = cwd(controller)
             if isnothing(file)
                 outputfile.name.val = "{yyyymmdd_HHMMSS}.label"
+            elseif !valid_filename(file)
+                @warn "Invalid filename; saving to date_time format instead" file
+                file = "{yyyymmdd_HHMMSS}.label"
+                savetofile(hub, file)
+                reset!(outputfile)
             elseif isfile(joinpath(dir, file))
                 twooptiondialog(hub, outputfile.merge,
                     "File already exists",

src/larvatagger.js

@@ -118,6 +118,7 @@ const LarvaTagger = (function () {
 		return false;
 	}
 
+	// TODO: validate filepath similarly to valid_filename in files.jl
 	function setOutputFilename(obs) {
 		var defaultfilepath = obs.value;
 		if (defaultfilepath === null) {

test/rest_client.sh

@@ -24,7 +24,7 @@ julia "+$JULIA_VERSION" --project="${larvatagger_jl_project_root}" -q -e "using
 # run and background the backend server
 JULIA_PROJECT="${larvatagger_project_root}/TaggingBackends" \
   julia "+$JULIA_VERSION" --project="${larvatagger_jl_project_root}" -i \
-  -e "using LarvaTagger.REST.Server; run_backend(\"${larvatagger_project_root}\"; port=${lt_backend_port})" &
+  -e "using LarvaTagger.REST.Server; run_backend(\"${larvatagger_project_root}\", 300; port=${lt_backend_port})" &
 lt_backend_pid=$!
 
 # run the frontend server
@@ -36,6 +36,8 @@ JULIA="julia +$JULIA_VERSION" ${larvatagger_jl_project_root}/scripts/larvatagger
 #   expected: a predicted.label is generated and the GUI reloads
 # * load a second tracking data file (binary if first was ascii or vice versa), select another model instance, click again on "Autotag";
 #   expected: a new token was issued + similar outcome as previous step, with tracking data file and tagging model properly identified in the predicted.label file
+# * wait for 5 min and click again on "Autotag";
+#   expected: the data directories corresponding to the previous tokens are empty
 
 kill $lt_backend_pid
 wait $lt_backend_pid