diff --git a/Manifest.toml b/Manifest.toml index 8b545901796d99e1d262398eab23dfa338d7766e..ee237c14ebdbcb0c0fa6f47fe6721f7f7949e049 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.9.0" manifest_format = "2.0" -project_hash = "2a3d26785ece5a51ad6d73d93147b49dbedd4551" +project_hash = "34327d7ea7e42bf13c02b6c70751c90982e1f1be" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -164,11 +164,6 @@ git-tree-sha1 = "d05d9e7b7aedff4e5b51a029dced05cfb6125781" uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" version = "0.6.2" -[[deps.Crayons]] -git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" -uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.1.1" - [[deps.DataAPI]] git-tree-sha1 = "8da84edb865b0b5b0100c0666a9bc9a0b71c553c" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" @@ -207,9 +202,9 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[deps.Distributions]] deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns", "Test"] -git-tree-sha1 = "5eeb2bd01e5065090ad591a205d8cad432ae6cb6" +git-tree-sha1 = "c72970914c8a21b36bbc244e9df0ed1834a0360b" uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.25.93" +version = "0.25.95" [deps.Distributions.extensions] DistributionsChainRulesCoreExt = "ChainRulesCore" @@ -294,9 +289,9 @@ uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" [[deps.FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] -git-tree-sha1 = "3cce72ec679a5e8e6a84ff09dd03b721de420cfe" +git-tree-sha1 = "589d3d3bff204bdd80ecc53293896b4f39175723" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "1.0.1" +version = "1.1.1" [[deps.FixedPointNumbers]] deps = ["Statistics"] @@ -342,9 +337,9 @@ version = "1.0.10+0" [[deps.GPUArraysCore]] deps = ["Adapt"] -git-tree-sha1 = "1cd7f0af1aa58abc02ea1d872953a97359cb87fa" +git-tree-sha1 = "2d6ca471a6c7b536127afccfa7564b5b39227fe0" uuid = "46192b85-c4d5-4398-a991-12ede77f4527" -version = "0.1.4" +version = "0.1.5" [[deps.GeoInterface]] deps = ["Extents"] @@ -401,9 +396,9 @@ version = "1.0.2" [[deps.HDF5]] deps = ["Compat", "HDF5_jll", "Libdl", "Mmap", "Random", "Requires", "UUIDs"] -git-tree-sha1 = "3dab31542b3da9f25a6a1d11159d4af8fdce7d67" +git-tree-sha1 = "c73fdc3d9da7700691848b78c61841274076932a" uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" -version = "0.16.14" +version = "0.16.15" [[deps.HDF5_jll]] deps = ["Artifacts", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenSSL_jll", "Pkg", "Zlib_jll"] @@ -532,9 +527,9 @@ uuid = "f1662d9f-8043-43de-a69a-05efc1cc6ff4" version = "0.1.1" [[deps.IterTools]] -git-tree-sha1 = "fa6287a4469f5e048d763df38279ee729fbd44e5" +git-tree-sha1 = "4ced6667f9974fc5c5943fa5e2ef1ca43ea9e450" uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" -version = "1.4.0" +version = "1.8.0" [[deps.IteratorInterfaceExtensions]] git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" @@ -554,10 +549,10 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.21.4" [[deps.JSON3]] -deps = ["Dates", "Mmap", "Parsers", "SnoopPrecompile", "StructTypes", "UUIDs"] -git-tree-sha1 = "84b10656a41ef564c39d2d477d7236966d2b5683" +deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] +git-tree-sha1 = "5b62d93f2582b09e469b3099d839c2d2ebf5066d" uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" -version = "1.12.0" +version = "1.13.1" [[deps.JSServe]] deps = ["Base64", "CodecZlib", "Colors", "HTTP", "Hyperscript", "JSON3", "LinearAlgebra", "Markdown", "MsgPack", "Observables", "RelocatableFolders", "SHA", "Sockets", "Tables", "Test", "UUIDs", "WebSockets", "WidgetsBase"] @@ -685,9 +680,9 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.LogExpFunctions]] deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] -git-tree-sha1 = "0a1b7c2863e44523180fdb3146534e265a91870b" +git-tree-sha1 = "c3ce8e7420b3a6e071e0fe4745f5d4300e37b13f" uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" -version = "0.3.23" +version = "0.3.24" [deps.LogExpFunctions.extensions] LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" @@ -704,9 +699,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.MAT]] deps = ["BufferedStreams", "CodecZlib", "HDF5", "SparseArrays"] -git-tree-sha1 = "6eff5740c8ab02c90065719579c7aa0eb40c9f69" +git-tree-sha1 = "79fd0b5ee384caf8ebba6c8fb3f365ca3e2c5493" uuid = "23992714-dd62-5051-b70f-ba57cb901cac" -version = "0.10.4" +version = "0.10.5" [[deps.MKL_jll]] deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] @@ -765,10 +760,10 @@ uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" version = "2.28.2+0" [[deps.Meshes]] -deps = ["Bessels", "CircularArrays", "Distances", "IterTools", "LinearAlgebra", "NearestNeighbors", "Random", "ReferenceFrameRotations", "SparseArrays", "StaticArrays", "StatsBase", "Tables", "TransformsBase"] -git-tree-sha1 = "5cd413ae2079f4a98aa8b506add26a1656cb0de5" +deps = ["Bessels", "CircularArrays", "Distances", "IterTools", "LinearAlgebra", "NearestNeighbors", "Random", "Rotations", "SparseArrays", "StaticArrays", "StatsBase", "Tables", "TransformsBase"] +git-tree-sha1 = "a1a152787767f3393362276452897605584964b1" uuid = "eacbb407-ea5a-433e-ab97-5258b1ca43fa" -version = "0.28.1" +version = "0.29.0" [[deps.Missings]] deps = ["DataAPI"] @@ -865,10 +860,10 @@ uuid = "05823500-19ac-5b8b-9628-191a04bc5112" version = "0.8.1+0" [[deps.OpenSSL_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "9ff31d101d987eb9d66bd8b176ac7c277beccd09" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1aa4b74f80b01c6bc2b89992b861b5f210e665b5" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "1.1.20+0" +version = "1.1.21+0" [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] @@ -941,11 +936,11 @@ version = "0.3.2" [[deps.PlanarLarvae]] deps = ["DelimitedFiles", "HDF5", "JSON3", "LinearAlgebra", "MAT", "Meshes", "OrderedCollections", "Random", "SHA", "StaticArrays", "Statistics", "StatsBase", "StructTypes"] -git-tree-sha1 = "0b711a20abae67f235e9f8f26fae41eddd04a766" +git-tree-sha1 = "ff28ccc1ff8bc94478df9d9bfc73a38f2273003e" repo-rev = "main" repo-url = "https://gitlab.pasteur.fr/nyx/planarlarvae.jl" uuid = "c2615984-ef14-4d40-b148-916c85b43307" -version = "0.11.1" +version = "0.11.2" [[deps.PlotUtils]] deps = ["ColorSchemes", "Colors", "Dates", "PrecompileTools", "Printf", "Random", "Reexport", "Statistics"] @@ -960,9 +955,9 @@ version = "0.1.2" [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "259e206946c293698122f63e2b513a7c99a244e8" +git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.1.1" +version = "1.1.2" [[deps.Preferences]] deps = ["TOML"] @@ -992,6 +987,12 @@ git-tree-sha1 = "6ec7ac8412e83d57e313393220879ede1740f9ee" uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" version = "2.8.2" +[[deps.Quaternions]] +deps = ["LinearAlgebra", "Random", "RealDot"] +git-tree-sha1 = "da095158bdc8eaccb7890f9884048555ab771019" +uuid = "94ee1d12-ae83-5a48-8b1c-48b8ff168ae0" +version = "0.7.4" + [[deps.REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" @@ -1007,25 +1008,25 @@ version = "0.3.2" [[deps.Ratios]] deps = ["Requires"] -git-tree-sha1 = "6d7bb727e76147ba18eed998700998e17b8e4911" +git-tree-sha1 = "1342a47bf3260ee108163042310d26f2be5ec90b" uuid = "c84ed2f1-dad5-54f0-aa8e-dbefe2724439" -version = "0.4.4" +version = "0.4.5" weakdeps = ["FixedPointNumbers"] [deps.Ratios.extensions] RatiosFixedPointNumbersExt = "FixedPointNumbers" +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + [[deps.Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" uuid = "189a3867-3050-52da-a836-e630ba90ab69" version = "1.2.2" -[[deps.ReferenceFrameRotations]] -deps = ["Crayons", "LinearAlgebra", "Printf", "Random", "StaticArrays"] -git-tree-sha1 = "ec9bde2e30bc221e05e20fcec9a36a9c315e04a6" -uuid = "74f56ac7-18b3-5285-802d-d4bd4f104033" -version = "3.0.0" - [[deps.RelocatableFolders]] deps = ["SHA", "Scratch"] git-tree-sha1 = "307761d71804208c0c62abdbd0ea6822aa5bbefd" @@ -1050,6 +1051,12 @@ git-tree-sha1 = "6ed52fdd3382cf21947b15e8870ac0ddbff736da" uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" version = "0.4.0+0" +[[deps.Rotations]] +deps = ["LinearAlgebra", "Quaternions", "Random", "StaticArrays"] +git-tree-sha1 = "54ccb4dbab4b1f69beb255a2c0ca5f65a9c82f08" +uuid = "6038ab10-8711-5258-84ad-4b1120ba62dc" +version = "1.5.1" + [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" @@ -1109,12 +1116,6 @@ git-tree-sha1 = "8fb59825be681d451c246a795117f317ecbcaa28" uuid = "45858cf5-a6b0-47a3-bbea-62219f50df47" version = "0.1.2" -[[deps.SnoopPrecompile]] -deps = ["Preferences"] -git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c" -uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" -version = "1.0.3" - [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" diff --git a/Project.toml b/Project.toml index c9af35f5768d227683bd00196caa229030257f57..7266aab1169e6fe2ee3fab70fb979e4e1a17d0cf 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "LarvaTagger" uuid = "8b3b36f1-dfed-446e-8561-ea19fe966a4d" authors = ["François Laurent", "Institut Pasteur"] -version = "0.15.0" +version = "0.15.1" [deps] Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" @@ -26,6 +26,8 @@ WGLMakie = "276b4fcb-3e11-5398-bf8b-a0c2d153d008" [compat] julia = "1.6" +Makie = "< 0.18.0" +PlanarLarvae = ">= 0.11.2" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/README.md b/README.md index 471cd4ce301c88cebb441f7b918fea557dbd56d7..e102b38f9ab06c8be6c54701c8f92a2bfefcad91 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,6 @@ julia --project=. -e 'using Pkg; Pkg.instantiate()' Calling `Pkg.instantiate` in a copy of the project is preferred over using `Pkg.add`, because `LarvaTagger.jl` depends on several unregistered packages. -### With Julia 1.6 - -`LarvaTagger.jl` is known to work also with `julia==1.6`. -However, with this version, `Pkg.instantiate` does not take into account version constraints. -In particular, `Makie` must be manually downgraded to version `0.17.13`: -``` -julia --project=. -e 'using Pkg; Pkg.add(name="Makie", version="0.17.13")' -``` - ### Alternative procedure Users who would prefer not to clone the repository or implicitly use the shipped `Manifest.toml` file @@ -58,9 +49,6 @@ julia --project=. -e 'using Pkg; Pkg.add([ \ PackageSpec(url="https://gitlab.pasteur.fr/nyx/LarvaTagger.jl")]) ``` -Note however that some versions of the dependencies may cause `LarvaTagger.jl` to break. -For example for now, `LarvaTagger.jl` does not support `Makie>=0.18`. - ### Upgrading From a local copy of the git repository, be sure to be on the appropriate branch (in most cases the *main* branch) and update (or *pull*) it: diff --git a/recipes/README.md b/recipes/README.md index 4fb588af797b08f1c42b9a02c537f3ae17c6b8e4..28e0fbabc71e1ba8d277c5e999a27b8138fbb8f9 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -163,7 +163,7 @@ docker pull flaur/larvatagger ``` Beware that images that ship with backends are relatively large files (>5GB on disk). -If you are not interested in automatic tagging, use the `flaur/larvatagger:0.15-standalone` image instead. +If you are not interested in automatic tagging, use the `flaur/larvatagger:0.15.1-standalone` image instead. ### Upgrading diff --git a/scripts/larvatagger.sh b/scripts/larvatagger.sh index db0d73a8f81db097f2d603f28a06caae5d464b9f..faaea47be6714aec8c03c9f08c0c887289fec275 100755 --- a/scripts/larvatagger.sh +++ b/scripts/larvatagger.sh @@ -132,8 +132,14 @@ eval "$docker run $DOCKER_ARGS$RUN_ARGS \"$LARVATAGGER_IMAGE\" $cmd \"/data/$fil train) +if [ "$(basename "$(dirname "$1")")" = "**" ]; then +parentdir=$(cd "$(dirname "$(dirname "$1")")"; pwd -P) +data_repository=$(basename "$(dirname "$1")")/$(basename "$1") +else parentdir=$(cd "$(dirname "$1")"; pwd -P) -data_repository=$(basename "$1"); shift +data_repository=$(basename "$1") +fi +shift RUN_ARGS="$RUN_ARGS -v \"$parentdir\":/data" diff --git a/src/Taggers.jl b/src/Taggers.jl index dbfccf2eaba740b45c9c0c01dc164cbf29bc8644..4503192cfdf89b0ebf73cae8368d8055b0615203 100644 --- a/src/Taggers.jl +++ b/src/Taggers.jl @@ -1,6 +1,6 @@ module Taggers -import PlanarLarvae.Formats +import PlanarLarvae.Formats, PlanarLarvae.Dataloaders export Tagger, isbackend, resetmodel, resetdata, train, predict @@ -80,7 +80,24 @@ function push(tagger::Tagger, inputdata::String) destination = nothing backend_name = basename(realpath(tagger.backend_dir)) raw_data_dir = datadir(tagger, "raw") - if isdir(inputdata) + if occursin("*", inputdata) + repository = Dataloaders.Repository(inputdata) + for file in Formats.find_associated_files(Dataloaders.files(repository)) + src_file = file.source + @debug "Pushing file to backend" backend=backend_name instance=tagger.model_instance file=src_file + dest_file = normpath(joinpath(raw_data_dir, relpath(src_file, repository.root))) + src_file = normpath(src_file) + if dest_file != src_file + dest_dir = dirname(dest_file) + mkpath(dest_dir) + open(src_file, "r") do f + open(dest_file, "w") do g + write(g, read(f)) + end + end + end + end + elseif isdir(inputdata) inputdata = realpath(inputdata) # strip the end slashes rm(raw_data_dir; force=true, recursive=true) destination = raw_data_dir diff --git a/test/scenarii.sh b/test/scenarii.sh index a48ccd48d29ba52ec5b46f78bcffcc7b090b1451..43bf428a766716e9bb6d8ddafa49c882850ce219 100755 --- a/test/scenarii.sh +++ b/test/scenarii.sh @@ -145,6 +145,33 @@ testTrainOneClassWithEncoder() { postTrain $tagger } +testTrainSelectedFiles() { + tagger="test_train_selected_files$tagger_suffix" + tmpdir=$(prepareTrainingData gui_imported.label $tagger trx.mat original_predictions.label partial_predictions.label) + mkdir "$tmpdir/subdir"; mv $tmpdir/*.label "$tmpdir/subdir/"; mv $tmpdir/trx.mat "$tmpdir/subdir/" + echo "(cd \"$tmpdir\" && find . -name gui_imported.label -print > filelist.txt)" + (cd "$tmpdir" && find . -name gui_imported.label -print > filelist.txt) + # run + cd "$tmpdir" + echo "\"$project_root/$larvataggerjl\" train \"$project_root/$maggotuba\" filelist.txt $tagger --seed $seed --labels=\"run_large,cast_large,hunch_large\" --balancing-strategy=maggotuba --iterations=100" + "$project_root/$larvataggerjl" train "$project_root/$maggotuba" filelist.txt $tagger --seed $seed --labels="run_large,cast_large,hunch_large" --balancing-strategy=maggotuba --iterations=100 + # test + cd "$project_root" + postTrain $tagger +} + +testTrainRecursiveSelection() { + tagger="test_train_recursive_selection$tagger_suffix" + tmpdir=$(prepareTrainingData gui_imported.label $tagger trx.mat original_predictions.label partial_predictions.label) + mkdir "$tmpdir/subdir"; mv $tmpdir/*.label "$tmpdir/subdir/"; mv $tmpdir/trx.mat "$tmpdir/subdir/" + # run + cd "$project_root" + echo "\"$larvataggerjl\" train \"$maggotuba\" \"$tmpdir/**/gui_imported.label\" $tagger --seed $seed --labels=\"run_large,cast_large,hunch_large\" --balancing-strategy=maggotuba --iterations=100" + "$larvataggerjl" train "$maggotuba" "$tmpdir/**/gui_imported.label" $tagger --seed $seed --labels="run_large,cast_large,hunch_large" --balancing-strategy=maggotuba --iterations=100 + # test + postTrain $tagger +} + postTrain() { tagger=$1 expected_tagger_dir="$datapath/$tagger"