diff --git a/Manifest.toml b/Manifest.toml index 3068263ed285b898e4ee090aea84968067804d1e..aed827e057d313b7f8fa5c0cbd1c2b2221de3dc1 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.8.2" manifest_format = "2.0" -project_hash = "d37bc0692e4cb75dce4365cae9dc31c02a907c73" +project_hash = "4e8a5dcb55ba02909578a76b57750f4d6fb0a766" [[deps.AbstractFFTs]] deps = ["ChainRulesCore", "LinearAlgebra"] @@ -147,6 +147,12 @@ deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" version = "0.5.2+0" +[[deps.Conda]] +deps = ["Downloads", "JSON", "VersionParsing"] +git-tree-sha1 = "6e47d11ea2776bc5627421d59cdcc1296c058071" +uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" +version = "1.7.0" + [[deps.Contour]] git-tree-sha1 = "d05d9e7b7aedff4e5b51a029dced05cfb6125781" uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" @@ -670,6 +676,12 @@ git-tree-sha1 = "2ce8695e1e699b68702c03402672a69f54b8aca9" uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" version = "2022.2.0+0" +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "42324d08725e200c23d4dfb549e0d5d89dede2d2" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.10" + [[deps.Makie]] deps = ["Animations", "Base64", "ColorBrewer", "ColorSchemes", "ColorTypes", "Colors", "Contour", "Distributions", "DocStringExtensions", "FFMPEG", "FileIO", "FixedPointNumbers", "Formatting", "FreeType", "FreeTypeAbstraction", "GeometryBasics", "GridLayoutBase", "ImageIO", "IntervalSets", "Isoband", "KernelDensity", "LaTeXStrings", "LinearAlgebra", "MakieCore", "Markdown", "Match", "MathTeXEngine", "Observables", "OffsetArrays", "Packing", "PlotUtils", "PolygonOps", "Printf", "Random", "RelocatableFolders", "Serialization", "Showoff", "SignedDistanceFields", "SparseArrays", "Statistics", "StatsBase", "StatsFuns", "StructArrays", "UnicodeFun"] git-tree-sha1 = "b0323393a7190c9bf5b03af442fc115756df8e59" @@ -714,6 +726,12 @@ deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" version = "2.28.0+0" +[[deps.Memoization]] +deps = ["MacroTools"] +git-tree-sha1 = "55dc27dc3d663900d1d768822528960acadc012a" +uuid = "6fafb56a-5788-4b4e-91ca-c0cea6611c73" +version = "0.1.14" + [[deps.Meshes]] deps = ["Bessels", "CircularArrays", "Distances", "IterTools", "IteratorInterfaceExtensions", "LinearAlgebra", "NearestNeighbors", "Random", "ReferenceFrameRotations", "SparseArrays", "StaticArrays", "StatsBase", "TableTraits", "Tables", "TransformsBase"] git-tree-sha1 = "589bc39cdaa212b352918b4c11cc4d2eaf899822" @@ -891,11 +909,11 @@ version = "0.3.2" [[deps.PlanarLarvae]] deps = ["DelimitedFiles", "HDF5", "JSON3", "MAT", "Meshes", "OrderedCollections", "SHA", "StaticArrays", "Statistics", "StatsBase", "StructTypes"] -git-tree-sha1 = "f11bd0f18657fb80dda4c054dad20551fd972fb3" -repo-rev = "dev" +git-tree-sha1 = "607572b4d9404105e64e5b9b0f2f047bd307eb17" +repo-rev = "main" repo-url = "https://gitlab.pasteur.fr/nyx/planarlarvae.jl" uuid = "c2615984-ef14-4d40-b148-916c85b43307" -version = "0.5.0" +version = "0.6.0" [[deps.PlotUtils]] deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "SnoopPrecompile", "Statistics"] @@ -924,6 +942,12 @@ git-tree-sha1 = "d7a7aef8f8f2d537104f170139553b14dfe39fe9" uuid = "92933f4c-e287-5a05-a399-4b506db050ca" version = "1.7.2" +[[deps.PyCall]] +deps = ["Conda", "Dates", "Libdl", "LinearAlgebra", "MacroTools", "Serialization", "VersionParsing"] +git-tree-sha1 = "53b8b07b721b77144a0fbbbc2675222ebf40a02d" +uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +version = "1.94.1" + [[deps.QOI]] deps = ["ColorTypes", "FileIO", "FixedPointNumbers"] git-tree-sha1 = "18e8f4d1426e965c7b532ddd260599e1510d26ce" @@ -1133,6 +1157,14 @@ git-tree-sha1 = "c79322d36826aa2f4fd8ecfa96ddb47b174ac78d" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" version = "1.10.0" +[[deps.TaggingBackends]] +deps = ["Dates", "HDF5", "LazyArtifacts", "MAT", "Memoization", "OrderedCollections", "PlanarLarvae", "PyCall", "Random", "StaticArrays", "Statistics"] +git-tree-sha1 = "610f42dc290ff67b54b4fbc84edebaec94eba2d8" +repo-rev = "main" +repo-url = "https://gitlab.pasteur.fr/nyx/TaggingBackends" +uuid = "e551f703-3b82-4335-b341-d497b48d519b" +version = "0.5.0" + [[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" @@ -1192,6 +1224,11 @@ git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf" uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1" version = "0.4.1" +[[deps.VersionParsing]] +git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868" +uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" +version = "1.3.0" + [[deps.WGLMakie]] deps = ["Colors", "FileIO", "FreeTypeAbstraction", "GeometryBasics", "Hyperscript", "ImageMagick", "JSServe", "LinearAlgebra", "Makie", "Observables", "RelocatableFolders", "ShaderAbstractions", "StaticArrays"] git-tree-sha1 = "0c7a980515d2072b1bc094668b5ca94671d020de" diff --git a/Project.toml b/Project.toml index 492c4be584ceb11bfbbf6c8adbd5a1275249f6fc..c480193e01a5a3f39f4bb7408449e31231b92f33 100644 --- a/Project.toml +++ b/Project.toml @@ -20,6 +20,7 @@ PlanarLarvae = "c2615984-ef14-4d40-b148-916c85b43307" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +TaggingBackends = "e551f703-3b82-4335-b341-d497b48d519b" TidyObservables = "c8131bbd-73a8-4254-a42d-d5d4c5febb31" WGLMakie = "276b4fcb-3e11-5398-bf8b-a0c2d153d008" diff --git a/recipes/Dockerfile b/recipes/Dockerfile index 9cdb28f932f66abc1090558a5a15045f4649c635..61a03ec76f965107d87bda1c48fe5124a1a703ac 100644 --- a/recipes/Dockerfile +++ b/recipes/Dockerfile @@ -1,19 +1,23 @@ FROM julia:1.8.2-bullseye -ENV DIRNAME app +ENV JULIA_PROJECT=/app +ENV JULIA_DEPOT_PATH=/usr/local/share/julia + ARG BRANCH=main ARG TIMEZONE=UTC RUN apt-get update \ && apt-get install -y git \ && rm -rf /var/lib/apt/lists/* \ - && git clone --depth 1 --single-branch -b $BRANCH https://gitlab.pasteur.fr/nyx/larvatagger.jl $DIRNAME \ - && julia --project=$DIRNAME -e 'using Pkg; Pkg.instantiate()' \ - && ln -s /$DIRNAME/scripts/larvatagger.jl /bin \ + && git clone --depth 1 --single-branch -b $BRANCH https://gitlab.pasteur.fr/nyx/larvatagger.jl $JULIA_PROJECT \ + && mkdir -p $JULIA_DEPOT_PATH \ + && echo "JULIA_DEPOT_PATH=$JULIA_DEPOT_PATH" >> /etc/environment \ + && julia -e 'using Pkg; Pkg.instantiate()' \ + && ln -s $JULIA_PROJECT/scripts/larvatagger.jl /bin \ && ln -snf /usr/share/zoneinfo/$TIMEZONE /etc/localtime \ && echo $TIMEZONE > /etc/timezone ARG BACKEND -RUN cd $DIRNAME; \ +RUN cd $JULIA_PROJECT; \ if [ "$BACKEND" = "MaggotUBA/20221005" ]; then \ apt-get update \ && apt-get install -y python3-pip \ @@ -22,7 +26,7 @@ RUN cd $DIRNAME; \ && cd MaggotUBA \ && python3 -m poetry install \ && python3 -m poetry run python -c 'import julia; julia.install()' \ - && python3 -m poetry run python -c 'from julia.api import Julia; Julia(compiled_modules=False); from julia import Pkg; Pkg.add(url="https://gitlab.pasteur.fr/nyx/planarlarvae.jl", rev="dev"); Pkg.add(url="https://gitlab.pasteur.fr/nyx/TaggingBackends", rev="dev")' \ + #&& python3 -m poetry run python -c "from julia.api import Julia; Julia(compiled_modules=False); from julia import Pkg; Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/planarlarvae.jl\", rev=\"$BRANCH\"); Pkg.add(url=\"https://gitlab.pasteur.fr/nyx/TaggingBackends\", rev=\"$BRANCH\")" \ && cd .. \ && apt-get clean \ && rm -rf /var/lib/apt/lists/*; \ diff --git a/scripts/larvatagger.sh b/scripts/larvatagger.sh index e878a3b2303e72550583ffc2fe4a6cb5ad1ef935..7ff894ced39e07918b88f6c577afb2b66b55a47b 100755 --- a/scripts/larvatagger.sh +++ b/scripts/larvatagger.sh @@ -2,7 +2,18 @@ cmd=$1; shift -if [ -z "$LARVATAGGER_IMAGE" ]; then LARVATAGGER_IMAGE=larvatagger; fi +if [ -z "$LARVATAGGER_IMAGE" ]; then +if [ "$cmd" = "build" -o -n "$(docker images | grep '^larvatagger ')" ]; then +LARVATAGGER_IMAGE=larvatagger +else +LARVATAGGER_IMAGE=flaur/larvatagger +fi +fi + +IO_UID=$(id -u $USER) +IO_GID=$(id -g $USER) +RUN_ARGS="-u $IO_UID:$IO_GID -v $(pwd):/data" +RUN_ARGS="-e IO_UID=$IO_UID -e IO_GID=$IO_GID -v $(pwd):/data" case "$cmd" in @@ -12,22 +23,13 @@ if ! [ -f recipes/Dockerfile ]; then echo "the build command can only be run from the project root directory" exit 1 fi -#mkdir -p ./external while ! [ -z "$1" ]; do if [ "$1" == "--dev" -o "$1" == "--stable" ]; then BUILD=$1; shift elif [ "$1" == "--get-backend" ]; then DOCKER_ARGS="--build-arg BACKEND=MaggotUBA/20221005 "; shift -#cd ./external -#if [ -d structured-temporal-convolution ]; then -#cd structured-temporal-convolution; git pull; cd .. -#else -#git clone --depth 1 --single-branch -b light-stable-for-tagging git@gitlab.pasteur.fr:les-larves/structured-temporal-convolution.git -#fi -#cd .. else echo "argument not supported: $1"; shift -#rm -rf ./external exit 1 fi done @@ -43,7 +45,6 @@ if ! [[ "$LARVATAGGER_IMAGE" == *:* ]]; then LARVATAGGER_IMAGE="${LARVATAGGER_IM if [ -z "$LARVATAGGER_DEFAULT_BRANCH" ]; then LARVATAGGER_DEFAULT_BRANCH=dev; fi docker build -t "$LARVATAGGER_IMAGE" -f recipes/Dockerfile ${DOCKER_ARGS}--build-arg BRANCH=$LARVATAGGER_DEFAULT_BRANCH . fi -#rm -rf ./external ;; open) @@ -56,20 +57,39 @@ DOCKER_ARGS="-p $LARVATAGGER_PORT:$LARVATAGGER_PORT" TAGGER_ARGS="--port=$LARVATAGGER_PORT" fi file=$1; shift -exec docker run -iv $(pwd):/data $DOCKER_ARGS "$LARVATAGGER_IMAGE" open "/data/$file" $TAGGER_ARGS $@ +exec docker run $RUN_ARGS -i $DOCKER_ARGS "$LARVATAGGER_IMAGE" open "/data/$file" $TAGGER_ARGS $@ ;; import) file=$1; shift -docker run -v $(pwd):/data "$LARVATAGGER_IMAGE" import "/data/$file" $@ +docker run $RUN_ARGS "$LARVATAGGER_IMAGE" import "/data/$file" $@ +;; + + train) + +backend_path=$1; shift +data_repository=$1; shift +tagger=$1; shift +docker run $RUN_ARGS "$LARVATAGGER_IMAGE" train "/data/$backend_path" "/data/$data_repository" "$tagger" $@ +;; + + predict) + +backend_path=$1; shift +tagger=$1; shift +data_file=$1; shift +docker run $RUN_ARGS "$LARVATAGGER_IMAGE" predict "/data/$backend_path" "$tagger" "/data/$data_file" $@ ;; *) -echo "usage: $0 build [--stable] [--dev] [--get-backend]" +echo "usage: $0 build [--get-backend]" echo " $0 open <filepath>" echo " $0 import <filepath> [<outputfilename>] [--id=<runid>] [--framerate=<fps>]" +echo " $0 train <backendpath> <datarepository> <taggername>" +echo " $0 predict <backendpath> <taggername> <datafile>" +echo 'see also `larvatagger.jl --help` for more arguments' ;; esac diff --git a/src/LarvaTagger.jl b/src/LarvaTagger.jl index 2c24b91e8272fd5ba0331d390d7e15fcb4087069..2c17d668ec916ae681c920a6aaee17977e3c0cd6 100644 --- a/src/LarvaTagger.jl +++ b/src/LarvaTagger.jl @@ -27,9 +27,9 @@ include("edits.jl") include("plots.jl") include("players.jl") include("controllers.jl") -include("files.jl") include("Taggers.jl") using .Taggers +include("files.jl") include("backends.jl") include("wgl.jl") diff --git a/src/Taggers.jl b/src/Taggers.jl index 834e0c71ad7bfb5132dd3aeea163cd62bd86ff1b..22de6b00be453a7d8e6437d76435e918a791af61 100644 --- a/src/Taggers.jl +++ b/src/Taggers.jl @@ -100,10 +100,28 @@ function pull(tagger::Tagger, dest_dir::String) write(g, read(f)) end end + check_permissions(dest_file) end return dest_file end +""" + check_permissions(filepath) + +Attempt to fix file ownership. +This is useful in a Docker container that runs as root. +Environment variables IO_UID and IO_GID are set in the container by the larvatagger.sh script +so that the actual user is known. +""" +function check_permissions(file) + try + uid, gid = ENV["IO_UID"], ENV["IO_GID"] + @info "Changing file ownership" uid gid file + chown(file, parse(Int, uid), parse(Int, gid)) + catch + end +end + function parsekwargs!(args, kwargs) for (key, value) in pairs(kwargs) isnothing(value) && continue diff --git a/src/files.jl b/src/files.jl index 6e02548ff3f0a9127c8b87385b6814336ec40881..e0453db578231a5c8d4839d704beb2c991d4eba0 100644 --- a/src/files.jl +++ b/src/files.jl @@ -302,6 +302,7 @@ function savetofile(controller, file; datafile=nothing) end end Datasets.to_json_file(filepath, dataset) + Taggers.check_permissions(filepath) end end