diff --git a/Manifest.toml b/Manifest.toml index 2639199df275899d2b00bd4beb00b26bd1757989..2b23b029f8d7c7584b90832b8b26e8cb1441dda2 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.8.5" +julia_version = "1.9.0" manifest_format = "2.0" project_hash = "2c20afabe03d014276e9478d0fdccbc2cdd634c1" @@ -10,10 +10,14 @@ uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" version = "0.4.4" [[deps.Adapt]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "0310e08cb19f5da31d08341c6120c047598f5b9c" +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "76289dc51920fdc6e0013c872ba9551d54961c24" uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "3.5.0" +version = "3.6.2" +weakdeps = ["StaticArrays"] + + [deps.Adapt.extensions] + AdaptStaticArraysExt = "StaticArrays" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -35,46 +39,38 @@ git-tree-sha1 = "bb065b14d7f941b8617bc323063dbe79f55d16ea" uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" version = "1.1.0" -[[deps.ChainRulesCore]] -deps = ["Compat", "LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "c6d890a52d2c4d55d326439580c3b8d0875a77d9" -uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "1.15.7" - -[[deps.ChangesOfVariables]] -deps = ["ChainRulesCore", "LinearAlgebra", "Test"] -git-tree-sha1 = "844b061c104c408b24537482469400af6075aae4" -uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" -version = "0.1.5" - [[deps.CircularArrays]] deps = ["OffsetArrays"] -git-tree-sha1 = "3587fdbecba8c44f7e7285a1957182711b95f580" +git-tree-sha1 = "61bc114e595167090b4cbcb7305ddeacd4274f16" uuid = "7a955b69-7140-5f4e-a0ed-f168c5e2e749" -version = "1.3.1" +version = "1.3.2" [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da" +git-tree-sha1 = "9c209fb7536406834aa938fb149964b985de6c83" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.0" +version = "0.7.1" [[deps.Compat]] -deps = ["Dates", "LinearAlgebra", "UUIDs"] -git-tree-sha1 = "3ca828fe1b75fa84b021a7860bd039eaea84d2f2" +deps = ["UUIDs"] +git-tree-sha1 = "7a60c856b9fa189eb34f5f8a6f6b5529b7942957" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.3.0" +version = "4.6.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.0.1+0" +version = "1.0.2+0" [[deps.Conda]] deps = ["Downloads", "JSON", "VersionParsing"] -git-tree-sha1 = "6e47d11ea2776bc5627421d59cdcc1296c058071" +git-tree-sha1 = "e32a90da027ca45d84678b826fffd3110bb3fc90" uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" -version = "1.7.0" +version = "1.8.0" [[deps.Crayons]] git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" @@ -82,9 +78,9 @@ uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" version = "4.1.1" [[deps.DataAPI]] -git-tree-sha1 = "e8119c1a33d267e16108be441a287a6981ba1630" +git-tree-sha1 = "8da84edb865b0b5b0100c0666a9bc9a0b71c553c" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.14.0" +version = "1.15.0" [[deps.DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] @@ -103,13 +99,15 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[deps.DelimitedFiles]] deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" [[deps.Distances]] deps = ["LinearAlgebra", "SparseArrays", "Statistics", "StatsAPI"] -git-tree-sha1 = "3258d0659f812acde79e8a74b11f17ac06d0ca04" +git-tree-sha1 = "49eba9ad9f7ead780bfb7ee319f962c811c6d3b2" uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" -version = "0.10.7" +version = "0.10.8" [[deps.DocStringExtensions]] deps = ["LibGit2"] @@ -126,10 +124,10 @@ version = "1.6.0" uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" [[deps.HDF5]] -deps = ["Compat", "HDF5_jll", "Libdl", "Mmap", "Random", "Requires"] -git-tree-sha1 = "899f041bf330ebeead3637073b2ca7477760edde" +deps = ["Compat", "HDF5_jll", "Libdl", "Mmap", "Random", "Requires", "UUIDs"] +git-tree-sha1 = "3dab31542b3da9f25a6a1d11159d4af8fdce7d67" uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" -version = "0.16.11" +version = "0.16.14" [[deps.HDF5_jll]] deps = ["Artifacts", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenSSL_jll", "Pkg", "Zlib_jll"] @@ -141,16 +139,10 @@ version = "1.12.2+2" deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -[[deps.InverseFunctions]] -deps = ["Test"] -git-tree-sha1 = "49510dfcb407e572524ba94aeae2fced1f3feb0f" -uuid = "3587e190-3f89-42d0-90ee-14403ec27112" -version = "0.1.8" - [[deps.IrrationalConstants]] -git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151" +git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" -version = "0.1.1" +version = "0.2.2" [[deps.IterTools]] git-tree-sha1 = "fa6287a4469f5e048d763df38279ee729fbd44e5" @@ -170,9 +162,9 @@ version = "1.4.1" [[deps.JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "3c837543ddb02250ef42f4738347454f95079d4e" +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.3" +version = "0.21.4" [[deps.JSON3]] deps = ["Dates", "Mmap", "Parsers", "SnoopPrecompile", "StructTypes", "UUIDs"] @@ -207,23 +199,33 @@ version = "1.10.2+0" uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [[deps.LinearAlgebra]] -deps = ["Libdl", "libblastrampoline_jll"] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.LogExpFunctions]] -deps = ["ChainRulesCore", "ChangesOfVariables", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"] -git-tree-sha1 = "680e733c3a0a9cea9e935c8c2184aea6a63fa0b5" +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "0a1b7c2863e44523180fdb3146534e265a91870b" uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" -version = "0.3.21" +version = "0.3.23" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.MAT]] deps = ["BufferedStreams", "CodecZlib", "HDF5", "SparseArrays"] -git-tree-sha1 = "971be550166fe3f604d28715302b58a3f7293160" +git-tree-sha1 = "6eff5740c8ab02c90065719579c7aa0eb40c9f69" uuid = "23992714-dd62-5051-b70f-ba57cb901cac" -version = "0.10.3" +version = "0.10.4" [[deps.MacroTools]] deps = ["Markdown", "Random"] @@ -238,19 +240,19 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.0+0" +version = "2.28.2+0" [[deps.Memoization]] deps = ["MacroTools"] -git-tree-sha1 = "55dc27dc3d663900d1d768822528960acadc012a" +git-tree-sha1 = "2f6913923a0cb8046134f5cbf8b4d7ba3c856a1d" uuid = "6fafb56a-5788-4b4e-91ca-c0cea6611c73" -version = "0.1.14" +version = "0.2.0" [[deps.Meshes]] deps = ["Bessels", "CircularArrays", "Distances", "IterTools", "LinearAlgebra", "NearestNeighbors", "Random", "ReferenceFrameRotations", "SparseArrays", "StaticArrays", "StatsBase", "Tables", "TransformsBase"] -git-tree-sha1 = "6c5b1552be2e2b76894211cd2083bdf6fe2d42c4" +git-tree-sha1 = "5cd413ae2079f4a98aa8b506add26a1656cb0de5" uuid = "eacbb407-ea5a-433e-ab97-5258b1ca43fa" -version = "0.27.0" +version = "0.28.1" [[deps.Missings]] deps = ["DataAPI"] @@ -263,7 +265,7 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2022.2.1" +version = "2022.10.11" [[deps.NearestNeighbors]] deps = ["Distances", "StaticArrays"] @@ -284,29 +286,29 @@ version = "1.12.9" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.20+0" +version = "0.3.21+4" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "e60321e3f2616584ff98f0a4f18d98ae6f89bbb3" +git-tree-sha1 = "9ff31d101d987eb9d66bd8b176ac7c277beccd09" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "1.1.17+0" +version = "1.1.20+0" [[deps.OrderedCollections]] -git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" +git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.4.1" +version = "1.6.0" [[deps.Parsers]] -deps = ["Dates"] -git-tree-sha1 = "6c01a9b494f6d2a9fc180a08b182fcb06f0958a0" +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "a5aef8d4a6e8d81f171b2bd4be5265b01384c74c" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.4.2" +version = "2.5.10" [[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.8.0" +version = "1.9.0" [[deps.PlanarLarvae]] deps = ["DelimitedFiles", "HDF5", "JSON3", "LinearAlgebra", "MAT", "Meshes", "OrderedCollections", "Random", "SHA", "StaticArrays", "Statistics", "StatsBase", "StructTypes"] @@ -316,11 +318,17 @@ repo-url = "https://gitlab.pasteur.fr/nyx/planarlarvae.jl" uuid = "c2615984-ef14-4d40-b148-916c85b43307" version = "0.10.0" +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "259e206946c293698122f63e2b513a7c99a244e8" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.1.1" + [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "47e5f437cc0e7ef2ce8406ce1e7e24d44915f88d" +git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.3.0" +version = "1.4.0" [[deps.Printf]] deps = ["Unicode"] @@ -328,9 +336,9 @@ uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" [[deps.PyCall]] deps = ["Conda", "Dates", "Libdl", "LinearAlgebra", "MacroTools", "Serialization", "VersionParsing"] -git-tree-sha1 = "53b8b07b721b77144a0fbbbc2675222ebf40a02d" +git-tree-sha1 = "62f417f6ad727987c755549e9cd88c46578da562" uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" -version = "1.94.1" +version = "1.95.1" [[deps.REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] @@ -375,14 +383,14 @@ uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" version = "1.1.0" [[deps.SparseArrays]] -deps = ["LinearAlgebra", "Random"] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.StaticArrays]] deps = ["LinearAlgebra", "Random", "StaticArraysCore", "Statistics"] -git-tree-sha1 = "f86b3a049e5d05227b10e15dbb315c5b90f14988" +git-tree-sha1 = "8982b3607a212b070a5e46eea83eb62b4744ae12" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.5.9" +version = "1.5.25" [[deps.StaticArraysCore]] git-tree-sha1 = "6b7ba252635a5eff6a0b0664a41ee140a1c9e72a" @@ -392,12 +400,13 @@ version = "1.4.0" [[deps.Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.9.0" [[deps.StatsAPI]] deps = ["LinearAlgebra"] -git-tree-sha1 = "f9af7f195fb13589dd2e2d57fdb401717d2eb1f6" +git-tree-sha1 = "45a7769a04a3cf80da1c1c7c60caf932e6f4c9f7" uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" -version = "1.5.0" +version = "1.6.0" [[deps.StatsBase]] deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] @@ -411,10 +420,15 @@ git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70" uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" version = "1.10.0" +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+6" + [[deps.TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.0" +version = "1.0.3" [[deps.TableTraits]] deps = ["IteratorInterfaceExtensions"] @@ -424,14 +438,14 @@ version = "1.0.1" [[deps.Tables]] deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] -git-tree-sha1 = "c79322d36826aa2f4fd8ecfa96ddb47b174ac78d" +git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.10.0" +version = "1.10.1" [[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.1" +version = "1.10.0" [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] @@ -439,9 +453,9 @@ uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[deps.TranscodingStreams]] deps = ["Random", "Test"] -git-tree-sha1 = "8a75929dcd3c38611db2f8d08546decb514fcadf" +git-tree-sha1 = "9a6ae7ed916312b41236fcef7e0af564ef934769" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.9.9" +version = "0.9.13" [[deps.TransformsBase]] deps = ["AbstractTrees"] @@ -464,12 +478,12 @@ version = "1.3.0" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.12+3" +version = "1.2.13+0" [[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl", "OpenBLAS_jll"] +deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.1.1+0" +version = "5.7.0+0" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] diff --git a/src/LarvaDatasets.jl b/src/LarvaDatasets.jl index 2f6f45c6d3e9aeaf4718d97e8bf3062c187366a9..e80675b0ba8396b9947717e374a58c3b5686e4fb 100644 --- a/src/LarvaDatasets.jl +++ b/src/LarvaDatasets.jl @@ -626,8 +626,14 @@ function new_write_larva_dataset_hdf5(output_dir, input_data; win = window_length # shorter name to keep next line within the allowed text width output_file = "larva_dataset_$(date)_$(win)_$(win)_$(total_sample_size).hdf5" output_file = joinpath(output_dir, output_file) + c = Threads.Condition() h5open(output_file, "w") do h5 - g = create_group(h5, "samples") + lock(c) + g = try + create_group(h5, "samples") + finally + unlock(c) + end sample(loader, :spine) do _, file, counts, segments sampleid, nsegments = counts @assert length(segments) == nsegments @@ -656,34 +662,45 @@ function new_write_larva_dataset_hdf5(output_dir, input_data; name = "sample_$sampleid" # transpose for compatibility with h5py # see issue https://github.com/JuliaIO/HDF5.jl/issues/785 - g[name] = permutedims(sample, reverse(1:ndims(sample))) - sampleid += 1 - # - d = g[name] - attributes(d)["larva_number"] = convert(Int, trackid) - # we should set `start_point` instead of `reference_time`, to comply with the - # original format, but this would not make sense here due to interpolation: - attributes(d)["reference_time"] = anchortime - attributes(d)["behavior"] = string(label) - attributes(d)["path"] = file.source + lock(c) + try + g[name] = permutedims(sample, reverse(1:ndims(sample))) + sampleid += 1 + # + d = g[name] + attributes(d)["larva_number"] = convert(Int, trackid) + # we should set `start_point` instead of `reference_time`, to comply with + # the original format, but this would not make sense here due to + # interpolation: + attributes(d)["reference_time"] = anchortime + attributes(d)["behavior"] = string(label) + attributes(d)["path"] = file.source + finally + unlock(c) + end end end - attributes(g)["len_traj"] = window_length - attributes(g)["len_pred"] = window_length - attributes(g)["n_samples"] = total_sample_size - # extensions - counts = Dataloaders.total(index.targetcounts) - if isnothing(labels) - h5["labels"] = string.(keys(classcounts)) - h5["label_counts"] = collect(values(classcounts)) - else - # ensure labels are ordered as provided in input; - # see https://gitlab.pasteur.fr/nyx/TaggingBackends/-/issues/24 - h5["labels"] = labels - h5["label_counts"] = [classcounts[Symbol(label)] for label in labels] - end - if !isnothing(frameinterval) - attributes(g)["frame_interval"] = frameinterval + lock(c) + try + attributes(g)["len_traj"] = window_length + attributes(g)["len_pred"] = window_length + attributes(g)["n_samples"] = total_sample_size + # extensions + counts = Dataloaders.total(index.targetcounts) + if isnothing(labels) + h5["labels"] = string.(keys(classcounts)) + h5["label_counts"] = collect(values(classcounts)) + else + # ensure labels are ordered as provided in input; + # see https://gitlab.pasteur.fr/nyx/TaggingBackends/-/issues/24 + h5["labels"] = labels + h5["label_counts"] = [classcounts[Symbol(label)] for label in labels] + end + if !isnothing(frameinterval) + attributes(g)["frame_interval"] = frameinterval + end + finally + unlock(c) end end return output_file