diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b4992efebe234b442fdc879a2fd987715c7222a..816aa6341432dddf862ded754ec8239a90aec8bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,17 +20,17 @@ Julia 1.6: image: julia:1.6 extends: - .script -Julia 1.8: - image: julia:1.8 - extends: - - .script Julia 1.9: image: julia:1.9 + extends: + - .script +Julia 1.10: + image: julia:1.10 extends: - .script - .coverage pages: - image: julia:1.9 + image: julia:1.10 stage: deploy script: - | diff --git a/Project.toml b/Project.toml index 2d1b72ce8e89e8dff6973700501cb4a4ac3910b7..cc796429ed65fcccba56db8c43dc9aefd045c37a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PlanarLarvae" uuid = "c2615984-ef14-4d40-b148-916c85b43307" authors = ["François Laurent", "Institut Pasteur"] -version = "0.15" +version = "0.16" [deps] DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" diff --git a/README.md b/README.md index f574a8c4652da33fe4e2012dd93383f32ba8974a..763b018eb7eee058568daddab1243827a5ef4de2 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,9 @@ Common fields in the `metadata` object are `software`->`tagger` and `camera`->(` Since LarvaTagger==0.10, LarvaTagger.jl relies on the presence or absence of a `metadata`->`software`->`tagger` object in the file to decide whether the tagging was automatic or manual respectively. +Since PlanarLarvae==0.16, if a label file lists several data dependencies, if available the `metadata->filename` element is used to identify a “primary” data dependency, and additional dependencies are inferred from this primary dependency. +This allows distinguishing between conflicting dependencies. Indeed, the `dependencies` object may list unrelated files if the label file was generated with MaggotUBA-adapter<0.19, including files that are valid but alternative tracking data. + Since PlanarLarvae==0.8 and LarvaTagger==0.10, an array-typed `secondarylabels` top-level element can be added along with `labels`. While the labels listed in the `labels` object are considered as primary tags, the `secondarylabels` array lists secondary tags that are expected to be used solely in combination with primary tags. In encoded labels, secondary labels are indexed like if they were listed in the same array as primary labels, but immediately after. diff --git a/src/Chore.jl b/src/Chore.jl index 63dbb92b674a81127f1d31227ed042e90e40ec42..147c3f41eb4b73ea38c5c38b5b7abd3209a50854 100644 --- a/src/Chore.jl +++ b/src/Chore.jl @@ -231,15 +231,18 @@ function parse_filename(filepath::String) stem, ext = splitext(filename) if ext == ".spine" || ext == ".outline" parts = split(stem, '@') - if length(parts) == 6 + if length(parts) == 6 && is_date_time(parts[1]) metadata[:date_time] = parts[1] metadata[:genotype] = parts[2] metadata[:effector] = parts[3] metadata[:tracker] = parts[4] metadata[:protocol] = parts[5] - @assert is_date_time(metadata[:date_time]) - @assert is_tracker(metadata[:tracker]) - @assert is_protocol(metadata[:protocol]) + if !is_tracker(metadata[:tracker]) + @warn "Unexpected tracker name" tracker=metadata[:tracker] + end + if !is_protocol(metadata[:protocol]) + @warn "Protocol does not include 4 #-separated fields" protocol=metadata[:protocol] + end else @warn "Failed to identify metadata in file stem: \"$stem\"" end diff --git a/src/Formats.jl b/src/Formats.jl index 0993c935d8238023b9d87497358934aaf86f6b26..b07ab65cd0bd64eacb932238ebead64b701fc023 100644 --- a/src/Formats.jl +++ b/src/Formats.jl @@ -314,11 +314,28 @@ end Vector of associated track data file names. """ -getdependencies(file) = String[] +getdependencies(_) = String[] getdependencies(path::String) = getdependencies(preload(path)) function getdependencies(file::JSONLabels) isempty(file.run) && load!(file) - return Datasets.getdependencies(file.run, file.source) + deps = Datasets.getdependencies(file.run, file.source) + if 1 < length(deps) + metadata = getmetadata(file) + if :filename in keys(metadata) + filename = metadata[:filename] + k = findfirst(deps) do dep + basename(dep) == filename + end + @assert !isnothing(k) "Filename in metadata does not match any data dependency" + if 1 < k + @debug "The primary data dependency does not come first" + # swap or shift? + #deps[1], deps[k] = deps[k], deps[1] + deps = deps[vcat(k, 1:(k-1), (k+1):end)] + end + end + end + return deps end """ @@ -331,7 +348,17 @@ Unlike [`getdependencies`](@ref), this function pre-loads all the data dependenc function getdependencies!(file::JSONLabels) deps = getdependencies(file) @assert !isempty(deps) - file.dependencies = preload.(deps; metadata=get(file.run.attributes, :metadata, nothing)) + isempty(file.dependencies) || empty!(file.dependencies) + foreach(deps) do dep + try + dep = preload(dep; metadata=get(file.run.attributes, :metadata, nothing)) + # try .. else syntax not supported in Julia 1.6 + push!(file.dependencies, dep) + catch + exc, _ = current_exceptions()[end] + @warn "Failed to preload data dependency" error=exc + end + end deps end diff --git a/test/runtests.jl b/test/runtests.jl index 7202e3dc581773a0d58d934a02c5ce9f4a2c9efc..a0f90fd0c14941831c380cd769bd2382c114c0da 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -937,8 +937,10 @@ if all_tests || "Dataloaders" in ARGS # default rng changed with Julia 1.7 @test lengths == if VERSION < v"1.7" [66, 66, 65, 66, 66, 66, 66, 66, 65, 65, 66, 65, 66, 66, 66, 66, 66, 65, 65, 66, 66, 66, 66, 65, 65, 66, 65, 66, 63, 64, 65, 67, 65, 66, 65, 65, 66, 66, 65, 66, 65, 65, 63, 63, 63, 66, 66, 65, 66, 66, 65, 65, 65, 66, 66, 66, 66, 65, 66, 66, 66, 66, 65, 65, 66, 66, 66, 65, 65, 65, 65, 63, 63, 65, 66, 66, 66, 66, 66, 65, 66, 66, 65, 65, 65, 65, 65, 66, 66, 66, 66, 63, 63, 65, 65, 65, 65, 63, 66, 67, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 65, 65, 65, 66, 65, 66, 66, 66, 65, 65, 65, 65, 65, 66, 65, 63, 63, 63, 63, 63, 66, 65, 66, 65, 66, 64, 65, 66, 64, 64, 64, 64, 65, 65, 65, 65, 66, 66, 66, 66, 63, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 63, 64, 63, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 64, 65, 64, 65, 66, 66, 65, 65, 66, 66, 66, 65, 65, 66, 66, 66, 66, 66, 65, 66, 66, 66, 65, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 63, 63, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 66, 66, 66, 65, 65, 66, 66, 66, 66, 65, 65, 66, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 65, 65, 65, 65, 65, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 65, 66, 66, 65, 65, 65, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 63, 63, 63, 63, 65, 63, 63, 64, 64, 64, 64, 64, 66, 66, 66, 65, 65, 65, 66, 66, 66, 66, 66, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 65, 65, 66, 66, 66, 65, 65, 65, 66, 65, 66, 65, 65, 65, 65, 65, 65, 65, 66, 65, 65, 66, 66, 65, 64, 64, 64, 65, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 63, 63, 66, 65, 63, 66, 66, 66, 66, 65, 66, 66, 66, 66, 66, 66, 66, 65, 65, 66, 66, 66, 66, 65, 63, 64, 64, 65, 64, 64, 65, 66, 66, 66, 66, 65, 66, 66, 66, 63, 63, 63, 64, 64, 64, 65, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 62, 62, 62, 63, 63, 64, 64, 66, 64, 64, 65, 65, 63, 65, 64, 64, 66, 65, 64, 64, 63] - else + elseif VERSION < v"1.10" [65, 65, 66, 64, 66, 66, 67, 66, 66, 66, 65, 66, 66, 65, 66, 65, 65, 66, 66, 65, 66, 66, 65, 65, 66, 66, 66, 65, 65, 66, 66, 66, 66, 66, 65, 65, 66, 64, 64, 66, 66, 66, 66, 66, 66, 65, 66, 65, 64, 63, 63, 64, 65, 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 65, 65, 66, 66, 65, 65, 64, 63, 63, 63, 62, 64, 66, 66, 66, 66, 66, 66, 65, 66, 66, 66, 65, 66, 66, 66, 65, 66, 65, 66, 64, 64, 63, 65, 65, 65, 64, 63, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 65, 66, 65, 66, 66, 66, 66, 65, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 63, 63, 66, 66, 66, 66, 66, 65, 65, 66, 66, 64, 63, 63, 64, 66, 65, 65, 63, 64, 64, 64, 63, 63, 63, 64, 64, 65, 66, 65, 65, 65, 64, 66, 65, 65, 66, 65, 66, 66, 65, 65, 65, 66, 65, 65, 65, 65, 65, 66, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 65, 65, 66, 65, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 63, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 64, 65, 65, 67, 65, 65, 63, 63, 63, 62, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 64, 66, 65, 66, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 65, 65, 66, 65, 65, 65, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 64, 64, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 63, 66, 66, 66, 66, 65, 66, 65, 65, 65, 65, 65, 66, 66, 64, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 63, 65, 66, 65, 66, 66, 66, 66, 65, 65, 66, 65, 65, 64, 63, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 62, 63, 63, 64, 64, 64, 65, 65, 64, 65, 63, 63, 64, 63, 66, 66, 65, 65, 66, 64, 65, 65, 64, 66, 66, 66, 65, 66, 66, 65, 66, 66, 65, 65, 68, 66, 66, 65, 66, 66, 66, 63, 66, 66, 66, 66, 66, 63, 63, 63, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 65, 66, 64, 65, 64, 63] + else + [65, 65, 66, 67, 66, 65, 65, 67, 65, 65, 66, 66, 65, 65, 66, 66, 65, 65, 66, 66, 66, 66, 65, 65, 65, 65, 66, 66, 66, 66, 65, 66, 65, 65, 65, 66, 66, 65, 65, 66, 65, 66, 63, 63, 65, 65, 66, 66, 63, 66, 66, 66, 66, 65, 66, 66, 66, 66, 66, 65, 65, 66, 66, 66, 66, 65, 65, 66, 65, 66, 65, 65, 65, 65, 65, 65, 66, 67, 66, 65, 66, 63, 65, 66, 66, 66, 65, 66, 66, 66, 65, 66, 65, 65, 65, 62, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 65, 66, 65, 65, 65, 65, 65, 65, 65, 66, 66, 65, 63, 66, 66, 65, 65, 65, 66, 63, 63, 65, 65, 65, 64, 64, 64, 64, 64, 63, 63, 63, 65, 66, 65, 66, 65, 66, 63, 65, 66, 66, 66, 66, 65, 66, 66, 65, 65, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 66, 65, 65, 66, 66, 66, 65, 65, 66, 66, 66, 63, 63, 63, 63, 63, 63, 63, 62, 62, 63, 63, 63, 63, 64, 63, 63, 66, 66, 65, 65, 66, 65, 65, 66, 66, 66, 66, 66, 65, 66, 67, 66, 66, 66, 64, 64, 63, 64, 65, 66, 66, 66, 66, 66, 65, 66, 66, 66, 66, 66, 65, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 65, 65, 65, 65, 64, 64, 64, 64, 64, 64, 63, 63, 63, 63, 66, 65, 65, 65, 65, 67, 66, 66, 66, 66, 66, 66, 66, 66, 66, 65, 66, 66, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 65, 65, 65, 65, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 65, 66, 66, 66, 65, 65, 66, 65, 65, 65, 65, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 64, 64, 64, 64, 64, 65, 65, 66, 66, 66, 66, 65, 65, 66, 65, 65, 65, 65, 65, 65, 66, 66, 66, 64, 63, 64, 64, 65, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 63, 66, 66, 66, 66, 65, 65, 65, 65, 66, 66, 66, 65, 63, 64, 64, 64, 64, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 65, 65, 65, 63, 63, 64, 63, 65, 66, 66, 66, 65, 66, 63, 66, 66, 66, 66, 65, 65, 66, 66, 65, 66, 66, 65, 65, 66, 66, 66, 65, 66, 65, 63, 64, 66, 66, 65, 66, 65, 65, 65, 64, 63, 63, 63, 63, 63, 64, 64, 64, 64, 65, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 65, 63, 66, 64, 64, 64, 64, 64, 63, 65, 64, 65, 63, 63, 64, 64, 64] end # repo5 = Repository(run1)