diff --git a/test/deploy_and_test.sh b/test/deploy_and_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..69b6dc7a266592052b574da080ab7049f88555dd --- /dev/null +++ b/test/deploy_and_test.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# Standalone script + +# on Maestro: +# SLURM_OPTS='-p dbc_pmo -q dbc -c 8' JULIA_THREADS=8 ./deploy_and_test.sh + +if [ -z "$(which julia)" ]; then + echo "\`julia\` not found" + exit 1 +fi + +if [ -n "$1" ] && [ "$1" = "-f" ] || [ "$1" = "--force" ]; then + rm -rf LarvaTagger +elif [ -d LarvaTagger ]; then + echo "\`$0\` needs to make a local LarvaTagger directory" + echo "but this directory already exists." + echo "Run \`$0 --force\` or pick another location." + exit 1 +fi + +if [ -n "$LAUNCHER" ]; then + launcher="$LAUNCHER " +elif [ -n "$(which srun)" ]; then + launcher="srun " + echo "Slurm detected" + if [ -z "$SLURM_OPTS" ]; then + echo "Options can be passed to \`srun\` with variable SLURM_OPTS" + else + launcher="$launcher$SLURM_OPTS " + fi +fi + +CURDIR=$(pwd) + +mkdir -p LarvaTagger && cd $_ + +# shunit2 is for tests specifically +if [ -z "$(which shunit2)" ]; then + git clone -c advice.detachedHead=false -b v2.1.8 https://github.com/kward/shunit2 +fi + +# comment out the below line to build a development environment +CLONE_OPTS=" --depth 1 --no-tags --single-branch -b dev" +git clone$CLONE_OPTS https://gitlab.pasteur.fr/nyx/planarlarvae.jl PlanarLarvae +git clone$CLONE_OPTS https://gitlab.pasteur.fr/nyx/LarvaTagger.jl +ln -s $(realpath LarvaTagger.jl) LarvaTagger +git clone$CLONE_OPTS https://gitlab.pasteur.fr/nyx/TaggingBackends +git clone$CLONE_OPTS https://gitlab.pasteur.fr/nyx/MaggotUBA-adapter +ln -s $(realpath MaggotUBA-adapter) MaggotUBA + +(cd TaggingBackends && julia --project=. -e 'using Pkg; Pkg.develop(path="../PlanarLarvae"); Pkg.instantiate()') + +(cd MaggotUBA && poetry install -v && poetry remove taggingbackends && poetry add ../TaggingBackends) + +cd LarvaTagger || exit 1 +julia --project=. -e 'using Pkg; Pkg.develop(path="../PlanarLarvae"); Pkg.instantiate()' + +# the remainder of this script is test-specific + +if [ -f ../../LarvaTagger_test_data.tgz ]; then + # To generate the test data, first run the tests with environment variable KEEP_MODEL_FILES=1 + # copy the LarvaTagger/MaggotUBA/models/test_train_* directories into LarvaTagger/LarvaTagger.jl/test/data + # and build the archive from the LarvaTagger/LarvaTagger.jl directory. + # For example: + # KEEP_MODEL_FILES=1 ./deploy_and_test.sh + # cd LarvaTagger/LarvaTagger.jl + # rm -rf test/data/test_train_* + # cp -ra ../MaggotUBA/models/test_train_* test/data/ + # tar zcvf LarvaTagger_test_data.tgz test/data/* + # mv LarvaTagger_test_data.tgz ../../ + tar zxvf ../../LarvaTagger_test_data.tgz +else + # Not recommended; reproducibility is not guarantee across hosts or architectures yet + wget -O- https://dl.pasteur.fr/fop/l3qruSQR/LarvaTagger_test_data.tgz | tar zxv +fi + +if [ "$LOCAL_SCENARII" = "1" ]; then + if [ -f "$CURDIR/scenarii.sh" ]; then cp "$CURDIR/scenarii.sh" test/ + elif [ -f "$CURDIR/test/scenarii.sh" ]; then cp "$CURDIR/test/scenarii.sh" test/ + else echo "Cannot find a local scenarii.sh file" + fi +fi + +if [ -z "$(which shunit2)" ]; then + ln -s "$CURDIR/LarvaTagger/shunit2/shunit2" test/ +fi + +echo "${launcher}test/scenarii.sh" +${launcher}test/scenarii.sh diff --git a/test/predict_and_retrain.sh b/test/predict_and_retrain.sh index 89f80e65ec8f29835346c4ef25e9c5b6ef875d27..e72d981d6597c39f6b1c004037abee40dd2e63d8 100755 --- a/test/predict_and_retrain.sh +++ b/test/predict_and_retrain.sh @@ -1,4 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash + +# this script performs a few tests with the Docker image +# TODO: implement more use cases as in scenarii.sh trap 'rm -rf "$TMPDIR"' EXIT diff --git a/test/scenarii.sh b/test/scenarii.sh index 5f8932ac7d3709e40f950be4ec1bdc84d585dca6..b0b07816be69a567403b674c680c2e4f11d7e2c2 100755 --- a/test/scenarii.sh +++ b/test/scenarii.sh @@ -1,16 +1,26 @@ -#!/bin/bash +#!/usr/bin/env bash -if [ -z "$(which shunit2)" ]; then - echo "shunit2 is required to run the tests" - exit 1 -fi +# This script performs tests for the scripts/larvatagger-toolkit.jl command. +# See also the deploy_and_test.sh standalone script. + +project_root=$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")") -if [ -z "$(which h5diff)" ]; then - echo "h5diff (hdf5-tools) is required to run the tests" +if [ -f "$project_root/test/shunit2" ]; then + SHUNIT2=$(realpath "$project_root/test/shunit2") +elif [ -z "$(which shunit2)" ]; then + echo "shUnit2 is required to run the tests (see https://github.com/kward/shunit2)" exit 1 +else + SHUNIT2=shunit2 fi -project_root=$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")") +# Since larva_dataset.hdf5 files cannot be compared because of absolute paths +# we no longer need `h5diff` + +# if [ -z "$(which h5diff)" ]; then +# echo "h5diff (hdf5-tools) is required to run the tests" +# exit 1 +# fi if [ -z "$JULIA_PROJECT" ]; then export JULIA_PROJECT=$(realpath "$project_root/../TaggingBackends") @@ -28,8 +38,6 @@ fi larvataggerjl="scripts/larvatagger-toolkit.jl" -# test data temporarily available at https://dl.pasteur.fr/fop/l3qruSQR/LarvaTagger_test_data.tgz - datadir="test/data" datapath="$project_root/$datadir" @@ -59,22 +67,26 @@ prepareTrainingData() { } testImportLabelFile() { + filename=result_imported.label # run cd "$project_root" - echo "\"$larvataggerjl\" import \"$datadir/partial_predictions.label\" imported.label --default-label=\"not back-up\"" - "$larvataggerjl" import "$datadir/partial_predictions.label" result_imported.label --default-label="not back-up" + echo "\"$larvataggerjl\" import \"$datadir/partial_predictions.label\" $filename --default-label=\"not back-up\"" + "$larvataggerjl" import "$datadir/partial_predictions.label" $filename --default-label="not back-up" # test - assertNull '\`import\` failed to reproduce the imported.label file' "$(cmp "$datapath/imported.label" "$datapath/result_imported.label")" + assertTrue '\`import\` failed to reproduce the imported.label file' '$(cmp "$datapath/imported.label" "$datapath/$filename")' # clean up - rm -f "$datapath/result_imported.label" + rm -f "$datapath/$filename" } testPredictDefault() { tagger="test_train_default$tagger_suffix" tmpdir=$(prepareTestData $tagger) # restore the model + mkdir -p "$maggotuba/models" rm -rf "$maggotuba/models/$tagger" + echo "cp -ra \"$datapath/$tagger\" \"$maggotuba/models/\"" cp -ra "$datapath/$tagger" "$maggotuba/models/" + [ -f "$maggotuba/models/$tagger/clf_config.json" ] || exit 1 # run cd "$project_root" echo "\"$larvataggerjl\" predict \"$maggotuba\" $tagger \"$tmpdir/sample.spine\"" @@ -84,7 +96,7 @@ testPredictDefault() { predictions="$tmpdir/$filename" expected_labels="$datapath/$tagger/$filename" assertFalse "\`predict\` failed to generate file $filename" '[ -z "$predictions" ]' - assertNull "\`predict\` failed to reproduce file $filename" "$(cmp "$expected_labels" "$predictions")" + assertTrue "\`predict\` failed to reproduce file $filename" '$(cmp "$expected_labels" "$predictions")' } testTrainDefault() { @@ -127,12 +139,13 @@ postTrain() { interim_dir="$maggotuba/data/interim/$tagger" # interim files - fn=larva_dataset.hdf5 + filename=larva_dataset.hdf5 larva_dataset="$interim_dir/$(cd "$interim_dir"; ls larva_dataset_*.hdf5)" - assertFalse "\`train\` failed to generate file $fn" '[ -z "$larva_dataset" ]' - assertNull "\`train\` failed to reproduce file $fn" "$(h5diff "$expected_tagger_dir/$fn" "$larva_dataset")" + assertFalse "\`train\` failed to generate file $filename" '[ -z "$larva_dataset" ]' + # fails because of absolute paths + #assertTrue "\`train\` failed to reproduce file $filename" '$(h5diff "$expected_tagger_dir/$filename" "$larva_dataset")' if [ "$KEEP_MODEL_FILES" = "1" ]; then - cp "$larva_dataset" "$tagger_dir/$fn" + cp "$larva_dataset" "$tagger_dir/$filename" fi # test model files @@ -148,17 +161,17 @@ postTrain() { assertFalse "\`train\` failed to generate file $enc_config" '[ -z "$enc_config_path" ]' assertFalse "\`train\` failed to generate file $clf_weights" '[ -z "$clf_weights_path" ]' assertFalse "\`train\` failed to generate file $clf_config" '[ -z "$clf_config_path" ]' - assertNull "\`train\` failed to reproduce file $enc_weights" "$(cmp "$expected_tagger_dir/$enc_weights" "$enc_weights_path")" - assertNull "\`train\` failed to reproduce file $enc_config" "$(cmp "$expected_tagger_dir/$enc_config" "$enc_config_path")" - assertNull "\`train\` failed to reproduce file $clf_weights" "$(cmp "$expected_tagger_dir/$clf_weights" "$clf_weights_path")" - assertNull "\`train\` failed to reproduce file $clf_config" "$(cmp "$expected_tagger_dir/$clf_config" "$clf_config_path")" + assertTrue "\`train\` failed to reproduce file $enc_weights" '$(cmp "$expected_tagger_dir/$enc_weights" "$enc_weights_path")' + assertTrue "\`train\` failed to reproduce file $enc_config" '$(cmp "$expected_tagger_dir/$enc_config" "$enc_config_path")' + assertTrue "\`train\` failed to reproduce file $clf_weights" '$(cmp "$expected_tagger_dir/$clf_weights" "$clf_weights_path")' + assertTrue "\`train\` failed to reproduce file $clf_config" '$(cmp "$expected_tagger_dir/$clf_config" "$clf_config_path")' # test the predictions echo "\"$larvataggerjl\" predict \"$maggotuba\" $tagger \"$tmpdir/sample.spine\"" "$larvataggerjl" predict "$maggotuba" $tagger "$tmpdir/sample.spine" predictions="$tmpdir/predicted.label" expected_labels="$datapath/$tagger/predicted.label" - assertNull "\`predict\` failed to reproduce file predicted.label" "$(cmp "$expected_labels" "$predictions")" + assertTrue "\`predict\` failed to reproduce file predicted.label" '$(cmp "$expected_labels" "$predictions")' # clean up if [ "$KEEP_MODEL_FILES" = "1" ]; then @@ -169,4 +182,4 @@ postTrain() { fi } -. shunit2 +. $SHUNIT2