Merge branch 'master' into 'update-frontend'

# Conflicts:
#   .gitlab-ci.yml

.gitlab-ci.yml

@@ -244,3 +244,107 @@ delete-dev-deployment:
     - echo "Removing $CI_COMMIT_REF_SLUG"
     - helm delete -n ${NAMESPACE} ${CI_COMMIT_REF_SLUG}
     - kubectl delete pvc -lapp.kubernetes.io/name=rabbitmq,app.kubernetes.io/instance=${CI_COMMIT_REF_SLUG}
+
+
+.upload-from-fex-to-somewhere:
+  stage: deploy
+  needs: []
+  when: manual
+  image: harbor.pasteur.fr/kube-system/helm-kubectl:3.14.0
+  variables:
+    FEX_URL: ""
+  script:
+    - mkdir -p removed added
+    - >
+      if [[ "$FEX_URL" =~ ^https://dl\.pasteur\.fr/.* ]]; then 
+        echo "Valid URL: $FEX_URL"; 
+      else
+        echo "FEX_URL invalide (FEX_URL=$FEX_URL), please provide a fex url starting with https://dl.pasteur.fr/..."; 
+        exit 1; 
+      fi
+    - wget $FEX_URL
+    - ls -lah .
+    - export HDF5_FILENAME=$(ls -1 *hdf5 | cut -d'/' -f2)
+    - >
+      if [ ! $HDF5_FILENAME ]; then
+        echo "No hdf5 file fetched, exiting"; 
+        exit 2; 
+      else
+        echo "Valid File: $HDF5_FILENAME"; 
+      fi
+    - kubectl apply -f chart/sidekick.yaml -f chart/pvc-shared-data.yaml
+    - kubectl wait -f chart/sidekick.yaml  --for=condition=Ready --timeout=3600s
+    - kubectl cp sidekick:/data/$HDF5_FILENAME ./removed/$HDF5_FILENAME || echo "no older version"
+    - kubectl cp ./$HDF5_FILENAME sidekick:/data/$HDF5_FILENAME
+    - mv ./$HDF5_FILENAME ./added/
+    - kubectl exec sidekick -- ls -lah /data
+    - kubectl delete -f chart/sidekick.yaml --force
+  artifacts:
+    when: always
+    paths:
+    - added
+    - removed
+    expire_in: 1 month
+
+
+upload-from-fex-to-dev:
+  extends: .upload-from-fex-to-somewhere
+  environment:
+    name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}"
+
+
+upload-from-fex-to-prod:
+  extends: .upload-from-fex-to-somewhere
+  only:
+    - master
+  environment:
+    name: "k8sprod-02/jass-prod/${CI_COMMIT_REF_SLUG}"
+
+
+.delete-data-from-somewhere:
+  stage: deploy
+  needs: []
+  when: manual
+  image: harbor.pasteur.fr/kube-system/helm-kubectl:3.14.0
+  variables:
+    HDF5_FILENAME: ""
+  script:
+    - kubectl apply -f chart/sidekick.yaml -f chart/pvc-shared-data.yaml
+    - kubectl wait -f chart/sidekick.yaml  --for=condition=Ready --timeout=3600s
+    - kubectl exec sidekick -- ls -lah /data
+    - >
+      if [ ! $HDF5_FILENAME ]; then
+        echo "No hdf5 file provided, exiting"; 
+        exit 0; 
+      else
+        echo "File to remove: $HDF5_FILENAME"; 
+      fi
+    - mkdir removed
+    - >
+      if [[ "$HDF5_FILENAME" =~ ^[^/]+\.hdf5$ ]]; then 
+        echo "Valid filename: $HDF5_FILENAME"; 
+      else
+        echo "HDF5_FILENAME invalide (HDF5_FILENAME=$HDF5_FILENAME), please provide an hdf5 file in /data";  
+      fi
+    - kubectl cp sidekick:/data/$HDF5_FILENAME ./removed/$HDF5_FILENAME
+    - kubectl exec sidekick -- rm -f /data/$HDF5_FILENAME
+    - kubectl delete -f chart/sidekick.yaml --force
+  artifacts:
+    when: always
+    paths:
+    - removed
+    expire_in: 1 month
+
+
+delete-data-from-dev:
+  extends: .delete-data-from-somewhere
+  environment:
+    name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}"
+
+
+delete-data-from-prod:
+  extends: .delete-data-from-somewhere
+  only:
+    - master
+  environment:
+    name: "k8sprod-02/jass-prod/${CI_COMMIT_REF_SLUG}"

doc/source/index.rst

@@ -16,6 +16,7 @@ JASS documentation
    generating_joint_analysis
    command_line_usage
    web_usage
+   web_admin
    develop
 
 Indices and tables

doc/source/web_admin.rst

+Web admin
+===============================================================================
+
+Updating data
+-------------------------------------------------------------------------------
+
+You can update data using kubectl, to do so see this `"How To" <https://gitlab.pasteur.fr/statistical-genetics/jass/-/blob/master/chart/HOWTO.md?ref_type=heads#updating-data-including-inittablehdf5>`_.
+
+An alternative using fex.pasteur.fr is also proposed, with no binary to install locally.
+
+Upload to fex
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Go to https://fex.pasteur.fr/fup, click on "fex yourself" and choose the file you want to send.
+In our example it is initTableTest.hdf5
+
+.. image:: _static/fex-upload.png
+  :width: 100%
+
+Get the fex link
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You then receive an email containing the url where the initTable is available, copy it.
+In our example it is https://dl.pasteur.fr/fop/6atgurR3/initTableTest.hdf5
+
+.. image:: _static/fex-email.png
+  :width: 100%
+
+Go to gitlab pipelines
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Go to https://gitlab.pasteur.fr/statistical-genetics/jass/-/pipelines
+
+Click on the last grey/green circle on the right (indicated 1 in the following image).
+Then on the task ``upload-from-fex-to-dev`` (indicated 2) if you want to send the file in dev.
+A job is also proposed related to master pipelines.
+
+.. image:: _static/gitlab-pipelines.png
+  :width: 100%
+
+Trigger while providing the URL
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Provide in an variable name ``FEX_URL`` the url.
+In our example ``FEX_URL=https://dl.pasteur.fr/fop/6atgurR3/initTableTest.hdf5``
+
+.. image:: _static/trigger-job-with-url.png
+  :width: 100%
+
+After some time the job is completed, you see with the ls that the file is now there.
+
+After the job is completed, in the right part you see "Job artifact", when browsing them you have the
+file you have added, and the previous one that was replaced.
+Artefact are keep 1 month as of March 2024.
+
+.. image:: _static/job-done-with-url.png
+  :width: 100%
+
+Deleting data
+-------------------------------------------------------------------------------
+
+You can also simply delete hdf5 file (and only them).
+
+Go to https://gitlab.pasteur.fr/statistical-genetics/jass/-/pipelines
+
+Click on the last grey/green circle on the right (indicated 1 in the following image).
+Then on the task ``delete-data-from-dev`` (indicated 2).
+
+.. image:: _static/gitlab-pipelines-delete-file.png
+  :width: 100%
+
+Trigger while providing the filename
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Provide in an variable name ``HDF5_FILENAME`` the hdf5 filename (no relative path allowed).
+In our example ``HDF5_FILENAME=initTableTest.hdf5``
+
+.. image:: _static/trigger-job-with-filename.png
+  :width: 100%
+
+Also in the right part you see "Job artifact", when browsing them you have the
+file you just removed.
+Artefact are keep 1 month as of March 2024.
+
+.. image:: _static/job-done-with-filename.png
+  :width: 100%
\ No newline at end of file