diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9800abf0ca5584d0084698915c1c8c2c901435cf..2f7f872fba56f247866e22fdb95943d770e3cf44 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -243,4 +243,108 @@ delete-dev-deployment: script: - echo "Removing $CI_COMMIT_REF_SLUG" - helm delete -n ${NAMESPACE} ${CI_COMMIT_REF_SLUG} - - kubectl delete pvc -lapp.kubernetes.io/name=rabbitmq,app.kubernetes.io/instance=${CI_COMMIT_REF_SLUG} \ No newline at end of file + - kubectl delete pvc -lapp.kubernetes.io/name=rabbitmq,app.kubernetes.io/instance=${CI_COMMIT_REF_SLUG} + + +.upload-from-fex-to-somewhere: + stage: deploy + needs: [] + when: manual + image: harbor.pasteur.fr/kube-system/helm-kubectl:3.14.0 + variables: + FEX_URL: "" + script: + - mkdir -p removed added + - > + if [[ "$FEX_URL" =~ ^https://dl\.pasteur\.fr/.* ]]; then + echo "Valid URL: $FEX_URL"; + else + echo "FEX_URL invalide (FEX_URL=$FEX_URL), please provide a fex url starting with https://dl.pasteur.fr/..."; + exit 1; + fi + - wget $FEX_URL + - ls -lah . + - export HDF5_FILENAME=$(ls -1 *hdf5 | cut -d'/' -f2) + - > + if [ ! $HDF5_FILENAME ]; then + echo "No hdf5 file fetched, exiting"; + exit 2; + else + echo "Valid File: $HDF5_FILENAME"; + fi + - kubectl apply -f chart/sidekick.yaml -f chart/pvc-shared-data.yaml + - kubectl wait -f chart/sidekick.yaml --for=condition=Ready --timeout=3600s + - kubectl cp sidekick:/data/$HDF5_FILENAME ./removed/$HDF5_FILENAME || echo "no older version" + - kubectl cp ./$HDF5_FILENAME sidekick:/data/$HDF5_FILENAME + - mv ./$HDF5_FILENAME ./added/ + - kubectl exec sidekick -- ls -lah /data + - kubectl delete -f chart/sidekick.yaml --force + artifacts: + when: always + paths: + - added + - removed + expire_in: 1 month + + +upload-from-fex-to-dev: + extends: .upload-from-fex-to-somewhere + environment: + name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}" + + +upload-from-fex-to-prod: + extends: .upload-from-fex-to-somewhere + only: + - master + environment: + name: "k8sprod-02/jass-prod/${CI_COMMIT_REF_SLUG}" + + +.delete-data-from-somewhere: + stage: deploy + needs: [] + when: manual + image: harbor.pasteur.fr/kube-system/helm-kubectl:3.14.0 + variables: + HDF5_FILENAME: "" + script: + - kubectl apply -f chart/sidekick.yaml -f chart/pvc-shared-data.yaml + - kubectl wait -f chart/sidekick.yaml --for=condition=Ready --timeout=3600s + - kubectl exec sidekick -- ls -lah /data + - > + if [ ! $HDF5_FILENAME ]; then + echo "No hdf5 file provided, exiting"; + exit 0; + else + echo "File to remove: $HDF5_FILENAME"; + fi + - mkdir removed + - > + if [[ "$HDF5_FILENAME" =~ ^[^/]+\.hdf5$ ]]; then + echo "Valid filename: $HDF5_FILENAME"; + else + echo "HDF5_FILENAME invalide (HDF5_FILENAME=$HDF5_FILENAME), please provide an hdf5 file in /data"; + fi + - kubectl cp sidekick:/data/$HDF5_FILENAME ./removed/$HDF5_FILENAME + - kubectl exec sidekick -- rm -f /data/$HDF5_FILENAME + - kubectl delete -f chart/sidekick.yaml --force + artifacts: + when: always + paths: + - removed + expire_in: 1 month + + +delete-data-from-dev: + extends: .delete-data-from-somewhere + environment: + name: "k8sdev-01/jass-dev/${CI_COMMIT_REF_SLUG}" + + +delete-data-from-prod: + extends: .delete-data-from-somewhere + only: + - master + environment: + name: "k8sprod-02/jass-prod/${CI_COMMIT_REF_SLUG}" diff --git a/doc/source/_static/fex-email.png b/doc/source/_static/fex-email.png new file mode 100644 index 0000000000000000000000000000000000000000..805895633e15354a90ded616656e7f2a1f05245a Binary files /dev/null and b/doc/source/_static/fex-email.png differ diff --git a/doc/source/_static/fex-upload.png b/doc/source/_static/fex-upload.png new file mode 100644 index 0000000000000000000000000000000000000000..0ebe5de3ba7efa84de7acb7053106a13b9f9dbff Binary files /dev/null and b/doc/source/_static/fex-upload.png differ diff --git a/doc/source/_static/gitlab-pipelines-delete-file.png b/doc/source/_static/gitlab-pipelines-delete-file.png new file mode 100644 index 0000000000000000000000000000000000000000..5755883efd524bf88773d7fc489195555cc42af4 Binary files /dev/null and b/doc/source/_static/gitlab-pipelines-delete-file.png differ diff --git a/doc/source/_static/gitlab-pipelines.png b/doc/source/_static/gitlab-pipelines.png new file mode 100644 index 0000000000000000000000000000000000000000..d937f929185d1ec8cb6700c5f4c042268e43e67c Binary files /dev/null and b/doc/source/_static/gitlab-pipelines.png differ diff --git a/doc/source/_static/job-done-with-filename.png b/doc/source/_static/job-done-with-filename.png new file mode 100644 index 0000000000000000000000000000000000000000..d5995d05c964c09d25f535073f96f65e997e470c Binary files /dev/null and b/doc/source/_static/job-done-with-filename.png differ diff --git a/doc/source/_static/job-done-with-url.png b/doc/source/_static/job-done-with-url.png new file mode 100644 index 0000000000000000000000000000000000000000..a980c320c29bd02455009b905a41dc0f4e96d29f Binary files /dev/null and b/doc/source/_static/job-done-with-url.png differ diff --git a/doc/source/_static/trigger-job-with-filename.png b/doc/source/_static/trigger-job-with-filename.png new file mode 100644 index 0000000000000000000000000000000000000000..cd5d4c9eb4cbed8892a5594589d18d23787dd022 Binary files /dev/null and b/doc/source/_static/trigger-job-with-filename.png differ diff --git a/doc/source/_static/trigger-job-with-url.png b/doc/source/_static/trigger-job-with-url.png new file mode 100644 index 0000000000000000000000000000000000000000..a7e87b80eb4ae58e5dc467a1cdcc99a8b41fa4b0 Binary files /dev/null and b/doc/source/_static/trigger-job-with-url.png differ diff --git a/doc/source/index.rst b/doc/source/index.rst index d49fff3e7ccac801b7973a949298e7ee1d3fa107..270f8c072ce9a715e642250b4daf5aa1b29772b3 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -16,6 +16,7 @@ JASS documentation generating_joint_analysis command_line_usage web_usage + web_admin develop Indices and tables diff --git a/doc/source/web_admin.rst b/doc/source/web_admin.rst new file mode 100644 index 0000000000000000000000000000000000000000..f2acb5df80c57487be9bb87b00fc1d40efee8221 --- /dev/null +++ b/doc/source/web_admin.rst @@ -0,0 +1,86 @@ +Web admin +=============================================================================== + +Updating data +------------------------------------------------------------------------------- + +You can update data using kubectl, to do so see this `"How To" <https://gitlab.pasteur.fr/statistical-genetics/jass/-/blob/master/chart/HOWTO.md?ref_type=heads#updating-data-including-inittablehdf5>`_. + +An alternative using fex.pasteur.fr is also proposed, with no binary to install locally. + +Upload to fex +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Go to https://fex.pasteur.fr/fup, click on "fex yourself" and choose the file you want to send. +In our example it is initTableTest.hdf5 + +.. image:: _static/fex-upload.png + :width: 100% + +Get the fex link +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You then receive an email containing the url where the initTable is available, copy it. +In our example it is https://dl.pasteur.fr/fop/6atgurR3/initTableTest.hdf5 + +.. image:: _static/fex-email.png + :width: 100% + +Go to gitlab pipelines +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Go to https://gitlab.pasteur.fr/statistical-genetics/jass/-/pipelines + +Click on the last grey/green circle on the right (indicated 1 in the following image). +Then on the task ``upload-from-fex-to-dev`` (indicated 2) if you want to send the file in dev. +A job is also proposed related to master pipelines. + +.. image:: _static/gitlab-pipelines.png + :width: 100% + +Trigger while providing the URL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Provide in an variable name ``FEX_URL`` the url. +In our example ``FEX_URL=https://dl.pasteur.fr/fop/6atgurR3/initTableTest.hdf5`` + +.. image:: _static/trigger-job-with-url.png + :width: 100% + +After some time the job is completed, you see with the ls that the file is now there. + +After the job is completed, in the right part you see "Job artifact", when browsing them you have the +file you have added, and the previous one that was replaced. +Artefact are keep 1 month as of March 2024. + +.. image:: _static/job-done-with-url.png + :width: 100% + +Deleting data +------------------------------------------------------------------------------- + +You can also simply delete hdf5 file (and only them). + +Go to https://gitlab.pasteur.fr/statistical-genetics/jass/-/pipelines + +Click on the last grey/green circle on the right (indicated 1 in the following image). +Then on the task ``delete-data-from-dev`` (indicated 2). + +.. image:: _static/gitlab-pipelines-delete-file.png + :width: 100% + +Trigger while providing the filename +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Provide in an variable name ``HDF5_FILENAME`` the hdf5 filename (no relative path allowed). +In our example ``HDF5_FILENAME=initTableTest.hdf5`` + +.. image:: _static/trigger-job-with-filename.png + :width: 100% + +Also in the right part you see "Job artifact", when browsing them you have the +file you just removed. +Artefact are keep 1 month as of March 2024. + +.. image:: _static/job-done-with-filename.png + :width: 100% \ No newline at end of file