From e50e0e6d325a7ff64aa298c056e07398b51d1174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Mon, 7 Apr 2025 18:19:29 +0200 Subject: [PATCH 01/24] feat: larvatagger backend --- .gitlab-ci.yml | 39 ++++------- Project.toml | 2 +- README.md | 12 +++- back/Containerfile.larvatagger | 11 +++ back/larvatagger-entrypoint.sh | 7 ++ back/larvatagger-no-build.sh | 21 ++++++ front/Containerfile | 2 +- front/Containerfile.local | 2 +- front/Manifest.toml | 16 +++-- front/build-run-local.sh | 2 +- front/build.sh | 2 +- front/entrypoint.sh | 3 + nyxui/Chart.yaml | 4 +- nyxui/templates/configmap-front.yaml | 12 ++++ nyxui/templates/deployment-front.yaml | 2 +- nyxui/templates/deployment-larvatagger.yaml | 68 +++++++++++++++++++ .../{service.yaml => service-front.yaml} | 3 +- nyxui/templates/service-larvatagger.yaml | 16 +++++ nyxui/values.yaml | 55 +++++++++------ src/apps/catalog/app.jl | 4 +- src/apps/larvatagger/app.jl | 11 ++- values-yaml.gotmpl | 67 +++++++++++------- 22 files changed, 270 insertions(+), 91 deletions(-) create mode 100644 back/Containerfile.larvatagger create mode 100755 back/larvatagger-entrypoint.sh create mode 100755 back/larvatagger-no-build.sh create mode 100644 nyxui/templates/configmap-front.yaml create mode 100644 nyxui/templates/deployment-larvatagger.yaml rename nyxui/templates/{service.yaml => service-front.yaml} (81%) create mode 100644 nyxui/templates/service-larvatagger.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79f586e..d07162b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ stages: - buildah build -t "$FQ_IMAGE_NAME" -f "$CONTAINERFILE" "$BUILD_CONTEXT" - buildah push "$FQ_IMAGE_NAME" -build on gitlab.pasteur.fr: +build front: extends: .build-with-buildah variables: PUBLIC_URL: "nyx.pasteur.cloud" @@ -32,7 +32,7 @@ build on gitlab.pasteur.fr: - if: ($CI_COMMIT_BRANCH == "main" && $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID) # gitlab.pasteur.fr only -build dev on gitlab.pasteur.fr: +build dev front: extends: .build-with-buildah variables: PUBLIC_URL: "nyx.dev.pasteur.cloud" @@ -40,25 +40,6 @@ build dev on gitlab.pasteur.fr: - if: ($CI_COMMIT_BRANCH == "dev" && $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID) # gitlab.pasteur.fr only -.deploy-with-manifests: - stage: deploy - image: docker.io/enix/ci-toolbox:1.21 - variables: - APP_NAME: nyxui - FQ_IMAGE_NAME: "$CI_REGISTRY_IMAGE/front:$CI_COMMIT_SHORT_SHA" - script: - - kubectl create secret - docker-registry registry-gitlab - --docker-server=registry-gitlab.pasteur.fr - --docker-username="$DOCKER_USER" - --docker-password="$DOCKER_TOKEN" - --docker-email=kubernetes@pasteur.fr - -n "$KUBE_NAMESPACE" - --dry-run=client -o yaml | kubectl apply -f - - - envsubst < k8s/front-deployment.yaml | kubectl apply -n "$KUBE_NAMESPACE" -f - - - envsubst < k8s/front-service.yaml | kubectl apply -n "$KUBE_NAMESPACE" -f - - - envsubst < k8s/ingress.yaml | kubectl apply -n "$KUBE_NAMESPACE" -f - - .deploy-with-helm: stage: deploy image: docker.io/enix/ci-toolbox:1.21 @@ -81,13 +62,11 @@ build dev on gitlab.pasteur.fr: deploy to pasteur.cloud: extends: .deploy-with-helm variables: - SERVICE_TARGET_PORT: "8080" IMAGE_SECRETS: "registry-gitlab" KUBE_NAMESPACE: "nyx-prod" PUBLIC_URL: "nyx.pasteur.cloud" INGRESS_CLASS: "external" INGRESS_URL: "nyx.pasteur.cloud" - SERVICE_PORT: "80" LIMITS_CPU: "2" LIMITS_MEMORY: "4Gi" LIMITS_STORAGE: "1Gi" @@ -102,13 +81,11 @@ deploy to pasteur.cloud: deploy to dev.pasteur.cloud: extends: .deploy-with-helm variables: - SERVICE_TARGET_PORT: "8080" IMAGE_SECRETS: "registry-gitlab" KUBE_NAMESPACE: "nyx-dev" PUBLIC_URL: "nyx.dev.pasteur.cloud" INGRESS_CLASS: "internal" INGRESS_URL: "nyx.dev.pasteur.cloud" - SERVICE_PORT: "80" LIMITS_CPU: "2" LIMITS_MEMORY: "4Gi" LIMITS_STORAGE: "1Gi" @@ -120,3 +97,15 @@ deploy to dev.pasteur.cloud: $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID) # gitlab.pasteur.fr only when: manual +register larvatagger on gitlab.pasteur.fr: + extends: .build-with-buildah + variables: + FQ_IMAGE_NAME: "$CI_REGISTRY_IMAGE/larvatagger:latest" + BUILD_CONTEXT: "back" + CONTAINERFILE: "back/Containerfile.larvatagger" + LARVATAGGER_PORT: "9286" + LARVATAGGER_IMAGE_TAG: "0.20-bigfat" + rules: + - if: $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID # gitlab.pasteur.fr only + when: manual # on LARVATAGGER_IMAGE_TAG update + diff --git a/Project.toml b/Project.toml index c0988dd..d4e91c2 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "NyxUI" uuid = "2c32e805-e4ca-4d0f-96f6-9d2d6204339d" authors = ["François Laurent <francois.laurent@pasteur.fr>"] -version = "0.2.0" +version = "0.2.1" [deps] Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8" diff --git a/README.md b/README.md index 4b6613d..f97582d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Web interface meant to be served at [nyx.pasteur.cloud](https://nyx.pasteur.clou It features an app catalog, with the following apps: * an editor for muscular activity programs, -* [LarvaTagger.jl](https://gitlab.pasteur.fr/nyx/larvatagger.jl) without any automating tagging backends. +* [LarvaTagger.jl](https://gitlab.pasteur.fr/nyx/larvatagger.jl) with two automating tagging backends. ## Local installation @@ -24,10 +24,11 @@ cp front/Manifest.toml . juliaup add lts juliaup default lts julia --project=. -e 'using Pkg; Pkg.instantiate()' +. back/larvatagger-no-build.sh julia --project=. routes.jl ``` -You may be asked whether to authorize ports 9284 and 9285; please give the app permission. +You may be asked whether to authorize ports 9284, 9285 and 9286; please give the app permission. From there, in a web browser, open http://localhost:9284/ to access the app. @@ -35,4 +36,9 @@ Note that the generated files can be found somewhere in directory *storage/expor The download buttons in the LarvaTagger app will not work in the web browser. To make them work, the reverse proxy setup in the container image called *front* is required. -If you have [Podman](https://podman.io/), as an alternative to the above installation procedure, you can simply run `front/build.sh --now` and, once the container is up and running, connect to http://localhost:8080/. +If you have [Podman](https://podman.io/), as an alternative to the above installation procedure, you can simply run the backend and frontend as follows: +``` +. back/larvatagger-no-build.sh +front/build.sh --now +``` +Once the frontend container is up and running, connect to http://localhost:8080/. diff --git a/back/Containerfile.larvatagger b/back/Containerfile.larvatagger new file mode 100644 index 0000000..d0b959a --- /dev/null +++ b/back/Containerfile.larvatagger @@ -0,0 +1,11 @@ +ARG LARVATAGGER_IMAGE_TAG=0.20-bigfat +FROM docker.io/flaur/larvatagger:${LARVATAGGER_IMAGE_TAG} + +ARG LARVATAGGER_PORT=9286 +ENV LARVATAGGER_PORT=${LARVATAGGER_PORT} +EXPOSE ${LARVATAGGER_PORT} + +COPY larvatagger-entrypoint.sh /app/ + +ENTRYPOINT [/app/larvatagger-entrypoint.sh] +CMD [] diff --git a/back/larvatagger-entrypoint.sh b/back/larvatagger-entrypoint.sh new file mode 100755 index 0000000..ac284ac --- /dev/null +++ b/back/larvatagger-entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +if [ -z "$LARVATAGGER_PORT" ]; then + LARVATAGGER_PORT=9286 +fi + +julia --project=/app -e "using LarvaTagger.REST.Server; run_backend(\"/app\"; host=\"0.0.0.0\", port=$LARVATAGGER_PORT)" diff --git a/back/larvatagger-no-build.sh b/back/larvatagger-no-build.sh new file mode 100755 index 0000000..f505281 --- /dev/null +++ b/back/larvatagger-no-build.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# for tests with lightweight deployment + +if [ -z "$LARVATAGGER_PORT" ]; then + LARVATAGGER_PORT=9286 +fi +if [ -z "$LARVATAGGER_IMAGE" ]; then + LARVATAGGER_IMAGE=docker.io/flaur/larvatagger:0.20-bigfat +fi + +podman run -d -p $LARVATAGGER_PORT:9285 --entrypoint=julia $LARVATAGGER_IMAGE \ + --project=/app -e 'using LarvaTagger.REST.Server; run_backend("/app"; host="0.0.0.0")' + +CONTAINER=`podman ps | grep $LARVATAGGER_IMAGE | cut -d' ' -f1` +if ! [ -z "$CONTAINER" ]; then + echo "To monitor the LarvaTagger backend:" + echo "podman logs -f $CONTAINER" +fi + +export LARVATAGGER_BACKEND=http://localhost:$LARVATAGGER_PORT diff --git a/front/Containerfile b/front/Containerfile index ca3adb2..c788eb3 100644 --- a/front/Containerfile +++ b/front/Containerfile @@ -16,7 +16,7 @@ ENV JULIA_PROJECT /app ENV JULIA_DEPOT_PATH /app/julia ENV JULIAUP_DEPOT_PATH /app/juliaup -ARG JULIA_VERSION=1.10.8 +ARG JULIA_VERSION=1.10.9 # UID/GID should match with same arguments defined in: # https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/mainline/debian/Dockerfile diff --git a/front/Containerfile.local b/front/Containerfile.local index 1449a18..d25b8dc 100644 --- a/front/Containerfile.local +++ b/front/Containerfile.local @@ -6,7 +6,7 @@ COPY ./NyxUI.jl/front/proxy.conf /etc/nginx/conf.d/default.conf ENV JULIAUP_DEPOT_PATH /juliaup ENV JULIA_DEPOT_PATH /julia -ARG JULIA_VERSION=1.10.8 +ARG JULIA_VERSION=1.10.9 RUN curl -fsSL https://install.julialang.org \ | sh -s -- --yes --default-channel $JULIA_VERSION diff --git a/front/Manifest.toml b/front/Manifest.toml index 7b87bea..db9a5df 100644 --- a/front/Manifest.toml +++ b/front/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.8" +julia_version = "1.10.9" manifest_format = "2.0" project_hash = "f30a6bd140cd9f9b66d3656725e83b8d83e18ee6" @@ -833,12 +833,12 @@ uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" version = "1.4.0" [[deps.LarvaTagger]] -deps = ["Bonito", "Colors", "Dates", "DocOpt", "Format", "LinearAlgebra", "Logging", "Makie", "Meshes", "NearestNeighbors", "NyxWidgets", "Observables", "ObservationPolicies", "OrderedCollections", "PlanarLarvae", "Random", "StaticArrays", "Statistics", "TidyObservables", "WGLMakie"] -git-tree-sha1 = "c9c185beb0bf417ed98afe9b8b9da557a0b886b7" +deps = ["Bonito", "Colors", "Dates", "DocOpt", "Format", "HTTP", "JSON3", "LinearAlgebra", "Logging", "Makie", "Meshes", "NearestNeighbors", "NyxWidgets", "Observables", "ObservationPolicies", "OrderedCollections", "Oxygen", "PlanarLarvae", "Random", "StaticArrays", "Statistics", "TidyObservables", "WGLMakie"] +git-tree-sha1 = "ee617c0f75eb1bb680ce8d100e7893012fc8590b" repo-rev = "main" repo-url = "https://gitlab.pasteur.fr/nyx/larvatagger.jl" uuid = "8b3b36f1-dfed-446e-8561-ea19fe966a4d" -version = "0.19.1" +version = "0.20.0" [[deps.LazyArtifacts]] deps = ["Artifacts", "Pkg"] @@ -1182,7 +1182,7 @@ version = "3.2.4+0" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" -version = "0.8.1+2" +version = "0.8.1+4" [[deps.OpenMPI_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] @@ -1219,6 +1219,12 @@ git-tree-sha1 = "12f1439c4f986bb868acda6ea33ebc78e19b95ad" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" version = "1.7.0" +[[deps.Oxygen]] +deps = ["DataStructures", "Dates", "HTTP", "JSON3", "MIMEs", "Reexport", "RelocatableFolders", "Requires", "Sockets", "Statistics", "StructTypes"] +git-tree-sha1 = "2ad010b0de6172faf1d09ed5e0837eb0b7355bd8" +uuid = "df9a0d86-3283-4920-82dc-4555fc0d1d8b" +version = "1.5.16" + [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" diff --git a/front/build-run-local.sh b/front/build-run-local.sh index 79c9f43..3aeaf2c 100755 --- a/front/build-run-local.sh +++ b/front/build-run-local.sh @@ -18,7 +18,7 @@ podman build --tag $IMAGE -f "$CONTAINERFILE" .. # --no-cache #mkdir -p public podman run -d -p 8081:80 -p 9484:9284 -p 9485:9285 \ - $IMAGE + -e LARVATAGGER_BACKEND $IMAGE CONTAINER=`podman ps | grep $IMAGE | cut -d' ' -f1` diff --git a/front/build.sh b/front/build.sh index 02619d9..1b9693e 100755 --- a/front/build.sh +++ b/front/build.sh @@ -14,7 +14,7 @@ podman rmi -f $IMAGE || true podman build --tag $IMAGE -f "$CONTAINERFILE" . if [ "$1" = "--now" ]; then - podman run -d -p 8080:8080 $IMAGE + podman run -d -p 8080:8080 -e LARVATAGGER_BACKEND $IMAGE CONTAINER=`podman ps | grep $IMAGE | cut -d' ' -f1` diff --git a/front/entrypoint.sh b/front/entrypoint.sh index ae9288c..90c3dee 100644 --- a/front/entrypoint.sh +++ b/front/entrypoint.sh @@ -2,6 +2,9 @@ set -m +[ -f "front/.env" ] && export `cat front/.env | xargs` +LARVATAGGER_BACKEND=`echo $LARVATAGGER_BACKEND | sed "s/localhost/host.containers.internal/"` + /docker-entrypoint.sh nginx -g "daemon off;" &>/dev/null & NGINX_PID=$! diff --git a/nyxui/Chart.yaml b/nyxui/Chart.yaml index 567ea86..0126bd7 100644 --- a/nyxui/Chart.yaml +++ b/nyxui/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 +version: 0.2.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.2.0" +appVersion: "0.2.1" diff --git a/nyxui/templates/configmap-front.yaml b/nyxui/templates/configmap-front.yaml new file mode 100644 index 0000000..16721db --- /dev/null +++ b/nyxui/templates/configmap-front.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nyxui.name" . }}-front-config + labels: + app.kubernetes.io/name: {{ include "nyxui.name" . }} + helm.sh/chart: {{ include "nyxui.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + .env: |- + LARVATAGGER_BACKEND=http://{{ include "nyxui.name" . }}-larvatagger:{{ .Values.service.port }} diff --git a/nyxui/templates/deployment-front.yaml b/nyxui/templates/deployment-front.yaml index e4319b6..f12b8bb 100644 --- a/nyxui/templates/deployment-front.yaml +++ b/nyxui/templates/deployment-front.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "nyxui.name" . }} + name: {{ include "nyxui.name" . }}-front labels: {{- include "nyxui.labels" . | nindent 4 }} spec: diff --git a/nyxui/templates/deployment-larvatagger.yaml b/nyxui/templates/deployment-larvatagger.yaml new file mode 100644 index 0000000..e0b6cfe --- /dev/null +++ b/nyxui/templates/deployment-larvatagger.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nyxui.name" . }}-larvatagger + labels: + {{- include "nyxui.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{ include "nyxui.name" . }}-larvatagger + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "nyxui.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "nyxui.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.larvatagger.image.repository }}/{{ .Values.larvatagger.image.name }}:{{ .Values.larvatagger.image.tag }}" + imagePullPolicy: {{ .Values.larvatagger.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.larvatagger.service.targetPort }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/nyxui/templates/service.yaml b/nyxui/templates/service-front.yaml similarity index 81% rename from nyxui/templates/service.yaml rename to nyxui/templates/service-front.yaml index 6086251..013c40d 100644 --- a/nyxui/templates/service.yaml +++ b/nyxui/templates/service-front.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "nyxui.name" . }} + name: {{ include "nyxui.name" . }}-front labels: {{- include "nyxui.labels" . | nindent 4 }} spec: @@ -13,3 +13,4 @@ spec: name: http selector: {{- include "nyxui.selectorLabels" . | nindent 4 }} + nyxui/component: front diff --git a/nyxui/templates/service-larvatagger.yaml b/nyxui/templates/service-larvatagger.yaml new file mode 100644 index 0000000..84772af --- /dev/null +++ b/nyxui/templates/service-larvatagger.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nyxui.name" . }}-larvatagger + labels: + {{- include "nyxui.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "nyxui.selectorLabels" . | nindent 4 }} + nyxui/component: larvatagger diff --git a/nyxui/values.yaml b/nyxui/values.yaml index c7bde4b..ea51f97 100644 --- a/nyxui/values.yaml +++ b/nyxui/values.yaml @@ -12,6 +12,33 @@ front: pullPolicy: Always service: targetPort: 8080 + resources: + limits: + cpu: "2" + memory: 4Gi + ephemeral-storage: 1Gi + requests: + cpu: "2" + memory: 4Gi + ephemeral-storage: 1Gi + +larvatagger: + image: + repository: registry-gitlab.pasteur.fr/nyx/nyxui.jl + name: larvatagger + tag: latest + pullPolicy: Always + service: + targetPort: 9286 + resources: + limits: + cpu: "2" + memory: 4Gi + ephemeral-storage: 4Gi + requests: + cpu: "2" + memory: 4Gi + ephemeral-storage: 4Gi imagePullSecrets: - name: registry-gitlab @@ -60,21 +87,12 @@ ingress: paths: - path: / pathType: Prefix + serviceName: nyxui-front tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local -resources: - limits: - cpu: "2" - memory: 4Gi - ephemeral-storage: 1Gi - requests: - cpu: "2" - memory: 4Gi - ephemeral-storage: 1Gi - autoscaling: enabled: false minReplicas: 1 @@ -83,17 +101,16 @@ autoscaling: # targetMemoryUtilizationPercentage: 80 # Additional volumes on the output Deployment definition. -volumes: [] -# - name: foo -# secret: -# secretName: mysecret -# optional: false +volumes: + - name: front-config + configMap: + name: nyxui-front # Additional volumeMounts on the output Deployment definition. -volumeMounts: [] -# - name: foo -# mountPath: "/etc/foo" -# readOnly: true +volumeMounts: + - name: front-config + mountPath: /app/front + subPath: .env nodeSelector: {} diff --git a/src/apps/catalog/app.jl b/src/apps/catalog/app.jl index cc5c45f..4ae5596 100644 --- a/src/apps/catalog/app.jl +++ b/src/apps/catalog/app.jl @@ -36,9 +36,7 @@ function app_catalog_view() ]))), item(item(class=app, Html.a(href=Router.link_to(:get_larvatagger), [ Html.h1("LarvaTagger", style=header2), - imageview(src="/images/LarvaTagger.png", style=img, - Html.div(class=caption, "Partially available"), - ), + imageview(src="/images/LarvaTagger.png", style=img), ]))), ]), ]) diff --git a/src/apps/larvatagger/app.jl b/src/apps/larvatagger/app.jl index dbdea62..b02e1aa 100644 --- a/src/apps/larvatagger/app.jl +++ b/src/apps/larvatagger/app.jl @@ -16,6 +16,7 @@ mutable struct Model sizefactor app # TODO: check whether explicitly closing apps helps appdata + kwargs end const bonito_models = Cache{String, Model}() @@ -28,8 +29,13 @@ const bonito_app = NamedApp("larvatagger", if haskey(bonito_models, session) bonito_models.cache[session] else + kwargs = Dict{Symbol, Any}() + if haskey(ENV, "LARVATAGGER_BACKEND") + kwargs[:backend_directory] = backend = ENV["LARVATAGGER_BACKEND"] + @info "Using environment variable" LARVATAGGER_BACKEND=backend + end inputfile = Ref{Union{Nothing, String}}(nothing) - bonito_models.cache[session] = Model(1.0, nothing, inputfile) + bonito_models.cache[session] = Model(1.0, nothing, inputfile, kwargs) end end exportdir = joinpath("public", session, "larvatagger") @@ -54,7 +60,8 @@ const bonito_app = NamedApp("larvatagger", prepare_download=prepare_download, enable_new_directories=true, enable_delete=true, - viewfactor=model.sizefactor) + viewfactor=model.sizefactor, + model.kwargs...) return app end ) diff --git a/values-yaml.gotmpl b/values-yaml.gotmpl index d2b4df2..1577fe2 100644 --- a/values-yaml.gotmpl +++ b/values-yaml.gotmpl @@ -7,11 +7,38 @@ replicaCount: 1 front: image: repository: {{ requiredEnv "CI_REGISTRY_IMAGE" }} - name: {{ requiredEnv "IMAGE_NAME" }} + name: {{ env "IMAGE_NAME" | default "front" }} tag: {{ requiredEnv "CI_COMMIT_SHORT_SHA" }} - pullPolicy: {{ requiredEnv "IMAGE_POLICY" }} + pullPolicy: {{ env "IMAGE_POLICY" | default "Always" }} service: - targetPort: {{ requiredEnv "SERVICE_TARGET_PORT" }} + targetPort: {{ env "SERVICE_TARGET_PORT" | default "8080" }} + resources: + limits: + cpu: {{ env "LIMITS_CPU" | default "2" }} + memory: {{ env "LIMITS_MEMORY" | default "4Gi" }} + ephemeral-storage: {{ env "LIMITS_STORAGE" | default "1Gi" }} + requests: + cpu: {{ env "LIMITS_CPU" | default "2" }} + memory: {{ env "LIMITS_MEMORY" | default "4Gi" }} + ephemeral-storage: {{ env "LIMITS_STORAGE" | default "1Gi" }} + +larvatagger: + image: + repository: {{ requiredEnv "CI_REGISTRY_IMAGE" }} + name: {{ env "LT_IMAGE_NAME" | default "larvatagger" }} + tag: {{ env "LT_IMAGE_TAG" | default "latest" }} + pullPolicy: {{ env "LT_IMAGE_POLICY" | default "Always" }} + service: + targetPort: {{ env "LARVATAGGER_PORT" | default "9286" }} + resources: + limits: + cpu: {{ env "LT_LIMITS_CPU" | default "2" }} + memory: {{ env "LT_LIMITS_MEMORY" | default "4Gi" }} + ephemeral-storage: {{ env "LT_LIMITS_STORAGE" | default "4Gi" }} + requests: + cpu: {{ env "LT_LIMITS_CPU" | default "2" }} + memory: {{ env "LT_LIMITS_MEMORY" | default "4Gi" }} + ephemeral-storage: {{ env "LT_LIMITS_STORAGE" | default "4Gi" }} imagePullSecrets: - name: {{ requiredEnv "IMAGE_SECRETS" }} @@ -45,14 +72,14 @@ securityContext: {} service: type: ClusterIP - port: {{ requiredEnv "SERVICE_PORT" }} + port: {{ env "SERVICE_PORT" | default "80" }} ingress: enabled: true name: nyxui - className: {{ requiredEnv "INGRESS_CLASS" }} + className: {{ env "INGRESS_CLASS" | default "internal" }} annotations: - meta.helm.sh/release-name: {{ requiredEnv "INGRESS_CLASS" }} + meta.helm.sh/release-name: nyxui meta.helm.sh/release-namespace: {{ requiredEnv "KUBE_NAMESPACE" }} nginx.ingress.kubernetes.io/proxy-body-size: "0" hosts: @@ -60,21 +87,12 @@ ingress: paths: - path: / pathType: Prefix + serviceName: nyxui-front tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local -resources: - limits: - cpu: {{ requiredEnv "LIMITS_CPU" }} - memory: {{ requiredEnv "LIMITS_MEMORY" }} - ephemeral-storage: {{ requiredEnv "LIMITS_STORAGE" }} - requests: - cpu: {{ requiredEnv "LIMITS_CPU" }} - memory: {{ requiredEnv "LIMITS_MEMORY" }} - ephemeral-storage: {{ requiredEnv "LIMITS_STORAGE" }} - autoscaling: enabled: false minReplicas: 1 @@ -83,17 +101,16 @@ autoscaling: # targetMemoryUtilizationPercentage: 80 # Additional volumes on the output Deployment definition. -volumes: [] -# - name: foo -# secret: -# secretName: mysecret -# optional: false +volumes: + - name: front-config + configMap: + name: nyxui-front # Additional volumeMounts on the output Deployment definition. -volumeMounts: [] -# - name: foo -# mountPath: "/etc/foo" -# readOnly: true +volumeMounts: + - name: front-config + mountPath: /app/front + subPath: .env nodeSelector: {} -- GitLab From 368dfd199fa183e5ae29c79a6943619dbf4a7244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Mon, 7 Apr 2025 19:07:44 +0200 Subject: [PATCH 02/24] fix: component and selectorLabels --- nyxui/templates/deployment-front.yaml | 1 + nyxui/templates/deployment-larvatagger.yaml | 3 ++- nyxui/templates/service-front.yaml | 2 +- nyxui/templates/service-larvatagger.yaml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nyxui/templates/deployment-front.yaml b/nyxui/templates/deployment-front.yaml index f12b8bb..cab0485 100644 --- a/nyxui/templates/deployment-front.yaml +++ b/nyxui/templates/deployment-front.yaml @@ -19,6 +19,7 @@ spec: {{- end }} labels: {{- include "nyxui.labels" . | nindent 8 }} + app.kubernetes.io/component: front {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/nyxui/templates/deployment-larvatagger.yaml b/nyxui/templates/deployment-larvatagger.yaml index e0b6cfe..b751b86 100644 --- a/nyxui/templates/deployment-larvatagger.yaml +++ b/nyxui/templates/deployment-larvatagger.yaml @@ -10,7 +10,7 @@ spec: {{- end }} selector: matchLabels: - {{ include "nyxui.name" . }}-larvatagger + {{- include "nyxui.selectorLabels" . | nindent 6 }} template: metadata: {{- with .Values.podAnnotations }} @@ -19,6 +19,7 @@ spec: {{- end }} labels: {{- include "nyxui.labels" . | nindent 8 }} + app.kubernetes.io/component: larvatagger {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/nyxui/templates/service-front.yaml b/nyxui/templates/service-front.yaml index 013c40d..5bdec4b 100644 --- a/nyxui/templates/service-front.yaml +++ b/nyxui/templates/service-front.yaml @@ -13,4 +13,4 @@ spec: name: http selector: {{- include "nyxui.selectorLabels" . | nindent 4 }} - nyxui/component: front + app.kubernetes.io/component: front diff --git a/nyxui/templates/service-larvatagger.yaml b/nyxui/templates/service-larvatagger.yaml index 84772af..32ac7f4 100644 --- a/nyxui/templates/service-larvatagger.yaml +++ b/nyxui/templates/service-larvatagger.yaml @@ -13,4 +13,4 @@ spec: name: http selector: {{- include "nyxui.selectorLabels" . | nindent 4 }} - nyxui/component: larvatagger + app.kubernetes.io/component: larvatagger -- GitLab From ec0b26a00f46fb48c2befbcff308869eec97e350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Mon, 7 Apr 2025 19:34:29 +0200 Subject: [PATCH 03/24] fix: skippable larvatagger registration --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d07162b..5e8fca7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -108,4 +108,5 @@ register larvatagger on gitlab.pasteur.fr: rules: - if: $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID # gitlab.pasteur.fr only when: manual # on LARVATAGGER_IMAGE_TAG update + allow_failure: true -- GitLab From 056a5b69af786bc26c7dea0bd90bcf75efbe75d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Tue, 8 Apr 2025 10:23:23 +0200 Subject: [PATCH 04/24] fix: configMap as a volume in front deployment only --- nyxui/templates/deployment-front.yaml | 10 ++++------ nyxui/values.yaml | 24 +++++++++++++----------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/nyxui/templates/deployment-front.yaml b/nyxui/templates/deployment-front.yaml index cab0485..d7dd22f 100644 --- a/nyxui/templates/deployment-front.yaml +++ b/nyxui/templates/deployment-front.yaml @@ -47,14 +47,12 @@ spec: {{- toYaml .Values.readinessProbe | nindent 12 }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.volumeMounts }} volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.volumes }} + {{- toYaml .Values.front.volumeMounts | nindent 12 }} + {{- toYaml .Values.volumeMounts | nindent 12 }} volumes: - {{- toYaml . | nindent 8 }} - {{- end }} + {{- toYaml .Values.front.volumes | nindent 8 }} + {{- toYaml .Values.volumes | nindent 8 }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/nyxui/values.yaml b/nyxui/values.yaml index ea51f97..233e739 100644 --- a/nyxui/values.yaml +++ b/nyxui/values.yaml @@ -12,6 +12,14 @@ front: pullPolicy: Always service: targetPort: 8080 + volumes: [] + - name: front-config + configMap: + name: nyxui-front-config + volumeMounts: [] + - name: front-config + mountPath: /app/front + subPath: .env resources: limits: cpu: "2" @@ -100,17 +108,11 @@ autoscaling: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 -# Additional volumes on the output Deployment definition. -volumes: - - name: front-config - configMap: - name: nyxui-front - -# Additional volumeMounts on the output Deployment definition. -volumeMounts: - - name: front-config - mountPath: /app/front - subPath: .env +# Additional volumes on the output Deployment definitions. +volumes: [] + +# Additional volumeMounts on the output Deployment definitions. +volumeMounts: [] nodeSelector: {} -- GitLab From 6d9e70d0f2251d164a982eb4b3d31f76f95ed689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Tue, 8 Apr 2025 10:32:39 +0200 Subject: [PATCH 05/24] fix: forgot to update values-yaml.gotmpl file --- nyxui/templates/deployment-front.yaml | 8 ++++++-- nyxui/values.yaml | 12 ++---------- values-yaml.gotmpl | 16 +++++----------- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/nyxui/templates/deployment-front.yaml b/nyxui/templates/deployment-front.yaml index d7dd22f..4084732 100644 --- a/nyxui/templates/deployment-front.yaml +++ b/nyxui/templates/deployment-front.yaml @@ -48,10 +48,14 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - {{- toYaml .Values.front.volumeMounts | nindent 12 }} + - name: front-config + mountPath: /app/front + subPath: .env {{- toYaml .Values.volumeMounts | nindent 12 }} volumes: - {{- toYaml .Values.front.volumes | nindent 8 }} + - name: front-config + configMap: + name: nyxui-front-config {{- toYaml .Values.volumes | nindent 8 }} {{- with .Values.nodeSelector }} nodeSelector: diff --git a/nyxui/values.yaml b/nyxui/values.yaml index 233e739..4eacdc4 100644 --- a/nyxui/values.yaml +++ b/nyxui/values.yaml @@ -12,14 +12,6 @@ front: pullPolicy: Always service: targetPort: 8080 - volumes: [] - - name: front-config - configMap: - name: nyxui-front-config - volumeMounts: [] - - name: front-config - mountPath: /app/front - subPath: .env resources: limits: cpu: "2" @@ -108,10 +100,10 @@ autoscaling: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 -# Additional volumes on the output Deployment definitions. +# Additional shared volumes on the output Deployment definitions. volumes: [] -# Additional volumeMounts on the output Deployment definitions. +# Additional shared volumeMounts on the output Deployment definitions. volumeMounts: [] nodeSelector: {} diff --git a/values-yaml.gotmpl b/values-yaml.gotmpl index 1577fe2..712200b 100644 --- a/values-yaml.gotmpl +++ b/values-yaml.gotmpl @@ -100,17 +100,11 @@ autoscaling: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 -# Additional volumes on the output Deployment definition. -volumes: - - name: front-config - configMap: - name: nyxui-front - -# Additional volumeMounts on the output Deployment definition. -volumeMounts: - - name: front-config - mountPath: /app/front - subPath: .env +# Additional shared volumes on the output Deployment definitions. +volumes: [] + +# Additional shared volumeMounts on the output Deployment definitions. +volumeMounts: [] nodeSelector: {} -- GitLab From 9d2dda219d0195a1fe2e084322a776c1bf8c4e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Tue, 8 Apr 2025 11:36:34 +0200 Subject: [PATCH 06/24] fix: syntax error --- nyxui/templates/deployment-front.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nyxui/templates/deployment-front.yaml b/nyxui/templates/deployment-front.yaml index 4084732..a0cc3e6 100644 --- a/nyxui/templates/deployment-front.yaml +++ b/nyxui/templates/deployment-front.yaml @@ -51,12 +51,16 @@ spec: - name: front-config mountPath: /app/front subPath: .env - {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} volumes: - name: front-config configMap: name: nyxui-front-config - {{- toYaml .Values.volumes | nindent 8 }} + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} -- GitLab From b9a5f22ff5d281453a7c21382bc77863af3b2139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Tue, 8 Apr 2025 12:36:09 +0200 Subject: [PATCH 07/24] fix: mountPath --- nyxui/templates/deployment-front.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nyxui/templates/deployment-front.yaml b/nyxui/templates/deployment-front.yaml index a0cc3e6..445709e 100644 --- a/nyxui/templates/deployment-front.yaml +++ b/nyxui/templates/deployment-front.yaml @@ -49,7 +49,7 @@ spec: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - name: front-config - mountPath: /app/front + mountPath: /app/front/.env subPath: .env {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} -- GitLab From f3fc0e51c98f7fa0bf362fa2bd013f2cb237f4e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Tue, 8 Apr 2025 18:05:12 +0200 Subject: [PATCH 08/24] fix: LT backend unprivileged image --- .gitlab-ci.yml | 10 ++++-- back/Containerfile.larvatagger | 59 +++++++++++++++++++++++++++++++--- back/larvatagger-build.sh | 28 ++++++++++++++++ back/larvatagger-entrypoint.sh | 2 +- 4 files changed, 91 insertions(+), 8 deletions(-) create mode 100755 back/larvatagger-build.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e8fca7..9df8b39 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -100,11 +100,15 @@ deploy to dev.pasteur.cloud: register larvatagger on gitlab.pasteur.fr: extends: .build-with-buildah variables: - FQ_IMAGE_NAME: "$CI_REGISTRY_IMAGE/larvatagger:latest" - BUILD_CONTEXT: "back" - CONTAINERFILE: "back/Containerfile.larvatagger" + REGISTRY_IMAGE: "$CI_REGISTRY_IMAGE/larvatagger" LARVATAGGER_PORT: "9286" LARVATAGGER_IMAGE_TAG: "0.20-bigfat" + FQ_IMAGE_NAME: "$REGISTRY_IMAGE:$LARVATAGGER_IMAGE_TAG" + BUILD_CONTEXT: "back" + CONTAINERFILE: "back/Containerfile.larvatagger" + after_script: + - buildah tag "$FQ_IMAGE_NAME" "$REGISTRY_IMAGE:latest" + - buildah push "$REGISTRY_IMAGE:latest" rules: - if: $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID # gitlab.pasteur.fr only when: manual # on LARVATAGGER_IMAGE_TAG update diff --git a/back/Containerfile.larvatagger b/back/Containerfile.larvatagger index d0b959a..10ac5b4 100644 --- a/back/Containerfile.larvatagger +++ b/back/Containerfile.larvatagger @@ -1,11 +1,62 @@ ARG LARVATAGGER_IMAGE_TAG=0.20-bigfat -FROM docker.io/flaur/larvatagger:${LARVATAGGER_IMAGE_TAG} +FROM docker.io/flaur/larvatagger:${LARVATAGGER_IMAGE_TAG} as src + + +FROM debian:bullseye as build + +# Python and Curl for Poetry +RUN apt-get update \ + && apt-get install --no-install-recommends -y python3-dev python3-pip curl \ + && rm -rf /var/lib/apt/lists/* + +# Unprivileged environment +ARG UID=1001 + +RUN useradd julia -u ${UID} --user-group --create-home + +COPY --chown=julia --from=src /app /app/ + +USER ${UID} + +# Poetry +ENV POETRY_VERSION=1.8.4 + +ENV POETRY_HOME=/app/poetry +ENV POETRY_VIRTUALENVS_IN_PROJECT=1 +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +ENV PIP_NO_CACHE_DIR=1 + +RUN curl -sSL https://install.python-poetry.org | python3 - + +# Julia +ARG JULIA_VERSION=1.10.9 + +ENV JULIA_PROJECT=/app +ENV JULIA_DEPOT_PATH=/app/julia +ENV JULIAUP_DEPOT_PATH=/app/juliaup + +RUN export HOME="${JULIA_PROJECT}" \ + && curl -fsSL https://install.julialang.org \ + | sh -s -- --yes --default-channel "${JULIA_VERSION}" \ + && ${JULIA_PROJECT}/.juliaup/bin/juliaup config versionsdbupdateinterval 0 + +ENV PATH "${PATH}:${JULIA_PROJECT}/.juliaup/bin:${POETRY_HOME}/bin" + +# LarvaTagger.jl/MaggotUBA-adapter/PasteurJanelia-adapter +RUN cd "${JULIA_PROJECT}" \ + && julia -e 'using Pkg; Pkg.instantiate()' \ + && cd MaggotUBA \ + && poetry install --only main \ + && cd ../PasteurJanelia \ + && poetry install --only main + + +FROM build AS base ARG LARVATAGGER_PORT=9286 ENV LARVATAGGER_PORT=${LARVATAGGER_PORT} -EXPOSE ${LARVATAGGER_PORT} -COPY larvatagger-entrypoint.sh /app/ +COPY --chown=julia larvatagger-entrypoint.sh /app/ -ENTRYPOINT [/app/larvatagger-entrypoint.sh] +ENTRYPOINT ["/app/larvatagger-entrypoint.sh"] CMD [] diff --git a/back/larvatagger-build.sh b/back/larvatagger-build.sh new file mode 100755 index 0000000..dcbdefd --- /dev/null +++ b/back/larvatagger-build.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +CONTAINERFILE=back/Containerfile.larvatagger + +if ! [ -f "$CONTAINERFILE" -a -f routes.jl ]; then + echo "Run $0 from the project root directory" + exit 1 +fi + +IMAGE=larvatagger-backend + +podman rmi -f $IMAGE || true + +podman build --tag $IMAGE -f "$CONTAINERFILE" back + +if [ "$1" = "--now" ]; then + podman run -d -p 9286:9286 $IMAGE + + CONTAINER=`podman ps | grep $IMAGE | cut -d' ' -f1` + + if [ -n "$CONTAINER" ]; then + echo "podman logs -f $CONTAINER" + sleep 1 + podman logs -f $CONTAINER + fi +fi + +export LARVATAGGER_BACKEND=http://localhost:9286 diff --git a/back/larvatagger-entrypoint.sh b/back/larvatagger-entrypoint.sh index ac284ac..70a99f8 100755 --- a/back/larvatagger-entrypoint.sh +++ b/back/larvatagger-entrypoint.sh @@ -4,4 +4,4 @@ if [ -z "$LARVATAGGER_PORT" ]; then LARVATAGGER_PORT=9286 fi -julia --project=/app -e "using LarvaTagger.REST.Server; run_backend(\"/app\"; host=\"0.0.0.0\", port=$LARVATAGGER_PORT)" +julia -e "using LarvaTagger.REST.Server; run_backend(\"/app\"; host=\"0.0.0.0\", port=$LARVATAGGER_PORT)" -- GitLab From d1b5cb0ea09265f873bc20cccaca1c1b90fec838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Wed, 9 Apr 2025 11:37:20 +0200 Subject: [PATCH 09/24] fix: double tags for images --- .gitlab-ci.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9df8b39..393c87e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,10 @@ stages: STORAGE_DRIVER: vfs BUILDAH_FORMAT: docker BUILDAH_ISOLATION: chroot - FQ_IMAGE_NAME: "$CI_REGISTRY_IMAGE/front:$CI_COMMIT_SHORT_SHA" + IMAGE_NAME: front + IMAGE_TAG: "$CI_COMMIT_SHORT_SHA" + REGISTRY_IMAGE: "$CI_REGISTRY_IMAGE/$IMAGE_NAME" + FQ_IMAGE_NAME: "$REGISTRY_IMAGE:$IMAGE_TAG" BUILD_CONTEXT: "." CONTAINERFILE: "front/Containerfile" before_script: @@ -23,6 +26,8 @@ stages: script: - buildah build -t "$FQ_IMAGE_NAME" -f "$CONTAINERFILE" "$BUILD_CONTEXT" - buildah push "$FQ_IMAGE_NAME" + - buildah tag "$FQ_IMAGE_NAME" "$REGISTRY_IMAGE:latest" + - buildah push "$REGISTRY_IMAGE:latest" build front: extends: .build-with-buildah @@ -87,8 +92,8 @@ deploy to dev.pasteur.cloud: INGRESS_CLASS: "internal" INGRESS_URL: "nyx.dev.pasteur.cloud" LIMITS_CPU: "2" - LIMITS_MEMORY: "4Gi" - LIMITS_STORAGE: "1Gi" + LIMITS_MEMORY: "8Gi" + LIMITS_STORAGE: "4Gi" environment: name: k8sdev-01/nyx-dev url: https://nyx.dev.pasteur.cloud @@ -97,18 +102,14 @@ deploy to dev.pasteur.cloud: $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID) # gitlab.pasteur.fr only when: manual -register larvatagger on gitlab.pasteur.fr: +build larvatagger backend: extends: .build-with-buildah variables: - REGISTRY_IMAGE: "$CI_REGISTRY_IMAGE/larvatagger" + IMAGE_NAME: "larvatagger" LARVATAGGER_PORT: "9286" - LARVATAGGER_IMAGE_TAG: "0.20-bigfat" - FQ_IMAGE_NAME: "$REGISTRY_IMAGE:$LARVATAGGER_IMAGE_TAG" + IMAGE_TAG: "0.20-bigfat" BUILD_CONTEXT: "back" CONTAINERFILE: "back/Containerfile.larvatagger" - after_script: - - buildah tag "$FQ_IMAGE_NAME" "$REGISTRY_IMAGE:latest" - - buildah push "$REGISTRY_IMAGE:latest" rules: - if: $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID # gitlab.pasteur.fr only when: manual # on LARVATAGGER_IMAGE_TAG update -- GitLab From 0e6a2ffa2e2a6983933d38fed65f8fcbcf8acf60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Wed, 9 Apr 2025 16:24:13 +0200 Subject: [PATCH 10/24] debugging configMap --- .gitlab-ci.yml | 8 ++++---- front/entrypoint.sh | 10 +++++++++- nyxui/templates/configmap-front.yaml | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 393c87e..6a875aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,12 +106,12 @@ build larvatagger backend: extends: .build-with-buildah variables: IMAGE_NAME: "larvatagger" - LARVATAGGER_PORT: "9286" IMAGE_TAG: "0.20-bigfat" BUILD_CONTEXT: "back" CONTAINERFILE: "back/Containerfile.larvatagger" rules: - - if: $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID # gitlab.pasteur.fr only - when: manual # on LARVATAGGER_IMAGE_TAG update - allow_failure: true + - if: ($CI_COMMIT_BRANCH == "dev" && + $CI_PROJECT_ID == $GITLAB_PASTEUR_PROJECT_ID) # gitlab.pasteur.fr only + when: manual # on IMAGE_TAG update only; alias latest is used and + allow_failure: true # should be confirmed to work when moving to branch main diff --git a/front/entrypoint.sh b/front/entrypoint.sh index 90c3dee..0358243 100644 --- a/front/entrypoint.sh +++ b/front/entrypoint.sh @@ -2,7 +2,15 @@ set -m -[ -f "front/.env" ] && export `cat front/.env | xargs` +cd /app + +if [ -f "front/.env" ]; then + echo "Found dotenv file:" + echo "------------------" + cat front/.env + echo "------------------" + export `cat front/.env | xargs` +fi LARVATAGGER_BACKEND=`echo $LARVATAGGER_BACKEND | sed "s/localhost/host.containers.internal/"` diff --git a/nyxui/templates/configmap-front.yaml b/nyxui/templates/configmap-front.yaml index 16721db..1ad6cf0 100644 --- a/nyxui/templates/configmap-front.yaml +++ b/nyxui/templates/configmap-front.yaml @@ -8,5 +8,5 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: - .env: |- + .env: | LARVATAGGER_BACKEND=http://{{ include "nyxui.name" . }}-larvatagger:{{ .Values.service.port }} -- GitLab From a7a223797ccd3d06f6b1823dd2d61fc624bb9c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Wed, 9 Apr 2025 18:14:40 +0200 Subject: [PATCH 11/24] fix: resources sections properly addressed --- back/Containerfile.larvatagger | 5 ++--- nyxui/templates/configmap-front.yaml | 5 +---- nyxui/templates/deployment-front.yaml | 2 +- nyxui/templates/deployment-larvatagger.yaml | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/back/Containerfile.larvatagger b/back/Containerfile.larvatagger index 10ac5b4..48fd059 100644 --- a/back/Containerfile.larvatagger +++ b/back/Containerfile.larvatagger @@ -2,7 +2,7 @@ ARG LARVATAGGER_IMAGE_TAG=0.20-bigfat FROM docker.io/flaur/larvatagger:${LARVATAGGER_IMAGE_TAG} as src -FROM debian:bullseye as build +FROM debian:bullseye as base # Python and Curl for Poetry RUN apt-get update \ @@ -50,8 +50,7 @@ RUN cd "${JULIA_PROJECT}" \ && cd ../PasteurJanelia \ && poetry install --only main - -FROM build AS base +# final stage ARG LARVATAGGER_PORT=9286 ENV LARVATAGGER_PORT=${LARVATAGGER_PORT} diff --git a/nyxui/templates/configmap-front.yaml b/nyxui/templates/configmap-front.yaml index 1ad6cf0..5505c88 100644 --- a/nyxui/templates/configmap-front.yaml +++ b/nyxui/templates/configmap-front.yaml @@ -3,10 +3,7 @@ kind: ConfigMap metadata: name: {{ include "nyxui.name" . }}-front-config labels: - app.kubernetes.io/name: {{ include "nyxui.name" . }} - helm.sh/chart: {{ include "nyxui.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "nyxui.labels" . | nindent 4 }} data: .env: | LARVATAGGER_BACKEND=http://{{ include "nyxui.name" . }}-larvatagger:{{ .Values.service.port }} diff --git a/nyxui/templates/deployment-front.yaml b/nyxui/templates/deployment-front.yaml index 445709e..d54718c 100644 --- a/nyxui/templates/deployment-front.yaml +++ b/nyxui/templates/deployment-front.yaml @@ -46,7 +46,7 @@ spec: readinessProbe: {{- toYaml .Values.readinessProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} + {{- toYaml .Values.front.resources | nindent 12 }} volumeMounts: - name: front-config mountPath: /app/front/.env diff --git a/nyxui/templates/deployment-larvatagger.yaml b/nyxui/templates/deployment-larvatagger.yaml index b751b86..e47504b 100644 --- a/nyxui/templates/deployment-larvatagger.yaml +++ b/nyxui/templates/deployment-larvatagger.yaml @@ -46,7 +46,7 @@ spec: readinessProbe: {{- toYaml .Values.readinessProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} + {{- toYaml .Values.larvatagger.resources | nindent 12 }} {{- with .Values.volumeMounts }} volumeMounts: {{- toYaml . | nindent 12 }} -- GitLab From ffacbc5f9caa92b28ccb591c29b2eae92600fd07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Wed, 9 Apr 2025 18:40:11 +0200 Subject: [PATCH 12/24] fix: lower memory requirements to fit within cluster's limits --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a875aa..d904d8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -74,7 +74,7 @@ deploy to pasteur.cloud: INGRESS_URL: "nyx.pasteur.cloud" LIMITS_CPU: "2" LIMITS_MEMORY: "4Gi" - LIMITS_STORAGE: "1Gi" + LIMITS_STORAGE: "4Gi" environment: name: k8sprod-02/nyx-prod url: https://nyx.pasteur.cloud @@ -92,7 +92,7 @@ deploy to dev.pasteur.cloud: INGRESS_CLASS: "internal" INGRESS_URL: "nyx.dev.pasteur.cloud" LIMITS_CPU: "2" - LIMITS_MEMORY: "8Gi" + LIMITS_MEMORY: "4Gi" LIMITS_STORAGE: "4Gi" environment: name: k8sdev-01/nyx-dev -- GitLab From cb578e5d351d4d8401992d75293cbd7bbe91d14b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Thu, 10 Apr 2025 15:45:06 +0200 Subject: [PATCH 13/24] fix: lower memory requirements to fit within cluster's limits (2) --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d904d8e..1cca675 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,9 +91,11 @@ deploy to dev.pasteur.cloud: PUBLIC_URL: "nyx.dev.pasteur.cloud" INGRESS_CLASS: "internal" INGRESS_URL: "nyx.dev.pasteur.cloud" - LIMITS_CPU: "2" + LIMITS_CPU: "3" LIMITS_MEMORY: "4Gi" LIMITS_STORAGE: "4Gi" + LT_LIMITS_MEMORY: "4Gi" + LT_LIMITS_STORAGE: "2Gi" environment: name: k8sdev-01/nyx-dev url: https://nyx.dev.pasteur.cloud -- GitLab From 720ea248d530f3beab356550f35fc80c2a558fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Thu, 10 Apr 2025 16:14:18 +0200 Subject: [PATCH 14/24] fix: lower compute resources to fit within cluster's limits (3) --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1cca675..598c21d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -94,6 +94,7 @@ deploy to dev.pasteur.cloud: LIMITS_CPU: "3" LIMITS_MEMORY: "4Gi" LIMITS_STORAGE: "4Gi" + LT_LIMITS_CPU: "1" LT_LIMITS_MEMORY: "4Gi" LT_LIMITS_STORAGE: "2Gi" environment: -- GitLab From 78a8cef5e64c4e0b60bb63b3d0f3210a6a6f7157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Thu, 10 Apr 2025 16:40:48 +0200 Subject: [PATCH 15/24] fix: serviceName in ingress --- nyxui/templates/ingress.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nyxui/templates/ingress.yaml b/nyxui/templates/ingress.yaml index 79522d8..03b430c 100644 --- a/nyxui/templates/ingress.yaml +++ b/nyxui/templates/ingress.yaml @@ -49,11 +49,11 @@ spec: backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: - name: {{ $fullName }} + name: {{ .serviceName }} port: number: {{ $svcPort }} {{- else }} - serviceName: {{ $fullName }} + serviceName: {{ .serviceName }} servicePort: {{ $svcPort }} {{- end }} {{- end }} -- GitLab From 579d3964da25f369b1535418119ce4b0c518e156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Fri, 11 Apr 2025 10:13:01 +0200 Subject: [PATCH 16/24] fix: resources adjustement (4) --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 598c21d..4235bfd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,10 +91,10 @@ deploy to dev.pasteur.cloud: PUBLIC_URL: "nyx.dev.pasteur.cloud" INGRESS_CLASS: "internal" INGRESS_URL: "nyx.dev.pasteur.cloud" - LIMITS_CPU: "3" - LIMITS_MEMORY: "4Gi" + LIMITS_CPU: "2" + LIMITS_MEMORY: "6Gi" LIMITS_STORAGE: "4Gi" - LT_LIMITS_CPU: "1" + LT_LIMITS_CPU: "2" LT_LIMITS_MEMORY: "4Gi" LT_LIMITS_STORAGE: "2Gi" environment: -- GitLab From dbbc853ddaea9f0402b824dbe46d9928d17d78b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Fri, 11 Apr 2025 10:48:12 +0200 Subject: [PATCH 17/24] fix: resources adjustement (5) --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4235bfd..20cae5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,10 +92,10 @@ deploy to dev.pasteur.cloud: INGRESS_CLASS: "internal" INGRESS_URL: "nyx.dev.pasteur.cloud" LIMITS_CPU: "2" - LIMITS_MEMORY: "6Gi" + LIMITS_MEMORY: "5Gi" LIMITS_STORAGE: "4Gi" LT_LIMITS_CPU: "2" - LT_LIMITS_MEMORY: "4Gi" + LT_LIMITS_MEMORY: "5Gi" LT_LIMITS_STORAGE: "2Gi" environment: name: k8sdev-01/nyx-dev -- GitLab From ec21b56e27155ba74b689697b2844a36b3719d73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Tue, 15 Apr 2025 17:43:34 +0200 Subject: [PATCH 18/24] feat: session expiry and data clean-up --- back/Containerfile.larvatagger | 3 - back/larvatagger-entrypoint.sh | 10 ++- back/larvatagger-no-build.sh | 13 +++- src/GenieExtras.jl | 130 ++++++++++++++++++++++++++++++++- src/apps/larvatagger/app.jl | 61 +++++++++------- src/apps/muscles/Backbone.jl | 60 ++++++--------- src/apps/muscles/app.jl | 9 ++- 7 files changed, 211 insertions(+), 75 deletions(-) diff --git a/back/Containerfile.larvatagger b/back/Containerfile.larvatagger index 48fd059..bc9e4af 100644 --- a/back/Containerfile.larvatagger +++ b/back/Containerfile.larvatagger @@ -52,9 +52,6 @@ RUN cd "${JULIA_PROJECT}" \ # final stage -ARG LARVATAGGER_PORT=9286 -ENV LARVATAGGER_PORT=${LARVATAGGER_PORT} - COPY --chown=julia larvatagger-entrypoint.sh /app/ ENTRYPOINT ["/app/larvatagger-entrypoint.sh"] diff --git a/back/larvatagger-entrypoint.sh b/back/larvatagger-entrypoint.sh index 70a99f8..aa31482 100755 --- a/back/larvatagger-entrypoint.sh +++ b/back/larvatagger-entrypoint.sh @@ -2,6 +2,14 @@ if [ -z "$LARVATAGGER_PORT" ]; then LARVATAGGER_PORT=9286 +else + echo "Using environment variable: LARVATAGGER_PORT= $LARVATAGGER_PORT" fi -julia -e "using LarvaTagger.REST.Server; run_backend(\"/app\"; host=\"0.0.0.0\", port=$LARVATAGGER_PORT)" +if [ -z "$LARVATAGGER_TOKEN_EXPIRY" ]; then + LARVATAGGER_TOKEN_EXPIRY=14400 +else + echo "Using environment variable: LARVATAGGER_TOKEN_EXPIRY= $LARVATAGGER_TOKEN_EXPIRY" +fi + +julia -e "using LarvaTagger.REST.Server; run_backend(\"/app\", $LARVATAGGER_TOKEN_EXPIRY; host=\"0.0.0.0\", port=$LARVATAGGER_PORT)" diff --git a/back/larvatagger-no-build.sh b/back/larvatagger-no-build.sh index f505281..7101dd7 100755 --- a/back/larvatagger-no-build.sh +++ b/back/larvatagger-no-build.sh @@ -4,13 +4,22 @@ if [ -z "$LARVATAGGER_PORT" ]; then LARVATAGGER_PORT=9286 +else + echo "Using environment variable: LARVATAGGER_PORT= $LARVATAGGER_PORT" fi if [ -z "$LARVATAGGER_IMAGE" ]; then LARVATAGGER_IMAGE=docker.io/flaur/larvatagger:0.20-bigfat +else + echo "Using environment variable: LARVATAGGER_IMAGE= $LARVATAGGER_IMAGE" +fi +if [ -z "$LARVATAGGER_TOKEN_EXPIRY" ]; then + LARVATAGGER_TOKEN_EXPIRY=3600 +else + echo "Using environment variable: LARVATAGGER_TOKEN_EXPIRY= $LARVATAGGER_TOKEN_EXPIRY" fi -podman run -d -p $LARVATAGGER_PORT:9285 --entrypoint=julia $LARVATAGGER_IMAGE \ - --project=/app -e 'using LarvaTagger.REST.Server; run_backend("/app"; host="0.0.0.0")' +podman run -d -p $LARVATAGGER_PORT:9285 --entrypoint=julia $LARVATAGGER_IMAGE --project=/app \ + -e "using LarvaTagger.REST.Server; run_backend(\"/app\", $LARVATAGGER_TOKEN_EXPIRY; host=\"0.0.0.0\")" CONTAINER=`podman ps | grep $LARVATAGGER_IMAGE | cut -d' ' -f1` if ! [ -z "$CONTAINER" ]; then diff --git a/src/GenieExtras.jl b/src/GenieExtras.jl index 40cf48a..ebe416c 100644 --- a/src/GenieExtras.jl +++ b/src/GenieExtras.jl @@ -1,8 +1,11 @@ module GenieExtras import Genie.Renderer.Html +import GenieSession: GenieSession, Session +using NyxWidgets.Base: Cache -export publish_css, appinfo, add_footer +export publish_css, appinfo, add_footer, Session, SessionRegistry, getsessionid, getsession, + clearexpired const project_root = dirname(Base.current_project()) @@ -49,4 +52,129 @@ add_footer(ui::AbstractString, footer=footer) = "$ui$(footer())" add_footer(ui::Function, footer=footer) = () -> add_footer(ui(), footer) +struct SessionRegistry + register + idlength + purgesession + purgesession_args +end + +function SessionRegistry(f, args=(:session_id,); idlength=32) + SessionRegistry(Cache{String, Dict{Symbol, Any}}(), idlength, f, args) +end + +Base.lock(f::Function, registry::SessionRegistry) = lock(f, registry.register) + +function check_session_id_length(registry, session_id) + if !isnothing(registry.idlength) && length(session_id) < registry.idlength + throw("wrong session_id length") + end +end + +function Base.haskey(registry::SessionRegistry, session_id::AbstractString) + lock(registry) do + for session_long_id in keys(registry.register.cache) + if startswith(session_long_id, session_id) + return true + end + end + return false + end +end + +function Base.getindex(registry::SessionRegistry, session_id::AbstractString) + check_session_id_length(registry, session_id) + lock(registry) do + for (session_long_id, records) in pairs(registry.register.cache) + if startswith(session_long_id, session_id) + records[:lastseen] = time() + return records + end + end + throw(KeyError(session_id)) + end +end + +Base.getindex(registry::SessionRegistry, session_id::AbstractString, record::Symbol + ) = registry[session_id][record] + +function getsessionid(session_long_id::String, idlength::Union{Nothing, <:Integer}) + session_id = session_long_id + if !isnothing(idlength) + session_id = string(session_id[1:idlength]) + end + return session_id +end + +function getsessionid(registry::SessionRegistry) + session = GenieSession.session() + session_long_id = session.id + session_id = getsessionid(session_long_id, registry.idlength) + # create the entry in the register if missing + records = lock(registry) do + register = registry.register.cache + if session_id != session_long_id && session_id in keys(register) + # if the session has been registered with its short id (and `getsession`), + # update the short id with the long one + register[session_long_id] = pop!(register, session_id) + else + # registry.register is a NyxWidgets.Cache and `getindex` behaves like `get!` + registry.register[session_long_id] + end + end + # register minimum information + records[:session] = session + records[:lastseen] = time() + # run maintenance routine + clearexpired(registry) + return session_id +end + +function getsession(registry::SessionRegistry, session_id::AbstractString) + if haskey(registry, session_id) # check for long id + registry[session_id] + else # call get! on short id (if session_id is short); not ideal + registry.register[session_id] + end +end + +function clearexpired(registry, expiry::Real=604800) + lock(registry) do + register = registry.register.cache + for (session_long_id, records) in pairs(register) + if !haskey(records, :lastseen) + @warn "Missing key: lastseen" + continue + end + expiry <= time() - records[:lastseen] || continue + # prepare to clear the session; collect the arguments to purgesession + session_id = getsessionid(session_long_id, registry.idlength) + available_args = Dict(:session_id=>session_id, + :session_long_id=>session_long_id, + records...) + @info "Purging session" session_id + pop!(register, session_long_id) + if haskey(records, :session) + session = records[:session] + # see learn.genieframework.com/framework/genie.jl/recipes/session + empty!(session.data) + else + @warn "Missing key: session" + end + # + function get′(arg) + get(available_args, arg) do + @warn "Missing argument for purgesession" arg + nothing + end + end + try + registry.purgesession(get′.(registry.purgesession_args)...) + catch + @error "\`purgesession\` callback failed" + end + end + end +end + end diff --git a/src/apps/larvatagger/app.jl b/src/apps/larvatagger/app.jl index b02e1aa..7b30816 100644 --- a/src/apps/larvatagger/app.jl +++ b/src/apps/larvatagger/app.jl @@ -3,7 +3,7 @@ module LarvaTagger using NyxUI, NyxUI.Storage, NyxUI.GenieExtras using NyxWidgets.Base: Cache import LarvaTagger as LT -using GenieSession, Stipple +using Stipple import Stipple: @app, @init, @private, @in, @onchange, @onbutton, @click import StippleUI: tooltip @@ -19,24 +19,30 @@ mutable struct Model kwargs end -const bonito_models = Cache{String, Model}() +function purgesession(model, session_id) + if isnothing(model) + @warn "Session data prematurely lost" + elseif !isnothing(model.app) + close(model.app) + end + purge_appdata(session_id, "larvatagger") + BonitoServer.addsession(bonito_app, session_id; restart=true) +end + +const session_registry = SessionRegistry(purgesession, (:model, :session_id)) const bonito_app = NamedApp("larvatagger", function (session) bucket = getbucket(session, "larvatagger", :read) mkpath(bucket) - model = lock(bonito_models) do - if haskey(bonito_models, session) - bonito_models.cache[session] - else - kwargs = Dict{Symbol, Any}() - if haskey(ENV, "LARVATAGGER_BACKEND") - kwargs[:backend_directory] = backend = ENV["LARVATAGGER_BACKEND"] - @info "Using environment variable" LARVATAGGER_BACKEND=backend - end - inputfile = Ref{Union{Nothing, String}}(nothing) - bonito_models.cache[session] = Model(1.0, nothing, inputfile, kwargs) + model = get!(getsession(session_registry, session), :model) do + kwargs = Dict{Symbol, Any}() + if !isempty(get(ENV, "LARVATAGGER_BACKEND", "")) + kwargs[:backend_directory] = backend = ENV["LARVATAGGER_BACKEND"] + @info "Using environment variable" LARVATAGGER_BACKEND=backend end + inputfile = Ref{Union{Nothing, String}}(nothing) + Model(1.0, nothing, inputfile, kwargs) end exportdir = joinpath("public", session, "larvatagger") function prepare_download(srcfile) @@ -66,13 +72,9 @@ const bonito_app = NamedApp("larvatagger", end ) -function purgesession(session) - model = lock(bonito_models) do - pop!(bonito_models.cache, session) - end - isnothing(model.app) || close(model.app) - purge_appdata(session, "larvatagger") - BonitoServer.addsession(bonito_app, session; restart=true) +function purgesession(session_id) + model = pop!(session_registry[session_id], :model, nothing) + purgesession(model, session_id) end @@ -82,10 +84,10 @@ end @in reset_bonito_session = false @onchange isready begin - genie_session = GenieSession.session() - bonito_session_id = genie_session.id[1:32] - if haskey(bonito_models, bonito_session_id) - sizefactor = sizefactors_str[bonito_models[bonito_session_id].sizefactor] + bonito_session_id = getsessionid(session_registry) + if haskey(session_registry[bonito_session_id], :model) + model = session_registry[bonito_session_id, :model] + sizefactor = sizefactors_str[model.sizefactor] else BonitoServer.addsession(bonito_app, bonito_session_id) end @@ -103,9 +105,10 @@ end @onchange sizefactor begin new_sizefactor = sizefactors[sizefactor] - if new_sizefactor != bonito_models[bonito_session_id].sizefactor + model = session_registry[bonito_session_id, :model] + if new_sizefactor != model.sizefactor @info "New size factor; refreshing the iframe" - bonito_models[bonito_session_id].sizefactor = new_sizefactor + model.sizefactor = new_sizefactor BonitoServer.addsession(bonito_app, bonito_session_id; restart=true) # restart width = appwidth[sizefactor] height = appheight[sizefactor] @@ -119,7 +122,11 @@ end end @onbutton reset_bonito_session begin - purgesession(bonito_session_id) + try + purgesession(bonito_session_id) + catch + @error "Error while resetting session" + end # reset UI to defaults bonito_session_id[!] = "" sizefactor[!] = "1.0" diff --git a/src/apps/muscles/Backbone.jl b/src/apps/muscles/Backbone.jl index 5547f55..e833beb 100644 --- a/src/apps/muscles/Backbone.jl +++ b/src/apps/muscles/Backbone.jl @@ -3,6 +3,7 @@ module Backbone using NyxWidgets.Base: Cache using NyxUI.MuscleActivities using NyxUI.Storage +using NyxUI.GenieExtras using Bonito include("MuscleWidgets.jl") @@ -32,37 +33,35 @@ export hasmodel, getmodel, setmodel, newsequence, getsequence, withsequences, ## models -const __backbone_cache__ = Cache{String, MuscleWidget}() -const __valid_sessions__ = Dict{String, Bool}() +function purgesession end + +const session_registry = SessionRegistry(purgesession, (:session_id, )) function app(persistent=""; title="Nyx muscle activity") App(; title=title) do session - # # embedded apps are always renderded twice; save time # EDIT: except on Ctrl+R - # valid = get(__valid_sessions__, persistent, false) - # __valid_sessions__[persistent] = !valid - # valid || return @info "New session" session.id cache_id = isempty(persistent) ? session.id : persistent - model = __backbone_cache__[cache_id] + model = session_registry[cache_id, :model] Bonito.jsrender(session, model) end end ## muscle model -getmodel(session_id) = __backbone_cache__[session_id]#.widget - -# for debugging -getmodel() = first(values(__backbone_cache__))#.widget +getmodel(session_id) = session_registry[session_id, :model] -hasmodel(session_id) = haskey(__backbone_cache__, session_id) +hasmodel(session_id) = haskey(session_registry[session_id], :model) function setmodel(session_id, sequence) - __backbone_cache__[session_id] = MuscleWidget(sequence) + records = getsession(session_registry, session_id) + records[:model] = model = MuscleWidget(sequence) + # init :sequences + get!(records, :sequences) do + Cache{Int, MuscleActivity}() + end + return model end -# getchannel(session_id) = __backbone_cache__[session_id].channel - ## sequence model function newsequencename(stem, existing) @@ -97,14 +96,13 @@ function newsequence(session_id, sequence_name, sequence_names, start_time, time return sequence end -const __sequences__ = Cache{String, Cache{Int, MuscleActivity}}() - function getsequence(session_id, sequence_name) model = getmodel(session_id) sequence = nothing if model.sequence[].program_name != sequence_name - lock(__sequences__[session_id]) do - for outer sequence in values(__sequences__[session_id]) + sequences = session_registry[session_id, :sequences] + lock(sequences) do + for outer sequence in values(sequences) sequence.program_name == sequence_name && break end end @@ -116,16 +114,16 @@ end getsequence(session_id) = getmodel(session_id).sequence[] function addsequence(session_id, sequence) - lock(__sequences__[session_id]) do - sequences = __sequences__[session_id] + sequences = session_registry[session_id, :sequences] + lock(sequences) do i = isempty(sequences) ? 0 : maximum(keys(sequences)) sequences[i+1] = sequence end end function withsequences(f, session_id) - lock(__sequences__[session_id]) do - sequences = __sequences__[session_id] + sequences = session_registry[session_id, :sequences] + lock(sequences) do return f(sequences) end end @@ -153,7 +151,7 @@ end function deletesequence(session_id) model = getmodel(session_id) current = model.sequence[].program_name - sequences = __sequences__[session_id] + sequences = session_registry[session_id, :sequences] lock(sequences) do for (ix, seq) in pairs(sequences) if seq.program_name == current @@ -167,19 +165,7 @@ function deletesequence(session_id) end function purgesession(session_id; appname="muscles") - model = lock(__backbone_cache__) do - if session_id in keys(__backbone_cache__.cache) - #pop!(__valid_sessions__, session_id) # not sure what to do with it - pop!(__backbone_cache__.cache, session_id) - end - end - lock(__sequences__) do - if session_id in keys(__sequences__.cache) - pop!(__sequences__.cache, session_id) - end - end - purge_appdata(session_id, appname) - return model + purge_appdata(session_id, appname) end end diff --git a/src/apps/muscles/app.jl b/src/apps/muscles/app.jl index 6140bd2..044e4cc 100644 --- a/src/apps/muscles/app.jl +++ b/src/apps/muscles/app.jl @@ -2,7 +2,7 @@ module MuscleApp using NyxUI, NyxUI.Storage, NyxUI.GenieExtras using Observables -using Genie, GenieSession, Stipple, StippleUI +using Genie, Stipple, StippleUI import Stipple: @app, @init, @private, @in, @out, @onchange, @onbutton, @notify include("Backbone.jl") @@ -20,6 +20,8 @@ const maxlength = getconfig("muscle-activity", "maxlength") const bonito_app = NamedApp(:inherit, Backbone.app) +Backbone.purgesession(session_id) = purgesession(session_id; appname=bonito_app.name) + @app begin @private bonito_session_id = "" @@ -95,7 +97,7 @@ const bonito_app = NamedApp(:inherit, Backbone.app) end @onbutton reset_bonito_session begin - model = purgesession(bonito_session_id; appname=bonito_app.name) + purgesession(bonito_session_id) # reset UI to defaults #bonito_session_id[!] = "" sequence[!] = MuscleActivity("") @@ -445,8 +447,7 @@ function init_model(muscle_model=nothing) muscle_model = @init #@show muscle_model end - genie_session = GenieSession.session() - session_id = genie_session.id[1:32] + session_id = getsessionid(Backbone.session_registry) if hasmodel(session_id) withsequences(session_id) do sequences ids = sort!(collect(keys(sequences))) -- GitLab From 5525091d62c85cc43c2a33d8620eba9c980ee5d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Wed, 16 Apr 2025 19:15:40 +0200 Subject: [PATCH 19/24] LT upgrade --- .gitlab-ci.yml | 7 +++++-- front/Manifest.toml | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20cae5f..829c297 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,8 +73,11 @@ deploy to pasteur.cloud: INGRESS_CLASS: "external" INGRESS_URL: "nyx.pasteur.cloud" LIMITS_CPU: "2" - LIMITS_MEMORY: "4Gi" + LIMITS_MEMORY: "5Gi" LIMITS_STORAGE: "4Gi" + LT_LIMITS_CPU: "2" + LT_LIMITS_MEMORY: "5Gi" + LT_LIMITS_STORAGE: "2Gi" environment: name: k8sprod-02/nyx-prod url: https://nyx.pasteur.cloud @@ -109,7 +112,7 @@ build larvatagger backend: extends: .build-with-buildah variables: IMAGE_NAME: "larvatagger" - IMAGE_TAG: "0.20-bigfat" + IMAGE_TAG: "0.20.1-bigfat" BUILD_CONTEXT: "back" CONTAINERFILE: "back/Containerfile.larvatagger" rules: diff --git a/front/Manifest.toml b/front/Manifest.toml index db9a5df..59993e6 100644 --- a/front/Manifest.toml +++ b/front/Manifest.toml @@ -834,11 +834,11 @@ version = "1.4.0" [[deps.LarvaTagger]] deps = ["Bonito", "Colors", "Dates", "DocOpt", "Format", "HTTP", "JSON3", "LinearAlgebra", "Logging", "Makie", "Meshes", "NearestNeighbors", "NyxWidgets", "Observables", "ObservationPolicies", "OrderedCollections", "Oxygen", "PlanarLarvae", "Random", "StaticArrays", "Statistics", "TidyObservables", "WGLMakie"] -git-tree-sha1 = "ee617c0f75eb1bb680ce8d100e7893012fc8590b" -repo-rev = "main" +git-tree-sha1 = "e4a16320c3526bbd0961d2c852549630ebc6841d" +repo-rev = "dev" repo-url = "https://gitlab.pasteur.fr/nyx/larvatagger.jl" uuid = "8b3b36f1-dfed-446e-8561-ea19fe966a4d" -version = "0.20.0" +version = "0.20.1" [[deps.LazyArtifacts]] deps = ["Artifacts", "Pkg"] -- GitLab From f30775590ca40918008a8d6b9d32f1556ace20bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Thu, 17 Apr 2025 12:33:59 +0200 Subject: [PATCH 20/24] fix: tag latest is never pulled despite Always pullPolicy --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 829c297..dfd29d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ variables: GITLAB_PASTEUR_PROJECT_ID: 6531 PROJECT_NAME: nyxui + LARVATAGGER_IMAGE_TAG: 0.20.1-bigfat stages: - build @@ -100,6 +101,7 @@ deploy to dev.pasteur.cloud: LT_LIMITS_CPU: "2" LT_LIMITS_MEMORY: "5Gi" LT_LIMITS_STORAGE: "2Gi" + LT_IMAGE_TAG: "$LARVATAGGER_IMAGE_TAG" environment: name: k8sdev-01/nyx-dev url: https://nyx.dev.pasteur.cloud @@ -112,7 +114,7 @@ build larvatagger backend: extends: .build-with-buildah variables: IMAGE_NAME: "larvatagger" - IMAGE_TAG: "0.20.1-bigfat" + IMAGE_TAG: "$LARVATAGGER_IMAGE_TAG" BUILD_CONTEXT: "back" CONTAINERFILE: "back/Containerfile.larvatagger" rules: -- GitLab From ed1e8996ed947ffc911b7c05ca4235cdaeb6e2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Thu, 17 Apr 2025 12:48:01 +0200 Subject: [PATCH 21/24] fix: revert previous commit and pass LARVATAGGER_IMAGE_TAG build arg --- .gitlab-ci.yml | 6 ++---- back/Containerfile.larvatagger | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dfd29d8..c3673a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,6 @@ variables: GITLAB_PASTEUR_PROJECT_ID: 6531 PROJECT_NAME: nyxui - LARVATAGGER_IMAGE_TAG: 0.20.1-bigfat stages: - build @@ -101,7 +100,6 @@ deploy to dev.pasteur.cloud: LT_LIMITS_CPU: "2" LT_LIMITS_MEMORY: "5Gi" LT_LIMITS_STORAGE: "2Gi" - LT_IMAGE_TAG: "$LARVATAGGER_IMAGE_TAG" environment: name: k8sdev-01/nyx-dev url: https://nyx.dev.pasteur.cloud @@ -114,8 +112,8 @@ build larvatagger backend: extends: .build-with-buildah variables: IMAGE_NAME: "larvatagger" - IMAGE_TAG: "$LARVATAGGER_IMAGE_TAG" - BUILD_CONTEXT: "back" + IMAGE_TAG: "0.20.1-bigfat" + BUILD_CONTEXT: "--build-arg LARVATAGGER_IMAGE_TAG=$IMAGE_TAG back" CONTAINERFILE: "back/Containerfile.larvatagger" rules: - if: ($CI_COMMIT_BRANCH == "dev" && diff --git a/back/Containerfile.larvatagger b/back/Containerfile.larvatagger index bc9e4af..963f042 100644 --- a/back/Containerfile.larvatagger +++ b/back/Containerfile.larvatagger @@ -1,4 +1,4 @@ -ARG LARVATAGGER_IMAGE_TAG=0.20-bigfat +ARG LARVATAGGER_IMAGE_TAG=0.20.1-bigfat FROM docker.io/flaur/larvatagger:${LARVATAGGER_IMAGE_TAG} as src -- GitLab From 669ac15be9530559ee6b5a301f7b68c0646d459e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Thu, 17 Apr 2025 13:48:17 +0200 Subject: [PATCH 22/24] fix: pass LARVATAGGER_IMAGE_TAG build arg (2) --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c3673a9..cbc6376 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -113,7 +113,7 @@ build larvatagger backend: variables: IMAGE_NAME: "larvatagger" IMAGE_TAG: "0.20.1-bigfat" - BUILD_CONTEXT: "--build-arg LARVATAGGER_IMAGE_TAG=$IMAGE_TAG back" + BUILD_CONTEXT: "--build-arg LARVATAGGER_IMAGE_TAG=0.20.1-bigfat back" CONTAINERFILE: "back/Containerfile.larvatagger" rules: - if: ($CI_COMMIT_BRANCH == "dev" && -- GitLab From a1f34a007874a14537f26c36aa300725da6f35d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Thu, 17 Apr 2025 14:08:14 +0200 Subject: [PATCH 23/24] fix: pass LARVATAGGER_IMAGE_TAG build arg (3) --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbc6376..fd8d8e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ stages: - export REGISTRY_AUTH_FILE="$HOME/auth.json" - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" script: - - buildah build -t "$FQ_IMAGE_NAME" -f "$CONTAINERFILE" "$BUILD_CONTEXT" + - buildah build -t "$FQ_IMAGE_NAME" -f "$CONTAINERFILE" $BUILD_CONTEXT - buildah push "$FQ_IMAGE_NAME" - buildah tag "$FQ_IMAGE_NAME" "$REGISTRY_IMAGE:latest" - buildah push "$REGISTRY_IMAGE:latest" @@ -113,7 +113,7 @@ build larvatagger backend: variables: IMAGE_NAME: "larvatagger" IMAGE_TAG: "0.20.1-bigfat" - BUILD_CONTEXT: "--build-arg LARVATAGGER_IMAGE_TAG=0.20.1-bigfat back" + BUILD_CONTEXT: --build-arg LARVATAGGER_IMAGE_TAG=0.20.1-bigfat back CONTAINERFILE: "back/Containerfile.larvatagger" rules: - if: ($CI_COMMIT_BRANCH == "dev" && -- GitLab From 73974f1e62db2eb4ad2719b3463f1111f3fc38ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= <francois.laurent@posteo.net> Date: Thu, 17 Apr 2025 18:14:38 +0200 Subject: [PATCH 24/24] cleanup --- front/Manifest.toml | 4 ++-- src/apps/muscles/Backbone.jl | 19 ------------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/front/Manifest.toml b/front/Manifest.toml index 59993e6..c55bc77 100644 --- a/front/Manifest.toml +++ b/front/Manifest.toml @@ -834,8 +834,8 @@ version = "1.4.0" [[deps.LarvaTagger]] deps = ["Bonito", "Colors", "Dates", "DocOpt", "Format", "HTTP", "JSON3", "LinearAlgebra", "Logging", "Makie", "Meshes", "NearestNeighbors", "NyxWidgets", "Observables", "ObservationPolicies", "OrderedCollections", "Oxygen", "PlanarLarvae", "Random", "StaticArrays", "Statistics", "TidyObservables", "WGLMakie"] -git-tree-sha1 = "e4a16320c3526bbd0961d2c852549630ebc6841d" -repo-rev = "dev" +git-tree-sha1 = "0b405b69b603ef4df736142907a32ab81df20215" +repo-rev = "main" repo-url = "https://gitlab.pasteur.fr/nyx/larvatagger.jl" uuid = "8b3b36f1-dfed-446e-8561-ea19fe966a4d" version = "0.20.1" diff --git a/src/apps/muscles/Backbone.jl b/src/apps/muscles/Backbone.jl index e833beb..320d214 100644 --- a/src/apps/muscles/Backbone.jl +++ b/src/apps/muscles/Backbone.jl @@ -12,25 +12,6 @@ using .MuscleWidgets export hasmodel, getmodel, setmodel, newsequence, getsequence, withsequences, exportsequence, loadsequence, deletesequence, purgesession -# mutable struct Backbone{W} -# widget::Union{Nothing, W} -# channel::Observable{Bonito.JSCode} -# end -# -# Backbone{W}() where {W} = Backbone{W}(nothing) -# -# Backbone{W}(widget::W) where {W} = Backbone{W}(widget, Observable(js"")) -# -# Backbone(widget::W) where {W} = Backbone{W}(widget) -# -# function Bonito.jsrender(session::Session, widget::Backbone) -# isnothing(widget.widget) && error("backbone widget not initialized") -# on(session, widget.channel) do jscode -# evaljs(session, jscode) -# end -# Bonito.jsrender(session, widget.widget) -# end - ## models function purgesession end -- GitLab