From 9d8444133462d9c3b2de86d02bf46428a2d2623d Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Thu, 21 Apr 2022 12:12:56 +0200 Subject: [PATCH 1/6] check free space, and remove older project is space is under the limit. No limit by default. Only enabled in k8s --- chart/templates/deployment-back.yaml | 4 ++++ chart/values.yaml | 1 + jass/config.py | 1 + jass/models/project.py | 21 +++++++++++++++++++++ jass/server.py | 3 ++- 5 files changed, 29 insertions(+), 1 deletion(-) diff --git a/chart/templates/deployment-back.yaml b/chart/templates/deployment-back.yaml index 9585dc5..14579b2 100644 --- a/chart/templates/deployment-back.yaml +++ b/chart/templates/deployment-back.yaml @@ -45,6 +45,10 @@ spec: value: {{ printf "%s-rabbitmq" .Release.Name }} - name: RABBITMQ_PORT value: '5672' + {{- if .Values.projects.minSizeToKeepFree }} + - name: MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB + value: {{ .Values.projects.minSizeToKeepFree }} + {{- end }} resources: {{- toYaml .Values.back.resources | nindent 12 }} volumeMounts: diff --git a/chart/values.yaml b/chart/values.yaml index e4bab34..71352e8 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -83,6 +83,7 @@ backCronTasks: projects: size: 30Gi + minSizeToKeepFree: 10240 # size is in MB storageClassName: isilon client: diff --git a/jass/config.py b/jass/config.py index b2d4cca..a1211ca 100644 --- a/jass/config.py +++ b/jass/config.py @@ -26,6 +26,7 @@ config = { os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "projects") ), "INITTABLE_CHUNKSIZE": 100000, + "MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB": int(os.environ.get('MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR', '0')), } if "JASS_HOST" in os.environ: config["HOST"] = os.environ["JASS_HOST"] diff --git a/jass/models/project.py b/jass/models/project.py index c375f01..7167ea1 100644 --- a/jass/models/project.py +++ b/jass/models/project.py @@ -4,6 +4,7 @@ compute joint statistics and generate plots for a given set of phenotypes """ from __future__ import absolute_import import abc +import shutil from datetime import datetime from pathlib import Path import json @@ -335,3 +336,23 @@ def get_projects_last_access(): ) ) return res + + +def ensure_space_in_project_dir(): + if config["MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB"] <= 0: + return + while True: + _, _, free = shutil.disk_usage(config["PROJECTS_DIR"]) + free /= 2 ** 20 + if free > config["MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB"]: + break + print(f'Free space is lower than minimum allowed ' + f'({free} MB<{config["MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB"]} MB), ' + f'removing older project') + try: + proj = sorted(get_projects_last_access(), key=lambda p: p['last_access'])[0] + print(f"Project {proj['project_id']} is the older project, last accessed on {proj['last_access']}.") + shutil.rmtree(proj['path']) + except IndexError: + print("No more project to remove") + break diff --git a/jass/server.py b/jass/server.py index fe89e9c..6e6fcf7 100644 --- a/jass/server.py +++ b/jass/server.py @@ -10,7 +10,7 @@ from jass import util from jass.config import config from jass.models.phenotype import Phenotype, get_available_phenotypes, PhenotypeIdList from jass.models.inittable import get_inittable_meta -from jass.models.project import GlobalProject, load_project +from jass.models.project import GlobalProject, load_project, ensure_space_in_project_dir from jass.tasks import create_project from fastapi import FastAPI @@ -51,6 +51,7 @@ def inittable_meta(): @app.post("/api/projects", response_model=GlobalProject) def project_create(phenotype_id_list: PhenotypeIdList): + ensure_space_in_project_dir() return create_project( phenotype_id_list.phenotypeID, get_available_phenotypes(os.path.join(config["DATA_DIR"], "initTable.hdf5")), -- GitLab From 32dccf6035ba35ae92b10d4f9de7ce7dc56752f8 Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Thu, 21 Apr 2022 12:13:45 +0200 Subject: [PATCH 2/6] less space is needed as now monitored before running a new project --- chart/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/values.yaml b/chart/values.yaml index 71352e8..36f31b1 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -82,7 +82,7 @@ backCronTasks: periodicity: "6 6 * * *" projects: - size: 30Gi + size: 20Gi minSizeToKeepFree: 10240 # size is in MB storageClassName: isilon -- GitLab From 71ae670dcdbf0e88803d0c09981e9584c4eea0d9 Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Thu, 21 Apr 2022 12:58:37 +0200 Subject: [PATCH 3/6] quote value --- chart/templates/deployment-back.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/templates/deployment-back.yaml b/chart/templates/deployment-back.yaml index 14579b2..7e34c0a 100644 --- a/chart/templates/deployment-back.yaml +++ b/chart/templates/deployment-back.yaml @@ -47,7 +47,7 @@ spec: value: '5672' {{- if .Values.projects.minSizeToKeepFree }} - name: MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB - value: {{ .Values.projects.minSizeToKeepFree }} + value: '{{ .Values.projects.minSizeToKeepFree }}' {{- end }} resources: {{- toYaml .Values.back.resources | nindent 12 }} -- GitLab From 345ee5b77dc5515ffaf5f47c55ada2f4578ed513 Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Thu, 21 Apr 2022 13:17:30 +0200 Subject: [PATCH 4/6] typo in env var name --- jass/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jass/config.py b/jass/config.py index a1211ca..ceb8019 100644 --- a/jass/config.py +++ b/jass/config.py @@ -26,7 +26,7 @@ config = { os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "projects") ), "INITTABLE_CHUNKSIZE": 100000, - "MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB": int(os.environ.get('MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR', '0')), + "MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB": int(os.environ.get('MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB', '0')), } if "JASS_HOST" in os.environ: config["HOST"] = os.environ["JASS_HOST"] -- GitLab From e781dec6fd45e846b205750ed0e3c5ed7a663d57 Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Thu, 21 Apr 2022 13:51:30 +0200 Subject: [PATCH 5/6] check space but keep project currently accessed in any case --- jass/models/project.py | 7 +++++-- jass/server.py | 3 +-- jass/tasks.py | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/jass/models/project.py b/jass/models/project.py index 7167ea1..26d6e73 100644 --- a/jass/models/project.py +++ b/jass/models/project.py @@ -338,7 +338,7 @@ def get_projects_last_access(): return res -def ensure_space_in_project_dir(): +def ensure_space_in_project_dir(*, except_project_id=None): if config["MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB"] <= 0: return while True: @@ -350,7 +350,10 @@ def ensure_space_in_project_dir(): f'({free} MB<{config["MIN_SIZE_TO_KEEP_IN_PROJECTS_DIR_IN_MB"]} MB), ' f'removing older project') try: - proj = sorted(get_projects_last_access(), key=lambda p: p['last_access'])[0] + projects = get_projects_last_access() + if except_project_id is not None: + projects = filter(lambda p: p.id != except_project_id, projects) + proj = sorted(projects, key=lambda p: p['last_access'])[0] print(f"Project {proj['project_id']} is the older project, last accessed on {proj['last_access']}.") shutil.rmtree(proj['path']) except IndexError: diff --git a/jass/server.py b/jass/server.py index 6e6fcf7..fe89e9c 100644 --- a/jass/server.py +++ b/jass/server.py @@ -10,7 +10,7 @@ from jass import util from jass.config import config from jass.models.phenotype import Phenotype, get_available_phenotypes, PhenotypeIdList from jass.models.inittable import get_inittable_meta -from jass.models.project import GlobalProject, load_project, ensure_space_in_project_dir +from jass.models.project import GlobalProject, load_project from jass.tasks import create_project from fastapi import FastAPI @@ -51,7 +51,6 @@ def inittable_meta(): @app.post("/api/projects", response_model=GlobalProject) def project_create(phenotype_id_list: PhenotypeIdList): - ensure_space_in_project_dir() return create_project( phenotype_id_list.phenotypeID, get_available_phenotypes(os.path.join(config["DATA_DIR"], "initTable.hdf5")), diff --git a/jass/tasks.py b/jass/tasks.py index d537397..a7f5f04 100644 --- a/jass/tasks.py +++ b/jass/tasks.py @@ -8,7 +8,7 @@ from fastapi import HTTPException from flask import Flask import jass.models.project -from jass.models.project import GlobalProject, Project +from jass.models.project import GlobalProject, Project, ensure_space_in_project_dir from jass.models.phenotype import Phenotype from jass.config import config @@ -117,6 +117,8 @@ def create_project( ] project = GlobalProject(phenotypes=phenotypes) + ensure_space_in_project_dir(except_project_id=project.id) + # if project.get_type_of_analysis() == Project.LOCAL_ANALYSIS: # # Local Analysis # global_plot_path = None -- GitLab From 50764d387934c4a30fdca660cdd0b378881ad0de Mon Sep 17 00:00:00 2001 From: Bryan Brancotte Date: Thu, 21 Apr 2022 15:19:10 +0200 Subject: [PATCH 6/6] keep log when crashing --- chart/templates/deployment-back.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/chart/templates/deployment-back.yaml b/chart/templates/deployment-back.yaml index 7e34c0a..71059b6 100644 --- a/chart/templates/deployment-back.yaml +++ b/chart/templates/deployment-back.yaml @@ -33,6 +33,7 @@ spec: image: {{ printf "%s/%s/%s:%s" .Values.CI_REGISTRY_IMAGE .Release.Name "backend" .Values.image.tag }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} + terminationMessagePolicy: "FallbackToLogsOnError" env: - name: RABBITMQ_PASSWORD valueFrom: -- GitLab