From 68b7a53e93225c6e86863675b47091393c34fde2 Mon Sep 17 00:00:00 2001
From: Bryan Brancotte <bryan.brancotte@pasteur.fr>
Date: Mon, 4 Apr 2022 16:19:53 +0200
Subject: [PATCH] move projects into a dedicated folder
---
.dockerignore | 1 +
.gitlab-ci.yml | 4 ++--
Dockerfile | 7 +++++--
chart/pvc-shared-data.yaml | 2 +-
chart/templates/deployment-back.yaml | 10 +++++-----
chart/templates/deployment-celery-worker.yaml | 13 +++++++++----
chart/templates/{pvc-data.yaml => pvc.yaml} | 6 +++---
chart/values.yaml | 4 ++--
docker-compose.yaml | 11 ++++-------
jass/config.py | 5 +++++
jass/models/project.py | 12 +++++++++---
projects/README.md | 1 +
12 files changed, 47 insertions(+), 29 deletions(-)
rename chart/templates/{pvc-data.yaml => pvc.yaml} (54%)
create mode 100644 projects/README.md
diff --git a/.dockerignore b/.dockerignore
index d3785139..6e142bde 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,3 +1,4 @@
data/
client/
+projects/
.venv/
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e8b9cbe1..edad1bb0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -41,8 +41,8 @@ test-backend-image:
- touch data/initTable.hdf5
- >
docker run
- -v $(pwd)/data:/code/data
- -v $(pwd)/data:/code/data_persistent
+ -v $(pwd)/data:/data
+ -v $(pwd)/projects:/projects
-v $(pwd)/.eggs:/code/.eggs
-v $(pwd)/jass.egg-info:/code/jass.egg-info
"$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG/backend:$CI_COMMIT_SHA"
diff --git a/Dockerfile b/Dockerfile
index ed07cacf..a11a1960 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,7 @@
FROM python:3.9 as backend
-ENV JASS_DATA_DIR '/code/data'
+ENV JASS_DATA_DIR '/data'
+ENV JASS_PROJECTS_DIR '/projects'
EXPOSE 8080
@@ -17,7 +18,9 @@ RUN addgroup --gid 1000 kiwi \
&& pip install --upgrade pip setuptools \
&& mkdir /code \
&& mkdir ${JASS_DATA_DIR} \
- && chown kiwi:kiwi ${JASS_DATA_DIR}
+ && chown kiwi:kiwi ${JASS_DATA_DIR} \
+ && mkdir ${JASS_PROJECTS_DIR} \
+ && chown kiwi:kiwi ${JASS_PROJECTS_DIR}
WORKDIR /code
COPY requirements*.txt /code/
diff --git a/chart/pvc-shared-data.yaml b/chart/pvc-shared-data.yaml
index 81d85165..24b77916 100644
--- a/chart/pvc-shared-data.yaml
+++ b/chart/pvc-shared-data.yaml
@@ -7,6 +7,6 @@ spec:
- ReadWriteMany
resources:
requests:
- storage: 25Gi
+ storage: 15Gi
storageClassName: isilon
status: {}
\ No newline at end of file
diff --git a/chart/templates/deployment-back.yaml b/chart/templates/deployment-back.yaml
index e97f920d..32bc8814 100644
--- a/chart/templates/deployment-back.yaml
+++ b/chart/templates/deployment-back.yaml
@@ -36,14 +36,14 @@ spec:
resources:
{{- toYaml .Values.back.resources | nindent 12 }}
volumeMounts:
- - mountPath: /code/data
- name: data-files
- - mountPath: /code/shared_data
+ - mountPath: /projects
+ name: projects-files
+ - mountPath: /data
name: shared-data-files
volumes:
- - name: data-files
+ - name: projects-files
persistentVolumeClaim:
- claimName: {{ printf "%s-data-files" .Release.Name }}
+ claimName: {{ printf "%s-projects-files" .Release.Name }}
- name: shared-data-files
persistentVolumeClaim:
claimName: "jass-shared-data-files"
\ No newline at end of file
diff --git a/chart/templates/deployment-celery-worker.yaml b/chart/templates/deployment-celery-worker.yaml
index cebf2772..dfab1f0b 100644
--- a/chart/templates/deployment-celery-worker.yaml
+++ b/chart/templates/deployment-celery-worker.yaml
@@ -47,9 +47,14 @@ spec:
resources:
{{- toYaml .Values.celery.resources | nindent 12 }}
volumeMounts:
- - mountPath: /code/data
- name: data-files
+ - mountPath: /projects
+ name: projects-files
+ - mountPath: /data
+ name: shared-data-files
volumes:
- - name: data-files
+ - name: projects-files
persistentVolumeClaim:
- claimName: {{ printf "%s-data-files" .Release.Name }}
\ No newline at end of file
+ claimName: {{ printf "%s-projects-files" .Release.Name }}
+ - name: shared-data-files
+ persistentVolumeClaim:
+ claimName: "jass-shared-data-files"
\ No newline at end of file
diff --git a/chart/templates/pvc-data.yaml b/chart/templates/pvc.yaml
similarity index 54%
rename from chart/templates/pvc-data.yaml
rename to chart/templates/pvc.yaml
index a7067e77..56d822b0 100644
--- a/chart/templates/pvc-data.yaml
+++ b/chart/templates/pvc.yaml
@@ -1,7 +1,7 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
- name: {{ printf "%s-data-files" .Release.Name }}
+ name: {{ printf "%s-projects-files" .Release.Name }}
labels:
{{- include "chart.labels" . | nindent 4 }}
spec:
@@ -9,6 +9,6 @@ spec:
- ReadWriteMany
resources:
requests:
- storage: {{ .Values.data.size }}
- storageClassName: {{ .Values.data.storageClassName }}
+ storage: {{ .Values.projects.size }}
+ storageClassName: {{ .Values.projects.storageClassName }}
status: {}
\ No newline at end of file
diff --git a/chart/values.yaml b/chart/values.yaml
index 0e2c186d..59d5d0b4 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -69,8 +69,8 @@ back:
memory: 4Gi
cpu: 2000m
-data:
- size: 50Gi
+projects:
+ size: 20Gi
storageClassName: isilon
client:
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 427f59e9..fa43015e 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -19,7 +19,8 @@ services:
JASS_RABBITMQ_URL: 'amqp://guest:guest@rabbitmq:5672'
WAIT_FOR_DATA_DIR_READY: 1
volumes:
- - ./data:/code/data
+ - ./projects:/projects
+ - ./data:/data
- ./jass:/code/jass
depends_on:
- rabbitmq
@@ -31,7 +32,8 @@ services:
environment:
JASS_RABBITMQ_URL: 'amqp://guest:guest@rabbitmq:5672'
volumes:
- - ./data:/code/data
+ - ./projects:/projects
+ - ./data:/data
- ./jass:/code/jass
ports:
- '8080:8080'
@@ -72,8 +74,3 @@ services:
# # - /client/node_modules
# ports:
# - '3001:8080'
-
-
-
-volumes:
- jass-persistent-data:
\ No newline at end of file
diff --git a/jass/config.py b/jass/config.py
index c8bd6ae4..b2d4ccad 100644
--- a/jass/config.py
+++ b/jass/config.py
@@ -22,6 +22,9 @@ config = {
"DATA_DIR": os.path.normpath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data")
),
+ "PROJECTS_DIR": os.path.normpath(
+ os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "projects")
+ ),
"INITTABLE_CHUNKSIZE": 100000,
}
if "JASS_HOST" in os.environ:
@@ -30,3 +33,5 @@ if "JASS_PORT" in os.environ:
config["PORT"] = os.environ["JASS_PORT"]
if "JASS_DATA_DIR" in os.environ:
config["DATA_DIR"] = os.environ["JASS_DATA_DIR"]
+if "JASS_PROJECTS_DIR" in os.environ:
+ config["PROJECTS_DIR"] = os.environ["JASS_PROJECTS_DIR"]
diff --git a/jass/models/project.py b/jass/models/project.py
index 8b2b7124..4280cb7d 100644
--- a/jass/models/project.py
+++ b/jass/models/project.py
@@ -110,10 +110,14 @@ class Project(BaseModel, abc.ABC):
def create_id(self):
return self.get_hash().hexdigest()
+ @classmethod
+ def get_folder_path_from_id(cls, project_id):
+ return os.path.join(config["PROJECTS_DIR"], f"project_{project_id}")
@classmethod
def load(cls, id):
- path = os.path.join(config["DATA_DIR"], f"project_{id}", "meta.json")
+ # path = os.path.join(config["DATA_DIR"], f"project_{id}", "meta.json")
+ path = os.path.join(Project.get_folder_path_from_id(id), "meta.json")
with open(path, "r") as fp:
return cls.parse_raw("".join(fp.readlines()))
@@ -126,7 +130,8 @@ class Project(BaseModel, abc.ABC):
"""
Get the path of the folder where the project data are stored
"""
- return os.path.join(config["DATA_DIR"], f"project_{self.id}")
+ # return os.path.join(config["DATA_DIR"], f"project_{self.id}")
+ return self.get_folder_path_from_id(self.id)
def get_meta_file_path(self):
"""
@@ -284,7 +289,8 @@ class LocalProject(Project):
def load_project(project_id):
- path = os.path.join(config["DATA_DIR"], f"project_{project_id}", "meta.json")
+ path = os.path.join(Project.get_folder_path_from_id(project_id), "meta.json")
+ # path = os.path.join(config["DATA_DIR"], f"project_{project_id}", "meta.json")
with open(path, "r") as fp:
project = parse_raw_as(
Union[
diff --git a/projects/README.md b/projects/README.md
new file mode 100644
index 00000000..10c46d81
--- /dev/null
+++ b/projects/README.md
@@ -0,0 +1 @@
+Projects are created in this directory.
\ No newline at end of file
--
GitLab