From bce6432d8531555e1a960ff51cbb91e4f64838bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9gis=20Behmo?= <regis@behmo.com>
Date: Wed, 25 Mar 2020 18:47:36 +0100
Subject: [PATCH] Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
---
CHANGELOG.md | 1 +
docs/plugins/api.rst | 4 +
requirements/base.in | 1 +
requirements/base.txt | 20 +-
requirements/dev.txt | 20 +-
requirements/docs.txt | 20 +-
tests/test_config.py | 6 +
tests/test_scripts.py | 14 --
tutor.spec | 1 +
tutor/commands/compose.py | 68 ++++++-
tutor/commands/dev.py | 27 ++-
tutor/commands/k8s.py | 187 ++++++++++++++++--
tutor/config.py | 5 +-
tutor/scripts.py | 37 ++--
tutor/serialize.py | 4 +
tutor/templates/k8s/deployments.yml | 9 +-
tutor/templates/k8s/ingress.yml | 11 +-
tutor/templates/k8s/jobs.yml | 106 ++++++++++
tutor/templates/kustomization.yml | 2 +
tutor/templates/local/docker-compose.jobs.yml | 37 ++++
tutor/templates/local/docker-compose.yml | 7 +-
21 files changed, 479 insertions(+), 108 deletions(-)
delete mode 100644 tests/test_scripts.py
create mode 100644 tutor/templates/k8s/jobs.yml
create mode 100644 tutor/templates/local/docker-compose.jobs.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6aa5c9d..34a47a0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ Note: Breaking changes between versions are indicated by "💥".
## Unreleased
- [Improvement] Fix tls certificate generation in k8s
+- [Improvement] Radically change the way jobs are run: we no longer "exec", but instead run a dedicated container.
- [Improvement] Upgrade k8s certificate issuer to cert-manager.io/v1alpha2
- [Feature] Add SCORM XBlock to default openedx docker image
diff --git a/docs/plugins/api.rst b/docs/plugins/api.rst
index 048a92e..f898776 100644
--- a/docs/plugins/api.rst
+++ b/docs/plugins/api.rst
@@ -83,6 +83,10 @@ Example::
During initialisation, "myservice1" and "myservice2" will be run in sequence with the commands defined in the templates ``myplugin/hooks/myservice1/init`` and ``myplugin/hooks/myservice2/init``.
+To initialise a "foo" service, Tutor runs the "foo-job" service that is found in the ``env/local/docker-compose.jobs.yml`` file. By default, Tutor comes with a few services in this file: mysql-job, lms-job, cms-job, forum-job. If your plugin requires running custom services during initialisation, you will need to add them to the ``docker-compose.jobs.yml`` template. To do so, just use the "local-docker-compose-jobs-services" patch.
+
+In Kubernetes, the approach is the same, except that jobs are implemented as actual job objects in the ``k8s/jobs.yml`` template. To add your own services there, your plugin should implement the "k8s-jobs" patch.
+
``pre-init``
++++++++++++
diff --git a/requirements/base.in b/requirements/base.in
index 8b547a6..3e7ed8d 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -2,4 +2,5 @@ appdirs
click>=7.0
click_repl
jinja2>=2.9
+kubernetes
pyyaml>=4.2b1
diff --git a/requirements/base.txt b/requirements/base.txt
index 33a5758..59162bb 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -5,11 +5,29 @@
# pip-compile requirements/base.in
#
appdirs==1.4.3
+cachetools==4.1.0 # via google-auth
+certifi==2020.4.5.1 # via kubernetes, requests
+chardet==3.0.4 # via requests
click-repl==0.1.6
click==7.1.1
+google-auth==1.14.0 # via kubernetes
+idna==2.9 # via requests
jinja2==2.11.1
+kubernetes==11.0.0
markupsafe==1.1.1 # via jinja2
+oauthlib==3.1.0 # via requests-oauthlib
prompt-toolkit==3.0.5 # via click-repl
+pyasn1-modules==0.2.8 # via google-auth
+pyasn1==0.4.8 # via pyasn1-modules, rsa
+python-dateutil==2.8.1 # via kubernetes
pyyaml==5.3.1
-six==1.14.0 # via click-repl
+requests-oauthlib==1.3.0 # via kubernetes
+requests==2.23.0 # via kubernetes, requests-oauthlib
+rsa==4.0 # via google-auth
+six==1.14.0 # via click-repl, google-auth, kubernetes, python-dateutil, websocket-client
+urllib3==1.25.9 # via kubernetes, requests
wcwidth==0.1.9 # via prompt-toolkit
+websocket-client==0.57.0 # via kubernetes
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/requirements/dev.txt b/requirements/dev.txt
index 87a44f1..a803e80 100644
--- a/requirements/dev.txt
+++ b/requirements/dev.txt
@@ -10,44 +10,54 @@ astroid==2.3.3 # via pylint
attrs==19.3.0 # via black
black==19.10b0
bleach==3.1.4 # via readme-renderer
-certifi==2019.11.28 # via requests
+cachetools==4.1.0
+certifi==2020.4.5.1
cffi==1.14.0 # via cryptography
-chardet==3.0.4 # via requests
+chardet==3.0.4
click-repl==0.1.6
click==7.1.1
cryptography==2.8 # via secretstorage
docutils==0.16 # via readme-renderer
-idna==2.9 # via requests
+google-auth==1.14.0
+idna==2.9
importlib-metadata==1.6.0 # via keyring, twine
isort==4.3.21 # via pylint
jeepney==0.4.3 # via keyring, secretstorage
jinja2==2.11.1
keyring==21.2.0 # via twine
+kubernetes==11.0.0
lazy-object-proxy==1.4.3 # via astroid
markupsafe==1.1.1
mccabe==0.6.1 # via pylint
+oauthlib==3.1.0
pathspec==0.7.0 # via black
pip-tools==4.5.1
pkginfo==1.5.0.1 # via twine
prompt-toolkit==3.0.5
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
pycparser==2.20 # via cffi
pygments==2.6.1 # via readme-renderer
pyinstaller==3.6
pylint==2.4.4
+python-dateutil==2.8.1
pyyaml==5.3.1
readme-renderer==25.0 # via twine
regex==2020.2.20 # via black
+requests-oauthlib==1.3.0
requests-toolbelt==0.9.1 # via twine
-requests==2.23.0 # via requests-toolbelt, twine
+requests==2.23.0
+rsa==4.0
secretstorage==3.1.2 # via keyring
six==1.14.0
toml==0.10.0 # via black
tqdm==4.44.1 # via twine
twine==3.1.1
typed-ast==1.4.1 # via astroid, black
-urllib3==1.25.8 # via requests
+urllib3==1.25.9
wcwidth==0.1.9
webencodings==0.5.1 # via bleach
+websocket-client==0.57.0
wrapt==1.11.2 # via astroid
zipp==3.1.0 # via importlib-metadata
diff --git a/requirements/docs.txt b/requirements/docs.txt
index bb7cd33..b91176b 100644
--- a/requirements/docs.txt
+++ b/requirements/docs.txt
@@ -7,22 +7,31 @@
alabaster==0.7.12 # via sphinx
appdirs==1.4.3
babel==2.8.0 # via sphinx
-certifi==2019.11.28 # via requests
-chardet==3.0.4 # via requests
+cachetools==4.1.0
+certifi==2020.4.5.1
+chardet==3.0.4
click-repl==0.1.6
click==7.1.1
docutils==0.16 # via sphinx
-idna==2.9 # via requests
+google-auth==1.14.0
+idna==2.9
imagesize==1.2.0 # via sphinx
jinja2==2.11.1
+kubernetes==11.0.0
markupsafe==1.1.1
+oauthlib==3.1.0
packaging==20.3 # via sphinx
prompt-toolkit==3.0.5
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
pygments==2.6.1 # via sphinx
pyparsing==2.4.6 # via packaging
+python-dateutil==2.8.1
pytz==2019.3 # via babel
pyyaml==5.3.1
-requests==2.23.0 # via sphinx
+requests-oauthlib==1.3.0
+requests==2.23.0
+rsa==4.0
six==1.14.0
snowballstemmer==2.0.0 # via sphinx
sphinx-rtd-theme==0.4.3
@@ -33,8 +42,9 @@ sphinxcontrib-htmlhelp==1.0.3 # via sphinx
sphinxcontrib-jsmath==1.0.1 # via sphinx
sphinxcontrib-qthelp==1.0.3 # via sphinx
sphinxcontrib-serializinghtml==1.1.4 # via sphinx
-urllib3==1.25.8 # via requests
+urllib3==1.25.9
wcwidth==0.1.9
+websocket-client==0.57.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools
diff --git a/tests/test_config.py b/tests/test_config.py
index 105b2b7..e73a671 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -72,3 +72,9 @@ class ConfigTests(unittest.TestCase):
self.assertNotIn("LMS_HOST", config)
self.assertEqual("www.myopenedx.com", defaults["LMS_HOST"])
self.assertEqual("studio.{{ LMS_HOST }}", defaults["CMS_HOST"])
+
+ def test_is_service_activated(self):
+ config = {"ACTIVATE_SERVICE1": True, "ACTIVATE_SERVICE2": False}
+
+ self.assertTrue(tutor_config.is_service_activated(config, "service1"))
+ self.assertFalse(tutor_config.is_service_activated(config, "service2"))
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
deleted file mode 100644
index dfaf4ae..0000000
--- a/tests/test_scripts.py
+++ /dev/null
@@ -1,14 +0,0 @@
-import unittest
-import unittest.mock
-
-from tutor import config as tutor_config
-from tutor import scripts
-
-
-class ScriptsTests(unittest.TestCase):
- def test_is_activated(self):
- config = {"ACTIVATE_SERVICE1": True, "ACTIVATE_SERVICE2": False}
- runner = scripts.BaseRunner("/tmp", config)
-
- self.assertTrue(runner.is_activated("service1"))
- self.assertFalse(runner.is_activated("service2"))
diff --git a/tutor.spec b/tutor.spec
index 68f6efa..cd362c0 100644
--- a/tutor.spec
+++ b/tutor.spec
@@ -27,6 +27,7 @@ hidden_imports.append("Crypto.Hash.SHA256")
hidden_imports.append("Crypto.PublicKey.RSA")
hidden_imports.append("Crypto.Random")
hidden_imports.append("Crypto.Signature.PKCS1_v1_5")
+hidden_imports.append("kubernetes")
hidden_imports.append("uuid")
diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py
index b9602fc..8412a1d 100644
--- a/tutor/commands/compose.py
+++ b/tutor/commands/compose.py
@@ -1,8 +1,10 @@
import click
from .. import config as tutor_config
+from .. import env as tutor_env
from .. import fmt
from .. import scripts
+from .. import serialize
from .. import utils
@@ -11,11 +13,62 @@ class ScriptRunner(scripts.BaseRunner):
super().__init__(root, config)
self.docker_compose_func = docker_compose_func
- def exec(self, service, command):
+ def run_job(self, service, command):
+ """
+ Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the
+ specified command. For backward-compatibility reasons, if the corresponding
+ service does not exist, run the service from good old regular
+ docker-compose.yml.
+ """
+ jobs_path = tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml")
+ job_service_name = "{}-job".format(service)
opts = [] if utils.is_a_tty() else ["-T"]
- self.docker_compose_func(
- self.root, self.config, "exec", *opts, service, "sh", "-e", "-c", command
- )
+ if job_service_name in serialize.load(open(jobs_path).read())["services"]:
+ self.docker_compose_func(
+ self.root,
+ self.config,
+ "-f",
+ jobs_path,
+ "run",
+ *opts,
+ "--rm",
+ job_service_name,
+ "sh",
+ "-e",
+ "-c",
+ command,
+ )
+ else:
+ fmt.echo_alert(
+ (
+ "The '{job_service_name}' service does not exist in {jobs_path}. "
+ "This might be caused by an older plugin. Tutor switched to a job "
+ "runner model for running one-time commands, such as database"
+ " initialisation. For the record, this is the command that we are "
+ "running:\n"
+ "\n"
+ " {command}\n"
+ "\n"
+ "Old-style job running will be deprecated soon. Please inform "
+ "your plugin maintainer!"
+ ).format(
+ job_service_name=job_service_name,
+ jobs_path=jobs_path,
+ command=command.replace("\n", "\n "),
+ )
+ )
+ self.docker_compose_func(
+ self.root,
+ self.config,
+ "run",
+ *opts,
+ "--rm",
+ service,
+ "sh",
+ "-e",
+ "-c",
+ command,
+ )
@click.command(help="Update docker images")
@@ -73,7 +126,7 @@ def restart(context, services):
pass
else:
for service in services:
- if "openedx" == service:
+ if service == "openedx":
if config["ACTIVATE_LMS"]:
command += ["lms", "lms-worker"]
if config["ACTIVATE_CMS"]:
@@ -138,7 +191,7 @@ def run_hook(context, service, path):
fmt.echo_info(
"Running '{}' hook in '{}' container...".format(".".join(path), service)
)
- runner.run(service, *path)
+ runner.run_job_from_template(service, *path)
@click.command(help="View output from containers")
@@ -171,11 +224,10 @@ def logs(context, follow, tail, service):
def createuser(context, superuser, staff, password, name, email):
config = tutor_config.load(context.root)
runner = ScriptRunner(context.root, config, context.docker_compose)
- runner.check_service_is_activated("lms")
command = scripts.create_user_command(
superuser, staff, name, email, password=password
)
- runner.exec("lms", command)
+ runner.run_job("lms", command)
@click.command(
diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py
index f030630..b2fd6d5 100644
--- a/tutor/commands/dev.py
+++ b/tutor/commands/dev.py
@@ -12,20 +12,19 @@ from .. import utils
class DevContext(Context):
@staticmethod
def docker_compose(root, config, *command):
- args = [
- "-f",
- tutor_env.pathjoin(root, "local", "docker-compose.yml"),
- ]
- override_path = tutor_env.pathjoin(root, "local", "docker-compose.override.yml")
- if os.path.exists(override_path):
- args += ["-f", override_path]
- args += [
- "-f",
- tutor_env.pathjoin(root, "dev", "docker-compose.yml"),
- ]
- override_path = tutor_env.pathjoin(root, "dev", "docker-compose.override.yml")
- if os.path.exists(override_path):
- args += ["-f", override_path]
+ args = []
+ for folder in ["local", "dev"]:
+ # Add docker-compose.yml and docker-compose.override.yml (if it exists)
+ # from "local" and "dev" folders
+ args += [
+ "-f",
+ tutor_env.pathjoin(root, folder, "docker-compose.yml"),
+ ]
+ override_path = tutor_env.pathjoin(
+ root, folder, "docker-compose.override.yml"
+ )
+ if os.path.exists(override_path):
+ args += ["-f", override_path]
return utils.docker_compose(
*args, "--project-name", config["DEV_PROJECT_NAME"], *command,
)
diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py
index bffa58d..b837d19 100644
--- a/tutor/commands/k8s.py
+++ b/tutor/commands/k8s.py
@@ -1,10 +1,15 @@
+from datetime import datetime
+from time import sleep
+
import click
from .. import config as tutor_config
from .. import env as tutor_env
+from .. import exceptions
from .. import fmt
from .. import interactive as interactive_config
from .. import scripts
+from .. import serialize
from .. import utils
@@ -47,6 +52,7 @@ def start(context):
"app.kubernetes.io/component=namespace",
)
# Create volumes
+ # TODO: instead, we should use StatefulSets
utils.kubectl(
"apply",
"--kustomize",
@@ -55,8 +61,14 @@ def start(context):
"--selector",
"app.kubernetes.io/component=volume",
)
- # Create everything else
- utils.kubectl("apply", "--kustomize", tutor_env.pathjoin(context.root))
+ # Create everything else except jobs
+ utils.kubectl(
+ "apply",
+ "--kustomize",
+ tutor_env.pathjoin(context.root),
+ "--selector",
+ "app.kubernetes.io/component!=job",
+ )
@click.command(help="Stop a running platform")
@@ -64,7 +76,9 @@ def start(context):
def stop(context):
config = tutor_config.load(context.root)
utils.kubectl(
- "delete", *resource_selector(config), "deployments,services,ingress,configmaps"
+ "delete",
+ *resource_selector(config),
+ "deployments,services,ingress,configmaps,jobs",
)
@@ -108,7 +122,7 @@ def init(context):
config = tutor_config.load(context.root)
runner = K8sScriptRunner(context.root, config)
for service in ["mysql", "elasticsearch", "mongodb"]:
- if runner.is_activated(service):
+ if tutor_config.is_service_activated(config, service):
wait_for_pod_ready(config, service)
scripts.initialise(runner)
@@ -126,8 +140,6 @@ def init(context):
@click.pass_obj
def createuser(context, superuser, staff, password, name, email):
config = tutor_config.load(context.root)
- runner = K8sScriptRunner(context.root, config)
- runner.check_service_is_activated("lms")
command = scripts.create_user_command(
superuser, staff, name, email, password=password
)
@@ -189,24 +201,161 @@ def logs(context, container, follow, tail, service):
utils.kubectl(*command)
+class K8sClients:
+ _instance = None
+
+ def __init__(self):
+ # Loading the kubernetes module here to avoid import overhead
+ from kubernetes import client, config # pylint: disable=import-outside-toplevel
+
+ config.load_kube_config()
+ self._batch_api = None
+ self._core_api = None
+ self._client = client
+
+ @classmethod
+ def instance(cls):
+ if cls._instance is None:
+ cls._instance = cls()
+ return cls._instance
+
+ @property
+ def batch_api(self):
+ if self._batch_api is None:
+ self._batch_api = self._client.BatchV1Api()
+ return self._batch_api
+
+ @property
+ def core_api(self):
+ if self._core_api is None:
+ self._core_api = self._client.CoreV1Api()
+ return self._core_api
+
+
class K8sScriptRunner(scripts.BaseRunner):
- def exec(self, service, command):
- kubectl_exec(self.config, service, command, attach=False)
+ def load_job(self, name):
+ jobs = self.render("k8s", "jobs.yml")
+ for job in serialize.load_all(jobs):
+ if job["metadata"]["name"] == name:
+ return job
+ raise ValueError("Could not find job '{}'".format(name))
+
+ def active_job_names(self):
+ """
+ Return a list of active job names
+ Docs:
+ https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#list-job-v1-batch
+ """
+ api = K8sClients.instance().batch_api
+ return [
+ job.metadata.name
+ for job in api.list_namespaced_job(self.config["K8S_NAMESPACE"]).items
+ if job.status.active
+ ]
+
+ def run_job(self, service, command):
+ job_name = "{}-job".format(service)
+ try:
+ job = self.load_job(job_name)
+ except ValueError:
+ message = (
+ "The '{job_name}' kubernetes job does not exist in the list of job "
+ "runners. This might be caused by an older plugin. Tutor switched to a"
+ " job runner model for running one-time commands, such as database"
+ " initialisation. For the record, this is the command that we are "
+ "running:\n"
+ "\n"
+ " {command}\n"
+ "\n"
+ "Old-style job running will be deprecated soon. Please inform "
+ "your plugin maintainer!"
+ ).format(
+ job_name=job_name,
+ command=command.replace("\n", "\n "),
+ )
+ fmt.echo_alert(message)
+ wait_for_pod_ready(self.config, service)
+ kubectl_exec(self.config, service, command)
+ return
+ # Create a unique job name to make it deduplicate jobs and make it easier to
+ # find later. Logs of older jobs will remain available for some time.
+ job_name += "-" + datetime.now().strftime("%Y%m%d%H%M%S")
+
+ # Wait until all other jobs are completed
+ while True:
+ active_jobs = self.active_job_names()
+ if not active_jobs:
+ break
+ fmt.echo_info(
+ "Waiting for active jobs to terminate: {}".format(" ".join(active_jobs))
+ )
+ sleep(5)
+
+ # Configure job
+ job["metadata"]["name"] = job_name
+ job["metadata"].setdefault("labels", {})
+ job["metadata"]["labels"]["app.kubernetes.io/name"] = job_name
+ job["spec"]["template"]["spec"]["containers"][0]["args"] = [
+ "sh",
+ "-e",
+ "-c",
+ command,
+ ]
+ job["spec"]["backoffLimit"] = 1
+ job["spec"]["ttlSecondsAfterFinished"] = 3600
+ # Save patched job to "jobs.yml" file
+ with open(tutor_env.pathjoin(self.root, "k8s", "jobs.yml"), "w") as job_file:
+ serialize.dump(job, job_file)
+ # We cannot use the k8s API to create the job: configMap and volume names need
+ # to be found with the right suffixes.
+ utils.kubectl(
+ "apply",
+ "--kustomize",
+ tutor_env.pathjoin(self.root),
+ "--selector",
+ "app.kubernetes.io/name={}".format(job_name),
+ )
+
+ message = (
+ "Job {job_name} is running. To view the logs from this job, run:\n\n"
+ """ kubectl logs --namespace={namespace} --follow $(kubectl get --namespace={namespace} pods """
+ """--selector=job-name={job_name} -o=jsonpath="{{.items[0].metadata.name}}")\n\n"""
+ "Waiting for job completion..."
+ ).format(job_name=job_name, namespace=self.config["K8S_NAMESPACE"])
+ fmt.echo_info(message)
+
+ # Wait for completion
+ field_selector = "metadata.name={}".format(job_name)
+ while True:
+ jobs = K8sClients.instance().batch_api.list_namespaced_job(
+ self.config["K8S_NAMESPACE"], field_selector=field_selector
+ )
+ if not jobs.items:
+ continue
+ job = jobs.items[0]
+ if not job.status.active:
+ if job.status.succeeded:
+ fmt.echo_info("Job {} successful.".format(job_name))
+ break
+ if job.status.failed:
+ raise exceptions.TutorError(
+ "Job {} failed. View the job logs to debug this issue.".format(
+ job_name
+ )
+ )
+ sleep(5)
def kubectl_exec(config, service, command, attach=False):
selector = "app.kubernetes.io/name={}".format(service)
-
- # Find pod in runner deployment
- wait_for_pod_ready(config, service)
- fmt.echo_info("Finding pod name for {} deployment...".format(service))
- pod = utils.check_output(
- "kubectl",
- "get",
- *resource_selector(config, selector),
- "pods",
- "-o=jsonpath={.items[0].metadata.name}",
+ pods = K8sClients.instance().core_api.list_namespaced_pod(
+ namespace=config["K8S_NAMESPACE"], label_selector=selector
)
+ if not pods.items:
+ raise exceptions.TutorError(
+ "Could not find an active pod for the {} service".format(service)
+ )
+ pod_name = pods.items[0].metadata.name
# Run command
attach_opts = ["-i", "-t"] if attach else []
@@ -215,7 +364,7 @@ def kubectl_exec(config, service, command, attach=False):
*attach_opts,
"--namespace",
config["K8S_NAMESPACE"],
- pod.decode(),
+ pod_name,
"--",
"sh",
"-e",
diff --git a/tutor/config.py b/tutor/config.py
index ddd5960..2869beb 100644
--- a/tutor/config.py
+++ b/tutor/config.py
@@ -1,4 +1,3 @@
-import json
import os
from . import exceptions
@@ -128,6 +127,10 @@ def load_plugins(config, defaults):
defaults[plugin.config_key(key)] = value
+def is_service_activated(config, service):
+ return config["ACTIVATE_" + service.upper()]
+
+
def upgrade_obsolete(config):
# Openedx-specific mysql passwords
if "MYSQL_PASSWORD" in config:
diff --git a/tutor/scripts.py b/tutor/scripts.py
index 5513416..acad508 100644
--- a/tutor/scripts.py
+++ b/tutor/scripts.py
@@ -1,5 +1,4 @@
from . import env
-from . import exceptions
from . import fmt
from . import plugins
@@ -14,34 +13,23 @@ class BaseRunner:
self.root = root
self.config = config
- def run(self, service, *path):
+ def run_job_from_template(self, service, *path):
command = self.render(*path)
- self.exec(service, command)
+ self.run_job(service, command)
def render(self, *path):
return env.render_file(self.config, *path).strip()
- def exec(self, service, command):
+ def run_job(self, service, command):
raise NotImplementedError
- def check_service_is_activated(self, service):
- if not self.is_activated(service):
- raise exceptions.TutorError(
- "This command may only be executed on the server where the {} is running".format(
- service
- )
- )
-
- def is_activated(self, service):
- return self.config["ACTIVATE_" + service.upper()]
-
def iter_plugin_hooks(self, hook):
yield from plugins.iter_hooks(self.config, hook)
def initialise(runner):
fmt.echo_info("Initialising all services...")
- runner.run("mysql", "hooks", "mysql", "init")
+ runner.run_job_from_template("mysql", "hooks", "mysql", "init")
for plugin_name, hook in runner.iter_plugin_hooks("pre-init"):
for service in hook:
fmt.echo_info(
@@ -49,17 +37,18 @@ def initialise(runner):
plugin_name, service
)
)
- runner.run(service, plugin_name, "hooks", service, "pre-init")
+ runner.run_job_from_template(
+ service, plugin_name, "hooks", service, "pre-init"
+ )
for service in ["lms", "cms", "forum"]:
- if runner.is_activated(service):
- fmt.echo_info("Initialising {}...".format(service))
- runner.run(service, "hooks", service, "init")
+ fmt.echo_info("Initialising {}...".format(service))
+ runner.run_job_from_template(service, "hooks", service, "init")
for plugin_name, hook in runner.iter_plugin_hooks("init"):
for service in hook:
fmt.echo_info(
"Plugin {}: running init for service {}...".format(plugin_name, service)
)
- runner.run(service, plugin_name, "hooks", service, "init")
+ runner.run_job_from_template(service, plugin_name, "hooks", service, "init")
fmt.echo_info("All services initialised.")
@@ -90,8 +79,7 @@ u.save()"
def import_demo_course(runner):
- runner.check_service_is_activated("cms")
- runner.run("cms", "hooks", "cms", "importdemocourse")
+ runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse")
def set_theme(theme_name, domain_name, runner):
@@ -108,5 +96,4 @@ site.themes.all().delete()
site.themes.create(theme_dir_name='{theme_name}')"
"""
command = command.format(theme_name=theme_name, domain_name=domain_name)
- runner.check_service_is_activated("lms")
- runner.exec("lms", command)
+ runner.run_job("lms", command)
diff --git a/tutor/serialize.py b/tutor/serialize.py
index 99a29da..98b7bdb 100644
--- a/tutor/serialize.py
+++ b/tutor/serialize.py
@@ -7,6 +7,10 @@ def load(stream):
return yaml.load(stream, Loader=yaml.SafeLoader)
+def load_all(stream):
+ return yaml.load_all(stream, Loader=yaml.SafeLoader)
+
+
def dump(content, fileobj):
yaml.dump(content, stream=fileobj, default_flow_style=False)
diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml
index 9ea7614..b3730b9 100644
--- a/tutor/templates/k8s/deployments.yml
+++ b/tutor/templates/k8s/deployments.yml
@@ -295,6 +295,7 @@ spec:
persistentVolumeClaim:
claimName: mongodb
{% endif %}
+{% if ACTIVATE_MYSQL %}
---
apiVersion: apps/v1
kind: Deployment
@@ -316,12 +317,7 @@ spec:
containers:
- name: mysql
image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
- {% if ACTIVATE_MYSQL %}
args: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci"]
- {% else %}
- command: ["sh", "-e", "-c"]
- args: ["echo 'ready'; while true; do sleep 60; done"]
- {% endif %}
env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
@@ -330,7 +326,6 @@ spec:
key: MYSQL_ROOT_PASSWORD
ports:
- containerPort: 3306
- {% if ACTIVATE_MYSQL %}
volumeMounts:
- mountPath: /var/lib/mysql
name: data
@@ -338,7 +333,7 @@ spec:
- name: data
persistentVolumeClaim:
claimName: mysql
- {% endif %}
+{% endif %}
{% if ACTIVATE_SMTP %}
---
apiVersion: apps/v1
diff --git a/tutor/templates/k8s/ingress.yml b/tutor/templates/k8s/ingress.yml
index 62c49fd..245605a 100644
--- a/tutor/templates/k8s/ingress.yml
+++ b/tutor/templates/k8s/ingress.yml
@@ -1,11 +1,12 @@
---{% set hosts = [LMS_HOST, "preview." + LMS_HOST, CMS_HOST] %}
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: web
labels:
app.kubernetes.io/name: web
annotations:
+ kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/proxy-body-size: 1000m
{% if ACTIVATE_HTTPS%}kubernetes.io/tls-acme: "true"
cert-manager.io/issuer: letsencrypt{% endif %}
@@ -22,9 +23,11 @@ spec:
{% if ACTIVATE_HTTPS %}
tls:
- hosts:
- {% for host in hosts %}
- - {{ host }}{% endfor %}
- {{ patch("k8s-ingress-tls-hosts")|indent(6) }}
+ {% for host in hosts %}
+ - {{ host }}{% endfor %}
+ {{ patch("k8s-ingress-tls-hosts")|indent(6) }}
+ # TODO maybe we should not take care of generating certificates ourselves
+ # and here just point to a tls secret
secretName: letsencrypt
{%endif%}
{% if ACTIVATE_HTTPS %}
diff --git a/tutor/templates/k8s/jobs.yml b/tutor/templates/k8s/jobs.yml
new file mode 100644
index 0000000..d5b5768
--- /dev/null
+++ b/tutor/templates/k8s/jobs.yml
@@ -0,0 +1,106 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: lms-job
+ labels:
+ app.kubernetes.io/component: job
+spec:
+ template:
+ spec:
+ restartPolicy: Never
+ containers:
+ - name: lms
+ image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+ volumeMounts:
+ - mountPath: /openedx/edx-platform/lms/envs/tutor/
+ name: settings-lms
+ - mountPath: /openedx/edx-platform/cms/envs/tutor/
+ name: settings-cms
+ - mountPath: /openedx/config
+ name: config
+ volumes:
+ - name: settings-lms
+ configMap:
+ name: openedx-settings-lms
+ - name: settings-cms
+ configMap:
+ name: openedx-settings-cms
+ - name: config
+ configMap:
+ name: openedx-config
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: cms-job
+ labels:
+ app.kubernetes.io/component: job
+spec:
+ template:
+ spec:
+ restartPolicy: Never
+ containers:
+ - name: cms
+ image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+ env:
+ - name: SERVICE_VARIANT
+ value: cms
+ volumeMounts:
+ - mountPath: /openedx/edx-platform/lms/envs/tutor/
+ name: settings-lms
+ - mountPath: /openedx/edx-platform/cms/envs/tutor/
+ name: settings-cms
+ - mountPath: /openedx/config
+ name: config
+ volumes:
+ - name: settings-lms
+ configMap:
+ name: openedx-settings-lms
+ - name: settings-cms
+ configMap:
+ name: openedx-settings-cms
+ - name: config
+ configMap:
+ name: openedx-config
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: mysql-job
+ labels:
+ app.kubernetes.io/component: job
+spec:
+ template:
+ spec:
+ restartPolicy: Never
+ containers:
+ - name: mysql
+ image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
+ command: []
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: forum-job
+ labels:
+ app.kubernetes.io/component: job
+spec:
+ template:
+ spec:
+ restartPolicy: Never
+ containers:
+ - name: forum
+ image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
+ env:
+ - name: SEARCH_SERVER
+ value: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}"
+ - name: MONGODB_AUTH
+ value: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}"
+ - name: MONGODB_HOST
+ value: "{{ MONGODB_HOST }}"
+ - name: MONGODB_PORT
+ value: "{{ MONGODB_PORT }}"
+
+{{ patch("k8s-jobs") }}
+
diff --git a/tutor/templates/kustomization.yml b/tutor/templates/kustomization.yml
index 04881b6..8a5048b 100644
--- a/tutor/templates/kustomization.yml
+++ b/tutor/templates/kustomization.yml
@@ -4,7 +4,9 @@ kind: Kustomization
resources:
- k8s/namespace.yml
- k8s/deployments.yml
+# TODO maybe we should not take care of ingress stuff and let the administrator do it
- k8s/ingress.yml
+- k8s/jobs.yml
- k8s/services.yml
- k8s/volumes.yml
{{ patch("kustomization-resources") }}
diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml
new file mode 100644
index 0000000..57a3084
--- /dev/null
+++ b/tutor/templates/local/docker-compose.jobs.yml
@@ -0,0 +1,37 @@
+version: "3.7"
+services:
+
+ mysql-job:
+ image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
+ entrypoint: []
+ command: ["echo", "done"]
+
+ lms-job:
+ image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+ environment:
+ SERVICE_VARIANT: lms
+ SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production}
+ volumes:
+ - ../apps/openedx/settings/lms/:/openedx/edx-platform/lms/envs/tutor/:ro
+ - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/:ro
+ - ../apps/openedx/config/:/openedx/config/:ro
+
+ cms-job:
+ image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+ environment:
+ SERVICE_VARIANT: cms
+ SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production}
+ volumes:
+ - ../apps/openedx/settings/lms/:/openedx/edx-platform/lms/envs/tutor/:ro
+ - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/:ro
+ - ../apps/openedx/config/:/openedx/config/:ro
+
+ forum-job:
+ image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
+ environment:
+ SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}"
+ MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}"
+ MONGODB_HOST: "{{ MONGODB_HOST }}"
+ MONGODB_PORT: "{{ MONGODB_PORT }}"
+
+ {{ patch("local-docker-compose-jobs-services")|indent(4) }}
\ No newline at end of file
diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml
index 977d2b9..1a802ea 100644
--- a/tutor/templates/local/docker-compose.yml
+++ b/tutor/templates/local/docker-compose.yml
@@ -19,18 +19,15 @@ services:
- ../../data/mongodb:/data/db
{% endif %}
+ {% if ACTIVATE_MYSQL %}
mysql:
image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
- {% if ACTIVATE_MYSQL %}
command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci
- {% else %}
- entrypoint: ["sh", "-e", "-c"]
- command: ["echo 'ready'; while true; do sleep 60; done"]
- {% endif %}
restart: unless-stopped
volumes:
- ../../data/mysql:/var/lib/mysql
env_file: ../apps/mysql/auth.env
+ {% endif %}
{% if ACTIVATE_ELASTICSEARCH %}
elasticsearch:
--
GitLab