diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9cb96742e60c0e99f3f841f5bf2119278a1a78b5..b30c832a4bc89ac8eb6a2224b0d6d3bc6cfb27ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ cache: stages: - get-data - format-data + - deploy-meilisearch - get-meili-key - build - deploy @@ -22,6 +23,47 @@ stages: - i=0; while [ "$i" -lt 12 ]; do docker info && break; sleep 5; i=$(( i + 1 )) ; done - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + + +################ DEPLOY MEILISEARCH ################# + +.deploy:meilisearch: + stage: deploy-meilisearch + image: harbor.pasteur.fr/kube-system/helm-kubectl:$HELM_VERSION + variables: + CI_DEBUG_TRACE: "false" + ENV: development + script: + - > + helm upgrade --install + ${CI_PROJECT_NAME}-${CI_ENVIRONMENT_NAME}-meiliseach + ./deploy/meilisearch + --namespace=${KUBE_NAMESPACE} + --set env="${ENV:-development}" + --values deploy/meilisearch/values.yaml + --values deploy/meilisearch/values.${ENV:-development}.yaml + # wait for it to start + - MEILI_POD=$(kubectl -n=${NAMESPACE} get po -l app.kubernetes.io\/instance=${CI_COMMIT_REF_SLUG},app=backend --output jsonpath='{.items[0].metadata.name}') + - | + until kubectl -n=${NAMESPACE} wait --for=condition=ready pod ${MEILI_POD} --timeout=1s + do + date + sleep 1 + kubectl -n=${NAMESPACE} get po + done + + +deploy:meilisearch:dev: + extends: .deploy:meilisearch + rules: + - if: $CI_COMMIT_BRANCH != "main" + variables: + CI_DEBUG_TRACE: "true" + ENV: "development" + PUBLIC_URL: "defense-finder-meilisearch.dev.pasteur.cloud" + + + ############# get-meili-key ############### .get-meili-api-key: @@ -41,6 +83,8 @@ stages: set-meili-env:dev: extends: .get-meili-api-key + needs: + - deploy:meilisearch:dev variables: MEILI_HOST: 'https://defense-finder-meilisearch.dev.pasteur.cloud' rules: @@ -102,6 +146,7 @@ format-pfam: - if: $CI_COMMIT_BRANCH == "main" + ################ BUILD ########################## .build: @@ -160,6 +205,8 @@ build:prod:wiki: variables: CI_DEBUG_TRACE: "false" TEAM_ID: "df" +# --set meilisearch.environment.MEILI_MASTER_KEY="${MEILI_MASTER_KEY}" +# --set meilisearch.environment.MEILI_ENV="${ENV}" script: - helm dependency update deploy/df-wiki/ - > @@ -172,8 +219,6 @@ build:prod:wiki: --set imagePullSecrets[0].name="registry-pull-secret-${CI_COMMIT_REF_SLUG}" --set image.repository="$CI_REGISTRY_IMAGE/$IMAGE_NAME" --set image.tag="$CI_COMMIT_SHORT_SHA" - --set meilisearch.environment.MEILI_MASTER_KEY="${MEILI_MASTER_KEY}" - --set meilisearch.environment.MEILI_ENV="${ENV}" --set env="${ENV:-development}" --values deploy/df-wiki/values.yaml --values deploy/df-wiki/values.${ENV:-development}.yaml diff --git a/deploy/df-wiki/Chart.lock b/deploy/df-wiki/Chart.lock deleted file mode 100644 index 8129ab1681f18bc6b5a63d6eee66ce356cf18eff..0000000000000000000000000000000000000000 --- a/deploy/df-wiki/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: meilisearch - repository: https://meilisearch.github.io/meilisearch-kubernetes - version: 0.2.7 -digest: sha256:f3fd738cebef7d3ff607381c966a2db3a01cfa01e9924bddbce9bc416955bc4b -generated: "2023-10-12T18:32:33.226266244+02:00" diff --git a/deploy/df-wiki/Chart.yaml b/deploy/df-wiki/Chart.yaml index 2eac809e4c1496761da37ae2b103a99f80b2454e..427f8bb578001a22f66aaf30439ed3e2336605eb 100644 --- a/deploy/df-wiki/Chart.yaml +++ b/deploy/df-wiki/Chart.yaml @@ -23,8 +23,3 @@ version: 0.1.0 # It is recommended to use it with quotes. appVersion: "1.16.0" - -dependencies: - - name: meilisearch - version: 0.2.7 - repository: "https://meilisearch.github.io/meilisearch-kubernetes" diff --git a/deploy/df-wiki/values.development.yaml b/deploy/df-wiki/values.development.yaml index 16e39d5ed35426ecbda4ee5b3fae09b8be95cd86..03f289ecae6a7950dd90d74f7ae3cc33fa3366ae 100644 --- a/deploy/df-wiki/values.development.yaml +++ b/deploy/df-wiki/values.development.yaml @@ -25,25 +25,3 @@ autoscaling: enabled: false -meilisearch: - environment: - MEILI_MASTER_KEY: "MASTER_KEY" - ingress: - enabled: true - className: internal - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: 100m - hosts: - - "defense-finder-meilisearch.dev.pasteur.cloud" - path: / - persistence: - enabled: true - storageClass: isilon - size: "1Gi" - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 800m - memory: 800Mi \ No newline at end of file diff --git a/deploy/df-wiki/values.production.yaml b/deploy/df-wiki/values.production.yaml index 6db74bea8c04a9ee255639739a276bf49da6ecdd..74a786562a3f2d85a6087e5a10881bda8b976bf8 100644 --- a/deploy/df-wiki/values.production.yaml +++ b/deploy/df-wiki/values.production.yaml @@ -28,25 +28,3 @@ autoscaling: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 -meilisearch: - environment: - MEILI_MASTER_KEY: "MASTER_KEY" - ingress: - enabled: true - className: internal - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: 100m - hosts: - - "defense-finder-meilisearch.pasteur.cloud" - path: / - persistence: - enabled: true - storageClass: isilon - size: "10Gi" - resources: - limits: - cpu: 1500m - memory: "1.5Gi" - requests: - cpu: 1000m - memory: 1Gi \ No newline at end of file diff --git a/deploy/df-wiki/values.yaml b/deploy/df-wiki/values.yaml index fd0eaf46d140d53b987f4a9d17cbc83e05172051..a2e8e55a3bba9a560cde12476e2e0df22a456c52 100644 --- a/deploy/df-wiki/values.yaml +++ b/deploy/df-wiki/values.yaml @@ -88,25 +88,25 @@ tolerations: [] affinity: {} -meilisearch: - environment: - MEILI_MASTER_KEY: "MASTER_KEY" - ingress: - enabled: true - className: internal - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: 100m - hosts: - - "defense-finder-meilisearch.dev.pasteur.cloud" - path: / - persistence: - enabled: true - storageClass: isilon - size: "1Gi" - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 800m - memory: 800Mi +# meilisearch: +# environment: +# MEILI_MASTER_KEY: "MASTER_KEY" +# ingress: +# enabled: true +# className: internal +# annotations: +# nginx.ingress.kubernetes.io/proxy-body-size: 100m +# hosts: +# - "defense-finder-meilisearch.dev.pasteur.cloud" +# path: / +# persistence: +# enabled: true +# storageClass: isilon +# size: "1Gi" +# resources: +# limits: +# cpu: 1000m +# memory: 1Gi +# requests: +# cpu: 800m +# memory: 800Mi diff --git a/deploy/meilisearch/.helmignore b/deploy/meilisearch/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..0e8a0eb36f4ca2c939201c0d54b5d82a1ea34778 --- /dev/null +++ b/deploy/meilisearch/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/meilisearch/Chart.yaml b/deploy/meilisearch/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9d0dea98545c77d5a09c91555236318281e8aa8 --- /dev/null +++ b/deploy/meilisearch/Chart.yaml @@ -0,0 +1,30 @@ +apiVersion: v2 +name: meilisearch +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" + +dependencies: + - name: meilisearch + version: 0.2.8 + repository: "https://meilisearch.github.io/meilisearch-kubernetes" + diff --git a/deploy/meilisearch/templates/NOTES.txt b/deploy/meilisearch/templates/NOTES.txt new file mode 100644 index 0000000000000000000000000000000000000000..801b83fe428dcd302f87d7cda8d29b2b07a177ff --- /dev/null +++ b/deploy/meilisearch/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "meilisearch.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "meilisearch.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "meilisearch.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "meilisearch.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/deploy/meilisearch/templates/_helpers.tpl b/deploy/meilisearch/templates/_helpers.tpl new file mode 100644 index 0000000000000000000000000000000000000000..740d06ae45189a1e73b6beaa659ac38e06b02f2c --- /dev/null +++ b/deploy/meilisearch/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "meilisearch.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "meilisearch.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "meilisearch.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "meilisearch.labels" -}} +helm.sh/chart: {{ include "meilisearch.chart" . }} +{{ include "meilisearch.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "meilisearch.selectorLabels" -}} +app.kubernetes.io/name: {{ include "meilisearch.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "meilisearch.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "meilisearch.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/meilisearch/templates/tests/test-connection.yaml b/deploy/meilisearch/templates/tests/test-connection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4ce1b6918c362520ca9240b179c40b3d5a913fb --- /dev/null +++ b/deploy/meilisearch/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "meilisearch.fullname" . }}-test-connection" + labels: + {{- include "meilisearch.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "meilisearch.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/deploy/meilisearch/values.development.yaml b/deploy/meilisearch/values.development.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4272bd234ca958190dac05da7f8409b44d0bf780 --- /dev/null +++ b/deploy/meilisearch/values.development.yaml @@ -0,0 +1,22 @@ +meilisearch: + environment: + MEILI_MASTER_KEY: "MASTER_KEY" + ingress: + enabled: true + className: internal + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: 100m + hosts: + - "defense-finder-meilisearch.dev.pasteur.cloud" + path: / + persistence: + enabled: true + storageClass: isilon + size: "1Gi" + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 800m + memory: 800Mi \ No newline at end of file diff --git a/deploy/meilisearch/values.production.yaml b/deploy/meilisearch/values.production.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3884eba6225bebc7f579248fa4c69dbb39b740a8 --- /dev/null +++ b/deploy/meilisearch/values.production.yaml @@ -0,0 +1,22 @@ +meilisearch: + environment: + MEILI_MASTER_KEY: "MASTER_KEY" + ingress: + enabled: true + className: internal + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: 100m + hosts: + - "defense-finder-meilisearch.pasteur.cloud" + path: / + persistence: + enabled: true + storageClass: isilon + size: "10Gi" + resources: + limits: + cpu: 1500m + memory: "1.5Gi" + requests: + cpu: 1000m + memory: 1Gi \ No newline at end of file diff --git a/deploy/meilisearch/values.yaml b/deploy/meilisearch/values.yaml index e4e735d5f839d80971b1c20d4d4595b08c4570b9..5b25a80b5bde96a6555b8094d7ff861123f8dd56 100644 --- a/deploy/meilisearch/values.yaml +++ b/deploy/meilisearch/values.yaml @@ -1,194 +1,98 @@ -# Default values for <CHARTNAME>. +# Default values for meilisearch. # This is a YAML-formatted file. # Declare variables to be passed into your templates. -startupProbe: - periodSeconds: 1 - InitialDelaySeconds: 1 - failureThreshold: 60 - -readinessProbe: - periodSeconds: 10 - InitialDelaySeconds: 0 - -livenessProbe: - periodSeconds: 10 - InitialDelaySeconds: 0 +replicaCount: 1 image: - # -- Meilisearch image name - repository: getmeili/meilisearch - # -- Meilisearch image tag - tag: v1.4.0 - # -- Meilisearch image pull policy + repository: nginx pullPolicy: IfNotPresent - # -- Secret to authenticate against the docker registry - pullSecret: + # Overrides the image tag whose default is the chart appVersion. + tag: "" -# -- String to partially override meilisearch.fullname +imagePullSecrets: [] nameOverride: "" -# -- String to fully override meilisearch.fullname fullnameOverride: "" -# Environment loaded into the configMap -environment: - # -- Deactivates analytics - MEILI_NO_ANALYTICS: true - # -- Sets the environment. Either **production** or **development** - MEILI_ENV: development - # For production deployment, the environment MEILI_MASTER_KEY is required. - # If MEILI_ENV is set to "production" without setting MEILI_MASTER_KEY, this - # chart will automatically create a secure MEILI_MASTER_KEY and push it as a - # secret. Otherwise the below value of MEILI_MASTER_KEY will be used instead. - # MEILI_MASTER_KEY: "" -auth: - # -- Use an existing Kubernetes secret for the MEILI_MASTER_KEY - existingMasterKeySecret: "" - -# -- Additional environment variables from ConfigMap or secrets -envFrom: [] - serviceAccount: - # You can use a specific pre-created service account instead of - # the one created by this chart. - # Set create to false to prevent creating a service account and use your own. - # -- Should this chart create a service account + # Specifies whether a service account should be created create: true - # -- Custom service account name, if not created by this chart - name: '' - # -- Additional annotations for created service account + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" podAnnotations: {} - -# -- Additional labels to add to the pod(s) only podLabels: {} -# -- Additional labels to add to all resources -customLabels: {} +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 -# -- Service HTTP port service: - # -- Kubernetes Service type type: ClusterIP - port: 7700 - # -- Additional annotations for service - annotations: {} - -container: - containerPort: 7700 - -podSecurityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - -securityContext: - capabilities: - drop: - - ALL - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true + port: 80 ingress: - # -- Enable ingress controller resource enabled: false - # -- Ingress ingressClassName - className: nginx - # -- Ingress annotations + className: "" annotations: {} # kubernetes.io/ingress.class: nginx - # -- Path within the host - path: / - # -- List of hostnames + # kubernetes.io/tls-acme: "true" hosts: - - meilisearch-example.local - # -- TLS specification + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local -persistence: - # -- Enable persistence using PVC - enabled: false - # -- PVC Access Mode - accessMode: ReadWriteOnce - ## Persistent Volume Storage Class - ## If defined, storageClassName: <storageClass> - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack) - ## - # -- PVC Storage Class - storageClass: "-" - - ## Data Persistent Volume existing claim name - ## Requires persistence.enabled: true - ## If defined, PVC must be created manually before volume will be bound - # -- Existing PVC - existingClaim: "" - # -- PVC Storage Request - size: 10Gi - # -- Additional annotations for PVC - annotations: {} - volume: - name: data - mountPath: /meili_data - -# -- Resources allocation (Requests and Limits) resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. # limits: - # cpu: 100m - # memory: 128Mi + # cpu: 100m + # memory: 128Mi # requests: - # cpu: 100m - # memory: 128Mi + # cpu: 100m + # memory: 128Mi -# -- Additional volumes for pod +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false -# -- Additional volumes to mount on pod +# Additional volumeMounts on the output Deployment definition. volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true -# -- Additional containers for pod -containers: [] - -# -- Node labels for pod assignment nodeSelector: {} -# -- Tolerations for pod assignment tolerations: [] -# -- Affinity for pod assignment affinity: {} - -# -- Pod command -command: [] - -# -- Monitoring with Prometheus Operator -serviceMonitor: - # -- Enable ServiceMonitor to configure scraping - enabled: false - # -- Set of labels to transfer from the Kubernetes Service onto the target - additionalLabels: {} - # -- Set scraping frequency - interval: 1m - # -- Set scraping timeout - scrapeTimeout: 10s - # -- Set path to metrics - telemetryPath: /metrics - # -- Set of labels to transfer from the Kubernetes Service onto the target - targetLabels: [] - # -- MetricRelabelConfigs to apply to samples before ingestion - metricRelabelings: [] - # -- Set relabel_configs as per https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config - relabelings: [] -