From d7f64a7763fe33891f3b90b316e303b90d7ddefc Mon Sep 17 00:00:00 2001
From: drpsyko101 <drpsyko101@gmail.com>
Date: Sat, 6 Apr 2024 22:22:24 +0800
Subject: [PATCH] Fix vector parsing event log
* add helm install notes
* add environment variable support to vector
* update image tags
* update README on vector volume
* fix vector volume error
---
charts/supabase/README.md | 1 +
charts/supabase/templates/NOTES.txt | 5 ++
charts/supabase/templates/vector/config.yaml | 58 +++++++++++--------
.../supabase/templates/vector/deployment.yaml | 13 ++++-
charts/supabase/values.example.yaml | 17 ++++--
5 files changed, 64 insertions(+), 30 deletions(-)
create mode 100644 charts/supabase/templates/NOTES.txt
diff --git a/charts/supabase/README.md b/charts/supabase/README.md
index 8bd462c..28c5966 100644
--- a/charts/supabase/README.md
+++ b/charts/supabase/README.md
@@ -271,3 +271,4 @@ docker run -it \
* Ingress are now limited to `kong` & `db` services. This is by design to limit entry to the stack through secure `kong` service.
* `kong.yaml` has been modified to follow [Docker kong.yaml](https://github.com/supabase/supabase/blob/master/docker/volumes/api/kong.yml) template.
* `supabase/storage` does not comes with pre-populated `/var/lib/storage`, therefore an `emptyDir` will be created if persistence is disabled. This might be incompatible with previous version if the persistent storage location is set to location other than specified above.
+* `supabase/vector` requires read access to the `/var/log/pods` directory. When run in a Kubernetes cluster this can be provided with a [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) volume.
diff --git a/charts/supabase/templates/NOTES.txt b/charts/supabase/templates/NOTES.txt
new file mode 100644
index 0000000..f06631a
--- /dev/null
+++ b/charts/supabase/templates/NOTES.txt
@@ -0,0 +1,5 @@
+---
+Thank you for installing {{ .Chart.Name }}!
+{{ if .Values.kong.ingress.enabled }}
+Visit the Studio dashboard at http://{{ (index .Values.kong.ingress.hosts 0).host }}
+{{- end }}
diff --git a/charts/supabase/templates/vector/config.yaml b/charts/supabase/templates/vector/config.yaml
index 1a483ff..df2e897 100644
--- a/charts/supabase/templates/vector/config.yaml
+++ b/charts/supabase/templates/vector/config.yaml
@@ -6,7 +6,23 @@ metadata:
labels:
{{- include "supabase.labels" . | nindent 4 }}
data:
+ secret.sh: |
+ #!/bin/sh
+ cat << EOF
+ {
+ "logflare_api_key": {
+ "value": "$LOGFLARE_API_KEY",
+ "error": null
+ }
+ }
+ EOF
vector.yml: |
+ secret:
+ credentials:
+ type: exec
+ command:
+ - /etc/vector/secret.sh
+
api:
enabled: true
address: 0.0.0.0:{{ .Values.vector.service.port }}
@@ -14,7 +30,7 @@ data:
sources:
kubernetes_host:
type: kubernetes_logs
- extra_label_selector: app.kubernetes.io/instance={{ .Release.Name }}
+ extra_label_selector: app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/name!={{ include "supabase.vector.name" . }}
transforms:
project_logs:
@@ -24,27 +40,23 @@ data:
source: |-
.project = "default"
.event_message = del(.message)
- .appname = del(.container_name)
- del(.container_created_at)
- del(.container_id)
+ .appname = del(.kubernetes.container_name)
+ del(.file)
+ del(.kubernetes)
del(.source_type)
del(.stream)
- del(.label)
- del(.image)
- del(.host)
- del(.stream)
router:
type: route
inputs:
- project_logs
route:
- kong: '.appname == {{ include "supabase.kong.fullname" . | quote }}'
- auth: '.appname == {{ include "supabase.auth.fullname" . | quote }}'
- rest: '.appname == {{ include "supabase.rest.fullname" . | quote }}'
- realtime: '.appname == {{ include "supabase.realtime.fullname" . | quote }}'
- storage: '.appname == {{ include "supabase.storage.fullname" . | quote }}'
- functions: '.appname == {{ include "supabase.functions.fullname" . | quote }}'
- db: '.appname == {{ include "supabase.db.fullname" . | quote }}'
+ kong: '.appname == {{ include "supabase.kong.name" . | quote }}'
+ auth: '.appname == {{ include "supabase.auth.name" . | quote }}'
+ rest: '.appname == {{ include "supabase.rest.name" . | quote }}'
+ realtime: '.appname == {{ include "supabase.realtime.name" . | quote }}'
+ storage: '.appname == {{ include "supabase.storage.name" . | quote }}'
+ functions: '.appname == {{ include "supabase.functions.name" . | quote }}'
+ db: '.appname == {{ include "supabase.db.name" . | quote }}'
# Ignores non nginx errors since they are related with kong booting up
kong_logs:
type: remap
@@ -109,7 +121,7 @@ data:
parsed, err = parse_regex(.event_message, r'^(?P<time>.*): (?P<msg>.*)$')
if err == null {
.event_message = parsed.msg
- .timestamp = to_timestamp!(parsed.time)
+ .timestamp = parse_timestamp!(parsed.time, format: "%e/%b/%Y %R %:z")
.metadata.host = .project
}
# Realtime logs are structured so we parse the severity level using regex (ignore time because it has no date)
@@ -174,7 +186,7 @@ data:
method: 'post'
request:
retry_max_duration_secs: 10
- uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=gotrue.logs.prod&api_key=$(LOGFLARE_API_KEY)'
+ uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=gotrue.logs.prod&api_key=SECRET[credentials.logflare_api_key]'
logflare_realtime:
type: 'http'
inputs:
@@ -184,7 +196,7 @@ data:
method: 'post'
request:
retry_max_duration_secs: 10
- uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=realtime.logs.prod&api_key=$(LOGFLARE_API_KEY)'
+ uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=realtime.logs.prod&api_key=SECRET[credentials.logflare_api_key]'
logflare_rest:
type: 'http'
inputs:
@@ -194,7 +206,7 @@ data:
method: 'post'
request:
retry_max_duration_secs: 10
- uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=postgREST.logs.prod&api_key=$(LOGFLARE_API_KEY)'
+ uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=postgREST.logs.prod&api_key=SECRET[credentials.logflare_api_key]'
logflare_db:
type: 'http'
inputs:
@@ -207,7 +219,7 @@ data:
# We must route the sink through kong because ingesting logs before logflare is fully initialised will
# lead to broken queries from studio. This works by the assumption that containers are started in the
# following order: vector > db > logflare > kong
- uri: 'http://{{ include "supabase.kong.fullname" . }}:{{ .Values.kong.service.port }}/analytics/v1/api/logs?source_name=postgres.logs&api_key=$(LOGFLARE_API_KEY)'
+ uri: 'http://{{ include "supabase.kong.fullname" . }}:{{ .Values.kong.service.port }}/analytics/v1/api/logs?source_name=postgres.logs&api_key=SECRET[credentials.logflare_api_key]'
logflare_functions:
type: 'http'
inputs:
@@ -217,7 +229,7 @@ data:
method: 'post'
request:
retry_max_duration_secs: 10
- uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=deno-relay-logs&api_key=$(LOGFLARE_API_KEY)'
+ uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=deno-relay-logs&api_key=SECRET[credentials.logflare_api_key]'
logflare_storage:
type: 'http'
inputs:
@@ -227,7 +239,7 @@ data:
method: 'post'
request:
retry_max_duration_secs: 10
- uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=storage.logs.prod.2&api_key=$(LOGFLARE_API_KEY)'
+ uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=storage.logs.prod.2&api_key=SECRET[credentials.logflare_api_key]'
logflare_kong:
type: 'http'
inputs:
@@ -238,6 +250,6 @@ data:
method: 'post'
request:
retry_max_duration_secs: 10
- uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=cloudflare.logs.prod&api_key=$(LOGFLARE_API_KEY)'
+ uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=cloudflare.logs.prod&api_key=SECRET[credentials.logflare_api_key]'
{{- end }}
{{- end }}
\ No newline at end of file
diff --git a/charts/supabase/templates/vector/deployment.yaml b/charts/supabase/templates/vector/deployment.yaml
index d68484b..5dae5fd 100644
--- a/charts/supabase/templates/vector/deployment.yaml
+++ b/charts/supabase/templates/vector/deployment.yaml
@@ -15,10 +15,11 @@ spec:
{{- include "supabase.vector.selectorLabels" . | nindent 6 }}
template:
metadata:
- {{- with .Values.vector.podAnnotations }}
annotations:
+ {{- with .Values.vector.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
+ checksum/config: {{ include (print $.Template.BasePath "/vector/config.yaml") . | sha256sum }}
labels:
{{- include "supabase.vector.selectorLabels" . | nindent 8 }}
spec:
@@ -39,6 +40,10 @@ spec:
image: "{{ .Values.vector.image.repository }}:{{ .Values.vector.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.vector.image.pullPolicy }}
env:
+ {{- range $key, $value := .Values.vector.environment }}
+ - name: {{ $key }}
+ value: {{ $value | quote }}
+ {{- end }}
- name: VECTOR_SELF_NODE_NAME
valueFrom:
fieldRef:
@@ -68,17 +73,21 @@ spec:
- mountPath: /etc/vector/vector.yml
name: vector-config
subPath: vector.yml
+ - mountPath: /etc/vector/secret.sh
+ name: vector-config
+ subPath: secret.sh
{{- with .Values.vector.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
volumes:
{{- with .Values.vector.volumes }}
- {{- toYaml . | nindent 10 }}
+ {{- toYaml . | nindent 8 }}
{{- end }}
- name: vector-config
configMap:
name: {{ include "supabase.vector.fullname" . }}-config
+ defaultMode: 0777
{{- with .Values.vector.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
diff --git a/charts/supabase/values.example.yaml b/charts/supabase/values.example.yaml
index e831c18..835c8b1 100644
--- a/charts/supabase/values.example.yaml
+++ b/charts/supabase/values.example.yaml
@@ -32,7 +32,7 @@ db:
studio:
image:
- tag: 20240205-b145c86
+ tag: 20240326-5e5586d
environment:
STUDIO_DEFAULT_ORGANIZATION: "My Organization"
STUDIO_DEFAULT_PROJECT: "My Project"
@@ -46,7 +46,7 @@ studio:
auth:
image:
- tag: v2.132.3
+ tag: v2.143.0
environment:
API_EXTERNAL_URL: http://example.com
GOTRUE_SITE_URL: http://example.com
@@ -63,7 +63,7 @@ rest:
realtime:
image:
- tag: v2.25.50
+ tag: v2.27.5
livenessProbe:
httpGet:
path: /
@@ -132,13 +132,20 @@ analytics:
vector:
image:
- tag: 0.28.1-alpine
+ tag: 0.34.0-alpine
livenessProbe:
httpGet:
path: /health
port: 9001
initialDelaySeconds: 3
+ volumeMounts:
+ - name: pod-logs
+ mountPath: /var/log/pods
+ volumes:
+ - name: pod-logs
+ hostPath:
+ path: /var/log/pods
functions:
image:
- tag: v1.36.1
+ tag: v1.41.2
--
GitLab