255 lines
9.2 KiB
YAML
255 lines
9.2 KiB
YAML
{{- if .Values.vector.enabled }}
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: {{ include "supabase.vector.fullname" . }}-config
|
|
labels:
|
|
{{- include "supabase.labels" . | nindent 4 }}
|
|
data:
|
|
secret.sh: |
|
|
#!/bin/sh
|
|
cat << EOF
|
|
{
|
|
"logflare_api_key": {
|
|
"value": "$LOGFLARE_API_KEY",
|
|
"error": null
|
|
}
|
|
}
|
|
EOF
|
|
vector.yml: |
|
|
secret:
|
|
credentials:
|
|
type: exec
|
|
command:
|
|
- /etc/vector/secret.sh
|
|
|
|
api:
|
|
enabled: true
|
|
address: 0.0.0.0:{{ .Values.vector.service.port }}
|
|
|
|
sources:
|
|
kubernetes_host:
|
|
type: kubernetes_logs
|
|
extra_label_selector: app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/name!={{ include "supabase.vector.name" . }}
|
|
|
|
transforms:
|
|
project_logs:
|
|
type: remap
|
|
inputs:
|
|
- kubernetes_host
|
|
source: |-
|
|
.project = "default"
|
|
.event_message = del(.message)
|
|
.appname = del(.kubernetes.container_name)
|
|
del(.file)
|
|
del(.kubernetes)
|
|
del(.source_type)
|
|
del(.stream)
|
|
router:
|
|
type: route
|
|
inputs:
|
|
- project_logs
|
|
route:
|
|
kong: '.appname == {{ include "supabase.kong.name" . | quote }}'
|
|
auth: '.appname == {{ include "supabase.auth.name" . | quote }}'
|
|
rest: '.appname == {{ include "supabase.rest.name" . | quote }}'
|
|
realtime: '.appname == {{ include "supabase.realtime.name" . | quote }}'
|
|
storage: '.appname == {{ include "supabase.storage.name" . | quote }}'
|
|
functions: '.appname == {{ include "supabase.functions.name" . | quote }}'
|
|
db: '.appname == {{ include "supabase.db.name" . | quote }}'
|
|
# Ignores non nginx errors since they are related with kong booting up
|
|
kong_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.kong
|
|
source: |-
|
|
req, err = parse_nginx_log(.event_message, "combined")
|
|
if err == null {
|
|
.timestamp = req.timestamp
|
|
.metadata.request.headers.referer = req.referer
|
|
.metadata.request.headers.user_agent = req.agent
|
|
.metadata.request.headers.cf_connecting_ip = req.client
|
|
.metadata.request.method = req.method
|
|
.metadata.request.path = req.path
|
|
.metadata.request.protocol = req.protocol
|
|
.metadata.response.status_code = req.status
|
|
}
|
|
if err != null {
|
|
abort
|
|
}
|
|
# Ignores non nginx errors since they are related with kong booting up
|
|
kong_err:
|
|
type: remap
|
|
inputs:
|
|
- router.kong
|
|
source: |-
|
|
.metadata.request.method = "GET"
|
|
.metadata.response.status_code = 200
|
|
parsed, err = parse_nginx_log(.event_message, "error")
|
|
if err == null {
|
|
.timestamp = parsed.timestamp
|
|
.severity = parsed.severity
|
|
.metadata.request.host = parsed.host
|
|
.metadata.request.headers.cf_connecting_ip = parsed.client
|
|
url, err = split(parsed.request, " ")
|
|
if err == null {
|
|
.metadata.request.method = url[0]
|
|
.metadata.request.path = url[1]
|
|
.metadata.request.protocol = url[2]
|
|
}
|
|
}
|
|
if err != null {
|
|
abort
|
|
}
|
|
# Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency.
|
|
auth_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.auth
|
|
source: |-
|
|
parsed, err = parse_json(.event_message)
|
|
if err == null {
|
|
.metadata.timestamp = parsed.time
|
|
.metadata = merge!(.metadata, parsed)
|
|
}
|
|
# PostgREST logs are structured so we separate timestamp from message using regex
|
|
rest_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.rest
|
|
source: |-
|
|
parsed, err = parse_regex(.event_message, r'^(?P<time>.*): (?P<msg>.*)$')
|
|
if err == null {
|
|
.event_message = parsed.msg
|
|
.timestamp = parse_timestamp!(parsed.time, format: "%e/%b/%Y %R %:z")
|
|
.metadata.host = .project
|
|
}
|
|
# Realtime logs are structured so we parse the severity level using regex (ignore time because it has no date)
|
|
realtime_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.realtime
|
|
source: |-
|
|
.metadata.project = del(.project)
|
|
.metadata.external_id = .metadata.project
|
|
parsed, err = parse_regex(.event_message, r'^(?P<time>\d+:\d+:\d+\.\d+) \[(?P<level>\w+)\] (?P<msg>.*)$')
|
|
if err == null {
|
|
.event_message = parsed.msg
|
|
.metadata.level = parsed.level
|
|
}
|
|
# Storage logs may contain json objects so we parse them for completeness
|
|
storage_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.storage
|
|
source: |-
|
|
.metadata.project = del(.project)
|
|
.metadata.tenantId = .metadata.project
|
|
parsed, err = parse_json(.event_message)
|
|
if err == null {
|
|
.event_message = parsed.msg
|
|
.metadata.level = parsed.level
|
|
.metadata.timestamp = parsed.time
|
|
.metadata.context[0].host = parsed.hostname
|
|
.metadata.context[0].pid = parsed.pid
|
|
}
|
|
# Postgres logs some messages to stderr which we map to warning severity level
|
|
db_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.db
|
|
source: |-
|
|
.metadata.host = "db-default"
|
|
.metadata.parsed.timestamp = .timestamp
|
|
|
|
parsed, err = parse_regex(.event_message, r'.*(?P<level>INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC?):.*', numeric_groups: true)
|
|
|
|
if err != null || parsed == null {
|
|
.metadata.parsed.error_severity = "info"
|
|
}
|
|
if parsed != null {
|
|
.metadata.parsed.error_severity = parsed.level
|
|
}
|
|
if .metadata.parsed.error_severity == "info" {
|
|
.metadata.parsed.error_severity = "log"
|
|
}
|
|
.metadata.parsed.error_severity = upcase!(.metadata.parsed.error_severity)
|
|
|
|
{{- if .Values.analytics.enabled }}
|
|
sinks:
|
|
logflare_auth:
|
|
type: 'http'
|
|
inputs:
|
|
- auth_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=gotrue.logs.prod&api_key=SECRET[credentials.logflare_api_key]'
|
|
logflare_realtime:
|
|
type: 'http'
|
|
inputs:
|
|
- realtime_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=realtime.logs.prod&api_key=SECRET[credentials.logflare_api_key]'
|
|
logflare_rest:
|
|
type: 'http'
|
|
inputs:
|
|
- rest_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=postgREST.logs.prod&api_key=SECRET[credentials.logflare_api_key]'
|
|
logflare_db:
|
|
type: 'http'
|
|
inputs:
|
|
- db_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
# We must route the sink through kong because ingesting logs before logflare is fully initialised will
|
|
# lead to broken queries from studio. This works by the assumption that containers are started in the
|
|
# following order: vector > db > logflare > kong
|
|
uri: 'http://{{ include "supabase.kong.fullname" . }}:{{ .Values.kong.service.port }}/analytics/v1/api/logs?source_name=postgres.logs&api_key=SECRET[credentials.logflare_api_key]'
|
|
logflare_functions:
|
|
type: 'http'
|
|
inputs:
|
|
- router.functions
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=deno-relay-logs&api_key=SECRET[credentials.logflare_api_key]'
|
|
logflare_storage:
|
|
type: 'http'
|
|
inputs:
|
|
- storage_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=storage.logs.prod.2&api_key=SECRET[credentials.logflare_api_key]'
|
|
logflare_kong:
|
|
type: 'http'
|
|
inputs:
|
|
- kong_logs
|
|
- kong_err
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.analytics.service.port }}/api/logs?source_name=cloudflare.logs.prod&api_key=SECRET[credentials.logflare_api_key]'
|
|
{{- end }}
|
|
{{- end }} |