Optional TSF persistent storage and memory/CPU TSF triggers

charts/cortex-agent/.helmignore

@@ -0,0 +1 @@
+tests/

charts/cortex-agent/Chart.yaml

@@ -5,7 +5,7 @@ description: Cortex XDR agent helm chart
 
 type: application
 
-version: 1.11.0
+version: 1.12.0
 
 appVersion: 1.0.0
 

charts/cortex-agent/README.md

@@ -19,6 +19,7 @@
 | 1.10.0        | >=7.5         | `daemonset.resources` split into `full`/`sensor` profiles, automatically selected by `agent.agentFlavor`
 | 1.11.0        | >=7.5         | Support for Flatcar Container Linux platform
 | 1.11.0        | >=7.5         | `agent.agentFlavor` added (1 = `standard`, 3 = `cloud/sensor`).
+| 1.12.0        | >=7.5         | Optional TSF persistent storage (`tsfPersistentStorage.*`) with memory/CPU TSF triggers
 
 ## Installing Cortex XDR helm chart
 
@@ -111,6 +112,12 @@ Even when using `--reuse-values` (which uses the values of the previous installa
 | `daemonset.resources.full`             | Resource requests/limits used when `agent.agentFlavor` is `1` (default)                                 | Since 1.10.0
 | `daemonset.resources.sensor`           | Resource requests/limits used when `agent.agentFlavor` is `3`                                            | Since 1.10.0
 | `agent.agentFlavor`                    | Agent flavor: `1` = standard (default), `3` = cloud/sensor. Flavor `3` also selects `daemonset.resources.sensor` | Since 1.11.0, agent >= 9.4
+| `tsfPersistentStorage.enabled`         | Copy TSFs to a customer PVC                                                                                | Since 1.12.0
+| `tsfPersistentStorage.pvcName`         | Existing ReadWriteMany PVC in `namespace.name` (required when enabled). Mounted only if Bound and RWX at install/upgrade time, otherwise skipped with a WARNING; always skipped by `helm template` | Since 1.12.0
+| `tsfPersistentStorage.path`            | Relative folder in the volume, e.g. `cortex/prod-cluster-1`; each node writes to `<path>/<node-name>`      | Since 1.12.0
+| `tsfPersistentStorage.autoTsfIntervalHours` | Minimum hours between automatic TSFs of any type, 1 to 168; empty means 24 | Since 1.12.0
+| `tsfPersistentStorage.memoryTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent memory stays at or above `percent` (default 90) of its limit for `seconds` (default 300, 1 to 86400). Requires `tsfPersistentStorage.enabled` | Since 1.12.0
+| `tsfPersistentStorage.cpuTsfTrigger.enabled` / `.percent` / `.seconds`    | Collect a TSF when agent CPU stays at or above `percent` (default 65) of its CPU limit for `seconds` (default 300, 1 to 86400). Requires `tsfPersistentStorage.enabled` | Since 1.12.0
 
 Note: Helm requires commas in arguments to be escaped.
 

charts/cortex-agent/templates/NOTES.txt

@@ -4,6 +4,10 @@ Deployed to namespace: {{ .Values.namespace.name }}
 Release name: {{ .Release.Name }}
 
 To view all agent pods run: kubectl get pods -n {{ .Values.namespace.name }}
+{{- if eq (include "cortex-xdr.tsfStorageState" .) "skipped" }}
+
+{{ include "cortex-xdr.tsfStorageSkippedWarning" . }}
+{{- end }}
 
 In case images are not pulled, make sure to set the right repository and the right
 pull secret value, visit "https://github.com/paloaltonetworks/cortex-helm/helm-charts/cortex-agent" for

charts/cortex-agent/templates/_helpers.tpl

@@ -84,6 +84,82 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{/*
+Validate tsfPersistentStorage. Fails install/upgrade on a missing or unsafe configuration.
+Numbers are checked as strings: an integer 0 counts as "empty" in Helm, so `if` would skip it.
+*/}}
+{{- define "cortex-xdr.validateTsfPersistentStorage" -}}
+{{- with .Values.tsfPersistentStorage }}
+{{- if .enabled }}
+{{- if not .pvcName }}{{ fail "tsfPersistentStorage.enabled=true requires tsfPersistentStorage.pvcName" }}{{ end }}
+{{- if not .path }}{{ fail "tsfPersistentStorage.enabled=true requires tsfPersistentStorage.path" }}{{ end }}
+{{- if or (hasPrefix "/" (toString .path)) (has ".." (splitList "/" (toString .path))) }}{{ fail "tsfPersistentStorage.path must be relative and must not contain '..'" }}{{ end }}
+{{- if not (regexMatch "^[A-Za-z0-9._/-]+$" (toString .path)) }}{{ fail "tsfPersistentStorage.path may contain only letters, digits, '.', '_', '-' and '/'" }}{{ end }}
+{{- end }}
+{{- if include "cortex-xdr.tsfAutoIntervalHours" $ }}
+{{- if not (regexMatch "^([1-9][0-9]?|1[0-5][0-9]|16[0-8])$" (include "cortex-xdr.tsfAutoIntervalHours" $)) }}{{ fail "tsfPersistentStorage.autoTsfIntervalHours must be an integer from 1 to 168" }}{{ end }}
+{{- end }}
+{{- range $name := list "memoryTsfTrigger" "cpuTsfTrigger" }}
+{{- $t := get $.Values.tsfPersistentStorage $name | default dict }}
+{{- if $t.enabled }}
+{{- if not $.Values.tsfPersistentStorage.enabled }}{{ fail (printf "tsfPersistentStorage.%s.enabled=true requires tsfPersistentStorage.enabled=true" $name) }}{{ end }}
+{{- if not (regexMatch "^([1-9][0-9]?|100)$" (toString $t.percent)) }}{{ fail (printf "tsfPersistentStorage.%s.percent must be an integer from 1 to 100" $name) }}{{ end }}
+{{- $sec := $t.seconds }}
+{{- if or (kindIs "bool" $sec) (ne (float64 $sec) (float64 (int64 $sec))) (lt (int64 $sec) 1) (gt (int64 $sec) 86400) }}{{ fail (printf "tsfPersistentStorage.%s.seconds must be an integer from 1 to 86400" $name) }}{{ end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+TSF persistent storage state.
+Returns "ready" when enabled and the PVC is Bound and ReadWriteMany at install/upgrade time,
+"skipped" when enabled but the PVC is not ready - missing, unbound, not RWX, or no cluster,
+and "" when disabled. Only "ready" mounts the PVC, so a bad PVC never blocks the agent pod
+at install/upgrade time.
+*/}}
+{{- define "cortex-xdr.tsfStorageState" -}}
+{{- $tsf := .Values.tsfPersistentStorage | default dict -}}
+{{- if $tsf.enabled -}}
+  {{- $pvc := lookup "v1" "PersistentVolumeClaim" .Values.namespace.name ($tsf.pvcName | toString) -}}
+  {{- $status := dig "status" dict $pvc -}}
+  {{- if and (eq (dig "phase" "" $status) "Bound") (has "ReadWriteMany" (dig "accessModes" list $status)) -}}
+ready
+  {{- else -}}
+skipped
+  {{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Warning shown when TSF persistent storage is enabled but skipped. %q escapes the user-supplied PVC name.
+*/}}
+{{- define "cortex-xdr.tsfStorageSkippedWarning" -}}
+{{- printf "WARNING: TSF persistent storage skipped: PVC %q in namespace %q was not found, is not Bound or is not ReadWriteMany (or no cluster, as with helm template). The agent runs without it; fix the PVC and run helm upgrade again." (.Values.tsfPersistentStorage.pvcName | toString) .Values.namespace.name -}}
+{{- end -}}
+
+{{/*
+tsfPersistentStorage.autoTsfIntervalHours as a string; empty when unset (the agent then uses 24 hours).
+*/}}
+{{- define "cortex-xdr.tsfAutoIntervalHours" -}}
+{{- $h := (.Values.tsfPersistentStorage | default dict).autoTsfIntervalHours -}}
+{{- if not (kindIs "invalid" $h) }}{{ toString $h }}{{ end -}}
+{{- end -}}
+
+{{/*
+Env vars for one TSF resource trigger, rendered only when the trigger is enabled.
+Args: list <root context> <values key> <env prefix>
+*/}}
+{{- define "cortex-xdr.tsfTriggerEnv" -}}
+{{- $t := get (index . 0).Values.tsfPersistentStorage (index . 1) | default dict }}
+{{- if $t.enabled }}
+- name: {{ index . 2 }}_PERCENT
+  value: {{ $t.percent | int | quote }}
+- name: {{ index . 2 }}_SECONDS
+  value: {{ $t.seconds | int | quote }}
+{{- end }}
+{{- end }}
+
 {{/*
 Create the path of the /var/log volume mount on the host fs
 */}}

charts/cortex-agent/templates/daemonset.yaml

@@ -1,3 +1,9 @@
+{{- include "cortex-xdr.validateTsfPersistentStorage" . }}
+{{- /* Env, mount and volume render only when the PVC is ready; see cortex-xdr.tsfStorageState. */ -}}
+{{- $tsfState := include "cortex-xdr.tsfStorageState" . }}
+{{- if eq $tsfState "skipped" }}
+# {{ include "cortex-xdr.tsfStorageSkippedWarning" . }}
+{{- end }}
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -120,6 +126,20 @@ spec:
         - name: XDR_AGENT_FLAVOR
           value: {{ $agentFlavor | quote }}
         {{- end }}
+        {{- if eq $tsfState "ready" }}
+        - name: XDR_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: XDR_TSF_STORAGE_DIR
+          value: "/var/lib/cortex-tsf"
+        {{- include "cortex-xdr.tsfTriggerEnv" (list . "memoryTsfTrigger" "XDR_TSF_MEMORY_TRIGGER") | nindent 8 }}
+        {{- include "cortex-xdr.tsfTriggerEnv" (list . "cpuTsfTrigger" "XDR_TSF_CPU_TRIGGER") | nindent 8 }}
+        {{- with include "cortex-xdr.tsfAutoIntervalHours" . }}
+        - name: XDR_TSF_AUTO_INTERVAL_SECONDS
+          value: {{ mul (atoi .) 3600 | toString | quote }}
+        {{- end }}
+        {{- end }}
 
         volumeMounts:
 
@@ -150,6 +170,12 @@ spec:
           readOnly: true
         {{- end }}
 
+        {{- if eq $tsfState "ready" }}
+        - name: tsf-storage
+          mountPath: /var/lib/cortex-tsf
+          subPathExpr: {{ printf "%s/$(XDR_NODE_NAME)" (trimSuffix "/" (toString .Values.tsfPersistentStorage.path)) | quote }}
+        {{- end }}
+
         resources:
           {{- include "cortex-xdr.daemonsetResources" . | nindent 10 }}
 
@@ -212,6 +238,12 @@ spec:
           secretName: {{ .Values.deploymentSecret.name | default (include "cortex-xdr.deploymentSecretName" .) }}
       {{- end }}
 
+      {{- if eq $tsfState "ready" }}
+      - name: tsf-storage
+        persistentVolumeClaim:
+          claimName: {{ .Values.tsfPersistentStorage.pvcName | quote }}
+      {{- end }}
+
       {{- if (include "cortex-xdr.dockerPullSecretName" .) }}
       imagePullSecrets:
       - name: {{ include "cortex-xdr.dockerPullSecretName" . }}

charts/cortex-agent/templates/openshift-scc.yaml

@@ -44,4 +44,7 @@ volumes:
   - downwardAPI
   - hostPath
   - secret
+{{- if eq (include "cortex-xdr.tsfStorageState" .) "ready" }}
+  - persistentVolumeClaim
+{{- end }}
 {{- end }}