From cd1e74d71cd7973f80871da09f8217b823978a3b Mon Sep 17 00:00:00 2001 From: Ran Gerson Date: Thu, 15 Oct 2026 15:08:27 +0000 Subject: [PATCH 1/6] linux-ll: Add optional TSF persistent storage and memory/CPU TSF triggers (#CPATR-41166) --- charts/cortex-agent/Chart.yaml | 2 +- charts/cortex-agent/README.md | 6 ++ charts/cortex-agent/templates/_helpers.tpl | 37 +++++++ charts/cortex-agent/templates/daemonset.yaml | 23 ++++ .../cortex-agent/templates/openshift-scc.yaml | 3 + charts/cortex-agent/values.yaml | 23 ++++ tests/test_tsf_persistent_storage.sh | 100 ++++++++++++++++++ 7 files changed, 193 insertions(+), 1 deletion(-) create mode 100755 tests/test_tsf_persistent_storage.sh diff --git a/charts/cortex-agent/Chart.yaml b/charts/cortex-agent/Chart.yaml index 8a1e576..2d8df45 100644 --- a/charts/cortex-agent/Chart.yaml +++ b/charts/cortex-agent/Chart.yaml @@ -5,7 +5,7 @@ description: Cortex XDR agent helm chart type: application -version: 1.11.0 +version: 1.12.0 appVersion: 1.0.0 diff --git a/charts/cortex-agent/README.md b/charts/cortex-agent/README.md index 8dca3bd..0258d64 100644 --- a/charts/cortex-agent/README.md +++ b/charts/cortex-agent/README.md @@ -19,6 +19,7 @@ | 1.10.0 | >=7.5 | `daemonset.resources` split into `full`/`sensor` profiles, automatically selected by `agent.agentFlavor` | 1.11.0 | >=7.5 | Support for Flatcar Container Linux platform | 1.11.0 | >=7.5 | `agent.agentFlavor` added (1 = `standard`, 3 = `cloud/sensor`). +| 1.12.0 | >=7.5 | Optional TSF persistent storage (`tsfPersistentStorage.*`) with memory/CPU TSF triggers ## Installing Cortex XDR helm chart @@ -111,6 +112,11 @@ Even when using `--reuse-values` (which uses the values of the previous installa | `daemonset.resources.full` | Resource requests/limits used when `agent.agentFlavor` is `1` (default) | Since 1.10.0 | `daemonset.resources.sensor` | Resource requests/limits used when `agent.agentFlavor` is `3` | Since 1.10.0 | `agent.agentFlavor` | Agent flavor: `1` = standard (default), `3` = cloud/sensor. Flavor `3` also selects `daemonset.resources.sensor` | Since 1.11.0, agent >= 9.4 +| `tsfPersistentStorage.enabled` | Copy TSFs to a customer PVC | Since 1.12.0 +| `tsfPersistentStorage.pvcName` | Existing ReadWriteMany PVC in the agent namespace (required when enabled) | Since 1.12.0 +| `tsfPersistentStorage.path` | Relative folder in the volume, e.g. `cortex/prod-cluster-1`; each node writes to `/` | Since 1.12.0 +| `tsfPersistentStorage.memoryTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent memory stays at or above `percent` (default 90) of its limit for `seconds` (default 300). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 +| `tsfPersistentStorage.cpuTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent CPU stays at or above `percent` (default 90) of its CPU limit for `seconds` (default 300). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 Note: Helm requires commas in arguments to be escaped. diff --git a/charts/cortex-agent/templates/_helpers.tpl b/charts/cortex-agent/templates/_helpers.tpl index fcc369a..40cab38 100644 --- a/charts/cortex-agent/templates/_helpers.tpl +++ b/charts/cortex-agent/templates/_helpers.tpl @@ -84,6 +84,43 @@ Create the name of the service account to use {{- end }} {{- end }} +{{/* +Validate tsfPersistentStorage. Fails install/upgrade on a missing or unsafe configuration. +Numbers are checked as strings: an integer 0 counts as "empty" in Helm, so `if` would skip it. +*/}} +{{- define "cortex-xdr.validateTsfPersistentStorage" -}} +{{- with .Values.tsfPersistentStorage }} +{{- if .enabled }} +{{- if not .pvcName }}{{ fail "tsfPersistentStorage.enabled=true requires tsfPersistentStorage.pvcName" }}{{ end }} +{{- if not .path }}{{ fail "tsfPersistentStorage.enabled=true requires tsfPersistentStorage.path" }}{{ end }} +{{- if or (hasPrefix "/" (toString .path)) (has ".." (splitList "/" (toString .path))) }}{{ fail "tsfPersistentStorage.path must be relative and must not contain '..'" }}{{ end }} +{{- if not (regexMatch "^[A-Za-z0-9._/-]+$" (toString .path)) }}{{ fail "tsfPersistentStorage.path may contain only letters, digits, '.', '_', '-' and '/'" }}{{ end }} +{{- end }} +{{- range $name := list "memoryTsfTrigger" "cpuTsfTrigger" }} +{{- $t := get $.Values.tsfPersistentStorage $name | default dict }} +{{- if $t.enabled }} +{{- if not $.Values.tsfPersistentStorage.enabled }}{{ fail (printf "tsfPersistentStorage.%s.enabled=true requires tsfPersistentStorage.enabled=true" $name) }}{{ end }} +{{- if not (regexMatch "^([1-9][0-9]?|100)$" (toString $t.percent)) }}{{ fail (printf "tsfPersistentStorage.%s.percent must be an integer from 1 to 100" $name) }}{{ end }} +{{- if not (regexMatch "^[1-9][0-9]*$" (toString $t.seconds)) }}{{ fail (printf "tsfPersistentStorage.%s.seconds must be a positive integer" $name) }}{{ end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Env vars for one TSF resource trigger, rendered only when the trigger is enabled. +Args: list +*/}} +{{- define "cortex-xdr.tsfTriggerEnv" -}} +{{- $t := get (index . 0).Values.tsfPersistentStorage (index . 1) | default dict }} +{{- if $t.enabled }} +- name: {{ index . 2 }}_PERCENT + value: {{ toString $t.percent | quote }} +- name: {{ index . 2 }}_SECONDS + value: {{ toString $t.seconds | quote }} +{{- end }} +{{- end }} + {{/* Create the path of the /var/log volume mount on the host fs */}} diff --git a/charts/cortex-agent/templates/daemonset.yaml b/charts/cortex-agent/templates/daemonset.yaml index 18c7147..c59406f 100644 --- a/charts/cortex-agent/templates/daemonset.yaml +++ b/charts/cortex-agent/templates/daemonset.yaml @@ -1,3 +1,4 @@ +{{- include "cortex-xdr.validateTsfPersistentStorage" . }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -120,6 +121,16 @@ spec: - name: XDR_AGENT_FLAVOR value: {{ $agentFlavor | quote }} {{- end }} + {{- if .Values.tsfPersistentStorage.enabled }} + - name: XDR_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: XDR_TSF_STORAGE_DIR + value: "/var/lib/cortex-tsf" + {{- include "cortex-xdr.tsfTriggerEnv" (list . "memoryTsfTrigger" "XDR_TSF_MEMORY_TRIGGER") | nindent 8 }} + {{- include "cortex-xdr.tsfTriggerEnv" (list . "cpuTsfTrigger" "XDR_TSF_CPU_TRIGGER") | nindent 8 }} + {{- end }} volumeMounts: @@ -150,6 +161,12 @@ spec: readOnly: true {{- end }} + {{- if .Values.tsfPersistentStorage.enabled }} + - name: tsf-storage + mountPath: /var/lib/cortex-tsf + subPathExpr: {{ printf "%s/$(XDR_NODE_NAME)" (trimSuffix "/" (toString .Values.tsfPersistentStorage.path)) | quote }} + {{- end }} + resources: {{- include "cortex-xdr.daemonsetResources" . | nindent 10 }} @@ -212,6 +229,12 @@ spec: secretName: {{ .Values.deploymentSecret.name | default (include "cortex-xdr.deploymentSecretName" .) }} {{- end }} + {{- if .Values.tsfPersistentStorage.enabled }} + - name: tsf-storage + persistentVolumeClaim: + claimName: {{ .Values.tsfPersistentStorage.pvcName | quote }} + {{- end }} + {{- if (include "cortex-xdr.dockerPullSecretName" .) }} imagePullSecrets: - name: {{ include "cortex-xdr.dockerPullSecretName" . }} diff --git a/charts/cortex-agent/templates/openshift-scc.yaml b/charts/cortex-agent/templates/openshift-scc.yaml index 39a04ee..6eab6cd 100644 --- a/charts/cortex-agent/templates/openshift-scc.yaml +++ b/charts/cortex-agent/templates/openshift-scc.yaml @@ -44,4 +44,7 @@ volumes: - downwardAPI - hostPath - secret +{{- if .Values.tsfPersistentStorage.enabled }} + - persistentVolumeClaim +{{- end }} {{- end }} diff --git a/charts/cortex-agent/values.yaml b/charts/cortex-agent/values.yaml index 4f3ba3b..07df41a 100644 --- a/charts/cortex-agent/values.yaml +++ b/charts/cortex-agent/values.yaml @@ -137,3 +137,26 @@ daemonset: namespace: create: true name: "cortex-xdr" + +# Optional: copy TSFs (crash, memory, CPU, manual; at most one per type) to +# customer-provisioned persistent storage, so they survive node replacement. +tsfPersistentStorage: + enabled: false + # Name of an existing PVC in the agent namespace (customer-created, ReadWriteMany). + # The pod stays in ContainerCreating if the PVC is missing, unbound or not RWX. + pvcName: "" + # Sub-directory inside the volume, relative and without "..", e.g. "cortex/prod-cluster-1". + # Each node writes only to /. + path: "" + # Collect a TSF when agent memory stays at or above `percent` of its limit for `seconds`. + # Requires tsfPersistentStorage.enabled=true. + memoryTsfTrigger: + enabled: false + percent: 90 + seconds: 300 + # Collect a TSF when agent CPU stays at or above `percent` of its CPU limit for `seconds`. + # Requires tsfPersistentStorage.enabled=true. + cpuTsfTrigger: + enabled: false + percent: 90 + seconds: 300 diff --git a/tests/test_tsf_persistent_storage.sh b/tests/test_tsf_persistent_storage.sh new file mode 100755 index 0000000..9703a0b --- /dev/null +++ b/tests/test_tsf_persistent_storage.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Render tests for tsfPersistentStorage. Needs helm and git. +# Usage: tests/test_tsf_persistent_storage.sh [base-ref] (default base-ref: origin/master) +set -uo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +CHART="$ROOT/charts/cortex-agent" +BASE_REF=${1:-origin/master} +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT +P=tsfPersistentStorage +ON=(--set $P.enabled=true --set $P.pvcName=cortex-tsf --set $P.path=tsf-test) +MEM=(--set $P.memoryTsfTrigger.enabled=true) +CPU=(--set $P.cpuTsfTrigger.enabled=true) +fails=0 + +ok() { echo "PASS $*"; } +bad() { echo "FAIL $*"; fails=$((fails + 1)); } +render() { helm template t "$@" 2>&1; } +has() { grep -qF -- "$2" <<<"$1"; } +env_is() { grep -A1 -- "name: $2\$" <<<"$1" | grep -qF -- "value: \"$3\""; } + +# P0 / R1: with the flag off, the output matches the base chart byte for byte, +# except the helm.sh/chart label, which carries the chart version. +git -C "$ROOT" archive "$BASE_REF" charts/cortex-agent | tar -x -C "$TMP" +render_nover() { render "$@" | grep -v '^ *helm.sh/chart: '; } +for extra in "" "--set serviceAccount.openshift.scc.create=true"; do + # shellcheck disable=SC2086 + if cmp -s <(render_nover "$TMP/charts/cortex-agent" $extra) <(render_nover "$CHART" $extra); then + ok "P0 flag off is identical to $BASE_REF ${extra:-(defaults)}" + else + bad "P0 flag off differs from $BASE_REF ${extra:-(defaults)}" + fi +done + +# P1: PVC on, triggers off: volume, per-node mount, env and OpenShift permission; no trigger env. +out=$(render "$CHART" "${ON[@]}" --set serviceAccount.openshift.scc.create=true) +for want in 'claimName: "cortex-tsf"' \ + 'subPathExpr: "tsf-test/$(XDR_NODE_NAME)"' \ + 'mountPath: /var/lib/cortex-tsf' \ + 'fieldPath: spec.nodeName' \ + '- persistentVolumeClaim'; do + has "$out" "$want" && ok "P1 renders: $want" || bad "P1 is missing: $want" +done +env_is "$out" XDR_TSF_STORAGE_DIR /var/lib/cortex-tsf && ok "P1 XDR_TSF_STORAGE_DIR" || bad "P1 XDR_TSF_STORAGE_DIR" +has "$out" "_TRIGGER_" && bad "P1 renders trigger env" || ok "P1 no trigger env" +has "$out" "LOG_SNAPSHOT" && bad "P1 renders old snapshot env" || ok "P1 no old snapshot env" + +# P2: memory trigger only, with defaults. +out=$(render "$CHART" "${ON[@]}" "${MEM[@]}") +env_is "$out" XDR_TSF_MEMORY_TRIGGER_PERCENT 90 && env_is "$out" XDR_TSF_MEMORY_TRIGGER_SECONDS 300 \ + && ok "P2 memory trigger defaults 90/300" || bad "P2 memory trigger defaults" +has "$out" "XDR_TSF_CPU_TRIGGER" && bad "P2 renders CPU trigger" || ok "P2 no CPU trigger" + +# P3: CPU trigger only, custom values. +out=$(render "$CHART" "${ON[@]}" "${CPU[@]}" --set $P.cpuTsfTrigger.percent=80 --set $P.cpuTsfTrigger.seconds=120) +env_is "$out" XDR_TSF_CPU_TRIGGER_PERCENT 80 && env_is "$out" XDR_TSF_CPU_TRIGGER_SECONDS 120 \ + && ok "P3 CPU trigger 80/120" || bad "P3 CPU trigger 80/120" +has "$out" "XDR_TSF_MEMORY_TRIGGER" && bad "P3 renders memory trigger" || ok "P3 no memory trigger" + +# P4: both triggers; percent 100 is accepted. +out=$(render "$CHART" "${ON[@]}" "${MEM[@]}" "${CPU[@]}" --set $P.memoryTsfTrigger.percent=100) +env_is "$out" XDR_TSF_MEMORY_TRIGGER_PERCENT 100 && env_is "$out" XDR_TSF_CPU_TRIGGER_PERCENT 90 \ + && ok "P4 both triggers rendered" || bad "P4 both triggers rendered" + +# Trailing slash in path is trimmed. +render "$CHART" "${ON[@]}" --set $P.path=a/b/ | grep -qF 'subPathExpr: "a/b/$(XDR_NODE_NAME)"' \ + && ok "trailing slash in path is trimmed" || bad "trailing slash in path is not trimmed" + +# Invalid configurations fail with a clear message. +expect_fail() { + local msg=$1 out; shift + # Capture first: helm exits non-zero here, and with pipefail a pipe into grep would report that. + out=$(render "$CHART" "$@") + if has "$out" "$msg"; then ok "rejected: $* -> $msg"; else bad "not rejected: $*"; fi +} +expect_fail "requires tsfPersistentStorage.pvcName" --set $P.enabled=true +expect_fail "requires tsfPersistentStorage.path" --set $P.enabled=true --set $P.pvcName=x +expect_fail "must be relative" "${ON[@]}" --set $P.path=/abs +expect_fail "must not contain '..'" "${ON[@]}" --set $P.path=a/../b +expect_fail "may contain only" "${ON[@]}" --set "$P.path=a\$b" +expect_fail "memoryTsfTrigger.enabled=true requires tsfPersistentStorage.enabled=true" "${MEM[@]}" +expect_fail "cpuTsfTrigger.enabled=true requires tsfPersistentStorage.enabled=true" "${CPU[@]}" +for v in 0 101 -5 abc 50.5; do + expect_fail "memoryTsfTrigger.percent must be an integer from 1 to 100" "${ON[@]}" "${MEM[@]}" --set $P.memoryTsfTrigger.percent=$v +done +for v in 0 -1 abc; do + expect_fail "cpuTsfTrigger.seconds must be a positive integer" "${ON[@]}" "${CPU[@]}" --set $P.cpuTsfTrigger.seconds=$v +done + +# A disabled trigger with bad values is ignored. +render "$CHART" "${ON[@]}" --set $P.cpuTsfTrigger.percent=0 >/dev/null \ + && ok "disabled trigger values are not validated" || bad "disabled trigger values are validated" + +# Valid paths that look unusual are accepted. +render "$CHART" "${ON[@]}" --set $P.path=cortex/prod-1.a_b >/dev/null \ + && ok "path cortex/prod-1.a_b is accepted" || bad "path cortex/prod-1.a_b is rejected" + +echo +if [ "$fails" -eq 0 ]; then echo "OVERALL: PASS"; else echo "OVERALL: FAIL ($fails)"; fi +[ "$fails" -eq 0 ] From 986a318eb84404299950dd7536b0e7d132a390b8 Mon Sep 17 00:00:00 2001 From: Ran Gerson Date: Sun, 25 Oct 2026 14:10:51 +0000 Subject: [PATCH 2/6] linux-ll: tsf-persistent: Auto TSF persistent feature's parameters for Agent installation (CPATR-38659) --- charts/cortex-agent/.helmignore | 1 + charts/cortex-agent/README.md | 2 +- charts/cortex-agent/templates/NOTES.txt | 4 + charts/cortex-agent/templates/_helpers.tpl | 27 +++ charts/cortex-agent/templates/daemonset.yaml | 11 +- .../cortex-agent/templates/openshift-scc.yaml | 2 +- .../tests/tsf_persistent_storage_test.yaml | 224 ++++++++++++++++++ charts/cortex-agent/values.yaml | 5 +- tests/test_tsf_persistent_storage.sh | 43 +--- 9 files changed, 281 insertions(+), 38 deletions(-) create mode 100644 charts/cortex-agent/.helmignore create mode 100644 charts/cortex-agent/tests/tsf_persistent_storage_test.yaml diff --git a/charts/cortex-agent/.helmignore b/charts/cortex-agent/.helmignore new file mode 100644 index 0000000..2bfa6a4 --- /dev/null +++ b/charts/cortex-agent/.helmignore @@ -0,0 +1 @@ +tests/ diff --git a/charts/cortex-agent/README.md b/charts/cortex-agent/README.md index 0258d64..32de54f 100644 --- a/charts/cortex-agent/README.md +++ b/charts/cortex-agent/README.md @@ -113,7 +113,7 @@ Even when using `--reuse-values` (which uses the values of the previous installa | `daemonset.resources.sensor` | Resource requests/limits used when `agent.agentFlavor` is `3` | Since 1.10.0 | `agent.agentFlavor` | Agent flavor: `1` = standard (default), `3` = cloud/sensor. Flavor `3` also selects `daemonset.resources.sensor` | Since 1.11.0, agent >= 9.4 | `tsfPersistentStorage.enabled` | Copy TSFs to a customer PVC | Since 1.12.0 -| `tsfPersistentStorage.pvcName` | Existing ReadWriteMany PVC in the agent namespace (required when enabled) | Since 1.12.0 +| `tsfPersistentStorage.pvcName` | Existing ReadWriteMany PVC in `namespace.name` (required when enabled). Mounted only if Bound and RWX at install/upgrade time, otherwise skipped with a WARNING; always skipped by `helm template` | Since 1.12.0 | `tsfPersistentStorage.path` | Relative folder in the volume, e.g. `cortex/prod-cluster-1`; each node writes to `/` | Since 1.12.0 | `tsfPersistentStorage.memoryTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent memory stays at or above `percent` (default 90) of its limit for `seconds` (default 300). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 | `tsfPersistentStorage.cpuTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent CPU stays at or above `percent` (default 90) of its CPU limit for `seconds` (default 300). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 diff --git a/charts/cortex-agent/templates/NOTES.txt b/charts/cortex-agent/templates/NOTES.txt index b34d91e..bfa97a3 100644 --- a/charts/cortex-agent/templates/NOTES.txt +++ b/charts/cortex-agent/templates/NOTES.txt @@ -4,6 +4,10 @@ Deployed to namespace: {{ .Values.namespace.name }} Release name: {{ .Release.Name }} To view all agent pods run: kubectl get pods -n {{ .Values.namespace.name }} +{{- if eq (include "cortex-xdr.tsfStorageState" .) "skipped" }} + +{{ include "cortex-xdr.tsfStorageSkippedWarning" . }} +{{- end }} In case images are not pulled, make sure to set the right repository and the right pull secret value, visit "https://github.com/paloaltonetworks/cortex-helm/helm-charts/cortex-agent" for diff --git a/charts/cortex-agent/templates/_helpers.tpl b/charts/cortex-agent/templates/_helpers.tpl index 40cab38..f8d9625 100644 --- a/charts/cortex-agent/templates/_helpers.tpl +++ b/charts/cortex-agent/templates/_helpers.tpl @@ -107,6 +107,33 @@ Numbers are checked as strings: an integer 0 counts as "empty" in Helm, so `if` {{- end }} {{- end }} +{{/* +TSF persistent storage state. +Returns "ready" when enabled and the PVC is Bound and ReadWriteMany at install/upgrade time, +"skipped" when enabled but the PVC is not ready - missing, unbound, not RWX, or no cluster, +and "" when disabled. Only "ready" mounts the PVC, so a bad PVC never blocks the agent pod +at install/upgrade time. +*/}} +{{- define "cortex-xdr.tsfStorageState" -}} +{{- $tsf := .Values.tsfPersistentStorage | default dict -}} +{{- if $tsf.enabled -}} + {{- $pvc := lookup "v1" "PersistentVolumeClaim" .Values.namespace.name ($tsf.pvcName | toString) -}} + {{- $status := dig "status" dict $pvc -}} + {{- if and (eq (dig "phase" "" $status) "Bound") (has "ReadWriteMany" (dig "accessModes" list $status)) -}} +ready + {{- else -}} +skipped + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Warning shown when TSF persistent storage is enabled but skipped. %q escapes the user-supplied PVC name. +*/}} +{{- define "cortex-xdr.tsfStorageSkippedWarning" -}} +{{- printf "WARNING: TSF persistent storage skipped: PVC %q in namespace %q was not found, is not Bound or is not ReadWriteMany (or no cluster, as with helm template). The agent runs without it; fix the PVC and run helm upgrade again." (.Values.tsfPersistentStorage.pvcName | toString) .Values.namespace.name -}} +{{- end -}} + {{/* Env vars for one TSF resource trigger, rendered only when the trigger is enabled. Args: list diff --git a/charts/cortex-agent/templates/daemonset.yaml b/charts/cortex-agent/templates/daemonset.yaml index c59406f..f1cc453 100644 --- a/charts/cortex-agent/templates/daemonset.yaml +++ b/charts/cortex-agent/templates/daemonset.yaml @@ -1,4 +1,9 @@ {{- include "cortex-xdr.validateTsfPersistentStorage" . }} +{{- /* Env, mount and volume render only when the PVC is ready; see cortex-xdr.tsfStorageState. */ -}} +{{- $tsfState := include "cortex-xdr.tsfStorageState" . }} +{{- if eq $tsfState "skipped" }} +# {{ include "cortex-xdr.tsfStorageSkippedWarning" . }} +{{- end }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -121,7 +126,7 @@ spec: - name: XDR_AGENT_FLAVOR value: {{ $agentFlavor | quote }} {{- end }} - {{- if .Values.tsfPersistentStorage.enabled }} + {{- if eq $tsfState "ready" }} - name: XDR_NODE_NAME valueFrom: fieldRef: @@ -161,7 +166,7 @@ spec: readOnly: true {{- end }} - {{- if .Values.tsfPersistentStorage.enabled }} + {{- if eq $tsfState "ready" }} - name: tsf-storage mountPath: /var/lib/cortex-tsf subPathExpr: {{ printf "%s/$(XDR_NODE_NAME)" (trimSuffix "/" (toString .Values.tsfPersistentStorage.path)) | quote }} @@ -229,7 +234,7 @@ spec: secretName: {{ .Values.deploymentSecret.name | default (include "cortex-xdr.deploymentSecretName" .) }} {{- end }} - {{- if .Values.tsfPersistentStorage.enabled }} + {{- if eq $tsfState "ready" }} - name: tsf-storage persistentVolumeClaim: claimName: {{ .Values.tsfPersistentStorage.pvcName | quote }} diff --git a/charts/cortex-agent/templates/openshift-scc.yaml b/charts/cortex-agent/templates/openshift-scc.yaml index 6eab6cd..b5f6ce7 100644 --- a/charts/cortex-agent/templates/openshift-scc.yaml +++ b/charts/cortex-agent/templates/openshift-scc.yaml @@ -44,7 +44,7 @@ volumes: - downwardAPI - hostPath - secret -{{- if .Values.tsfPersistentStorage.enabled }} +{{- if eq (include "cortex-xdr.tsfStorageState" .) "ready" }} - persistentVolumeClaim {{- end }} {{- end }} diff --git a/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml b/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml new file mode 100644 index 0000000..83e72b4 --- /dev/null +++ b/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml @@ -0,0 +1,224 @@ +# Tests for tsfPersistentStorage against a fake cluster. The PVC is mounted only +# when `lookup` finds it Bound and ReadWriteMany, so a bad PVC never blocks the agent pod. +# The "skipped" WARNING comment is asserted by tests/test_tsf_persistent_storage.sh: +# helm-unittest parses manifests and drops comments. +suite: tsf persistent storage +templates: + - templates/daemonset.yaml + - templates/openshift-scc.yaml +set: + serviceAccount.openshift.scc.create: true +# PVCs live in namespace.name (default "cortex-xdr"). +kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: { name: cortex-tsf, namespace: cortex-xdr } + status: { phase: Bound, accessModes: [ReadWriteMany] } + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: { name: pending-tsf, namespace: cortex-xdr } + status: { phase: Pending } + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: { name: rwo-tsf, namespace: cortex-xdr } + status: { phase: Bound, accessModes: [ReadWriteOnce] } + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: { name: cortex-tsf, namespace: other } + status: { phase: Bound, accessModes: [ReadWriteMany] } + +tests: + - it: renders nothing when disabled (default) + asserts: + - notContains: + path: spec.template.spec.volumes + content: { name: tsf-storage } + any: true + template: templates/daemonset.yaml + - notContains: + path: volumes + content: persistentVolumeClaim + template: templates/openshift-scc.yaml + + - it: storage on, triggers off - PVC volume, per-node mount, env, SCC + set: &storageOn + tsfPersistentStorage.enabled: true + tsfPersistentStorage.pvcName: cortex-tsf + tsfPersistentStorage.path: tsf-test + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: XDR_NODE_NAME + valueFrom: { fieldRef: { fieldPath: spec.nodeName } } + template: templates/daemonset.yaml + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_STORAGE_DIR, value: /var/lib/cortex-tsf } + template: templates/daemonset.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: tsf-storage + mountPath: /var/lib/cortex-tsf + subPathExpr: tsf-test/$(XDR_NODE_NAME) + template: templates/daemonset.yaml + - contains: + path: spec.template.spec.volumes + content: + name: tsf-storage + persistentVolumeClaim: { claimName: cortex-tsf } + template: templates/daemonset.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT } + any: true + template: templates/daemonset.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_CPU_TRIGGER_PERCENT } + any: true + template: templates/daemonset.yaml + - contains: + path: volumes + content: persistentVolumeClaim + template: templates/openshift-scc.yaml + + - it: memory trigger only, with defaults 90/300 + set: + <<: *storageOn + tsfPersistentStorage.memoryTsfTrigger.enabled: true + template: templates/daemonset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT, value: "90" } + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_MEMORY_TRIGGER_SECONDS, value: "300" } + - notContains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_CPU_TRIGGER_PERCENT } + any: true + + - it: CPU trigger only, custom 80/120 + set: + <<: *storageOn + tsfPersistentStorage.cpuTsfTrigger.enabled: true + tsfPersistentStorage.cpuTsfTrigger.percent: 80 + tsfPersistentStorage.cpuTsfTrigger.seconds: 120 + template: templates/daemonset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_CPU_TRIGGER_PERCENT, value: "80" } + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_CPU_TRIGGER_SECONDS, value: "120" } + - notContains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT } + any: true + + - it: both triggers; percent 100 is accepted + set: + <<: *storageOn + tsfPersistentStorage.memoryTsfTrigger.enabled: true + tsfPersistentStorage.memoryTsfTrigger.percent: 100 + tsfPersistentStorage.cpuTsfTrigger.enabled: true + template: templates/daemonset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT, value: "100" } + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_CPU_TRIGGER_PERCENT, value: "90" } + + - it: trims a trailing slash in path + set: + <<: *storageOn + tsfPersistentStorage.path: a/b/ + template: templates/daemonset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: tsf-storage + mountPath: /var/lib/cortex-tsf + subPathExpr: a/b/$(XDR_NODE_NAME) + + # A PVC that is missing, not Bound, not RWX or in another namespace is skipped: + # no env, mount, volume or SCC entry. The agent DaemonSet still renders. + - it: skips a missing PVC and still renders the agent + set: + <<: *storageOn + tsfPersistentStorage.pvcName: no-such-pvc + tsfPersistentStorage.memoryTsfTrigger.enabled: true + asserts: + - isKind: { of: DaemonSet } + template: templates/daemonset.yaml + - notContains: + path: spec.template.spec.volumes + content: { name: tsf-storage } + any: true + template: templates/daemonset.yaml + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: { name: tsf-storage } + any: true + template: templates/daemonset.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_STORAGE_DIR } + any: true + template: templates/daemonset.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT } + any: true + template: templates/daemonset.yaml + - notContains: + path: volumes + content: persistentVolumeClaim + template: templates/openshift-scc.yaml + + - it: skips a PVC that is not Bound + set: + <<: *storageOn + tsfPersistentStorage.pvcName: pending-tsf + template: templates/daemonset.yaml + asserts: + - notContains: + path: spec.template.spec.volumes + content: { name: tsf-storage } + any: true + + - it: skips a PVC that is not ReadWriteMany + set: + <<: *storageOn + tsfPersistentStorage.pvcName: rwo-tsf + template: templates/daemonset.yaml + asserts: + - notContains: + path: spec.template.spec.volumes + content: { name: tsf-storage } + any: true + + - it: skips a PVC that exists only in another namespace + set: + <<: *storageOn + namespace.name: cortex-other + template: templates/daemonset.yaml + asserts: + - notContains: + path: spec.template.spec.volumes + content: { name: tsf-storage } + any: true diff --git a/charts/cortex-agent/values.yaml b/charts/cortex-agent/values.yaml index 07df41a..fb8609d 100644 --- a/charts/cortex-agent/values.yaml +++ b/charts/cortex-agent/values.yaml @@ -142,8 +142,9 @@ namespace: # customer-provisioned persistent storage, so they survive node replacement. tsfPersistentStorage: enabled: false - # Name of an existing PVC in the agent namespace (customer-created, ReadWriteMany). - # The pod stays in ContainerCreating if the PVC is missing, unbound or not RWX. + # Name of an existing PVC in namespace.name (customer-created, ReadWriteMany). + # Mounted only if it is Bound and RWX at install/upgrade time; otherwise the feature is skipped with a WARNING. + # Always skipped by helm template / GitOps renders (no cluster to look up). pvcName: "" # Sub-directory inside the volume, relative and without "..", e.g. "cortex/prod-cluster-1". # Each node writes only to /. diff --git a/tests/test_tsf_persistent_storage.sh b/tests/test_tsf_persistent_storage.sh index 9703a0b..4d14b45 100755 --- a/tests/test_tsf_persistent_storage.sh +++ b/tests/test_tsf_persistent_storage.sh @@ -32,39 +32,20 @@ for extra in "" "--set serviceAccount.openshift.scc.create=true"; do fi done -# P1: PVC on, triggers off: volume, per-node mount, env and OpenShift permission; no trigger env. -out=$(render "$CHART" "${ON[@]}" --set serviceAccount.openshift.scc.create=true) -for want in 'claimName: "cortex-tsf"' \ - 'subPathExpr: "tsf-test/$(XDR_NODE_NAME)"' \ - 'mountPath: /var/lib/cortex-tsf' \ - 'fieldPath: spec.nodeName' \ - '- persistentVolumeClaim'; do - has "$out" "$want" && ok "P1 renders: $want" || bad "P1 is missing: $want" +# S1: helm template has no cluster, so the PVC lookup is empty and the feature is skipped: +# a WARNING comment, the agent still renders, and no TSF env, mount, volume or SCC entry. +# The mounted path (PVC Bound and RWX) is covered by charts/cortex-agent/tests (helm unittest). +out=$(render "$CHART" "${ON[@]}" "${MEM[@]}" "${CPU[@]}" --set serviceAccount.openshift.scc.create=true) +has "$out" '# WARNING: TSF persistent storage skipped: PVC "cortex-tsf" in namespace "cortex-xdr"' \ + && ok "S1 skipped WARNING rendered" || bad "S1 skipped WARNING missing" +has "$out" "kind: DaemonSet" && ok "S1 agent DaemonSet rendered" || bad "S1 agent DaemonSet missing" +for unwanted in 'tsf-storage' 'XDR_TSF_' 'XDR_NODE_NAME' '- persistentVolumeClaim'; do + has "$out" "$unwanted" && bad "S1 renders: $unwanted" || ok "S1 does not render: $unwanted" done -env_is "$out" XDR_TSF_STORAGE_DIR /var/lib/cortex-tsf && ok "P1 XDR_TSF_STORAGE_DIR" || bad "P1 XDR_TSF_STORAGE_DIR" -has "$out" "_TRIGGER_" && bad "P1 renders trigger env" || ok "P1 no trigger env" -has "$out" "LOG_SNAPSHOT" && bad "P1 renders old snapshot env" || ok "P1 no old snapshot env" -# P2: memory trigger only, with defaults. -out=$(render "$CHART" "${ON[@]}" "${MEM[@]}") -env_is "$out" XDR_TSF_MEMORY_TRIGGER_PERCENT 90 && env_is "$out" XDR_TSF_MEMORY_TRIGGER_SECONDS 300 \ - && ok "P2 memory trigger defaults 90/300" || bad "P2 memory trigger defaults" -has "$out" "XDR_TSF_CPU_TRIGGER" && bad "P2 renders CPU trigger" || ok "P2 no CPU trigger" - -# P3: CPU trigger only, custom values. -out=$(render "$CHART" "${ON[@]}" "${CPU[@]}" --set $P.cpuTsfTrigger.percent=80 --set $P.cpuTsfTrigger.seconds=120) -env_is "$out" XDR_TSF_CPU_TRIGGER_PERCENT 80 && env_is "$out" XDR_TSF_CPU_TRIGGER_SECONDS 120 \ - && ok "P3 CPU trigger 80/120" || bad "P3 CPU trigger 80/120" -has "$out" "XDR_TSF_MEMORY_TRIGGER" && bad "P3 renders memory trigger" || ok "P3 no memory trigger" - -# P4: both triggers; percent 100 is accepted. -out=$(render "$CHART" "${ON[@]}" "${MEM[@]}" "${CPU[@]}" --set $P.memoryTsfTrigger.percent=100) -env_is "$out" XDR_TSF_MEMORY_TRIGGER_PERCENT 100 && env_is "$out" XDR_TSF_CPU_TRIGGER_PERCENT 90 \ - && ok "P4 both triggers rendered" || bad "P4 both triggers rendered" - -# Trailing slash in path is trimmed. -render "$CHART" "${ON[@]}" --set $P.path=a/b/ | grep -qF 'subPathExpr: "a/b/$(XDR_NODE_NAME)"' \ - && ok "trailing slash in path is trimmed" || bad "trailing slash in path is not trimmed" +# S2: the PVC name in the WARNING is escaped. +render "$CHART" "${ON[@]}" --set "$P.pvcName=a\"b" | grep -qF 'PVC "a\"b"' \ + && ok "S2 PVC name is escaped in the WARNING" || bad "S2 PVC name is not escaped in the WARNING" # Invalid configurations fail with a clear message. expect_fail() { From 5894de7cd4cbdf0a4aa137ecf7590349cbdea795 Mon Sep 17 00:00:00 2001 From: Ran Gerson Date: Sun, 8 Nov 2026 18:08:30 +0000 Subject: [PATCH 3/6] linux-ll: tsf-persistent: Only automatic TSFs are copied to persistent storage (CPATR-38659) --- charts/cortex-agent/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cortex-agent/values.yaml b/charts/cortex-agent/values.yaml index fb8609d..322e8b9 100644 --- a/charts/cortex-agent/values.yaml +++ b/charts/cortex-agent/values.yaml @@ -138,7 +138,7 @@ namespace: create: true name: "cortex-xdr" -# Optional: copy TSFs (crash, memory, CPU, manual; at most one per type) to +# Optional: copy automatic TSFs (crash, memory, CPU; at most one per type) to # customer-provisioned persistent storage, so they survive node replacement. tsfPersistentStorage: enabled: false From 5dc6dfb842b58bf0711f7cbd2cc1e50f067714f9 Mon Sep 17 00:00:00 2001 From: Ran Gerson Date: Mon, 9 Nov 2026 11:14:59 +0000 Subject: [PATCH 4/6] inux-ll: tsf-persistent: Windows interval config by helm val, def 24H, change CPU def 65 (CPATR-38659) --- charts/cortex-agent/README.md | 3 +- charts/cortex-agent/templates/_helpers.tpl | 11 +++++ charts/cortex-agent/templates/daemonset.yaml | 4 ++ .../tests/tsf_persistent_storage_test.yaml | 44 ++++++++++++++++++- charts/cortex-agent/values.yaml | 4 +- 5 files changed, 63 insertions(+), 3 deletions(-) diff --git a/charts/cortex-agent/README.md b/charts/cortex-agent/README.md index 32de54f..3288499 100644 --- a/charts/cortex-agent/README.md +++ b/charts/cortex-agent/README.md @@ -115,8 +115,9 @@ Even when using `--reuse-values` (which uses the values of the previous installa | `tsfPersistentStorage.enabled` | Copy TSFs to a customer PVC | Since 1.12.0 | `tsfPersistentStorage.pvcName` | Existing ReadWriteMany PVC in `namespace.name` (required when enabled). Mounted only if Bound and RWX at install/upgrade time, otherwise skipped with a WARNING; always skipped by `helm template` | Since 1.12.0 | `tsfPersistentStorage.path` | Relative folder in the volume, e.g. `cortex/prod-cluster-1`; each node writes to `/` | Since 1.12.0 +| `tsfPersistentStorage.autoTsfIntervalHours` | Minimum hours between automatic TSFs of any type, 1 to 168; empty means 24 | Since 1.12.0 | `tsfPersistentStorage.memoryTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent memory stays at or above `percent` (default 90) of its limit for `seconds` (default 300). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 -| `tsfPersistentStorage.cpuTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent CPU stays at or above `percent` (default 90) of its CPU limit for `seconds` (default 300). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 +| `tsfPersistentStorage.cpuTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent CPU stays at or above `percent` (default 65) of its CPU limit for `seconds` (default 300). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 Note: Helm requires commas in arguments to be escaped. diff --git a/charts/cortex-agent/templates/_helpers.tpl b/charts/cortex-agent/templates/_helpers.tpl index f8d9625..dccb12a 100644 --- a/charts/cortex-agent/templates/_helpers.tpl +++ b/charts/cortex-agent/templates/_helpers.tpl @@ -96,6 +96,9 @@ Numbers are checked as strings: an integer 0 counts as "empty" in Helm, so `if` {{- if or (hasPrefix "/" (toString .path)) (has ".." (splitList "/" (toString .path))) }}{{ fail "tsfPersistentStorage.path must be relative and must not contain '..'" }}{{ end }} {{- if not (regexMatch "^[A-Za-z0-9._/-]+$" (toString .path)) }}{{ fail "tsfPersistentStorage.path may contain only letters, digits, '.', '_', '-' and '/'" }}{{ end }} {{- end }} +{{- if include "cortex-xdr.tsfAutoIntervalHours" $ }} +{{- if not (regexMatch "^([1-9][0-9]?|1[0-5][0-9]|16[0-8])$" (include "cortex-xdr.tsfAutoIntervalHours" $)) }}{{ fail "tsfPersistentStorage.autoTsfIntervalHours must be an integer from 1 to 168" }}{{ end }} +{{- end }} {{- range $name := list "memoryTsfTrigger" "cpuTsfTrigger" }} {{- $t := get $.Values.tsfPersistentStorage $name | default dict }} {{- if $t.enabled }} @@ -134,6 +137,14 @@ Warning shown when TSF persistent storage is enabled but skipped. %q escapes the {{- printf "WARNING: TSF persistent storage skipped: PVC %q in namespace %q was not found, is not Bound or is not ReadWriteMany (or no cluster, as with helm template). The agent runs without it; fix the PVC and run helm upgrade again." (.Values.tsfPersistentStorage.pvcName | toString) .Values.namespace.name -}} {{- end -}} +{{/* +tsfPersistentStorage.autoTsfIntervalHours as a string; empty when unset (the agent then uses 24 hours). +*/}} +{{- define "cortex-xdr.tsfAutoIntervalHours" -}} +{{- $h := (.Values.tsfPersistentStorage | default dict).autoTsfIntervalHours -}} +{{- if not (kindIs "invalid" $h) }}{{ toString $h }}{{ end -}} +{{- end -}} + {{/* Env vars for one TSF resource trigger, rendered only when the trigger is enabled. Args: list diff --git a/charts/cortex-agent/templates/daemonset.yaml b/charts/cortex-agent/templates/daemonset.yaml index f1cc453..8c9500c 100644 --- a/charts/cortex-agent/templates/daemonset.yaml +++ b/charts/cortex-agent/templates/daemonset.yaml @@ -135,6 +135,10 @@ spec: value: "/var/lib/cortex-tsf" {{- include "cortex-xdr.tsfTriggerEnv" (list . "memoryTsfTrigger" "XDR_TSF_MEMORY_TRIGGER") | nindent 8 }} {{- include "cortex-xdr.tsfTriggerEnv" (list . "cpuTsfTrigger" "XDR_TSF_CPU_TRIGGER") | nindent 8 }} + {{- with include "cortex-xdr.tsfAutoIntervalHours" . }} + - name: XDR_TSF_AUTO_INTERVAL_SECONDS + value: {{ mul (atoi .) 3600 | toString | quote }} + {{- end }} {{- end }} volumeMounts: diff --git a/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml b/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml index 83e72b4..5d18ae2 100644 --- a/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml +++ b/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml @@ -140,7 +140,49 @@ tests: content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT, value: "100" } - contains: path: spec.template.spec.containers[0].env - content: { name: XDR_TSF_CPU_TRIGGER_PERCENT, value: "90" } + content: { name: XDR_TSF_CPU_TRIGGER_PERCENT, value: "65" } + - notContains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_AUTO_INTERVAL_SECONDS } + any: true + + - it: autoTsfIntervalHours renders seconds; bounds 1 and 168 accepted + set: + <<: *storageOn + tsfPersistentStorage.autoTsfIntervalHours: 1 + template: templates/daemonset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_AUTO_INTERVAL_SECONDS, value: "3600" } + + - it: autoTsfIntervalHours 168 renders 604800 + set: + <<: *storageOn + tsfPersistentStorage.autoTsfIntervalHours: 168 + template: templates/daemonset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_AUTO_INTERVAL_SECONDS, value: "604800" } + + - it: rejects autoTsfIntervalHours 0 + set: + <<: *storageOn + tsfPersistentStorage.autoTsfIntervalHours: 0 + template: templates/daemonset.yaml + asserts: + - failedTemplate: + errorMessage: tsfPersistentStorage.autoTsfIntervalHours must be an integer from 1 to 168 + + - it: rejects autoTsfIntervalHours 169 + set: + <<: *storageOn + tsfPersistentStorage.autoTsfIntervalHours: 169 + template: templates/daemonset.yaml + asserts: + - failedTemplate: + errorMessage: tsfPersistentStorage.autoTsfIntervalHours must be an integer from 1 to 168 - it: trims a trailing slash in path set: diff --git a/charts/cortex-agent/values.yaml b/charts/cortex-agent/values.yaml index 322e8b9..f93ca3e 100644 --- a/charts/cortex-agent/values.yaml +++ b/charts/cortex-agent/values.yaml @@ -149,6 +149,8 @@ tsfPersistentStorage: # Sub-directory inside the volume, relative and without "..", e.g. "cortex/prod-cluster-1". # Each node writes only to /. path: "" + # Minimum hours between automatic TSFs of any type (crash, memory, CPU), 1 to 168. Empty: 24. + autoTsfIntervalHours: "" # Collect a TSF when agent memory stays at or above `percent` of its limit for `seconds`. # Requires tsfPersistentStorage.enabled=true. memoryTsfTrigger: @@ -159,5 +161,5 @@ tsfPersistentStorage: # Requires tsfPersistentStorage.enabled=true. cpuTsfTrigger: enabled: false - percent: 90 + percent: 65 seconds: 300 From 4bf4954cb9417e547bdf68e45d1bf22cddfaa54f Mon Sep 17 00:00:00 2001 From: Ran Gerson Date: Mon, 9 Nov 2026 12:14:29 +0000 Subject: [PATCH 5/6] inux-ll: tsf-persistent: CR fix, render trigger values as number, add cap (CPATR-38659) --- charts/cortex-agent/README.md | 4 ++-- charts/cortex-agent/templates/_helpers.tpl | 7 ++++--- .../tests/tsf_persistent_storage_test.yaml | 21 +++++++++++++++++++ tests/test_tsf_persistent_storage.sh | 8 +++++-- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/charts/cortex-agent/README.md b/charts/cortex-agent/README.md index 3288499..5550009 100644 --- a/charts/cortex-agent/README.md +++ b/charts/cortex-agent/README.md @@ -116,8 +116,8 @@ Even when using `--reuse-values` (which uses the values of the previous installa | `tsfPersistentStorage.pvcName` | Existing ReadWriteMany PVC in `namespace.name` (required when enabled). Mounted only if Bound and RWX at install/upgrade time, otherwise skipped with a WARNING; always skipped by `helm template` | Since 1.12.0 | `tsfPersistentStorage.path` | Relative folder in the volume, e.g. `cortex/prod-cluster-1`; each node writes to `/` | Since 1.12.0 | `tsfPersistentStorage.autoTsfIntervalHours` | Minimum hours between automatic TSFs of any type, 1 to 168; empty means 24 | Since 1.12.0 -| `tsfPersistentStorage.memoryTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent memory stays at or above `percent` (default 90) of its limit for `seconds` (default 300). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 -| `tsfPersistentStorage.cpuTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent CPU stays at or above `percent` (default 65) of its CPU limit for `seconds` (default 300). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 +| `tsfPersistentStorage.memoryTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent memory stays at or above `percent` (default 90) of its limit for `seconds` (default 300, 1 to 86400). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 +| `tsfPersistentStorage.cpuTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent CPU stays at or above `percent` (default 65) of its CPU limit for `seconds` (default 300, 1 to 86400). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 Note: Helm requires commas in arguments to be escaped. diff --git a/charts/cortex-agent/templates/_helpers.tpl b/charts/cortex-agent/templates/_helpers.tpl index dccb12a..53fa73d 100644 --- a/charts/cortex-agent/templates/_helpers.tpl +++ b/charts/cortex-agent/templates/_helpers.tpl @@ -104,7 +104,8 @@ Numbers are checked as strings: an integer 0 counts as "empty" in Helm, so `if` {{- if $t.enabled }} {{- if not $.Values.tsfPersistentStorage.enabled }}{{ fail (printf "tsfPersistentStorage.%s.enabled=true requires tsfPersistentStorage.enabled=true" $name) }}{{ end }} {{- if not (regexMatch "^([1-9][0-9]?|100)$" (toString $t.percent)) }}{{ fail (printf "tsfPersistentStorage.%s.percent must be an integer from 1 to 100" $name) }}{{ end }} -{{- if not (regexMatch "^[1-9][0-9]*$" (toString $t.seconds)) }}{{ fail (printf "tsfPersistentStorage.%s.seconds must be a positive integer" $name) }}{{ end }} +{{- $sec := $t.seconds }} +{{- if or (kindIs "bool" $sec) (ne (float64 $sec) (float64 (int64 $sec))) (lt (int64 $sec) 1) (gt (int64 $sec) 86400) }}{{ fail (printf "tsfPersistentStorage.%s.seconds must be an integer from 1 to 86400" $name) }}{{ end }} {{- end }} {{- end }} {{- end }} @@ -153,9 +154,9 @@ Args: list {{- $t := get (index . 0).Values.tsfPersistentStorage (index . 1) | default dict }} {{- if $t.enabled }} - name: {{ index . 2 }}_PERCENT - value: {{ toString $t.percent | quote }} + value: {{ $t.percent | int | quote }} - name: {{ index . 2 }}_SECONDS - value: {{ toString $t.seconds | quote }} + value: {{ $t.seconds | int | quote }} {{- end }} {{- end }} diff --git a/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml b/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml index 5d18ae2..14ebf55 100644 --- a/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml +++ b/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml @@ -127,6 +127,27 @@ tests: content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT } any: true + - it: seconds 86400 (maximum) is accepted + set: + <<: *storageOn + tsfPersistentStorage.cpuTsfTrigger.enabled: true + tsfPersistentStorage.cpuTsfTrigger.seconds: 86400 + template: templates/daemonset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: { name: XDR_TSF_CPU_TRIGGER_SECONDS, value: "86400" } + + - it: rejects seconds above 86400 + set: + <<: *storageOn + tsfPersistentStorage.cpuTsfTrigger.enabled: true + tsfPersistentStorage.cpuTsfTrigger.seconds: 1000000 + template: templates/daemonset.yaml + asserts: + - failedTemplate: + errorPattern: "cpuTsfTrigger.seconds must be an integer from 1 to 86400" + - it: both triggers; percent 100 is accepted set: <<: *storageOn diff --git a/tests/test_tsf_persistent_storage.sh b/tests/test_tsf_persistent_storage.sh index 4d14b45..2092a6f 100755 --- a/tests/test_tsf_persistent_storage.sh +++ b/tests/test_tsf_persistent_storage.sh @@ -64,10 +64,14 @@ expect_fail "cpuTsfTrigger.enabled=true requires tsfPersistentStorage.enabled=tr for v in 0 101 -5 abc 50.5; do expect_fail "memoryTsfTrigger.percent must be an integer from 1 to 100" "${ON[@]}" "${MEM[@]}" --set $P.memoryTsfTrigger.percent=$v done -for v in 0 -1 abc; do - expect_fail "cpuTsfTrigger.seconds must be a positive integer" "${ON[@]}" "${CPU[@]}" --set $P.cpuTsfTrigger.seconds=$v +for v in 0 -1 abc 1.5 86401 1000000; do + expect_fail "cpuTsfTrigger.seconds must be an integer from 1 to 86400" "${ON[@]}" "${CPU[@]}" --set $P.cpuTsfTrigger.seconds=$v done +# A values file parses numbers as float64 (toString gives 1e+06); the rendered digits are in helm unittest. +printf '%s:\n cpuTsfTrigger:\n seconds: 1000000\n' "$P" > "$TMP/huge.yaml" +expect_fail "cpuTsfTrigger.seconds must be an integer from 1 to 86400" "${ON[@]}" "${CPU[@]}" -f "$TMP/huge.yaml" + # A disabled trigger with bad values is ignored. render "$CHART" "${ON[@]}" --set $P.cpuTsfTrigger.percent=0 >/dev/null \ && ok "disabled trigger values are not validated" || bad "disabled trigger values are validated" From e7c25cbe501b5d7db60e2795739410aa019a1f92 Mon Sep 17 00:00:00 2001 From: Ran Gerson Date: Wed, 11 Nov 2026 08:57:11 +0000 Subject: [PATCH 6/6] linux-ll: tsf-persistent: Set trigger defaults to memory 65%, CPU 90% (CPATR-38659) --- charts/cortex-agent/README.md | 4 ++-- charts/cortex-agent/tests/tsf_persistent_storage_test.yaml | 6 +++--- charts/cortex-agent/values.yaml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/cortex-agent/README.md b/charts/cortex-agent/README.md index 5550009..6eb8067 100644 --- a/charts/cortex-agent/README.md +++ b/charts/cortex-agent/README.md @@ -116,8 +116,8 @@ Even when using `--reuse-values` (which uses the values of the previous installa | `tsfPersistentStorage.pvcName` | Existing ReadWriteMany PVC in `namespace.name` (required when enabled). Mounted only if Bound and RWX at install/upgrade time, otherwise skipped with a WARNING; always skipped by `helm template` | Since 1.12.0 | `tsfPersistentStorage.path` | Relative folder in the volume, e.g. `cortex/prod-cluster-1`; each node writes to `/` | Since 1.12.0 | `tsfPersistentStorage.autoTsfIntervalHours` | Minimum hours between automatic TSFs of any type, 1 to 168; empty means 24 | Since 1.12.0 -| `tsfPersistentStorage.memoryTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent memory stays at or above `percent` (default 90) of its limit for `seconds` (default 300, 1 to 86400). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 -| `tsfPersistentStorage.cpuTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent CPU stays at or above `percent` (default 65) of its CPU limit for `seconds` (default 300, 1 to 86400). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 +| `tsfPersistentStorage.memoryTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent memory stays at or above `percent` (default 65) of its limit for `seconds` (default 300, 1 to 86400). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 +| `tsfPersistentStorage.cpuTsfTrigger.enabled` / `.percent` / `.seconds` | Collect a TSF when agent CPU stays at or above `percent` (default 90) of its CPU limit for `seconds` (default 300, 1 to 86400). Requires `tsfPersistentStorage.enabled` | Since 1.12.0 Note: Helm requires commas in arguments to be escaped. diff --git a/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml b/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml index 14ebf55..f640c41 100644 --- a/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml +++ b/charts/cortex-agent/tests/tsf_persistent_storage_test.yaml @@ -91,7 +91,7 @@ tests: content: persistentVolumeClaim template: templates/openshift-scc.yaml - - it: memory trigger only, with defaults 90/300 + - it: memory trigger only, with defaults 65/300 set: <<: *storageOn tsfPersistentStorage.memoryTsfTrigger.enabled: true @@ -99,7 +99,7 @@ tests: asserts: - contains: path: spec.template.spec.containers[0].env - content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT, value: "90" } + content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT, value: "65" } - contains: path: spec.template.spec.containers[0].env content: { name: XDR_TSF_MEMORY_TRIGGER_SECONDS, value: "300" } @@ -161,7 +161,7 @@ tests: content: { name: XDR_TSF_MEMORY_TRIGGER_PERCENT, value: "100" } - contains: path: spec.template.spec.containers[0].env - content: { name: XDR_TSF_CPU_TRIGGER_PERCENT, value: "65" } + content: { name: XDR_TSF_CPU_TRIGGER_PERCENT, value: "90" } - notContains: path: spec.template.spec.containers[0].env content: { name: XDR_TSF_AUTO_INTERVAL_SECONDS } diff --git a/charts/cortex-agent/values.yaml b/charts/cortex-agent/values.yaml index f93ca3e..d65d768 100644 --- a/charts/cortex-agent/values.yaml +++ b/charts/cortex-agent/values.yaml @@ -155,11 +155,11 @@ tsfPersistentStorage: # Requires tsfPersistentStorage.enabled=true. memoryTsfTrigger: enabled: false - percent: 90 + percent: 65 seconds: 300 # Collect a TSF when agent CPU stays at or above `percent` of its CPU limit for `seconds`. # Requires tsfPersistentStorage.enabled=true. cpuTsfTrigger: enabled: false - percent: 65 + percent: 90 seconds: 300