1# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
7# http://www.apache.org/licenses/LICENSE-2.0
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
16 repository: cgr.dev/scratch-images/test-tmp/dcgm-exporter
17 pullPolicy: IfNotPresent
18 # Image tag defaults to AppVersion, but you can use the tag key
19 # for the image tag, e.g:
20 tag: 4.6.0.4.8.3-r2@sha256:60219daea21f202548b9786fd571c99c75e936682d74606816c587d77848a244
21 # Optional digest pin. When set, the chart renders repository@digest and
22 # ignores tag for the image reference.
24# Add dcgm-exporter command line arguments here. When empty, the exporter uses
25# its built-in default collectors path.
27# NOTE: in general, add any command line arguments to arguments above and they
28# will be passed through.
29# Use "-r", "<HOST>:<PORT>" or a DCGM connection string to connect to an already running hostengine.
30# Connection string formats: "tcp://<HOST>:<PORT>", "unix:///<SOCKET_PATH>", or "vsock://<CID>:<PORT>".
31# Example arguments: ["-r", "host123:5555"]
32# Example VSOCK arguments: ["-r", "vsock://3:5555"]
33# Use "-n" to remove the hostname tag from the output.
34# Example arguments: ["-n"]
35# Use "-d" to specify the devices to monitor. -d must be followed by a string
36# in the following format: f, g[:numeric_range], i[:numeric_range], or
37# g[:numeric_range]+i[:numeric_range]. i+g ordering is also accepted.
38# The flex option f is the default and cannot be combined with g or i.
39# Where a numeric range is something like 0-4 or 0,2,4, etc.
40# Example arguments: ["-d", "f"] to monitor all GPUs or GPU instances, or
41# ["-d", "g:0-3"] to monitor GPUs 0-3, or ["-d", "i"] to monitor GPU instances.
42# Use "-m" to specify the namespace and name of a configmap containing
43# the watched exporter fields.
44# Example arguments: ["-m", "default:exporter-metrics-config-map"]
45# If adding "--enable-pprof", also enable tlsServerConfig and/or basicAuth.
46# Pprof requires a web config file so /debug/pprof/ is protected by
47# exporter-toolkit authentication or TLS.
49# Optional dcgm-exporter YAML configuration. YAML is read at exporter startup.
50# Changes to the YAML file require restarting the exporter pod.
53 # When true, the chart creates a ConfigMap from config.data. When false,
54 # config.name must reference an existing ConfigMap.
56 # Defaults to "<release-name>-dcgm-exporter-config" when empty.
59 mountPath: /etc/dcgm-exporter/config.yaml
63 file: /etc/dcgm-exporter/default-counters.csv
66# Image pull secrets for container images
68# Overrides the chart's name
70# Overrides the chart's computed fullname
72# Overrides the deployment namespace
74# hostPID allows the DCGM-Exporter container to see processes on the host node
77# Defines the runtime class that will be used by the pod
79# Defines the priority class that will be used by the pod
80priorityClassName: "system-node-critical"
81# Enable or disable host network
83# Defines serviceAccount names for components.
85 # Specifies whether a service account should be created
87 # Annotations to add to the service account
89 # The name of the service account to use.
90 # If not set and create is true, a name is generated using the fullname template
92 # Whether to mount the SA token on the DaemonSet pod and the ServiceAccount.
93 # If not set, derived from kubernetes.enablePodLabels/enablePodUID with rbac.create, or kubernetesDRA.enabled.
94 # Setting this to false disables in-cluster Kubernetes API authentication for those features.
95 automountServiceAccountToken:
97 # Specifies maximum number of DaemonSet pods that can be unavailable during the update
99 # Specifies maximum number of nodes with an existing available DaemonSet pod that can have an updated DaemonSet pod during during an update
101# Labels to be added to dcgm-exporter pods
103# Annotations to be added to dcgm-exporter pods
105# Using this annotation which is required for prometheus scraping
106# prometheus.io/scrape: "true"
107# prometheus.io/port: "9400"
109# The SecurityContext for the dcgm-exporter pods
110podSecurityContext: {}
113# The SecurityContext for the dcgm-exporter containers
118 add: ["SYS_ADMIN"] # Required for profiling metrics (DCGM_FI_PROF_*)
120 allowPrivilegeEscalation: false
121 # readOnlyRootFilesystem: true # Enable if tmpfs volumes are configured
122 # Note: For non-root without profiling metrics, use:
123 # runAsNonRoot: true, runAsUser: 1000, and remove SYS_ADMIN from capabilities.add
124# Defines the dcgm-exporter service
126 # When enabled, the helm chart will create service
129 # Accepts either "Cluster" or "Local", choose Local if you want to route internal traffic within the node only
130 internalTrafficPolicy: Cluster
134 # HTTP server read timeout for inbound scrape requests
136 # HTTP server write timeout for generating and returning scrape responses
138 # Annotations to add to the service
140# Allows to control pod resources
149 apiVersion: "monitoring.coreos.com/v1"
155 #monitoring: prometheus
157 # - sourceLabels: [__meta_kubernetes_pod_node_name]
160 # targetLabel: nodename
163 metricRelabelings: []
165 # regex: original_metrics_name
166 # replacement: new_metrics_name
169 # targetLabel: __name__
174 - key: node-role.kubernetes.io/control-plane
181# requiredDuringSchedulingIgnoredDuringExecution:
188#- name: host-binaries
191extraConfigMapVolumes:
192 - name: exporter-metrics-volume
194 name: exporter-metrics-config-map
197 path: default-counters.csv
199 - name: exporter-metrics-volume
200 mountPath: /etc/dcgm-exporter/default-counters.csv
201 subPath: default-counters.csv
203# Path to the kubelet socket for /pod-resources
204kubeletPath: "/var/lib/kubelet/pod-resources"
207 # Enable or disable HTTPS configuration
209 # Use autogenerated self-signed TLS certificates. Not recommended for production environments.
211 # Existing secret containing your own server key and certificate
213 # Certificate file name
214 certFilename: "tls.crt"
216 keyFilename: "tls.key"
217 # CA certificate file name
219 # Server policy for client authentication. Maps to ClientAuth Policies.
220 # For more detail on clientAuth options:
221 # https://golang.org/pkg/crypto/tls/#ClientAuthType
223 # NOTE: If you want to enable client authentication, you need to use
224 # RequireAndVerifyClientCert. Other values are insecure.
226 # TLS Key for HTTPS - ignored if existingSecret is provided
228 # TLS Certificate for HTTPS - ignored if existingSecret is provided
230 # CA Certificate for HTTPS - ignored if existingSecret is provided
233 #Object containing <user>:<passwords> key-value pairs for each user that will have access via basic authentication
235# Kubernetes integration settings
237 # Enable Kubernetes pod labels in metrics
238 # When enabled, metrics will include labels from the pods that are using the GPUs
239 # This requires cluster-level read permissions to pods
240 enablePodLabels: false
241 # Enable Kubernetes pod UID in metrics
242 # When enabled, metrics will include the pod UID as an attribute for the pods that are using the GPUs
243 # This requires cluster-level read permissions to pods
245 # Pod label filtering configuration
246 # Filter which pod labels are included in metrics using regex patterns
247 # Empty list means all labels are included (default behavior)
249 # podLabelAllowlistRegex:
250 # - "^app$" # Exact match for "app" label
251 # - "^app\\.kubernetes\\.io/.*" # All labels starting with app.kubernetes.io/
252 # - "^(tier|environment|version)$" # Match tier, environment, or version labels
253 podLabelAllowlistRegex: []
254 # RBAC settings for Kubernetes integration
256 # Automatically creates ClusterRole and ClusterRoleBinding for pod access when enablePodLabels or enablePodUID is true
257 # Set to false if you want to manage RBAC resources manually
259# Runtime object debug dump configuration for troubleshooting
261 # Enable file-based debugging dumps for troubleshooting
263 # Directory to store debug dump files
264 directory: "/tmp/dcgm-exporter-debug"
265 # Retention period for debug dump files in hours (0 = no cleanup)
267 # Use gzip compression for debug dump files
269# Enable DRA to capture metrics associated with GPUs managed by NVIDIA k8s-dra-driver (https://github.com/NVIDIA/k8s-dra-driver-gpu)
273# Customized list of metrics to emit. Expected to be in the same format (CSV) as the default list.
274# Must be the complete list and is not additive. If unset, the default list will take effect.
277# If line starts with a '#' it is considered a comment
278# DCGM FIELD, Prometheus metric type, help message
281# DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz).
282# DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).
285# DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
286# DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C).
289# DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W).
290# DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).
293# DCGM_FI_DEV_PCIE_TX_THROUGHPUT, counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML.
294# DCGM_FI_DEV_PCIE_RX_THROUGHPUT, counter, Total number of bytes received through PCIe RX (in KB) via NVML.
295# DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.
297# Utilization (the sample period varies depending on the product)
298# DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %).
299# DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
300# DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %).
301# DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %).
303# Errors and violations
304# DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered.
305# DCGM_EXP_CLOCK_EVENTS_TOTAL, counter, cumulative clock events observed since exporter start (edge-counted)
306# DCGM_EXP_XID_ERRORS_TOTAL, counter, cumulative XID errors observed since exporter start
307# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in ns).
308# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in ns).
309# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in ns).
310# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in ns).
311# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in ns).
312# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in ns).
315# DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB).
316# DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB).
319# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.
320# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.
321# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.
322# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.
325# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors.
326# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors.
327# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.
330# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors.
331# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors.
332# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries.
333# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors.
334# DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, gauge, Total number of NVLink bandwidth counters for all lanes.
335# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload.
338# DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status
341# DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors
342# DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors
343# DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed
346# DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active.
347# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned.
348# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM.
349# DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active.
350# DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data.
351# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active.
352# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active.
353# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active.
354# DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second.
355# DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second.
356# DCGM_FI_PROF_SM_CYCLES_ELAPSED_TOTAL, counter, Total elapsed SM cycles.
357# DCGM_FI_PROF_SM_CYCLES_ACTIVE_TOTAL, counter, Total SM cycles with active warps.
358# DCGM_FI_PROF_MMA_CYCLES_ACTIVE_TOTAL, counter, Total MMA tensor cycles active.
359# DCGM_FI_PROF_DMMA_CYCLES_ACTIVE_TOTAL, counter, Total DMMA tensor cycles active.
360# DCGM_FI_PROF_HMMA_CYCLES_ACTIVE_TOTAL, counter, Total HMMA tensor cycles active.
361# DCGM_FI_PROF_IMMA_CYCLES_ACTIVE_TOTAL, counter, Total IMMA tensor cycles active.
362# DCGM_FI_PROF_DFMA_CYCLES_ACTIVE_TOTAL, counter, Total DFMA tensor cycles active.
363# DCGM_FI_PROF_PCIE_TX_BYTES_TOTAL, counter, Total PCIe transmitted bytes.
364# DCGM_FI_PROF_PCIE_RX_BYTES_TOTAL, counter, Total PCIe received bytes.
365# DCGM_FI_PROF_INT_CYCLES_ACTIVE_TOTAL, counter, Total integer pipe cycles active.
366# DCGM_FI_PROF_FP64_CYCLES_ACTIVE_TOTAL, counter, Total FP64 pipe cycles active.
367# DCGM_FI_PROF_FP32_CYCLES_ACTIVE_TOTAL, counter, Total FP32 pipe cycles active.
368# DCGM_FI_PROF_FP16_CYCLES_ACTIVE_TOTAL, counter, Total FP16 pipe cycles active.
370 initialDelaySeconds: 45
373 initialDelaySeconds: 45