#ifndef DCGM_ERRORS_H

#define DCGM_ERRORS_H

#include "dcgm_api_export.h"

#include "dcgm_structs.h"

typedef enum dcgmError_enum

} dcgmError_t;

typedef enum dcgmErrorSeverity_enum

} dcgmErrorSeverity_t;

typedef enum dcgmErrorCategory_enum

} dcgmErrorCategory_t;

typedef struct

{

dcgmError_t errorId;

const char *msgFormat;

const char *suggestion;

int severity;

int category;

} dcgm_error_meta_t;

extern dcgm_error_meta_t dcgmErrorMeta[];

#define TRIAGE_RUN_FIELD_DIAG_MSG "Run a field diagnostic on the GPU."

#define DEBUG_COOLING_MSG \

"Verify that the cooling on this machine is functional, including external, " \

"thermal material interface, fans, and any other components."

#define BUG_REPORT_MSG "Please capture an nvidia-bug-report and send it to NVIDIA."

#define SYSTEM_TRIAGE_MSG "Check DCGM and system logs for errors. Reset GPU. Restart DCGM. Rerun diagnostics."

#define CONFIG_MSG "Check DCGM and system configuration. This error may be eliminated with an updated configuration."

#define DCGM_FR_OK_MSG "The operation completed successfully."

#define DCGM_FR_UNKNOWN_MSG "Unknown error."

#define DCGM_FR_UNRECOGNIZED_MSG "Unrecognized error code."

#define DCGM_FR_PCI_REPLAY_RATE_MSG "Detected more than %u PCIe replays per minute for GPU %u : %d"

#define DCGM_FR_VOLATILE_DBE_DETECTED_MSG "Detected %d volatile double-bit ECC error(s) in GPU %u."

#define DCGM_FR_VOLATILE_SBE_DETECTED_MSG "More than %u single-bit ECC error(s) detected in GPU %u Volatile SBEs: %lld"

#define DCGM_FR_PENDING_PAGE_RETIREMENTS_MSG "A pending retired page has been detected in GPU %u."

#define DCGM_FR_RETIRED_PAGES_LIMIT_MSG "%u or more retired pages have been detected in GPU %u. "

#define DCGM_FR_RETIRED_PAGES_DBE_LIMIT_MSG \

#define DCGM_FR_CORRUPT_INFOROM_MSG "A corrupt InfoROM has been detected in GPU %u."

#define DCGM_FR_CLOCKS_EVENT_THERMAL_MSG "Detected clocks event due to thermal violation in GPU %u."

#define DCGM_FR_POWER_UNREADABLE_MSG "Cannot reliably read the power usage for GPU %u."

#define DCGM_FR_CLOCKS_EVENT_POWER_MSG "Detected clocks event due to power violation in GPU %u."

#define DCGM_FR_NVLINK_ERROR_THRESHOLD_MSG \

"Detected %ld %s NvLink errors on GPU %u's NVLink which exceeds " \

"threshold of %u"

#define DCGM_FR_NVLINK_DOWN_MSG "GPU %u's NvLink link %d is currently down"

#define DCGM_FR_GPU_EXPECTED_NVLINKS_UP_MSG "Only %u NvLinks are up out of the expected %u"

#define DCGM_FR_NVSWITCH_EXPECTED_NVLINKS_UP_MSG "NvSwitch %u - Only %u NvLinks are up out of the expected %u"

#define DCGM_FR_NVSWITCH_FATAL_ERROR_MSG "Detected fatal errors on NvSwitch %u link %u"

#define DCGM_FR_NVSWITCH_NON_FATAL_ERROR_MSG "Detected nonfatal errors on NvSwitch %u link %u"

#define DCGM_FR_NVSWITCH_DOWN_MSG "NvSwitch physical ID %u's NvLink port %d is currently down."

#define DCGM_FR_NO_ACCESS_TO_FILE_MSG "File %s could not be accessed directly: %s"

#define DCGM_FR_NVML_API_MSG "Error calling NVML API %s: %s"

#define DCGM_FR_DEVICE_COUNT_MISMATCH_MSG \

"The number of devices NVML returns is different than the number " \

"of devices in /dev."

#define DCGM_FR_BAD_PARAMETER_MSG "Bad parameter to function %s cannot be processed"

#define DCGM_FR_CANNOT_OPEN_LIB_MSG "Cannot open library %s: '%s'"

#define DCGM_FR_DENYLISTED_DRIVER_MSG "Found driver on the denylist: %s"

#define DCGM_FR_NVML_LIB_BAD_MSG "Cannot get pointer to %s from libnvidia-ml.so"

#define DCGM_FR_SRAM_THRESHOLD_MSG "SRAM Threshold Count exceeded on GPU %d: %ld"

#define DCGM_FR_GRAPHICS_PROCESSES_MSG \

"NVVS has detected processes with graphics contexts open running on at least one " \

"GPU. This may cause some tests to fail."

#define DCGM_FR_HOSTENGINE_CONN_MSG "Could not connect to the host engine: '%s'"

#define DCGM_FR_FIELD_QUERY_MSG "Could not query field %s for GPU %u"

#define DCGM_FR_BAD_CUDA_ENV_MSG "Found CUDA performance-limiting environment variable '%s'."

#define DCGM_FR_PERSISTENCE_MODE_MSG "Persistence mode for GPU %u is disabled."

#define DCGM_FR_LOW_BANDWIDTH_MSG \

"Bandwidth of GPU %u in direction %s of %.2f did not exceed " \

"minimum required bandwidth of %.2f."

#define DCGM_FR_HIGH_LATENCY_MSG \

"Latency type %s of GPU %u value %.2f exceeded maximum allowed " \

"latency of %.2f."

#define DCGM_FR_CANNOT_GET_FIELD_TAG_MSG "Unable to get field information for field id %hu"

#define DCGM_FR_FIELD_VIOLATION_MSG "Detected %ld %s for GPU %u"

#define DCGM_FR_FIELD_THRESHOLD_MSG "Detected %ld %s for GPU %u which is above the threshold %ld"

#define DCGM_FR_FIELD_VIOLATION_DBL_MSG "Detected %.1f %s for GPU %u"

#define DCGM_FR_FIELD_THRESHOLD_DBL_MSG "Detected %.1f %s for %s:%u which is above the threshold %.1f"

#define DCGM_FR_UNSUPPORTED_FIELD_TYPE_MSG \

"Field %s is not supported by this API because it is neither an " \

"int64 nor a double type."

#define DCGM_FR_FIELD_THRESHOLD_TS_MSG \

"%s met or exceeded the threshold of %lu per second: %lu at " \

"%.1f seconds into the test."

#define DCGM_FR_FIELD_THRESHOLD_TS_DBL_MSG \

"%s met or exceeded the threshold of %.1f per second: %.1f at " \

"%.1f seconds into the test."

#define DCGM_FR_THERMAL_VIOLATIONS_MSG "There were thermal violations totaling %.1f seconds for GPU %u"

#define DCGM_FR_THERMAL_VIOLATIONS_TS_MSG \

"Thermal violations totaling %.1f seconds started at %.1f seconds " \

"into the test for GPU %u"

#define DCGM_FR_TEMP_VIOLATION_MSG \

"Temperature %lld of %s %u exceeded user-specified maximum " \

"allowed temperature %lld"

#define DCGM_FR_CLOCKS_EVENT_VIOLATION_MSG \

"Clocks event for GPU %u because of clocks event " \

"starting %.1f seconds into the test. %s"

#define DCGM_FR_INTERNAL_MSG "There was an internal error during the test: '%s'"

#define DCGM_FR_PCIE_GENERATION_MSG \

"GPU %u is running at PCI link generation %d, which is below " \

"the minimum allowed link generation of %d (parameter '%s')"

#define DCGM_FR_PCIE_WIDTH_MSG \

"GPU %u is running at PCI link width %dX, which is below the " \

"minimum allowed link generation of %d (parameter '%s')"

#define DCGM_FR_ABORTED_MSG "Test was aborted early due to user signal"

#define DCGM_FR_TEST_DISABLED_MSG "The %s test is skipped."

#define DCGM_FR_CANNOT_GET_STAT_MSG "Unable to generate / collect stat %s for GPU %u"

#define DCGM_FR_STRESS_LEVEL_MSG \

"Max stress level of %.1f did not reach desired stress level of " \

"%.1f for GPU %u"

#define DCGM_FR_CUDA_API_MSG "Error using CUDA API %s"

#define DCGM_FR_FAULTY_MEMORY_MSG "Found %lld faulty memory elements on GPU %u"

#define DCGM_FR_CANNOT_SET_WATCHES_MSG "Unable to add field watches to DCGM: %s"

#define DCGM_FR_CUDA_UNBOUND_MSG "Cuda GPU %d is no longer bound to a CUDA context...Aborting"

#define DCGM_FR_ECC_DISABLED_MSG "Skipping test %s because ECC is not enabled on GPU %u"

#define DCGM_FR_MEMORY_ALLOC_MSG "Couldn't allocate at least %.1f%% of GPU memory on GPU %u"

#define DCGM_FR_CUDA_DBE_MSG \

"CUDA APIs have indicated that a double-bit ECC error has " \

"occured on GPU %u."

#define DCGM_FR_MEMORY_MISMATCH_MSG \

"A memory mismatch was detected on GPU %u, but no error was " \

"reported by CUDA or NVML."

#define DCGM_FR_CUDA_DEVICE_MSG "Unable to find a corresponding CUDA device for GPU %u: '%s'"

#define DCGM_FR_ECC_UNSUPPORTED_MSG "ECC Memory is not turned on or is unsupported. Skipping test."

#define DCGM_FR_ECC_PENDING_MSG "ECC memory for GPU %u is in a pending state."

#define DCGM_FR_MEMORY_BANDWIDTH_MSG \

"GPU %u only achieved a memory bandwidth of %.2f GB/s, failing " \

"to meet %.2f GB/s for test %d"

#define DCGM_FR_TARGET_POWER_MSG \

"Max power of %.1f did not reach desired power minimum %s of " \

"%.1f for GPU %u"

#define DCGM_FR_API_FAIL_MSG "API call %s failed: '%s'"

#define DCGM_FR_API_FAIL_GPU_MSG "API call %s failed for GPU %u: '%s'"

#define DCGM_FR_CUDA_CONTEXT_MSG "GPU %u failed to create a CUDA context: %s"

#define DCGM_FR_DCGM_API_MSG "Error using DCGM API %s"

#define DCGM_FR_CONCURRENT_GPUS_MSG \

"Unable to run concurrent pair bandwidth test without 2 or more " \

"gpus. Skipping"

#define DCGM_FR_TOO_MANY_ERRORS_MSG \

#define DCGM_FR_NVLINK_CRC_ERROR_THRESHOLD_MSG \

"%.1f %s NvLink errors found occuring per second on GPU %u, " \

"exceeding the limit of 100 per second."

#define DCGM_FR_NVLINK_ERROR_CRITICAL_MSG "Detected %ld %s NvLink errors on GPU %u's NVLink (should be 0)"

#define DCGM_FR_NVLINK_EFFECTIVE_BER_THRESHOLD_MSG \

"Detected effective BER %.2e exceeds minimum threshold on GPU %u's NVLink."

#define DCGM_FR_NVLINK_SYMBOL_BER_THRESHOLD_MSG "Detected symbol BER %.2e exceeds minimum threshold on GPU %u's NVLink."

#define DCGM_FR_ENFORCED_POWER_LIMIT_MSG \

"Enforced power limit on GPU %u set to %.1f, which is too low to " \

"attempt to achieve target power %.1f"

#define DCGM_FR_MEMORY_ALLOC_HOST_MSG "Cannot allocate %zu bytes on the host"

#define DCGM_FR_GPU_OP_MODE_MSG "Skipping plugin due to a GPU being in GPU Operating Mode: LOW_DP."

#define DCGM_FR_NO_MEMORY_CLOCKS_MSG "No memory clocks <= %u MHZ were found in %u supported memory clocks."

#define DCGM_FR_NO_GRAPHICS_CLOCKS_MSG \

"No graphics clocks <= %u MHZ were found in %u supported graphics clocks for memory clock %u MHZ."

#define DCGM_FR_HAD_TO_RESTORE_STATE_MSG "Had to restore GPU state on NVML GPU(s): %s"

#define DCGM_FR_L1TAG_UNSUPPORTED_MSG "This card does not support the L1 cache test. Skipping test."

#define DCGM_FR_L1TAG_MISCOMPARE_MSG "Detected a miscompare failure in the L1 cache."

#define DCGM_FR_ROW_REMAP_FAILURE_MSG "GPU %u had uncorrectable memory errors and row remapping failed."

#define DCGM_FR_UNCONTAINED_ERROR_MSG "GPU had an uncontained error (XID 95)"

#define DCGM_FR_CONTAINED_ERROR_MSG "GPU had a contained error."

#define DCGM_FR_EMPTY_GPU_LIST_MSG "No valid GPUs passed to plugin"

#define DCGM_FR_DBE_PENDING_PAGE_RETIREMENTS_MSG "Pending page retirements together with a DBE were detected on GPU %u."

#define DCGM_FR_UNCORRECTABLE_ROW_REMAP_MSG "GPU %u had uncorrectable memory errors and %u rows were remapped"

#define DCGM_FR_UNCORRECTABLE_ROW_REMAP_LIMIT_MSG "GPU %u exceeded uncorrectable row remap limit with %u remapped rows"

#define DCGM_FR_PENDING_ROW_REMAP_MSG "GPU %u had memory errors and row remappings are pending"

#define DCGM_FR_BROKEN_P2P_MEMORY_DEVICE_MSG "GPU %u was unsuccessfully written to by GPU %u in a peer-to-peer test: %s"

#define DCGM_FR_BROKEN_P2P_WRITER_DEVICE_MSG "GPU %u unsuccessfully wrote data to GPU %u in a peer-to-peer test: %s"

#define DCGM_FR_BROKEN_P2P_PCIE_MEMORY_DEVICE_MSG \

"GPU %u was unsuccessfully written to by GPU %u over PCIe in a peer-to-peer test: %s"

#define DCGM_FR_BROKEN_P2P_PCIE_WRITER_DEVICE_MSG \

"GPU %u unsuccessfully wrote data to GPU %u over PCIe in a peer-to-peer test: %s"

#define DCGM_FR_BROKEN_P2P_NVLINK_MEMORY_DEVICE_MSG \

"GPU %u was unsuccessfully written to by GPU %u over NVLink in a peer-to-peer test: %s"

#define DCGM_FR_BROKEN_P2P_NVLINK_WRITER_DEVICE_MSG \

"GPU %u unsuccessfully wrote data to GPU %u over NVLink in a peer-to-peer test: %s"

#define DCGM_FR_NVSWITCH_NVLINK_DOWN_MSG "NVSwitch %u's NvLink %u is down."

#define DCGM_FR_EUD_BINARY_PERMISSIONS_MSG "" /* See message inplace */

#define DCGM_FR_EUD_NON_ROOT_USER_MSG "" /* See message inplace */

#define DCGM_FR_EUD_SPAWN_FAILURE_MSG "" /* See message inplace */

#define DCGM_FR_EUD_TIMEOUT_MSG "" /* See message inplace */

#define DCGM_FR_EUD_ZOMBIE_MSG "" /* See message inplace */

#define DCGM_FR_EUD_NON_ZERO_EXIT_CODE_MSG "" /* See message inplace */

#define DCGM_FR_EUD_TEST_FAILED_MSG "" /* See message inplace */

#define DCGM_FR_FILE_CREATE_PERMISSIONS_MSG \

"The DCGM Diagnostic does not have permissions to create a file in directory '%s'"

#define DCGM_FR_PAUSE_RESUME_FAILED_MSG "" /* See message inplace */

#define DCGM_FR_PCIE_H_REPLAY_VIOLATION_MSG "GPU %u host-side PCIe replay violation, see dmesg for more information"

#define DCGM_FR_XID_ERROR_MSG "Detected XID %u for GPU %u"

#define DCGM_FR_SBE_VIOLATION_MSG "Detected %ld %s for GPU %u"

#define DCGM_FR_DBE_VIOLATION_MSG "Detected %ld %s for GPU %u"

#define DCGM_FR_PCIE_REPLAY_VIOLATION_MSG "Detected %ld %s for GPU %u"

#define DCGM_FR_SBE_THRESHOLD_VIOLATION_MSG "Detected %ld %s for GPU %u which is above the threshold %ld"

#define DCGM_FR_DBE_THRESHOLD_VIOLATION_MSG "Detected %ld %s for GPU %u which is above the threshold %ld"

#define DCGM_FR_PCIE_REPLAY_THRESHOLD_VIOLATION_MSG "Detected %ld %s for GPU %u which is above the threshold %ld"

#define DCGM_FR_CUDA_FM_NOT_INITIALIZED_MSG ""

#define DCGM_FR_SXID_ERROR_MSG "Detected fatal NvSwitch SXID %u"

#define DCGM_FR_GFLOPS_THRESHOLD_VIOLATION_MSG "Detected %.2f %s for GPU %u which is below the threshold %.2f"

#define DCGM_FR_NAN_VALUE_MSG "Found %lld NaN-value memory elements on GPU %u"

#define DCGM_FR_FABRIC_MANAGER_TRAINING_ERROR_MSG \

"Fabric Manager (Cluster UUID: %s, Clique ID: %ld, Health Mask: %#lx): %s."

#define DCGM_FR_TEST_SKIPPED_MSG "Test %s was skipped."

#define DCGM_FR_FALLEN_OFF_BUS_MSG "GPU %d has fallen off the bus"

#define DCGM_FR_IMEX_UNHEALTHY_MSG "IMEX %s status is %s (%s)"

#define DCGM_FR_FABRIC_PROBE_STATE_MSG "GPU %u: Fabric State is %s (%lld)."

#define DCGM_FR_BINARY_PERMISSIONS_MSG "" /* See message inplace */

#define DCGM_FR_GPU_RECOVERY_RESET_MSG \

"GPU %u requires a reset to recover from a fault. Recovery action: %ld (GPU_RESET)."

#define DCGM_FR_GPU_RECOVERY_REBOOT_MSG \

"GPU %u fault may have left the OS in an inconsistent state. Recovery action: %ld (NODE_REBOOT)."

#define DCGM_FR_GPU_RECOVERY_DRAIN_P2P_MSG \

"GPU %u requires peer-to-peer traffic to be quiesced. Recovery action: %ld (DRAIN_P2P)."

#define DCGM_FR_GPU_RECOVERY_DRAIN_RESET_MSG \

"GPU %u operating at reduced capacity due to a fault. Recovery action: %ld (DRAIN_AND_RESET)."

#define DCGM_FR_NCCL_ERROR_MSG "Detected NCCL error: %s Recovery action: %ld (DRAIN_AND_RESET)."

#define DCGM_FR_RETEST_REQUESTED_MSG "" /* See message inplace */

#define DCGM_FR_ERROR_SENTINEL_MSG "" /* See message inplace */

#define DCGM_FR_OK_NEXT "N/A"

#define DCGM_FR_UNKNOWN_NEXT ""

#define DCGM_FR_UNRECOGNIZED_NEXT ""

#define DCGM_FR_PCI_REPLAY_RATE_NEXT \

#define DCGM_FR_VOLATILE_DBE_DETECTED_NEXT "Drain the GPU and reset it or reboot the node."

#define DCGM_FR_VOLATILE_SBE_DETECTED_NEXT "Monitor - this GPU can still perform workload."

#define DCGM_FR_PENDING_PAGE_RETIREMENTS_NEXT "Monitor - this GPU can still perform workload"

#define DCGM_FR_RETIRED_PAGES_LIMIT_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_RETIRED_PAGES_DBE_LIMIT_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_CORRUPT_INFOROM_NEXT "Flash the InfoROM to clear this corruption."

#define DCGM_FR_CLOCKS_EVENT_THERMAL_NEXT DEBUG_COOLING_MSG

#define DCGM_FR_POWER_UNREADABLE_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_CLOCKS_EVENT_POWER_NEXT "Monitor the power conditions. This GPU can still perform workload."

#define DCGM_FR_NVLINK_ERROR_THRESHOLD_NEXT "Monitor the NVLink. It can still perform workload."

#define DCGM_FR_NVLINK_DOWN_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_NVSWITCH_FATAL_ERROR_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_NVSWITCH_NON_FATAL_ERROR_NEXT "Monitor the NVSwitch. It can still perform workload."

#define DCGM_FR_NVSWITCH_DOWN_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_NO_ACCESS_TO_FILE_NEXT "Check relevant permissions, access, and existence of the file."

#define DCGM_FR_GPU_EXPECTED_NVLINKS_UP_NEXT \

"Ensure Fabric Manager is running. Check system logs, dmesg, and fabric-manager logs for more info."

#define DCGM_FR_NVSWITCH_EXPECTED_NVLINKS_UP_NEXT \

"Ensure Fabric Manager is running. Check system logs, dmesg, and fabric-manager logs for more info."

#define DCGM_FR_NVML_API_NEXT \

"Check the error condition and ensure that appropriate libraries " \

"are present and accessible."

#define DCGM_FR_DEVICE_COUNT_MISMATCH_NEXT \

"Check for the presence of cgroups, operating system blocks, and " \

"or unsupported / older cards"

#define DCGM_FR_BAD_PARAMETER_NEXT BUG_REPORT_MSG

#define DCGM_FR_CANNOT_OPEN_LIB_NEXT \

"Check for the existence of the library and set LD_LIBRARY_PATH " \

"if needed."

#define DCGM_FR_DENYLISTED_DRIVER_NEXT "Please load the appropriate driver."

#define DCGM_FR_NVML_LIB_BAD_NEXT \

"Make sure that the required version of libnvidia-ml.so " \

"is present and accessible on the system."

#define DCGM_FR_SRAM_THRESHOLD_NEXT "Check memory"

#define DCGM_FR_GRAPHICS_PROCESSES_NEXT \

"Stop the graphics processes or run this diagnostic on a server " \

"that is not being used for display purposes."

#define DCGM_FR_HOSTENGINE_CONN_NEXT \

"If hostengine is run separately, please ensure that it is up " \

"and responsive."

#define DCGM_FR_FIELD_QUERY_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_BAD_CUDA_ENV_NEXT "Please unset this environment variable to address test failures."

#define DCGM_FR_PERSISTENCE_MODE_NEXT \

"Enable persistence mode by running \"nvidia-smi -i <gpuId> -pm " \

"1 \" as root."

#define DCGM_FR_LOW_BANDWIDTH_NEXT \

#define DCGM_FR_HIGH_LATENCY_NEXT \

#define DCGM_FR_CANNOT_GET_FIELD_TAG_NEXT ""

#define DCGM_FR_FIELD_VIOLATION_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_FIELD_THRESHOLD_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_FIELD_VIOLATION_DBL_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_FIELD_THRESHOLD_DBL_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_UNSUPPORTED_FIELD_TYPE_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_FIELD_THRESHOLD_TS_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_FIELD_THRESHOLD_TS_DBL_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_THERMAL_VIOLATIONS_NEXT DEBUG_COOLING_MSG

#define DCGM_FR_THERMAL_VIOLATIONS_TS_NEXT DEBUG_COOLING_MSG

#define DCGM_FR_TEMP_VIOLATION_NEXT \

"Verify that the user-specified temperature maximum is set " \

"correctly. If it is, check the cooling for this GPU and node: " DEBUG_COOLING_MSG

#define DCGM_FR_CLOCKS_EVENT_VIOLATION_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_INTERNAL_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_PCIE_GENERATION_NEXT CONFIG_MSG

#define DCGM_FR_PCIE_WIDTH_NEXT CONFIG_MSG

#define DCGM_FR_ABORTED_NEXT ""

#define DCGM_FR_TEST_DISABLED_NEXT CONFIG_MSG

#define DCGM_FR_CANNOT_GET_STAT_NEXT \

"If running a standalone nv-hostengine, verify that it is up " \

"and responsive."

#define DCGM_FR_STRESS_LEVEL_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_CUDA_API_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_FAULTY_MEMORY_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_CANNOT_SET_WATCHES_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_CUDA_UNBOUND_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_ECC_DISABLED_NEXT \

"Enable ECC memory by running \"nvidia-smi -i <gpuId> -e 1\" " \

"to enable. This may require a GPU reset or reboot to take effect."

#define DCGM_FR_MEMORY_ALLOC_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_CUDA_DBE_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_MEMORY_MISMATCH_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_CUDA_DEVICE_NEXT \

#define DCGM_FR_ECC_UNSUPPORTED_NEXT CONFIG_MSG

#define DCGM_FR_ECC_PENDING_NEXT "Reboot to complete activation of the ECC memory."

#define DCGM_FR_MEMORY_BANDWIDTH_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_TARGET_POWER_NEXT "Verify that the clock speeds and GPU utilization are high."

#define DCGM_FR_API_FAIL_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_API_FAIL_GPU_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_CUDA_CONTEXT_NEXT \

"Please make sure the correct driver version is installed and " \

"verify that no conflicting libraries are present."

#define DCGM_FR_DCGM_API_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_CONCURRENT_GPUS_NEXT CONFIG_MSG

#define DCGM_FR_TOO_MANY_ERRORS_NEXT ""

#define DCGM_FR_NVLINK_CRC_ERROR_THRESHOLD_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_NVLINK_ERROR_CRITICAL_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_ENFORCED_POWER_LIMIT_NEXT \

#define DCGM_FR_MEMORY_ALLOC_HOST_NEXT "Manually kill processes or restart your machine."

#define DCGM_FR_GPU_OP_MODE_NEXT \

"Fix by running nvidia-smi as root with: nvidia-smi --gom=0 -i " \

"<gpu index>"

#define DCGM_FR_NO_MEMORY_CLOCKS_NEXT ""

#define DCGM_FR_NO_GRAPHICS_CLOCKS_NEXT ""

#define DCGM_FR_HAD_TO_RESTORE_STATE_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_L1TAG_UNSUPPORTED_NEXT CONFIG_MSG

#define DCGM_FR_L1TAG_MISCOMPARE_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_ROW_REMAP_FAILURE_NEXT \

"Row remapping failure indicates unrecoverable memory hardware damage. Reset the GPU or reboot the node immediately."

#define DCGM_FR_UNCONTAINED_ERROR_NEXT DCGM_FR_VOLATILE_DBE_DETECTED_NEXT

#define DCGM_FR_CONTAINED_ERROR_NEXT \

"Restart the application that encountered the error. Other applications on the GPU can continue running. GPU reset can be deferred until a convenient time."

#define DCGM_FR_DBE_PENDING_PAGE_RETIREMENTS_NEXT "Drain the GPU and reset it or reboot the node to resolve this issue."

#define DCGM_FR_EMPTY_GPU_LIST_NEXT CONFIG_MSG

#define DCGM_FR_UNCORRECTABLE_ROW_REMAP_NEXT ""

#define DCGM_FR_UNCORRECTABLE_ROW_REMAP_LIMIT_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_PENDING_ROW_REMAP_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_BROKEN_P2P_MEMORY_DEVICE_NEXT BUG_REPORT_MSG

#define DCGM_FR_BROKEN_P2P_WRITER_DEVICE_NEXT BUG_REPORT_MSG

#define DCGM_FR_BROKEN_P2P_PCIE_MEMORY_DEVICE_NEXT BUG_REPORT_MSG

#define DCGM_FR_BROKEN_P2P_PCIE_WRITER_DEVICE_NEXT BUG_REPORT_MSG

#define DCGM_FR_BROKEN_P2P_NVLINK_MEMORY_DEVICE_NEXT BUG_REPORT_MSG

#define DCGM_FR_BROKEN_P2P_NVLINK_WRITER_DEVICE_NEXT BUG_REPORT_MSG

#define DCGM_FR_NVSWITCH_NVLINK_DOWN_NEXT \

"Please check fabric manager and initialization logs to figure out why the link is down. " \

"You may also need to run a field diagnostic."

#define DCGM_FR_EUD_BINARY_PERMISSIONS_NEXT "" /* See message inplace */

#define DCGM_FR_EUD_NON_ROOT_USER_NEXT "" /* See message inplace */

#define DCGM_FR_EUD_SPAWN_FAILURE_NEXT "" /* See message inplace */

#define DCGM_FR_EUD_TIMEOUT_NEXT "" /* See message inplace */

#define DCGM_FR_EUD_ZOMBIE_NEXT "" /* See message inplace */

#define DCGM_FR_EUD_NON_ZERO_EXIT_CODE_NEXT "" /* See message inplace */

#define DCGM_FR_EUD_TEST_FAILED_NEXT "" /* See message inplace */

#define DCGM_FR_FILE_CREATE_PERMISSIONS_NEXT \

"Please restart the hostengine with parameter --home-dir to specify a different home directory for the " \

"diagnostic or change permissions in the current directory to allow the user to write files there."

#define DCGM_FR_PAUSE_RESUME_FAILED_NEXT "" /* See message inplace */

#define DCGM_FR_PCIE_H_REPLAY_VIOLATION_NEXT "" /* See message inplace */

#define DCGM_FR_XID_ERROR_NEXT "Please consult the documentation for details of this XID."

#define DCGM_FR_SBE_VIOLATION_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_DBE_VIOLATION_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_PCIE_REPLAY_VIOLATION_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_SBE_THRESHOLD_VIOLATION_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_DBE_THRESHOLD_VIOLATION_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_PCIE_REPLAY_THRESHOLD_VIOLATION_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_CUDA_FM_NOT_INITIALIZED_NEXT "Ensure that the FabricManager is running without errors."

#define DCGM_FR_SXID_ERROR_NEXT SYSTEM_TRIAGE_MSG

#define DCGM_FR_GFLOPS_THRESHOLD_VIOLATION_NEXT \

"Please verify your user-specified variance tolerance is set appropriately; " \

"if so, and if errors are persistent, please run a field diagnostic."

#define DCGM_FR_NAN_VALUE_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_FABRIC_MANAGER_TRAINING_ERROR_NEXT DCGM_FR_CUDA_FM_NOT_INITIALIZED_NEXT

#define DCGM_FR_TEST_SKIPPED_NEXT ""

#define DCGM_FR_NVLINK_EFFECTIVE_BER_THRESHOLD_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_NVLINK_SYMBOL_BER_THRESHOLD_NEXT TRIAGE_RUN_FIELD_DIAG_MSG

#define DCGM_FR_FALLEN_OFF_BUS_NEXT \

"Please re-seat the GPU, check for thermal and power issues, and verify that there is no outstanding bug against your driver or BIOS versions. If the issue persists, please run a field diagnostic on the GPU."

#define DCGM_FR_IMEX_UNHEALTHY_NEXT \

"Check IMEX installation, configuration, domain and daemon status, and network connectivity."

#define DCGM_FR_FABRIC_PROBE_STATE_NEXT DCGM_FR_CUDA_FM_NOT_INITIALIZED_NEXT

#define DCGM_FR_BINARY_PERMISSIONS_NEXT "" /* See message inplace */

#define DCGM_FR_GPU_RECOVERY_RESET_NEXT "Terminate all GPU processes and reset the GPU."

#define DCGM_FR_GPU_RECOVERY_REBOOT_NEXT "Reboot the operating system to restore a consistent state."

#define DCGM_FR_GPU_RECOVERY_DRAIN_P2P_NEXT \

"Terminate GPU processes conducting peer-to-peer traffic and disable UVM persistence mode. Check GPU health status again after draining."

#define DCGM_FR_GPU_RECOVERY_DRAIN_RESET_NEXT \

"Do not schedule new work on this GPU. Reset the GPU after existing work has drained."

#define DCGM_FR_NCCL_ERROR_NEXT "Attempt to reset the GPUs and reboot the machines if that fails."

#define DCGM_FR_RETEST_REQUESTED_NEXT "" /* See message inplace */

#define DCGM_FR_ERROR_SENTINEL_NEXT "" /* See message inplace */

#ifdef __cplusplus

extern "C" {

DCGM_PUBLIC_API dcgmErrorSeverity_t dcgmErrorGetPriorityByCode(unsigned int code);

DCGM_PUBLIC_API dcgmErrorCategory_t dcgmErrorGetCategoryByCode(unsigned int code);

DCGM_PUBLIC_API const char *dcgmErrorGetFormatMsgByCode(unsigned int code);

DCGM_PUBLIC_API const dcgm_error_meta_t *dcgmGetErrorMeta(dcgmError_t error);

DCGM_PUBLIC_API const char *errorString(dcgmReturn_t result);

#ifdef __cplusplus

} // extern "C"

#endif // DCGM_ERRORS_H