github/tinygrad
1
1
Fork 0
mirror of https://github.com/tinygrad/tinygrad.git synced 2026-01-09 15:08:02 -05:00
Code Issues Packages Projects Releases Wiki Activity

nv print info about faults (#7057)

Browse Source 
* nv print info about faults

* unrelated changes

* nv_gpu.GT200_DEBUGGER in mockgpu

* regen with ocrrect version

* spacing
This commit is contained in:
nimlgen
2024-10-14 21:49:38 +03:00
committed by GitHub
parent fbaab30fe3
commit 8094340221
6 changed files with 809 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
autogen_stubs.sh
View File

@@ -117,11 +117,13 @@ generate_nv() {
$NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc56f.h \
$NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc56f.h \
$NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc56f.h \
$NVKERN_SRC/src/common/sdk/nvidia/inc/class/cl83de.h \
$NVKERN_SRC/src/nvidia/generated/g_allclasses.h \
$NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc6c0.h \
$NVKERN_SRC/kernel-open/nvidia-uvm/clc6b5.h \
$NVKERN_SRC/kernel-open/nvidia-uvm/uvm_ioctl.h \
$NVKERN_SRC/kernel-open/nvidia-uvm/uvm_linux_ioctl.h \
$NVKERN_SRC/kernel-open/nvidia-uvm/hwref/ampere/ga100/dev_fault.h \
$NVKERN_SRC/src/nvidia/arch/nvalloc/unix/include/nv_escape.h \
$NVKERN_SRC/src/nvidia/arch/nvalloc/unix/include/nv-ioctl.h \
$NVKERN_SRC/src/nvidia/arch/nvalloc/unix/include/nv-ioctl-numbers.h \

3
extra/mockgpu/nv/nvdriver.py
View File

@@ -129,6 +129,9 @@ class NVDriver(VirtDriver):
self.object_by_handle[struct.hObjectNew] = NVGPFIFO(gpu, gpfifo_token)
elif struct.hClass == nv_gpu.AMPERE_DMA_COPY_B or struct.hClass == nv_gpu.ADA_COMPUTE_A:
assert struct.hObjectParent in self.object_by_handle and isinstance(self.object_by_handle[struct.hObjectParent], NVGPFIFO)
struct.hObjectNew = self._alloc_handle()
elif struct.hClass == nv_gpu.GT200_DEBUGGER:
struct.hObjectNew = self._alloc_handle()
else: raise RuntimeError(f"Unknown {struct.hClass} to rm_alloc")
return 0

4
extra/nv_gpu_driver/nv_ioctl.py
View File

@@ -103,13 +103,15 @@ def ioctl(fd, request, argp):
# print(f"{(st-start)*1000:7.2f} ms +{et*1000.:7.2f} ms : {ret:2d} = {name:40s}", ' '.join(format_struct(s)))
elif nr == nv_gpu.NV_ESC_RM_ALLOC:
s = get_struct(argp, nv_gpu.NVOS21_PARAMETERS)
print(f"NV_ESC_RM_ALLOC hClass={nvclasses.get(s.hClass, 'unk'):30s}, hRoot={s.hRoot}, hObjectParent={s.hObjectParent}, pAllocParms={s.pAllocParms}, hObjectNew={s.hObjectNew}")
print(f"NV_ESC_RM_ALLOC hClass={nvclasses.get(s.hClass, f'unk=0x{s.hClass:X}'):30s}, hRoot={s.hRoot}, hObjectParent={s.hObjectParent}, pAllocParms={s.pAllocParms}, hObjectNew={s.hObjectNew} status={s.status}")
if s.pAllocParms is not None:
if s.hClass == nv_gpu.NV01_DEVICE_0: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV0080_ALLOC_PARAMETERS))
if s.hClass == nv_gpu.FERMI_VASPACE_A: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV_VASPACE_ALLOCATION_PARAMETERS))
if s.hClass == nv_gpu.NV50_MEMORY_VIRTUAL: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV_MEMORY_ALLOCATION_PARAMS))
if s.hClass == nv_gpu.NV1_MEMORY_USER: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV_MEMORY_ALLOCATION_PARAMS))
if s.hClass == nv_gpu.NV1_MEMORY_SYSTEM: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV_MEMORY_ALLOCATION_PARAMS))
# if s.hClass == nv_gpu.NV1_EVENT_OS_EVENT: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV0005_ALLOC_PARAMETERS))
if s.hClass == nv_gpu.GT200_DEBUGGER: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV83DE_ALLOC_PARAMETERS))
if s.hClass == nv_gpu.AMPERE_CHANNEL_GPFIFO_A:
sx = get_struct(s.pAllocParms, nv_gpu.NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS)
dump_struct(sx)

760
tinygrad/runtime/autogen/nv_gpu.py
View File

@@ -1820,6 +1820,19 @@ struct_Nvc56fControl_struct._fields_ = [
Nvc56fControl = struct_Nvc56fControl_struct
AmpereAControlGPFifo = struct_Nvc56fControl_struct
GT200_DEBUGGER = (0x000083de) # macro
NV83DE_ALLOC_PARAMETERS_MESSAGE_ID = (0x83de) # macro
class struct_NV83DE_ALLOC_PARAMETERS(Structure):
pass
struct_NV83DE_ALLOC_PARAMETERS._pack_ = 1 # source:False
struct_NV83DE_ALLOC_PARAMETERS._fields_ = [
('hDebuggerClient_Obsolete', ctypes.c_uint32),
('hAppClient', ctypes.c_uint32),
('hClass3dObject', ctypes.c_uint32),
]
NV83DE_ALLOC_PARAMETERS = struct_NV83DE_ALLOC_PARAMETERS
NV01_ROOT = (0x0) # macro
NV1_ROOT = (0x00000000) # macro
NV01_NULL_OBJECT = (0x00000000) # macro
@@ -1984,7 +1997,6 @@ HOPPER_A = (0x0000cb97) # macro
HOPPER_COMPUTE_A = (0x0000cbc0) # macro
NV40_DEBUG_BUFFER = (0x000000db) # macro
RM_USER_SHARED_DATA = (0x000000de) # macro
GT200_DEBUGGER = (0x000083de) # macro
NV40_I2C = (0x0000402c) # macro
KEPLER_DEVICE_VGPU = (0x0000a080) # macro
NVA081_VGPU_CONFIG = (0x0000a081) # macro
@@ -3824,6 +3836,459 @@ struct_c__SA_UVM_INITIALIZE_PARAMS._fields_ = [
]
UVM_INITIALIZE_PARAMS = struct_c__SA_UVM_INITIALIZE_PARAMS
__ga100_dev_fault_h__ = True # macro
NV_PFAULT = True # macro
NV_PFAULT_MMU_ENG_ID_GRAPHICS = 64 # macro
NV_PFAULT_MMU_ENG_ID_DISPLAY = 1 # macro
NV_PFAULT_MMU_ENG_ID_GSP = 2 # macro
NV_PFAULT_MMU_ENG_ID_IFB = 9 # macro
NV_PFAULT_MMU_ENG_ID_FLA = 4 # macro
NV_PFAULT_MMU_ENG_ID_BAR1 = 128 # macro
NV_PFAULT_MMU_ENG_ID_BAR2 = 192 # macro
NV_PFAULT_MMU_ENG_ID_SEC = 14 # macro
NV_PFAULT_MMU_ENG_ID_PERF = 8 # macro
NV_PFAULT_MMU_ENG_ID_NVDEC = 25 # macro
NV_PFAULT_MMU_ENG_ID_NVDEC0 = 25 # macro
NV_PFAULT_MMU_ENG_ID_NVDEC1 = 26 # macro
NV_PFAULT_MMU_ENG_ID_NVDEC2 = 27 # macro
NV_PFAULT_MMU_ENG_ID_NVDEC3 = 28 # macro
NV_PFAULT_MMU_ENG_ID_NVDEC4 = 29 # macro
NV_PFAULT_MMU_ENG_ID_NVJPG0 = 30 # macro
NV_PFAULT_MMU_ENG_ID_GRCOPY = 15 # macro
NV_PFAULT_MMU_ENG_ID_CE0 = 15 # macro
NV_PFAULT_MMU_ENG_ID_CE1 = 16 # macro
NV_PFAULT_MMU_ENG_ID_CE2 = 17 # macro
NV_PFAULT_MMU_ENG_ID_CE3 = 18 # macro
NV_PFAULT_MMU_ENG_ID_CE4 = 19 # macro
NV_PFAULT_MMU_ENG_ID_CE5 = 20 # macro
NV_PFAULT_MMU_ENG_ID_CE6 = 21 # macro
NV_PFAULT_MMU_ENG_ID_CE7 = 22 # macro
NV_PFAULT_MMU_ENG_ID_CE8 = 23 # macro
NV_PFAULT_MMU_ENG_ID_CE9 = 24 # macro
NV_PFAULT_MMU_ENG_ID_PWR_PMU = 6 # macro
NV_PFAULT_MMU_ENG_ID_PTP = 3 # macro
NV_PFAULT_MMU_ENG_ID_NVENC0 = 11 # macro
NV_PFAULT_MMU_ENG_ID_NVENC1 = 12 # macro
NV_PFAULT_MMU_ENG_ID_NVENC2 = 13 # macro
NV_PFAULT_MMU_ENG_ID_OFA0 = 10 # macro
NV_PFAULT_MMU_ENG_ID_PHYSICAL = 31 # macro
NV_PFAULT_MMU_ENG_ID_HOST0 = 32 # macro
NV_PFAULT_MMU_ENG_ID_HOST1 = 33 # macro
NV_PFAULT_MMU_ENG_ID_HOST2 = 34 # macro
NV_PFAULT_MMU_ENG_ID_HOST3 = 35 # macro
NV_PFAULT_MMU_ENG_ID_HOST4 = 36 # macro
NV_PFAULT_MMU_ENG_ID_HOST5 = 37 # macro
NV_PFAULT_MMU_ENG_ID_HOST6 = 38 # macro
NV_PFAULT_MMU_ENG_ID_HOST7 = 39 # macro
NV_PFAULT_MMU_ENG_ID_HOST8 = 40 # macro
NV_PFAULT_MMU_ENG_ID_HOST9 = 41 # macro
NV_PFAULT_MMU_ENG_ID_HOST10 = 42 # macro
NV_PFAULT_MMU_ENG_ID_HOST11 = 43 # macro
NV_PFAULT_MMU_ENG_ID_HOST12 = 44 # macro
NV_PFAULT_MMU_ENG_ID_HOST13 = 45 # macro
NV_PFAULT_MMU_ENG_ID_HOST14 = 46 # macro
NV_PFAULT_MMU_ENG_ID_HOST15 = 47 # macro
NV_PFAULT_MMU_ENG_ID_HOST16 = 48 # macro
NV_PFAULT_MMU_ENG_ID_HOST17 = 49 # macro
NV_PFAULT_MMU_ENG_ID_HOST18 = 50 # macro
NV_PFAULT_MMU_ENG_ID_HOST19 = 51 # macro
NV_PFAULT_MMU_ENG_ID_HOST20 = 52 # macro
NV_PFAULT_MMU_ENG_ID_HOST21 = 53 # macro
NV_PFAULT_MMU_ENG_ID_HOST22 = 54 # macro
NV_PFAULT_MMU_ENG_ID_HOST23 = 55 # macro
NV_PFAULT_MMU_ENG_ID_HOST24 = 56 # macro
NV_PFAULT_MMU_ENG_ID_HOST25 = 57 # macro
NV_PFAULT_MMU_ENG_ID_HOST26 = 58 # macro
NV_PFAULT_MMU_ENG_ID_HOST27 = 59 # macro
NV_PFAULT_MMU_ENG_ID_HOST28 = 60 # macro
NV_PFAULT_MMU_ENG_ID_HOST29 = 61 # macro
NV_PFAULT_MMU_ENG_ID_HOST30 = 62 # macro
NV_PFAULT_MMU_ENG_ID_HOST31 = 63 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN0 = 128 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN1 = 129 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN2 = 130 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN3 = 131 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN4 = 132 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN5 = 133 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN6 = 134 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN7 = 135 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN8 = 136 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN9 = 137 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN10 = 138 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN11 = 139 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN12 = 140 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN13 = 141 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN14 = 142 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN15 = 143 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN16 = 144 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN17 = 145 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN18 = 146 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN19 = 147 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN20 = 148 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN21 = 149 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN22 = 150 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN23 = 151 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN24 = 152 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN25 = 153 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN26 = 154 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN27 = 155 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN28 = 156 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN29 = 157 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN30 = 158 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN31 = 159 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN32 = 160 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN33 = 161 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN34 = 162 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN35 = 163 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN36 = 164 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN37 = 165 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN38 = 166 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN39 = 167 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN40 = 168 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN41 = 169 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN42 = 170 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN43 = 171 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN44 = 172 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN45 = 173 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN46 = 174 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN47 = 175 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN48 = 176 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN49 = 177 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN50 = 178 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN51 = 179 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN52 = 180 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN53 = 181 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN54 = 182 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN55 = 183 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN56 = 184 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN57 = 185 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN58 = 186 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN59 = 187 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN60 = 188 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN61 = 189 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN62 = 190 # macro
NV_PFAULT_MMU_ENG_ID_BAR1_FN63 = 191 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN0 = 192 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN1 = 193 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN2 = 194 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN3 = 195 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN4 = 196 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN5 = 197 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN6 = 198 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN7 = 199 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN8 = 200 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN9 = 201 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN10 = 202 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN11 = 203 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN12 = 204 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN13 = 205 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN14 = 206 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN15 = 207 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN16 = 208 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN17 = 209 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN18 = 210 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN19 = 211 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN20 = 212 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN21 = 213 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN22 = 214 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN23 = 215 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN24 = 216 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN25 = 217 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN26 = 218 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN27 = 219 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN28 = 220 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN29 = 221 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN30 = 222 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN31 = 223 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN32 = 224 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN33 = 225 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN34 = 226 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN35 = 227 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN36 = 228 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN37 = 229 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN38 = 230 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN39 = 231 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN40 = 232 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN41 = 233 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN42 = 234 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN43 = 235 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN44 = 236 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN45 = 237 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN46 = 238 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN47 = 239 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN48 = 240 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN49 = 241 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN50 = 242 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN51 = 243 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN52 = 244 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN53 = 245 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN54 = 246 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN55 = 247 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN56 = 248 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN57 = 249 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN58 = 250 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN59 = 251 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN60 = 252 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN61 = 253 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN62 = 254 # macro
NV_PFAULT_MMU_ENG_ID_BAR2_FN63 = 255 # macro
# NV_PFAULT_FAULT_TYPE = 4 : 0 # macro
NV_PFAULT_FAULT_TYPE_PDE = 0x00000000 # macro
NV_PFAULT_FAULT_TYPE_PDE_SIZE = 0x00000001 # macro
NV_PFAULT_FAULT_TYPE_PTE = 0x00000002 # macro
NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION = 0x00000003 # macro
NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK = 0x00000004 # macro
NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION = 0x00000005 # macro
NV_PFAULT_FAULT_TYPE_RO_VIOLATION = 0x00000006 # macro
NV_PFAULT_FAULT_TYPE_WO_VIOLATION = 0x00000007 # macro
NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION = 0x00000008 # macro
NV_PFAULT_FAULT_TYPE_WORK_CREATION = 0x00000009 # macro
NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE = 0x0000000a # macro
NV_PFAULT_FAULT_TYPE_COMPRESSION_FAILURE = 0x0000000b # macro
NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND = 0x0000000c # macro
NV_PFAULT_FAULT_TYPE_REGION_VIOLATION = 0x0000000d # macro
NV_PFAULT_FAULT_TYPE_POISONED = 0x0000000e # macro
NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION = 0x0000000f # macro
# NV_PFAULT_CLIENT = 14 : 8 # macro
NV_PFAULT_CLIENT_GPC_T1_0 = 0x00000000 # macro
NV_PFAULT_CLIENT_GPC_T1_1 = 0x00000001 # macro
NV_PFAULT_CLIENT_GPC_T1_2 = 0x00000002 # macro
NV_PFAULT_CLIENT_GPC_T1_3 = 0x00000003 # macro
NV_PFAULT_CLIENT_GPC_T1_4 = 0x00000004 # macro
NV_PFAULT_CLIENT_GPC_T1_5 = 0x00000005 # macro
NV_PFAULT_CLIENT_GPC_T1_6 = 0x00000006 # macro
NV_PFAULT_CLIENT_GPC_T1_7 = 0x00000007 # macro
NV_PFAULT_CLIENT_GPC_PE_0 = 0x00000008 # macro
NV_PFAULT_CLIENT_GPC_PE_1 = 0x00000009 # macro
NV_PFAULT_CLIENT_GPC_PE_2 = 0x0000000A # macro
NV_PFAULT_CLIENT_GPC_PE_3 = 0x0000000B # macro
NV_PFAULT_CLIENT_GPC_PE_4 = 0x0000000C # macro
NV_PFAULT_CLIENT_GPC_PE_5 = 0x0000000D # macro
NV_PFAULT_CLIENT_GPC_PE_6 = 0x0000000E # macro
NV_PFAULT_CLIENT_GPC_PE_7 = 0x0000000F # macro
NV_PFAULT_CLIENT_GPC_RAST = 0x00000010 # macro
NV_PFAULT_CLIENT_GPC_GCC = 0x00000011 # macro
NV_PFAULT_CLIENT_GPC_GPCCS = 0x00000012 # macro
NV_PFAULT_CLIENT_GPC_PROP_0 = 0x00000013 # macro
NV_PFAULT_CLIENT_GPC_PROP_1 = 0x00000014 # macro
NV_PFAULT_CLIENT_GPC_PROP_2 = 0x00000015 # macro
NV_PFAULT_CLIENT_GPC_PROP_3 = 0x00000016 # macro
NV_PFAULT_CLIENT_GPC_T1_8 = 0x00000021 # macro
NV_PFAULT_CLIENT_GPC_T1_9 = 0x00000022 # macro
NV_PFAULT_CLIENT_GPC_T1_10 = 0x00000023 # macro
NV_PFAULT_CLIENT_GPC_T1_11 = 0x00000024 # macro
NV_PFAULT_CLIENT_GPC_T1_12 = 0x00000025 # macro
NV_PFAULT_CLIENT_GPC_T1_13 = 0x00000026 # macro
NV_PFAULT_CLIENT_GPC_T1_14 = 0x00000027 # macro
NV_PFAULT_CLIENT_GPC_T1_15 = 0x00000028 # macro
NV_PFAULT_CLIENT_GPC_TPCCS_0 = 0x00000029 # macro
NV_PFAULT_CLIENT_GPC_TPCCS_1 = 0x0000002A # macro
NV_PFAULT_CLIENT_GPC_TPCCS_2 = 0x0000002B # macro
NV_PFAULT_CLIENT_GPC_TPCCS_3 = 0x0000002C # macro
NV_PFAULT_CLIENT_GPC_TPCCS_4 = 0x0000002D # macro
NV_PFAULT_CLIENT_GPC_TPCCS_5 = 0x0000002E # macro
NV_PFAULT_CLIENT_GPC_TPCCS_6 = 0x0000002F # macro
NV_PFAULT_CLIENT_GPC_TPCCS_7 = 0x00000030 # macro
NV_PFAULT_CLIENT_GPC_PE_8 = 0x00000031 # macro
NV_PFAULT_CLIENT_GPC_PE_9 = 0x00000032 # macro
NV_PFAULT_CLIENT_GPC_TPCCS_8 = 0x00000033 # macro
NV_PFAULT_CLIENT_GPC_TPCCS_9 = 0x00000034 # macro
NV_PFAULT_CLIENT_GPC_T1_16 = 0x00000035 # macro
NV_PFAULT_CLIENT_GPC_T1_17 = 0x00000036 # macro
NV_PFAULT_CLIENT_GPC_T1_18 = 0x00000037 # macro
NV_PFAULT_CLIENT_GPC_T1_19 = 0x00000038 # macro
NV_PFAULT_CLIENT_GPC_PE_10 = 0x00000039 # macro
NV_PFAULT_CLIENT_GPC_PE_11 = 0x0000003A # macro
NV_PFAULT_CLIENT_GPC_TPCCS_10 = 0x0000003B # macro
NV_PFAULT_CLIENT_GPC_TPCCS_11 = 0x0000003C # macro
NV_PFAULT_CLIENT_GPC_T1_20 = 0x0000003D # macro
NV_PFAULT_CLIENT_GPC_T1_21 = 0x0000003E # macro
NV_PFAULT_CLIENT_GPC_T1_22 = 0x0000003F # macro
NV_PFAULT_CLIENT_GPC_T1_23 = 0x00000040 # macro
NV_PFAULT_CLIENT_GPC_PE_12 = 0x00000041 # macro
NV_PFAULT_CLIENT_GPC_PE_13 = 0x00000042 # macro
NV_PFAULT_CLIENT_GPC_TPCCS_12 = 0x00000043 # macro
NV_PFAULT_CLIENT_GPC_TPCCS_13 = 0x00000044 # macro
NV_PFAULT_CLIENT_GPC_T1_24 = 0x00000045 # macro
NV_PFAULT_CLIENT_GPC_T1_25 = 0x00000046 # macro
NV_PFAULT_CLIENT_GPC_T1_26 = 0x00000047 # macro
NV_PFAULT_CLIENT_GPC_T1_27 = 0x00000048 # macro
NV_PFAULT_CLIENT_GPC_PE_14 = 0x00000049 # macro
NV_PFAULT_CLIENT_GPC_PE_15 = 0x0000004A # macro
NV_PFAULT_CLIENT_GPC_TPCCS_14 = 0x0000004B # macro
NV_PFAULT_CLIENT_GPC_TPCCS_15 = 0x0000004C # macro
NV_PFAULT_CLIENT_GPC_T1_28 = 0x0000004D # macro
NV_PFAULT_CLIENT_GPC_T1_29 = 0x0000004E # macro
NV_PFAULT_CLIENT_GPC_T1_30 = 0x0000004F # macro
NV_PFAULT_CLIENT_GPC_T1_31 = 0x00000050 # macro
NV_PFAULT_CLIENT_GPC_PE_16 = 0x00000051 # macro
NV_PFAULT_CLIENT_GPC_PE_17 = 0x00000052 # macro
NV_PFAULT_CLIENT_GPC_TPCCS_16 = 0x00000053 # macro
NV_PFAULT_CLIENT_GPC_TPCCS_17 = 0x00000054 # macro
NV_PFAULT_CLIENT_GPC_T1_32 = 0x00000055 # macro
NV_PFAULT_CLIENT_GPC_T1_33 = 0x00000056 # macro
NV_PFAULT_CLIENT_GPC_T1_34 = 0x00000057 # macro
NV_PFAULT_CLIENT_GPC_T1_35 = 0x00000058 # macro
NV_PFAULT_CLIENT_GPC_PE_18 = 0x00000059 # macro
NV_PFAULT_CLIENT_GPC_PE_19 = 0x0000005A # macro
NV_PFAULT_CLIENT_GPC_TPCCS_18 = 0x0000005B # macro
NV_PFAULT_CLIENT_GPC_TPCCS_19 = 0x0000005C # macro
NV_PFAULT_CLIENT_GPC_T1_36 = 0x0000005D # macro
NV_PFAULT_CLIENT_GPC_T1_37 = 0x0000005E # macro
NV_PFAULT_CLIENT_GPC_T1_38 = 0x0000005F # macro
NV_PFAULT_CLIENT_GPC_T1_39 = 0x00000060 # macro
NV_PFAULT_CLIENT_GPC_ROP_0 = 0x00000070 # macro
NV_PFAULT_CLIENT_GPC_ROP_1 = 0x00000071 # macro
NV_PFAULT_CLIENT_GPC_ROP_2 = 0x00000072 # macro
NV_PFAULT_CLIENT_GPC_ROP_3 = 0x00000073 # macro
NV_PFAULT_CLIENT_GPC_GPM = 0x00000017 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_0 = 0x00000018 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_1 = 0x00000019 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_2 = 0x0000001A # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_3 = 0x0000001B # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_4 = 0x0000001C # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_5 = 0x0000001D # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_6 = 0x0000001E # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_7 = 0x0000001F # macro
NV_PFAULT_CLIENT_GPC_RGG_UTLB = 0x00000020 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_8 = 0x00000031 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_9 = 0x00000032 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_10 = 0x00000033 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_11 = 0x00000034 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_12 = 0x00000035 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_13 = 0x00000036 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_14 = 0x00000037 # macro
NV_PFAULT_CLIENT_GPC_LTP_UTLB_15 = 0x00000038 # macro
NV_PFAULT_CLIENT_HUB_VIP = 0x00000000 # macro
NV_PFAULT_CLIENT_HUB_CE0 = 0x00000001 # macro
NV_PFAULT_CLIENT_HUB_CE1 = 0x00000002 # macro
NV_PFAULT_CLIENT_HUB_DNISO = 0x00000003 # macro
NV_PFAULT_CLIENT_HUB_DISPNISO = 0x00000003 # macro
NV_PFAULT_CLIENT_HUB_FE0 = 0x00000004 # macro
NV_PFAULT_CLIENT_HUB_FE = 0x00000004 # macro
NV_PFAULT_CLIENT_HUB_FECS0 = 0x00000005 # macro
NV_PFAULT_CLIENT_HUB_FECS = 0x00000005 # macro
NV_PFAULT_CLIENT_HUB_HOST = 0x00000006 # macro
NV_PFAULT_CLIENT_HUB_HOST_CPU = 0x00000007 # macro
NV_PFAULT_CLIENT_HUB_HOST_CPU_NB = 0x00000008 # macro
NV_PFAULT_CLIENT_HUB_ISO = 0x00000009 # macro
NV_PFAULT_CLIENT_HUB_MMU = 0x0000000A # macro
NV_PFAULT_CLIENT_HUB_NVDEC0 = 0x0000000B # macro
NV_PFAULT_CLIENT_HUB_NVDEC = 0x0000000B # macro
NV_PFAULT_CLIENT_HUB_NVENC1 = 0x0000000D # macro
NV_PFAULT_CLIENT_HUB_NISO = 0x0000000E # macro
NV_PFAULT_CLIENT_HUB_ACTRS = 0x0000000E # macro
NV_PFAULT_CLIENT_HUB_P2P = 0x0000000F # macro
NV_PFAULT_CLIENT_HUB_PD = 0x00000010 # macro
NV_PFAULT_CLIENT_HUB_PERF0 = 0x00000011 # macro
NV_PFAULT_CLIENT_HUB_PERF = 0x00000011 # macro
NV_PFAULT_CLIENT_HUB_PMU = 0x00000012 # macro
NV_PFAULT_CLIENT_HUB_RASTERTWOD = 0x00000013 # macro
NV_PFAULT_CLIENT_HUB_SCC = 0x00000014 # macro
NV_PFAULT_CLIENT_HUB_SCC_NB = 0x00000015 # macro
NV_PFAULT_CLIENT_HUB_SEC = 0x00000016 # macro
NV_PFAULT_CLIENT_HUB_SSYNC = 0x00000017 # macro
NV_PFAULT_CLIENT_HUB_GRCOPY = 0x00000018 # macro
NV_PFAULT_CLIENT_HUB_CE2 = 0x00000018 # macro
NV_PFAULT_CLIENT_HUB_XV = 0x00000019 # macro
NV_PFAULT_CLIENT_HUB_MMU_NB = 0x0000001A # macro
NV_PFAULT_CLIENT_HUB_NVENC0 = 0x0000001B # macro
NV_PFAULT_CLIENT_HUB_NVENC = 0x0000001B # macro
NV_PFAULT_CLIENT_HUB_DFALCON = 0x0000001C # macro
NV_PFAULT_CLIENT_HUB_SKED0 = 0x0000001D # macro
NV_PFAULT_CLIENT_HUB_SKED = 0x0000001D # macro
NV_PFAULT_CLIENT_HUB_AFALCON = 0x0000001E # macro
NV_PFAULT_CLIENT_HUB_DONT_CARE = 0x0000001F # macro
NV_PFAULT_CLIENT_HUB_HSCE0 = 0x00000020 # macro
NV_PFAULT_CLIENT_HUB_HSCE1 = 0x00000021 # macro
NV_PFAULT_CLIENT_HUB_HSCE2 = 0x00000022 # macro
NV_PFAULT_CLIENT_HUB_HSCE3 = 0x00000023 # macro
NV_PFAULT_CLIENT_HUB_HSCE4 = 0x00000024 # macro
NV_PFAULT_CLIENT_HUB_HSCE5 = 0x00000025 # macro
NV_PFAULT_CLIENT_HUB_HSCE6 = 0x00000026 # macro
NV_PFAULT_CLIENT_HUB_HSCE7 = 0x00000027 # macro
NV_PFAULT_CLIENT_HUB_HSCE8 = 0x00000028 # macro
NV_PFAULT_CLIENT_HUB_HSCE9 = 0x00000029 # macro
NV_PFAULT_CLIENT_HUB_HSHUB = 0x0000002A # macro
NV_PFAULT_CLIENT_HUB_PTP_X0 = 0x0000002B # macro
NV_PFAULT_CLIENT_HUB_PTP_X1 = 0x0000002C # macro
NV_PFAULT_CLIENT_HUB_PTP_X2 = 0x0000002D # macro
NV_PFAULT_CLIENT_HUB_PTP_X3 = 0x0000002E # macro
NV_PFAULT_CLIENT_HUB_PTP_X4 = 0x0000002F # macro
NV_PFAULT_CLIENT_HUB_PTP_X5 = 0x00000030 # macro
NV_PFAULT_CLIENT_HUB_PTP_X6 = 0x00000031 # macro
NV_PFAULT_CLIENT_HUB_PTP_X7 = 0x00000032 # macro
NV_PFAULT_CLIENT_HUB_NVENC2 = 0x00000033 # macro
NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER0 = 0x00000034 # macro
NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER1 = 0x00000035 # macro
NV_PFAULT_CLIENT_HUB_DWBIF = 0x00000036 # macro
NV_PFAULT_CLIENT_HUB_FBFALCON = 0x00000037 # macro
NV_PFAULT_CLIENT_HUB_CE_SHIM = 0x00000038 # macro
NV_PFAULT_CLIENT_HUB_GSP = 0x00000039 # macro
NV_PFAULT_CLIENT_HUB_NVDEC1 = 0x0000003A # macro
NV_PFAULT_CLIENT_HUB_NVDEC2 = 0x0000003B # macro
NV_PFAULT_CLIENT_HUB_NVJPG0 = 0x0000003C # macro
NV_PFAULT_CLIENT_HUB_NVDEC3 = 0x0000003D # macro
NV_PFAULT_CLIENT_HUB_NVDEC4 = 0x0000003E # macro
NV_PFAULT_CLIENT_HUB_OFA0 = 0x0000003F # macro
NV_PFAULT_CLIENT_HUB_HSCE10 = 0x00000040 # macro
NV_PFAULT_CLIENT_HUB_HSCE11 = 0x00000041 # macro
NV_PFAULT_CLIENT_HUB_HSCE12 = 0x00000042 # macro
NV_PFAULT_CLIENT_HUB_HSCE13 = 0x00000043 # macro
NV_PFAULT_CLIENT_HUB_HSCE14 = 0x00000044 # macro
NV_PFAULT_CLIENT_HUB_HSCE15 = 0x00000045 # macro
NV_PFAULT_CLIENT_HUB_PTP_X8 = 0x00000046 # macro
NV_PFAULT_CLIENT_HUB_PTP_X9 = 0x00000047 # macro
NV_PFAULT_CLIENT_HUB_PTP_X10 = 0x00000048 # macro
NV_PFAULT_CLIENT_HUB_PTP_X11 = 0x00000049 # macro
NV_PFAULT_CLIENT_HUB_PTP_X12 = 0x0000004A # macro
NV_PFAULT_CLIENT_HUB_PTP_X13 = 0x0000004B # macro
NV_PFAULT_CLIENT_HUB_PTP_X14 = 0x0000004C # macro
NV_PFAULT_CLIENT_HUB_PTP_X15 = 0x0000004D # macro
NV_PFAULT_CLIENT_HUB_FE1 = 0x0000004E # macro
NV_PFAULT_CLIENT_HUB_FE2 = 0x0000004F # macro
NV_PFAULT_CLIENT_HUB_FE3 = 0x00000050 # macro
NV_PFAULT_CLIENT_HUB_FE4 = 0x00000051 # macro
NV_PFAULT_CLIENT_HUB_FE5 = 0x00000052 # macro
NV_PFAULT_CLIENT_HUB_FE6 = 0x00000053 # macro
NV_PFAULT_CLIENT_HUB_FE7 = 0x00000054 # macro
NV_PFAULT_CLIENT_HUB_FECS1 = 0x00000055 # macro
NV_PFAULT_CLIENT_HUB_FECS2 = 0x00000056 # macro
NV_PFAULT_CLIENT_HUB_FECS3 = 0x00000057 # macro
NV_PFAULT_CLIENT_HUB_FECS4 = 0x00000058 # macro
NV_PFAULT_CLIENT_HUB_FECS5 = 0x00000059 # macro
NV_PFAULT_CLIENT_HUB_FECS6 = 0x0000005A # macro
NV_PFAULT_CLIENT_HUB_FECS7 = 0x0000005B # macro
NV_PFAULT_CLIENT_HUB_SKED1 = 0x0000005C # macro
NV_PFAULT_CLIENT_HUB_SKED2 = 0x0000005D # macro
NV_PFAULT_CLIENT_HUB_SKED3 = 0x0000005E # macro
NV_PFAULT_CLIENT_HUB_SKED4 = 0x0000005F # macro
NV_PFAULT_CLIENT_HUB_SKED5 = 0x00000060 # macro
NV_PFAULT_CLIENT_HUB_SKED6 = 0x00000061 # macro
NV_PFAULT_CLIENT_HUB_SKED7 = 0x00000062 # macro
NV_PFAULT_CLIENT_HUB_ESC = 0x00000063 # macro
# NV_PFAULT_ACCESS_TYPE = 19 : 16 # macro
NV_PFAULT_ACCESS_TYPE_READ = 0x00000000 # macro
NV_PFAULT_ACCESS_TYPE_WRITE = 0x00000001 # macro
NV_PFAULT_ACCESS_TYPE_ATOMIC = 0x00000002 # macro
NV_PFAULT_ACCESS_TYPE_PREFETCH = 0x00000003 # macro
NV_PFAULT_ACCESS_TYPE_VIRT_READ = 0x00000000 # macro
NV_PFAULT_ACCESS_TYPE_VIRT_WRITE = 0x00000001 # macro
NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC = 0x00000002 # macro
NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_STRONG = 0x00000002 # macro
NV_PFAULT_ACCESS_TYPE_VIRT_PREFETCH = 0x00000003 # macro
NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_WEAK = 0x00000004 # macro
NV_PFAULT_ACCESS_TYPE_PHYS_READ = 0x00000008 # macro
NV_PFAULT_ACCESS_TYPE_PHYS_WRITE = 0x00000009 # macro
NV_PFAULT_ACCESS_TYPE_PHYS_ATOMIC = 0x0000000a # macro
NV_PFAULT_ACCESS_TYPE_PHYS_PREFETCH = 0x0000000b # macro
# NV_PFAULT_MMU_CLIENT_TYPE = 20 : 20 # macro
NV_PFAULT_MMU_CLIENT_TYPE_GPC = 0x00000000 # macro
NV_PFAULT_MMU_CLIENT_TYPE_HUB = 0x00000001 # macro
# NV_PFAULT_GPC_ID = 28 : 24 # macro
# NV_PFAULT_PROTECTED_MODE = 29 : 29 # macro
# NV_PFAULT_REPLAYABLE_FAULT_EN = 30 : 30 # macro
# NV_PFAULT_VALID = 31 : 31 # macro
NV_ESCAPE_H_INCLUDED = True # macro
NV_ESC_RM_ALLOC_MEMORY = 0x27 # macro
NV_ESC_RM_ALLOC_OBJECT = 0x28 # macro
@@ -29631,6 +30096,7 @@ __all__ = \
'NV50VAIO_CHANNELPIO_ALLOCATION_PARAMETERS',
'NV50_CHANNEL_GPFIFO', 'NV50_DEFERRED_API_CLASS',
'NV50_MEMORY_VIRTUAL', 'NV50_P2P', 'NV50_THIRD_PARTY_P2P',
'NV83DE_ALLOC_PARAMETERS', 'NV83DE_ALLOC_PARAMETERS_MESSAGE_ID',
'NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES',
'NV83DE_CTRL_CMD_DEBUG_CLEAR_SINGLE_SM_ERROR_STATE',
'NV83DE_CTRL_CMD_DEBUG_EXEC_REG_OPS',
@@ -31202,7 +31668,294 @@ __all__ = \
'NV_MEMORY_MULTICAST_FABRIC', 'NV_ME_ALLOCATION_PARAMETERS',
'NV_MSENC_ALLOCATION_PARAMETERS',
'NV_NVJPG_ALLOCATION_PARAMETERS', 'NV_OFA_ALLOCATION_PARAMETERS',
'NV_OS_DESC_MEMORY_ALLOCATION_PARAMS',
'NV_OS_DESC_MEMORY_ALLOCATION_PARAMS', 'NV_PFAULT',
'NV_PFAULT_ACCESS_TYPE_ATOMIC',
'NV_PFAULT_ACCESS_TYPE_PHYS_ATOMIC',
'NV_PFAULT_ACCESS_TYPE_PHYS_PREFETCH',
'NV_PFAULT_ACCESS_TYPE_PHYS_READ',
'NV_PFAULT_ACCESS_TYPE_PHYS_WRITE',
'NV_PFAULT_ACCESS_TYPE_PREFETCH', 'NV_PFAULT_ACCESS_TYPE_READ',
'NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC',
'NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_STRONG',
'NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_WEAK',
'NV_PFAULT_ACCESS_TYPE_VIRT_PREFETCH',
'NV_PFAULT_ACCESS_TYPE_VIRT_READ',
'NV_PFAULT_ACCESS_TYPE_VIRT_WRITE', 'NV_PFAULT_ACCESS_TYPE_WRITE',
'NV_PFAULT_CLIENT_GPC_GCC', 'NV_PFAULT_CLIENT_GPC_GPCCS',
'NV_PFAULT_CLIENT_GPC_GPM', 'NV_PFAULT_CLIENT_GPC_LTP_UTLB_0',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_1',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_10',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_11',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_12',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_13',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_14',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_15',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_2',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_3',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_4',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_5',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_6',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_7',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_8',
'NV_PFAULT_CLIENT_GPC_LTP_UTLB_9', 'NV_PFAULT_CLIENT_GPC_PE_0',
'NV_PFAULT_CLIENT_GPC_PE_1', 'NV_PFAULT_CLIENT_GPC_PE_10',
'NV_PFAULT_CLIENT_GPC_PE_11', 'NV_PFAULT_CLIENT_GPC_PE_12',
'NV_PFAULT_CLIENT_GPC_PE_13', 'NV_PFAULT_CLIENT_GPC_PE_14',
'NV_PFAULT_CLIENT_GPC_PE_15', 'NV_PFAULT_CLIENT_GPC_PE_16',
'NV_PFAULT_CLIENT_GPC_PE_17', 'NV_PFAULT_CLIENT_GPC_PE_18',
'NV_PFAULT_CLIENT_GPC_PE_19', 'NV_PFAULT_CLIENT_GPC_PE_2',
'NV_PFAULT_CLIENT_GPC_PE_3', 'NV_PFAULT_CLIENT_GPC_PE_4',
'NV_PFAULT_CLIENT_GPC_PE_5', 'NV_PFAULT_CLIENT_GPC_PE_6',
'NV_PFAULT_CLIENT_GPC_PE_7', 'NV_PFAULT_CLIENT_GPC_PE_8',
'NV_PFAULT_CLIENT_GPC_PE_9', 'NV_PFAULT_CLIENT_GPC_PROP_0',
'NV_PFAULT_CLIENT_GPC_PROP_1', 'NV_PFAULT_CLIENT_GPC_PROP_2',
'NV_PFAULT_CLIENT_GPC_PROP_3', 'NV_PFAULT_CLIENT_GPC_RAST',
'NV_PFAULT_CLIENT_GPC_RGG_UTLB', 'NV_PFAULT_CLIENT_GPC_ROP_0',
'NV_PFAULT_CLIENT_GPC_ROP_1', 'NV_PFAULT_CLIENT_GPC_ROP_2',
'NV_PFAULT_CLIENT_GPC_ROP_3', 'NV_PFAULT_CLIENT_GPC_T1_0',
'NV_PFAULT_CLIENT_GPC_T1_1', 'NV_PFAULT_CLIENT_GPC_T1_10',
'NV_PFAULT_CLIENT_GPC_T1_11', 'NV_PFAULT_CLIENT_GPC_T1_12',
'NV_PFAULT_CLIENT_GPC_T1_13', 'NV_PFAULT_CLIENT_GPC_T1_14',
'NV_PFAULT_CLIENT_GPC_T1_15', 'NV_PFAULT_CLIENT_GPC_T1_16',
'NV_PFAULT_CLIENT_GPC_T1_17', 'NV_PFAULT_CLIENT_GPC_T1_18',
'NV_PFAULT_CLIENT_GPC_T1_19', 'NV_PFAULT_CLIENT_GPC_T1_2',
'NV_PFAULT_CLIENT_GPC_T1_20', 'NV_PFAULT_CLIENT_GPC_T1_21',
'NV_PFAULT_CLIENT_GPC_T1_22', 'NV_PFAULT_CLIENT_GPC_T1_23',
'NV_PFAULT_CLIENT_GPC_T1_24', 'NV_PFAULT_CLIENT_GPC_T1_25',
'NV_PFAULT_CLIENT_GPC_T1_26', 'NV_PFAULT_CLIENT_GPC_T1_27',
'NV_PFAULT_CLIENT_GPC_T1_28', 'NV_PFAULT_CLIENT_GPC_T1_29',
'NV_PFAULT_CLIENT_GPC_T1_3', 'NV_PFAULT_CLIENT_GPC_T1_30',
'NV_PFAULT_CLIENT_GPC_T1_31', 'NV_PFAULT_CLIENT_GPC_T1_32',
'NV_PFAULT_CLIENT_GPC_T1_33', 'NV_PFAULT_CLIENT_GPC_T1_34',
'NV_PFAULT_CLIENT_GPC_T1_35', 'NV_PFAULT_CLIENT_GPC_T1_36',
'NV_PFAULT_CLIENT_GPC_T1_37', 'NV_PFAULT_CLIENT_GPC_T1_38',
'NV_PFAULT_CLIENT_GPC_T1_39', 'NV_PFAULT_CLIENT_GPC_T1_4',
'NV_PFAULT_CLIENT_GPC_T1_5', 'NV_PFAULT_CLIENT_GPC_T1_6',
'NV_PFAULT_CLIENT_GPC_T1_7', 'NV_PFAULT_CLIENT_GPC_T1_8',
'NV_PFAULT_CLIENT_GPC_T1_9', 'NV_PFAULT_CLIENT_GPC_TPCCS_0',
'NV_PFAULT_CLIENT_GPC_TPCCS_1', 'NV_PFAULT_CLIENT_GPC_TPCCS_10',
'NV_PFAULT_CLIENT_GPC_TPCCS_11', 'NV_PFAULT_CLIENT_GPC_TPCCS_12',
'NV_PFAULT_CLIENT_GPC_TPCCS_13', 'NV_PFAULT_CLIENT_GPC_TPCCS_14',
'NV_PFAULT_CLIENT_GPC_TPCCS_15', 'NV_PFAULT_CLIENT_GPC_TPCCS_16',
'NV_PFAULT_CLIENT_GPC_TPCCS_17', 'NV_PFAULT_CLIENT_GPC_TPCCS_18',
'NV_PFAULT_CLIENT_GPC_TPCCS_19', 'NV_PFAULT_CLIENT_GPC_TPCCS_2',
'NV_PFAULT_CLIENT_GPC_TPCCS_3', 'NV_PFAULT_CLIENT_GPC_TPCCS_4',
'NV_PFAULT_CLIENT_GPC_TPCCS_5', 'NV_PFAULT_CLIENT_GPC_TPCCS_6',
'NV_PFAULT_CLIENT_GPC_TPCCS_7', 'NV_PFAULT_CLIENT_GPC_TPCCS_8',
'NV_PFAULT_CLIENT_GPC_TPCCS_9', 'NV_PFAULT_CLIENT_HUB_ACTRS',
'NV_PFAULT_CLIENT_HUB_AFALCON', 'NV_PFAULT_CLIENT_HUB_CE0',
'NV_PFAULT_CLIENT_HUB_CE1', 'NV_PFAULT_CLIENT_HUB_CE2',
'NV_PFAULT_CLIENT_HUB_CE_SHIM', 'NV_PFAULT_CLIENT_HUB_DFALCON',
'NV_PFAULT_CLIENT_HUB_DISPNISO', 'NV_PFAULT_CLIENT_HUB_DNISO',
'NV_PFAULT_CLIENT_HUB_DONT_CARE', 'NV_PFAULT_CLIENT_HUB_DWBIF',
'NV_PFAULT_CLIENT_HUB_ESC', 'NV_PFAULT_CLIENT_HUB_FBFALCON',
'NV_PFAULT_CLIENT_HUB_FE', 'NV_PFAULT_CLIENT_HUB_FE0',
'NV_PFAULT_CLIENT_HUB_FE1', 'NV_PFAULT_CLIENT_HUB_FE2',
'NV_PFAULT_CLIENT_HUB_FE3', 'NV_PFAULT_CLIENT_HUB_FE4',
'NV_PFAULT_CLIENT_HUB_FE5', 'NV_PFAULT_CLIENT_HUB_FE6',
'NV_PFAULT_CLIENT_HUB_FE7', 'NV_PFAULT_CLIENT_HUB_FECS',
'NV_PFAULT_CLIENT_HUB_FECS0', 'NV_PFAULT_CLIENT_HUB_FECS1',
'NV_PFAULT_CLIENT_HUB_FECS2', 'NV_PFAULT_CLIENT_HUB_FECS3',
'NV_PFAULT_CLIENT_HUB_FECS4', 'NV_PFAULT_CLIENT_HUB_FECS5',
'NV_PFAULT_CLIENT_HUB_FECS6', 'NV_PFAULT_CLIENT_HUB_FECS7',
'NV_PFAULT_CLIENT_HUB_GRCOPY', 'NV_PFAULT_CLIENT_HUB_GSP',
'NV_PFAULT_CLIENT_HUB_HOST', 'NV_PFAULT_CLIENT_HUB_HOST_CPU',
'NV_PFAULT_CLIENT_HUB_HOST_CPU_NB', 'NV_PFAULT_CLIENT_HUB_HSCE0',
'NV_PFAULT_CLIENT_HUB_HSCE1', 'NV_PFAULT_CLIENT_HUB_HSCE10',
'NV_PFAULT_CLIENT_HUB_HSCE11', 'NV_PFAULT_CLIENT_HUB_HSCE12',
'NV_PFAULT_CLIENT_HUB_HSCE13', 'NV_PFAULT_CLIENT_HUB_HSCE14',
'NV_PFAULT_CLIENT_HUB_HSCE15', 'NV_PFAULT_CLIENT_HUB_HSCE2',
'NV_PFAULT_CLIENT_HUB_HSCE3', 'NV_PFAULT_CLIENT_HUB_HSCE4',
'NV_PFAULT_CLIENT_HUB_HSCE5', 'NV_PFAULT_CLIENT_HUB_HSCE6',
'NV_PFAULT_CLIENT_HUB_HSCE7', 'NV_PFAULT_CLIENT_HUB_HSCE8',
'NV_PFAULT_CLIENT_HUB_HSCE9', 'NV_PFAULT_CLIENT_HUB_HSHUB',
'NV_PFAULT_CLIENT_HUB_ISO', 'NV_PFAULT_CLIENT_HUB_MMU',
'NV_PFAULT_CLIENT_HUB_MMU_NB', 'NV_PFAULT_CLIENT_HUB_NISO',
'NV_PFAULT_CLIENT_HUB_NVDEC', 'NV_PFAULT_CLIENT_HUB_NVDEC0',
'NV_PFAULT_CLIENT_HUB_NVDEC1', 'NV_PFAULT_CLIENT_HUB_NVDEC2',
'NV_PFAULT_CLIENT_HUB_NVDEC3', 'NV_PFAULT_CLIENT_HUB_NVDEC4',
'NV_PFAULT_CLIENT_HUB_NVENC', 'NV_PFAULT_CLIENT_HUB_NVENC0',
'NV_PFAULT_CLIENT_HUB_NVENC1', 'NV_PFAULT_CLIENT_HUB_NVENC2',
'NV_PFAULT_CLIENT_HUB_NVJPG0', 'NV_PFAULT_CLIENT_HUB_OFA0',
'NV_PFAULT_CLIENT_HUB_P2P', 'NV_PFAULT_CLIENT_HUB_PD',
'NV_PFAULT_CLIENT_HUB_PERF', 'NV_PFAULT_CLIENT_HUB_PERF0',
'NV_PFAULT_CLIENT_HUB_PMU', 'NV_PFAULT_CLIENT_HUB_PTP_X0',
'NV_PFAULT_CLIENT_HUB_PTP_X1', 'NV_PFAULT_CLIENT_HUB_PTP_X10',
'NV_PFAULT_CLIENT_HUB_PTP_X11', 'NV_PFAULT_CLIENT_HUB_PTP_X12',
'NV_PFAULT_CLIENT_HUB_PTP_X13', 'NV_PFAULT_CLIENT_HUB_PTP_X14',
'NV_PFAULT_CLIENT_HUB_PTP_X15', 'NV_PFAULT_CLIENT_HUB_PTP_X2',
'NV_PFAULT_CLIENT_HUB_PTP_X3', 'NV_PFAULT_CLIENT_HUB_PTP_X4',
'NV_PFAULT_CLIENT_HUB_PTP_X5', 'NV_PFAULT_CLIENT_HUB_PTP_X6',
'NV_PFAULT_CLIENT_HUB_PTP_X7', 'NV_PFAULT_CLIENT_HUB_PTP_X8',
'NV_PFAULT_CLIENT_HUB_PTP_X9', 'NV_PFAULT_CLIENT_HUB_RASTERTWOD',
'NV_PFAULT_CLIENT_HUB_SCC', 'NV_PFAULT_CLIENT_HUB_SCC_NB',
'NV_PFAULT_CLIENT_HUB_SEC', 'NV_PFAULT_CLIENT_HUB_SKED',
'NV_PFAULT_CLIENT_HUB_SKED0', 'NV_PFAULT_CLIENT_HUB_SKED1',
'NV_PFAULT_CLIENT_HUB_SKED2', 'NV_PFAULT_CLIENT_HUB_SKED3',
'NV_PFAULT_CLIENT_HUB_SKED4', 'NV_PFAULT_CLIENT_HUB_SKED5',
'NV_PFAULT_CLIENT_HUB_SKED6', 'NV_PFAULT_CLIENT_HUB_SKED7',
'NV_PFAULT_CLIENT_HUB_SSYNC', 'NV_PFAULT_CLIENT_HUB_VIP',
'NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER0',
'NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER1', 'NV_PFAULT_CLIENT_HUB_XV',
'NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION',
'NV_PFAULT_FAULT_TYPE_COMPRESSION_FAILURE',
'NV_PFAULT_FAULT_TYPE_PDE', 'NV_PFAULT_FAULT_TYPE_PDE_SIZE',
'NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION',
'NV_PFAULT_FAULT_TYPE_POISONED',
'NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION', 'NV_PFAULT_FAULT_TYPE_PTE',
'NV_PFAULT_FAULT_TYPE_REGION_VIOLATION',
'NV_PFAULT_FAULT_TYPE_RO_VIOLATION',
'NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK',
'NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE',
'NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND',
'NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION',
'NV_PFAULT_FAULT_TYPE_WORK_CREATION',
'NV_PFAULT_FAULT_TYPE_WO_VIOLATION',
'NV_PFAULT_MMU_CLIENT_TYPE_GPC', 'NV_PFAULT_MMU_CLIENT_TYPE_HUB',
'NV_PFAULT_MMU_ENG_ID_BAR1', 'NV_PFAULT_MMU_ENG_ID_BAR1_FN0',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN1', 'NV_PFAULT_MMU_ENG_ID_BAR1_FN10',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN11',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN12',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN13',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN14',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN15',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN16',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN17',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN18',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN19', 'NV_PFAULT_MMU_ENG_ID_BAR1_FN2',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN20',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN21',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN22',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN23',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN24',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN25',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN26',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN27',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN28',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN29', 'NV_PFAULT_MMU_ENG_ID_BAR1_FN3',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN30',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN31',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN32',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN33',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN34',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN35',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN36',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN37',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN38',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN39', 'NV_PFAULT_MMU_ENG_ID_BAR1_FN4',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN40',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN41',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN42',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN43',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN44',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN45',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN46',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN47',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN48',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN49', 'NV_PFAULT_MMU_ENG_ID_BAR1_FN5',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN50',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN51',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN52',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN53',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN54',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN55',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN56',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN57',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN58',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN59', 'NV_PFAULT_MMU_ENG_ID_BAR1_FN6',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN60',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN61',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN62',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN63', 'NV_PFAULT_MMU_ENG_ID_BAR1_FN7',
'NV_PFAULT_MMU_ENG_ID_BAR1_FN8', 'NV_PFAULT_MMU_ENG_ID_BAR1_FN9',
'NV_PFAULT_MMU_ENG_ID_BAR2', 'NV_PFAULT_MMU_ENG_ID_BAR2_FN0',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN1', 'NV_PFAULT_MMU_ENG_ID_BAR2_FN10',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN11',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN12',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN13',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN14',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN15',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN16',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN17',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN18',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN19', 'NV_PFAULT_MMU_ENG_ID_BAR2_FN2',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN20',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN21',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN22',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN23',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN24',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN25',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN26',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN27',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN28',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN29', 'NV_PFAULT_MMU_ENG_ID_BAR2_FN3',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN30',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN31',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN32',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN33',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN34',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN35',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN36',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN37',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN38',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN39', 'NV_PFAULT_MMU_ENG_ID_BAR2_FN4',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN40',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN41',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN42',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN43',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN44',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN45',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN46',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN47',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN48',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN49', 'NV_PFAULT_MMU_ENG_ID_BAR2_FN5',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN50',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN51',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN52',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN53',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN54',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN55',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN56',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN57',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN58',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN59', 'NV_PFAULT_MMU_ENG_ID_BAR2_FN6',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN60',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN61',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN62',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN63', 'NV_PFAULT_MMU_ENG_ID_BAR2_FN7',
'NV_PFAULT_MMU_ENG_ID_BAR2_FN8', 'NV_PFAULT_MMU_ENG_ID_BAR2_FN9',
'NV_PFAULT_MMU_ENG_ID_CE0', 'NV_PFAULT_MMU_ENG_ID_CE1',
'NV_PFAULT_MMU_ENG_ID_CE2', 'NV_PFAULT_MMU_ENG_ID_CE3',
'NV_PFAULT_MMU_ENG_ID_CE4', 'NV_PFAULT_MMU_ENG_ID_CE5',
'NV_PFAULT_MMU_ENG_ID_CE6', 'NV_PFAULT_MMU_ENG_ID_CE7',
'NV_PFAULT_MMU_ENG_ID_CE8', 'NV_PFAULT_MMU_ENG_ID_CE9',
'NV_PFAULT_MMU_ENG_ID_DISPLAY', 'NV_PFAULT_MMU_ENG_ID_FLA',
'NV_PFAULT_MMU_ENG_ID_GRAPHICS', 'NV_PFAULT_MMU_ENG_ID_GRCOPY',
'NV_PFAULT_MMU_ENG_ID_GSP', 'NV_PFAULT_MMU_ENG_ID_HOST0',
'NV_PFAULT_MMU_ENG_ID_HOST1', 'NV_PFAULT_MMU_ENG_ID_HOST10',
'NV_PFAULT_MMU_ENG_ID_HOST11', 'NV_PFAULT_MMU_ENG_ID_HOST12',
'NV_PFAULT_MMU_ENG_ID_HOST13', 'NV_PFAULT_MMU_ENG_ID_HOST14',
'NV_PFAULT_MMU_ENG_ID_HOST15', 'NV_PFAULT_MMU_ENG_ID_HOST16',
'NV_PFAULT_MMU_ENG_ID_HOST17', 'NV_PFAULT_MMU_ENG_ID_HOST18',
'NV_PFAULT_MMU_ENG_ID_HOST19', 'NV_PFAULT_MMU_ENG_ID_HOST2',
'NV_PFAULT_MMU_ENG_ID_HOST20', 'NV_PFAULT_MMU_ENG_ID_HOST21',
'NV_PFAULT_MMU_ENG_ID_HOST22', 'NV_PFAULT_MMU_ENG_ID_HOST23',
'NV_PFAULT_MMU_ENG_ID_HOST24', 'NV_PFAULT_MMU_ENG_ID_HOST25',
'NV_PFAULT_MMU_ENG_ID_HOST26', 'NV_PFAULT_MMU_ENG_ID_HOST27',
'NV_PFAULT_MMU_ENG_ID_HOST28', 'NV_PFAULT_MMU_ENG_ID_HOST29',
'NV_PFAULT_MMU_ENG_ID_HOST3', 'NV_PFAULT_MMU_ENG_ID_HOST30',
'NV_PFAULT_MMU_ENG_ID_HOST31', 'NV_PFAULT_MMU_ENG_ID_HOST4',
'NV_PFAULT_MMU_ENG_ID_HOST5', 'NV_PFAULT_MMU_ENG_ID_HOST6',
'NV_PFAULT_MMU_ENG_ID_HOST7', 'NV_PFAULT_MMU_ENG_ID_HOST8',
'NV_PFAULT_MMU_ENG_ID_HOST9', 'NV_PFAULT_MMU_ENG_ID_IFB',
'NV_PFAULT_MMU_ENG_ID_NVDEC', 'NV_PFAULT_MMU_ENG_ID_NVDEC0',
'NV_PFAULT_MMU_ENG_ID_NVDEC1', 'NV_PFAULT_MMU_ENG_ID_NVDEC2',
'NV_PFAULT_MMU_ENG_ID_NVDEC3', 'NV_PFAULT_MMU_ENG_ID_NVDEC4',
'NV_PFAULT_MMU_ENG_ID_NVENC0', 'NV_PFAULT_MMU_ENG_ID_NVENC1',
'NV_PFAULT_MMU_ENG_ID_NVENC2', 'NV_PFAULT_MMU_ENG_ID_NVJPG0',
'NV_PFAULT_MMU_ENG_ID_OFA0', 'NV_PFAULT_MMU_ENG_ID_PERF',
'NV_PFAULT_MMU_ENG_ID_PHYSICAL', 'NV_PFAULT_MMU_ENG_ID_PTP',
'NV_PFAULT_MMU_ENG_ID_PWR_PMU', 'NV_PFAULT_MMU_ENG_ID_SEC',
'NV_PPP_ALLOCATION_PARAMETERS', 'NV_RM_API_VERSION_CMD_QUERY',
'NV_RM_API_VERSION_CMD_RELAXED', 'NV_RM_API_VERSION_CMD_STRICT',
'NV_RM_API_VERSION_REPLY_RECOGNIZED',
@@ -31380,7 +32133,7 @@ __all__ = \
'UVM_VALIDATE_VA_RANGE_PARAMS', 'VOLTA_CHANNEL_GPFIFO_A',
'VOLTA_USERMODE_A', 'VPR_REQUEST_PARAMS', 'VPR_STATUS_PARAMS',
'_NV_UNIX_NVOS_PARAMS_WRAPPERS_H_', '_UVM_IOCTL_H',
'_UVM_LINUX_IOCTL_H', '__CLC6C0QMD_H__',
'_UVM_LINUX_IOCTL_H', '__CLC6C0QMD_H__', '__ga100_dev_fault_h__',
'_cl2080_notification_h_', '_cl_ampere_compute_a_h_',
'_clc56f_h_', '_clc6b5_h_', 'ampere_dma_copy_aControlPio',
'c__EA_NV_VIDMEM_ACCESS_BIT_ALLOCATION_PARAMS_ADDR_SPACE',
@@ -32324,6 +33077,7 @@ __all__ = \
'struct_NV2080_NOCAT_JOURNAL_RECORD',
'struct_NV2080_VGPU_FB_USAGE', 'struct_NV2080_VGPU_GUEST',
'struct_NV30F1_CTRL_GSYNC_GET_OPTIMIZED_TIMING_PARAMS',
'struct_NV83DE_ALLOC_PARAMETERS',
'struct_NV83DE_CTRL_CMD_DEBUG_SUSPEND_ALL_CONTEXTS_FOR_CLIENT_PARAMS',
'struct_NV83DE_CTRL_DEBUG_ACCESS_MEMORY_ENTRY',
'struct_NV83DE_CTRL_DEBUG_ACCESS_MEMORY_PARAMS',

44
tinygrad/runtime/ops_nv.py
View File

@@ -16,6 +16,9 @@ if MOCKGPU:=getenv("MOCKGPU"): import extra.mockgpu.mockgpu # noqa: F401 # pylin
def get_error_str(status): return f"{status}: {nv_gpu.nv_status_codes.get(status, 'Unknown error')}"
NV_PFAULT_FAULT_TYPE = {dt:name for name,dt in nv_gpu.__dict__.items() if name.startswith("NV_PFAULT_FAULT_TYPE_")}
NV_PFAULT_ACCESS_TYPE = {dt:name.split("_")[-1] for name,dt in nv_gpu.__dict__.items() if name.startswith("NV_PFAULT_ACCESS_TYPE_")}
def nv_iowr(fd, nr, args):
ret = fcntl.ioctl(fd, (3 << 30) | (ctypes.sizeof(args) & 0x1FFF) << 16 | (ord('F') & 0xFF) << 8 | (nr & 0xFF), args)
if ret != 0: raise RuntimeError(f"ioctl returned {ret}")
@@ -36,8 +39,8 @@ def rm_control(cmd, sttyp, fd, client, obj, **kwargs):
def make_rmctrl_type():
return type("NVRMCTRL", (object,), {name[name.find("_CTRL_CMD_")+10:].lower(): functools.partial(rm_control, dt, sttyp)
for name,dt in nv_gpu.__dict__.items() if name.find("_CTRL_CMD_")>=0 and
(sttyp:=getattr(nv_gpu, name.replace("_CTRL_CMD_", "_CTRL_")+"_PARAMS", getattr(nv_gpu, name+"_PARAMS", None)))})
for name,dt in nv_gpu.__dict__.items() if name.find("_CTRL_CMD_")>=0 and (sttyp:=getattr(nv_gpu, name.replace("_CTRL_CMD_", "_CTRL_")+"_PARAMS", \
getattr(nv_gpu, name+"_PARAMS", getattr(nv_gpu, name.replace("_CTRL_CMD_", "_CTRL_DEBUG_")+"_PARAMETERS", None))))})
rmctrl = make_rmctrl_type()
def uvm_ioctl(cmd, sttyp, fd, **kwargs):
@@ -465,7 +468,7 @@ class NVDevice(HCQCompiled):
ctxshare_params = nv_gpu.NV_CTXSHARE_ALLOCATION_PARAMETERS(hVASpace=vaspace, flags=nv_gpu.NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_ASYNC)
ctxshare = rm_alloc(self.fd_ctl, nv_gpu.FERMI_CONTEXT_SHARE_A, self.root, channel_group, ctxshare_params).hObjectNew
self.compute_gpfifo = self._new_gpu_fifo(gpfifo_area, ctxshare, channel_group, offset=0, entries=0x10000)
self.compute_gpfifo = self._new_gpu_fifo(gpfifo_area, ctxshare, channel_group, offset=0, entries=0x10000, enable_debug=True)
self.dma_gpfifo = self._new_gpu_fifo(gpfifo_area, ctxshare, channel_group, offset=0x100000, entries=0x10000)
rmctrl.gpfifo_schedule(self.fd_ctl, self.root, channel_group, bEnable=1)
@@ -484,15 +487,20 @@ class NVDevice(HCQCompiled):
self._setup_gpfifos()
def _new_gpu_fifo(self, gpfifo_area, ctxshare, channel_group, offset=0, entries=0x400) -> GPFifo:
def _new_gpu_fifo(self, gpfifo_area, ctxshare, channel_group, offset=0, entries=0x400, enable_debug=False) -> GPFifo:
notifier = self._gpu_system_alloc(48 << 20)
params = nv_gpu.NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS(hObjectError=notifier.hMemory, hObjectBuffer=gpfifo_area.hMemory,
gpFifoOffset=gpfifo_area.va_addr+offset, gpFifoEntries=entries, hContextShare=ctxshare,
hUserdMemory=(ctypes.c_uint32*8)(gpfifo_area.hMemory), userdOffset=(ctypes.c_uint64*8)(entries*8+offset))
gpfifo = rm_alloc(self.fd_ctl, nv_gpu.AMPERE_CHANNEL_GPFIFO_A, self.root, channel_group, params).hObjectNew
rm_alloc(self.fd_ctl, self.compute_class, self.root, gpfifo, None)
comp = rm_alloc(self.fd_ctl, self.compute_class, self.root, gpfifo, None).hObjectNew
rm_alloc(self.fd_ctl, nv_gpu.AMPERE_DMA_COPY_B, self.root, gpfifo, None)
if enable_debug:
self.debug_compute_obj, self.debug_channel = comp, gpfifo
debugger_params = nv_gpu.NV83DE_ALLOC_PARAMETERS(hAppClient=self.root, hClass3dObject=self.debug_compute_obj)
self.debugger = rm_alloc(self.fd_ctl, nv_gpu.GT200_DEBUGGER, self.root, self.device, debugger_params).hObjectNew
ws_token_params = rmctrl.gpfifo_get_work_submit_token(self.fd_ctl, self.root, gpfifo, workSubmitToken=-1)
assert ws_token_params.workSubmitToken != -1
@@ -536,3 +544,29 @@ class NVDevice(HCQCompiled):
rmctrl.fb_flush_gpu_cache(self.fd_ctl, self.root, self.subdevice,
flags=((nv_gpu.NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_WRITE_BACK_YES << 2) | (nv_gpu.NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_INVALIDATE_YES << 3) |
(nv_gpu.NV2080_CTRL_FB_FLUSH_GPU_CACHE_FLAGS_FLUSH_MODE_FULL_CACHE << 4)))
def on_device_hang(self):
# Prepare fault report.
# TODO: Restore the GPU using NV83DE_CTRL_CMD_CLEAR_ALL_SM_ERROR_STATES if needed.
report = []
sm_errors = rmctrl.debug_read_all_sm_error_states(self.fd_ctl, self.root, self.debugger, hTargetChannel=self.debug_channel, numSMsToRead=100)
if sm_errors.mmuFault.valid:
mmu_info = rmctrl.debug_read_mmu_fault_info(self.fd_ctl, self.root, self.debugger)
for i in range(mmu_info.count):
valo, vahi = ((pfaddr:=(pf:=mmu_info.mmuFaultInfoList[i]).faultAddress) - (1 << 30)) & ~0xffffff, round_up(pfaddr + (1 << 30), 0x1000000)
report += [f"MMU fault: 0x{pfaddr:X} | {NV_PFAULT_FAULT_TYPE[pf.faultType]} | {NV_PFAULT_ACCESS_TYPE[pf.accessType]}"]
# Scan mapped regions (1GB around the faulted area).
report += [f"Valid mapping on range 0x{valo:X} - 0x{vahi:X}:"]
while valo != vahi:
mappings = rmctrl.get_mappings(self.fd_ctl, self.root, self.debugger, vaLo=valo, vaHi=vahi)
for i in range(mappings.count): report += [f"\t0x{mappings.opsBuffer[-1].gpuVA:X} | size: 0x{mappings.opsBuffer[-1].size:X}"]
if mappings.hasMore: valo = mappings.opsBuffer[-1].gpuVA + mappings.opsBuffer[-1].size
else: break
else:
for i, e in enumerate(sm_errors.smErrorStateArray):
if e.hwwGlobalEsr or e.hwwWarpEsr: report += [f"SM{i} fault: esr={e.hwwGlobalEsr} warp_esr={e.hwwWarpEsr} warp_pc={e.hwwWarpEsrPc64}"]
raise RuntimeError("\n".join(report))

7
tinygrad/runtime/support/hcq.py
View File

@@ -296,7 +296,7 @@ class HCQProgram:
q.signal(self.device.timeline_signal, self.device.timeline_value).submit(self.device)
self.device.timeline_value += 1
if wait: self.device.timeline_signal.wait(self.device.timeline_value - 1)
if wait: self.device.synchronize()
return (float(sig_en.timestamp - sig_st.timestamp) / 1e6) if wait else None
class ProfileLogger:
@@ -363,7 +363,10 @@ class HCQCompiled(Compiled):
self.devices.append(self)
def synchronize(self):
self.timeline_signal.wait(self.timeline_value - 1) if not hasattr(self, '_syncdev') else self._syncdev()
try: self.timeline_signal.wait(self.timeline_value - 1) if not hasattr(self, '_syncdev') else self._syncdev()
except RuntimeError as e:
if hasattr(self, 'on_device_hang'): self.on_device_hang()
else: raise e
if self.timeline_value > (1 << 31): self._wrap_timeline_signal()
if PROFILE: