Refactor ops_amd.py (MI300X prereq) (#9428)

autogen_stubs.sh

@@ -308,11 +308,27 @@ generate_am() {
   sed -i "s\(int64_t)\ \g" $BASE/am/am.py
   sed -i "s\AMDGPU_PTE_MTYPE_VG10(2)\AMDGPU_PTE_MTYPE_VG10(0, 2)\g" $BASE/am/am.py # incorrect parsing (TODO: remove when clang2py is gone).
 
+  clang2py -k cdefstum \
+    extra/hip_gpu_driver/kfd_pm4_headers_ai.h \
+    extra/hip_gpu_driver/nvd.h \
+    -o $BASE/am/pm4_nv.py
+  fixup $BASE/am/pm4_nv.py
+
+  clang2py -k cdefstum \
+    extra/amdpci/headers/navi10_enum.h \
+    -o $BASE/am/navi10.py
+  fixup $BASE/am/navi10.py
+
   clang2py -k cdefstum \
     extra/amdpci/headers/soc21_enum.h \
     -o $BASE/am/soc21.py
   fixup $BASE/am/soc21.py
 
+  clang2py -k cdefstum \
+    extra/amdpci/headers/soc24_enum.h \
+    -o $BASE/am/soc24.py
+  fixup $BASE/am/soc24.py
+
   clang2py -k cdefstum \
     extra/amdpci/headers/mp_13_0_0_offset.h \
     extra/amdpci/headers/mp_13_0_0_sh_mask.h \
@@ -325,12 +341,38 @@ generate_am() {
     -o $BASE/am/mp_11_0.py
   fixup $BASE/am/mp_11_0.py
 
+  clang2py -k cdefstum \
+    extra/amdpci/headers/gc_10_3_0_offset.h \
+    extra/amdpci/headers/gc_10_3_0_sh_mask.h \
+    -o $BASE/am/gc_10_3_0.py
+  fixup $BASE/am/gc_10_3_0.py
+
   clang2py -k cdefstum \
     extra/amdpci/headers/gc_11_0_0_offset.h \
     extra/amdpci/headers/gc_11_0_0_sh_mask.h \
     -o $BASE/am/gc_11_0_0.py
   fixup $BASE/am/gc_11_0_0.py
 
+  clang2py -k cdefstum \
+    extra/amdpci/headers/gc_12_0_0_offset.h \
+    extra/amdpci/headers/gc_12_0_0_sh_mask.h \
+    -o $BASE/am/gc_12_0_0.py
+  fixup $BASE/am/gc_12_0_0.py
+
+  clang2py -k cdefstum \
+    extra/hip_gpu_driver/sdma_registers.h \
+    extra/hip_gpu_driver/navi10_sdma_pkt_open.h \
+    --clang-args="-I/opt/rocm/include -x c++" \
+    -o $BASE/am/sdma_5_0_0.py
+  fixup $BASE/am/sdma_5_0_0.py
+
+  clang2py -k cdefstum \
+    extra/hip_gpu_driver/sdma_registers.h \
+    extra/hip_gpu_driver/sdma_v6_0_0_pkt_open.h \
+    --clang-args="-I/opt/rocm/include -x c++" \
+    -o $BASE/am/sdma_6_0_0.py
+  fixup $BASE/am/sdma_6_0_0.py
+
   clang2py -k cdefstum \
     extra/amdpci/headers/mmhub_3_0_0_offset.h \
     extra/amdpci/headers/mmhub_3_0_0_sh_mask.h \
@@ -343,12 +385,24 @@ generate_am() {
     -o $BASE/am/mmhub_3_0_2.py
   fixup $BASE/am/mmhub_3_0_2.py
 
+  clang2py -k cdefstum \
+    extra/amdpci/headers/nbio_2_3_offset.h \
+    extra/amdpci/headers/nbio_2_3_sh_mask.h \
+    -o $BASE/am/nbio_2_3_0.py
+  fixup $BASE/am/nbio_2_3_0.py
+
   clang2py -k cdefstum \
     extra/amdpci/headers/nbio_4_3_0_offset.h \
     extra/amdpci/headers/nbio_4_3_0_sh_mask.h \
     -o $BASE/am/nbio_4_3_0.py
   fixup $BASE/am/nbio_4_3_0.py
 
+  clang2py -k cdefstum \
+    extra/amdpci/headers/nbif_6_3_1_offset.h \
+    extra/amdpci/headers/nbif_6_3_1_sh_mask.h \
+    -o $BASE/am/nbif_6_3_1.py
+  fixup $BASE/am/nbif_6_3_1.py
+
   clang2py -k cdefstum \
     extra/amdpci/headers/osssys_6_0_0_offset.h \
     extra/amdpci/headers/osssys_6_0_0_sh_mask.h \

test/mockgpu/amd/amddriver.py

@@ -1,5 +1,6 @@
 import pathlib, re, ctypes, mmap, collections, functools, copy, os
 import tinygrad.runtime.autogen.kfd as kfd
+import tinygrad.runtime.autogen.am.am as am
 from tinygrad.helpers import from_mv
 from test.mockgpu.driver import VirtDriver, VirtFileDesc, TextFileDesc, DirFileDesc, VirtFile
 from test.mockgpu.amd.amdgpu import AMDGPU, gpu_props
@@ -82,6 +83,23 @@ class AMDDriver(VirtDriver):
       VirtFile(f'/sys/devices/virtual/kfd/kfd/topology/nodes/{gpu_id}/gpu_id', functools.partial(TextFileDesc, text=f"{gpu_id}")),
       VirtFile(f'/sys/devices/virtual/kfd/kfd/topology/nodes/{gpu_id}/properties',
         functools.partial(TextFileDesc, text=gpu_props.format(drm_render_minor=gpu_id))),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0',
+               functools.partial(DirFileDesc, child_names=[str(am.GC_HWID), str(am.SDMA0_HWID), str(am.NBIF_HWID)])),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.GC_HWID}/0/major', functools.partial(TextFileDesc, text='11')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.GC_HWID}/0/minor', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.GC_HWID}/0/revision', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.GC_HWID}/0/base_addr',
+               functools.partial(TextFileDesc, text='0x00001260\n0x0000A000\n0x0001C000\n0x02402C00')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.SDMA0_HWID}/0/major', functools.partial(TextFileDesc, text='6')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.SDMA0_HWID}/0/minor', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.SDMA0_HWID}/0/revision', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.SDMA0_HWID}/0/base_addr',
+               functools.partial(TextFileDesc, text='0x00001260\n0x0000A000\n0x0001C000\n0x02402C00')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.NBIF_HWID}/0/major', functools.partial(TextFileDesc, text='4')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.NBIF_HWID}/0/minor', functools.partial(TextFileDesc, text='3')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.NBIF_HWID}/0/revision', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.NBIF_HWID}/0/base_addr',
+               functools.partial(TextFileDesc, text='0x00000000\n0x00000014\n0x00000D20\n0x00010400\n0x0241B000\n0x04040000')),
       VirtFile(f'/dev/dri/renderD{gpu_id}', functools.partial(DRMFileDesc, driver=self, gpu=f"{self.gpus[gpu_id]}")),
     ]
 

tinygrad/runtime/autogen/am/pm4_nv.py

@@ -0,0 +1,962 @@
+# mypy: ignore-errors
+# -*- coding: utf-8 -*-
+#
+# TARGET arch is: []
+# WORD_SIZE is: 8
+# POINTER_SIZE is: 8
+# LONGDOUBLE_SIZE is: 16
+#
+import ctypes
+
+
+class AsDictMixin:
+    @classmethod
+    def as_dict(cls, self):
+        result = {}
+        if not isinstance(self, AsDictMixin):
+            # not a structure, assume it's already a python object
+            return self
+        if not hasattr(cls, "_fields_"):
+            return result
+        # sys.version_info >= (3, 5)
+        # for (field, *_) in cls._fields_:  # noqa
+        for field_tuple in cls._fields_:  # noqa
+            field = field_tuple[0]
+            if field.startswith('PADDING_'):
+                continue
+            value = getattr(self, field)
+            type_ = type(value)
+            if hasattr(value, "_length_") and hasattr(value, "_type_"):
+                # array
+                if not hasattr(type_, "as_dict"):
+                    value = [v for v in value]
+                else:
+                    type_ = type_._type_
+                    value = [type_.as_dict(v) for v in value]
+            elif hasattr(value, "contents") and hasattr(value, "_type_"):
+                # pointer
+                try:
+                    if not hasattr(type_, "as_dict"):
+                        value = value.contents
+                    else:
+                        type_ = type_._type_
+                        value = type_.as_dict(value.contents)
+                except ValueError:
+                    # nullptr
+                    value = None
+            elif isinstance(value, AsDictMixin):
+                # other structure
+                value = type_.as_dict(value)
+            result[field] = value
+        return result
+
+
+class Structure(ctypes.Structure, AsDictMixin):
+
+    def __init__(self, *args, **kwds):
+        # We don't want to use positional arguments fill PADDING_* fields
+
+        args = dict(zip(self.__class__._field_names_(), args))
+        args.update(kwds)
+        super(Structure, self).__init__(**args)
+
+    @classmethod
+    def _field_names_(cls):
+        if hasattr(cls, '_fields_'):
+            return (f[0] for f in cls._fields_ if not f[0].startswith('PADDING'))
+        else:
+            return ()
+
+    @classmethod
+    def get_type(cls, field):
+        for f in cls._fields_:
+            if f[0] == field:
+                return f[1]
+        return None
+
+    @classmethod
+    def bind(cls, bound_fields):
+        fields = {}
+        for name, type_ in cls._fields_:
+            if hasattr(type_, "restype"):
+                if name in bound_fields:
+                    if bound_fields[name] is None:
+                        fields[name] = type_()
+                    else:
+                        # use a closure to capture the callback from the loop scope
+                        fields[name] = (
+                            type_((lambda callback: lambda *args: callback(*args))(
+                                bound_fields[name]))
+                        )
+                    del bound_fields[name]
+                else:
+                    # default callback implementation (does nothing)
+                    try:
+                        default_ = type_(0).restype().value
+                    except TypeError:
+                        default_ = None
+                    fields[name] = type_((
+                        lambda default_: lambda *args: default_)(default_))
+            else:
+                # not a callback function, use default initialization
+                if name in bound_fields:
+                    fields[name] = bound_fields[name]
+                    del bound_fields[name]
+                else:
+                    fields[name] = type_()
+        if len(bound_fields) != 0:
+            raise ValueError(
+                "Cannot bind the following unknown callback(s) {}.{}".format(
+                    cls.__name__, bound_fields.keys()
+            ))
+        return cls(**fields)
+
+
+class Union(ctypes.Union, AsDictMixin):
+    pass
+
+
+
+
+
+F32_MES_PM4_PACKETS_H = True # macro
+uint32_t = True # macro
+int32_t = True # macro
+PM4_MES_HEADER_DEFINED = True # macro
+PM4_MEC_RELEASE_MEM_DEFINED = True # macro
+PM4_MEC_WRITE_DATA_DEFINED = True # macro
+class union_PM4_MES_TYPE_3_HEADER(Union):
+    pass
+
+class struct_PM4_MES_TYPE_3_HEADER_0(Structure):
+    pass
+
+struct_PM4_MES_TYPE_3_HEADER_0._pack_ = 1 # source:False
+struct_PM4_MES_TYPE_3_HEADER_0._fields_ = [
+    ('reserved1', ctypes.c_uint32, 8),
+    ('opcode', ctypes.c_uint32, 8),
+    ('count', ctypes.c_uint32, 14),
+    ('type', ctypes.c_uint32, 2),
+]
+
+union_PM4_MES_TYPE_3_HEADER._pack_ = 1 # source:False
+union_PM4_MES_TYPE_3_HEADER._anonymous_ = ('_0',)
+union_PM4_MES_TYPE_3_HEADER._fields_ = [
+    ('_0', struct_PM4_MES_TYPE_3_HEADER_0),
+    ('u32All', ctypes.c_uint32),
+]
+
+
+# values for enumeration 'c_uint32'
+c_uint32__enumvalues = {
+    5: 'event_index__mec_release_mem__end_of_pipe',
+    6: 'event_index__mec_release_mem__shader_done',
+}
+event_index__mec_release_mem__end_of_pipe = 5
+event_index__mec_release_mem__shader_done = 6
+c_uint32 = ctypes.c_uint32 # enum
+
+# values for enumeration 'c_uint32'
+c_uint32__enumvalues = {
+    0: 'cache_policy__mec_release_mem__lru',
+    1: 'cache_policy__mec_release_mem__stream',
+}
+cache_policy__mec_release_mem__lru = 0
+cache_policy__mec_release_mem__stream = 1
+c_uint32 = ctypes.c_uint32 # enum
+
+# values for enumeration 'c_uint32'
+c_uint32__enumvalues = {
+    0: 'pq_exe_status__mec_release_mem__default',
+    1: 'pq_exe_status__mec_release_mem__phase_update',
+}
+pq_exe_status__mec_release_mem__default = 0
+pq_exe_status__mec_release_mem__phase_update = 1
+c_uint32 = ctypes.c_uint32 # enum
+
+# values for enumeration 'c_uint32'
+c_uint32__enumvalues = {
+    0: 'dst_sel__mec_release_mem__memory_controller',
+    1: 'dst_sel__mec_release_mem__tc_l2',
+    2: 'dst_sel__mec_release_mem__queue_write_pointer_register',
+    3: 'dst_sel__mec_release_mem__queue_write_pointer_poll_mask_bit',
+}
+dst_sel__mec_release_mem__memory_controller = 0
+dst_sel__mec_release_mem__tc_l2 = 1
+dst_sel__mec_release_mem__queue_write_pointer_register = 2
+dst_sel__mec_release_mem__queue_write_pointer_poll_mask_bit = 3
+c_uint32 = ctypes.c_uint32 # enum
+
+# values for enumeration 'c_uint32'
+c_uint32__enumvalues = {
+    0: 'int_sel__mec_release_mem__none',
+    1: 'int_sel__mec_release_mem__send_interrupt_only',
+    2: 'int_sel__mec_release_mem__send_interrupt_after_write_confirm',
+    3: 'int_sel__mec_release_mem__send_data_after_write_confirm',
+    4: 'int_sel__mec_release_mem__unconditionally_send_int_ctxid',
+    5: 'int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_32_bit_compare',
+    6: 'int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_64_bit_compare',
+}
+int_sel__mec_release_mem__none = 0
+int_sel__mec_release_mem__send_interrupt_only = 1
+int_sel__mec_release_mem__send_interrupt_after_write_confirm = 2
+int_sel__mec_release_mem__send_data_after_write_confirm = 3
+int_sel__mec_release_mem__unconditionally_send_int_ctxid = 4
+int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_32_bit_compare = 5
+int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_64_bit_compare = 6
+c_uint32 = ctypes.c_uint32 # enum
+
+# values for enumeration 'c_uint32'
+c_uint32__enumvalues = {
+    0: 'data_sel__mec_release_mem__none',
+    1: 'data_sel__mec_release_mem__send_32_bit_low',
+    2: 'data_sel__mec_release_mem__send_64_bit_data',
+    3: 'data_sel__mec_release_mem__send_gpu_clock_counter',
+    4: 'data_sel__mec_release_mem__send_cp_perfcounter_hi_lo',
+    5: 'data_sel__mec_release_mem__store_gds_data_to_memory',
+}
+data_sel__mec_release_mem__none = 0
+data_sel__mec_release_mem__send_32_bit_low = 1
+data_sel__mec_release_mem__send_64_bit_data = 2
+data_sel__mec_release_mem__send_gpu_clock_counter = 3
+data_sel__mec_release_mem__send_cp_perfcounter_hi_lo = 4
+data_sel__mec_release_mem__store_gds_data_to_memory = 5
+c_uint32 = ctypes.c_uint32 # enum
+class struct_pm4_mec_release_mem(Structure):
+    pass
+
+class union_pm4_mec_release_mem_0(Union):
+    pass
+
+union_pm4_mec_release_mem_0._pack_ = 1 # source:False
+union_pm4_mec_release_mem_0._fields_ = [
+    ('header', union_PM4_MES_TYPE_3_HEADER),
+    ('ordinal1', ctypes.c_uint32),
+]
+
+class union_pm4_mec_release_mem_1(Union):
+    pass
+
+class struct_pm4_mec_release_mem_1_bitfields2(Structure):
+    pass
+
+struct_pm4_mec_release_mem_1_bitfields2._pack_ = 1 # source:False
+struct_pm4_mec_release_mem_1_bitfields2._fields_ = [
+    ('event_type', ctypes.c_uint32, 6),
+    ('reserved1', ctypes.c_uint32, 2),
+    ('event_index', c_uint32, 4),
+    ('tcl1_vol_action_ena', ctypes.c_uint32, 1),
+    ('tc_vol_action_ena', ctypes.c_uint32, 1),
+    ('reserved2', ctypes.c_uint32, 1),
+    ('tc_wb_action_ena', ctypes.c_uint32, 1),
+    ('tcl1_action_ena', ctypes.c_uint32, 1),
+    ('tc_action_ena', ctypes.c_uint32, 1),
+    ('reserved3', ctypes.c_uint32, 1),
+    ('tc_nc_action_ena', ctypes.c_uint32, 1),
+    ('tc_wc_action_ena', ctypes.c_uint32, 1),
+    ('tc_md_action_ena', ctypes.c_uint32, 1),
+    ('reserved4', ctypes.c_uint32, 3),
+    ('cache_policy', c_uint32, 2),
+    ('reserved5', ctypes.c_uint32, 2),
+    ('pq_exe_status', c_uint32, 1),
+    ('reserved6', ctypes.c_uint32, 2),
+]
+
+union_pm4_mec_release_mem_1._pack_ = 1 # source:False
+union_pm4_mec_release_mem_1._fields_ = [
+    ('bitfields2', struct_pm4_mec_release_mem_1_bitfields2),
+    ('ordinal2', ctypes.c_uint32),
+]
+
+class union_pm4_mec_release_mem_2(Union):
+    pass
+
+class struct_pm4_mec_release_mem_2_bitfields3(Structure):
+    pass
+
+struct_pm4_mec_release_mem_2_bitfields3._pack_ = 1 # source:False
+struct_pm4_mec_release_mem_2_bitfields3._fields_ = [
+    ('reserved7', ctypes.c_uint32, 16),
+    ('dst_sel', c_uint32, 2),
+    ('reserved8', ctypes.c_uint32, 6),
+    ('int_sel', c_uint32, 3),
+    ('reserved9', ctypes.c_uint32, 2),
+    ('data_sel', c_uint32, 3),
+]
+
+union_pm4_mec_release_mem_2._pack_ = 1 # source:False
+union_pm4_mec_release_mem_2._fields_ = [
+    ('bitfields3', struct_pm4_mec_release_mem_2_bitfields3),
+    ('ordinal3', ctypes.c_uint32),
+]
+
+class union_pm4_mec_release_mem_3(Union):
+    pass
+
+class struct_pm4_mec_release_mem_3_bitfields4(Structure):
+    pass
+
+struct_pm4_mec_release_mem_3_bitfields4._pack_ = 1 # source:False
+struct_pm4_mec_release_mem_3_bitfields4._fields_ = [
+    ('reserved10', ctypes.c_uint32, 2),
+    ('address_lo_32b', ctypes.c_uint32, 30),
+]
+
+class struct_pm4_mec_release_mem_3_bitfields4b(Structure):
+    pass
+
+struct_pm4_mec_release_mem_3_bitfields4b._pack_ = 1 # source:False
+struct_pm4_mec_release_mem_3_bitfields4b._fields_ = [
+    ('reserved11', ctypes.c_uint32, 3),
+    ('address_lo_64b', ctypes.c_uint32, 29),
+]
+
+union_pm4_mec_release_mem_3._pack_ = 1 # source:False
+union_pm4_mec_release_mem_3._fields_ = [
+    ('bitfields4', struct_pm4_mec_release_mem_3_bitfields4),
+    ('bitfields4b', struct_pm4_mec_release_mem_3_bitfields4b),
+    ('reserved12', ctypes.c_uint32),
+    ('ordinal4', ctypes.c_uint32),
+]
+
+class union_pm4_mec_release_mem_4(Union):
+    pass
+
+union_pm4_mec_release_mem_4._pack_ = 1 # source:False
+union_pm4_mec_release_mem_4._fields_ = [
+    ('address_hi', ctypes.c_uint32),
+    ('reserved13', ctypes.c_uint32),
+    ('ordinal5', ctypes.c_uint32),
+]
+
+class union_pm4_mec_release_mem_5(Union):
+    pass
+
+class struct_pm4_mec_release_mem_5_bitfields6c(Structure):
+    pass
+
+struct_pm4_mec_release_mem_5_bitfields6c._pack_ = 1 # source:False
+struct_pm4_mec_release_mem_5_bitfields6c._fields_ = [
+    ('dw_offset', ctypes.c_uint32, 16),
+    ('num_dwords', ctypes.c_uint32, 16),
+]
+
+union_pm4_mec_release_mem_5._pack_ = 1 # source:False
+union_pm4_mec_release_mem_5._fields_ = [
+    ('data_lo', ctypes.c_uint32),
+    ('cmp_data_lo', ctypes.c_uint32),
+    ('bitfields6c', struct_pm4_mec_release_mem_5_bitfields6c),
+    ('reserved14', ctypes.c_uint32),
+    ('ordinal6', ctypes.c_uint32),
+]
+
+class union_pm4_mec_release_mem_6(Union):
+    pass
+
+union_pm4_mec_release_mem_6._pack_ = 1 # source:False
+union_pm4_mec_release_mem_6._fields_ = [
+    ('data_hi', ctypes.c_uint32),
+    ('cmp_data_hi', ctypes.c_uint32),
+    ('reserved15', ctypes.c_uint32),
+    ('reserved16', ctypes.c_uint32),
+    ('ordinal7', ctypes.c_uint32),
+]
+
+struct_pm4_mec_release_mem._pack_ = 1 # source:False
+struct_pm4_mec_release_mem._anonymous_ = ('_0', '_1', '_2', '_3', '_4', '_5', '_6',)
+struct_pm4_mec_release_mem._fields_ = [
+    ('_0', union_pm4_mec_release_mem_0),
+    ('_1', union_pm4_mec_release_mem_1),
+    ('_2', union_pm4_mec_release_mem_2),
+    ('_3', union_pm4_mec_release_mem_3),
+    ('_4', union_pm4_mec_release_mem_4),
+    ('_5', union_pm4_mec_release_mem_5),
+    ('_6', union_pm4_mec_release_mem_6),
+    ('int_ctxid', ctypes.c_uint32),
+]
+
+
+# values for enumeration 'WRITE_DATA_dst_sel_enum'
+WRITE_DATA_dst_sel_enum__enumvalues = {
+    0: 'dst_sel___write_data__mem_mapped_register',
+    2: 'dst_sel___write_data__tc_l2',
+    3: 'dst_sel___write_data__gds',
+    5: 'dst_sel___write_data__memory',
+    6: 'dst_sel___write_data__memory_mapped_adc_persistent_state',
+}
+dst_sel___write_data__mem_mapped_register = 0
+dst_sel___write_data__tc_l2 = 2
+dst_sel___write_data__gds = 3
+dst_sel___write_data__memory = 5
+dst_sel___write_data__memory_mapped_adc_persistent_state = 6
+WRITE_DATA_dst_sel_enum = ctypes.c_uint32 # enum
+
+# values for enumeration 'WRITE_DATA_addr_incr_enum'
+WRITE_DATA_addr_incr_enum__enumvalues = {
+    0: 'addr_incr___write_data__increment_address',
+    1: 'addr_incr___write_data__do_not_increment_address',
+}
+addr_incr___write_data__increment_address = 0
+addr_incr___write_data__do_not_increment_address = 1
+WRITE_DATA_addr_incr_enum = ctypes.c_uint32 # enum
+
+# values for enumeration 'WRITE_DATA_wr_confirm_enum'
+WRITE_DATA_wr_confirm_enum__enumvalues = {
+    0: 'wr_confirm___write_data__do_not_wait_for_write_confirmation',
+    1: 'wr_confirm___write_data__wait_for_write_confirmation',
+}
+wr_confirm___write_data__do_not_wait_for_write_confirmation = 0
+wr_confirm___write_data__wait_for_write_confirmation = 1
+WRITE_DATA_wr_confirm_enum = ctypes.c_uint32 # enum
+
+# values for enumeration 'WRITE_DATA_cache_policy_enum'
+WRITE_DATA_cache_policy_enum__enumvalues = {
+    0: 'cache_policy___write_data__lru',
+    1: 'cache_policy___write_data__stream',
+}
+cache_policy___write_data__lru = 0
+cache_policy___write_data__stream = 1
+WRITE_DATA_cache_policy_enum = ctypes.c_uint32 # enum
+class struct_pm4_mec_write_data_mmio(Structure):
+    pass
+
+class union_pm4_mec_write_data_mmio_0(Union):
+    pass
+
+union_pm4_mec_write_data_mmio_0._pack_ = 1 # source:False
+union_pm4_mec_write_data_mmio_0._fields_ = [
+    ('header', union_PM4_MES_TYPE_3_HEADER),
+    ('ordinal1', ctypes.c_uint32),
+]
+
+class union_pm4_mec_write_data_mmio_1(Union):
+    pass
+
+class struct_pm4_mec_write_data_mmio_1_bitfields2(Structure):
+    pass
+
+struct_pm4_mec_write_data_mmio_1_bitfields2._pack_ = 1 # source:False
+struct_pm4_mec_write_data_mmio_1_bitfields2._fields_ = [
+    ('reserved1', ctypes.c_uint32, 8),
+    ('dst_sel', ctypes.c_uint32, 4),
+    ('reserved2', ctypes.c_uint32, 4),
+    ('addr_incr', ctypes.c_uint32, 1),
+    ('reserved3', ctypes.c_uint32, 2),
+    ('resume_vf', ctypes.c_uint32, 1),
+    ('wr_confirm', ctypes.c_uint32, 1),
+    ('reserved4', ctypes.c_uint32, 4),
+    ('cache_policy', ctypes.c_uint32, 2),
+    ('reserved5', ctypes.c_uint32, 5),
+]
+
+union_pm4_mec_write_data_mmio_1._pack_ = 1 # source:False
+union_pm4_mec_write_data_mmio_1._fields_ = [
+    ('bitfields2', struct_pm4_mec_write_data_mmio_1_bitfields2),
+    ('ordinal2', ctypes.c_uint32),
+]
+
+class union_pm4_mec_write_data_mmio_2(Union):
+    pass
+
+class struct_pm4_mec_write_data_mmio_2_bitfields3(Structure):
+    pass
+
+struct_pm4_mec_write_data_mmio_2_bitfields3._pack_ = 1 # source:False
+struct_pm4_mec_write_data_mmio_2_bitfields3._fields_ = [
+    ('dst_mmreg_addr', ctypes.c_uint32, 18),
+    ('reserved6', ctypes.c_uint32, 14),
+]
+
+union_pm4_mec_write_data_mmio_2._pack_ = 1 # source:False
+union_pm4_mec_write_data_mmio_2._fields_ = [
+    ('bitfields3', struct_pm4_mec_write_data_mmio_2_bitfields3),
+    ('ordinal3', ctypes.c_uint32),
+]
+
+struct_pm4_mec_write_data_mmio._pack_ = 1 # source:False
+struct_pm4_mec_write_data_mmio._anonymous_ = ('_0', '_1', '_2',)
+struct_pm4_mec_write_data_mmio._fields_ = [
+    ('_0', union_pm4_mec_write_data_mmio_0),
+    ('_1', union_pm4_mec_write_data_mmio_1),
+    ('_2', union_pm4_mec_write_data_mmio_2),
+    ('reserved7', ctypes.c_uint32),
+    ('data', ctypes.c_uint32),
+]
+
+
+# values for enumeration 'c__Ea_CACHE_FLUSH_AND_INV_TS_EVENT'
+c__Ea_CACHE_FLUSH_AND_INV_TS_EVENT__enumvalues = {
+    20: 'CACHE_FLUSH_AND_INV_TS_EVENT',
+}
+CACHE_FLUSH_AND_INV_TS_EVENT = 20
+c__Ea_CACHE_FLUSH_AND_INV_TS_EVENT = ctypes.c_uint32 # enum
+NVD_H = True # macro
+PACKET_TYPE0 = 0 # macro
+PACKET_TYPE1 = 1 # macro
+PACKET_TYPE2 = 2 # macro
+PACKET_TYPE3 = 3 # macro
+def CP_PACKET_GET_TYPE(h):  # macro
+   return (((h)>>30)&3)
+def CP_PACKET_GET_COUNT(h):  # macro
+   return (((h)>>16)&0x3FFF)
+def CP_PACKET0_GET_REG(h):  # macro
+   return ((h)&0xFFFF)
+def CP_PACKET3_GET_OPCODE(h):  # macro
+   return (((h)>>8)&0xFF)
+def PACKET0(reg, n):  # macro
+   return ((0<<30)|((reg)&0xFFFF)|((n)&0x3FFF)<<16)
+CP_PACKET2 = 0x80000000 # macro
+PACKET2_PAD_SHIFT = 0 # macro
+PACKET2_PAD_MASK = (0x3fffffff<<0) # macro
+# def PACKET2(v):  # macro
+#    return (0x80000000|REG_SET(PACKET2_PAD,(v)))
+def PACKET3(op, n):  # macro
+   return ((3<<30)|(((op)&0xFF)<<8)|((n)&0x3FFF)<<16)
+def PACKET3_COMPUTE(op, n):  # macro
+   return (PACKET3(op,n)|1<<1)
+PACKET3_NOP = 0x10 # macro
+PACKET3_SET_BASE = 0x11 # macro
+def PACKET3_BASE_INDEX(x):  # macro
+   return ((x)<<0)
+CE_PARTITION_BASE = 3 # macro
+PACKET3_CLEAR_STATE = 0x12 # macro
+PACKET3_INDEX_BUFFER_SIZE = 0x13 # macro
+PACKET3_DISPATCH_DIRECT = 0x15 # macro
+PACKET3_DISPATCH_INDIRECT = 0x16 # macro
+PACKET3_INDIRECT_BUFFER_END = 0x17 # macro
+PACKET3_INDIRECT_BUFFER_CNST_END = 0x19 # macro
+PACKET3_ATOMIC_GDS = 0x1D # macro
+PACKET3_ATOMIC_MEM = 0x1E # macro
+PACKET3_OCCLUSION_QUERY = 0x1F # macro
+PACKET3_SET_PREDICATION = 0x20 # macro
+PACKET3_REG_RMW = 0x21 # macro
+PACKET3_COND_EXEC = 0x22 # macro
+PACKET3_PRED_EXEC = 0x23 # macro
+PACKET3_DRAW_INDIRECT = 0x24 # macro
+PACKET3_DRAW_INDEX_INDIRECT = 0x25 # macro
+PACKET3_INDEX_BASE = 0x26 # macro
+PACKET3_DRAW_INDEX_2 = 0x27 # macro
+PACKET3_CONTEXT_CONTROL = 0x28 # macro
+PACKET3_INDEX_TYPE = 0x2A # macro
+PACKET3_DRAW_INDIRECT_MULTI = 0x2C # macro
+PACKET3_DRAW_INDEX_AUTO = 0x2D # macro
+PACKET3_NUM_INSTANCES = 0x2F # macro
+PACKET3_DRAW_INDEX_MULTI_AUTO = 0x30 # macro
+PACKET3_INDIRECT_BUFFER_PRIV = 0x32 # macro
+PACKET3_INDIRECT_BUFFER_CNST = 0x33 # macro
+PACKET3_COND_INDIRECT_BUFFER_CNST = 0x33 # macro
+PACKET3_STRMOUT_BUFFER_UPDATE = 0x34 # macro
+PACKET3_DRAW_INDEX_OFFSET_2 = 0x35 # macro
+PACKET3_DRAW_PREAMBLE = 0x36 # macro
+PACKET3_WRITE_DATA = 0x37 # macro
+def WRITE_DATA_DST_SEL(x):  # macro
+   return ((x)<<8)
+WR_ONE_ADDR = (1<<16) # macro
+WR_CONFIRM = (1<<20) # macro
+def WRITE_DATA_CACHE_POLICY(x):  # macro
+   return ((x)<<25)
+def WRITE_DATA_ENGINE_SEL(x):  # macro
+   return ((x)<<30)
+PACKET3_DRAW_INDEX_INDIRECT_MULTI = 0x38 # macro
+PACKET3_MEM_SEMAPHORE = 0x39 # macro
+PACKET3_SEM_USE_MAILBOX = (0x1<<16) # macro
+PACKET3_SEM_SEL_SIGNAL_TYPE = (0x1<<20) # macro
+PACKET3_SEM_SEL_SIGNAL = (0x6<<29) # macro
+PACKET3_SEM_SEL_WAIT = (0x7<<29) # macro
+PACKET3_DRAW_INDEX_MULTI_INST = 0x3A # macro
+PACKET3_COPY_DW = 0x3B # macro
+PACKET3_WAIT_REG_MEM = 0x3C # macro
+def WAIT_REG_MEM_FUNCTION(x):  # macro
+   return ((x)<<0)
+def WAIT_REG_MEM_MEM_SPACE(x):  # macro
+   return ((x)<<4)
+def WAIT_REG_MEM_OPERATION(x):  # macro
+   return ((x)<<6)
+def WAIT_REG_MEM_ENGINE(x):  # macro
+   return ((x)<<8)
+PACKET3_INDIRECT_BUFFER = 0x3F # macro
+INDIRECT_BUFFER_VALID = (1<<23) # macro
+def INDIRECT_BUFFER_CACHE_POLICY(x):  # macro
+   return ((x)<<28)
+def INDIRECT_BUFFER_PRE_ENB(x):  # macro
+   return ((x)<<21)
+def INDIRECT_BUFFER_PRE_RESUME(x):  # macro
+   return ((x)<<30)
+PACKET3_COND_INDIRECT_BUFFER = 0x3F # macro
+PACKET3_COPY_DATA = 0x40 # macro
+PACKET3_CP_DMA = 0x41 # macro
+PACKET3_PFP_SYNC_ME = 0x42 # macro
+PACKET3_SURFACE_SYNC = 0x43 # macro
+PACKET3_ME_INITIALIZE = 0x44 # macro
+PACKET3_COND_WRITE = 0x45 # macro
+PACKET3_EVENT_WRITE = 0x46 # macro
+def EVENT_TYPE(x):  # macro
+   return ((x)<<0)
+def EVENT_INDEX(x):  # macro
+   return ((x)<<8)
+PACKET3_EVENT_WRITE_EOP = 0x47 # macro
+PACKET3_EVENT_WRITE_EOS = 0x48 # macro
+PACKET3_RELEASE_MEM = 0x49 # macro
+def PACKET3_RELEASE_MEM_EVENT_TYPE(x):  # macro
+   return ((x)<<0)
+def PACKET3_RELEASE_MEM_EVENT_INDEX(x):  # macro
+   return ((x)<<8)
+PACKET3_RELEASE_MEM_GCR_GLM_WB = (1<<12) # macro
+PACKET3_RELEASE_MEM_GCR_GLM_INV = (1<<13) # macro
+PACKET3_RELEASE_MEM_GCR_GLV_INV = (1<<14) # macro
+PACKET3_RELEASE_MEM_GCR_GL1_INV = (1<<15) # macro
+PACKET3_RELEASE_MEM_GCR_GL2_US = (1<<16) # macro
+PACKET3_RELEASE_MEM_GCR_GL2_RANGE = (1<<17) # macro
+PACKET3_RELEASE_MEM_GCR_GL2_DISCARD = (1<<19) # macro
+PACKET3_RELEASE_MEM_GCR_GL2_INV = (1<<20) # macro
+PACKET3_RELEASE_MEM_GCR_GL2_WB = (1<<21) # macro
+PACKET3_RELEASE_MEM_GCR_SEQ = (1<<22) # macro
+def PACKET3_RELEASE_MEM_CACHE_POLICY(x):  # macro
+   return ((x)<<25)
+PACKET3_RELEASE_MEM_EXECUTE = (1<<28) # macro
+def PACKET3_RELEASE_MEM_DATA_SEL(x):  # macro
+   return ((x)<<29)
+def PACKET3_RELEASE_MEM_INT_SEL(x):  # macro
+   return ((x)<<24)
+def PACKET3_RELEASE_MEM_DST_SEL(x):  # macro
+   return ((x)<<16)
+PACKET3_PREAMBLE_CNTL = 0x4A # macro
+PACKET3_PREAMBLE_BEGIN_CLEAR_STATE = (2<<28) # macro
+PACKET3_PREAMBLE_END_CLEAR_STATE = (3<<28) # macro
+PACKET3_DMA_DATA = 0x50 # macro
+def PACKET3_DMA_DATA_ENGINE(x):  # macro
+   return ((x)<<0)
+def PACKET3_DMA_DATA_SRC_CACHE_POLICY(x):  # macro
+   return ((x)<<13)
+def PACKET3_DMA_DATA_DST_SEL(x):  # macro
+   return ((x)<<20)
+def PACKET3_DMA_DATA_DST_CACHE_POLICY(x):  # macro
+   return ((x)<<25)
+def PACKET3_DMA_DATA_SRC_SEL(x):  # macro
+   return ((x)<<29)
+PACKET3_DMA_DATA_CP_SYNC = (1<<31) # macro
+PACKET3_DMA_DATA_CMD_SAS = (1<<26) # macro
+PACKET3_DMA_DATA_CMD_DAS = (1<<27) # macro
+PACKET3_DMA_DATA_CMD_SAIC = (1<<28) # macro
+PACKET3_DMA_DATA_CMD_DAIC = (1<<29) # macro
+PACKET3_DMA_DATA_CMD_RAW_WAIT = (1<<30) # macro
+PACKET3_CONTEXT_REG_RMW = 0x51 # macro
+PACKET3_GFX_CNTX_UPDATE = 0x52 # macro
+PACKET3_BLK_CNTX_UPDATE = 0x53 # macro
+PACKET3_INCR_UPDT_STATE = 0x55 # macro
+PACKET3_ACQUIRE_MEM = 0x58 # macro
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(x):  # macro
+   return ((x)<<0)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_RANGE(x):  # macro
+   return ((x)<<2)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(x):  # macro
+   return ((x)<<4)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(x):  # macro
+   return ((x)<<5)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_WB(x):  # macro
+   return ((x)<<6)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(x):  # macro
+   return ((x)<<7)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(x):  # macro
+   return ((x)<<8)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(x):  # macro
+   return ((x)<<9)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_US(x):  # macro
+   return ((x)<<10)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_RANGE(x):  # macro
+   return ((x)<<11)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_DISCARD(x):  # macro
+   return ((x)<<13)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(x):  # macro
+   return ((x)<<14)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(x):  # macro
+   return ((x)<<15)
+def PACKET3_ACQUIRE_MEM_GCR_CNTL_SEQ(x):  # macro
+   return ((x)<<16)
+PACKET3_ACQUIRE_MEM_GCR_RANGE_IS_PA = (1<<18) # macro
+PACKET3_REWIND = 0x59 # macro
+PACKET3_INTERRUPT = 0x5A # macro
+PACKET3_GEN_PDEPTE = 0x5B # macro
+PACKET3_INDIRECT_BUFFER_PASID = 0x5C # macro
+PACKET3_PRIME_UTCL2 = 0x5D # macro
+PACKET3_LOAD_UCONFIG_REG = 0x5E # macro
+PACKET3_LOAD_SH_REG = 0x5F # macro
+PACKET3_LOAD_CONFIG_REG = 0x60 # macro
+PACKET3_LOAD_CONTEXT_REG = 0x61 # macro
+PACKET3_LOAD_COMPUTE_STATE = 0x62 # macro
+PACKET3_LOAD_SH_REG_INDEX = 0x63 # macro
+PACKET3_SET_CONFIG_REG = 0x68 # macro
+PACKET3_SET_CONFIG_REG_START = 0x00002000 # macro
+PACKET3_SET_CONFIG_REG_END = 0x00002c00 # macro
+PACKET3_SET_CONTEXT_REG = 0x69 # macro
+PACKET3_SET_CONTEXT_REG_START = 0x0000a000 # macro
+PACKET3_SET_CONTEXT_REG_END = 0x0000a400 # macro
+PACKET3_SET_CONTEXT_REG_INDEX = 0x6A # macro
+PACKET3_SET_VGPR_REG_DI_MULTI = 0x71 # macro
+PACKET3_SET_SH_REG_DI = 0x72 # macro
+PACKET3_SET_CONTEXT_REG_INDIRECT = 0x73 # macro
+PACKET3_SET_SH_REG_DI_MULTI = 0x74 # macro
+PACKET3_GFX_PIPE_LOCK = 0x75 # macro
+PACKET3_SET_SH_REG = 0x76 # macro
+PACKET3_SET_SH_REG_START = 0x00002c00 # macro
+PACKET3_SET_SH_REG_END = 0x00003000 # macro
+PACKET3_SET_SH_REG_OFFSET = 0x77 # macro
+PACKET3_SET_QUEUE_REG = 0x78 # macro
+PACKET3_SET_UCONFIG_REG = 0x79 # macro
+PACKET3_SET_UCONFIG_REG_START = 0x0000c000 # macro
+PACKET3_SET_UCONFIG_REG_END = 0x0000c400 # macro
+PACKET3_SET_UCONFIG_REG_INDEX = 0x7A # macro
+PACKET3_FORWARD_HEADER = 0x7C # macro
+PACKET3_SCRATCH_RAM_WRITE = 0x7D # macro
+PACKET3_SCRATCH_RAM_READ = 0x7E # macro
+PACKET3_LOAD_CONST_RAM = 0x80 # macro
+PACKET3_WRITE_CONST_RAM = 0x81 # macro
+PACKET3_DUMP_CONST_RAM = 0x83 # macro
+PACKET3_INCREMENT_CE_COUNTER = 0x84 # macro
+PACKET3_INCREMENT_DE_COUNTER = 0x85 # macro
+PACKET3_WAIT_ON_CE_COUNTER = 0x86 # macro
+PACKET3_WAIT_ON_DE_COUNTER_DIFF = 0x88 # macro
+PACKET3_SWITCH_BUFFER = 0x8B # macro
+PACKET3_DISPATCH_DRAW_PREAMBLE = 0x8C # macro
+PACKET3_DISPATCH_DRAW_PREAMBLE_ACE = 0x8C # macro
+PACKET3_DISPATCH_DRAW = 0x8D # macro
+PACKET3_DISPATCH_DRAW_ACE = 0x8D # macro
+PACKET3_GET_LOD_STATS = 0x8E # macro
+PACKET3_DRAW_MULTI_PREAMBLE = 0x8F # macro
+PACKET3_FRAME_CONTROL = 0x90 # macro
+FRAME_TMZ = (1<<0) # macro
+def FRAME_CMD(x):  # macro
+   return ((x)<<28)
+PACKET3_INDEX_ATTRIBUTES_INDIRECT = 0x91 # macro
+PACKET3_WAIT_REG_MEM64 = 0x93 # macro
+PACKET3_COND_PREEMPT = 0x94 # macro
+PACKET3_HDP_FLUSH = 0x95 # macro
+PACKET3_COPY_DATA_RB = 0x96 # macro
+PACKET3_INVALIDATE_TLBS = 0x98 # macro
+def PACKET3_INVALIDATE_TLBS_DST_SEL(x):  # macro
+   return ((x)<<0)
+def PACKET3_INVALIDATE_TLBS_ALL_HUB(x):  # macro
+   return ((x)<<4)
+def PACKET3_INVALIDATE_TLBS_PASID(x):  # macro
+   return ((x)<<5)
+PACKET3_AQL_PACKET = 0x99 # macro
+PACKET3_DMA_DATA_FILL_MULTI = 0x9A # macro
+PACKET3_SET_SH_REG_INDEX = 0x9B # macro
+PACKET3_DRAW_INDIRECT_COUNT_MULTI = 0x9C # macro
+PACKET3_DRAW_INDEX_INDIRECT_COUNT_MULTI = 0x9D # macro
+PACKET3_DUMP_CONST_RAM_OFFSET = 0x9E # macro
+PACKET3_LOAD_CONTEXT_REG_INDEX = 0x9F # macro
+PACKET3_SET_RESOURCES = 0xA0 # macro
+def PACKET3_SET_RESOURCES_VMID_MASK(x):  # macro
+   return ((x)<<0)
+def PACKET3_SET_RESOURCES_UNMAP_LATENTY(x):  # macro
+   return ((x)<<16)
+def PACKET3_SET_RESOURCES_QUEUE_TYPE(x):  # macro
+   return ((x)<<29)
+PACKET3_MAP_PROCESS = 0xA1 # macro
+PACKET3_MAP_QUEUES = 0xA2 # macro
+def PACKET3_MAP_QUEUES_QUEUE_SEL(x):  # macro
+   return ((x)<<4)
+def PACKET3_MAP_QUEUES_VMID(x):  # macro
+   return ((x)<<8)
+def PACKET3_MAP_QUEUES_QUEUE(x):  # macro
+   return ((x)<<13)
+def PACKET3_MAP_QUEUES_PIPE(x):  # macro
+   return ((x)<<16)
+def PACKET3_MAP_QUEUES_ME(x):  # macro
+   return ((x)<<18)
+def PACKET3_MAP_QUEUES_QUEUE_TYPE(x):  # macro
+   return ((x)<<21)
+def PACKET3_MAP_QUEUES_ALLOC_FORMAT(x):  # macro
+   return ((x)<<24)
+def PACKET3_MAP_QUEUES_ENGINE_SEL(x):  # macro
+   return ((x)<<26)
+def PACKET3_MAP_QUEUES_NUM_QUEUES(x):  # macro
+   return ((x)<<29)
+def PACKET3_MAP_QUEUES_CHECK_DISABLE(x):  # macro
+   return ((x)<<1)
+def PACKET3_MAP_QUEUES_DOORBELL_OFFSET(x):  # macro
+   return ((x)<<2)
+PACKET3_UNMAP_QUEUES = 0xA3 # macro
+def PACKET3_UNMAP_QUEUES_ACTION(x):  # macro
+   return ((x)<<0)
+def PACKET3_UNMAP_QUEUES_QUEUE_SEL(x):  # macro
+   return ((x)<<4)
+def PACKET3_UNMAP_QUEUES_ENGINE_SEL(x):  # macro
+   return ((x)<<26)
+def PACKET3_UNMAP_QUEUES_NUM_QUEUES(x):  # macro
+   return ((x)<<29)
+def PACKET3_UNMAP_QUEUES_PASID(x):  # macro
+   return ((x)<<0)
+def PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(x):  # macro
+   return ((x)<<2)
+def PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET1(x):  # macro
+   return ((x)<<2)
+def PACKET3_UNMAP_QUEUES_RB_WPTR(x):  # macro
+   return ((x)<<0)
+def PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET2(x):  # macro
+   return ((x)<<2)
+def PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET3(x):  # macro
+   return ((x)<<2)
+PACKET3_QUERY_STATUS = 0xA4 # macro
+def PACKET3_QUERY_STATUS_CONTEXT_ID(x):  # macro
+   return ((x)<<0)
+def PACKET3_QUERY_STATUS_INTERRUPT_SEL(x):  # macro
+   return ((x)<<28)
+def PACKET3_QUERY_STATUS_COMMAND(x):  # macro
+   return ((x)<<30)
+def PACKET3_QUERY_STATUS_PASID(x):  # macro
+   return ((x)<<0)
+def PACKET3_QUERY_STATUS_DOORBELL_OFFSET(x):  # macro
+   return ((x)<<2)
+def PACKET3_QUERY_STATUS_ENG_SEL(x):  # macro
+   return ((x)<<25)
+PACKET3_RUN_LIST = 0xA5 # macro
+PACKET3_MAP_PROCESS_VM = 0xA6 # macro
+PACKET3_SET_Q_PREEMPTION_MODE = 0xF0 # macro
+def PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(x):  # macro
+   return ((x)<<0)
+PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM = (1<<0) # macro
+__all__ = \
+    ['CACHE_FLUSH_AND_INV_TS_EVENT', 'CE_PARTITION_BASE',
+    'CP_PACKET2', 'F32_MES_PM4_PACKETS_H', 'FRAME_TMZ',
+    'INDIRECT_BUFFER_VALID', 'NVD_H', 'PACKET2_PAD_MASK',
+    'PACKET2_PAD_SHIFT', 'PACKET3_ACQUIRE_MEM',
+    'PACKET3_ACQUIRE_MEM_GCR_RANGE_IS_PA', 'PACKET3_AQL_PACKET',
+    'PACKET3_ATOMIC_GDS', 'PACKET3_ATOMIC_MEM',
+    'PACKET3_BLK_CNTX_UPDATE', 'PACKET3_CLEAR_STATE',
+    'PACKET3_COND_EXEC', 'PACKET3_COND_INDIRECT_BUFFER',
+    'PACKET3_COND_INDIRECT_BUFFER_CNST', 'PACKET3_COND_PREEMPT',
+    'PACKET3_COND_WRITE', 'PACKET3_CONTEXT_CONTROL',
+    'PACKET3_CONTEXT_REG_RMW', 'PACKET3_COPY_DATA',
+    'PACKET3_COPY_DATA_RB', 'PACKET3_COPY_DW', 'PACKET3_CP_DMA',
+    'PACKET3_DISPATCH_DIRECT', 'PACKET3_DISPATCH_DRAW',
+    'PACKET3_DISPATCH_DRAW_ACE', 'PACKET3_DISPATCH_DRAW_PREAMBLE',
+    'PACKET3_DISPATCH_DRAW_PREAMBLE_ACE', 'PACKET3_DISPATCH_INDIRECT',
+    'PACKET3_DMA_DATA', 'PACKET3_DMA_DATA_CMD_DAIC',
+    'PACKET3_DMA_DATA_CMD_DAS', 'PACKET3_DMA_DATA_CMD_RAW_WAIT',
+    'PACKET3_DMA_DATA_CMD_SAIC', 'PACKET3_DMA_DATA_CMD_SAS',
+    'PACKET3_DMA_DATA_CP_SYNC', 'PACKET3_DMA_DATA_FILL_MULTI',
+    'PACKET3_DRAW_INDEX_2', 'PACKET3_DRAW_INDEX_AUTO',
+    'PACKET3_DRAW_INDEX_INDIRECT',
+    'PACKET3_DRAW_INDEX_INDIRECT_COUNT_MULTI',
+    'PACKET3_DRAW_INDEX_INDIRECT_MULTI',
+    'PACKET3_DRAW_INDEX_MULTI_AUTO', 'PACKET3_DRAW_INDEX_MULTI_INST',
+    'PACKET3_DRAW_INDEX_OFFSET_2', 'PACKET3_DRAW_INDIRECT',
+    'PACKET3_DRAW_INDIRECT_COUNT_MULTI',
+    'PACKET3_DRAW_INDIRECT_MULTI', 'PACKET3_DRAW_MULTI_PREAMBLE',
+    'PACKET3_DRAW_PREAMBLE', 'PACKET3_DUMP_CONST_RAM',
+    'PACKET3_DUMP_CONST_RAM_OFFSET', 'PACKET3_EVENT_WRITE',
+    'PACKET3_EVENT_WRITE_EOP', 'PACKET3_EVENT_WRITE_EOS',
+    'PACKET3_FORWARD_HEADER', 'PACKET3_FRAME_CONTROL',
+    'PACKET3_GEN_PDEPTE', 'PACKET3_GET_LOD_STATS',
+    'PACKET3_GFX_CNTX_UPDATE', 'PACKET3_GFX_PIPE_LOCK',
+    'PACKET3_HDP_FLUSH', 'PACKET3_INCREMENT_CE_COUNTER',
+    'PACKET3_INCREMENT_DE_COUNTER', 'PACKET3_INCR_UPDT_STATE',
+    'PACKET3_INDEX_ATTRIBUTES_INDIRECT', 'PACKET3_INDEX_BASE',
+    'PACKET3_INDEX_BUFFER_SIZE', 'PACKET3_INDEX_TYPE',
+    'PACKET3_INDIRECT_BUFFER', 'PACKET3_INDIRECT_BUFFER_CNST',
+    'PACKET3_INDIRECT_BUFFER_CNST_END', 'PACKET3_INDIRECT_BUFFER_END',
+    'PACKET3_INDIRECT_BUFFER_PASID', 'PACKET3_INDIRECT_BUFFER_PRIV',
+    'PACKET3_INTERRUPT', 'PACKET3_INVALIDATE_TLBS',
+    'PACKET3_LOAD_COMPUTE_STATE', 'PACKET3_LOAD_CONFIG_REG',
+    'PACKET3_LOAD_CONST_RAM', 'PACKET3_LOAD_CONTEXT_REG',
+    'PACKET3_LOAD_CONTEXT_REG_INDEX', 'PACKET3_LOAD_SH_REG',
+    'PACKET3_LOAD_SH_REG_INDEX', 'PACKET3_LOAD_UCONFIG_REG',
+    'PACKET3_MAP_PROCESS', 'PACKET3_MAP_PROCESS_VM',
+    'PACKET3_MAP_QUEUES', 'PACKET3_MEM_SEMAPHORE',
+    'PACKET3_ME_INITIALIZE', 'PACKET3_NOP', 'PACKET3_NUM_INSTANCES',
+    'PACKET3_OCCLUSION_QUERY', 'PACKET3_PFP_SYNC_ME',
+    'PACKET3_PREAMBLE_BEGIN_CLEAR_STATE', 'PACKET3_PREAMBLE_CNTL',
+    'PACKET3_PREAMBLE_END_CLEAR_STATE', 'PACKET3_PRED_EXEC',
+    'PACKET3_PRIME_UTCL2', 'PACKET3_QUERY_STATUS', 'PACKET3_REG_RMW',
+    'PACKET3_RELEASE_MEM', 'PACKET3_RELEASE_MEM_EXECUTE',
+    'PACKET3_RELEASE_MEM_GCR_GL1_INV',
+    'PACKET3_RELEASE_MEM_GCR_GL2_DISCARD',
+    'PACKET3_RELEASE_MEM_GCR_GL2_INV',
+    'PACKET3_RELEASE_MEM_GCR_GL2_RANGE',
+    'PACKET3_RELEASE_MEM_GCR_GL2_US',
+    'PACKET3_RELEASE_MEM_GCR_GL2_WB',
+    'PACKET3_RELEASE_MEM_GCR_GLM_INV',
+    'PACKET3_RELEASE_MEM_GCR_GLM_WB',
+    'PACKET3_RELEASE_MEM_GCR_GLV_INV', 'PACKET3_RELEASE_MEM_GCR_SEQ',
+    'PACKET3_REWIND', 'PACKET3_RUN_LIST', 'PACKET3_SCRATCH_RAM_READ',
+    'PACKET3_SCRATCH_RAM_WRITE', 'PACKET3_SEM_SEL_SIGNAL',
+    'PACKET3_SEM_SEL_SIGNAL_TYPE', 'PACKET3_SEM_SEL_WAIT',
+    'PACKET3_SEM_USE_MAILBOX', 'PACKET3_SET_BASE',
+    'PACKET3_SET_CONFIG_REG', 'PACKET3_SET_CONFIG_REG_END',
+    'PACKET3_SET_CONFIG_REG_START', 'PACKET3_SET_CONTEXT_REG',
+    'PACKET3_SET_CONTEXT_REG_END', 'PACKET3_SET_CONTEXT_REG_INDEX',
+    'PACKET3_SET_CONTEXT_REG_INDIRECT',
+    'PACKET3_SET_CONTEXT_REG_START', 'PACKET3_SET_PREDICATION',
+    'PACKET3_SET_QUEUE_REG', 'PACKET3_SET_Q_PREEMPTION_MODE',
+    'PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM',
+    'PACKET3_SET_RESOURCES', 'PACKET3_SET_SH_REG',
+    'PACKET3_SET_SH_REG_DI', 'PACKET3_SET_SH_REG_DI_MULTI',
+    'PACKET3_SET_SH_REG_END', 'PACKET3_SET_SH_REG_INDEX',
+    'PACKET3_SET_SH_REG_OFFSET', 'PACKET3_SET_SH_REG_START',
+    'PACKET3_SET_UCONFIG_REG', 'PACKET3_SET_UCONFIG_REG_END',
+    'PACKET3_SET_UCONFIG_REG_INDEX', 'PACKET3_SET_UCONFIG_REG_START',
+    'PACKET3_SET_VGPR_REG_DI_MULTI', 'PACKET3_STRMOUT_BUFFER_UPDATE',
+    'PACKET3_SURFACE_SYNC', 'PACKET3_SWITCH_BUFFER',
+    'PACKET3_UNMAP_QUEUES', 'PACKET3_WAIT_ON_CE_COUNTER',
+    'PACKET3_WAIT_ON_DE_COUNTER_DIFF', 'PACKET3_WAIT_REG_MEM',
+    'PACKET3_WAIT_REG_MEM64', 'PACKET3_WRITE_CONST_RAM',
+    'PACKET3_WRITE_DATA', 'PACKET_TYPE0', 'PACKET_TYPE1',
+    'PACKET_TYPE2', 'PACKET_TYPE3', 'PM4_MEC_RELEASE_MEM_DEFINED',
+    'PM4_MEC_WRITE_DATA_DEFINED', 'PM4_MES_HEADER_DEFINED',
+    'WRITE_DATA_addr_incr_enum', 'WRITE_DATA_cache_policy_enum',
+    'WRITE_DATA_dst_sel_enum', 'WRITE_DATA_wr_confirm_enum',
+    'WR_CONFIRM', 'WR_ONE_ADDR',
+    'addr_incr___write_data__do_not_increment_address',
+    'addr_incr___write_data__increment_address',
+    'c__Ea_CACHE_FLUSH_AND_INV_TS_EVENT', 'c_uint32', 'c_uint32',
+    'c_uint32', 'c_uint32', 'c_uint32', 'c_uint32',
+    'cache_policy___write_data__lru',
+    'cache_policy___write_data__stream',
+    'cache_policy__mec_release_mem__lru',
+    'cache_policy__mec_release_mem__stream',
+    'data_sel__mec_release_mem__none',
+    'data_sel__mec_release_mem__send_32_bit_low',
+    'data_sel__mec_release_mem__send_64_bit_data',
+    'data_sel__mec_release_mem__send_cp_perfcounter_hi_lo',
+    'data_sel__mec_release_mem__send_gpu_clock_counter',
+    'data_sel__mec_release_mem__store_gds_data_to_memory',
+    'dst_sel___write_data__gds',
+    'dst_sel___write_data__mem_mapped_register',
+    'dst_sel___write_data__memory',
+    'dst_sel___write_data__memory_mapped_adc_persistent_state',
+    'dst_sel___write_data__tc_l2',
+    'dst_sel__mec_release_mem__memory_controller',
+    'dst_sel__mec_release_mem__queue_write_pointer_poll_mask_bit',
+    'dst_sel__mec_release_mem__queue_write_pointer_register',
+    'dst_sel__mec_release_mem__tc_l2',
+    'event_index__mec_release_mem__end_of_pipe',
+    'event_index__mec_release_mem__shader_done', 'int32_t',
+    'int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_32_bit_compare',
+    'int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_64_bit_compare',
+    'int_sel__mec_release_mem__none',
+    'int_sel__mec_release_mem__send_data_after_write_confirm',
+    'int_sel__mec_release_mem__send_interrupt_after_write_confirm',
+    'int_sel__mec_release_mem__send_interrupt_only',
+    'int_sel__mec_release_mem__unconditionally_send_int_ctxid',
+    'pq_exe_status__mec_release_mem__default',
+    'pq_exe_status__mec_release_mem__phase_update',
+    'struct_PM4_MES_TYPE_3_HEADER_0', 'struct_pm4_mec_release_mem',
+    'struct_pm4_mec_release_mem_1_bitfields2',
+    'struct_pm4_mec_release_mem_2_bitfields3',
+    'struct_pm4_mec_release_mem_3_bitfields4',
+    'struct_pm4_mec_release_mem_3_bitfields4b',
+    'struct_pm4_mec_release_mem_5_bitfields6c',
+    'struct_pm4_mec_write_data_mmio',
+    'struct_pm4_mec_write_data_mmio_1_bitfields2',
+    'struct_pm4_mec_write_data_mmio_2_bitfields3', 'uint32_t',
+    'union_PM4_MES_TYPE_3_HEADER', 'union_pm4_mec_release_mem_0',
+    'union_pm4_mec_release_mem_1', 'union_pm4_mec_release_mem_2',
+    'union_pm4_mec_release_mem_3', 'union_pm4_mec_release_mem_4',
+    'union_pm4_mec_release_mem_5', 'union_pm4_mec_release_mem_6',
+    'union_pm4_mec_write_data_mmio_0',
+    'union_pm4_mec_write_data_mmio_1',
+    'union_pm4_mec_write_data_mmio_2',
+    'wr_confirm___write_data__do_not_wait_for_write_confirmation',
+    'wr_confirm___write_data__wait_for_write_confirmation']

tinygrad/runtime/ops_amd.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from typing import Any, cast, ClassVar
-import os, ctypes, ctypes.util, struct, hashlib, functools, mmap, errno, array, contextlib, sys, select
+import os, ctypes, ctypes.util, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, select
 assert sys.platform != 'win32'
 from dataclasses import dataclass
 from tinygrad.runtime.support.hcq import HCQCompiled, HCQAllocator, HCQBuffer, HWQueue, CLikeArgsState, HCQSignal, HCQProgram, HWInterface
@@ -9,31 +9,19 @@ from tinygrad.device import Compiled, ProfileEvent, BufferSpec, CPUProgram, PROF
 from tinygrad.helpers import getenv, to_mv, round_up, data64_le, mv_address, DEBUG, OSX
 from tinygrad.renderer.cstyle import AMDRenderer
 from tinygrad.renderer.llvmir import AMDLLVMRenderer
-from tinygrad.runtime.autogen import kfd, hsa, amd_gpu, libc, pci, vfio, sqtt
-from tinygrad.runtime.autogen.am import am, gc_11_0_0
+from tinygrad.runtime.autogen import kfd, hsa, libc, pci, vfio, sqtt
+from tinygrad.runtime.autogen.am import am
 from tinygrad.runtime.support.compiler_amd import HIPCompiler, AMDLLVMCompiler
 from tinygrad.runtime.support.elf import elf_loader
 from tinygrad.runtime.support.am.amdev import AMDev, AMMapping
+from tinygrad.runtime.support.amd import AMDRegBase, collect_registers, import_module
 if getenv("IOCTL"): import extra.hip_gpu_driver.hip_ioctl  # noqa: F401 # pylint: disable=unused-import
 
-regBIF_BX_PF1_GPU_HDP_FLUSH_REQ, regBIF_BX_PF1_GPU_HDP_FLUSH_DONE = 0x0106, 0x0107
-
 EVENT_INDEX_PARTIAL_FLUSH = 4 # based on a comment in nvd.h
 WAIT_REG_MEM_FUNCTION_EQ  = 3 # ==
 WAIT_REG_MEM_FUNCTION_NEQ = 4 # !=
 WAIT_REG_MEM_FUNCTION_GEQ = 5 # >=
 
-COMPUTE_SHADER_EN, FORCE_START_AT_000, CS_W32_EN = (1 << 0), (1 << 2), (1 << 15)
-
-def gfxreg(reg): return reg + amd_gpu.GC_BASE__INST0_SEG0 - amd_gpu.PACKET3_SET_SH_REG_START
-def ucfgreg(reg, pkt3_set:bool=True): return reg + amd_gpu.GC_BASE__INST0_SEG1 - (amd_gpu.PACKET3_SET_UCONFIG_REG_START if pkt3_set else 0)
-def nbioreg(reg): return reg + amd_gpu.NBIO_BASE__INST0_SEG2
-
-# This can potentially be shared with AMRegister._parse_kwargs. NOTE: This is hardcoded to gfx11, bitfields might be different in other gfxvers.
-# Currently not a problem because this is only used by sqtt and sqtt is only supported on 7900xtx
-def encode_bitfields(regname: str, **kwargs) -> int:
-  return functools.reduce(lambda x,y: x|y, [v << getattr(gc_11_0_0, f'{regname}__{k.upper()}__SHIFT') for k,v in kwargs.items()], 0)
-
 class AMDSignal(HCQSignal):
   def __init__(self, base_addr:int|None=None, **kwargs):
     super().__init__(base_addr, **kwargs, timestamp_divider=100, dev_t=AMDDevice)
@@ -43,61 +31,71 @@ class AMDSignal(HCQSignal):
     if time_spent_waiting_ms > 2000 and self.timeline_for_device is not None: self.timeline_for_device.dev_iface.sleep(200)
 
 class AMDComputeQueue(HWQueue):
+  def __init__(self, dev:AMDDevice):
+    self.soc, self.pm4, self.gc, self.nbio = dev.soc, dev.pm4, dev.gc, dev.nbio
+    super().__init__()
+
   def __del__(self):
     if self.binded_device is not None:
       self.binded_device.allocator.free(self.hw_page, self.hw_page.size, BufferSpec(cpu_access=True, nolru=True, uncached=True))
 
-  def pkt3(self, cmd, *vals): self.q(amd_gpu.PACKET3(cmd, len(vals) - 1), *vals)
+  def pkt3(self, cmd, *vals): self.q(self.pm4.PACKET3(cmd, len(vals) - 1), *vals)
+
+  def gfxreg(self, reg:AMDReg): return reg.addr - self.pm4.PACKET3_SET_SH_REG_START
+  def ucfgreg(self, reg:AMDReg): return reg.addr - self.pm4.PACKET3_SET_UCONFIG_REG_START
 
   def sqtt_userdata(self, data, *extra_dwords):
     data_ints = [x[0] for x in struct.iter_unpack('<I', bytes(data))] + list(extra_dwords)
     for i in range(0, len(data_ints), 2):
-      self.pkt3(amd_gpu.PACKET3_SET_UCONFIG_REG, ucfgreg(amd_gpu.regSQ_THREAD_TRACE_USERDATA_2), *data_ints[i:i+2])
+      self.pkt3(self.pm4.PACKET3_SET_UCONFIG_REG, self.ucfgreg(self.gc.regSQ_THREAD_TRACE_USERDATA_2), *data_ints[i:i+2])
 
   def wait_reg_mem(self, value, mask=0xffffffff, mem=None, reg_req=None, reg_done=None):
-    wrm_info_dw = amd_gpu.WAIT_REG_MEM_MEM_SPACE(int(mem is not None)) | amd_gpu.WAIT_REG_MEM_OPERATION(int(mem is None)) \
-                | amd_gpu.WAIT_REG_MEM_FUNCTION(WAIT_REG_MEM_FUNCTION_GEQ) | amd_gpu.WAIT_REG_MEM_ENGINE(0)
+    wrm_info_dw = self.pm4.WAIT_REG_MEM_MEM_SPACE(int(mem is not None)) | self.pm4.WAIT_REG_MEM_OPERATION(int(mem is None)) \
+                | self.pm4.WAIT_REG_MEM_FUNCTION(WAIT_REG_MEM_FUNCTION_GEQ) | self.pm4.WAIT_REG_MEM_ENGINE(0)
 
-    self.pkt3(amd_gpu.PACKET3_WAIT_REG_MEM, wrm_info_dw, *(data64_le(mem) if mem is not None else (reg_req, reg_done)), value, mask, 4)
+    self.pkt3(self.pm4.PACKET3_WAIT_REG_MEM, wrm_info_dw, *(data64_le(mem) if mem is not None else (reg_req, reg_done)), value, mask, 4)
 
   def acquire_mem(self, addr=0x0, sz=(1 << 64)-1, gli=1, glm=1, glk=1, glv=1, gl1=1, gl2=1):
-    cache_flags_dw = amd_gpu.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(gli) \
-                   | amd_gpu.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(glm) | amd_gpu.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(glm) \
-                   | amd_gpu.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(glk) | amd_gpu.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_WB(glk) \
-                   | amd_gpu.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(glv) | amd_gpu.PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(gl1) \
-                   | amd_gpu.PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(gl2) | amd_gpu.PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(gl2)
+    cache_flags_dw = self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(gli) \
+                   | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(glm) | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(glm) \
+                   | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(glk) | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_WB(glk) \
+                   | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(glv) | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(gl1) \
+                   | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(gl2) | self.pm4.PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(gl2)
 
-    self.pkt3(amd_gpu.PACKET3_ACQUIRE_MEM, 0, *data64_le(sz), *data64_le(addr), 0, cache_flags_dw)
+    self.pkt3(self.pm4.PACKET3_ACQUIRE_MEM, 0, *data64_le(sz), *data64_le(addr), 0, cache_flags_dw)
 
   def release_mem(self, address, value, data_sel, int_sel, ctxid=0, cache_flush=False):
-    cache_flags_dw = 0 if not cache_flush else (amd_gpu.PACKET3_RELEASE_MEM_GCR_GLV_INV | amd_gpu.PACKET3_RELEASE_MEM_GCR_GL1_INV \
-                   | amd_gpu.PACKET3_RELEASE_MEM_GCR_GL2_INV | amd_gpu.PACKET3_RELEASE_MEM_GCR_GLM_WB \
-                   | amd_gpu.PACKET3_RELEASE_MEM_GCR_GLM_INV | amd_gpu.PACKET3_RELEASE_MEM_GCR_GL2_WB | amd_gpu.PACKET3_RELEASE_MEM_GCR_SEQ)
+    cache_flags_dw = 0 if not cache_flush else (self.pm4.PACKET3_RELEASE_MEM_GCR_GLV_INV | self.pm4.PACKET3_RELEASE_MEM_GCR_GL1_INV \
+                   | self.pm4.PACKET3_RELEASE_MEM_GCR_GL2_INV | self.pm4.PACKET3_RELEASE_MEM_GCR_GLM_WB \
+                   | self.pm4.PACKET3_RELEASE_MEM_GCR_GLM_INV | self.pm4.PACKET3_RELEASE_MEM_GCR_GL2_WB | self.pm4.PACKET3_RELEASE_MEM_GCR_SEQ)
 
-    event_dw = amd_gpu.PACKET3_RELEASE_MEM_EVENT_TYPE(amd_gpu.CACHE_FLUSH_AND_INV_TS_EVENT) \
-             | amd_gpu.PACKET3_RELEASE_MEM_EVENT_INDEX(amd_gpu.event_index__mec_release_mem__end_of_pipe)
+    event_dw = self.pm4.PACKET3_RELEASE_MEM_EVENT_TYPE(self.pm4.CACHE_FLUSH_AND_INV_TS_EVENT) \
+             | self.pm4.PACKET3_RELEASE_MEM_EVENT_INDEX(self.pm4.event_index__mec_release_mem__end_of_pipe)
 
-    memsel_dw = amd_gpu.PACKET3_RELEASE_MEM_DATA_SEL(data_sel) | amd_gpu.PACKET3_RELEASE_MEM_INT_SEL(int_sel) | amd_gpu.PACKET3_RELEASE_MEM_DST_SEL(0)
+    memsel_dw = self.pm4.PACKET3_RELEASE_MEM_DATA_SEL(data_sel) | self.pm4.PACKET3_RELEASE_MEM_INT_SEL(int_sel) \
+              | self.pm4.PACKET3_RELEASE_MEM_DST_SEL(0)
 
-    self.pkt3(amd_gpu.PACKET3_RELEASE_MEM, event_dw | cache_flags_dw, memsel_dw, *data64_le(address), *data64_le(value), ctxid)
+    self.pkt3(self.pm4.PACKET3_RELEASE_MEM, event_dw | cache_flags_dw, memsel_dw, *data64_le(address), *data64_le(value), ctxid)
 
   def memory_barrier(self):
-    self.wait_reg_mem(reg_req=nbioreg(regBIF_BX_PF1_GPU_HDP_FLUSH_REQ), reg_done=nbioreg(regBIF_BX_PF1_GPU_HDP_FLUSH_DONE), value=0xffffffff)
+    self.wait_reg_mem(reg_req=self.nbio.regBIF_BX_PF0_GPU_HDP_FLUSH_REQ.addr, reg_done=self.nbio.regBIF_BX_PF0_GPU_HDP_FLUSH_DONE.addr,
+                      value=0xffffffff)
     self.acquire_mem()
     return self
 
   def spi_config(self, tracing:bool):
-    spi_config_cntl = encode_bitfields('SPI_CONFIG_CNTL', ps_pkr_priority_cntl=3, exp_priority_order=3, gpr_write_priority=0x2c688,
-                                       enable_sqg_bop_events=int(tracing), enable_sqg_top_events=int(tracing))
-    self.pkt3(amd_gpu.PACKET3_SET_UCONFIG_REG, ucfgreg(amd_gpu.regSPI_CONFIG_CNTL), spi_config_cntl)
+    spi_config_cntl = self.gc.regSPI_CONFIG_CNTL.encode(ps_pkr_priority_cntl=3, exp_priority_order=3, gpr_write_priority=0x2c688,
+                                                        enable_sqg_bop_events=int(tracing), enable_sqg_top_events=int(tracing))
+    self.pkt3(self.pm4.PACKET3_SET_UCONFIG_REG, self.ucfgreg(self.gc.regSPI_CONFIG_CNTL), spi_config_cntl)
 
   def sqtt_config(self, tracing:bool):
-    sq_thread_trace_ctrl = encode_bitfields('SQ_THREAD_TRACE_CTRL', draw_event_en=1, spi_stall_en=1, sq_stall_en=1, reg_at_hwm=2, hiwater=1,
-                                            rt_freq=amd_gpu.SQ_TT_RT_FREQ_4096_CLK, util_timer=amd_gpu.SQ_TT_UTIL_TIMER_250_CLK, mode=int(tracing))
-    self.pkt3(amd_gpu.PACKET3_SET_UCONFIG_REG, ucfgreg(amd_gpu.regSQ_THREAD_TRACE_CTRL), sq_thread_trace_ctrl)
+    sq_thread_trace_ctrl = self.gc.regSQ_THREAD_TRACE_CTRL.encode(draw_event_en=1, spi_stall_en=1, sq_stall_en=1, reg_at_hwm=2, hiwater=1,
+                                                                  rt_freq=self.soc.SQ_TT_RT_FREQ_4096_CLK,
+                                                                  util_timer=self.soc.SQ_TT_UTIL_TIMER_250_CLK, mode=int(tracing))
+    self.pkt3(self.pm4.PACKET3_SET_UCONFIG_REG, self.ucfgreg(self.gc.regSQ_THREAD_TRACE_CTRL), sq_thread_trace_ctrl)
 
   def grbm_gfx_index(self, **kwargs):
-    self.pkt3(amd_gpu.PACKET3_SET_UCONFIG_REG, ucfgreg(amd_gpu.regGRBM_GFX_INDEX), encode_bitfields('GRBM_GFX_INDEX', **kwargs))
+    self.pkt3(self.pm4.PACKET3_SET_UCONFIG_REG, self.ucfgreg(self.gc.regGRBM_GFX_INDEX), self.gc.regGRBM_GFX_INDEX.encode(**kwargs))
 
   # Magic values from mesa/src/amd/vulkan/radv_sqtt.c:radv_emit_spi_config_cntl and src/amd/common/ac_sqtt.c:ac_sqtt_emit_start
   def start_trace(self, buf0s:list[HCQBuffer], se_mask:int):
@@ -107,31 +105,31 @@ class AMDComputeQueue(HWQueue):
     for se in range(len(buf0s)):
       self.grbm_gfx_index(se_index=se, instance_broadcast_writes=1) # select se, broadcast to all instances in that se
       buf0_lo, buf0_hi = data64_le(buf0s[se].va_addr>>12)
-      self.pkt3(amd_gpu.PACKET3_SET_UCONFIG_REG, ucfgreg(amd_gpu.regSQ_THREAD_TRACE_BUF0_SIZE),
-                encode_bitfields('SQ_THREAD_TRACE_BUF0_SIZE', base_hi=buf0_hi, size=buf0s[se].size>>12))
-      self.pkt3(amd_gpu.PACKET3_SET_UCONFIG_REG, ucfgreg(amd_gpu.regSQ_THREAD_TRACE_BUF0_BASE), buf0_lo)
+      self.pkt3(self.pm4.PACKET3_SET_UCONFIG_REG, self.ucfgreg(self.gc.regSQ_THREAD_TRACE_BUF0_SIZE),
+                self.gc.regSQ_THREAD_TRACE_BUF0_SIZE.encode(base_hi=buf0_hi, size=buf0s[se].size>>12))
+      self.pkt3(self.pm4.PACKET3_SET_UCONFIG_REG, self.ucfgreg(self.gc.regSQ_THREAD_TRACE_BUF0_BASE), buf0_lo)
       # NOTE: SQTT can only trace instructions on one simd per se, this selects first simd in first wgp in first sa.
       # For RGP to display instruction trace it has to see it on first SE. Howerver ACE/MEC/whatever does the dispatching starting with second se,
       # and on amdgpu/non-AM it also does weird things with dispatch order inside se: around 7 times out of 10 it starts from the last cu, but
       # sometimes not, especially if the kernel has more than one wavefront which means that kernels with small global size might get unlucky and
       # be dispatched on something else and not be seen in instruction tracing tab. You can force the wavefronts of a kernel to be dispatched on the
       # CUs you want to by disabling other CUs via bits in regCOMPUTE_STATIC_THREAD_MGMT_SE<x> and trace even kernels that only have one wavefront.
-      self.pkt3(amd_gpu.PACKET3_SET_UCONFIG_REG, ucfgreg(amd_gpu.regSQ_THREAD_TRACE_MASK),
-                encode_bitfields('SQ_THREAD_TRACE_MASK', wtype_include=amd_gpu.SQ_TT_WTYPE_INCLUDE_CS_BIT, simd_sel=0, wgp_sel=0, sa_sel=0))
-      REG_INCLUDE = amd_gpu.SQ_TT_TOKEN_MASK_SQDEC_BIT | amd_gpu.SQ_TT_TOKEN_MASK_SHDEC_BIT | amd_gpu.SQ_TT_TOKEN_MASK_GFXUDEC_BIT | \
-                    amd_gpu.SQ_TT_TOKEN_MASK_COMP_BIT | amd_gpu.SQ_TT_TOKEN_MASK_CONTEXT_BIT | amd_gpu.SQ_TT_TOKEN_MASK_CONTEXT_BIT
-      TOKEN_EXCLUDE = 1 << amd_gpu.SQ_TT_TOKEN_EXCLUDE_PERF_SHIFT
+      self.pkt3(self.pm4.PACKET3_SET_UCONFIG_REG, self.ucfgreg(self.gc.regSQ_THREAD_TRACE_MASK),
+                self.gc.regSQ_THREAD_TRACE_MASK.encode(wtype_include=self.soc.SQ_TT_WTYPE_INCLUDE_CS_BIT, simd_sel=0, wgp_sel=0, sa_sel=0))
+      REG_INCLUDE = self.soc.SQ_TT_TOKEN_MASK_SQDEC_BIT | self.soc.SQ_TT_TOKEN_MASK_SHDEC_BIT | self.soc.SQ_TT_TOKEN_MASK_GFXUDEC_BIT | \
+                    self.soc.SQ_TT_TOKEN_MASK_COMP_BIT | self.soc.SQ_TT_TOKEN_MASK_CONTEXT_BIT | self.soc.SQ_TT_TOKEN_MASK_CONTEXT_BIT
+      TOKEN_EXCLUDE = 1 << self.soc.SQ_TT_TOKEN_EXCLUDE_PERF_SHIFT
       if not (se_mask >> se) & 0b1:
-        TOKEN_EXCLUDE |= 1 << amd_gpu.SQ_TT_TOKEN_EXCLUDE_VMEMEXEC_SHIFT | 1 << amd_gpu.SQ_TT_TOKEN_EXCLUDE_ALUEXEC_SHIFT | \
-                         1 << amd_gpu.SQ_TT_TOKEN_EXCLUDE_VALUINST_SHIFT | 1 << amd_gpu.SQ_TT_TOKEN_EXCLUDE_IMMEDIATE_SHIFT | \
-                         1 << amd_gpu.SQ_TT_TOKEN_EXCLUDE_INST_SHIFT
-      self.pkt3(amd_gpu.PACKET3_SET_UCONFIG_REG, ucfgreg(amd_gpu.regSQ_THREAD_TRACE_TOKEN_MASK),
-                encode_bitfields('SQ_THREAD_TRACE_TOKEN_MASK', reg_include=REG_INCLUDE, token_exclude=TOKEN_EXCLUDE, bop_events_token_include=1))
+        TOKEN_EXCLUDE |= 1 << self.soc.SQ_TT_TOKEN_EXCLUDE_VMEMEXEC_SHIFT | 1 << self.soc.SQ_TT_TOKEN_EXCLUDE_ALUEXEC_SHIFT | \
+                         1 << self.soc.SQ_TT_TOKEN_EXCLUDE_VALUINST_SHIFT | 1 << self.soc.SQ_TT_TOKEN_EXCLUDE_IMMEDIATE_SHIFT | \
+                         1 << self.soc.SQ_TT_TOKEN_EXCLUDE_INST_SHIFT
+      self.pkt3(self.pm4.PACKET3_SET_UCONFIG_REG, self.ucfgreg(self.gc.regSQ_THREAD_TRACE_TOKEN_MASK),
+                self.gc.regSQ_THREAD_TRACE_TOKEN_MASK.encode(reg_include=REG_INCLUDE, token_exclude=TOKEN_EXCLUDE, bop_events_token_include=1))
       # Enable SQTT
       self.sqtt_config(tracing=True)
     # Restore global broadcasting
     self.grbm_gfx_index(se_broadcast_writes=1, sa_broadcast_writes=1, instance_broadcast_writes=1)
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_THREAD_TRACE_ENABLE), 1)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_THREAD_TRACE_ENABLE), 1)
     self.memory_barrier()
     return self
 
@@ -139,24 +137,24 @@ class AMDComputeQueue(HWQueue):
   def stop_trace(self, ses: int, wptrs: HCQBuffer):
     self.memory_barrier()
     # Start shutting everything down
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_THREAD_TRACE_ENABLE), 0)
-    self.pkt3(amd_gpu.PACKET3_EVENT_WRITE, amd_gpu.EVENT_TYPE(amd_gpu.THREAD_TRACE_FINISH) | amd_gpu.EVENT_INDEX(0))
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_THREAD_TRACE_ENABLE), 0)
+    self.pkt3(self.pm4.PACKET3_EVENT_WRITE, self.pm4.EVENT_TYPE(self.soc.THREAD_TRACE_FINISH) | self.pm4.EVENT_INDEX(0))
     # For each SE wait for finish to complete and copy regSQ_THREAD_TRACE_WPTR to know where in the buffer trace data ends
     for se in range(ses):
       self.grbm_gfx_index(se_index=se, instance_broadcast_writes=1) # select se, broadcast to all instances in that se
       # Wait for FINISH_PENDING==0
-      self.pkt3(amd_gpu.PACKET3_WAIT_REG_MEM, amd_gpu.WAIT_REG_MEM_FUNCTION(WAIT_REG_MEM_FUNCTION_EQ),
-                ucfgreg(amd_gpu.regSQ_THREAD_TRACE_STATUS, False), 0, 0, gc_11_0_0.SQ_THREAD_TRACE_STATUS__FINISH_PENDING_MASK, 4)
+      self.pkt3(self.pm4.PACKET3_WAIT_REG_MEM, self.pm4.WAIT_REG_MEM_FUNCTION(WAIT_REG_MEM_FUNCTION_EQ),
+                self.gc.regSQ_THREAD_TRACE_STATUS.addr, 0, 0, self.gc.SQ_THREAD_TRACE_STATUS__FINISH_PENDING_MASK, 4)
       # Wait for FINISH_DONE!=0
-      self.pkt3(amd_gpu.PACKET3_WAIT_REG_MEM, amd_gpu.WAIT_REG_MEM_FUNCTION(WAIT_REG_MEM_FUNCTION_NEQ),
-                ucfgreg(amd_gpu.regSQ_THREAD_TRACE_STATUS, False), 0, 0, gc_11_0_0.SQ_THREAD_TRACE_STATUS__FINISH_DONE_MASK, 4)
+      self.pkt3(self.pm4.PACKET3_WAIT_REG_MEM, self.pm4.WAIT_REG_MEM_FUNCTION(WAIT_REG_MEM_FUNCTION_NEQ),
+                self.gc.regSQ_THREAD_TRACE_STATUS.addr, 0, 0, self.gc.SQ_THREAD_TRACE_STATUS__FINISH_DONE_MASK, 4)
       # Disable SQTT
       self.sqtt_config(tracing=False)
       # Wait for BUSY==0
-      self.pkt3(amd_gpu.PACKET3_WAIT_REG_MEM, amd_gpu.WAIT_REG_MEM_FUNCTION(WAIT_REG_MEM_FUNCTION_EQ),
-                ucfgreg(amd_gpu.regSQ_THREAD_TRACE_STATUS, False), 0, 0, gc_11_0_0.SQ_THREAD_TRACE_STATUS__BUSY_MASK, 4)
-      # Copy WPTR to memory (src_sel = perf, dst_sel = tc_l2, wr_confirm = True), ucfgreg with False adds GC_BASE__INST0_SEG1 but not pkt3 reg offset
-      self.pkt3(amd_gpu.PACKET3_COPY_DATA, 1 << 20 | 2 << 8 | 4, ucfgreg(amd_gpu.regSQ_THREAD_TRACE_WPTR, False), 0, *data64_le(wptrs.va_addr+(se*4)))
+      self.pkt3(self.pm4.PACKET3_WAIT_REG_MEM, self.pm4.WAIT_REG_MEM_FUNCTION(WAIT_REG_MEM_FUNCTION_EQ),
+                self.gc.regSQ_THREAD_TRACE_STATUS.addr, 0, 0, self.gc.SQ_THREAD_TRACE_STATUS__BUSY_MASK, 4)
+      # Copy WPTR to memory (src_sel = perf, dst_sel = tc_l2, wr_confirm = True)
+      self.pkt3(self.pm4.PACKET3_COPY_DATA, 1 << 20 | 2 << 8 | 4, self.gc.regSQ_THREAD_TRACE_WPTR.addr, 0, *data64_le(wptrs.va_addr+(se*4)))
     # Restore global broadcasting
     self.grbm_gfx_index(se_broadcast_writes=1, sa_broadcast_writes=1, instance_broadcast_writes=1)
     self.spi_config(tracing=False)
@@ -198,25 +196,26 @@ class AMDComputeQueue(HWQueue):
       ), *global_size)
       prg.dev.cmd_id += 1
 
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_PGM_LO), *data64_le(prg.prog_addr >> 8))
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_PGM_RSRC1), prg.rsrc1, prg.rsrc2)
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_PGM_RSRC3), 0)
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_TMPRING_SIZE), prg.dev.tmpring_size)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_PGM_LO), *data64_le(prg.prog_addr >> 8))
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_PGM_RSRC1), prg.rsrc1, prg.rsrc2)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_PGM_RSRC3), 0)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_TMPRING_SIZE), prg.dev.tmpring_size)
     if prg.dev.has_scratch_base_registers:
-      self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_DISPATCH_SCRATCH_BASE_LO), *data64_le(prg.dev.scratch.va_addr >> 8))
-    if prg.dev.target < 110000: self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.mmCP_COHER_START_DELAY), 0x20)
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_RESTART_X), 0, 0, 0)
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xFFFFFFFF, 0xFFFFFFFF)
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xFFFFFFFF, 0xFFFFFFFF)
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF)
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_USER_DATA_0), *user_regs)
+      self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_DISPATCH_SCRATCH_BASE_LO), *data64_le(prg.dev.scratch.va_addr >> 8))
+    if prg.dev.target < 110000: self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.mmCP_COHER_START_DELAY), 0x20)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_RESTART_X), 0, 0, 0)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xFFFFFFFF, 0xFFFFFFFF)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xFFFFFFFF, 0xFFFFFFFF)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_USER_DATA_0), *user_regs)
 
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_START_X), 0, 0, 0, *local_size, 0, 0)
-    self.pkt3(amd_gpu.PACKET3_SET_SH_REG, gfxreg(amd_gpu.regCOMPUTE_RESOURCE_LIMITS), 0)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_START_X), 0, 0, 0, *local_size, 0, 0)
+    self.pkt3(self.pm4.PACKET3_SET_SH_REG, self.gfxreg(self.gc.regCOMPUTE_RESOURCE_LIMITS), 0)
 
-    self.pkt3(amd_gpu.PACKET3_DISPATCH_DIRECT, *global_size, CS_W32_EN | FORCE_START_AT_000 | COMPUTE_SHADER_EN)
-    if prg.dev.sqtt_enabled: self.pkt3(amd_gpu.PACKET3_EVENT_WRITE, amd_gpu.EVENT_TYPE(amd_gpu.THREAD_TRACE_MARKER) | amd_gpu.EVENT_INDEX(0))
-    self.pkt3(amd_gpu.PACKET3_EVENT_WRITE, amd_gpu.EVENT_TYPE(amd_gpu.CS_PARTIAL_FLUSH) | amd_gpu.EVENT_INDEX(EVENT_INDEX_PARTIAL_FLUSH))
+    DISPATCH_INITIATOR = self.gc.regCOMPUTE_DISPATCH_INITIATOR.encode(cs_w32_en=1, force_start_at_000=1, compute_shader_en=1)
+    self.pkt3(self.pm4.PACKET3_DISPATCH_DIRECT, *global_size, DISPATCH_INITIATOR)
+    if prg.dev.sqtt_enabled: self.pkt3(self.pm4.PACKET3_EVENT_WRITE, self.pm4.EVENT_TYPE(self.soc.THREAD_TRACE_MARKER) | self.pm4.EVENT_INDEX(0))
+    self.pkt3(self.pm4.PACKET3_EVENT_WRITE, self.pm4.EVENT_TYPE(self.soc.CS_PARTIAL_FLUSH) | self.pm4.EVENT_INDEX(EVENT_INDEX_PARTIAL_FLUSH))
     return self
 
   def wait(self, signal:AMDSignal, value:sint=0):
@@ -224,17 +223,17 @@ class AMDComputeQueue(HWQueue):
     return self
 
   def timestamp(self, signal:AMDSignal):
-    self.release_mem(signal.timestamp_addr, 0, amd_gpu.data_sel__mec_release_mem__send_gpu_clock_counter, amd_gpu.int_sel__mec_release_mem__none)
+    self.release_mem(signal.timestamp_addr, 0, self.pm4.data_sel__mec_release_mem__send_gpu_clock_counter, self.pm4.int_sel__mec_release_mem__none)
     return self
 
   def signal(self, signal:AMDSignal, value:sint=0):
     # NOTE: this needs an EOP buffer on the queue or it will NULL pointer
-    self.release_mem(signal.value_addr, value, amd_gpu.data_sel__mec_release_mem__send_32_bit_low,
-                     amd_gpu.int_sel__mec_release_mem__send_interrupt_after_write_confirm, cache_flush=True)
+    self.release_mem(signal.value_addr, value, self.pm4.data_sel__mec_release_mem__send_32_bit_low,
+                     self.pm4.int_sel__mec_release_mem__send_interrupt_after_write_confirm, cache_flush=True)
 
     if not AMDDevice.driverless and (dev:=signal.timeline_for_device) is not None:
-      self.release_mem(dev.queue_event_mailbox_ptr, dev.queue_event.event_id, amd_gpu.data_sel__mec_release_mem__send_32_bit_low,
-                       amd_gpu.int_sel__mec_release_mem__send_interrupt_after_write_confirm, ctxid=dev.queue_event.event_id)
+      self.release_mem(dev.queue_event_mailbox_ptr, dev.queue_event.event_id, self.pm4.data_sel__mec_release_mem__send_32_bit_low,
+                       self.pm4.int_sel__mec_release_mem__send_interrupt_after_write_confirm, ctxid=dev.queue_event.event_id)
     return self
 
   def bind(self, dev:AMDDevice):
@@ -243,8 +242,8 @@ class AMDComputeQueue(HWQueue):
     hw_view = to_mv(self.hw_page.va_addr, self.hw_page.size).cast("I")
     for i, value in enumerate(self._q): hw_view[i] = value
 
-    self.indirect_cmd = [amd_gpu.PACKET3(amd_gpu.PACKET3_INDIRECT_BUFFER, 2), *data64_le(self.hw_page.va_addr),
-                         len(self._q) | amd_gpu.INDIRECT_BUFFER_VALID]
+    self.indirect_cmd = [self.pm4.PACKET3(self.pm4.PACKET3_INDIRECT_BUFFER, 2), *data64_le(self.hw_page.va_addr),
+                         len(self._q) | self.pm4.INDIRECT_BUFFER_VALID]
     self._q = hw_view
     return self
 
@@ -257,8 +256,8 @@ class AMDComputeQueue(HWQueue):
     dev.compute_queue.signal_doorbell(dev)
 
 class AMDCopyQueue(HWQueue):
-  def __init__(self, max_copy_size=0x40000000):
-    self.internal_cmd_sizes, self.max_copy_size = [], max_copy_size
+  def __init__(self, dev, max_copy_size=0x40000000):
+    self.sdma, self.internal_cmd_sizes, self.max_copy_size = dev.sdma, [], max_copy_size
     super().__init__()
 
   def q(self, *arr):
@@ -271,30 +270,30 @@ class AMDCopyQueue(HWQueue):
     for _ in range(copy_commands):
       step_copy_size = min(copy_size - copied, self.max_copy_size)
 
-      self.q(amd_gpu.SDMA_OP_COPY | amd_gpu.SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(amd_gpu.SDMA_SUBOP_COPY_LINEAR),
-        amd_gpu.SDMA_PKT_COPY_LINEAR_COUNT_COUNT(step_copy_size - 1), 0, *data64_le(src + copied), *data64_le(dest + copied))
+      self.q(self.sdma.SDMA_OP_COPY | self.sdma.SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(self.sdma.SDMA_SUBOP_COPY_LINEAR),
+        self.sdma.SDMA_PKT_COPY_LINEAR_COUNT_COUNT(step_copy_size - 1), 0, *data64_le(src + copied), *data64_le(dest + copied))
 
       copied += step_copy_size
     return self
 
   def signal(self, signal:AMDSignal, value:sint=0):
-    self.q(amd_gpu.SDMA_OP_FENCE | amd_gpu.SDMA_PKT_FENCE_HEADER_MTYPE(3), *data64_le(signal.value_addr), value)
+    self.q(self.sdma.SDMA_OP_FENCE | self.sdma.SDMA_PKT_FENCE_HEADER_MTYPE(3), *data64_le(signal.value_addr), value)
 
     if not AMDDevice.driverless and (dev:=signal.timeline_for_device) is not None:
-      self.q(amd_gpu.SDMA_OP_FENCE | amd_gpu.SDMA_PKT_FENCE_HEADER_MTYPE(3), *data64_le(dev.queue_event_mailbox_ptr), dev.queue_event.event_id)
-      self.q(amd_gpu.SDMA_OP_TRAP, amd_gpu.SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(dev.queue_event.event_id))
-    elif AMDDevice.driverless: self.q(amd_gpu.SDMA_OP_TRAP, amd_gpu.SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0))
+      self.q(self.sdma.SDMA_OP_FENCE | self.sdma.SDMA_PKT_FENCE_HEADER_MTYPE(3), *data64_le(dev.queue_event_mailbox_ptr), dev.queue_event.event_id)
+      self.q(self.sdma.SDMA_OP_TRAP, self.sdma.SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(dev.queue_event.event_id))
+    elif AMDDevice.driverless: self.q(self.sdma.SDMA_OP_TRAP, self.sdma.SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0))
 
     return self
 
   def wait(self, signal:AMDSignal, value:sint=0):
-    self.q(amd_gpu.SDMA_OP_POLL_REGMEM | amd_gpu.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(WAIT_REG_MEM_FUNCTION_GEQ) | \
-           amd_gpu.SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1), *data64_le(signal.value_addr), value, 0xffffffff,
-           amd_gpu.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(0x04) | amd_gpu.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff))
+    self.q(self.sdma.SDMA_OP_POLL_REGMEM | self.sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(WAIT_REG_MEM_FUNCTION_GEQ) | \
+           self.sdma.SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1), *data64_le(signal.value_addr), value, 0xffffffff,
+           self.sdma.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(0x04) | self.sdma.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff))
     return self
 
   def timestamp(self, signal:AMDSignal):
-    self.q(amd_gpu.SDMA_OP_TIMESTAMP | amd_gpu.SDMA_PKT_TIMESTAMP_GET_HEADER_SUB_OP(amd_gpu.SDMA_SUBOP_TIMESTAMP_GET_GLOBAL),
+    self.q(self.sdma.SDMA_OP_TIMESTAMP | self.sdma.SDMA_PKT_TIMESTAMP_GET_HEADER_SUB_OP(self.sdma.SDMA_SUBOP_TIMESTAMP_GET_GLOBAL),
            *data64_le(signal.timestamp_addr))
     return self
 
@@ -306,7 +305,8 @@ class AMDCopyQueue(HWQueue):
     hw_view = to_mv(self.hw_page.va_addr, self.hw_page.size).cast("I")
     for i in range(qsz): hw_view[i] = self._q[i] if i < len(self._q) else 0
 
-    self.indirect_cmd = [amd_gpu.SDMA_OP_INDIRECT | amd_gpu.SDMA_PKT_INDIRECT_HEADER_VMID(0), *data64_le(self.hw_page.va_addr), qsz, *data64_le(0)]
+    self.indirect_cmd = [self.sdma.SDMA_OP_INDIRECT | self.sdma.SDMA_PKT_INDIRECT_HEADER_VMID(0), *data64_le(self.hw_page.va_addr), qsz,
+                         *data64_le(0)]
     self._q, self.cmd_sizes = hw_view, [len(self.indirect_cmd)]
 
   def _submit(self, dev:AMDDevice):
@@ -415,6 +415,25 @@ class AMDQueueDesc:
     if dev.driverless and getenv("AMD_ALLOC_QUEUE_DEV_MEM", 1): dev.dev_iface.adev.gmc.flush_hdp()
     self.doorbell[0] = self.put_value
 
+@dataclass(frozen=True)
+class AMDReg(AMDRegBase):
+  ip: AMDIP
+  @property
+  def addr(self): return self.ip.bases[self.segment] + self.offset
+
+@dataclass(frozen=True)
+class AMDIP:
+  name: str
+  version: tuple[int, ...]
+  bases: tuple[int, ...]
+  @functools.cached_property
+  def module(self): return import_module(self.name, self.version)
+  @functools.cached_property
+  def regs(self): return collect_registers(self.module, cls=functools.partial(AMDReg, ip=self))
+  def __getattr__(self, name:str):
+    if name in self.regs: return self.regs[name]
+    return getattr(self.module, name)
+
 class KFDIface:
   kfd:HWInterface|None = None
   event_page:HCQBuffer|None = None
@@ -441,6 +460,12 @@ class KFDIface:
 
     self.gpu_id = int(HWInterface(f"{kfd_topo_path}/{KFDIface.gpus[device_id]}/gpu_id").read())
     self.props = {l.split()[0]: int(l.split()[1]) for l in HWInterface(f"{kfd_topo_path}/{KFDIface.gpus[device_id]}/properties").read().splitlines()}
+    ip_base = f"/sys/class/drm/renderD{self.props['drm_render_minor']}/device/ip_discovery/die/0"
+    id2ip = {am.GC_HWID: am.GC_HWIP, am.SDMA0_HWID: am.SDMA0_HWIP, am.NBIF_HWID: am.NBIF_HWIP}
+    self.ip_versions = {id2ip[int(hwid)]:tuple(int(HWInterface(f'{ip_base}/{hwid}/0/{part}').read()) for part in ['major', 'minor', 'revision'])
+                        for hwid in HWInterface(ip_base).listdir() if hwid.isnumeric() and int(hwid) in id2ip}
+    self.ip_offsets = {id2ip[int(hwid)]:tuple(int(x, 16) for x in HWInterface(f'{ip_base}/{hwid}/0/base_addr').read().splitlines())
+                        for hwid in HWInterface(ip_base).listdir() if hwid.isnumeric() and int(hwid) in id2ip}
     self.drm_fd = HWInterface(f"/dev/dri/renderD{self.props['drm_render_minor']}", os.O_RDWR)
 
     kfd.AMDKFD_IOC_ACQUIRE_VM(KFDIface.kfd, drm_fd=self.drm_fd.fd, gpu_id=self.gpu_id)
@@ -609,6 +634,8 @@ class PCIIface:
     self.bar_info = {j:(int(start,16), int(end,16), int(flgs,16)) for j,(start,end,flgs) in enumerate(l.split() for l in bar_info)}
 
     self.adev = AMDev(self.pcibus, self._map_pci_range(0), dbell:=self._map_pci_range(2).cast('Q'), self._map_pci_range(5).cast('I'))
+    self.ip_versions = self.adev.ip_ver
+    self.ip_offsets = {hwip: tuple(instances[0]) for hwip,instances in self.adev.regs_offset.items()}
     self.doorbell_cpu_addr = mv_address(dbell)
 
     pci_cmd = int.from_bytes(self.cfg_fd.read(2, binary=True, offset=pci.PCI_COMMAND), byteorder='little') | pci.PCI_COMMAND_MASTER
@@ -702,6 +729,12 @@ class AMDDevice(HCQCompiled):
     if self.target//10000 == 10: ctl_stack_size = min(ctl_stack_size, 0x7000)
     debug_memory_size = round_up((self.max_cu_id + 1) * (self.max_wave_id + 1) * 32, 64)
 
+    self.soc = importlib.import_module(f"tinygrad.runtime.autogen.am.{({10: 'navi10', 11: 'soc21', 12: 'soc24'}[self.target//10000])}")
+    self.pm4 = importlib.import_module("tinygrad.runtime.autogen.am.pm4_nv")
+    self.sdma = import_module('sdma', self.dev_iface.ip_versions[am.SDMA0_HWIP])
+    self.gc = AMDIP('gc', self.dev_iface.ip_versions[am.GC_HWIP], self.dev_iface.ip_offsets[am.GC_HWIP])
+    self.nbio = AMDIP('nbio' if self.target < 120000 else 'nbif', self.dev_iface.ip_versions[am.NBIF_HWIP], self.dev_iface.ip_offsets[am.NBIF_HWIP])
+
     self.compute_queue = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_COMPUTE, 0x800000, ctx_save_restore_size=wg_data_size + ctl_stack_size,
                                            eop_buffer_size=0x1000, ctl_stack_size=ctl_stack_size, debug_memory_size=debug_memory_size)
 
@@ -709,7 +742,7 @@ class AMDDevice(HCQCompiled):
 
     super().__init__(device, AMDAllocator(self), AMDLLVMRenderer() if getenv("AMD_LLVM", 0) else AMDRenderer(self.arch),
                      AMDLLVMCompiler(self.arch) if getenv("AMD_LLVM", 0) else HIPCompiler(self.arch), functools.partial(AMDProgram, self),
-                     AMDSignal, AMDComputeQueue, AMDCopyQueue)
+                     AMDSignal, functools.partial(AMDComputeQueue, self), functools.partial(AMDCopyQueue, self))
 
     # Scratch setup
     self.max_private_segment_size = 0
@@ -728,7 +761,7 @@ class AMDDevice(HCQCompiled):
       self.sqtt_buffers = [self.allocator.alloc(SQTT_BUFFER_SIZE*1024*1024, BufferSpec(cpu_access=True, nolru=True)) for _ in range(SQTT_NUM)]
       self.sqtt_itrace_se_mask = getenv("SQTT_ITRACE_SE_MASK", 2) # -1 enable all, 0 disable all, >0 bitmask for where to enable instruction tracing
       self.cmd_id = 0
-      AMDComputeQueue().start_trace(self.sqtt_buffers, self.sqtt_itrace_se_mask).submit(self)
+      AMDComputeQueue(self).start_trace(self.sqtt_buffers, self.sqtt_itrace_se_mask).submit(self)
 
   def create_queue(self, queue_type, ring_size, ctx_save_restore_size=0, eop_buffer_size=0, ctl_stack_size=0, debug_memory_size=0):
     ring = self.dev_iface.alloc(ring_size, uncached=True, cpu_access=True)
@@ -753,7 +786,7 @@ class AMDDevice(HCQCompiled):
       self.max_private_segment_size = required
 
   def invalidate_caches(self):
-    AMDComputeQueue().memory_barrier().signal(self.timeline_signal, self.next_timeline()).submit(self)
+    AMDComputeQueue(self).memory_barrier().signal(self.timeline_signal, self.next_timeline()).submit(self)
     self.synchronize()
 
   def on_device_hang(self): self.dev_iface.on_device_hang()
@@ -762,7 +795,7 @@ class AMDDevice(HCQCompiled):
     if self.sqtt_enabled:
       wptrs_buf = self.allocator.alloc(round_up(len(self.sqtt_buffers), 0x1000), BufferSpec(cpu_access=True, nolru=True))
       wptrs = to_mv(wptrs_buf.va_addr, wptrs_buf.size)
-      AMDComputeQueue().stop_trace(len(self.sqtt_buffers), wptrs_buf).signal(self.timeline_signal, self.next_timeline()).submit(self)
+      AMDComputeQueue(self).stop_trace(len(self.sqtt_buffers), wptrs_buf).signal(self.timeline_signal, self.next_timeline()).submit(self)
       self.synchronize()
       if DEBUG>=2: print('Saving SQTT in profile...')
       for i,buf0 in enumerate(self.sqtt_buffers):

tinygrad/runtime/support/hcq.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from typing import cast, Type, TypeVar, Generic, Any, ClassVar
+from typing import cast, Callable, Type, TypeVar, Generic, Any, ClassVar
 import contextlib, decimal, statistics, time, ctypes, array, os, fcntl
 from tinygrad.helpers import PROFILE, from_mv, getenv, to_mv, round_up
 from tinygrad.renderer import Renderer
@@ -255,7 +255,7 @@ class HCQSignal(Generic[DeviceType]):
     if self.value < value: raise RuntimeError(f"Wait timeout: {timeout} ms! (the signal is not set to {value}, but {self.value})")
 
 @contextlib.contextmanager
-def hcq_profile(dev:HCQCompiled, enabled, desc, queue_type:Type[HWQueue]|None=None, queue:HWQueue|None=None):
+def hcq_profile(dev:HCQCompiled, enabled, desc, queue_type:Callable[[], HWQueue]|None=None, queue:HWQueue|None=None):
   st, en = (dev.signal_t(), dev.signal_t()) if enabled else (None, None)
 
   if enabled and queue is not None: queue.timestamp(st)
@@ -341,7 +341,7 @@ class HCQCompiled(Compiled, Generic[SignalType]):
   signal_pool: ClassVar[list[int]] = []
 
   def __init__(self, device:str, allocator:HCQAllocatorBase, renderer:Renderer, compiler:Compiler, runtime, signal_t:Type[SignalType],
-               comp_queue_t:Type[HWQueue], copy_queue_t:Type[HWQueue]|None):
+               comp_queue_t:Callable[[], HWQueue], copy_queue_t:Callable[[], HWQueue]|None):
     self.device_id:int = int(device.split(":")[1]) if ":" in device else 0
 
     from tinygrad.runtime.graph.hcq import HCQGraph
@@ -384,7 +384,7 @@ class HCQCompiled(Compiled, Generic[SignalType]):
     return cls.signal_pool.pop()
 
   def _at_profile_finalize(self):
-    def _sync(d:HCQCompiled, q_t:Type[HWQueue]):
+    def _sync(d:HCQCompiled, q_t:Callable[[], HWQueue]):
       q_t().timestamp(d.timeline_signal).signal(d.timeline_signal, d.next_timeline()).submit(d)
       st = time.perf_counter_ns()
       d.timeline_signal.wait(d.timeline_value - 1)  # average of the two