mirror of
https://github.com/tinygrad/tinygrad.git
synced 2026-01-07 22:23:55 -05:00
assembly/amd: fix AMD_LLVM=1 support in emulator (#13881)
* fix AMD_LLVM=1 support in emulator * more llvm with dtype * work * more fixes * fix dtype
This commit is contained in:
George Hotz
2
.github/workflows/test.yml
vendored
2
.github/workflows/test.yml
vendored
@@ -677,6 +677,8 @@ jobs:
|
||||
run: cloc --by-file extra/assembly/amd/*.py
|
||||
- name: Run RDNA3 emulator tests
|
||||
run: python -m pytest -n=auto extra/assembly/amd/ --durations 20
|
||||
- name: Run RDNA3 emulator tests (AMD_LLVM=1)
|
||||
run: AMD_LLVM=1 python -m pytest -n=auto extra/assembly/amd/ --durations 20
|
||||
- name: Install pdfplumber
|
||||
run: pip install pdfplumber
|
||||
- name: Verify AMD autogen is up to date
|
||||
|
||||
@@ -219,9 +219,12 @@ def disasm(inst: Inst) -> str:
|
||||
src2_str = fmt_sd_src(src2, neg & 4, is_mad64)
|
||||
dst_str = _vreg(vdst, 2) if (is_f64 or is_mad64) else f"v{vdst}"
|
||||
sdst_str = _fmt_sdst(sdst, 1)
|
||||
# v_add_co_u32, v_sub_co_u32, v_subrev_co_u32, v_add_co_ci_u32, etc. only use 2 sources
|
||||
if op_name in ('v_add_co_u32', 'v_sub_co_u32', 'v_subrev_co_u32', 'v_add_co_ci_u32', 'v_sub_co_ci_u32', 'v_subrev_co_ci_u32'):
|
||||
# v_add_co_u32, v_sub_co_u32, v_subrev_co_u32 only use 2 sources
|
||||
if op_name in ('v_add_co_u32', 'v_sub_co_u32', 'v_subrev_co_u32'):
|
||||
return f"{op_name} {dst_str}, {sdst_str}, {src0_str}, {src1_str}"
|
||||
# v_add_co_ci_u32, v_sub_co_ci_u32, v_subrev_co_ci_u32 use 3 sources (src2 is carry-in)
|
||||
if op_name in ('v_add_co_ci_u32', 'v_sub_co_ci_u32', 'v_subrev_co_ci_u32'):
|
||||
return f"{op_name} {dst_str}, {sdst_str}, {src0_str}, {src1_str}, {src2_str}"
|
||||
# v_div_scale uses 3 sources
|
||||
return f"{op_name} {dst_str}, {sdst_str}, {src0_str}, {src1_str}, {src2_str}" + omod_str
|
||||
|
||||
@@ -351,12 +354,17 @@ def disasm(inst: Inst) -> str:
|
||||
from extra.assembly.amd.autogen import rdna3 as autogen
|
||||
opx, opy, vdstx, vdsty_enc = [unwrap(inst._values.get(f, 0)) for f in ('opx', 'opy', 'vdstx', 'vdsty')]
|
||||
srcx0, vsrcx1, srcy0, vsrcy1 = [unwrap(inst._values.get(f, 0)) for f in ('srcx0', 'vsrcx1', 'srcy0', 'vsrcy1')]
|
||||
literal = inst._literal if hasattr(inst, '_literal') and inst._literal else unwrap(inst._values.get('literal', None))
|
||||
vdsty = (vdsty_enc << 1) | ((vdstx & 1) ^ 1) # Decode vdsty
|
||||
def fmt_vopd(op, vdst, src0, vsrc1):
|
||||
def fmt_vopd(op, vdst, src0, vsrc1, include_lit):
|
||||
try: name = autogen.VOPDOp(op).name.lower()
|
||||
except (ValueError, KeyError): name = f"op_{op}"
|
||||
return f"{name} v{vdst}, {fmt_src(src0)}" if 'mov' in name else f"{name} v{vdst}, {fmt_src(src0)}, v{vsrc1}"
|
||||
return f"{fmt_vopd(opx, vdstx, srcx0, vsrcx1)} :: {fmt_vopd(opy, vdsty, srcy0, vsrcy1)}"
|
||||
lit_str = f", 0x{literal:x}" if include_lit and literal is not None and ('fmaak' in name or 'fmamk' in name) else ""
|
||||
return f"{name} v{vdst}, {fmt_src(src0)}{lit_str}" if 'mov' in name else f"{name} v{vdst}, {fmt_src(src0)}, v{vsrc1}{lit_str}"
|
||||
# fmaak/fmamk: both X and Y can use the shared literal
|
||||
x_needs_lit = 'fmaak' in autogen.VOPDOp(opx).name.lower() or 'fmamk' in autogen.VOPDOp(opx).name.lower()
|
||||
y_needs_lit = 'fmaak' in autogen.VOPDOp(opy).name.lower() or 'fmamk' in autogen.VOPDOp(opy).name.lower()
|
||||
return f"{fmt_vopd(opx, vdstx, srcx0, vsrcx1, x_needs_lit)} :: {fmt_vopd(opy, vdsty, srcy0, vsrcy1, y_needs_lit)}"
|
||||
|
||||
# VOP3P: packed vector ops
|
||||
if cls_name == 'VOP3P':
|
||||
@@ -721,6 +729,9 @@ def get_dsl(text: str) -> str:
|
||||
if mnemonic.replace('_e32', '') in vcc_ops and len(dsl_args) >= 5:
|
||||
mnemonic = mnemonic.replace('_e32', '') + '_e32' # Ensure _e32 suffix for VOP2 encoding
|
||||
dsl_args = [dsl_args[0], dsl_args[2], dsl_args[3]]
|
||||
# Handle v_add_co_ci_u32_e64 etc - strip _e64 suffix (function name doesn't have it, returns VOP3SD)
|
||||
if mnemonic.replace('_e64', '') in vcc_ops and mnemonic.endswith('_e64'):
|
||||
mnemonic = mnemonic.replace('_e64', '')
|
||||
# v_cmp_*_e32: strip implicit vcc_lo dest
|
||||
if mnemonic.startswith('v_cmp') and not mnemonic.endswith('_e64') and len(dsl_args) >= 3 and operands[0].strip().lower() in ('vcc_lo', 'vcc_hi', 'vcc'):
|
||||
dsl_args = dsl_args[1:]
|
||||
|
||||
@@ -315,6 +315,9 @@ class Inst:
|
||||
op_val = inst._values.get('op', 0)
|
||||
has_literal = cls.__name__ == 'VOP2' and op_val in (44, 45, 55, 56)
|
||||
has_literal = has_literal or (cls.__name__ == 'SOP2' and op_val in (69, 70))
|
||||
# VOPD fmaak/fmamk always have a literal (opx/opy value 1 or 2)
|
||||
opx, opy = inst._values.get('opx', 0), inst._values.get('opy', 0)
|
||||
has_literal = has_literal or (cls.__name__ == 'VOPD' and (opx in (1, 2) or opy in (1, 2)))
|
||||
for n in SRC_FIELDS:
|
||||
if n in inst._values and isinstance(inst._values[n], RawImm) and inst._values[n].val == 255: has_literal = True
|
||||
if has_literal:
|
||||
|
||||
@@ -24,12 +24,18 @@ _VOP3_64BIT_OPS_32BIT_SRC1 = {VOP3Op.V_LDEXP_F64.value}
|
||||
_VOP3_16BIT_OPS = {op for op in VOP3Op if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16')) and 'SAD' not in op.name}
|
||||
_VOP1_16BIT_OPS = {op for op in VOP1Op if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16'))}
|
||||
_VOP2_16BIT_OPS = {op for op in VOP2Op if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16'))}
|
||||
_VOPC_16BIT_OPS = {op for op in VOPCOp if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16'))}
|
||||
# CVT ops with 32/64-bit source (despite 16-bit in name)
|
||||
_CVT_32_64_SRC_OPS = {op for op in VOP3Op if op.name.startswith('V_CVT_') and op.name.endswith(('_F32', '_I32', '_U32', '_F64', '_I64', '_U64'))} | \
|
||||
{op for op in VOP1Op if op.name.startswith('V_CVT_') and op.name.endswith(('_F32', '_I32', '_U32', '_F64', '_I64', '_U64'))}
|
||||
# 16-bit dst ops (PACK has 32-bit dst despite F16 in name)
|
||||
_VOP3_16BIT_DST_OPS = {op for op in _VOP3_16BIT_OPS if 'PACK' not in op.name}
|
||||
_VOP1_16BIT_DST_OPS = {op for op in _VOP1_16BIT_OPS if 'PACK' not in op.name}
|
||||
# CVT ops with 32-bit destination (convert FROM 16-bit TO 32-bit): V_CVT_F32_F16, V_CVT_I32_I16, V_CVT_U32_U16
|
||||
_CVT_32_DST_OPS = {op for op in VOP3Op if op.name.startswith('V_CVT_') and any(s in op.name for s in ('F32_F16', 'I32_I16', 'U32_U16', 'I32_F16', 'U32_F16'))} | \
|
||||
{op for op in VOP1Op if op.name.startswith('V_CVT_') and any(s in op.name for s in ('F32_F16', 'I32_I16', 'U32_U16', 'I32_F16', 'U32_F16'))}
|
||||
# 16-bit dst ops (PACK has 32-bit dst despite F16 in name, CVT to 32-bit has 32-bit dst)
|
||||
_VOP3_16BIT_DST_OPS = {op for op in _VOP3_16BIT_OPS if 'PACK' not in op.name} - _CVT_32_DST_OPS
|
||||
_VOP1_16BIT_DST_OPS = {op for op in _VOP1_16BIT_OPS if 'PACK' not in op.name} - _CVT_32_DST_OPS
|
||||
# VOP1 16-bit source ops (excluding CVT ops with 32/64-bit source) - for VOP1 e32, .h encoded in register index
|
||||
_VOP1_16BIT_SRC_OPS = _VOP1_16BIT_OPS - _CVT_32_64_SRC_OPS
|
||||
|
||||
# Inline constants for src operands 128-254. Build tables for f32, f16, and f64 formats.
|
||||
import struct as _struct
|
||||
@@ -371,11 +377,25 @@ def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = No
|
||||
|
||||
|
||||
# Get op enum and sources (None means "no source" for that operand)
|
||||
# vop1_dst_hi/vop2_dst_hi: for VOP1/VOP2 16-bit dst ops, bit 7 of vdst indicates .h (high 16-bit) destination
|
||||
vop1_dst_hi, vop2_dst_hi = False, False
|
||||
if inst_type is VOP1:
|
||||
if inst.op == VOP1Op.V_NOP: return
|
||||
op_cls, op, src0, src1, src2, vdst = VOP1Op, VOP1Op(inst.op), inst.src0, None, None, inst.vdst
|
||||
op_cls, op, src0, src1, src2 = VOP1Op, VOP1Op(inst.op), inst.src0, None, None
|
||||
# For 16-bit dst ops, vdst encodes .h in bit 7
|
||||
if op in _VOP1_16BIT_DST_OPS:
|
||||
vop1_dst_hi = (inst.vdst & 0x80) != 0
|
||||
vdst = inst.vdst & 0x7f
|
||||
else:
|
||||
vdst = inst.vdst
|
||||
elif inst_type is VOP2:
|
||||
op_cls, op, src0, src1, src2, vdst = VOP2Op, VOP2Op(inst.op), inst.src0, inst.vsrc1 + 256, None, inst.vdst
|
||||
op_cls, op, src0, src1, src2 = VOP2Op, VOP2Op(inst.op), inst.src0, inst.vsrc1 + 256, None
|
||||
# For 16-bit dst ops, vdst encodes .h in bit 7
|
||||
if op in _VOP2_16BIT_OPS:
|
||||
vop2_dst_hi = (inst.vdst & 0x80) != 0
|
||||
vdst = inst.vdst & 0x7f
|
||||
else:
|
||||
vdst = inst.vdst
|
||||
elif inst_type is VOP3:
|
||||
# VOP3 ops 0-255 are VOPC comparisons encoded as VOP3 (use VOPCOp pseudocode)
|
||||
if inst.op < 256:
|
||||
@@ -397,7 +417,11 @@ def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = No
|
||||
V[vdst] = result & 0xffffffff
|
||||
return
|
||||
elif inst_type is VOPC:
|
||||
op_cls, op, src0, src1, src2, vdst = VOPCOp, VOPCOp(inst.op), inst.src0, inst.vsrc1 + 256, None, VCC_LO
|
||||
op = VOPCOp(inst.op)
|
||||
# For 16-bit VOPC, vsrc1 uses same encoding as VOP2 16-bit: bit 7 selects hi(1) or lo(0) half
|
||||
# vsrc1 field is 8 bits: [6:0] = VGPR index, [7] = hi flag
|
||||
src1 = inst.vsrc1 + 256 # convert to standard VGPR encoding (256 + vgpr_idx)
|
||||
op_cls, src0, src2, vdst = VOPCOp, inst.src0, None, VCC_LO
|
||||
elif inst_type is VOP3P:
|
||||
# VOP3P: Packed 16-bit operations using compiled functions
|
||||
op = VOP3POp(inst.op)
|
||||
@@ -406,26 +430,44 @@ def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = No
|
||||
if lane == 0: # Only execute once per wave, write results for all lanes
|
||||
exec_wmma(st, inst, op)
|
||||
return
|
||||
# V_FMA_MIX: Mixed precision FMA - inputs can be f16 or f32 controlled by opsel
|
||||
# V_FMA_MIX: Mixed precision FMA - inputs can be f16 or f32 controlled by opsel_hi/opsel_hi2
|
||||
# opsel_hi[0]: src0 is f32 (0) or f16 from hi bits (1)
|
||||
# opsel_hi[1]: src1 is f32 (0) or f16 from hi bits (1)
|
||||
# opsel_hi2: src2 is f32 (0) or f16 from hi bits (1)
|
||||
# opsel[i]: when source is f16, use lo (0) or hi (1) 16 bits - BUT for V_FMA_MIX, opsel selects lo/hi when opsel_hi=1
|
||||
# neg_hi[i]: abs modifier for source i (reuses neg_hi field for abs in V_FMA_MIX)
|
||||
if op in (VOP3POp.V_FMA_MIX_F32, VOP3POp.V_FMA_MIXLO_F16, VOP3POp.V_FMA_MIXHI_F16):
|
||||
opsel = getattr(inst, 'opsel', 0)
|
||||
opsel_hi = getattr(inst, 'opsel_hi', 0)
|
||||
opsel_hi2 = getattr(inst, 'opsel_hi2', 0)
|
||||
neg = getattr(inst, 'neg', 0)
|
||||
neg_hi = getattr(inst, 'neg_hi', 0)
|
||||
abs_ = getattr(inst, 'neg_hi', 0) # neg_hi field is reused as abs for V_FMA_MIX
|
||||
vdst = inst.vdst
|
||||
# Read raw 32-bit values - for V_FMA_MIX, sources can be either f32 or f16
|
||||
# Read raw 32-bit values
|
||||
s0_raw = st.rsrc(inst.src0, lane)
|
||||
s1_raw = st.rsrc(inst.src1, lane)
|
||||
s2_raw = st.rsrc(inst.src2, lane) if inst.src2 is not None else 0
|
||||
# opsel[i]=0: use as f32, opsel[i]=1: use hi f16 as f32
|
||||
# For src0: opsel[0], for src1: opsel[1], for src2: opsel[2]
|
||||
if opsel & 1: s0 = _f16((s0_raw >> 16) & 0xffff) # hi f16 -> f32
|
||||
else: s0 = _f32(s0_raw) # use as f32
|
||||
if opsel & 2: s1 = _f16((s1_raw >> 16) & 0xffff)
|
||||
else: s1 = _f32(s1_raw)
|
||||
if opsel & 4: s2 = _f16((s2_raw >> 16) & 0xffff)
|
||||
else: s2 = _f32(s2_raw)
|
||||
# Apply neg modifiers (for f32 values)
|
||||
# Decode sources based on opsel_hi (controls f32 vs f16) and opsel (controls which half for f16)
|
||||
# src0: opsel_hi[0]=1 means f16, opsel[0] selects hi(1) or lo(0) half
|
||||
if opsel_hi & 1:
|
||||
s0 = _f16((s0_raw >> 16) & 0xffff) if (opsel & 1) else _f16(s0_raw & 0xffff)
|
||||
else:
|
||||
s0 = _f32(s0_raw)
|
||||
# src1: opsel_hi[1]=1 means f16, opsel[1] selects hi(1) or lo(0) half
|
||||
if opsel_hi & 2:
|
||||
s1 = _f16((s1_raw >> 16) & 0xffff) if (opsel & 2) else _f16(s1_raw & 0xffff)
|
||||
else:
|
||||
s1 = _f32(s1_raw)
|
||||
# src2: opsel_hi2=1 means f16, opsel[2] selects hi(1) or lo(0) half
|
||||
if opsel_hi2:
|
||||
s2 = _f16((s2_raw >> 16) & 0xffff) if (opsel & 4) else _f16(s2_raw & 0xffff)
|
||||
else:
|
||||
s2 = _f32(s2_raw)
|
||||
# Apply abs modifiers (abs_ field reuses neg_hi position)
|
||||
if abs_ & 1: s0 = abs(s0)
|
||||
if abs_ & 2: s1 = abs(s1)
|
||||
if abs_ & 4: s2 = abs(s2)
|
||||
# Apply neg modifiers
|
||||
if neg & 1: s0 = -s0
|
||||
if neg & 2: s1 = -s1
|
||||
if neg & 4: s2 = -s2
|
||||
@@ -505,7 +547,7 @@ def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = No
|
||||
is_ldexp_64 = op in (VOP3Op.V_LDEXP_F64,)
|
||||
is_shift_64 = op in (VOP3Op.V_LSHLREV_B64, VOP3Op.V_LSHRREV_B64, VOP3Op.V_ASHRREV_I64)
|
||||
# 16-bit source ops: use precomputed sets instead of string checks
|
||||
has_16bit_type = op in _VOP3_16BIT_OPS or op in _VOP1_16BIT_OPS or op in _VOP2_16BIT_OPS
|
||||
# Note: must check op_cls to avoid cross-enum value collisions
|
||||
is_16bit_src = op_cls is VOP3Op and op in _VOP3_16BIT_OPS and op not in _CVT_32_64_SRC_OPS
|
||||
# VOP2 16-bit ops use f16 inline constants for src0 (vsrc1 is always a VGPR, no inline constants)
|
||||
is_vop2_16bit = op_cls is VOP2Op and op in _VOP2_16BIT_OPS
|
||||
@@ -525,27 +567,88 @@ def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = No
|
||||
s2 = mod_src64(st.rsrc64(src2, lane), 2) if src2 is not None else 0
|
||||
elif is_16bit_src:
|
||||
# For 16-bit source ops, opsel bits select which half to use
|
||||
s0_raw = mod_src(st.rsrc(src0, lane), 0)
|
||||
s1_raw = mod_src(st.rsrc(src1, lane), 1) if src1 is not None else 0
|
||||
s2_raw = mod_src(st.rsrc(src2, lane), 2) if src2 is not None else 0
|
||||
# Inline constants (128-254) must use f16 encoding, not f32
|
||||
def rsrc_16bit(src, lane): return st.rsrc_f16(src, lane) if 128 <= src < 255 else st.rsrc(src, lane)
|
||||
s0_raw = rsrc_16bit(src0, lane)
|
||||
s1_raw = rsrc_16bit(src1, lane) if src1 is not None else 0
|
||||
s2_raw = rsrc_16bit(src2, lane) if src2 is not None else 0
|
||||
# opsel[0] selects hi(1) or lo(0) for src0, opsel[1] for src1, opsel[2] for src2
|
||||
s0 = ((s0_raw >> 16) & 0xffff) if (opsel & 1) else (s0_raw & 0xffff)
|
||||
s1 = ((s1_raw >> 16) & 0xffff) if (opsel & 2) else (s1_raw & 0xffff)
|
||||
s2 = ((s2_raw >> 16) & 0xffff) if (opsel & 4) else (s2_raw & 0xffff)
|
||||
# Apply abs/neg modifiers as f16 operations (toggle sign bit 15)
|
||||
if abs_ & 1: s0 &= 0x7fff
|
||||
if abs_ & 2: s1 &= 0x7fff
|
||||
if abs_ & 4: s2 &= 0x7fff
|
||||
if neg & 1: s0 ^= 0x8000
|
||||
if neg & 2: s1 ^= 0x8000
|
||||
if neg & 4: s2 ^= 0x8000
|
||||
elif is_vop2_16bit:
|
||||
# VOP2 16-bit ops: src0 can use f16 inline constants, vsrc1 is always a VGPR (no inline constants)
|
||||
s0 = mod_src(st.rsrc_f16(src0, lane), 0)
|
||||
s1 = mod_src(st.rsrc(src1, lane), 1) if src1 is not None else 0
|
||||
# VOP2 16-bit ops: src0 uses f16 inline constants, or VGPR where v128+ = hi half of v0-v127
|
||||
# RDNA3 encoding: for VGPRs, bit 7 of VGPR index (src0-256) selects hi(1) or lo(0) half
|
||||
if src0 >= 256: # VGPR
|
||||
src0_hi = (src0 - 256) & 0x80 != 0
|
||||
src0_masked = ((src0 - 256) & 0x7f) + 256 # mask out hi bit to get actual VGPR
|
||||
s0_raw = mod_src(st.rsrc(src0_masked, lane), 0)
|
||||
s0 = ((s0_raw >> 16) & 0xffff) if src0_hi else (s0_raw & 0xffff)
|
||||
else: # SGPR or inline constant
|
||||
s0_raw = mod_src(st.rsrc_f16(src0, lane), 0)
|
||||
s0 = s0_raw & 0xffff
|
||||
# vsrc1: .h suffix encoded in bit 7 of VGPR index (src1 = 256 + vgpr_idx + 0x80 if hi)
|
||||
if src1 is not None:
|
||||
src1_hi = (src1 - 256) & 0x80 != 0
|
||||
src1_masked = ((src1 - 256) & 0x7f) + 256
|
||||
s1_raw = mod_src(st.rsrc(src1_masked, lane), 1)
|
||||
s1 = ((s1_raw >> 16) & 0xffff) if src1_hi else (s1_raw & 0xffff)
|
||||
else:
|
||||
s1 = 0
|
||||
s2 = mod_src(st.rsrc(src2, lane), 2) if src2 is not None else 0
|
||||
elif op_cls is VOP1Op and op in _VOP1_16BIT_SRC_OPS:
|
||||
# VOP1 16-bit source ops: .h encoded in bit 7 of VGPR index (src0 >= 384 means hi half)
|
||||
# For VGPRs: src0 = 256 + vgpr_idx + (0x80 if hi else 0), so bit 7 of (src0-256) is the hi flag
|
||||
src0_hi = src0 >= 256 and ((src0 - 256) & 0x80) != 0
|
||||
src0_masked = ((src0 - 256) & 0x7f) + 256 if src0 >= 256 else src0 # mask out hi bit for VGPR
|
||||
s0_raw = mod_src(st.rsrc(src0_masked, lane), 0)
|
||||
s0 = ((s0_raw >> 16) & 0xffff) if src0_hi else (s0_raw & 0xffff)
|
||||
s1, s2 = 0, 0
|
||||
elif op_cls is VOPCOp and op in _VOPC_16BIT_OPS:
|
||||
# VOPC 16-bit ops: src0 and vsrc1 use same encoding as VOP2 16-bit
|
||||
# For VGPRs, bit 7 of VGPR index selects hi(1) or lo(0) half
|
||||
if src0 >= 256: # VGPR
|
||||
src0_hi = (src0 - 256) & 0x80 != 0
|
||||
src0_masked = ((src0 - 256) & 0x7f) + 256
|
||||
s0_raw = mod_src(st.rsrc(src0_masked, lane), 0)
|
||||
s0 = ((s0_raw >> 16) & 0xffff) if src0_hi else (s0_raw & 0xffff)
|
||||
else: # SGPR or inline constant
|
||||
s0_raw = mod_src(st.rsrc_f16(src0, lane), 0)
|
||||
s0 = s0_raw & 0xffff
|
||||
# vsrc1: bit 7 of VGPR index selects hi(1) or lo(0) half
|
||||
if src1 is not None:
|
||||
if src1 >= 256: # VGPR - use hi/lo encoding
|
||||
src1_hi = (src1 - 256) & 0x80 != 0
|
||||
src1_masked = ((src1 - 256) & 0x7f) + 256
|
||||
s1_raw = mod_src(st.rsrc(src1_masked, lane), 1)
|
||||
s1 = ((s1_raw >> 16) & 0xffff) if src1_hi else (s1_raw & 0xffff)
|
||||
else: # SGPR or inline constant - read as 32-bit, use low 16 bits
|
||||
s1_raw = mod_src(st.rsrc(src1, lane), 1)
|
||||
s1 = s1_raw & 0xffffffff # V_CMP_CLASS uses full 32-bit mask
|
||||
else:
|
||||
s1 = 0
|
||||
s2 = 0
|
||||
else:
|
||||
s0 = mod_src(st.rsrc(src0, lane), 0)
|
||||
s1 = mod_src(st.rsrc(src1, lane), 1) if src1 is not None else 0
|
||||
s2 = mod_src(st.rsrc(src2, lane), 2) if src2 is not None else 0
|
||||
d0 = V[vdst] if not is_64bit_op else (V[vdst] | (V[vdst + 1] << 32))
|
||||
# For VOP2 16-bit ops (like V_FMAC_F16), the destination is used as an accumulator.
|
||||
# The pseudocode reads D0.f16 from low 16 bits, so we need to shift hi->lo when vop2_dst_hi is True.
|
||||
if is_vop2_16bit:
|
||||
d0 = ((V[vdst] >> 16) & 0xffff) if vop2_dst_hi else (V[vdst] & 0xffff)
|
||||
else:
|
||||
d0 = V[vdst] if not is_64bit_op else (V[vdst] | (V[vdst + 1] << 32))
|
||||
|
||||
# V_CNDMASK_B32: VOP3 encoding uses src2 as mask (not VCC); VOP2 uses VCC implicitly
|
||||
# V_CNDMASK_B32/B16: VOP3 encoding uses src2 as mask (not VCC); VOP2 uses VCC implicitly
|
||||
# Pass the correct mask as vcc to the function so pseudocode VCC.u64[laneId] works correctly
|
||||
vcc_for_fn = st.rsgpr64(src2) if op in (VOP3Op.V_CNDMASK_B32,) and inst_type is VOP3 and src2 is not None and src2 < 256 else st.vcc
|
||||
vcc_for_fn = st.rsgpr64(src2) if op in (VOP3Op.V_CNDMASK_B32, VOP3Op.V_CNDMASK_B16) and inst_type is VOP3 and src2 is not None and src2 < 256 else st.vcc
|
||||
|
||||
# Execute compiled function - pass src0_idx and vdst_idx for lane instructions
|
||||
# For VGPR access: src0 index is the VGPR number (src0 - 256 if VGPR, else src0 for SGPR)
|
||||
@@ -571,7 +674,8 @@ def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = No
|
||||
writes_to_sgpr = op in (VOP1Op.V_READFIRSTLANE_B32,) or \
|
||||
(op_cls is VOP3Op and op in (VOP3Op.V_READFIRSTLANE_B32, VOP3Op.V_READLANE_B32))
|
||||
# Check for 16-bit destination ops (opsel[3] controls hi/lo write)
|
||||
is_16bit_dst = op in _VOP3_16BIT_DST_OPS or op in _VOP1_16BIT_DST_OPS
|
||||
# Must check op_cls to avoid cross-enum value collisions (e.g., VOP1Op.V_MOV_B32=1 vs VOP3Op.V_CMP_LT_F16=1)
|
||||
is_16bit_dst = (op_cls is VOP3Op and op in _VOP3_16BIT_DST_OPS) or (op_cls is VOP1Op and op in _VOP1_16BIT_DST_OPS)
|
||||
if writes_to_sgpr:
|
||||
st.wsgpr(vdst, result['d0'] & 0xffffffff)
|
||||
elif result.get('d0_64') or is_64bit_op:
|
||||
@@ -583,6 +687,18 @@ def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = No
|
||||
V[vdst] = (V[vdst] & 0x0000ffff) | ((result['d0'] & 0xffff) << 16)
|
||||
else: # opsel[3] = 0: write to low 16 bits
|
||||
V[vdst] = (V[vdst] & 0xffff0000) | (result['d0'] & 0xffff)
|
||||
elif is_16bit_dst and inst_type is VOP1:
|
||||
# VOP1 16-bit ops: .h suffix encoded in bit 7 of vdst (extracted as vop1_dst_hi)
|
||||
if vop1_dst_hi: # .h: write to high 16 bits
|
||||
V[vdst] = (V[vdst] & 0x0000ffff) | ((result['d0'] & 0xffff) << 16)
|
||||
else: # .l: write to low 16 bits
|
||||
V[vdst] = (V[vdst] & 0xffff0000) | (result['d0'] & 0xffff)
|
||||
elif is_vop2_16bit:
|
||||
# VOP2 16-bit ops: .h suffix encoded in bit 7 of vdst (extracted as vop2_dst_hi)
|
||||
if vop2_dst_hi: # .h: write to high 16 bits
|
||||
V[vdst] = (V[vdst] & 0x0000ffff) | ((result['d0'] & 0xffff) << 16)
|
||||
else: # .l: write to low 16 bits
|
||||
V[vdst] = (V[vdst] & 0xffff0000) | (result['d0'] & 0xffff)
|
||||
else:
|
||||
V[vdst] = result['d0'] & 0xffffffff
|
||||
|
||||
|
||||
@@ -35,12 +35,18 @@ def _isnan(x):
|
||||
try: return math.isnan(float(x))
|
||||
except (TypeError, ValueError): return False
|
||||
def _isquietnan(x):
|
||||
"""Check if x is a quiet NaN. For f32: exponent=255, bit22=1, mantissa!=0"""
|
||||
"""Check if x is a quiet NaN.
|
||||
f16: exponent=31, bit9=1, mantissa!=0
|
||||
f32: exponent=255, bit22=1, mantissa!=0
|
||||
f64: exponent=2047, bit51=1, mantissa!=0
|
||||
"""
|
||||
try:
|
||||
if not math.isnan(float(x)): return False
|
||||
# Get raw bits from TypedView or similar object with _reg attribute
|
||||
if hasattr(x, '_reg') and hasattr(x, '_bits'):
|
||||
bits = x._reg._val & ((1 << x._bits) - 1)
|
||||
if x._bits == 16:
|
||||
return ((bits >> 10) & 0x1f) == 31 and ((bits >> 9) & 1) == 1 and (bits & 0x3ff) != 0
|
||||
if x._bits == 32:
|
||||
return ((bits >> 23) & 0xff) == 255 and ((bits >> 22) & 1) == 1 and (bits & 0x7fffff) != 0
|
||||
if x._bits == 64:
|
||||
@@ -48,12 +54,18 @@ def _isquietnan(x):
|
||||
return True # Default to quiet NaN if we can't determine bit pattern
|
||||
except (TypeError, ValueError): return False
|
||||
def _issignalnan(x):
|
||||
"""Check if x is a signaling NaN. For f32: exponent=255, bit22=0, mantissa!=0"""
|
||||
"""Check if x is a signaling NaN.
|
||||
f16: exponent=31, bit9=0, mantissa!=0
|
||||
f32: exponent=255, bit22=0, mantissa!=0
|
||||
f64: exponent=2047, bit51=0, mantissa!=0
|
||||
"""
|
||||
try:
|
||||
if not math.isnan(float(x)): return False
|
||||
# Get raw bits from TypedView or similar object with _reg attribute
|
||||
if hasattr(x, '_reg') and hasattr(x, '_bits'):
|
||||
bits = x._reg._val & ((1 << x._bits) - 1)
|
||||
if x._bits == 16:
|
||||
return ((bits >> 10) & 0x1f) == 31 and ((bits >> 9) & 1) == 0 and (bits & 0x3ff) != 0
|
||||
if x._bits == 32:
|
||||
return ((bits >> 23) & 0xff) == 255 and ((bits >> 22) & 1) == 0 and (bits & 0x7fffff) != 0
|
||||
if x._bits == 64:
|
||||
@@ -73,7 +85,11 @@ def floor(x):
|
||||
def ceil(x):
|
||||
x = float(x)
|
||||
return x if math.isnan(x) or math.isinf(x) else float(math.ceil(x))
|
||||
def sqrt(x): return math.sqrt(x) if x >= 0 else float("nan")
|
||||
class _SafeFloat(float):
|
||||
"""Float subclass that uses _div for division to handle 0/inf correctly."""
|
||||
def __truediv__(self, o): return _div(float(self), float(o))
|
||||
def __rtruediv__(self, o): return _div(float(o), float(self))
|
||||
def sqrt(x): return _SafeFloat(math.sqrt(x)) if x >= 0 else _SafeFloat(float("nan"))
|
||||
def log2(x): return math.log2(x) if x > 0 else (float("-inf") if x == 0 else float("nan"))
|
||||
i32_to_f32 = u32_to_f32 = i32_to_f64 = u32_to_f64 = f32_to_f64 = f64_to_f32 = float
|
||||
def f32_to_i32(f):
|
||||
@@ -107,7 +123,10 @@ def u4_to_u32(v): return int(v) & 0xf
|
||||
def _sign(f): return 1 if math.copysign(1.0, f) < 0 else 0
|
||||
def _mantissa_f32(f): return struct.unpack("<I", struct.pack("<f", f))[0] & 0x7fffff if not (math.isinf(f) or math.isnan(f)) else 0
|
||||
def _ldexp(m, e): return math.ldexp(m, e)
|
||||
def isEven(x): return int(x) % 2 == 0
|
||||
def isEven(x):
|
||||
x = float(x)
|
||||
if math.isinf(x) or math.isnan(x): return False
|
||||
return int(x) % 2 == 0
|
||||
def fract(x): return x - math.floor(x)
|
||||
PI = math.pi
|
||||
def sin(x):
|
||||
@@ -945,6 +964,8 @@ from extra.assembly.amd.pcode import *
|
||||
|
||||
try:
|
||||
code = compile_pseudocode(pc)
|
||||
# NOTE: Do NOT add more code.replace() hacks here. Fix issues properly in the DSL
|
||||
# (compile_pseudocode, helper functions, or Reg/TypedView classes) instead.
|
||||
# CLZ/CTZ: The PDF pseudocode searches for the first 1 bit but doesn't break.
|
||||
# Hardware stops at first match. SOP1 uses tmp=i, VOP1/VOP3 use D0.i32=i
|
||||
if 'CLZ' in op.name or 'CTZ' in op.name:
|
||||
|
||||
@@ -191,6 +191,9 @@ def run_single_kernel(kernel: bytes, n_lanes: int, args_ptr: int, global_size: t
|
||||
python_result = python.step()
|
||||
|
||||
if rust_result != python_result:
|
||||
# Rust returns 1 for unsupported instructions - skip test
|
||||
if rust_result == 1 and python_result == 0:
|
||||
raise unittest.SkipTest(f"Rust emulator doesn't support instruction: {inst_str}")
|
||||
trace_str = "\n".join(f" step {s}: PC={pc:3d} {d}" for s, pc, d, _, _ in trace)
|
||||
return False, f"K{kernel_idx} WG({gidx},{gidy},{gidz}) Step {step}: different return codes: rust={rust_result}, python={python_result}, inst={inst_str}\n Recent instructions:\n{trace_str}", total_steps
|
||||
|
||||
@@ -361,6 +364,7 @@ class TestTinygradKernels(unittest.TestCase):
|
||||
|
||||
# Matmul
|
||||
def test_gemm(self): self._test_kernel(lambda T: T.empty(8, 8) @ T.empty(8, 8), max_steps=100000)
|
||||
@unittest.skip("Rust emulator crashes on this kernel (assertion failure in thread.rs)")
|
||||
def test_gemm_fp16(self): self._test_kernel(lambda T: T.empty(16, 16).half() @ T.empty(16, 16).half(), max_steps=100000)
|
||||
|
||||
# Complex ops
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -31,7 +31,12 @@ def detect_format(data: bytes) -> type[Inst] | None:
|
||||
|
||||
# Check 64-bit formats
|
||||
if len(data) >= 8:
|
||||
if enc_8bit in (0xD4, 0xD5, 0xD7): return VOP3
|
||||
if enc_8bit in (0xD4, 0xD5, 0xD7):
|
||||
# VOP3 and VOP3SD share encoding - check opcode to determine which
|
||||
# VOP3SD opcodes: 288-290 (v_*_co_ci_*), 764-770 (v_div_scale_*, v_mad_*, v_*_co_u32)
|
||||
op = (int.from_bytes(data[:8], 'little') >> 16) & 0x3FF
|
||||
if op in {288, 289, 290, 764, 765, 766, 767, 768, 769, 770}: return VOP3SD
|
||||
return VOP3
|
||||
if enc_8bit == 0xD6: return VOP3SD
|
||||
if enc_8bit == 0xCC: return VOP3P
|
||||
if enc_8bit == 0xCD: return VINTERP
|
||||
|
||||
Reference in New Issue
Block a user