feat(hpu) ISC Trace support

2026-01-09 14:47:56 -05:00 · 2025-01-20 16:10:53 +00:00
parent d8ac34aa44
commit f4b2ba2cc6
17 changed files with 41122 additions and 4 deletions
--- a/backends/tfhe-hpu-backend/Cargo.toml
+++ b/backends/tfhe-hpu-backend/Cargo.toml
@@ -11,8 +11,8 @@ default = []
 hw-xrt = []
 hw-aved = []
 io-dump = ["num-traits"]
-utils = ["clap", "clap-num"]
 rtl_graph = ["dot2"]
+utils = ["clap", "clap-num", "packed_struct", "bitvec", "serde_json"]

 [build-dependencies]
 cxx-build = "1.0"
@@ -55,6 +55,10 @@ nix = { version = "0.29.0", features = ["ioctl", "uio"] }
 # Dependencies used for rtl_graph features
 dot2 = { version = "*", optional=true }

+bitvec = { version = "*", optional=true }
+packed_struct = { path = "src/packed_struct", optional=true }
+serde_json = { version = "*", optional=true }
+
 # Binary for manual debugging
 # Enable to access Hpu register and drive some custom sequence by hand
 [[bin]]
--- a/backends/tfhe-hpu-backend/config_store/aved/custom_iop/cust_4.asm
+++ b/backends/tfhe-hpu-backend/config_store/aved/custom_iop/cust_4.asm
@@ -0,0 +1,264 @@
+# CUST_4
+# Just to check if this batch times out
+LD        R0               TS[0].31
+LD        R1               TS[1].31
+LD        R3               TS[0].27
+LD        R4               TS[1].27
+LD        R6               TS[0].30
+LD        R7               TS[1].30
+LD        R9               TS[0].28
+LD        R10              TS[1].28
+LD        R12              TS[0].29
+LD        R13              TS[1].29
+LD        R15              TS[0].23
+LD        R16              TS[1].23
+LD        R18              TS[0].26
+LD        R19              TS[1].26
+LD        R21              TS[0].24
+LD        R22              TS[1].24
+LD        R24              TS[0].20
+LD        R25              TS[1].20
+LD        R27              TS[0].13
+LD        R28              TS[1].13
+LD        R30              TS[0].25
+LD        R31              TS[1].25
+LD        R33              TS[0].22
+LD        R34              TS[1].22
+LD        R36              TS[0].17
+LD        R37              TS[1].17
+LD        R39              TS[0].19
+LD        R40              TS[1].19
+LD        R42              TS[0].15
+LD        R43              TS[1].15
+LD        R45              TS[0].12
+LD        R46              TS[1].12
+LD        R48              TS[0].7
+LD        R49              TS[1].7
+LD        R51              TS[0].6
+LD        R52              TS[1].6
+LD        R54              TS[0].10
+LD        R55              TS[1].10
+LD        R57              TS[0].14
+LD        R58              TS[1].14
+LD        R60              TS[0].11
+LD        R61              TS[1].11
+ADD       R2               R0               R1
+ADD       R5               R3               R4
+LD        R63              TS[0].18
+LD        R3               TS[1].18
+ADD       R8               R6               R7
+ST        TH.0             R6
+ST        TH.1             R7
+ADD       R11              R9               R10
+ST        TH.2             R11
+LD        R9               TH.2
+ADD       R14              R12              R13
+ST        TH.3             R12
+ST        TH.4             R13
+ADD       R17              R15              R16
+ST        TH.5             R17
+ADD       R20              R18              R19
+ST        TH.6             R18
+ST        TH.7             R19
+LD        R15              TH.5
+ADD       R23              R21              R22
+ST        TH.8             R23
+LD        R21              TH.8
+ADD       R26              R24              R25
+ST        TH.9             R24
+ST        TH.10            R25
+ADD       R29              R27              R28
+ST        TH.11            R29
+LD        R27              TH.11
+ADD       R32              R30              R31
+ST        TH.12            R30
+ST        TH.13            R31
+ADD       R35              R33              R34
+ST        TH.14            R35
+ADD       R38              R36              R37
+ST        TH.15            R36
+ST        TH.16            R37
+LD        R33              TH.14
+PBS_ML2   R0               R2               PbsManyGenProp
+PBS_ML2   R6               R5               PbsManyGenProp
+PBS_ML2   R10              R9               PbsManyGenProp
+PBS_ML2   R12              R8               PbsManyGenProp
+PBS_ML2   R16              R14              PbsManyGenProp
+PBS_ML2   R18              R15              PbsManyGenProp
+PBS_ML2   R22              R21              PbsManyGenProp
+PBS_ML2   R24              R20              PbsManyGenProp
+PBS_ML2   R28              R27              PbsManyGenProp
+PBS_ML2   R30              R26              PbsManyGenProp
+PBS_ML2   R34              R32              PbsManyGenProp
+PBS_ML2_F R36              R33              PbsManyGenProp
+ADD       R41              R39              R40
+LD        R39              TS[0].16
+LD        R40              TS[1].16
+ST        TH.17            R38
+ST        TH.18            R33
+LD        R33              TS[0].1
+ST        TH.19            R32
+LD        R32              TS[1].1
+ST        TH.20            R26
+ST        TH.21            R27
+LD        R27              TS[0].21
+ST        TH.22            R20
+LD        R20              TS[1].21
+ST        TH.23            R21
+ST        TH.24            R15
+LD        R15              TS[0].0
+ST        TH.25            R14
+LD        R14              TS[1].0
+ST        TH.26            R8
+ST        TH.27            R9
+LD        R9               TS[0].3
+ST        TH.28            R5
+LD        R5               TS[1].3
+ST        TH.29            R2
+ADD       R44              R42              R43
+LD        R42              TS[0].2
+LD        R43              TS[1].2
+ST        TH.30            R41
+ADD       R47              R45              R46
+LD        R45              TS[0].9
+LD        R46              TS[1].9
+ST        TH.31            R44
+ADD       R50              R48              R49
+LD        R48              TS[0].5
+LD        R49              TS[1].5
+ST        TH.32            R47
+ADD       R53              R51              R52
+LD        R51              TS[0].4
+LD        R52              TS[1].4
+ST        TH.33            R50
+ADD       R56              R54              R55
+LD        R54              TS[0].8
+LD        R55              TS[1].8
+ST        TH.34            R53
+ADD       R59              R57              R58
+ADD       R62              R60              R61
+ADD       R4               R63              R3
+ADD       R38              R39              R40
+ADD       R26              R33              R32
+ADD       R21              R27              R20
+ADD       R8               R15              R14
+ADD       R2               R9               R5
+ADD       R41              R42              R43
+ADD       R44              R45              R46
+ADD       R47              R48              R49
+ADD       R50              R51              R52
+ADD       R53              R54              R55
+MAC       R57              R11              R7               2
+LD        R58              TH.31
+LD        R63              TH.32
+LD        R3               TH.17
+ST        TH.35            R41
+LD        R39              TH.30
+ST        TH.36            R21
+ST        TH.37            R47
+ST        TH.38            R53
+ST        TH.39            R44
+ST        TH.40            R50
+ST        TH.41            R0
+LD        R27              TH.35
+ST        TH.42            R12
+ST        TH.43            R13
+LD        R9               TH.39
+ST        TH.44            R16
+ST        TH.45            R17
+LD        R5               TH.37
+ST        TH.46            R18
+ST        TH.47            R19
+ST        TH.48            R6
+LD        R6               TH.40
+ST        TH.49            R22
+ST        TH.50            R23
+ST        TH.51            R10
+LD        R10              TH.38
+ST        TH.52            R24
+ST        TH.53            R25
+ST        TH.54            R28
+LD        R28              TH.33
+ST        TH.55            R30
+ST        TH.56            R31
+ST        TH.57            R29
+LD        R29              TH.36
+ST        TH.58            R34
+ST        TH.59            R35
+ST        TH.60            R36
+LD        R36              TH.34
+PBS_ML2   R60              R58              PbsManyGenProp
+PBS_ML2   R32              R38              PbsManyGenProp
+PBS_ML2   R14              R63              PbsManyGenProp
+PBS_ML2   R42              R8               PbsManyGenProp
+PBS_ML2   R48              R3               PbsManyGenProp
+PBS_ML2   R54              R62              PbsManyGenProp
+PBS_ML2   R40              R39              PbsManyGenProp
+PBS_ML2   R20              R4               PbsManyGenProp
+PBS_ML2   R46              R59              PbsManyGenProp
+PBS_ML2   R52              R26              PbsManyGenProp
+PBS_ML2   R44              R56              PbsManyGenProp
+PBS_ML2_F R50              R2               PbsManyGenProp
+LD        R11              TH.45
+ST        TH.61            R37
+ST        TH.62            R2
+LD        R2               TH.53
+ST        TH.63            R56
+LD        R56              TH.59
+ST        TH.64            R26
+ST        TH.65            R59
+LD        R59              TH.43
+ST        TH.66            R4
+MAC       R37              R11              R57              4
+MAC       R26              R2               R56              2
+MAC       R4               R59              R11              2
+MAC       R2               R4               R57              4
+MAC       R59              R33              R61              2
+LD        R58              TH.57
+LD        R62              TH.56
+ADDS      R4               R42              0
+MAC       R38              R47              R58              2
+MAC       R63              R49              R59              4
+MAC       R8               R21              R49              2
+MULS      R3               R43              2
+ADDS      R3               R3               0
+MAC       R39              R62              R41              2
+MAC       R42              R8               R59              4
+MAC       R21              R53              R3               4
+PBS_ML2   R0               R27              PbsManyGenProp
+PBS_ML2   R12              R9               PbsManyGenProp
+PBS_ML2   R16              R5               PbsManyGenProp
+PBS_ML2   R18              R6               PbsManyGenProp
+PBS_ML2   R22              R10              PbsManyGenProp
+PBS_ML2   R24              R28              PbsManyGenProp
+PBS_ML2   R30              R29              PbsManyGenProp
+PBS_ML2   R34              R36              PbsManyGenProp
+PBS       R11              R2               PbsReduceCarryPad
+PBS       R33              R4               PbsGenPropAdd
+PBS       R47              R3               PbsReduceCarry2
+PBS_F     R49              R42              PbsReduceCarryPad
+MAC       R43              R1               R53              2
+ST        TD[0].0          R33
+LD        R29              TH.61
+MAC       R8               R47              R52              4
+ADDS      R27              R11              1
+MAC       R9               R31              R39              4
+ADDS      R5               R49              1
+MAC       R6               R43              R3               4
+MAC       R10              R45              R13              2
+MAC       R28              R23              R25              2
+MAC       R36              R29              R31              2
+MAC       R2               R19              R51              2
+MAC       R4               R35              R17              2
+MAC       R1               R13              R28              4
+MAC       R53              R10              R28              4
+MAC       R47              R36              R39              4
+MAC       R52              R17              R2               4
+MAC       R11              R4               R2               4
+PBS       R62              R21              PbsReduceCarry3
+PBS       R42              R8               PbsGenPropAdd
+PBS       R33              R6               PbsReduceCarryPad
+PBS       R49              R53              PbsReduceCarryPad
+PBS       R43              R47              PbsReduceCarryPad
+PBS_F     R3               R11              PbsReduceCarryPad
+MAC       R45              R62              R0               4
--- a/backends/tfhe-hpu-backend/config_store/aved/hpu_config.toml
+++ b/backends/tfhe-hpu-backend/config_store/aved/hpu_config.toml
@@ -56,19 +56,21 @@
    {Hbm={pc=23}}
  ]

-trace_pc = {Hbm={pc=35}}
+  trace_pc = {Hbm={pc=35}}
+  trace_depth = 32 # In MB

 [firmware]
  integer_w=[16]
  # Set to true for the firmware scheduler to try to fill the PE batch FIFO
  # first even at the expense of missing dependencies
  fill_batch_fifo = true
-  min_batch_size = true
+  min_batch_size = false
  kogge_cfg            = "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/kogge_cfg.toml"
  custom_iop.'IOP[0]'  = "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/custom_iop/cust_0.asm"
  custom_iop.'IOP[1]'  = "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/custom_iop/cust_1.asm"
  custom_iop.'IOP[2]'  = "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/custom_iop/cust_2.asm"
  custom_iop.'IOP[3]'  = "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/custom_iop/cust_3.asm"
+  custom_iop.'IOP[4]'  = "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/custom_iop/cust_4.asm"
  custom_iop.'IOP[8]'  = "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/custom_iop/cust_8.asm"
  custom_iop.'IOP[9]'  = "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/custom_iop/cust_9.asm"
  custom_iop.'IOP[16]' = "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/custom_iop/cust_16.asm"
--- a/backends/tfhe-hpu-backend/config_store/sim/hpu_config.toml
+++ b/backends/tfhe-hpu-backend/config_store/sim/hpu_config.toml
@@ -53,7 +53,8 @@
    {Hbm={pc=23}}
  ]

-trace_pc = {Hbm={pc=35}}
+  trace_pc = {Hbm={pc=35}}
+  trace_depth = 32 # In MB

 [firmware]
  integer_w=[16]
--- a/backends/tfhe-hpu-backend/config_store/u55c_gf64/hpu_config.toml
+++ b/backends/tfhe-hpu-backend/config_store/u55c_gf64/hpu_config.toml
@@ -48,6 +48,7 @@
  ]

  trace_pc = {Hbm={pc=0}}
+  trace_depth = 4 # In MB

 [firmware]
  integer_w=[16]
--- a/backends/tfhe-hpu-backend/python/lib/example.json
+++ b/backends/tfhe-hpu-backend/python/lib/example.json
--- a/backends/tfhe-hpu-backend/python/lib/isctrace.py
+++ b/backends/tfhe-hpu-backend/python/lib/isctrace.py
@@ -0,0 +1,60 @@
+import json
+from pandas import DataFrame
+
+
+class Instruction:
+    def __init__(self, asm, duration, timestamp):
+        self.asm = asm
+        self.duration = duration
+        self.timestamp = timestamp
+
+    def __repr__(self):
+        return f"{self.name} {self.duration} {self.timestamp}"
+
+    def as_dict(self):
+        return self.__dict__
+
+
+class Event:
+    def __init__(self, trace_dict):
+        self.__dict__.update(
+                {x: trace_dict[x] for x in ("cmd", "insn_asm", "timestamp")})
+
+    def as_dict(self):
+        return self.__dict__
+
+
+class Trace:
+    def __init__(self, filename):
+        with open(filename, 'r') as fd: 
+            self.events = [Event(x) for x in json.load(fd)]
+
+    def __iter__(self):
+        return iter(self.events)
+
+    def to_pandas(self):
+        return DataFrame.from_records([x.as_dict() for x in self],
+        index='timestamp')
+
+    def instructions(self):
+        isn_map = {}
+        for event in filter(lambda x: x.cmd in ("ISSUE", "RETIRE"), self):
+            asm = event.insn_asm
+            if (event.cmd == "RETIRE" and asm in isn_map):
+                yield Instruction(asm, event.timestamp - isn_map[asm],
+                                  event.timestamp)
+                del isn_map[asm]
+            elif (event.cmd == "ISSUE"):
+                isn_map[asm] = event.timestamp
+
+    def to_pandas_delta(self):
+        return DataFrame.from_records([x.as_dict() for x in self.instructions()])
+
+
+if __name__ == "__main__":
+    trace = Trace("example.json")
+    df = trace.to_pandas()
+    df = df[df.cmd == "RETIRE"]
+    print(df.to_string())
+    print(trace.to_pandas_delta().to_string())
+
--- a/backends/tfhe-hpu-backend/src/interface/config.rs
+++ b/backends/tfhe-hpu-backend/src/interface/config.rs
@@ -99,6 +99,8 @@ pub struct BoardConfig {
    pub fw_pc: ffi::MemKind,
    /// Depict the memory connected to trace manager
    pub trace_pc: ffi::MemKind,
+    /// The trace memory depth in MB
+    pub trace_depth: usize,

    /// Depict the hbm_pc connected to bsk master_axi
    pub bsk_pc: Vec<ffi::MemKind>,
--- a/backends/tfhe-hpu-backend/src/isc_trace.rs
+++ b/backends/tfhe-hpu-backend/src/isc_trace.rs
@@ -0,0 +1,51 @@
+pub mod fmt;
+
+use crate::ffi;
+pub use crate::isc_trace::fmt::{IscTraceStream, IscQueryCmd, TRACE_W};
+
+pub struct TraceDump {
+    trace: Vec<u8>
+}
+
+use tracing::trace;
+
+impl TraceDump {
+    pub fn new_from(hpu_hw: &mut ffi::HpuHw, regmap: &hw_regmap::FlatRegmap,
+                    depth: usize) -> TraceDump {
+        let size_b = ((depth * 1024 * 1024)/TRACE_W) * TRACE_W;
+
+        let mut trace: Vec<u8> = vec![0;size_b];
+
+        let offset_reg: Vec<usize> = ["addr_lsb", "addr_msb"].into_iter().
+            map(|name| {
+                let reg = regmap
+                    .register()
+                    .get(&format!("Trace::{}", name))
+                    .expect("Unknow register, check regmap definition");
+                hpu_hw.read_reg(*reg.offset() as u64) as usize
+            }).collect();
+        let offset = offset_reg[0] + (offset_reg[1] << 32);
+
+        trace!(target="TraceDump", "Reading @0x{:x} size_b: {}",
+               offset, size_b);
+
+        let cut_props = ffi::MemZoneProperties {
+            mem_kind: ffi::MemKind::Ddr { offset },
+            size_b,
+        };
+        let mut mz = hpu_hw.alloc(cut_props);
+        mz.sync(ffi::SyncMode::Device2Host);
+        mz.read(0, trace.as_mut_slice());
+        TraceDump{trace}
+    }
+}
+
+impl From<TraceDump> for IscTraceStream {
+    fn from(value: TraceDump) -> Self {
+        IscTraceStream(
+            IscTraceStream::from_bytes(value.trace.as_slice()).0
+            .into_iter()
+            .filter(|i| i.cmd != IscQueryCmd::NONE)
+            .collect())
+    }
+}
--- a/backends/tfhe-hpu-backend/src/isc_trace/fmt.rs
+++ b/backends/tfhe-hpu-backend/src/isc_trace/fmt.rs
@@ -0,0 +1,159 @@
+use crate::asm::dop::DOp;
+use bitvec::prelude::*;
+use packed_struct::{Len, NoMoreBits, PackedStructLsb};
+use serde::Serialize;
+use std::{error::Error, fmt::Display};
+
+pub static TRACE_W: usize = 16;
+
+#[derive(Debug, Serialize, PartialEq, Eq)]
+pub enum IscQueryCmd {
+    NONE,
+    RDUNLOCK,
+    RETIRE,
+    REFILL,
+    ISSUE,
+}
+
+#[derive(Debug)]
+struct BadCmd;
+
+impl Display for BadCmd {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.to_string().as_str())
+    }
+}
+
+impl Error for BadCmd {
+    fn description(&self) -> &str {
+        "No such command"
+    }
+}
+
+impl Len for IscQueryCmd {
+    fn len() -> usize {
+        3
+    }
+}
+
+impl<O> PackedStructLsb<O> for IscQueryCmd
+where
+    O: bitvec::store::BitStore,
+{
+    fn from_bit_slice_le(slice: &BitSlice<O, Lsb0>) -> Result<Self, Box<dyn Error>> {
+        let bits = slice.get(0..3).ok_or(NoMoreBits)?.load::<u8>();
+        match bits {
+            0 => Ok(IscQueryCmd::NONE),
+            1 => Ok(IscQueryCmd::RDUNLOCK),
+            2 => Ok(IscQueryCmd::RETIRE),
+            3 => Ok(IscQueryCmd::REFILL),
+            4 => Ok(IscQueryCmd::ISSUE),
+            _ => Err(BadCmd.into()),
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct IscPoolState {
+    pub(super) pdg: bool,
+    pub(super) rd_pdg: bool,
+    pub(super) vld: bool,
+    pub(super) wr_lock: u32,
+    pub(super) rd_lock: u32,
+    pub(super) sync_id: u32,
+}
+
+impl Len for IscPoolState {
+    fn len() -> usize {
+        21
+    }
+}
+
+impl<O> PackedStructLsb<O> for IscPoolState
+where
+    O: bitvec::store::BitStore,
+{
+    fn from_bit_slice_le(slice: &BitSlice<O, Lsb0>) -> Result<Self, Box<dyn Error>> {
+        Ok(IscPoolState {
+            pdg: *(slice.get(0).ok_or(NoMoreBits)?),
+            rd_pdg: *(slice.get(1).ok_or(NoMoreBits)?),
+            vld: *(slice.get(2).ok_or(NoMoreBits)?),
+            wr_lock: slice.get(3..10).ok_or(NoMoreBits)?.load::<u32>(),
+            rd_lock: slice.get(10..17).ok_or(NoMoreBits)?.load::<u32>(),
+            sync_id: slice.get(17..21).ok_or(NoMoreBits)?.load::<u32>(),
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct IscTrace {
+    pub(super) state: IscPoolState,
+    pub(super) cmd: IscQueryCmd,
+    pub(super) insn: Option<DOp>,
+    pub(super) insn_asm: Option<String>,
+    pub(super) timestamp: u32,
+}
+
+impl<O> PackedStructLsb<O> for IscTrace
+where
+    O: bitvec::store::BitStore,
+{
+    fn from_bit_slice_le(slice: &BitSlice<O, Lsb0>) -> Result<Self, Box<dyn Error>> {
+        let lwe_k_w = 10;
+        let slice = slice.get(lwe_k_w..).ok_or(NoMoreBits)?;
+
+        let state = IscPoolState::from_bit_slice_le(slice)?;
+        let slice = slice.get(IscPoolState::len()..).ok_or(NoMoreBits)?;
+
+        let cmd = IscQueryCmd::from_bit_slice_le(slice)?;
+        let slice = slice.get(IscQueryCmd::len()..).ok_or(NoMoreBits)?;
+
+        let insn = match cmd {
+            IscQueryCmd::REFILL | IscQueryCmd::NONE => None,
+            _ => {
+                let insn = u32::from_bit_slice_le(slice)?;
+                let dop = DOp::from_hex(insn)?;
+                Some(dop)
+            }
+        };
+
+        let slice = slice.get(u32::len()..).ok_or(NoMoreBits)?;
+        let timestamp = u32::from_bit_slice_le(&slice)?;
+        let insn_asm = insn.as_ref().map(|dop| format!("{dop}"));
+
+        Ok(IscTrace {
+            state,
+            cmd,
+            insn,
+            insn_asm,
+            timestamp,
+        })
+    }
+}
+
+impl Len for IscTrace {
+    fn len() -> usize {
+        TRACE_W
+    }
+}
+
+#[derive(Serialize, Debug)]
+pub struct IscTraceStream(pub(super) Vec<IscTrace>);
+
+impl IscTraceStream {
+    pub fn sort(&mut self) {
+        self.0.sort_by_key(|k| k.timestamp)
+    }
+
+    pub fn from_bytes(bytes: &[u8]) -> IscTraceStream {
+        let view = bytes.view_bits::<Lsb0>();
+        IscTraceStream(
+            view.chunks(TRACE_W * 8)
+                .filter_map(|c| IscTrace::from_bit_slice_le(&c).ok())
+                .collect(),
+        )
+    }
+}
+
+#[cfg(test)]
+mod test;
--- a/backends/tfhe-hpu-backend/src/isc_trace/fmt/test/data.rs
+++ b/backends/tfhe-hpu-backend/src/isc_trace/fmt/test/data.rs
@@ -0,0 +1,109 @@
+pub(super) static V80_TEST_DATA: [u16; 856] = [
+0x0000, 0x8000, 0x0001, 0x0000, 0x4bd4, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0002, 0x0000, 0x4bf2, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4c3c, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0806, 0x0000, 0x4c5a, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4ca4, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4d0c, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4d74, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4ddc, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4e44, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0216, 0x0000, 0x4e62, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4eac, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0a1a, 0x0000, 0x4eca, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4f14, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4f7c, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x4fe4, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x504c, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x50b4, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x511c, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x042a, 0x0000, 0x513a, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x5184, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0c2e, 0x0000, 0x51a2, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x51ec, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x5254, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0000, 0x0000, 0x528a, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0001, 0x0000, 0x52d6, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x5320, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x5388, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x53f0, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x5458, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x063e, 0x0000, 0x5476, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x54c0, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0e42, 0x0000, 0x54de, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x5528, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x5590, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x55f8, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x5660, 0x3524, 0x0000, 0x0000,
+0x0000, 0x8000, 0x0001, 0x0000, 0x56b0, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0804, 0x0000, 0x59a2, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0805, 0x0000, 0x59ee, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x000a, 0x1001, 0x5a08, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0214, 0x0000, 0x5e4a, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0215, 0x0000, 0x5e96, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0a18, 0x0000, 0x62f2, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0a19, 0x0000, 0x633e, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0a1e, 0x1006, 0x6358, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0428, 0x0000, 0x67ee, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0429, 0x0000, 0x683a, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0c2c, 0x0000, 0x6efe, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0c2d, 0x0000, 0x6f4a, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x1432, 0x100b, 0x6f64, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x063c, 0x0000, 0x73c2, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x063d, 0x0000, 0x740e, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0e40, 0x0000, 0x7912, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0e41, 0x0000, 0x795e, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x1e46, 0x1010, 0x7978, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0008, 0x1001, 0x7b34, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0009, 0x1001, 0x7b80, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x040e, 0x0003, 0x7b9f, 0x3524, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0412, 0x0001, 0x7bbb, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x040c, 0x0003, 0x7d23, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0410, 0x0001, 0x7e37, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0a1c, 0x1006, 0x9bb8, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0a1d, 0x1006, 0x9c04, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x1430, 0x100b, 0xbc44, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x1431, 0x100b, 0xbc90, 0x3524, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x1e44, 0x1010, 0xdcb4, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x1e45, 0x1010, 0xdd00, 0x3524, 0x0000, 0x0000,
+0x1400, 0x0000, 0x040d, 0x0003, 0xadd7, 0x3540, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0e1e, 0x1003, 0xadf0, 0x3540, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0411, 0x0001, 0xaf1f, 0x3540, 0x0000, 0x0000,
+0x1800, 0x0000, 0x1012, 0x1000, 0xaf3a, 0x3540, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x1010, 0x1000, 0xb5fe, 0x3540, 0x0000, 0x0000,
+0x1400, 0x0000, 0x1011, 0x1000, 0xb64a, 0x3540, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0e1c, 0x1003, 0xcf34, 0x3540, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0e1d, 0x1003, 0xcf80, 0x3540, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0e22, 0x0003, 0xcf9f, 0x3540, 0x0000, 0x0000,
+0x1800, 0x0000, 0x0e26, 0x0001, 0xcfbb, 0x3540, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0e20, 0x0003, 0xd123, 0x3540, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x0e24, 0x0001, 0xd21b, 0x3540, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0e21, 0x0003, 0x2f37, 0x355c, 0x0000, 0x0000,
+0x1800, 0x0000, 0x1832, 0x1008, 0x2f50, 0x355c, 0x0000, 0x0000,
+0x1400, 0x0000, 0x0e25, 0x0001, 0x3063, 0x355c, 0x0000, 0x0000,
+0x1800, 0x0000, 0x1226, 0x1000, 0x307e, 0x355c, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x1224, 0x1000, 0x3742, 0x355c, 0x0000, 0x0000,
+0x1400, 0x0000, 0x1225, 0x1000, 0x378e, 0x355c, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x1830, 0x1008, 0x5094, 0x355c, 0x0000, 0x0000,
+0x1400, 0x0000, 0x1831, 0x1008, 0x50e0, 0x355c, 0x0000, 0x0000,
+0x1800, 0x0000, 0x1836, 0x0003, 0x50ff, 0x355c, 0x0000, 0x0000,
+0x1800, 0x0000, 0x183a, 0x0001, 0x511b, 0x355c, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x1834, 0x0003, 0x5283, 0x355c, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x1838, 0x0001, 0x537b, 0x355c, 0x0000, 0x0000,
+0x1400, 0x0000, 0x1835, 0x0003, 0x4d0f, 0x3577, 0x0000, 0x0000,
+0x1800, 0x0000, 0x2246, 0x100d, 0x4d28, 0x3577, 0x0000, 0x0000,
+0x1400, 0x0000, 0x1839, 0x0001, 0x4e57, 0x3577, 0x0000, 0x0000,
+0x1800, 0x0000, 0x143a, 0x1000, 0x4e72, 0x3577, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x1438, 0x1000, 0x5536, 0x3577, 0x0000, 0x0000,
+0x1400, 0x0000, 0x1439, 0x1000, 0x5582, 0x3577, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x2244, 0x100d, 0x6e6c, 0x3577, 0x0000, 0x0000,
+0x1400, 0x0000, 0x2245, 0x100d, 0x6eb8, 0x3577, 0x0000, 0x0000,
+0x1800, 0x0000, 0x224a, 0x0001, 0x6ed7, 0x3577, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x2248, 0x0001, 0x705b, 0x3577, 0x0000, 0x0000,
+0x1400, 0x0000, 0x2249, 0x0001, 0x8b67, 0x3592, 0x0000, 0x0000,
+0x1800, 0x0000, 0x164a, 0x1000, 0x8b82, 0x3592, 0x0000, 0x0000,
+0x1c00, 0x8000, 0x1648, 0x1000, 0x9492, 0x3592, 0x0000, 0x0000,
+0x1400, 0x0000, 0x1649, 0x1000, 0x94de, 0x3592, 0x0000, 0x0000,
+0x1800, 0x0000, 0xfffe, 0x0003, 0x94f9, 0x3592, 0x0000, 0x0000,
+0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+];
--- a/backends/tfhe-hpu-backend/src/isc_trace/fmt/test/mod.rs
+++ b/backends/tfhe-hpu-backend/src/isc_trace/fmt/test/mod.rs
@@ -0,0 +1,25 @@
+use super::*;
+mod data;
+
+#[test]
+fn simple() {
+    let bytes: Vec<u64> = vec![0x180000000, 0xf1b6e5ec, 0x2000000200001800,
+    0xf1b6e608, 0x180000000, 0xf1b6e670, 0x180000000, 0xf1b6e710,
+    0x180000000, 0xf1b6e760, 0x2000000080001c00, 0xf1b7074c,
+    0x2000000100001400, 0xf1b70798, 0x2000000200001800, 0xf1b707b4,
+    0x2000000080001c00, 0xf1b728f0, 0x2000000100001400, 0xf1b7293c,
+    0x2000000200001800, 0xf1b72958, 0x2000000080001c00, 0xf1b74a94,
+    0x2000000100001400, 0xf1b74ae0, 0x3fffe00001800, 0xf1b74afd];
+
+    let byte_view = bytemuck::try_cast_slice::<_, u8>(bytes.as_slice()).unwrap();
+    let stream = IscTraceStream::from_bytes(&byte_view);
+    print!("stream: {:?}\n", stream);
+}
+
+#[test]
+fn v80() {
+    let bytes = &data::V80_TEST_DATA;
+    let byte_view = bytemuck::try_cast_slice::<_, u8>(bytes.as_slice()).unwrap();
+    let stream = IscTraceStream::from_bytes(&byte_view);
+    print!("stream: {:?}\n", stream);
+}
--- a/backends/tfhe-hpu-backend/src/lib.rs
+++ b/backends/tfhe-hpu-backend/src/lib.rs
@@ -4,6 +4,8 @@ mod entities;
 mod ffi;
 #[cfg(feature = "utils")]
 pub mod ffi;
+#[cfg(feature = "utils")]
+pub mod isc_trace;

 pub mod interface;

--- a/backends/tfhe-hpu-backend/src/packed_struct/Cargo.toml
+++ b/backends/tfhe-hpu-backend/src/packed_struct/Cargo.toml
@@ -0,0 +1,7 @@
+[package]
+name = "packed_struct"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+bitvec = "1"
--- a/backends/tfhe-hpu-backend/src/packed_struct/src/lib.rs
+++ b/backends/tfhe-hpu-backend/src/packed_struct/src/lib.rs
@@ -0,0 +1,121 @@
+use std::error::Error;
+use bitvec::prelude::*;
+use std::fmt;
+use std::fmt::{Display, Debug, Formatter};
+
+mod macros;
+
+#[derive(Debug)]
+pub struct NoMoreBits;
+
+impl Error for NoMoreBits {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        None
+    }
+
+    fn description(&self) -> &str {
+        "No more bits to unpack"
+    }
+}
+
+impl Display for NoMoreBits {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+pub trait Len {
+    fn len() -> usize;
+}
+
+pub trait PackedStructLsb<O>
+where O: bitvec::store::BitStore,
+      Self: Sized + Len
+{
+    fn from_bit_slice_le(slice: &BitSlice<O,Lsb0>) -> 
+            Result<Self, Box<dyn Error>>;
+    fn to_bit_slice_le(&self, _dst: &mut BitSlice<O,Lsb0>) ->
+        Result<(), Box<dyn Error>> {
+        Ok(())
+    }
+}
+
+impl Len for bool {
+    fn len() -> usize { 1 }
+}
+
+impl <O> PackedStructLsb<O> for bool
+where O: bitvec::store::BitStore
+{
+    fn from_bit_slice_le(slice: &BitSlice<O,Lsb0>) -> 
+        Result<Self, Box<dyn Error>> {
+            if slice.len() != 0 {
+                Ok(slice[0])
+            } else {
+                Err(NoMoreBits)?
+            }
+        }
+    fn to_bit_slice_le(&self, dst: &mut BitSlice<O,Lsb0>) ->
+        Result<(), Box<dyn Error>> {
+            if dst.len() > bool::len() {
+                Err(NoMoreBits)?
+            } else {
+                dst.set(0, *self);
+                Ok(())
+            }
+        }
+}
+
+integer_packed_struct!(u8);
+integer_packed_struct!(u16);
+integer_packed_struct!(u32);
+integer_packed_struct!(u64);
+
+#[cfg(test)]
+mod packed_struct_tests {
+    use super::*;
+
+    #[test]
+    fn simple() {
+        let mut bytes: [u8;3] = [0x00, 0x00, 0x00];
+        let out_view = bytes.view_bits_mut::<Lsb0>();
+
+        let byte0: u8 = 0xFF;
+        let byte1: u8 = 0x00;
+        let byte2: u8 = 0xF0;
+
+        byte0.to_bit_slice_le(&mut out_view[0..7]).unwrap();
+        byte1.to_bit_slice_le(&mut out_view[7..15]).unwrap();
+        byte2.to_bit_slice_le(&mut out_view[15..23]).unwrap();
+
+        print!("Struct partially deserialized 0x{:?}\n", bytes);
+
+        let bytes: [u8;3] = [0xBA, 0xBE, 0x12];
+        let mut view = bytes.view_bits::<Lsb0>();
+
+        for _ in bytes {
+            print!("next u8: {:X}\n", u8::from_bit_slice_le(&view).unwrap());
+            view = &view[u8::len()..];
+        }
+    }
+
+    #[test]
+    fn bitvec() {
+        use bitvec::prelude::*;
+
+        let raw = [
+            0x8_Fu8,
+            //  7 0
+            0x0_1u8,
+            // 15 8
+            0b1111_0010u8,
+            //       ^ sign bit
+            // 23       16
+        ];
+        let asd = &raw.view_bits::<Lsb0>() [4 .. 20];
+        assert_eq!(
+            asd.load_le::<u16>(),
+            0x2018u16,
+            );
+    }
+}
--- a/backends/tfhe-hpu-backend/src/packed_struct/src/macros.rs
+++ b/backends/tfhe-hpu-backend/src/packed_struct/src/macros.rs
@@ -0,0 +1,42 @@
+#[macro_export]
+macro_rules! integer_packed_struct {
+    ($x: ident) => 
+        {
+            impl <O> PackedStructLsb<O> for $x 
+            where O: bitvec::store::BitStore
+            {
+                fn from_bit_slice_le(slice: &BitSlice<O,Lsb0>) -> 
+                    Result<Self, Box<dyn Error>> {
+                        if slice.len() != 0 {
+                            Ok(slice[0..($x::BITS as usize).min(slice.len())]
+                                .load::<$x>())
+                        } else {
+                            Err(NoMoreBits)?
+                        }
+                    }
+                fn to_bit_slice_le(&self, dst: &mut BitSlice<O,Lsb0>) ->
+                    Result<(), Box<dyn Error>> {
+                        if dst.len() == 0 {
+                            Err(NoMoreBits)?
+                        } else {
+                            let size = dst.len().min($x::len());
+                            dst[0..size].clone_from_bitslice(
+                                &self.try_view_bits::<Lsb0>()?[0..size]);
+                            Ok(())
+                        }
+                }
+            }
+
+            impl Len for $x {
+                fn len() -> usize {
+                    $x::BITS as usize
+                }
+            }
+        }
+}
+
+#[macro_export]
+macro_rules! integer_len {
+    ($x: ident) => {
+    }
+}
--- a/backends/tfhe-hpu-backend/src/utils/hputil.rs
+++ b/backends/tfhe-hpu-backend/src/utils/hputil.rs
@@ -6,9 +6,16 @@ use tfhe_hpu_backend::interface::rtl;
 use tfhe_hpu_backend::interface::rtl::FromRtl;
 use tfhe_hpu_backend::prelude::*;

+use tfhe_hpu_backend::isc_trace::{IscTraceStream, TraceDump};
+
 use clap::{Parser, Subcommand, ValueEnum};
 use clap_num::maybe_hex;

+use serde_json;
+use std::fs::File;
+
+use tracing_subscriber::fmt::MakeWriter;
+
 #[derive(Clone, Debug, Subcommand)]
 pub enum Command {
    #[clap(about = "Read register")]
@@ -67,6 +74,12 @@ pub enum Command {
        #[arg(short, long, value_parser=maybe_hex::<u8>)]
        pattern: u8,
    },
+
+    #[clap(about = "Trace Dump")]
+    TraceDump {
+        #[arg(short, long, default_value_t = String::from("trace.json"))]
+        file: String,
+    },
 }

 #[derive(Clone, Debug, ValueEnum)]
@@ -129,6 +142,10 @@ fn main() {
        hw_regmap::FlatRegmap::from_file(&regmap_str)
    };

+    // Init the memory backend
+    let params = HpuParameters::from_rtl(&mut hpu_hw, &regmap);
+    hpu_hw.init_mem(&config, &params);
+
    // Handle user command --------------------------------------------------
    match args.cmd {
        Command::Read { name, range } => {
@@ -259,5 +276,13 @@ fn main() {
            mz.write(0, bfr.as_slice());
            mz.sync(ffi::SyncMode::Host2Device);
        }
+        Command::TraceDump { file: filename } => {
+            let trace = TraceDump::new_from(&mut hpu_hw, &regmap, config.board.trace_depth);
+            let parsed = IscTraceStream::from(trace);
+
+            let file = File::create(filename).expect("Failed to create or open trace dump file");
+            serde_json::to_writer_pretty(file.make_writer(), &parsed)
+                .expect("Could not write trace dump");
+        }
    }
 }