Implement Burst Memory Access for RTIO DMA & Analyzer

artiq/firmware/runtime/analyzer.rs

@@ -10,7 +10,7 @@ use core::cell::RefCell;
 
 const BUFFER_SIZE: usize = 512 * 1024;
 
-#[repr(align(64))]
+#[repr(align(2048))]
 struct Buffer {
     data: [u8; BUFFER_SIZE],
 }

artiq/gateware/rtio/analyzer.py

@@ -151,23 +151,56 @@ def __init__(self, membus, cpu_dw):
                                        alignment_bits=data_alignment)
         self.byte_count = CSRStatus(64)  # only read when shut down
 
-        self.sink = stream.Endpoint(
-            [("data", dw),
-             ("valid_token_count", bits_for(messages_per_dw))])
+        layout = [
+            ("data", dw),
+            ("valid_token_count", bits_for(messages_per_dw))
+        ]
+
+        self.sink = stream.Endpoint(layout)
+        self.done = Signal()
 
         # # #
 
+        lookahead_buf = stream.Endpoint(layout)
+        buffer_open = Signal()
+
+        self.comb += [
+            buffer_open.eq(~lookahead_buf.stb | lookahead_buf.ack),
+            self.sink.ack.eq(buffer_open),
+        ]
+        self.sync += [
+            If(buffer_open,
+                self.sink.connect(lookahead_buf, omit={"ack"}),
+            ),
+        ]
+
+        # RULE 4.30: Set End-Of-Burst to signal the end of the current burst.
+        last = Signal()
+        stale_last = Signal()
+        self.comb += last.eq(lookahead_buf.eop | (lookahead_buf.stb & ~self.sink.stb))
+        self.sync += \
+            If(~buffer_open,
+                If(last, stale_last.eq(1)),
+            ).Else(
+                stale_last.eq(0)
+            )
+
+        # Write operations completes once EoP is transmitted
+        self.comb += self.done.eq(
+            lookahead_buf.stb & lookahead_buf.ack & lookahead_buf.eop)
+
         self.comb += [
-            membus.cyc.eq(self.sink.stb),
-            membus.stb.eq(self.sink.stb),
-            self.sink.ack.eq(membus.ack),
+            membus.cyc.eq(lookahead_buf.stb),
+            membus.stb.eq(lookahead_buf.stb),
+            membus.cti.eq(Mux(last | stale_last, 0b111, 0b010)),
+            lookahead_buf.ack.eq(membus.ack),
             membus.we.eq(1),
-            membus.dat_w.eq(dma.convert_signal(self.sink.data, cpu_dw//8))
+            membus.dat_w.eq(dma.convert_signal(lookahead_buf.data, cpu_dw//8))
         ]
         if messages_per_dw > 1:
             for i in range(dw//8):
                 self.comb += membus.sel[i].eq(
-                    self.sink.valid_token_count >= i//(256//8))
+                    lookahead_buf.valid_token_count >= i//(256//8))
         else:
             self.comb += membus.sel.eq(2**(dw//8)-1)
 
@@ -189,7 +222,7 @@ def __init__(self, membus, cpu_dw):
         self.sync += [
             If(self.reset.re, message_count.eq(0)),
             If(membus.ack, message_count.eq(
-                message_count + self.sink.valid_token_count))
+                message_count + lookahead_buf.valid_token_count))
         ]
 
 
@@ -201,8 +234,9 @@ def __init__(self, tsc, cri, membus, fifo_depth=128, cpu_dw=32):
 
         self.submodules.message_encoder = MessageEncoder(
             tsc, cri, self.enable.storage)
+        hi_wm = 64 if fifo_depth > 64 else None
         self.submodules.fifo = stream.SyncFIFO(
-            [("data", message_len)], fifo_depth, True)
+            [("data", message_len)], fifo_depth, True, hi_wm=hi_wm)
         self.submodules.converter = stream.Converter(
             message_len, len(membus.dat_w), reverse=True,
             report_valid_token_count=True)
@@ -213,8 +247,8 @@ def __init__(self, tsc, cri, membus, fifo_depth=128, cpu_dw=32):
             enable_r.eq(self.enable.storage),
             If(self.enable.storage & ~enable_r,
                 self.busy.status.eq(1)),
-            If(self.dma.sink.stb & self.dma.sink.ack & self.dma.sink.eop,
-                self.busy.status.eq(0))
+            If(self.dma.done,
+                self.busy.status.eq(0)),
         ]
 
         self.comb += [

artiq/gateware/rtio/dma.py

@@ -35,23 +35,38 @@ def __init__(self, bus, cpu_dw):
         # # #
 
         bus_stb = Signal()
-        data_reg_loaded = Signal()
+
+        transfer_cyc = Signal(max=64, reset=64-1)
+        transfer_cyc_ce = Signal()
+        transfer_cyc_rst = Signal()
+        self.sync += [
+            If(transfer_cyc_rst,
+                transfer_cyc.eq(transfer_cyc.reset),
+            ).Elif(transfer_cyc_ce,
+                transfer_cyc.eq(transfer_cyc - 1),
+            )
+        ]
+
+        last = Signal()
 
         self.comb += [
-            bus_stb.eq(self.sink.stb & (~data_reg_loaded | self.source.ack)),
+            # source ack (from FIFO) signals FIFO space availability
+            bus_stb.eq(self.sink.stb & self.source.ack),
+            last.eq(transfer_cyc == 0),
+
+            transfer_cyc_rst.eq(self.source.stb & self.source.ack & (self.sink.eop | last)),
+            transfer_cyc_ce.eq(self.source.stb & self.source.ack),
+
             bus.cyc.eq(bus_stb),
             bus.stb.eq(bus_stb),
+            bus.cti.eq(Mux((self.sink.eop | last), 0b111, 0b010)),
             bus.adr.eq(self.sink.address),
+
             self.sink.ack.eq(bus.ack),
-            self.source.stb.eq(data_reg_loaded),
-        ]
-        self.sync += [
-            If(self.source.ack, data_reg_loaded.eq(0)),
-            If(bus.ack,
-                data_reg_loaded.eq(1),
-                self.source.data.eq(convert_signal(bus.dat_r, cpu_dw//8)),
-                self.source.eop.eq(self.sink.eop)
-            )
+            self.source.stb.eq(bus.ack),
+
+            self.source.data.eq(convert_signal(bus.dat_r, cpu_dw//8)),
+            self.source.eop.eq(self.sink.eop),
         ]
 
 
@@ -341,13 +356,16 @@ def __init__(self, membus, cpu_dw):
 
         flow_enable = Signal()
         self.submodules.dma = DMAReader(membus, flow_enable, cpu_dw)
+        self.submodules.fifo = stream.SyncFIFO(
+            [("data", len(membus.dat_w))], 128, True, lo_wm=64)
         self.submodules.slicer = RecordSlicer(len(membus.dat_w))
         self.submodules.time_offset = TimeOffset()
         self.submodules.cri_master = CRIMaster()
         self.cri = self.cri_master.cri
 
         self.comb += [
-            self.dma.source.connect(self.slicer.sink),
+            self.dma.source.connect(self.fifo.sink),
+            self.fifo.source.connect(self.slicer.sink),
             self.slicer.source.connect(self.time_offset.sink),
             self.time_offset.source.connect(self.cri_master.sink)
         ]

artiq/test/coredevice/test_rtio.py

@@ -720,13 +720,6 @@ def test_dma_record_time(self):
         self.assertLess(dt/count, 11*us)
 
     def test_dma_playback_time(self):
-        # Skip on Kasli until #946 is resolved.
-        try:
-            # hack to detect Kasli.
-            self.device_mgr.get_desc("ad9914dds0")
-        except KeyError:
-            raise unittest.SkipTest("skipped on Kasli for now")
-
         exp = self.create(_DMA)
         is_zynq = exp.core.target_cls == CortexA9Target
         count = 20000