axi_dmac: early abort support

Data mover/ src axis changes
  Request rewind ID if TLAST received during non-last burst
  Consume (ignore) descriptors until last segment received
  Block descriptors towards destination until last segment received

Request generator changes
  Rewind the burst ID if rewind request received
  Consume (ignore) descriptors until last segment received
  If TLAST happened on last segment replay next transfer (in progress or
   completed) with the adjusted ID
  Create completion requests for ignored segments

Response generator changes
  Track requests
  Complete segments which got ignored

library/axi_dmac/Makefile

@@ -49,6 +49,7 @@ XILINX_LIB_DEPS += util_cdc
 ALTERA_DEPS += ../util_axis_fifo/util_axis_fifo.v
 ALTERA_DEPS += ../util_axis_fifo/address_sync.v
 ALTERA_DEPS += ../util_cdc/sync_bits.v
+ALTERA_DEPS += ../util_cdc/sync_event.v
 ALTERA_DEPS += axi_dmac_constr.sdc
 ALTERA_DEPS += axi_dmac_hw.tcl
 

library/axi_dmac/axi_dmac_constr.ttcl

@@ -60,6 +60,34 @@ set_max_delay -quiet -datapath_only \
 	-to $src_clk \
 	[get_property -min PERIOD $src_clk]
 
+set_max_delay -quiet -datapath_only \
+	-from $src_clk \
+	-to [get_cells -quiet -hier *cdc_sync_stage1_reg* \
+		-filter {NAME =~ *i_rewind_req_fifo/i_waddr_sync* && IS_SEQUENTIAL}] \
+	[get_property -min PERIOD $src_clk]
+
+set_max_delay -quiet -datapath_only \
+	-from $dest_clk \
+	-to [get_cells -quiet -hier *cdc_sync_stage1_reg* \
+		-filter {NAME =~ *i_rewind_req_fifo/i_raddr_sync* && IS_SEQUENTIAL}] \
+	[get_property -min PERIOD $dest_clk]
+
+set_max_delay -quiet -datapath_only \
+	-from [get_cells -quiet -hier *cdc_sync_fifo_ram_reg* \
+		-filter {NAME =~ *i_rewind_req_fifo* && IS_SEQUENTIAL}] \
+	-to $dest_clk \
+	[get_property -min PERIOD $dest_clk]
+
+set_false_path -quiet \
+	-from $req_clk \
+	-to [get_cells -quiet -hier *cdc_sync_stage1_reg* \
+		-filter {NAME =~ *sync_rewind/i_sync_out* && IS_SEQUENTIAL}]
+
+set_false_path -quiet \
+	-from $src_clk \
+	-to [get_cells -quiet -hier *cdc_sync_stage1_reg* \
+		-filter {NAME =~ *sync_rewind/i_sync_in* && IS_SEQUENTIAL}]
+
 <: } :>
 <: if {$async_dest_req} { :>
 set_max_delay -quiet -datapath_only \

library/axi_dmac/axi_dmac_hw.tcl

@@ -16,6 +16,7 @@ set_module_property VALIDATION_CALLBACK axi_dmac_validate
 
 ad_ip_files axi_dmac [list \
   $ad_hdl_dir/library/util_cdc/sync_bits.v \
+  $ad_hdl_dir/library/util_cdc/sync_event.v \
   $ad_hdl_dir/library/common/up_axi.v \
   $ad_hdl_dir/library/util_axis_fifo/util_axis_fifo.v \
   $ad_hdl_dir/library/util_axis_fifo/address_sync.v \

library/axi_dmac/axi_dmac_response_manager.v

@@ -60,17 +60,22 @@ module axi_dmac_response_manager #(
   output reg [BYTES_PER_BURST_WIDTH-1:0] measured_burst_length = 'h0,
   output response_partial,
   output reg response_valid = 1'b0,
-  input response_ready
+  input response_ready,
 
   // Interface to requester side
+  input completion_req_valid,
+  input completion_req_last,
+  input [1:0] completion_transfer_id
 );
 
-localparam STATE_IDLE         = 2'h0;
-localparam STATE_ACC          = 2'h1;
-localparam STATE_WRITE_RESPR  = 2'h2;
+localparam STATE_IDLE         = 3'h0;
+localparam STATE_ACC          = 3'h1;
+localparam STATE_WRITE_RESPR  = 3'h2;
+localparam STATE_ZERO_COMPL   = 3'h3;
+localparam STATE_WRITE_ZRCMPL = 3'h4;
 
-reg [1:0] state = STATE_IDLE;
-reg [1:0] nx_state;
+reg [2:0] state = STATE_IDLE;
+reg [2:0] nx_state;
 
 localparam DEST_SRC_RATIO = DMA_DATA_WIDTH_DEST/DMA_DATA_WIDTH_SRC;
 
@@ -86,6 +91,7 @@ localparam BYTES_PER_BEAT_WIDTH = DEST_SRC_RATIO_WIDTH + BYTES_PER_BEAT_WIDTH_SR
 localparam BURST_LEN_WIDTH = BYTES_PER_BURST_WIDTH - BYTES_PER_BEAT_WIDTH;
 
 wire do_acc_st;
+wire do_compl;
 reg req_eot = 1'b0;
 reg req_response_partial = 1'b0;
 reg [BYTES_PER_BURST_WIDTH-1:0] req_response_dest_data_burst_length = 'h0;
@@ -99,6 +105,10 @@ wire [BYTES_PER_BURST_WIDTH-1:0] response_dest_data_burst_length;
 wire [BURST_LEN_WIDTH-1:0] burst_lenght;
 reg [BURST_LEN_WIDTH-1:0] burst_pointer_end;
 
+reg [1:0] to_complete_count = 'h0;
+reg [1:0] transfer_id = 'h0;
+reg completion_req_last_found = 1'b0;
+
 util_axis_fifo #(
   .DATA_WIDTH(BYTES_PER_BURST_WIDTH+1+1),
   .ADDRESS_WIDTH(0),
@@ -150,7 +160,7 @@ begin
   if (req_resetn == 1'b0) begin
     response_valid <= 1'b0;
   end else begin
-    if (nx_state == STATE_WRITE_RESPR) begin
+    if (nx_state == STATE_WRITE_RESPR || nx_state == STATE_WRITE_ZRCMPL) begin
       response_valid <= 1'b1;
     end else if (response_ready == 1'b1) begin
       response_valid <= 1'b0;
@@ -188,6 +198,9 @@ always @(*) begin
     STATE_IDLE: begin
       if (response_dest_valid == 1'b1) begin
         nx_state = STATE_ACC;
+      end else if (|to_complete_count) begin
+        if (transfer_id == completion_transfer_id)
+          nx_state = STATE_ZERO_COMPL;
       end
     end
     STATE_ACC: begin
@@ -198,6 +211,20 @@ always @(*) begin
         nx_state = STATE_IDLE;
       end
     end
+    STATE_ZERO_COMPL: begin
+      if (|to_complete_count) begin
+        nx_state = STATE_WRITE_ZRCMPL;
+      end else begin
+        if (completion_req_last_found == 1'b1) begin
+          nx_state = STATE_IDLE;
+        end
+      end
+    end
+    STATE_WRITE_ZRCMPL:begin
+      if (response_ready == 1'b1) begin
+        nx_state = STATE_ZERO_COMPL;
+      end
+    end
     default: begin
       nx_state = STATE_IDLE;
     end
@@ -212,4 +239,39 @@ always @(posedge req_clk) begin
   end
 end
 
+assign do_compl = (state == STATE_WRITE_ZRCMPL) && response_ready;
+
+// Once the last completion request from request generator is received 
+// we can wait for completions from the destination side
+always @(posedge req_clk) begin
+  if (req_resetn == 1'b0) begin
+    completion_req_last_found <= 1'b0;
+  end else if (completion_req_valid) begin
+    completion_req_last_found <= completion_req_last;
+  end else if (state ==STATE_ZERO_COMPL && ~(|to_complete_count)) begin
+    completion_req_last_found <= 1'b0;
+  end
+end
+
+// Track transfers so we can tell when did the destination completed all its
+// transfers  
+always @(posedge req_clk) begin
+  if (req_resetn == 1'b0) begin
+    transfer_id <= 'h0;
+  end else if ((state == STATE_ACC && req_eot) || do_compl) begin
+    transfer_id <= transfer_id + 1;
+  end
+end
+
+// Count how many transfers we need to complete 
+always @(posedge req_clk) begin
+  if (req_resetn == 1'b0) begin
+    to_complete_count <= 'h0;
+  end else if (completion_req_valid & ~do_compl) begin
+    to_complete_count <= to_complete_count + 1;
+  end else if (~completion_req_valid & do_compl) begin
+    to_complete_count <= to_complete_count - 1;
+  end
+end
+
 endmodule

library/axi_dmac/data_mover.v

@@ -47,10 +47,16 @@ module dmac_data_mover #(
   output [ID_WIDTH-1:0] response_id,
   input eot,
 
+  output rewind_req_valid,
+  input rewind_req_ready,
+  output [ID_WIDTH+3-1:0] rewind_req_data,
+
   output reg bl_valid = 'b0,
   input bl_ready,
   output reg [BEATS_PER_BURST_WIDTH-1:0] measured_last_burst_length,
 
+  output block_descr_to_dst,
+
   output [ID_WIDTH-1:0] source_id,
   output source_eot,
 
@@ -97,36 +103,46 @@ wire transfer_abort_s;
 
 wire last_load;
 wire last;
+wire early_tlast;
 
 assign xfer_req = active;
 
 assign response_id = id;
 
 assign source_id = id;
-assign source_eot = eot;
+assign source_eot = eot || early_tlast;
 
 assign last = eot ? last_eot : last_non_eot;
 
 assign s_axi_ready = (pending_burst & active) & ~transfer_abort_s;
-assign m_axi_valid = (s_axi_sync_valid | transfer_abort_s) & pending_burst & active;
-assign m_axi_data = transfer_abort_s == 1'b1 ? {DATA_WIDTH{1'b0}} : s_axi_data;
-assign m_axi_last = last;
+assign m_axi_valid = s_axi_sync_valid & s_axi_ready;
+assign m_axi_data = s_axi_data;
+assign m_axi_last = last || early_tlast;
+assign m_axi_partial_burst = early_tlast;
+
+assign block_descr_to_dst = transfer_abort_s;
 
 generate if (ALLOW_ABORT == 1) begin
+  wire programmed_last;
+
   reg transfer_abort = 1'b0;
   reg req_xlast_d = 1'b0;
+  reg [1:0] transfer_id = 2'b0;
 
+  assign programmed_last = (last == 1'b1 && eot == 1'b1 && req_xlast_d == 1'b1);
   /*
    * A 'last' on the external interface indicates the end of an packet. If such a
    * 'last' indicator is observed before the end of the current transfer stop
-   * accepting data on the external interface and complete the current transfer by
-   * writing zeros to the buffer.
+   * accepting data on the external interface until a new descriptor is
+   * received that is the first segment of a transfer. 
    */
   always @(posedge clk) begin
     if (resetn == 1'b0) begin
       transfer_abort <= 1'b0;
+    end else if (req_valid == 1'b1 && req_ready == 1'b1 && req_xlast_d == 1'b1) begin
+      transfer_abort <= 1'b0;
     end else if (m_axi_valid == 1'b1) begin
-      if (last == 1'b1 && eot == 1'b1 && req_xlast_d == 1'b1) begin
+      if (programmed_last == 1'b1) begin
         transfer_abort <= 1'b0;
       end else if (s_axi_last == 1'b1) begin
         transfer_abort <= 1'b1;
@@ -135,18 +151,33 @@ generate if (ALLOW_ABORT == 1) begin
   end
 
   always @(posedge clk) begin
-    if (req_ready == 1'b1) begin
+    if (req_ready == 1'b1 && req_valid == 1'b1) begin
       req_xlast_d <= req_xlast;
     end
   end
 
   assign transfer_abort_s = transfer_abort;
-  assign m_axi_partial_burst = (transfer_abort == 1'b0) && (s_axi_last == 1'b1) &&
-                              !(last == 1'b1 && eot == 1'b1 && req_xlast_d == 1'b1);
+  assign early_tlast = (s_axi_ready == 1'b1) && (m_axi_valid == 1'b1) &&
+                       (s_axi_last == 1'b1) && (programmed_last == 1'b0);
+
+  assign rewind_req_valid = early_tlast;
+  assign rewind_req_data = {transfer_id,req_xlast_d,id_next};
+
+  // The width of the id must fit the number of transfers that can be in flight 
+  // in the burst memory
+  always @(posedge clk) begin
+    if (resetn == 1'b0) begin
+      transfer_id <= 2'b0;
+    end else if (req_valid == 1'b1 && req_ready == 1'b1) begin
+      transfer_id <= transfer_id + 1'b1;
+    end
+  end
 
 end else begin
   assign transfer_abort_s = 1'b0;
-  assign m_axi_partial_burst = 1'b0;
+  assign early_tlast = 1'b0;
+  assign rewind_req_valid = 1'b0;
+  assign rewind_req_data = 'h0;
 end endgenerate
 
 /*
@@ -164,7 +195,7 @@ end
 // If we want to support zero delay between transfers we have to assert
 // req_ready on the same cycle on which the last load happens.
 assign last_load = m_axi_valid && last_eot && eot;
-assign req_ready = last_load || ~active;
+assign req_ready = last_load || ~active || (transfer_abort_s & rewind_req_ready);
 
 always @(posedge clk) begin
   if (req_ready) begin
@@ -192,7 +223,7 @@ always @(posedge clk) begin
 end
 
 always @(posedge clk) begin
-  if (last_load) begin
+  if (last_load || early_tlast) begin
     bl_valid <= 1'b1;
     measured_last_burst_length <= beat_counter_minus_one;
   end else if (bl_ready) begin
@@ -212,7 +243,7 @@ end
 
 always @(*)
 begin
-  if (m_axi_valid == 1'b1 && last == 1'b1)
+  if (m_axi_valid == 1'b1 && (last == 1'b1 || early_tlast == 1'b1))
     id_next <= inc_id(id);
   else
     id_next <= id;