feat: DataBus notion with calldata/return data (#5504)

The DataBus will eventually contain 3 columns: calldata, return data,
and a 3rd for storing "function" calldata/returndata. Prior to this PR,
we only had calldata (implemented as a proof of concept). This PR
introduced a more proper notion of the `DataBus` and adds return data to
its functionality. (No 3rd column yet).

Each column (`BusVector`) of the `DataBus` essentially behaves the same
way so the bulk of the work in this PR is simply making that logic
generic/shareable in the `GoblinUltraCircuitBuilder` and the
`DataBusLookupRelation`.

Note: In prior work I collected some of the generic log-derivative
lookup logic (originally introduced by Zac for the ECCVM) in
`logderivative_library.hpp`. I'm no longer using that shared logic for
the Databus for a few reasons: 1) the library is now also used by the
AVM so any changes would have touched many avm files, 2) the DataBus
relation needs functionality for multiple "tables" (bus columns) whereas
others only have one, and 3) the library methods were over generalized
in a way not needed for the DataBus (multiple read/write terms) that
obscured the simple structure of the Databus lookup relation.

barretenberg/cpp/src/barretenberg/stdlib/honk_recursion/verifier/protogalaxy_recursive_verifier.cpp

@@ -47,6 +47,10 @@ void ProtoGalaxyRecursiveVerifier_<VerifierInstances>::receive_and_finalise_inst
             transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.calldata);
         witness_commitments.calldata_read_counts =
             transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.calldata_read_counts);
+        witness_commitments.return_data =
+            transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.return_data);
+        witness_commitments.return_data_read_counts = transcript->template receive_from_prover<Commitment>(
+            domain_separator + "_" + labels.return_data_read_counts);
     }
 
     // Get challenge for sorted list batching and wire four memory records commitment
@@ -62,8 +66,10 @@ void ProtoGalaxyRecursiveVerifier_<VerifierInstances>::receive_and_finalise_inst
 
     // If Goblin (i.e. using DataBus) receive commitments to log-deriv inverses polynomial
     if constexpr (IsGoblinFlavor<Flavor>) {
-        witness_commitments.lookup_inverses = transcript->template receive_from_prover<Commitment>(
-            domain_separator + "_" + commitment_labels.lookup_inverses);
+        witness_commitments.calldata_inverses = transcript->template receive_from_prover<Commitment>(
+            domain_separator + "_" + commitment_labels.calldata_inverses);
+        witness_commitments.return_data_inverses = transcript->template receive_from_prover<Commitment>(
+            domain_separator + "_" + commitment_labels.return_data_inverses);
     }
 
     witness_commitments.z_perm =

barretenberg/cpp/src/barretenberg/stdlib/honk_recursion/verifier/ultra_recursive_verifier.cpp

@@ -68,6 +68,9 @@ std::array<typename Flavor::GroupElement, 2> UltraRecursiveVerifier_<Flavor>::ve
         commitments.calldata = transcript->template receive_from_prover<Commitment>(commitment_labels.calldata);
         commitments.calldata_read_counts =
             transcript->template receive_from_prover<Commitment>(commitment_labels.calldata_read_counts);
+        commitments.return_data = transcript->template receive_from_prover<Commitment>(commitment_labels.return_data);
+        commitments.return_data_read_counts =
+            transcript->template receive_from_prover<Commitment>(commitment_labels.return_data_read_counts);
     }
 
     // Get challenge for sorted list batching and wire four memory records
@@ -85,8 +88,10 @@ std::array<typename Flavor::GroupElement, 2> UltraRecursiveVerifier_<Flavor>::ve
 
     // If Goblin (i.e. using DataBus) receive commitments to log-deriv inverses polynomial
     if constexpr (IsGoblinFlavor<Flavor>) {
-        commitments.lookup_inverses =
-            transcript->template receive_from_prover<Commitment>(commitment_labels.lookup_inverses);
+        commitments.calldata_inverses =
+            transcript->template receive_from_prover<Commitment>(commitment_labels.calldata_inverses);
+        commitments.return_data_inverses =
+            transcript->template receive_from_prover<Commitment>(commitment_labels.return_data_inverses);
     }
 
     const FF public_input_delta = compute_public_input_delta<Flavor>(

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/databus.hpp

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <cstdint>
+namespace bb {
+
+using namespace bb;
+
+/**
+ * @brief A DataBus column
+ *
+ */
+struct BusVector {
+
+    /**
+     * @brief Add an element to the data defining this bus column
+     *
+     * @param idx Index of the element in the variables vector of a builder
+     */
+    void append(const uint32_t& idx)
+    {
+        data.emplace_back(idx);
+        read_counts.emplace_back(0);
+    }
+
+    size_t size() const { return data.size(); }
+
+    const uint32_t& operator[](size_t idx) const
+    {
+        ASSERT(idx < size());
+        return data[idx];
+    }
+
+    const uint32_t& get_read_count(size_t idx) const
+    {
+        ASSERT(idx < read_counts.size());
+        return read_counts[idx];
+    }
+
+    void increment_read_count(size_t idx)
+    {
+        ASSERT(idx < read_counts.size());
+        read_counts[idx]++;
+    }
+
+  private:
+    std::vector<uint32_t> read_counts; // count of reads at each index into data
+    std::vector<uint32_t> data;        // variable indices corresponding to data in this bus vector
+};
+
+/**
+ * @brief The DataBus; facilitates storage of public circuit input/output
+ * @details The DataBus is designed to facilitate efficient transfer of large amounts of public data between circuits.
+ * It is expected that only a small subset of the data being passed needs to be used in any single circuit, thus we
+ * provide a read mechanism (implemented through a Builder) that results in prover work proportional to only the data
+ * that is used. (The prover must still commit to all data in each bus vector but we do not need to hash all data
+ * in-circuit as we would with public inputs).
+ *
+ */
+struct DataBus {
+    BusVector calldata;    // the public input to the circuit
+    BusVector return_data; // the public output of the circuit
+};
+
+} // namespace bb

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/goblin_ultra_circuit_builder.cpp

@@ -33,10 +33,16 @@ template <typename FF> void GoblinUltraCircuitBuilder_<FF>::add_gates_to_ensure_
 
     // Create an arbitrary calldata read gate
     add_public_calldata(FF(25)); // ensure there is at least one entry in calldata
-    uint32_t raw_read_idx = 0;   // read first entry in calldata
+    auto raw_read_idx = static_cast<uint32_t>(databus.calldata.size()) - 1; // read data that was just added
     auto read_idx = this->add_variable(raw_read_idx);
     read_calldata(read_idx);
 
+    // Create an arbitrary return data read gate
+    add_public_return_data(FF(17)); // ensure there is at least one entry in return data
+    raw_read_idx = static_cast<uint32_t>(databus.return_data.size()) - 1; // read data that was just added
+    read_idx = this->add_variable(raw_read_idx);
+    read_return_data(read_idx);
+
     // mock a poseidon external gate, with all zeros as input
     this->blocks.poseidon_external.populate_wires(this->zero_idx, this->zero_idx, this->zero_idx, this->zero_idx);
     this->blocks.poseidon_external.q_m().emplace_back(0);
@@ -219,40 +225,40 @@ template <typename FF> void GoblinUltraCircuitBuilder_<FF>::set_goblin_ecc_op_co
 }
 
 /**
- * @brief Read from calldata
- * @details Creates a calldata lookup gate based on the read data
+ * @brief Read from a databus column
+ * @details Creates a databus lookup gate based on the input index and read result
  *
  * @tparam FF
  * @param read_idx_witness_idx Variable index of the read index
  * @return uint32_t Variable index of the result of the read
  */
-template <typename FF> uint32_t GoblinUltraCircuitBuilder_<FF>::read_calldata(const uint32_t& read_idx_witness_idx)
+template <typename FF>
+uint32_t GoblinUltraCircuitBuilder_<FF>::read_bus_vector(BusVector& bus_vector, const uint32_t& read_idx_witness_idx)
 {
-    // Get the raw index into the calldata
+    // Get the raw index into the databus column
     const uint32_t read_idx = static_cast<uint32_t>(uint256_t(this->get_variable(read_idx_witness_idx)));
 
-    // Ensure that the read index is valid
-    ASSERT(read_idx < public_calldata.size());
+    ASSERT(read_idx < bus_vector.size()); // Ensure that the read index is valid
+    // NOTE(https://github.com/AztecProtocol/barretenberg/issues/937): Multiple reads at same index is not supported.
+    ASSERT(bus_vector.get_read_count(read_idx) < 1);
 
     // Create a variable corresponding to the result of the read. Note that we do not in general connect reads from
-    // calldata via copy constraints (i.e. we create a unique variable for the result of each read)
-    FF calldata_value = this->get_variable(public_calldata[read_idx]);
-    uint32_t value_witness_idx = this->add_variable(calldata_value);
+    // databus via copy constraints (i.e. we create a unique variable for the result of each read)
+    FF value = this->get_variable(bus_vector[read_idx]);
+    uint32_t value_witness_idx = this->add_variable(value);
 
-    create_calldata_read_gate({ read_idx_witness_idx, value_witness_idx });
-    calldata_read_counts[read_idx]++;
+    bus_vector.increment_read_count(read_idx);
 
     return value_witness_idx;
 }
 
 /**
- * @brief Create a calldata lookup/read gate
+ * @brief Create a databus lookup/read gate
  *
  * @tparam FF
- * @param databus_lookup_gate_ witness indices corresponding to: calldata index, calldata value
+ * @param databus_lookup_gate_ witness indices corresponding to: read index, result value
  */
-template <typename FF>
-void GoblinUltraCircuitBuilder_<FF>::create_calldata_read_gate(const databus_lookup_gate_<FF>& in)
+template <typename FF> void GoblinUltraCircuitBuilder_<FF>::create_databus_read_gate(const databus_lookup_gate_<FF>& in)
 {
     auto& block = this->blocks.busread;
     block.populate_wires(in.value, in.index, this->zero_idx, this->zero_idx);
@@ -277,6 +283,36 @@ void GoblinUltraCircuitBuilder_<FF>::create_calldata_read_gate(const databus_loo
     ++this->num_gates;
 }
 
+/**
+ * @brief Create a databus calldata lookup/read gate
+ *
+ * @tparam FF
+ * @param databus_lookup_gate_ witness indices corresponding to: calldata index, calldata value
+ */
+template <typename FF>
+void GoblinUltraCircuitBuilder_<FF>::create_calldata_read_gate(const databus_lookup_gate_<FF>& in)
+{
+    // Create generic read gate then set q_1 = 1 to specify a calldata read
+    create_databus_read_gate(in);
+    auto& block = this->blocks.busread;
+    block.q_1()[block.size() - 1] = 1;
+}
+
+/**
+ * @brief Create a databus return data lookup/read gate
+ *
+ * @tparam FF
+ * @param databus_lookup_gate_ witness indices corresponding to: read index, result value
+ */
+template <typename FF>
+void GoblinUltraCircuitBuilder_<FF>::create_return_data_read_gate(const databus_lookup_gate_<FF>& in)
+{
+    // Create generic read gate then set q_2 = 1 to specify a return data read
+    create_databus_read_gate(in);
+    auto& block = this->blocks.busread;
+    block.q_2()[block.size() - 1] = 1;
+}
+
 /**
  * @brief Poseidon2 external round gate, activates the q_poseidon2_external selector and relation
  */

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/goblin_ultra_circuit_builder.hpp

@@ -2,6 +2,7 @@
 #include "barretenberg/execution_trace/execution_trace.hpp"
 #include "barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp"
 #include "barretenberg/stdlib_circuit_builders/op_queue/ecc_op_queue.hpp"
+#include "databus.hpp"
 #include "ultra_circuit_builder.hpp"
 
 namespace bb {
@@ -26,10 +27,8 @@ template <typename FF> class GoblinUltraCircuitBuilder_ : public UltraCircuitBui
     uint32_t mul_accum_op_idx;
     uint32_t equality_op_idx;
 
-    // DataBus call/return data arrays
-    std::vector<uint32_t> public_calldata;
-    std::vector<uint32_t> calldata_read_counts;
-    std::vector<uint32_t> public_return_data;
+    // Container for public calldata/returndata
+    DataBus databus;
 
     // Functions for adding ECC op queue "gates"
     ecc_op_tuple queue_ecc_add_accum(const g1::affine_element& point);
@@ -40,7 +39,16 @@ template <typename FF> class GoblinUltraCircuitBuilder_ : public UltraCircuitBui
     void populate_ecc_op_wires(const ecc_op_tuple& in);
     ecc_op_tuple decompose_ecc_operands(uint32_t op, const g1::affine_element& point, const FF& scalar = FF::zero());
     void set_goblin_ecc_op_code_constant_variables();
+    uint32_t read_bus_vector(BusVector& bus_vector, const uint32_t& read_idx_witness_idx);
+    void create_databus_read_gate(const databus_lookup_gate_<FF>& in);
     void create_calldata_read_gate(const databus_lookup_gate_<FF>& in);
+    void create_return_data_read_gate(const databus_lookup_gate_<FF>& in);
+    uint32_t append_to_bus_vector(BusVector& bus_vector, const FF& in)
+    {
+        const uint32_t index = this->add_variable(in);
+        bus_vector.append(index);
+        return index;
+    }
 
   public:
     GoblinUltraCircuitBuilder_(const size_t size_hint = 0,
@@ -126,18 +134,40 @@ template <typename FF> class GoblinUltraCircuitBuilder_ : public UltraCircuitBui
     /**
      * @brief Add a witness variable to the public calldata.
      *
-     * @param in Value to be added to calldata.
      * */
-    uint32_t add_public_calldata(const FF& in)
+    uint32_t add_public_calldata(const FF& in) { return append_to_bus_vector(databus.calldata, in); }
+
+    /**
+     * @brief Add a witness variable to the public return_data.
+     *
+     * */
+    uint32_t add_public_return_data(const FF& in) { return append_to_bus_vector(databus.return_data, in); }
+
+    /**
+     * @brief Read from calldata and create a corresponding databus read gate
+     *
+     * @param read_idx_witness_idx Witness index for the calldata read index
+     * @return uint32_t Witness index for the result of the read
+     */
+    uint32_t read_calldata(const uint32_t& read_idx_witness_idx)
     {
-        const uint32_t index = this->add_variable(in);
-        public_calldata.emplace_back(index);
-        // Note: this is a bit inefficent to do every time but for safety these need to be coupled
-        calldata_read_counts.resize(public_calldata.size());
-        return index;
-    }
+        uint32_t value_witness_idx = read_bus_vector(databus.calldata, read_idx_witness_idx);
+        create_calldata_read_gate({ read_idx_witness_idx, value_witness_idx });
+        return value_witness_idx;
+    };
 
-    uint32_t read_calldata(const uint32_t& read_idx_witness_idx);
+    /**
+     * @brief Read from return_data and create a corresponding databus read gate
+     *
+     * @param read_idx_witness_idx Witness index for the return_data read index
+     * @return uint32_t Witness index for the result of the read
+     */
+    uint32_t read_return_data(const uint32_t& read_idx_witness_idx)
+    {
+        uint32_t value_witness_idx = read_bus_vector(databus.return_data, read_idx_witness_idx);
+        create_return_data_read_gate({ read_idx_witness_idx, value_witness_idx });
+        return value_witness_idx;
+    };
 
     void create_poseidon2_external_gate(const poseidon2_external_gate_<FF>& in);
     void create_poseidon2_internal_gate(const poseidon2_internal_gate_<FF>& in);