diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index bc9cbfa2670..557f7c7ca3e 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -233,19 +233,16 @@ test-clang-format: test: ARG hardware_concurrency="" # prefetch - BUILD +test-binaries BUILD +preset-release-assert-test BUILD +test-clang-format BUILD ./srs_db/+build # prefetch - FROM +source - COPY --dir +test-binaries/build build FROM +preset-release-assert-test COPY --dir ./srs_db/+build/. srs_db # limit hardware concurrency, if provided IF [ "$HARDWARE_CONCURRENCY" != "" ] ENV HARDWARE_CONCURRENCY=$hardware_concurrency END - RUN cd build && GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure + RUN cd build && ./bin/stdlib_client_ivc_verifier_tests --gtest_filter=ClientIVCRecursionTests.ClientTubeBase #GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure vm-full-test: ARG hardware_concurrency="" diff --git a/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/main.cpp b/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/main.cpp index 2b6e2caf4a0..e16cdbaee35 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/main.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/pippenger_bench/main.cpp @@ -72,7 +72,7 @@ int pippenger() scalar_multiplication::pippenger_runtime_state state(NUM_POINTS); std::chrono::steady_clock::time_point time_start = std::chrono::steady_clock::now(); g1::element result = scalar_multiplication::pippenger_unsafe( - { &scalars[0], /*size*/ NUM_POINTS }, reference_string->get_monomial_points(), NUM_POINTS, state); + { &scalars[0], /*size*/ NUM_POINTS }, reference_string->get_monomial_points(), state); std::chrono::steady_clock::time_point time_end = std::chrono::steady_clock::now(); std::chrono::microseconds diff = std::chrono::duration_cast(time_end - time_start); std::cout << "run time: " << diff.count() << "us" << std::endl; diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp index f0b20c0a03c..c2cd8ec68b3 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp @@ -129,6 +129,22 @@ template void bench_commit_random(::benchmark::State& state) key->commit(polynomial); } } +// Commit to a polynomial with dense random nonzero entries but NOT our happiest case of an exact power of 2 +// Note this used to be a 50% regression just subtracting a power of 2 by 1. +template void bench_commit_random_non_power_of_2(::benchmark::State& state) +{ + using Fr = typename Curve::ScalarField; + auto key = create_commitment_key(MAX_NUM_POINTS); + + const size_t num_points = 1 << state.range(0); + auto polynomial = Polynomial(num_points - 1); + for (auto& coeff : polynomial) { + coeff = Fr::random_element(); + } + for (auto _ : state) { + key->commit(polynomial); + } +} BENCHMARK(bench_commit_zero) ->DenseRange(MIN_LOG_NUM_POINTS, MAX_LOG_NUM_POINTS) @@ -148,6 +164,9 @@ BENCHMARK(bench_commit_sparse_random_preprocessed) BENCHMARK(bench_commit_random) ->DenseRange(MIN_LOG_NUM_POINTS, MAX_LOG_NUM_POINTS) ->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_random_non_power_of_2) + ->DenseRange(MIN_LOG_NUM_POINTS, MAX_LOG_NUM_POINTS) + ->Unit(benchmark::kMillisecond); } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index 4509ba3e405..746ebbe3786 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -9,6 +9,7 @@ #include "barretenberg/common/op_count.hpp" #include "barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp" +#include "barretenberg/numeric/bitop/get_msb.hpp" #include "barretenberg/numeric/bitop/pow.hpp" #include "barretenberg/polynomials/polynomial_arithmetic.hpp" #include "barretenberg/srs/factories/crs_factory.hpp" @@ -34,6 +35,17 @@ template class CommitmentKey { using Fr = typename Curve::ScalarField; using Commitment = typename Curve::AffineElement; using G1 = typename Curve::AffineElement; + static constexpr size_t EXTRA_SRS_POINTS_FOR_ECCVM_IPA = 1; + + static size_t get_num_needed_srs_points(size_t num_points) + { + // NOTE 1: Currently we must round up internal space for points as our pippenger algorithm (specifically, + // pippenger_unsafe_optimized_for_non_dyadic_polys) will use next power of 2. This is used to simplify the + // recursive halving scheme. We do, however allow the polynomial to not be fully formed. Pippenger internally + // will pad 0s into the runtime state. + // NOTE 2: We then add one for ECCVM to provide for IPA verification + return numeric::round_up_power_2(num_points) + EXTRA_SRS_POINTS_FOR_ECCVM_IPA; + } public: scalar_multiplication::pippenger_runtime_state pippenger_runtime_state; @@ -50,9 +62,9 @@ template class CommitmentKey { * */ CommitmentKey(const size_t num_points) - : pippenger_runtime_state(num_points) + : pippenger_runtime_state(get_num_needed_srs_points(num_points)) , crs_factory(srs::get_crs_factory()) - , srs(crs_factory->get_prover_crs(num_points)) + , srs(crs_factory->get_prover_crs(get_num_needed_srs_points(num_points))) {} // Note: This constructor is to be used only by Plonk; For Honk the srs lives in the CommitmentKey @@ -70,16 +82,17 @@ template class CommitmentKey { Commitment commit(std::span polynomial) { BB_OP_COUNT_TIME(); - const size_t degree = polynomial.size(); - if (degree > srs->get_monomial_size()) { - info("Attempting to commit to a polynomial of degree ", - degree, - " with an SRS of size ", + // See constructor, we must round up the number of used srs points to a power of 2. + const size_t consumed_srs = numeric::round_up_power_2(polynomial.size()); + if (consumed_srs > srs->get_monomial_size()) { + info("Attempting to commit to a polynomial that needs ", + consumed_srs, + " points with an SRS of size ", srs->get_monomial_size()); ASSERT(false); } - return scalar_multiplication::pippenger_unsafe( - polynomial, srs->get_monomial_points(), degree, pippenger_runtime_state); + return scalar_multiplication::pippenger_unsafe_optimized_for_non_dyadic_polys( + polynomial, { srs->get_monomial_points(), srs->get_monomial_size() }, pippenger_runtime_state); }; /** @@ -145,8 +158,7 @@ template class CommitmentKey { } // Call the version of pippenger which assumes all points are distinct - return scalar_multiplication::pippenger_unsafe( - scalars, points.data(), scalars.size(), pippenger_runtime_state); + return scalar_multiplication::pippenger_unsafe(scalars, points.data(), pippenger_runtime_state); } }; diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp index 10e8bfbe80f..fa0ed6e5f96 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp @@ -215,13 +215,13 @@ template class IPA { // Step 6.a (using letters, because doxygen automaticall converts the sublist counters to letters :( ) // L_i = < a_vec_lo, G_vec_hi > + inner_prod_L * aux_generator L_i = bb::scalar_multiplication::pippenger_without_endomorphism_basis_points( - {&a_vec[0], /*size*/ round_size}, &G_vec_local[round_size], round_size, ck->pippenger_runtime_state); + {&a_vec[0], /*size*/ round_size}, &G_vec_local[round_size], ck->pippenger_runtime_state); L_i += aux_generator * inner_prod_L; // Step 6.b // R_i = < a_vec_hi, G_vec_lo > + inner_prod_R * aux_generator R_i = bb::scalar_multiplication::pippenger_without_endomorphism_basis_points( - {&a_vec[round_size], /*size*/ round_size}, &G_vec_local[0], round_size, ck->pippenger_runtime_state); + {&a_vec[round_size], /*size*/ round_size}, &G_vec_local[0], ck->pippenger_runtime_state); R_i += aux_generator * inner_prod_R; // Step 6.c @@ -345,7 +345,7 @@ template class IPA { // Step 5. // Compute C₀ = C' + ∑_{j ∈ [k]} u_j^{-1}L_j + ∑_{j ∈ [k]} u_jR_j GroupElement LR_sums = bb::scalar_multiplication::pippenger_without_endomorphism_basis_points( - {&msm_scalars[0], /*size*/ pippenger_size}, &msm_elements[0], pippenger_size, vk->pippenger_runtime_state); + {&msm_scalars[0], /*size*/ pippenger_size}, &msm_elements[0], vk->pippenger_runtime_state); GroupElement C_zero = C_prime + LR_sums; // Step 6. @@ -394,7 +394,7 @@ template class IPA { // Step 8. // Compute G₀ Commitment G_zero = bb::scalar_multiplication::pippenger_without_endomorphism_basis_points( - {&s_vec[0], /*size*/ poly_length}, &G_vec_local[0], poly_length, vk->pippenger_runtime_state); + {&s_vec[0], /*size*/ poly_length}, &G_vec_local[0], vk->pippenger_runtime_state); // Step 9. // Receive a₀ from the prover diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/runtime_states.hpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/runtime_states.hpp index 19f686f2f3b..d6b659bd25f 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/runtime_states.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/runtime_states.hpp @@ -64,7 +64,7 @@ constexpr size_t get_num_rounds(const size_t num_points) } template struct affine_product_runtime_state { - typename Curve::AffineElement* points; + const typename Curve::AffineElement* points; typename Curve::AffineElement* point_pairs_1; typename Curve::AffineElement* point_pairs_2; typename Curve::BaseField* scratch_space; diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.cpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.cpp index 329b8138a1e..3bd52afd563 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.cpp +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -12,6 +13,7 @@ #include "barretenberg/common/op_count.hpp" #include "barretenberg/common/thread.hpp" #include "barretenberg/common/throw_or_abort.hpp" +#include "barretenberg/ecc/curves/bn254/bn254.hpp" #include "barretenberg/ecc/groups/wnaf.hpp" #include "barretenberg/numeric/bitop/get_msb.hpp" @@ -101,7 +103,7 @@ namespace bb::scalar_multiplication { * to use the curve endomorphism for faster scalar multiplication. See below for more details. */ template -void generate_pippenger_point_table(typename Curve::AffineElement* points, +void generate_pippenger_point_table(const typename Curve::AffineElement* points, typename Curve::AffineElement* table, size_t num_points) { @@ -127,7 +129,7 @@ void generate_pippenger_point_table(typename Curve::AffineElement* points, *points we're multiplying. Once we have the number of rounds, `m`, we need to split our scalar into `m` bit-slices. *Each pippenger round will work on one bit-slice. * - * Pippenger's algorithm works by, for each round, iterating over the points we're multplying. For each point, we + * Pippenger's algorithm works by, for each round, iterating over the points we're multiplying. For each point, we *examing the point's scalar multiplier and extract the bit-slice associated with the current pippenger round (we start *with the most significant slice). We then use the bit-slice to index a 'bucket', which we add the point into. For *example, if the bit slice is 01101, we add the corresponding point into bucket[13]. @@ -193,7 +195,7 @@ void generate_pippenger_point_table(typename Curve::AffineElement* points, * @param input_skew_table Pointer to the output array with all skews * @param round_counts The number of points in each round * @param scalars The pointer to the region with initial scalars that need to be converted into WNAF - * @param num_initial_points The number of points before the endomorphism split + * @param num_initial_points The number of points before the endomorphism split. A rounded up power of 2. **/ template void compute_wnaf_states(uint64_t* point_schedule, @@ -220,30 +222,46 @@ void compute_wnaf_states(uint64_t* point_schedule, } parallel_for(num_threads, [&](size_t i) { - Fr T0; uint64_t* wnaf_table = &point_schedule[(2 * i) * num_initial_points_per_thread]; - const Fr* thread_scalars = &scalars[i * num_initial_points_per_thread]; bool* skew_table = &input_skew_table[(2 * i) * num_initial_points_per_thread]; - uint64_t offset = i * num_points_per_thread; - - for (uint64_t j = 0; j < num_initial_points_per_thread; ++j) { - T0 = thread_scalars[j].from_montgomery_form(); - Fr::split_into_endomorphism_scalars(T0, T0, *(Fr*)&T0.data[2]); - - wnaf::fixed_wnaf_with_counts(&T0.data[0], - &wnaf_table[(j << 1UL)], - skew_table[j << 1ULL], + // Our offsets for this thread + const uint64_t point_offset = i * num_points_per_thread; + const size_t scalar_offset = i * num_initial_points_per_thread; + + // How many defined scalars are there? + const size_t defined_extent = std::min(scalar_offset + num_initial_points_per_thread, scalars.size()); + const size_t defined_scalars = defined_extent > scalar_offset ? defined_extent - scalar_offset : 0; + + auto wnaf_first_half = [&](const uint64_t* scalar, size_t j) { + wnaf::fixed_wnaf_with_counts(scalar, + &wnaf_table[j * 2], + skew_table[j * 2], &thread_round_counts[i][0], - ((j << 1ULL) + offset) << 32ULL, + (j * 2ULL + point_offset) << 32ULL, num_points, wnaf_bits); - wnaf::fixed_wnaf_with_counts(&T0.data[2], - &wnaf_table[(j << 1UL) + 1], - skew_table[(j << 1UL) + 1], + }; + auto wnaf_second_half = [&](const uint64_t* scalar, size_t j) { + wnaf::fixed_wnaf_with_counts(scalar, + &wnaf_table[j * 2 + 1], + skew_table[j * 2 + 1], &thread_round_counts[i][0], - ((j << 1UL) + offset + 1) << 32UL, + (j * 2ULL + point_offset + 1ULL) << 32ULL, num_points, wnaf_bits); + }; + for (size_t j = 0; j < defined_scalars; j++) { + Fr T0 = scalars[scalar_offset + j].from_montgomery_form(); + Fr::split_into_endomorphism_scalars(T0, T0, *(Fr*)&T0.data[2]); + + wnaf_first_half(&T0.data[0], j); + wnaf_second_half(&T0.data[2], j); + } + for (size_t j = defined_scalars; j < num_initial_points_per_thread; j++) { + // If we are trying to use a non-power-of-2 + static const uint64_t PADDING_ZEROES[] = { 0, 0 }; + wnaf_first_half(PADDING_ZEROES, j); + wnaf_second_half(PADDING_ZEROES, j); } }); @@ -740,7 +758,7 @@ uint32_t construct_addition_chains(affine_product_runtime_state& state, b template typename Curve::Element evaluate_pippenger_rounds(pippenger_runtime_state& state, - typename Curve::AffineElement* points, + const typename Curve::AffineElement* points, const size_t num_points, bool handle_edge_cases) { @@ -828,7 +846,7 @@ typename Curve::Element evaluate_pippenger_rounds(pippenger_runtime_state if (i == (num_rounds - 1)) { const size_t num_points_per_thread = num_points / num_threads; bool* skew_table = &state.skew_table[j * num_points_per_thread]; - AffineElement* point_table = &points[j * num_points_per_thread]; + const AffineElement* point_table = &points[j * num_points_per_thread]; AffineElement addition_temporary; for (size_t k = 0; k < num_points_per_thread; ++k) { if (skew_table[k]) { @@ -873,7 +891,6 @@ typename Curve::Element pippenger_internal(typename Curve::AffineElement* points template typename Curve::Element pippenger(std::span scalars, typename Curve::AffineElement* points, - const size_t num_initial_points, pippenger_runtime_state& state, bool handle_edge_cases) { @@ -886,7 +903,7 @@ typename Curve::Element pippenger(std::span s // For 8 threads, this neatly coincides with the threshold where Strauss scalar multiplication outperforms // Pippenger const size_t threshold = get_num_cpus_pow2() * 8; - + size_t num_initial_points = scalars.size(); if (num_initial_points == 0) { Element out = Group::one; out.self_set_infinity(); @@ -911,16 +928,39 @@ typename Curve::Element pippenger(std::span s Element result = pippenger_internal(points, scalars, num_slice_points, state, handle_edge_cases); if (num_slice_points != num_initial_points) { - const uint64_t leftover_points = num_initial_points - num_slice_points; return result + pippenger(scalars.subspan(num_slice_points), points + static_cast(num_slice_points * 2), - static_cast(leftover_points), state, handle_edge_cases); } return result; } +/* @brief Used for commits. +The main reason for this existing alone as this has one assumption: +The number of points is equal to or larger than #scalars rounded up to next power of 2. +Pippenger above can behavely poorly with numbers with many bits set.*/ + +template +typename Curve::Element pippenger_unsafe_optimized_for_non_dyadic_polys( + std::span scalars, + std::span points, + pippenger_runtime_state& state) +{ + BB_OP_COUNT_TIME(); + + // our windowed non-adjacent form algorthm requires that each thread can work on at least 8 points. + const size_t threshold = get_num_cpus_pow2() * 8; + // Delegate edge-cases to normal pippenger_unsafe(). + if (scalars.size() <= threshold) { + return pippenger_unsafe(scalars, &points[0], state); + } + // We need a padding of scalars. + ASSERT(numeric::round_up_power_2(scalars.size()) <= points.size()); + // We do not optimize for the small case at all. + return pippenger_internal(&points[0], scalars, numeric::round_up_power_2(scalars.size()), state, false); +} + /** * It's pippenger! But this one has go-faster stripes and a prediliction for questionable life choices. * We use affine-addition formula in this method, which paradoxically is ~45% faster than the mixed addition @@ -939,27 +979,25 @@ typename Curve::Element pippenger(std::span s template typename Curve::Element pippenger_unsafe(std::span scalars, typename Curve::AffineElement* points, - const size_t num_initial_points, pippenger_runtime_state& state) { - return pippenger(scalars, points, num_initial_points, state, false); + return pippenger(scalars, points, state, false); } template typename Curve::Element pippenger_without_endomorphism_basis_points( std::span scalars, typename Curve::AffineElement* points, - const size_t num_initial_points, pippenger_runtime_state& state) { - std::vector G_mod(num_initial_points * 2); - bb::scalar_multiplication::generate_pippenger_point_table(points, &G_mod[0], num_initial_points); - return pippenger(scalars, &G_mod[0], num_initial_points, state, false); + std::vector G_mod(scalars.size() * 2); + bb::scalar_multiplication::generate_pippenger_point_table(points, &G_mod[0], scalars.size()); + return pippenger(scalars, &G_mod[0], state, false); } // Explicit instantiation // BN254 -template void generate_pippenger_point_table(curve::BN254::AffineElement* points, +template void generate_pippenger_point_table(const curve::BN254::AffineElement* points, curve::BN254::AffineElement* table, size_t num_points); @@ -967,7 +1005,7 @@ template uint32_t construct_addition_chains(affine_product_runtime bool empty_bucket_counts = true); template void add_affine_points(curve::BN254::AffineElement* points, - const size_t num_points, + size_t num_points, curve::BN254::BaseField* scratch_space); template void add_affine_points_with_edge_cases(curve::BN254::AffineElement* points, @@ -984,7 +1022,7 @@ template curve::BN254::Element pippenger_internal(curve::BN254::Af bool handle_edge_cases); template curve::BN254::Element evaluate_pippenger_rounds(pippenger_runtime_state& state, - curve::BN254::AffineElement* points, + const curve::BN254::AffineElement* points, const size_t num_points, bool handle_edge_cases = false); @@ -994,23 +1032,25 @@ template curve::BN254::AffineElement* reduce_buckets(affine_produc template curve::BN254::Element pippenger(std::span scalars, curve::BN254::AffineElement* points, - const size_t num_points, pippenger_runtime_state& state, bool handle_edge_cases = true); template curve::BN254::Element pippenger_unsafe(std::span scalars, curve::BN254::AffineElement* points, - const size_t num_initial_points, pippenger_runtime_state& state); +template curve::BN254::Element pippenger_unsafe_optimized_for_non_dyadic_polys( + std::span scalars, + std::span points, + pippenger_runtime_state& state); + template curve::BN254::Element pippenger_without_endomorphism_basis_points( std::span scalars, curve::BN254::AffineElement* points, - const size_t num_initial_points, pippenger_runtime_state& state); // Grumpkin -template void generate_pippenger_point_table(curve::Grumpkin::AffineElement* points, +template void generate_pippenger_point_table(const curve::Grumpkin::AffineElement* points, curve::Grumpkin::AffineElement* table, size_t num_points); @@ -1037,7 +1077,7 @@ template curve::Grumpkin::Element pippenger_internal( template curve::Grumpkin::Element evaluate_pippenger_rounds( pippenger_runtime_state& state, - curve::Grumpkin::AffineElement* points, + const curve::Grumpkin::AffineElement* points, const size_t num_points, bool handle_edge_cases = false); @@ -1046,20 +1086,21 @@ template curve::Grumpkin::AffineElement* reduce_buckets( template curve::Grumpkin::Element pippenger(std::span scalars, curve::Grumpkin::AffineElement* points, - const size_t num_points, pippenger_runtime_state& state, bool handle_edge_cases = true); template curve::Grumpkin::Element pippenger_unsafe( std::span scalars, curve::Grumpkin::AffineElement* points, - const size_t num_initial_points, + pippenger_runtime_state& state); +template curve::Grumpkin::Element pippenger_unsafe_optimized_for_non_dyadic_polys( + std::span scalars, + std::span points, pippenger_runtime_state& state); template curve::Grumpkin::Element pippenger_without_endomorphism_basis_points( std::span scalars, curve::Grumpkin::AffineElement* points, - const size_t num_initial_points, pippenger_runtime_state& state); } // namespace bb::scalar_multiplication diff --git a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp index f178023b325..a4604f652ab 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp @@ -93,7 +93,7 @@ void compute_wnaf_states(uint64_t* point_schedule, size_t num_initial_points); template -void generate_pippenger_point_table(typename Curve::AffineElement* points, +void generate_pippenger_point_table(const typename Curve::AffineElement* points, typename Curve::AffineElement* table, size_t num_points); @@ -142,7 +142,7 @@ typename Curve::Element pippenger_internal(typename Curve::AffineElement* points template typename Curve::Element evaluate_pippenger_rounds(pippenger_runtime_state& state, - typename Curve::AffineElement* points, + const typename Curve::AffineElement* points, size_t num_points, bool handle_edge_cases = false); @@ -154,21 +154,26 @@ typename Curve::AffineElement* reduce_buckets(affine_product_runtime_state typename Curve::Element pippenger(std::span scalars, typename Curve::AffineElement* points, - size_t num_initial_points, pippenger_runtime_state& state, bool handle_edge_cases = true); template typename Curve::Element pippenger_unsafe(std::span scalars, typename Curve::AffineElement* points, - size_t num_initial_points, pippenger_runtime_state& state); template typename Curve::Element pippenger_without_endomorphism_basis_points( std::span scalars, typename Curve::AffineElement* points, - size_t num_initial_points, + pippenger_runtime_state& state); + +// NOTE: pippenger_unsafe_optimized_for_non_dyadic_polys requires SRS to have #scalars +// rounded up to nearest power of 2 or above points. +template +typename Curve::Element pippenger_unsafe_optimized_for_non_dyadic_polys( + std::span scalars, + std::span points, pippenger_runtime_state& state); // Explicit instantiation diff --git a/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp b/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp index 6ecd4a7b5f9..418e89d0de7 100644 --- a/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp @@ -130,7 +130,7 @@ template class ProvingKey_ { { if (commitment_key == nullptr) { ZoneScopedN("init commitment key"); - this->commitment_key = std::make_shared(circuit_size + 1); + this->commitment_key = std::make_shared(circuit_size); } else { // Don't create another commitment key if we already have one this->commitment_key = commitment_key; @@ -439,7 +439,7 @@ MegaRecursiveFlavor_, TranslatorRecursiveFlavor_, TranslatorRecursiveFlavor_, TranslatorRecursiveFlavor_, -ECCVMRecursiveFlavor_, +ECCVMRecursiveFlavor_, AvmRecursiveFlavor_>; template concept IsECCVMRecursiveFlavor = IsAnyOf>; diff --git a/barretenberg/cpp/src/barretenberg/numeric/bitop/get_msb.hpp b/barretenberg/cpp/src/barretenberg/numeric/bitop/get_msb.hpp index fc456a68a8e..3bf24ca9e70 100644 --- a/barretenberg/cpp/src/barretenberg/numeric/bitop/get_msb.hpp +++ b/barretenberg/cpp/src/barretenberg/numeric/bitop/get_msb.hpp @@ -43,4 +43,10 @@ template constexpr inline T get_msb(const T in) return (sizeof(T) <= 4) ? get_msb32(in) : get_msb64(in); } +template constexpr inline T round_up_power_2(const T in) +{ + auto lower_bound = T(1) << get_msb(in); + return (lower_bound == in || lower_bound == 1) ? in : lower_bound * 2; +} + } // namespace bb::numeric \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/verifier/verifier.cpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/verifier/verifier.cpp index dd8c4d2a6f3..cea9ea412d2 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/verifier/verifier.cpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/verifier/verifier.cpp @@ -182,8 +182,8 @@ template bool VerifierBase::verify g1::element P[2]; - P[0] = bb::scalar_multiplication::pippenger( - { &scalars[0], num_elements }, &elements[0], num_elements, state); + P[0] = + bb::scalar_multiplication::pippenger({ &scalars[0], /*size*/ num_elements }, &elements[0], state); P[1] = -(g1::element(PI_Z_OMEGA) * separator_challenge + PI_Z); if (key->contains_recursive_proof) { diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/verifier/verifier.test.cpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/verifier/verifier.test.cpp index 427108062ec..87c1037625e 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/verifier/verifier.test.cpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/verifier/verifier.test.cpp @@ -36,7 +36,6 @@ plonk::Verifier generate_verifier(std::shared_ptr circuit_proving_k commitments[i] = g1::affine_element(scalar_multiplication::pippenger( { poly_coefficients[i].get(), circuit_proving_key->circuit_size }, circuit_proving_key->reference_string->get_monomial_points(), - circuit_proving_key->circuit_size, state)); } diff --git a/barretenberg/cpp/src/barretenberg/plonk/work_queue/work_queue.cpp b/barretenberg/cpp/src/barretenberg/plonk/work_queue/work_queue.cpp index e6308a8343f..2b5eef768b5 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/work_queue/work_queue.cpp +++ b/barretenberg/cpp/src/barretenberg/plonk/work_queue/work_queue.cpp @@ -214,7 +214,7 @@ void work_queue::process_queue() // Run pippenger multi-scalar multiplication. auto runtime_state = bb::scalar_multiplication::pippenger_runtime_state(msm_size); bb::g1::affine_element result(bb::scalar_multiplication::pippenger_unsafe( - { item.mul_scalars.get(), msm_size }, srs_points, msm_size, runtime_state)); + { item.mul_scalars.get(), msm_size }, srs_points, runtime_state)); transcript->add_element(item.tag, result.to_buffer()); diff --git a/barretenberg/cpp/src/barretenberg/polynomials/legacy_polynomials.bench.cpp b/barretenberg/cpp/src/barretenberg/polynomials/legacy_polynomials.bench.cpp index 716c79d8600..abad771e626 100644 --- a/barretenberg/cpp/src/barretenberg/polynomials/legacy_polynomials.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/polynomials/legacy_polynomials.bench.cpp @@ -124,7 +124,7 @@ void pippenger_bench(State& state) noexcept state.ResumeTiming(); // uint64_t before = rdtsc(); scalar_multiplication::pippenger( - { &globals.scalars[0], /*size*/ num_points }, &globals.monomials[0], num_points, run_state); + { &globals.scalars[0], /*size*/ num_points }, &globals.monomials[0], run_state); // uint64_t after = rdtsc(); // count += (after - before); // ++i; @@ -169,23 +169,23 @@ void new_plonk_scalar_multiplications_bench(State& state) noexcept uint64_t before = rdtsc(); g1::element a = scalar_multiplication::pippenger( - { &globals.scalars[0], /*size*/ MAX_GATES }, &globals.monomials[0], MAX_GATES, run_state); + { &globals.scalars[0], /*size*/ MAX_GATES }, &globals.monomials[0], run_state); g1::element b = scalar_multiplication::pippenger( - { &globals.scalars[1], /*size*/ MAX_GATES }, &globals.monomials[0], MAX_GATES, run_state); + { &globals.scalars[1], /*size*/ MAX_GATES }, &globals.monomials[0], run_state); g1::element c = scalar_multiplication::pippenger( - { &globals.scalars[2], /*size*/ MAX_GATES }, &globals.monomials[0], MAX_GATES, run_state); + { &globals.scalars[2], /*size*/ MAX_GATES }, &globals.monomials[0], run_state); g1::element d = scalar_multiplication::pippenger( - { &globals.scalars[3], /*size*/ MAX_GATES }, &globals.monomials[0], MAX_GATES, run_state); + { &globals.scalars[3], /*size*/ MAX_GATES }, &globals.monomials[0], run_state); g1::element e = scalar_multiplication::pippenger( - { &globals.scalars[4], /*size*/ MAX_GATES }, &globals.monomials[0], MAX_GATES, run_state); + { &globals.scalars[4], /*size*/ MAX_GATES }, &globals.monomials[0], run_state); g1::element f = scalar_multiplication::pippenger( - { &globals.scalars[5], /*size*/ MAX_GATES }, &globals.monomials[0], MAX_GATES, run_state); + { &globals.scalars[5], /*size*/ MAX_GATES }, &globals.monomials[0], run_state); g1::element g = scalar_multiplication::pippenger( - { &globals.scalars[6], /*size*/ MAX_GATES }, &globals.monomials[0], MAX_GATES, run_state); + { &globals.scalars[6], /*size*/ MAX_GATES }, &globals.monomials[0], run_state); g1::element h = scalar_multiplication::pippenger( - { &globals.scalars[7], /*size*/ MAX_GATES }, &globals.monomials[0], MAX_GATES, run_state); + { &globals.scalars[7], /*size*/ MAX_GATES }, &globals.monomials[0], run_state); g1::element i = scalar_multiplication::pippenger( - { &globals.scalars[8], /*size*/ MAX_GATES }, &globals.monomials[0], MAX_GATES, run_state); + { &globals.scalars[8], /*size*/ MAX_GATES }, &globals.monomials[0], run_state); uint64_t after = rdtsc(); count += (after - before); ++k; diff --git a/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.cpp b/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.cpp index 823c3ebdc0c..bb4ddcf5f42 100644 --- a/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.cpp @@ -55,8 +55,7 @@ FileCrsFactory::FileCrsFactory(std::string path, size_t initial_degree) template std::shared_ptr> FileCrsFactory::get_prover_crs(size_t degree) { - if (degree != degree_ || !prover_crs_) { - ZoneScopedN("get_prover_crs"); + if (degree_ < degree || !prover_crs_) { prover_crs_ = std::make_shared>(degree, path_); degree_ = degree; } @@ -66,7 +65,7 @@ std::shared_ptr> FileCrsFactory::get template std::shared_ptr> FileCrsFactory::get_verifier_crs(size_t degree) { - if (degree != degree_ || !verifier_crs_) { + if (degree_ < degree || !verifier_crs_) { verifier_crs_ = std::make_shared>(path_, degree); degree_ = degree; } diff --git a/barretenberg/cpp/src/barretenberg/srs/factories/mem_grumpkin_crs_factory.cpp b/barretenberg/cpp/src/barretenberg/srs/factories/mem_grumpkin_crs_factory.cpp index bbec0dccf0d..8d0741cc920 100644 --- a/barretenberg/cpp/src/barretenberg/srs/factories/mem_grumpkin_crs_factory.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/factories/mem_grumpkin_crs_factory.cpp @@ -1,5 +1,6 @@ #include "mem_grumpkin_crs_factory.hpp" #include "../io.hpp" +#include "barretenberg/common/throw_or_abort.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "barretenberg/ecc/scalar_multiplication/point_table.hpp" #include "barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp" @@ -42,13 +43,19 @@ MemGrumpkinCrsFactory::MemGrumpkinCrsFactory(std::vector(points)) {} -std::shared_ptr> MemGrumpkinCrsFactory::get_prover_crs(size_t) +std::shared_ptr> MemGrumpkinCrsFactory::get_prover_crs(size_t degree) { + if (prover_crs_->get_monomial_size() < degree) { + throw_or_abort("prover trying to get too many points in MemGrumpkinCrsFactory!"); + } return prover_crs_; } -std::shared_ptr> MemGrumpkinCrsFactory::get_verifier_crs(size_t) +std::shared_ptr> MemGrumpkinCrsFactory::get_verifier_crs(size_t degree) { + if (prover_crs_->get_monomial_size() < degree) { + throw_or_abort("verifier trying to get too many points in MemGrumpkinCrsFactory!"); + } return verifier_crs_; } diff --git a/barretenberg/cpp/src/barretenberg/srs/io.hpp b/barretenberg/cpp/src/barretenberg/srs/io.hpp index 893f5bdd443..bd0b8a1444c 100644 --- a/barretenberg/cpp/src/barretenberg/srs/io.hpp +++ b/barretenberg/cpp/src/barretenberg/srs/io.hpp @@ -87,7 +87,8 @@ template class IO { file.read((char*)&manifest, sizeof(Manifest)); if (!file) { ptrdiff_t read = file.gcount(); - throw_or_abort(format("Only read ", read, " bytes from file but expected ", sizeof(Manifest), ".")); + throw_or_abort(format( + "Only read ", read, " bytes from manifest file ", filename, " but expected ", sizeof(Manifest), ".")); } file.close(); @@ -130,7 +131,8 @@ template class IO { file.read(buffer, (int)size); if (!file) { ptrdiff_t read = file.gcount(); - throw_or_abort(format("Only read ", read, " bytes from file but expected ", size, ".")); + throw_or_abort( + format("Only read ", read, " bytes from transcript file ", filename, " but expected ", size, ".")); } file.close(); diff --git a/barretenberg/cpp/src/barretenberg/srs/scalar_multiplication.test.cpp b/barretenberg/cpp/src/barretenberg/srs/scalar_multiplication.test.cpp index 660339f674e..247911339ae 100644 --- a/barretenberg/cpp/src/barretenberg/srs/scalar_multiplication.test.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/scalar_multiplication.test.cpp @@ -681,8 +681,7 @@ TYPED_TEST(ScalarMultiplicationTests, OversizedInputs) } scalar_multiplication::pippenger_runtime_state state(target_degree); - Element first = - scalar_multiplication::pippenger({ scalars, /*size*/ target_degree }, monomials, target_degree, state); + Element first = scalar_multiplication::pippenger({ scalars, /*size*/ target_degree }, monomials, state); first = first.normalize(); for (size_t i = 0; i < target_degree; ++i) { @@ -690,8 +689,7 @@ TYPED_TEST(ScalarMultiplicationTests, OversizedInputs) } scalar_multiplication::pippenger_runtime_state state_2(target_degree); - Element second = - scalar_multiplication::pippenger({ scalars, /*size*/ target_degree }, monomials, target_degree, state_2); + Element second = scalar_multiplication::pippenger({ scalars, /*size*/ target_degree }, monomials, state_2); second = second.normalize(); EXPECT_EQ((first.z == second.z), true); @@ -734,8 +732,7 @@ TYPED_TEST(ScalarMultiplicationTests, UndersizedInputs) scalar_multiplication::pippenger_runtime_state state(num_points); - Element result = - scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points, num_points, state); + Element result = scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points, state); result = result.normalize(); aligned_free(scalars); @@ -772,8 +769,7 @@ TYPED_TEST(ScalarMultiplicationTests, PippengerSmall) scalar_multiplication::generate_pippenger_point_table(points, points, num_points); scalar_multiplication::pippenger_runtime_state state(num_points); - Element result = - scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points, num_points, state); + Element result = scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points, state); result = result.normalize(); aligned_free(scalars); @@ -812,8 +808,7 @@ TYPED_TEST(ScalarMultiplicationTests, PippengerEdgeCaseDbl) } scalar_multiplication::generate_pippenger_point_table(points, points, num_points); scalar_multiplication::pippenger_runtime_state state(num_points); - Element result = - scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points, num_points, state); + Element result = scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points, state); result = result.normalize(); aligned_free(scalars); @@ -871,8 +866,7 @@ TYPED_TEST(ScalarMultiplicationTests, PippengerShortInputs) scalar_multiplication::generate_pippenger_point_table(points.get(), points.get(), num_points); scalar_multiplication::pippenger_runtime_state state(num_points); - Element result = - scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points.get(), num_points, state); + Element result = scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points.get(), state); result = result.normalize(); aligned_free(scalars); @@ -908,8 +902,8 @@ TYPED_TEST(ScalarMultiplicationTests, PippengerUnsafe) scalar_multiplication::generate_pippenger_point_table(points.get(), points.get(), num_points); scalar_multiplication::pippenger_runtime_state state(num_points); - Element result = scalar_multiplication::pippenger_unsafe( - { scalars, /*size*/ num_points }, points.get(), num_points, state); + Element result = + scalar_multiplication::pippenger_unsafe({ scalars, /*size*/ num_points }, points.get(), state); result = result.normalize(); aligned_free(scalars); @@ -966,8 +960,7 @@ TYPED_TEST(ScalarMultiplicationTests, PippengerUnsafeShortInputs) scalar_multiplication::generate_pippenger_point_table(points, points, num_points); scalar_multiplication::pippenger_runtime_state state(num_points); - Element result = - scalar_multiplication::pippenger_unsafe({ scalars, /*size*/ num_points }, points, num_points, state); + Element result = scalar_multiplication::pippenger_unsafe({ scalars, /*size*/ num_points }, points, state); result = result.normalize(); aligned_free(scalars); @@ -1004,8 +997,7 @@ TYPED_TEST(ScalarMultiplicationTests, PippengerOne) scalar_multiplication::generate_pippenger_point_table(points, points, num_points); scalar_multiplication::pippenger_runtime_state state(num_points); - Element result = - scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points, num_points, state); + Element result = scalar_multiplication::pippenger({ scalars, /*size*/ num_points }, points, state); result = result.normalize(); aligned_free(scalars); @@ -1026,7 +1018,7 @@ TYPED_TEST(ScalarMultiplicationTests, PippengerZeroPoints) AffineElement* points = (AffineElement*)aligned_alloc(32, sizeof(AffineElement) * (2 + 1)); scalar_multiplication::pippenger_runtime_state state(0); - Element result = scalar_multiplication::pippenger({ scalars, /*size*/ 0 }, points, 0, state); + Element result = scalar_multiplication::pippenger({ scalars, /*size*/ 0 }, points, state); aligned_free(scalars); aligned_free(points); @@ -1051,7 +1043,7 @@ TYPED_TEST(ScalarMultiplicationTests, PippengerMulByZero) scalar_multiplication::generate_pippenger_point_table(points, points, 1); scalar_multiplication::pippenger_runtime_state state(1); - Element result = scalar_multiplication::pippenger({ scalars, /*size*/ 1 }, points, 1, state); + Element result = scalar_multiplication::pippenger({ scalars, /*size*/ 1 }, points, state); aligned_free(scalars); aligned_free(points);