Skip to content

feat: shplemini claim batcher #11614

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ledwards2225 merged 11 commits into from
Jan 31, 2025
Merged

feat: shplemini claim batcher #11614

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
849c546
basic claim batcher concept
ledwards2225 Jan 29, 2025
5a61de9
move it up a level
ledwards2225 Jan 29, 2025
2ed762b
using the batch slcalar methods
ledwards2225 Jan 29, 2025
bda7005
Shplem interface now takes claim batcher
ledwards2225 Jan 29, 2025
da6c22d
Merge branch 'master' into lde/concatenate_4
ledwards2225 Jan 30, 2025
ad2bdfd
comments
ledwards2225 Jan 30, 2025
29aef8d
more comments
ledwards2225 Jan 30, 2025
f1b84f0
minor reworking plus comments
ledwards2225 Jan 30, 2025
71b4ed8
remove unneeded member
ledwards2225 Jan 30, 2025
02cdd6e
improving some naming in batcher
ledwards2225 Jan 31, 2025
df3cb4d
Merge branch 'master' into lde/concatenate_4
ledwards2225 Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 10 additions & 19 deletions barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ class IPATest : public CommitmentTest<Curve> {
using GeminiProver = GeminiProver_<Curve>;
using GeminiVerifier = GeminiVerifier_<Curve>;
using ShpleminiVerifier = ShpleminiVerifier_<Curve>;
using ClaimBatcher = ShpleminiVerifier::ClaimBatcher;
using ClaimBatch = ShpleminiVerifier::ClaimBatch;

static std::shared_ptr<CK> ck;
static std::shared_ptr<VK> vk;
Expand Down Expand Up @@ -296,15 +298,8 @@ TEST_F(IPATest, ShpleminiIPAWithShift)

auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);

const auto batch_opening_claim =
ShpleminiVerifier::compute_batch_opening_claim(small_n,
RefVector(instance_witness.unshifted_commitments),
RefVector(instance_witness.to_be_shifted_commitments),
RefVector(instance_witness.unshifted_evals),
RefVector(instance_witness.shifted_evals),
mle_opening_point,
vk->get_g1_identity(),
verifier_transcript);
const auto batch_opening_claim = ShpleminiVerifier::compute_batch_opening_claim(
small_n, instance_witness.claim_batcher, mle_opening_point, vk->get_g1_identity(), verifier_transcript);

auto result = PCS::reduce_verify_batch_opening_claim(batch_opening_claim, vk, verifier_transcript);
// auto result = PCS::reduce_verify(vk, shplonk_verifier_claim, verifier_transcript);
Expand Down Expand Up @@ -350,16 +345,12 @@ TEST_F(IPATest, ShpleminiIPAShiftsRemoval)
// vectors corresponding to the "shifted" commitment
auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);

const auto batch_opening_claim =
ShpleminiVerifier::compute_batch_opening_claim(small_n,
RefVector(instance_witness.unshifted_commitments),
RefVector(instance_witness.to_be_shifted_commitments),
RefVector(instance_witness.unshifted_evals),
RefVector(instance_witness.shifted_evals),
mle_opening_point,
vk->get_g1_identity(),
verifier_transcript,
repeated_commitments);
const auto batch_opening_claim = ShpleminiVerifier::compute_batch_opening_claim(small_n,
instance_witness.claim_batcher,
mle_opening_point,
vk->get_g1_identity(),
verifier_transcript,
repeated_commitments);

auto result = PCS::reduce_verify_batch_opening_claim(batch_opening_claim, vk, verifier_transcript);
EXPECT_EQ(result, true);
Expand Down
33 changes: 10 additions & 23 deletions barretenberg/cpp/src/barretenberg/commitment_schemes/kzg/kzg.test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,15 +186,9 @@ TEST_F(KZGTest, ShpleminiKzgWithShift)

// Gemini verifier output:
// - claim: d+1 commitments to Fold_{r}^(0), Fold_{-r}^(0), Fold^(l), d+1 evaluations a_0_pos, a_l, l = 0:d-1
const auto batch_opening_claim =
ShpleminiVerifier::compute_batch_opening_claim(n,
RefVector(instance_witness.unshifted_commitments),
RefVector(instance_witness.to_be_shifted_commitments),
RefVector(instance_witness.unshifted_evals),
RefVector(instance_witness.shifted_evals),
mle_opening_point,
vk->get_g1_identity(),
verifier_transcript);
const auto batch_opening_claim = ShpleminiVerifier::compute_batch_opening_claim(
n, instance_witness.claim_batcher, mle_opening_point, vk->get_g1_identity(), verifier_transcript);

const auto pairing_points = PCS::reduce_verify_batch_opening_claim(batch_opening_claim, verifier_transcript);
// Final pairing check: e([Q] - [Q_z] + z[W], [1]_2) = e([W], [x]_2)

Expand Down Expand Up @@ -243,10 +237,7 @@ TEST_F(KZGTest, ShpleminiKzgWithShiftAndConcatenation)
// - claim: d+1 commitments to Fold_{r}^(0), Fold_{-r}^(0), Fold^(l), d+1 evaluations a_0_pos, a_l, l = 0:d-1
const auto batch_opening_claim =
ShpleminiVerifier::compute_batch_opening_claim(n,
RefVector(instance_witness.unshifted_commitments),
RefVector(instance_witness.to_be_shifted_commitments),
RefVector(instance_witness.unshifted_evals),
RefVector(instance_witness.shifted_evals),
instance_witness.claim_batcher,
mle_opening_point,
vk->get_g1_identity(),
verifier_transcript,
Expand Down Expand Up @@ -309,16 +300,12 @@ TEST_F(KZGTest, ShpleminiKzgShiftsRemoval)

// Gemini verifier output:
// - claim: d+1 commitments to Fold_{r}^(0), Fold_{-r}^(0), Fold^(l), d+1 evaluations a_0_pos, a_l, l = 0:d-1
const auto batch_opening_claim =
ShpleminiVerifier::compute_batch_opening_claim(n,
RefVector(instance_witness.unshifted_commitments),
RefVector(instance_witness.to_be_shifted_commitments),
RefVector(instance_witness.unshifted_evals),
RefVector(instance_witness.shifted_evals),
mle_opening_point,
vk->get_g1_identity(),
verifier_transcript,
repeated_commitments);
const auto batch_opening_claim = ShpleminiVerifier::compute_batch_opening_claim(n,
instance_witness.claim_batcher,
mle_opening_point,
vk->get_g1_identity(),
verifier_transcript,
repeated_commitments);

const auto pairing_points = PCS::reduce_verify_batch_opening_claim(batch_opening_claim, verifier_transcript);

Expand Down
165 changes: 115 additions & 50 deletions barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,13 +196,112 @@ template <typename Curve> class ShpleminiVerifier_ {
using GeminiVerifier = GeminiVerifier_<Curve>;

public:
struct ClaimBatch {
RefVector<Commitment> commitments;
RefVector<Fr> evaluations;
// scalar used for batching the claims, excluding the power of batching challenge \rho

@maramihali maramihali Jan 31, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i wonder whether we can somehow link the corresponding power of rho in here. I found myself several times checking whether the incorrect power could be the bug

@ledwards2225 ledwards2225 Jan 31, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would be nice I agree. I'll explore that when I'm adding the k-shifts here

Fr scalar = 0;
};

/**
* @brief Logic to support batching opening claims for unshifted and shifted polynomials in Shplemini
* @details Stores references to the commitments/evaluations of unshifted and shifted polynomials to be batched
* opened via Shplemini. Aggregates the commitments and batching scalars for each batch into the corresponding
* containers for Shplemini. Computes the batched evaluation. Contains logic for computing the per-batch scalars
* used to batch each set of claims (see details below).
* @note This class performs the actual batching of the evaluations but not of the commitments. The latter are
* simply appended to a larger container, along with the scalars used to batch them. This is because Shplemini
* is optimized to perform a single batch mul that includes all commitments from each stage of the PCS. See
* description of ShpleminiVerifier for more details.
*
*/
struct ClaimBatcher {
std::optional<ClaimBatch> unshifted; // commitments and evaluations of unshifted polynomials

@ledwards2225 ledwards2225 Jan 30, 2025 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefer this std::optional container and if (container) to the if (container.size() > 0) syntax I used in the PolynomialBatcher. I think it makes the intent a bit more clear. (Or course right now all variants use both unshifted and shifted but eventually this will also contain k-shifts and concatenations which will only be used in certain protocols). If this stays as it is I'll plan to make that class match this one in that sense.

@maramihali maramihali Jan 31, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I somehow prefer the has_unshifted() / has_shifted() pattern but no strong feelings. I'm worried this std::optional will get misused perhaps and we'll end up with the situation we had in sumcheck with the optional verified boolean.

@ledwards2225 ledwards2225 Jan 31, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hated that too but after some thorough googling I'm convinced that this is a very idomatic usage of optionals. I think the problem with the sumcheck use case was (1) it was a bad reason (just bc the prover and verifier shared a struct I think?) and (2) the external user had to know about the presence of optionals. Here its all encapsulated in the class and no one else has to know

std::optional<ClaimBatch> shifted; // commitments of to-be-shifted-by-1 polys, evals of their shifts

Fr get_unshifted_batch_scalar() const { return unshifted ? unshifted->scalar : Fr{ 0 }; }

/**
* @brief Compute scalars used to batch each set of claims, excluding contribution from batching challenge \rho
* @details Computes scalars s_0 and s_1 given by
* \f[
* - s_0 = \left(\frac{1}{z-r} + \nu \times \frac{1}{z+r}\right) \f],
* - s_1 = \left(\frac{1}{z-r} - \nu \times \frac{1}{z+r}\right)
* \f]
* where the scalars used to batch the claims are given by
* \f[
* \left(
* - s_0,
* \ldots,
* - \rho^{i+k-1} \times s_0,
* - \rho^{i+k} \times \frac{1}{r} \times s_1,
* \ldots,
* - \rho^{k+m-1} \times \frac{1}{r} \times s_1
* \right)
* \f]
*
* @param inverse_vanishing_eval_pos 1/(z-r)
* @param inverse_vanishing_eval_neg 1/(z+r)
* @param nu_challenge ν (shplonk batching challenge)
* @param r_challenge r (gemini evaluation challenge)
*/
void compute_scalars_for_each_batch(const Fr& inverse_vanishing_eval_pos,
const Fr& inverse_vanishing_eval_neg,
const Fr& nu_challenge,
const Fr& r_challenge)
{
if (unshifted) {
// (1/(z−r) + ν/(z+r))
unshifted->scalar = inverse_vanishing_eval_pos + nu_challenge * inverse_vanishing_eval_neg;
}
if (shifted) {
// r⁻¹ ⋅ (1/(z−r) − ν/(z+r))
shifted->scalar =
r_challenge.invert() * (inverse_vanishing_eval_pos - nu_challenge * inverse_vanishing_eval_neg);
}
}

/**
* @brief Append the commitments and scalars from each batch of claims to the Shplemini batch mul input vectors;
* update the batched evaluation and the running batching challenge (power of rho) in place.
*
* @param commitments commitment inputs to the single Shplemini batch mul
* @param scalars scalar inputs to the single Shplemini batch mul
* @param batched_evaluation running batched evaluation of the committed multilinear polynomials
* @param rho multivariate batching challenge \rho
* @param rho_power current power of \rho used in the batching scalar
*/
void update_batch_mul_inputs_and_batched_evaluation(std::vector<Commitment>& commitments,
std::vector<Fr>& scalars,
Fr& batched_evaluation,
const Fr& rho,
Fr& rho_power)
{
// Append the commitments/scalars from a given batch to the corresponding containers; update the batched
// evaluation and the running batching challenge in place
auto aggregate_claim_data_and_update_batched_evaluation = [&](const ClaimBatch& batch, Fr& rho_power) {
for (auto [commitment, evaluation] : zip_view(batch.commitments, batch.evaluations)) {
commitments.emplace_back(std::move(commitment));
scalars.emplace_back(-batch.scalar * rho_power);
batched_evaluation += evaluation * rho_power;
rho_power *= rho;
}
};

// Incorporate the claim data from each batch of claims that is present
if (unshifted) {
aggregate_claim_data_and_update_batched_evaluation(*unshifted, rho_power);
}
if (shifted) {
aggregate_claim_data_and_update_batched_evaluation(*shifted, rho_power);
}
}
};

template <typename Transcript>
static BatchOpeningClaim<Curve> compute_batch_opening_claim(
const Fr N,
RefSpan<Commitment> unshifted_commitments,
RefSpan<Commitment> shifted_commitments,
RefSpan<Fr> unshifted_evaluations,
RefSpan<Fr> shifted_evaluations,
ClaimBatcher& claim_batcher,

@maramihali maramihali Jan 31, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is great :)

const std::vector<Fr>& multivariate_challenge,
const Commitment& g1_identity,
const std::shared_ptr<Transcript>& transcript,
Expand Down Expand Up @@ -291,16 +390,11 @@ template <typename Curve> class ShpleminiVerifier_ {
log_circuit_size + 1, shplonk_evaluation_challenge, gemini_eval_challenge_powers);

// Compute the additional factors to be multiplied with unshifted and shifted commitments when lazily
// reconstructing thec commitment of Q_z

// i-th unshifted commitment is multiplied by −ρⁱ and the unshifted_scalar ( 1/(z−r) + ν/(z+r) )
const Fr unshifted_scalar =
inverse_vanishing_evals[0] + shplonk_batching_challenge * inverse_vanishing_evals[1];

// j-th shifted commitment is multiplied by −ρᵏ⁺ʲ⁻¹ and the shifted_scalar r⁻¹ ⋅ (1/(z−r) − ν/(z+r))
const Fr shifted_scalar =
gemini_evaluation_challenge.invert() *
(inverse_vanishing_evals[0] - shplonk_batching_challenge * inverse_vanishing_evals[1]);
// reconstructing the commitment of Q_z
claim_batcher.compute_scalars_for_each_batch(inverse_vanishing_evals[0],
inverse_vanishing_evals[1],
shplonk_batching_challenge,
gemini_evaluation_challenge);

std::vector<Fr> concatenation_scalars;
if (!concatenation_group_commitments.empty()) {
Expand All @@ -326,18 +420,13 @@ template <typename Curve> class ShpleminiVerifier_ {

if (has_zk) {
commitments.emplace_back(hiding_polynomial_commitment);
scalars.emplace_back(-unshifted_scalar); // corresponds to ρ⁰
scalars.emplace_back(-claim_batcher.get_unshifted_batch_scalar()); // corresponds to ρ⁰

@maramihali maramihali Jan 31, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

wonder whether negation here can be somehow encompassed in the functionality of claim_batcher

@ledwards2225 ledwards2225 Jan 31, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll think about this when I think about the inclusion of rho / rho power in the batcher

}

// Place the commitments to prover polynomials in the commitments vector. Compute the evaluation of the
// batched multilinear polynomial. Populate the vector of scalars for the final batch mul
batch_multivariate_opening_claims(unshifted_commitments,
shifted_commitments,
unshifted_evaluations,
shifted_evaluations,
batch_multivariate_opening_claims(claim_batcher,
multivariate_batching_challenge,
unshifted_scalar,
shifted_scalar,
commitments,
scalars,
batched_evaluation,
Expand Down Expand Up @@ -459,13 +548,8 @@ template <typename Curve> class ShpleminiVerifier_ {
* @param concatenated_evaluations Evaluations of the full concatenated polynomials.
*/
static void batch_multivariate_opening_claims(
RefSpan<Commitment> unshifted_commitments,
RefSpan<Commitment> shifted_commitments,
RefSpan<Fr> unshifted_evaluations,
RefSpan<Fr> shifted_evaluations,
ClaimBatcher& claim_batcher,
const Fr& multivariate_batching_challenge,
const Fr& unshifted_scalar,
const Fr& shifted_scalar,
std::vector<Commitment>& commitments,
std::vector<Fr>& scalars,
Fr& batched_evaluation,
Expand All @@ -481,30 +565,11 @@ template <typename Curve> class ShpleminiVerifier_ {
current_batching_challenge *= multivariate_batching_challenge;
}

for (auto [unshifted_commitment, unshifted_evaluation] :
zip_view(unshifted_commitments, unshifted_evaluations)) {
// Move unshifted commitments to the 'commitments' vector
commitments.emplace_back(std::move(unshifted_commitment));
// Compute −ρⁱ ⋅ (1/(z−r) + ν/(z+r)) and place into 'scalars'
scalars.emplace_back(-unshifted_scalar * current_batching_challenge);
// Accumulate the evaluation of ∑ ρⁱ ⋅ fᵢ at the sumcheck challenge
batched_evaluation += unshifted_evaluation * current_batching_challenge;
// Update the batching challenge
current_batching_challenge *= multivariate_batching_challenge;
}
for (auto [shifted_commitment, shifted_evaluation] : zip_view(shifted_commitments, shifted_evaluations)) {
// Move shifted commitments to the 'commitments' vector
commitments.emplace_back(std::move(shifted_commitment));
// Compute −ρ⁽ᵏ⁺ʲ⁾ ⋅ r⁻¹ ⋅ (1/(z−r) − ν/(z+r)) and place into 'scalars'
scalars.emplace_back(-shifted_scalar * current_batching_challenge);
// Accumulate the evaluation of ∑ ρ⁽ᵏ⁺ʲ⁾ ⋅ f_shift at the sumcheck challenge
batched_evaluation += shifted_evaluation * current_batching_challenge;
// Update the batching challenge ρ
current_batching_challenge *= multivariate_batching_challenge;
}
claim_batcher.update_batch_mul_inputs_and_batched_evaluation(
commitments, scalars, batched_evaluation, multivariate_batching_challenge, current_batching_challenge);

// If we are performing an opening verification for the translator, add the contributions from the concatenation
// commitments and evaluations to the result
// If we are performing an opening verification for the translator, add the contributions from the
// concatenation commitments and evaluations to the result
ASSERT(concatenated_evaluations.size() == concatenation_group_commitments.size());
if (!concatenation_group_commitments.empty()) {
size_t concatenation_group_size = concatenation_group_commitments[0].size();
Expand Down
Loading