diff --git a/tfhe/benches/core_crypto/pbs_bench.rs b/tfhe/benches/core_crypto/pbs_bench.rs index 3ed40e77c0..70aae52d18 100644 --- a/tfhe/benches/core_crypto/pbs_bench.rs +++ b/tfhe/benches/core_crypto/pbs_bench.rs @@ -13,27 +13,28 @@ use tfhe::core_crypto::prelude::*; use tfhe::keycache::NamedParam; use tfhe::shortint::parameters::*; -const SHORTINT_BENCH_PARAMS: [ClassicPBSParameters; 19] = [ - PARAM_MESSAGE_1_CARRY_0_KS_PBS, - PARAM_MESSAGE_1_CARRY_1_KS_PBS, - PARAM_MESSAGE_2_CARRY_0_KS_PBS, - PARAM_MESSAGE_2_CARRY_1_KS_PBS, - PARAM_MESSAGE_2_CARRY_2_KS_PBS, - PARAM_MESSAGE_3_CARRY_0_KS_PBS, - PARAM_MESSAGE_3_CARRY_2_KS_PBS, - PARAM_MESSAGE_3_CARRY_3_KS_PBS, - PARAM_MESSAGE_4_CARRY_0_KS_PBS, - PARAM_MESSAGE_4_CARRY_3_KS_PBS, - PARAM_MESSAGE_4_CARRY_4_KS_PBS, - PARAM_MESSAGE_5_CARRY_0_KS_PBS, - PARAM_MESSAGE_6_CARRY_0_KS_PBS, - PARAM_MESSAGE_7_CARRY_0_KS_PBS, - PARAM_MESSAGE_8_CARRY_0_KS_PBS, - PARAM_MESSAGE_1_CARRY_1_PBS_KS, - PARAM_MESSAGE_2_CARRY_2_PBS_KS, - PARAM_MESSAGE_3_CARRY_3_PBS_KS, - PARAM_MESSAGE_4_CARRY_4_PBS_KS, -]; +// const SHORTINT_BENCH_PARAMS: [ClassicPBSParameters; 19] = [ +// PARAM_MESSAGE_1_CARRY_0_KS_PBS, +// PARAM_MESSAGE_1_CARRY_1_KS_PBS, +// PARAM_MESSAGE_2_CARRY_0_KS_PBS, +// PARAM_MESSAGE_2_CARRY_1_KS_PBS, +// PARAM_MESSAGE_2_CARRY_2_KS_PBS, +// PARAM_MESSAGE_3_CARRY_0_KS_PBS, +// PARAM_MESSAGE_3_CARRY_2_KS_PBS, +// PARAM_MESSAGE_3_CARRY_3_KS_PBS, +// PARAM_MESSAGE_4_CARRY_0_KS_PBS, +// PARAM_MESSAGE_4_CARRY_3_KS_PBS, +// PARAM_MESSAGE_4_CARRY_4_KS_PBS, +// PARAM_MESSAGE_5_CARRY_0_KS_PBS, +// PARAM_MESSAGE_6_CARRY_0_KS_PBS, +// PARAM_MESSAGE_7_CARRY_0_KS_PBS, +// PARAM_MESSAGE_8_CARRY_0_KS_PBS, +// PARAM_MESSAGE_1_CARRY_1_PBS_KS, +// PARAM_MESSAGE_2_CARRY_2_PBS_KS, +// PARAM_MESSAGE_3_CARRY_3_PBS_KS, +// PARAM_MESSAGE_4_CARRY_4_PBS_KS, +// ]; +const SHORTINT_BENCH_PARAMS: [ClassicPBSParameters; 1] = [PARAM_MESSAGE_2_CARRY_2_KS_PBS]; const BOOLEAN_BENCH_PARAMS: [(&str, BooleanParameters); 2] = [ ("BOOLEAN_DEFAULT_PARAMS", DEFAULT_PARAMETERS), @@ -216,10 +217,133 @@ fn mem_optimized_pbs + Serialize>( { bench_group.bench_function(&id, |b| { b.iter(|| { - programmable_bootstrap_lwe_ciphertext_mem_optimized( + for _ in 0..10 { + programmable_bootstrap_lwe_ciphertext_mem_optimized( + &lwe_ciphertext_in, + &mut out_pbs_ct, + &accumulator.as_view(), + &fourier_bsk, + fft, + buffers.stack(), + ); + black_box(&mut out_pbs_ct); + } + }) + }); + } + + let bit_size = (params.message_modulus.unwrap_or(2) as u32).ilog2(); + write_to_json( + &id, + *params, + name, + "pbs", + &OperatorType::Atomic, + bit_size, + vec![bit_size], + ); + } +} + +fn mem_optimized_batched_pbs + Serialize>( + c: &mut Criterion, + parameters: &[(String, CryptoParametersRecord)], +) { + let bench_name = "core_crypto::batched_pbs_mem_optimized"; + let mut bench_group = c.benchmark_group(bench_name); + bench_group + .sample_size(15) + .measurement_time(std::time::Duration::from_secs(10)); + + // Create the PRNG + let mut seeder = new_seeder(); + let seeder = seeder.as_mut(); + let mut encryption_generator = + EncryptionRandomGenerator::::new(seeder.seed(), seeder); + let mut secret_generator = + SecretRandomGenerator::::new(seeder.seed()); + + for (name, params) in parameters.iter() { + // Create the LweSecretKey + let input_lwe_secret_key = allocate_and_generate_new_binary_lwe_secret_key( + params.lwe_dimension.unwrap(), + &mut secret_generator, + ); + let output_glwe_secret_key: GlweSecretKeyOwned = + allocate_and_generate_new_binary_glwe_secret_key( + params.glwe_dimension.unwrap(), + params.polynomial_size.unwrap(), + &mut secret_generator, + ); + let output_lwe_secret_key = output_glwe_secret_key.into_lwe_secret_key(); + + // Create the empty bootstrapping key in the Fourier domain + let fourier_bsk = FourierLweBootstrapKey::new( + params.lwe_dimension.unwrap(), + params.glwe_dimension.unwrap().to_glwe_size(), + params.polynomial_size.unwrap(), + params.pbs_base_log.unwrap(), + params.pbs_level.unwrap(), + ); + + let count = 10; // FIXME Is it a representative value (big enough?) + + // Allocate a new LweCiphertext and encrypt our plaintext + let mut lwe_ciphertext_in = LweCiphertextListOwned::::new( + Scalar::ZERO, + input_lwe_secret_key.lwe_dimension().to_lwe_size(), + LweCiphertextCount(count), + params.ciphertext_modulus.unwrap(), + ); + + encrypt_lwe_ciphertext_list( + &input_lwe_secret_key, + &mut lwe_ciphertext_in, + &PlaintextList::from_container(vec![Scalar::ZERO; count]), + params.lwe_noise_distribution.unwrap(), + &mut encryption_generator, + ); + + let accumulator = GlweCiphertextList::new( + Scalar::ZERO, + params.glwe_dimension.unwrap().to_glwe_size(), + params.polynomial_size.unwrap(), + GlweCiphertextCount(count), + params.ciphertext_modulus.unwrap(), + ); + + // Allocate the LweCiphertext to store the result of the PBS + let mut out_pbs_ct = LweCiphertextList::new( + Scalar::ZERO, + output_lwe_secret_key.lwe_dimension().to_lwe_size(), + LweCiphertextCount(count), + params.ciphertext_modulus.unwrap(), + ); + + let mut buffers = ComputationBuffers::new(); + + let fft = Fft::new(fourier_bsk.polynomial_size()); + let fft = fft.as_view(); + + buffers.resize( + batch_programmable_bootstrap_lwe_ciphertext_mem_optimized_requirement::( + fourier_bsk.glwe_size(), + fourier_bsk.polynomial_size(), + fft, + ) + .unwrap() + .unaligned_bytes_required() + * count, + ); + + let id = format!("{bench_name}::{name}"); + { + bench_group.bench_function(&id, |b| { + b.iter(|| { + batched_programmable_bootstrap_lwe_ciphertext_mem_optimized( &lwe_ciphertext_in, &mut out_pbs_ct, - &accumulator.as_view(), + &accumulator, &fourier_bsk, fft, buffers.stack(), @@ -1310,6 +1434,7 @@ pub fn pbs_group() { mem_optimized_pbs(&mut criterion, &benchmark_parameters_64bits()); mem_optimized_pbs(&mut criterion, &benchmark_parameters_32bits()); mem_optimized_pbs_ntt(&mut criterion); + mem_optimized_batched_pbs(&mut criterion, &benchmark_parameters_64bits()); } pub fn multi_bit_pbs_group() { diff --git a/tfhe/src/core_crypto/algorithms/lwe_programmable_bootstrapping/fft64.rs b/tfhe/src/core_crypto/algorithms/lwe_programmable_bootstrapping/fft64.rs index da5f449f42..c2179018f1 100644 --- a/tfhe/src/core_crypto/algorithms/lwe_programmable_bootstrapping/fft64.rs +++ b/tfhe/src/core_crypto/algorithms/lwe_programmable_bootstrapping/fft64.rs @@ -1071,3 +1071,77 @@ pub fn programmable_bootstrap_lwe_ciphertext_mem_optimized_requirement Result { bootstrap_scratch::(glwe_size, polynomial_size, fft) } + +/// Memory optimized version of [`batch_programmable_bootstrap_lwe_ciphertext`], the caller must provide +/// a properly configured [`FftView`] object and a `PodStack` used as a memory buffer having a +/// capacity at least as large as the result of +/// [`crate::core_crypto::algorithms::batch_programmable_bootstrap_lwe_ciphertext_mem_optimized_requirement`]. +pub fn batch_programmable_bootstrap_lwe_ciphertext_mem_optimized< + InputScalar, + OutputScalar, + InputCont, + OutputCont, + AccCont, + KeyCont, +>( + input: &LweCiphertextList, + output: &mut LweCiphertextList, + accumulator: &GlweCiphertextList, + fourier_bsk: &FourierLweBootstrapKey, + fft: FftView<'_>, + stack: PodStack<'_>, +) where + // CastInto required for PBS modulus switch which returns a usize + InputScalar: UnsignedTorus + CastInto, + OutputScalar: UnsignedTorus, + InputCont: Container, + OutputCont: ContainerMut, + AccCont: Container, + KeyCont: Container, +{ + assert_eq!( + accumulator.ciphertext_modulus(), + output.ciphertext_modulus(), + "Mismatched moduli between accumulator ({:?}) and output ({:?})", + accumulator.ciphertext_modulus(), + output.ciphertext_modulus() + ); + + assert_eq!( + fourier_bsk.input_lwe_dimension(), + input.lwe_size().to_lwe_dimension(), + "Mismatched input LweDimension. \ + FourierLweBootstrapKey input LweDimension: {:?}, input LweCiphertext LweDimension {:?}.", + fourier_bsk.input_lwe_dimension(), + input.lwe_size().to_lwe_dimension(), + ); + assert_eq!( + fourier_bsk.output_lwe_dimension(), + output.lwe_size().to_lwe_dimension(), + "Mismatched output LweDimension. \ + FourierLweBootstrapKey input LweDimension: {:?}, input LweCiphertext LweDimension {:?}.", + fourier_bsk.output_lwe_dimension(), + output.lwe_size().to_lwe_dimension(), + ); + + fourier_bsk.as_view().batch_bootstrap( + output.as_mut_view(), + input.as_view(), + &accumulator.as_view(), + fft, + stack, + ); +} + +/// Return the required memory for [`batch_programmable_bootstrap_lwe_ciphertext_mem_optimized`]. +pub fn batch_programmable_bootstrap_lwe_ciphertext_mem_optimized_requirement( + glwe_size: GlweSize, + polynomial_size: PolynomialSize, + fft: FftView<'_>, +) -> Result { + programmable_bootstrap_lwe_ciphertext_mem_optimized_requirement::( + glwe_size, + polynomial_size, + fft, + ) +} diff --git a/tfhe/src/core_crypto/algorithms/test/lwe_programmable_bootstrapping.rs b/tfhe/src/core_crypto/algorithms/test/lwe_programmable_bootstrapping.rs index c26e54b18d..95df74977f 100644 --- a/tfhe/src/core_crypto/algorithms/test/lwe_programmable_bootstrapping.rs +++ b/tfhe/src/core_crypto/algorithms/test/lwe_programmable_bootstrapping.rs @@ -164,6 +164,149 @@ where create_parametrized_test!(lwe_encrypt_pbs_decrypt_custom_mod); +fn lwe_encrypt_batch_pbs_decrypt_custom_mod(params: ClassicTestParams) +where + Scalar: UnsignedTorus + + Sync + + Send + + CastFrom + + CastInto + + Serialize + + DeserializeOwned, + ClassicTestParams: KeyCacheAccess>, +{ + let lwe_noise_distribution = params.lwe_noise_distribution; + let ciphertext_modulus = params.ciphertext_modulus; + let message_modulus_log = params.message_modulus_log; + let msg_modulus = Scalar::ONE.shl(message_modulus_log.0); + let encoding_with_padding = get_encoding_with_padding(ciphertext_modulus); + let glwe_dimension = params.glwe_dimension; + let polynomial_size = params.polynomial_size; + + let ciphertext_count = 2; + + let mut rsc = TestResources::new(); + + let f = |x: Scalar| x; + + let delta: Scalar = encoding_with_padding / msg_modulus; + let mut msg = msg_modulus; + + let accumulator = generate_programmable_bootstrap_glwe_lut( + polynomial_size, + glwe_dimension.to_glwe_size(), + msg_modulus.cast_into(), + ciphertext_modulus, + delta, + f, + ); + + assert!(check_encrypted_content_respects_mod( + &accumulator, + ciphertext_modulus + )); + + while msg != Scalar::ZERO { + msg = msg.wrapping_sub(Scalar::ONE); + + let mut keys_gen = |params| generate_keys(params, &mut rsc); + let keys = gen_keys_or_get_from_cache_if_enabled(params, &mut keys_gen); + let (input_lwe_secret_key, output_lwe_secret_key, fbsk) = + (keys.small_lwe_sk, keys.big_lwe_sk, keys.fbsk); + + for _ in 0..NB_TESTS { + let plaintext = msg * delta; + + let mut lwe_ciphertext_in = LweCiphertextListOwned::::new( + Scalar::ZERO, + input_lwe_secret_key.lwe_dimension().to_lwe_size(), + LweCiphertextCount(ciphertext_count), + ciphertext_modulus, + ); + + encrypt_lwe_ciphertext_list( + &input_lwe_secret_key, + &mut lwe_ciphertext_in, + &PlaintextList::from_container(vec![plaintext; ciphertext_count]), + lwe_noise_distribution, + &mut rsc.encryption_random_generator, + ); + + assert!(lwe_ciphertext_in + .iter() + .all(|ct| check_encrypted_content_respects_mod(&ct, ciphertext_modulus))); + + let mut accumulator_list = GlweCiphertextList::new( + Scalar::ZERO, + glwe_dimension.to_glwe_size(), + polynomial_size, + GlweCiphertextCount(ciphertext_count), + ciphertext_modulus, + ); + + for mut glwe in accumulator_list.iter_mut() { + glwe.as_mut().copy_from_slice(accumulator.as_ref()); + } + + // Allocate the LweCiphertext to store the result of the PBS + let mut out_pbs_ct = LweCiphertextList::new( + Scalar::ZERO, + output_lwe_secret_key.lwe_dimension().to_lwe_size(), + LweCiphertextCount(ciphertext_count), + ciphertext_modulus, + ); + + let mut buffers = ComputationBuffers::new(); + + let fft = Fft::new(fbsk.polynomial_size()); + let fft = fft.as_view(); + + buffers.resize( + batch_programmable_bootstrap_lwe_ciphertext_mem_optimized_requirement::( + fbsk.glwe_size(), + fbsk.polynomial_size(), + fft, + ) + .unwrap() + .unaligned_bytes_required() + * ciphertext_count, + ); + + batched_programmable_bootstrap_lwe_ciphertext_mem_optimized( + &lwe_ciphertext_in, + &mut out_pbs_ct, + &accumulator_list, + &fbsk, + fft, + buffers.stack(), + ); + + assert!(out_pbs_ct + .iter() + .all(|ct| check_encrypted_content_respects_mod(&ct, ciphertext_modulus))); + + let mut decrypted_list = + PlaintextList::new(Scalar::ZERO, PlaintextCount(ciphertext_count)); + + decrypt_lwe_ciphertext_list(&output_lwe_secret_key, &out_pbs_ct, &mut decrypted_list); + + let decoded_list = decrypted_list + .iter() + .map(|ct| round_decode(*ct.0, delta) % msg_modulus) + .collect::>(); + + assert!(decoded_list.iter().all(|ct| *ct == f(msg))); + } + + // In coverage, we break after one while loop iteration, changing message values does not + // yield higher coverage + #[cfg(tarpaulin)] + break; + } +} + +create_parametrized_test!(lwe_encrypt_batch_pbs_decrypt_custom_mod); + // Here we will define a helper function to generate a many lut accumulator for a PBS fn generate_accumulator_many_lut>( polynomial_size: PolynomialSize, diff --git a/tfhe/src/core_crypto/entities/glwe_ciphertext_list.rs b/tfhe/src/core_crypto/entities/glwe_ciphertext_list.rs index 5229009099..a7ee118c84 100644 --- a/tfhe/src/core_crypto/entities/glwe_ciphertext_list.rs +++ b/tfhe/src/core_crypto/entities/glwe_ciphertext_list.rs @@ -33,6 +33,18 @@ impl> AsMut<[T]> for GlweCipher } } +impl> GlweCiphertextList { + /// Mutable variant of [`GlweCiphertext::as_view`]. + pub fn as_mut_view(&mut self) -> GlweCiphertextList<&'_ mut [Scalar]> { + GlweCiphertextList { + data: self.data.as_mut(), + glwe_size: self.glwe_size, + polynomial_size: self.polynomial_size, + ciphertext_modulus: self.ciphertext_modulus, + } + } +} + impl> GlweCiphertextList { /// Create a [`GlweCiphertextList`] from an existing container. /// @@ -164,6 +176,30 @@ impl> GlweCiphertextList pub fn ciphertext_modulus(&self) -> CiphertextModulus { self.ciphertext_modulus } + + /// Return a view of the [`GlweCiphertext`]. This is useful if an algorithm takes a view by + /// value. + pub fn as_view(&self) -> GlweCiphertextList<&'_ [Scalar]> { + GlweCiphertextList { + data: self.data.as_ref(), + glwe_size: self.glwe_size, + polynomial_size: self.polynomial_size, + ciphertext_modulus: self.ciphertext_modulus, + } + } + + /// Return an iterator over the GLWE ciphertexts. + pub fn into_ciphertexts(self) -> impl DoubleEndedIterator> + where + C: Split, + { + let ciphertext_modulus = self.ciphertext_modulus(); + let count = self.glwe_ciphertext_count().0; + let polynomial_size = self.polynomial_size(); + self.data.split_into(count).map(move |slice| { + GlweCiphertext::from_container(slice, polynomial_size, ciphertext_modulus) + }) + } } /// A [`GlweCiphertextList`] owning the memory for its own storage. diff --git a/tfhe/src/core_crypto/entities/lwe_ciphertext_list.rs b/tfhe/src/core_crypto/entities/lwe_ciphertext_list.rs index a813763f36..730cc70a11 100644 --- a/tfhe/src/core_crypto/entities/lwe_ciphertext_list.rs +++ b/tfhe/src/core_crypto/entities/lwe_ciphertext_list.rs @@ -214,6 +214,18 @@ impl> LweCiphertextList< self.ciphertext_modulus(), ) } + + /// Return an iterator over the LWE ciphertexts. + pub fn into_ciphertexts(self) -> impl DoubleEndedIterator> + where + C: Split, + { + let ciphertext_modulus = self.ciphertext_modulus(); + let count = self.lwe_ciphertext_count().0; + self.data + .split_into(count) + .map(move |slice| LweCiphertext::from_container(slice, ciphertext_modulus)) + } } impl> LweCiphertextList { diff --git a/tfhe/src/core_crypto/fft_impl/fft64/crypto/bootstrap.rs b/tfhe/src/core_crypto/fft_impl/fft64/crypto/bootstrap.rs index 8710d9ce4a..e602c9ce17 100644 --- a/tfhe/src/core_crypto/fft_impl/fft64/crypto/bootstrap.rs +++ b/tfhe/src/core_crypto/fft_impl/fft64/crypto/bootstrap.rs @@ -332,6 +332,109 @@ impl<'a> FourierLweBootstrapKeyView<'a> { } } + // CastInto required for PBS modulus switch which returns a usize + pub fn batch_blind_rotate_assign( + self, + mut lut_list: GlweCiphertextListMutView<'_, OutputScalar>, + lwe_list: LweCiphertextListView<'_, InputScalar>, + fft: FftView<'_>, + mut stack: PodStack<'_>, + ) where + InputScalar: UnsignedTorus + CastInto, + OutputScalar: UnsignedTorus, + { + let lut_poly_size = lut_list.polynomial_size(); + let ciphertext_modulus = lut_list.ciphertext_modulus(); + assert!(ciphertext_modulus.is_compatible_with_native_modulus()); + + for (mut lut, lwe) in izip!(lut_list.as_mut_view().iter_mut(), lwe_list.iter()) { + let lwe = lwe.as_ref(); + let lwe_body = *lwe.last().unwrap(); + + let monomial_degree = MonomialDegree(pbs_modulus_switch(lwe_body, lut_poly_size)); + + lut.as_mut_polynomial_list() + .iter_mut() + .for_each(|mut poly| { + let (tmp_poly, _) = stack + .rb_mut() + .make_aligned_raw(poly.as_ref().len(), CACHELINE_ALIGN); + + let mut tmp_poly = Polynomial::from_container(&mut *tmp_poly); + tmp_poly.as_mut().copy_from_slice(poly.as_ref()); + polynomial_wrapping_monic_monomial_div(&mut poly, &tmp_poly, monomial_degree); + }); + } + + // We initialize the ct_0 used for the successive cmuxes + let mut ct0_list = lut_list; + let (ct1_list, mut stack) = + stack.make_aligned_raw(ct0_list.as_ref().len(), CACHELINE_ALIGN); + let mut ct1_list = GlweCiphertextListMutView::from_container( + &mut *ct1_list, + ct0_list.glwe_size(), + lut_poly_size, + ciphertext_modulus, + ); + + for (idx, bootstrap_key_ggsw) in self.into_ggsw_iter().enumerate() { + for (mut ct0, mut ct1, lwe) in izip!( + ct0_list.as_mut_view().iter_mut(), + ct1_list.as_mut_view().iter_mut(), + lwe_list.iter() + ) { + let lwe = lwe.as_ref(); + let lwe_mask_element = lwe[idx]; + if lwe_mask_element != InputScalar::ZERO { + let monomial_degree = + MonomialDegree(pbs_modulus_switch(lwe_mask_element, lut_poly_size)); + + // we effectively inline the body of cmux here, merging the initial subtraction + // operation with the monic polynomial multiplication, then performing the + // external product manually + + // We rotate ct_1 and subtract ct_0 (first step of cmux) by performing + // ct_1 <- (ct_0 * X^{a_hat}) - ct_0 + for (mut ct1_poly, ct0_poly) in izip!( + ct1.as_mut_polynomial_list().iter_mut(), + ct0.as_polynomial_list().iter(), + ) { + polynomial_wrapping_monic_monomial_mul_and_subtract( + &mut ct1_poly, + &ct0_poly, + monomial_degree, + ); + } + + // as_mut_view is required to keep borrow rules consistent + // second step of cmux + add_external_product_assign( + ct0.as_mut_view(), + bootstrap_key_ggsw, + ct1.as_view(), + fft, + stack.rb_mut(), + ); + } + } + } + + if !ciphertext_modulus.is_native_modulus() { + // When we convert back from the fourier domain, integer values will contain up to 53 + // MSBs with information. In our representation of power of 2 moduli < native modulus we + // fill the MSBs and leave the LSBs empty, this usage of the signed decomposer allows to + // round while keeping the data in the MSBs + let signed_decomposer = SignedDecomposer::new( + DecompositionBaseLog(ciphertext_modulus.get_custom_modulus().ilog2() as usize), + DecompositionLevelCount(1), + ); + ct0_list + .as_mut() + .iter_mut() + .for_each(|x| *x = signed_decomposer.closest_representable(*x)); + } + } + pub fn bootstrap( self, mut lwe_out: LweCiphertextMutView<'_, OutputScalar>, @@ -366,6 +469,45 @@ impl<'a> FourierLweBootstrapKeyView<'a> { MonomialDegree(0), ); } + + pub fn batch_bootstrap( + self, + mut lwe_out: LweCiphertextListMutView<'_, OutputScalar>, + lwe_in: LweCiphertextListView<'_, InputScalar>, + accumulator: &GlweCiphertextListView<'_, OutputScalar>, + fft: FftView<'_>, + stack: PodStack<'_>, + ) where + // CastInto required for PBS modulus switch which returns a usize + InputScalar: UnsignedTorus + CastInto, + OutputScalar: UnsignedTorus, + { + assert!(lwe_in.ciphertext_modulus().is_power_of_two()); + assert!(lwe_out.ciphertext_modulus().is_power_of_two()); + assert_eq!( + lwe_out.ciphertext_modulus(), + accumulator.ciphertext_modulus() + ); + + let (local_accumulator_data, stack) = + stack.collect_aligned(CACHELINE_ALIGN, accumulator.as_ref().iter().copied()); + let mut local_accumulator = GlweCiphertextListMutView::from_container( + &mut *local_accumulator_data, + accumulator.glwe_size(), + accumulator.polynomial_size(), + accumulator.ciphertext_modulus(), + ); + self.batch_blind_rotate_assign(local_accumulator.as_mut_view(), lwe_in, fft, stack); + + for (mut lwe_out, local_accumulator) in izip!(lwe_out.iter_mut(), local_accumulator.iter()) + { + extract_lwe_sample_from_glwe_ciphertext( + &local_accumulator, + &mut lwe_out, + MonomialDegree(0), + ); + } + } } impl FourierBootstrapKey for FourierLweBootstrapKeyOwned