Merge pull request #959 from filecoin-project/simpler-windows

refactor: unify window size handling

Author: dignifiedquire

.circleci/config.yml

@@ -181,7 +181,7 @@ jobs:
       - run:
           name: Ensure existence of Groth parameters and keys on remote host
           command: |
-            ./fil-proofs-tooling/scripts/paramcache-remote.sh "${CIRCLE_BRANCH}" "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" "-z=$((1024*1024*1024)):$(((128*1024*1024)/32))"
+            ./fil-proofs-tooling/scripts/paramcache-remote.sh "${CIRCLE_BRANCH}" "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" "-z=$((1024*1024*1024))"
           no_output_timeout: 60m
       - run:
           name: Run hash-constraints benchmarks on remote host

fil-proofs-tooling/src/bin/benchy/election_post.rs

@@ -52,12 +52,8 @@ impl Report {
     }
 }
 
-pub fn run(sector_size: usize, window_size_nodes: usize) -> anyhow::Result<()> {
-    info!(
-        "Benchy Election PoSt: sector-size={} - window_size={}",
-        sector_size,
-        window_size_nodes * 32
-    );
+pub fn run(sector_size: usize) -> anyhow::Result<()> {
+    info!("Benchy Election PoSt: sector-size={}", sector_size,);
 
     let sector_size_unpadded_bytes_ammount =
         UnpaddedBytesAmount::from(PaddedBytesAmount(sector_size as u64));
@@ -102,7 +98,6 @@ pub fn run(sector_size: usize, window_size_nodes: usize) -> anyhow::Result<()> {
     let porep_config = PoRepConfig {
         sector_size: SectorSize(sector_size as u64),
         partitions: N_PARTITIONS,
-        window_size_nodes,
     };
     let cache_dir = tempfile::tempdir().unwrap();
     let sector_id = SectorId::from(SECTOR_ID);
@@ -147,7 +142,6 @@ pub fn run(sector_size: usize, window_size_nodes: usize) -> anyhow::Result<()> {
     // Measure PoSt generation and verification.
     let post_config = PoStConfig {
         sector_size: SectorSize(sector_size as u64),
-        window_size_nodes,
     };
 
     let challenge_count = 1u64;

fil-proofs-tooling/src/bin/benchy/main.rs

@@ -112,13 +112,6 @@ fn main() {
                 .required(true)
                 .help("The data size in KiB")
                 .takes_value(true),
-        )
-        .arg(
-            Arg::with_name("window-size")
-                .long("window-size")
-                .help("The window size in bytes")
-                .default_value("4096")
-                .takes_value(true),
         );
 
     let hash_cmd = SubCommand::with_name("hash-constraints")
@@ -164,10 +157,7 @@ fn main() {
             let sector_size_kibs = value_t!(m, "size", usize)
                 .expect("could not convert `size` CLI argument to `usize`");
             let sector_size = sector_size_kibs * 1024;
-            let window_size_bytes = value_t!(m, "window-size", usize)
-                .expect("could not convert `window-size` CLI argument to `usize`");
-            let window_size_nodes = window_size_bytes / 32;
-            election_post::run(sector_size, window_size_nodes).expect("election-post failed");
+            election_post::run(sector_size).expect("election-post failed");
         }
         ("hash-constraints", Some(_m)) => {
             hash_fns::run().expect("hash-constraints failed");

filecoin-proofs/src/api/mod.rs

@@ -73,7 +73,6 @@ pub fn get_unsealed_range<T: Into<PathBuf> + AsRef<Path>>(
     let pp = public_params(
         PaddedBytesAmount::from(porep_config),
         usize::from(PoRepProofPartitions::from(porep_config)),
-        porep_config.window_size_nodes,
     );
 
     let offset_padded: PaddedBytesAmount = UnpaddedBytesAmount::from(offset).into();
@@ -289,9 +288,7 @@ mod tests {
     use storage_proofs::fr32::bytes_into_fr;
     use tempfile::NamedTempFile;
 
-    use crate::constants::{
-        SECTOR_SIZE_ONE_KIB, SINGLE_PARTITION_PROOF_LEN, WINDOW_SIZE_NODES_ONE_KIB,
-    };
+    use crate::constants::{SECTOR_SIZE_ONE_KIB, SINGLE_PARTITION_PROOF_LEN};
     use crate::types::{PoStConfig, SectorSize};
 
     static INIT_LOGGER: Once = Once::new();
@@ -316,7 +313,6 @@ mod tests {
                 PoRepConfig {
                     sector_size: SectorSize(SECTOR_SIZE_ONE_KIB),
                     partitions: PoRepProofPartitions(2),
-                    window_size_nodes: WINDOW_SIZE_NODES_ONE_KIB,
                 },
                 not_convertible_to_fr_bytes,
                 convertible_to_fr_bytes,
@@ -345,7 +341,6 @@ mod tests {
                 PoRepConfig {
                     sector_size: SectorSize(SECTOR_SIZE_ONE_KIB),
                     partitions: PoRepProofPartitions(2),
-                    window_size_nodes: WINDOW_SIZE_NODES_ONE_KIB,
                 },
                 convertible_to_fr_bytes,
                 not_convertible_to_fr_bytes,
@@ -393,7 +388,6 @@ mod tests {
         let result = verify_post(
             PoStConfig {
                 sector_size: SectorSize(SECTOR_SIZE_ONE_KIB),
-                window_size_nodes: WINDOW_SIZE_NODES_ONE_KIB,
             },
             &[0; 32],
             1,
@@ -424,7 +418,6 @@ mod tests {
         let rng = &mut XorShiftRng::from_seed(crate::TEST_SEED);
 
         let sector_size = SECTOR_SIZE_ONE_KIB;
-        let window_size_nodes = WINDOW_SIZE_NODES_ONE_KIB;
 
         let number_of_bytes_in_piece =
             UnpaddedBytesAmount::from(PaddedBytesAmount(sector_size.clone()));
@@ -456,7 +449,6 @@ mod tests {
         let config = PoRepConfig {
             sector_size: SectorSize(sector_size.clone()),
             partitions: PoRepProofPartitions(2),
-            window_size_nodes,
         };
 
         let cache_dir = tempfile::tempdir().unwrap();

filecoin-proofs/src/api/seal.rs

@@ -77,7 +77,6 @@ pub fn seal_pre_commit<R: AsRef<Path>, T: AsRef<Path>, S: AsRef<Path>>(
         vanilla_params: setup_params(
             PaddedBytesAmount::from(porep_config),
             usize::from(PoRepProofPartitions::from(porep_config)),
-            porep_config.window_size_nodes,
         ),
         partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))),
     };
@@ -220,7 +219,6 @@ pub fn seal_commit<T: AsRef<Path>>(
         vanilla_params: setup_params(
             PaddedBytesAmount::from(porep_config),
             usize::from(PoRepProofPartitions::from(porep_config)),
-            porep_config.window_size_nodes,
         ),
         partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))),
     };
@@ -292,7 +290,6 @@ pub fn verify_seal(
         vanilla_params: setup_params(
             PaddedBytesAmount::from(porep_config),
             usize::from(PoRepProofPartitions::from(porep_config)),
-            porep_config.window_size_nodes,
         ),
         partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))),
     };

filecoin-proofs/src/bin/paramcache.rs

@@ -18,11 +18,11 @@ use storage_proofs::stacked::StackedDrg;
 
 const POREP_PROOF_PARTITION_CHOICES: [PoRepProofPartitions; 1] = [PoRepProofPartitions(2)];
 
-const PUBLISHED_SECTOR_SIZES: [(u64, usize); 4] = [
-    (SECTOR_SIZE_ONE_KIB, WINDOW_SIZE_NODES_ONE_KIB),
-    (SECTOR_SIZE_16_MIB, WINDOW_SIZE_NODES_16_MIB),
-    (SECTOR_SIZE_256_MIB, WINDOW_SIZE_NODES_256_MIB),
-    (SECTOR_SIZE_1_GIB, WINDOW_SIZE_NODES_1_GIB),
+const PUBLISHED_SECTOR_SIZES: [u64; 4] = [
+    SECTOR_SIZE_ONE_KIB,
+    SECTOR_SIZE_16_MIB,
+    SECTOR_SIZE_256_MIB,
+    SECTOR_SIZE_1_GIB,
 ];
 
 fn cache_porep_params(porep_config: PoRepConfig) {
@@ -35,7 +35,6 @@ fn cache_porep_params(porep_config: PoRepConfig) {
     let public_params = public_params(
         PaddedBytesAmount::from(porep_config),
         usize::from(PoRepProofPartitions::from(porep_config)),
-        porep_config.window_size_nodes,
     );
 
     {
@@ -134,35 +133,24 @@ pub fn main() {
         )
         .get_matches();
 
-    let sizes: HashSet<(u64, usize)> = if matches.is_present("params-for-sector-sizes") {
-        values_t!(matches.values_of("params-for-sector-sizes"), String)
+    let sizes: HashSet<u64> = if matches.is_present("params-for-sector-sizes") {
+        values_t!(matches.values_of("params-for-sector-sizes"), u64)
             .unwrap()
             .into_iter()
-            .map(|v| {
-                let parts = v.split(':').take(2).collect::<Vec<_>>();
-                assert_eq!(
-                    parts.len(),
-                    2,
-                    "invalid param, expecting sector_size:window_size_nodes"
-                );
-                (parts[0].parse().unwrap(), parts[1].parse().unwrap())
-            })
             .collect()
     } else {
         PUBLISHED_SECTOR_SIZES.iter().cloned().collect()
     };
 
-    for (sector_size, window_size_nodes) in sizes {
+    for sector_size in sizes {
         cache_post_params(PoStConfig {
             sector_size: SectorSize(sector_size),
-            window_size_nodes,
         });
 
         for p in &POREP_PROOF_PARTITION_CHOICES {
             cache_porep_params(PoRepConfig {
                 sector_size: SectorSize(sector_size),
                 partitions: *p,
-                window_size_nodes,
             });
         }
     }

filecoin-proofs/src/caches.rs

@@ -81,7 +81,6 @@ pub fn get_stacked_params(porep_config: PoRepConfig) -> Result<Arc<groth16::Para
     let public_params = public_params(
         PaddedBytesAmount::from(porep_config),
         usize::from(PoRepProofPartitions::from(porep_config)),
-        porep_config.window_size_nodes,
     );
 
     let parameters_generator = || {
@@ -127,7 +126,6 @@ pub fn get_stacked_verifying_key(porep_config: PoRepConfig) -> Result<Arc<Bls12V
     let public_params = public_params(
         PaddedBytesAmount::from(porep_config),
         usize::from(PoRepProofPartitions::from(porep_config)),
-        porep_config.window_size_nodes,
     );
 
     let vk_generator = || {

filecoin-proofs/src/parameters.rs

@@ -1,3 +1,4 @@
+use anyhow::Result;
 use storage_proofs::drgraph::{DefaultTreeHasher, BASE_DEGREE};
 use storage_proofs::election_post::{self, ElectionPoSt};
 use storage_proofs::proof::ProofScheme;
@@ -21,16 +22,26 @@ pub type PostPublicParams = election_post::PublicParams;
 pub fn public_params(
     sector_bytes: PaddedBytesAmount,
     partitions: usize,
-    window_size_nodes: usize,
 ) -> stacked::PublicParams<DefaultTreeHasher> {
     StackedDrg::<DefaultTreeHasher, DefaultPieceHasher>::setup(&setup_params(
         sector_bytes,
         partitions,
-        window_size_nodes,
     ))
     .unwrap()
 }
 
+pub fn window_size_nodes_for_sector_bytes(sector_size: PaddedBytesAmount) -> Result<usize> {
+    use crate::constants::*;
+
+    match u64::from(sector_size) {
+        SECTOR_SIZE_ONE_KIB => Ok(WINDOW_SIZE_NODES_ONE_KIB),
+        SECTOR_SIZE_16_MIB => Ok(WINDOW_SIZE_NODES_16_MIB),
+        SECTOR_SIZE_256_MIB => Ok(WINDOW_SIZE_NODES_256_MIB),
+        SECTOR_SIZE_1_GIB => Ok(WINDOW_SIZE_NODES_1_GIB),
+        _ => Err(anyhow!("Unknown sector size {:?}", sector_size)),
+    }
+}
+
 pub fn post_public_params(post_config: PoStConfig) -> PostPublicParams {
     ElectionPoSt::<DefaultTreeHasher>::setup(&post_setup_params(post_config)).unwrap()
 }
@@ -43,16 +54,12 @@ pub fn post_setup_params(post_config: PoStConfig) -> PostSetupParams {
     }
 }
 
-pub fn setup_params(
-    sector_bytes: PaddedBytesAmount,
-    partitions: usize,
-    window_size_nodes: usize,
-) -> stacked::SetupParams {
-    let sector_bytes = usize::from(sector_bytes);
-
+pub fn setup_params(sector_bytes: PaddedBytesAmount, partitions: usize) -> stacked::SetupParams {
     let window_challenges = select_challenges(partitions, POREP_WINDOW_MINIMUM_CHALLENGES, LAYERS);
     let wrapper_challenges =
         select_challenges(partitions, POREP_WRAPPER_MINIMUM_CHALLENGES, LAYERS);
+    let window_size_nodes = window_size_nodes_for_sector_bytes(sector_bytes).unwrap();
+    let sector_bytes = usize::from(sector_bytes);
 
     let config = StackedConfig {
         window_challenges,

filecoin-proofs/src/types/porep_config.rs

@@ -12,7 +12,6 @@ use crate::types::*;
 pub struct PoRepConfig {
     pub sector_size: SectorSize,
     pub partitions: PoRepProofPartitions,
-    pub window_size_nodes: usize,
 }
 
 impl From<PoRepConfig> for PaddedBytesAmount {
@@ -49,11 +48,8 @@ impl From<PoRepConfig> for SectorSize {
 impl PoRepConfig {
     /// Returns the cache identifier as used by `storage-proofs::paramater_cache`.
     pub fn get_cache_identifier(&self) -> String {
-        let params = crate::parameters::public_params(
-            self.sector_size.into(),
-            self.partitions.into(),
-            self.window_size_nodes,
-        );
+        let params =
+            crate::parameters::public_params(self.sector_size.into(), self.partitions.into());
 
         <StackedCompound as CacheableParameters<
             Bls12,

filecoin-proofs/src/types/post_config.rs

@@ -10,7 +10,6 @@ use crate::types::*;
 #[derive(Clone, Copy, Debug)]
 pub struct PoStConfig {
     pub sector_size: SectorSize,
-    pub window_size_nodes: usize,
 }
 
 impl From<PoStConfig> for PaddedBytesAmount {