feat(storage-proofs): implement memory reduced stacked replication

Author: dignifiedquire

filecoin-proofs/Cargo.toml

@@ -53,7 +53,7 @@ rexpect = "0.3.0"
 [features]
 default = []
 cpu-profile = []
-heap-profile = []
+heap-profile = ["gperftools/heap"]
 simd = ["storage-proofs/simd"]
 asm = ["storage-proofs/asm"]
 gpu = ["storage-proofs/gpu", "bellperson/gpu", "fil-sapling-crypto/gpu", "phase21/gpu"]

storage-proofs/Cargo.toml

@@ -15,7 +15,7 @@ bench = false
 bitvec = "0.5"
 rand = "0.4"
 libc = "0.2"
-merkletree = "0.10"
+merkletree = "0.11"
 failure = "0.1"
 byteorder = "1"
 config = "0.9.3"

storage-proofs/src/circuit/pedersen.rs

@@ -174,7 +174,8 @@ mod tests {
             assert_eq!(
                 expected,
                 out.get_value().unwrap(),
-                "circuit and non circuit do not match"
+                "circuit and non circuit do not match {} bytes",
+                bytes
             );
         }
     }

storage-proofs/src/crypto/pedersen.rs

@@ -1,4 +1,3 @@
-use bitvec::{self, BitVec};
 use ff::PrimeFieldRepr;
 use fil_sapling_crypto::jubjub::JubjubBls12;
 use fil_sapling_crypto::pedersen_hash::{pedersen_hash, Personalization};
@@ -20,64 +19,229 @@ pub const PEDERSEN_BLOCK_SIZE: usize = 256;
 pub const PEDERSEN_BLOCK_BYTES: usize = PEDERSEN_BLOCK_SIZE / 8;
 
 pub fn pedersen(data: &[u8]) -> Fr {
-    pedersen_hash::<Bls12, _>(
-        Personalization::None,
-        BitVec::<bitvec::LittleEndian, u8>::from(data)
-            .iter()
-            .take(data.len() * 8),
-        &JJ_PARAMS,
-    )
-    .into_xy()
-    .0
+    pedersen_bits(Bits::new(data))
+}
+
+pub fn pedersen_bits<'a, S: Iterator<Item = &'a [u8]>>(data: Bits<&'a [u8], S>) -> Fr {
+    pedersen_hash::<Bls12, _>(Personalization::None, data, &JJ_PARAMS)
+        .into_xy()
+        .0
 }
 
 /// Pedersen hashing for inputs that have length mulitple of the block size `256`. Based on pedersen hashes and a Merkle-Damgard construction.
 pub fn pedersen_md_no_padding(data: &[u8]) -> Fr {
-    assert!(
-        data.len() >= 2 * PEDERSEN_BLOCK_BYTES,
-        "must be at least 2 block sizes long, got {}bits",
-        data.len()
-    );
-    assert_eq!(
-        data.len() % PEDERSEN_BLOCK_BYTES,
-        0,
-        "input must be a multiple of the blocksize"
-    );
-    let mut chunks = data.chunks(PEDERSEN_BLOCK_BYTES);
-    let mut cur = Vec::with_capacity(2 * PEDERSEN_BLOCK_BYTES);
-    cur.resize(PEDERSEN_BLOCK_BYTES, 0);
-    cur[0..PEDERSEN_BLOCK_BYTES].copy_from_slice(chunks.nth(0).unwrap());
+    pedersen_md_no_padding_bits(Bits::new(data))
+}
 
-    for block in chunks {
-        cur.resize(2 * PEDERSEN_BLOCK_BYTES, 0);
-        cur[PEDERSEN_BLOCK_BYTES..].copy_from_slice(block);
-        pedersen_compression(&mut cur);
+pub fn pedersen_md_no_padding_bits<T: AsRef<[u8]>, S: Iterator<Item = T>>(
+    mut data: Bits<T, S>,
+) -> Fr {
+    let mut cur = Vec::with_capacity(PEDERSEN_BLOCK_SIZE);
+
+    // hash the first two blocks
+    let first = pedersen_compression_bits(data.ref_take(2 * PEDERSEN_BLOCK_SIZE));
+    first
+        .write_le(&mut cur)
+        .expect("failed to write result hash");
+
+    while !data.is_done() {
+        let r = data.ref_take(PEDERSEN_BLOCK_SIZE);
+        let x = pedersen_compression_bits(Bits::new(&cur).chain(r));
+
+        cur.truncate(0);
+        x.write_le(&mut cur).expect("failed to write result hash");
     }
 
-    let frs = bytes_into_frs::<Bls12>(&cur[0..PEDERSEN_BLOCK_BYTES])
-        .expect("pedersen must generate valid fr elements");
+    let frs = bytes_into_frs::<Bls12>(&cur).expect("pedersen must generate valid fr elements");
     assert_eq!(frs.len(), 1);
     frs[0]
 }
 
-pub fn pedersen_compression(bytes: &mut Vec<u8>) {
-    let bits = BitVec::<bitvec::LittleEndian, u8>::from(&bytes[..]);
-    let (x, _) = pedersen_hash::<Bls12, _>(
-        Personalization::None,
-        bits.iter().take(bytes.len() * 8),
-        &JJ_PARAMS,
-    )
-    .into_xy();
-    let x: FrRepr = x.into();
+fn pedersen_compression_bits<T>(bits: T) -> FrRepr
+where
+    T: IntoIterator<Item = bool>,
+{
+    let (x, _) = pedersen_hash::<Bls12, _>(Personalization::None, bits, &JJ_PARAMS).into_xy();
+    x.into()
+}
+
+/// Creates an iterator over the byte slices in little endian format.
+#[derive(Debug, Clone)]
+pub struct Bits<K: AsRef<[u8]>, S: Iterator<Item = K>> {
+    /// The individual parts that make up the data that is being iterated over.
+    parts: ManyOrSingle<K, S>,
+    /// How many bytes we are into the `current_part`
+    position_byte: usize,
+    /// How many bits we are into the `current_byte`.
+    position_bit: u8,
+    /// The current part we are reading from.
+    current_part: Option<K>,
+    /// Track the first iteration.
+    first: bool,
+    /// Are we done yet?
+    done: bool,
+}
+
+/// Abstraction over either an iterator or a single element.
+#[derive(Debug, Clone)]
+enum ManyOrSingle<T, S = <Vec<T> as IntoIterator>::IntoIter>
+where
+    S: Iterator<Item = T>,
+{
+    Many(S),
+    Single(Option<T>),
+}
+
+impl<T: AsRef<[u8]>> Bits<T, <Vec<T> as IntoIterator>::IntoIter> {
+    pub fn new(parts: T) -> Self {
+        Bits {
+            parts: ManyOrSingle::<T, <Vec<T> as IntoIterator>::IntoIter>::Single(Some(parts)),
+            position_byte: 0,
+            position_bit: 0,
+            current_part: None,
+            first: true,
+            done: false,
+        }
+    }
+}
+
+impl<T: AsRef<[u8]>, S: Iterator<Item = T>> Bits<T, S> {
+    pub fn new_many(parts: S) -> Self {
+        Bits {
+            parts: ManyOrSingle::Many(parts),
+            position_byte: 0,
+            position_bit: 0,
+            current_part: None,
+            first: true,
+            done: false,
+        }
+    }
+
+    pub fn is_done(&self) -> bool {
+        self.done
+    }
+
+    fn inc_part(&mut self) {
+        self.current_part = match self.parts {
+            ManyOrSingle::Many(ref mut parts) => {
+                if self.first {
+                    self.first = false;
+                }
+                parts.next()
+            }
+            ManyOrSingle::Single(ref mut part) => {
+                if self.first {
+                    self.first = false;
+                    part.take()
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    /// Increments the inner positions by 1 bit.
+    fn inc(&mut self) {
+        if self.position_bit < 7 {
+            self.position_bit += 1;
+            return;
+        }
+
+        self.position_bit = 0;
+        if let Some(ref part) = self.current_part {
+            if self.position_byte + 1 < part.as_ref().len() {
+                self.position_byte += 1;
+                return;
+            }
+        }
+
+        self.inc_part();
+        self.position_byte = 0;
+        self.done = self.current_part.is_none();
+    }
+
+    fn ref_take(&mut self, take: usize) -> BitsTake<'_, T, S> {
+        BitsTake::new(self, take)
+    }
+}
+
+#[derive(Debug)]
+struct BitsTake<'a, T: AsRef<[u8]>, S: Iterator<Item = T>> {
+    iter: &'a mut Bits<T, S>,
+    take: usize,
+}
+
+impl<'a, T: AsRef<[u8]>, S: Iterator<Item = T>> BitsTake<'a, T, S> {
+    pub fn new(iter: &'a mut Bits<T, S>, take: usize) -> Self {
+        BitsTake { iter, take }
+    }
+}
+
+impl<'a, T: AsRef<[u8]>, S: Iterator<Item = T> + std::iter::FusedIterator> std::iter::FusedIterator
+    for BitsTake<'a, T, S>
+{
+}
+
+impl<'a, T: AsRef<[u8]>, S: Iterator<Item = T>> Iterator for BitsTake<'a, T, S> {
+    type Item = bool;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.take == 0 {
+            return None;
+        }
+
+        self.take -= 1;
+        self.iter.next()
+    }
+}
+
+impl<T: AsRef<[u8]>, S: Iterator<Item = T> + std::iter::FusedIterator> std::iter::FusedIterator
+    for Bits<T, S>
+{
+}
+
+impl<T: AsRef<[u8]>, S: Iterator<Item = T>> Iterator for Bits<T, S> {
+    type Item = bool;
 
-    bytes.truncate(0);
-    x.write_le(bytes).expect("failed to write result hash");
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.done {
+            return None;
+        }
+
+        if self.first {
+            // first time
+            self.inc_part();
+        }
+
+        let byte = match self.current_part {
+            Some(ref part) => part.as_ref()[self.position_byte],
+            None => {
+                self.done = true;
+                return None;
+            }
+        };
+
+        let res = (byte >> self.position_bit) & 1u8 == 1u8;
+        self.inc();
+
+        Some(res)
+    }
+
+    // optimized nth method so we can use it to skip forward easily
+    fn nth(&mut self, n: usize) -> Option<Self::Item> {
+        for _ in 0..n {
+            // TODO: implement optimized inc for n bits.
+            self.inc();
+        }
+        self.next()
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::util::bytes_into_bits;
+    use bitvec::{self, BitVec};
     use ff::Field;
     use paired::bls12_381::Fr;
     use rand::{Rng, SeedableRng, XorShiftRng};
@@ -98,10 +262,12 @@ mod tests {
 
     #[test]
     fn test_pedersen_compression() {
-        let bytes = b"some bytes";
-        let mut data = vec![0; bytes.len()];
-        data.copy_from_slice(&bytes[..]);
-        pedersen_compression(&mut data);
+        let bytes = Bits::new(b"some bytes");
+
+        let x = pedersen_compression_bits(bytes);
+        let mut data = Vec::new();
+        x.write_le(&mut data).unwrap();
+
         let expected = vec![
             237, 70, 41, 231, 39, 180, 131, 120, 36, 36, 119, 199, 200, 225, 153, 242, 106, 116,
             70, 9, 12, 249, 169, 84, 105, 38, 225, 115, 165, 188, 98, 25,
@@ -119,4 +285,45 @@ mod tests {
             assert_ne!(hashed, Fr::zero());
         }
     }
+
+    #[test]
+    fn test_bits_collect() {
+        let bytes = b"hello";
+        let bits = bytes_into_bits(bytes);
+
+        let bits_iter = Bits::new(bytes);
+        let bits_iter_collected: Vec<bool> = bits_iter.collect();
+
+        assert_eq!(bits, bits_iter_collected);
+
+        let bytes = b"hello world these are some bytes";
+        let bits = bytes_into_bits(bytes);
+
+        let parts: Vec<&[u8]> = vec![b"hello ", b"world", b" these are some bytes"];
+        let bits_iter = Bits::new_many(parts.into_iter());
+
+        let bits_iter_collected: Vec<bool> = bits_iter.collect();
+
+        assert_eq!(bits, bits_iter_collected);
+    }
+
+    #[test]
+    fn test_bits_take() {
+        let bytes = b"hello world these are some bytes";
+        let bits = bytes_into_bits(bytes);
+
+        let parts: Vec<&[u8]> = vec![b"hello ", b"world", b" these are some bytes"];
+        let mut bits_iter = Bits::new_many(parts.into_iter());
+
+        let bits_collected: Vec<bool> = vec![
+            bits_iter.ref_take(8).collect::<Vec<bool>>(),
+            bits_iter.ref_take(8).collect::<Vec<bool>>(),
+            bits_iter.ref_take(bits.len() - 16).collect::<Vec<bool>>(),
+        ]
+        .into_iter()
+        .flatten()
+        .collect();
+
+        assert_eq!(bits, bits_collected);
+    }
 }

storage-proofs/src/drgraph.rs

@@ -9,7 +9,7 @@ use crate::error::*;
 use crate::fr32::bytes_into_fr_repr_safe;
 use crate::hasher::pedersen::PedersenHasher;
 use crate::hasher::{Domain, Hasher};
-use crate::merkle::{MerkleStore, MerkleTree};
+use crate::merkle::MerkleTree;
 use crate::parameter_cache::ParameterSetMetadata;
 use crate::util::{data_at_node, data_at_node_offset, NODE_SIZE};
 use merkletree::merkle::FromIndexedParallelIterator;
@@ -71,16 +71,6 @@ pub trait Graph<H: Hasher>: ::std::fmt::Debug + Clone + PartialEq + Eq {
         }
     }
 
-    /// Builds a merkle tree based on the given leaves store.
-    // FIXME: Add the parallel case (if it's worth it).
-    fn merkle_tree_from_leaves(
-        &self,
-        leaves: MerkleStore<H::Domain>,
-        leafs_number: usize,
-    ) -> Result<MerkleTree<H::Domain, H::Function>> {
-        Ok(MerkleTree::from_leaves_store(leaves, leafs_number))
-    }
-
     /// Returns the merkle tree depth.
     fn merkle_tree_depth(&self) -> u64 {
         graph_height(self.size()) as u64