feat(filecoin-proofs): switch generate_piece_commitment to accept io::Read

Author: dignifiedquire

filecoin-proofs/src/api/mod.rs

@@ -381,14 +381,13 @@ pub fn verify_piece_inclusion_proof(
 
 /// Takes a piece file at `unpadded_piece_path` and the size of the piece and returns the comm_p.
 ///
-pub fn generate_piece_commitment<T: Into<PathBuf> + AsRef<Path>>(
-    unpadded_piece_path: T,
+pub fn generate_piece_commitment<T: std::io::Read>(
+    unpadded_piece_file: T,
     unpadded_piece_size: UnpaddedBytesAmount,
 ) -> error::Result<Commitment> {
-    let mut unpadded_piece_file = File::open(unpadded_piece_path)?;
     let mut padded_piece_file = tempfile()?;
 
-    let (_, mut source) = get_aligned_source(&mut unpadded_piece_file, &[], unpadded_piece_size);
+    let (_, mut source) = get_aligned_source(unpadded_piece_file, &[], unpadded_piece_size);
     write_padded(&mut source, &mut padded_piece_file)?;
 
     let _ = padded_piece_file.seek(SeekFrom::Start(0))?;
@@ -455,6 +454,7 @@ fn pad_safe_fr(unpadded: &FrSafe) -> Fr32Ary {
 #[cfg(test)]
 mod tests {
     use std::collections::BTreeMap;
+    use std::io::{Seek, SeekFrom};
 
     use crate::constants::SECTOR_SIZE_ONE_KIB;
     use crate::error::ExpectWithBacktrace;
@@ -470,8 +470,10 @@ mod tests {
     fn generate_comm_p(data: &[u8]) -> Result<Commitment, failure::Error> {
         let mut file = NamedTempFile::new().expects("could not create named temp file");
         file.write_all(data)?;
+        file.seek(SeekFrom::Start(0))?;
+
         let comm_p =
-            generate_piece_commitment(file.path(), UnpaddedBytesAmount(data.len() as u64))?;
+            generate_piece_commitment(file.as_file_mut(), UnpaddedBytesAmount(data.len() as u64))?;
         Ok(comm_p)
     }
 
@@ -704,9 +706,11 @@ mod tests {
 
         let mut piece_file = NamedTempFile::new()?;
         piece_file.write_all(&piece_bytes)?;
-        piece_file.seek(SeekFrom::Start(0))?;
+        piece_file.as_file_mut().sync_all()?;
+        piece_file.as_file_mut().seek(SeekFrom::Start(0))?;
 
-        let comm_p = generate_piece_commitment(&piece_file.path(), number_of_bytes_in_piece)?;
+        let comm_p = generate_piece_commitment(piece_file.as_file_mut(), number_of_bytes_in_piece)?;
+        piece_file.as_file_mut().seek(SeekFrom::Start(0))?;
 
         let mut staged_sector_file = NamedTempFile::new()?;
         add_piece(

filecoin-proofs/src/fr32.rs

@@ -650,11 +650,21 @@ where
     let mut buffer = [0; CHUNK_SIZE];
     let mut written = 0;
 
-    while let Ok(bytes_read) = source.read(&mut buffer) {
-        if bytes_read == 0 {
-            break;
+    loop {
+        match source.read(&mut buffer) {
+            Ok(bytes_read) => {
+                if bytes_read == 0 {
+                    break;
+                }
+                written += write_padded_aux(&FR32_PADDING_MAP, &buffer[..bytes_read], target)?;
+            }
+            Err(err) => {
+                if err.kind() == io::ErrorKind::Interrupted {
+                    continue;
+                }
+                return Err(err);
+            }
         }
-        written += write_padded_aux(&FR32_PADDING_MAP, &buffer[..bytes_read], target)?;
     }
 
     Ok(written)
@@ -1174,6 +1184,46 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_write_padded_chained_byte_source() {
+        let random_bytes: Vec<u8> = (0..127).map(|_| rand::random::<u8>()).collect();
+
+        // read 127 bytes from a non-chained source
+        let (n, output_x) = {
+            let mut input_x = Cursor::new(random_bytes.clone());
+            let mut output_x = Cursor::new(Vec::new());
+
+            let n_x = write_padded(&mut input_x, &mut output_x).unwrap();
+            output_x.seek(SeekFrom::Start(0)).expect("could not seek");
+
+            let mut buf_x = Vec::new();
+            output_x.read_to_end(&mut buf_x).expect("could not seek");
+
+            (n_x, buf_x)
+        };
+
+        // read 127 bytes from a 32-byte buffer and then a 95-byte buffer
+        let (m, output_y) = {
+            let mut input_y =
+                Cursor::new(random_bytes.iter().take(32).cloned().collect::<Vec<u8>>()).chain(
+                    Cursor::new(random_bytes.iter().skip(32).cloned().collect::<Vec<u8>>()),
+                );
+
+            let mut output_y = Cursor::new(Vec::new());
+
+            let n_y = write_padded(&mut input_y, &mut output_y).unwrap();
+            output_y.seek(SeekFrom::Start(0)).expect("could not seek");
+
+            let mut buf_y = Vec::new();
+            output_y.read_to_end(&mut buf_y).expect("could not seek");
+
+            (n_y, buf_y)
+        };
+
+        assert_eq!(n, m, "should have written same number of bytes");
+        assert_eq!(output_x, output_y, "should have written same bytes")
+    }
+
     // `write_padded` for 127 bytes of 1s, splitting it in two calls of varying
     // sizes, from 0 to the full size, generating many unaligned calls, check padding.
     #[test]