Merge branch 'master' into fix/prove/request-parents-once

Author: schomatis

filecoin-proofs/src/api/internal.rs

@@ -341,7 +341,7 @@ pub unsafe extern "C" fn get_max_user_bytes_per_staged_sector(
     let mut response: responses::GetMaxStagedBytesPerSector = Default::default();
 
     response.status_code = FCPResponseStatus::FCPNoError;
-    response.max_staged_bytes_per_sector = (*ptr).get_max_user_bytes_per_staged_sector();;
+    response.max_staged_bytes_per_sector = (*ptr).get_max_user_bytes_per_staged_sector().into();
 
     raw_ptr(response)
 }
@@ -377,7 +377,7 @@ pub unsafe extern "C" fn get_seal_status(
                         .iter()
                         .map(|p| FFIPieceMetadata {
                             piece_key: rust_str_to_c_str(p.piece_key.to_string()),
-                            num_bytes: p.num_bytes,
+                            num_bytes: p.num_bytes.into(),
                         })
                         .collect::<Vec<FFIPieceMetadata>>();
 
@@ -426,7 +426,7 @@ pub unsafe extern "C" fn get_sealed_sectors(
                         .iter()
                         .map(|p| FFIPieceMetadata {
                             piece_key: rust_str_to_c_str(p.piece_key.to_string()),
-                            num_bytes: p.num_bytes,
+                            num_bytes: p.num_bytes.into(),
                         })
                         .collect::<Vec<FFIPieceMetadata>>();
 
@@ -480,7 +480,7 @@ pub unsafe extern "C" fn get_staged_sectors(
                         .iter()
                         .map(|p| FFIPieceMetadata {
                             piece_key: rust_str_to_c_str(p.piece_key.to_string()),
-                            num_bytes: p.num_bytes,
+                            num_bytes: p.num_bytes.into(),
                         })
                         .collect::<Vec<FFIPieceMetadata>>();
 

filecoin-proofs/src/api/mod.rs

@@ -1,11 +1,13 @@
+use std::sync::Arc;
+
 use crate::api::sector_builder::errors::*;
 use crate::api::sector_builder::metadata::sum_piece_bytes;
 use crate::api::sector_builder::metadata::StagedSectorMetadata;
 use crate::api::sector_builder::state::StagedState;
 use crate::api::sector_builder::*;
 use crate::error;
+use sector_base::api::bytes_amount::UnpaddedBytesAmount;
 use sector_base::api::sector_store::SectorManager;
-use std::sync::Arc;
 
 pub fn add_piece(
     sector_store: &Arc<WrappedSectorStore>,
@@ -16,7 +18,7 @@ pub fn add_piece(
     let sector_mgr = sector_store.inner.manager();
     let sector_max = sector_store.inner.config().max_unsealed_bytes_per_sector();
 
-    let piece_bytes_len = piece_bytes.len() as u64;
+    let piece_bytes_len = UnpaddedBytesAmount(piece_bytes.len() as u64);
 
     let opt_dest_sector_id = {
         let candidates: Vec<StagedSectorMetadata> = staged_state
@@ -41,7 +43,10 @@ pub fn add_piece(
             .map_err(|err| err.into())
             .and_then(|num_bytes_written| {
                 if num_bytes_written != piece_bytes_len {
-                    Err(err_inc_write(num_bytes_written, piece_bytes_len).into())
+                    Err(
+                        err_inc_write(u64::from(num_bytes_written), u64::from(piece_bytes_len))
+                            .into(),
+                    )
                 } else {
                     Ok(s.sector_id)
                 }
@@ -63,11 +68,11 @@ pub fn add_piece(
 // first staged sector into which the bytes will fit.
 fn compute_destination_sector_id(
     candidate_sectors: &[StagedSectorMetadata],
-    max_bytes_per_sector: u64,
-    num_bytes_in_piece: u64,
+    max_bytes_per_sector: UnpaddedBytesAmount,
+    num_bytes_in_piece: UnpaddedBytesAmount,
 ) -> error::Result<Option<SectorId>> {
     if num_bytes_in_piece > max_bytes_per_sector {
-        Err(err_overflow(num_bytes_in_piece, max_bytes_per_sector).into())
+        Err(err_overflow(num_bytes_in_piece.into(), max_bytes_per_sector.into()).into())
     } else {
         Ok(candidate_sectors
             .iter()
@@ -116,47 +121,63 @@ mod tests {
 
         sealed_sector_a.pieces.push(PieceMetadata {
             piece_key: String::from("x"),
-            num_bytes: 5,
+            num_bytes: UnpaddedBytesAmount(5),
         });
 
         sealed_sector_a.pieces.push(PieceMetadata {
             piece_key: String::from("x"),
-            num_bytes: 10,
+            num_bytes: UnpaddedBytesAmount(10),
         });
 
         let mut sealed_sector_b: StagedSectorMetadata = Default::default();
 
         sealed_sector_b.pieces.push(PieceMetadata {
             piece_key: String::from("x"),
-            num_bytes: 5,
+            num_bytes: UnpaddedBytesAmount(5),
         });
 
         let staged_sectors = vec![sealed_sector_a.clone(), sealed_sector_b.clone()];
 
         // piece takes up all remaining space in first sector
-        match compute_destination_sector_id(&staged_sectors, 100, 85) {
+        match compute_destination_sector_id(
+            &staged_sectors,
+            UnpaddedBytesAmount(100),
+            UnpaddedBytesAmount(85),
+        ) {
             Ok(Some(destination_sector_id)) => {
                 assert_eq!(destination_sector_id, sealed_sector_a.sector_id)
             }
             _ => panic!(),
         }
 
         // piece doesn't fit into the first, but does the second
-        match compute_destination_sector_id(&staged_sectors, 100, 90) {
+        match compute_destination_sector_id(
+            &staged_sectors,
+            UnpaddedBytesAmount(100),
+            UnpaddedBytesAmount(90),
+        ) {
             Ok(Some(destination_sector_id)) => {
                 assert_eq!(destination_sector_id, sealed_sector_b.sector_id)
             }
             _ => panic!(),
         }
 
         // piece doesn't fit into any in the list
-        match compute_destination_sector_id(&staged_sectors, 100, 100) {
+        match compute_destination_sector_id(
+            &staged_sectors,
+            UnpaddedBytesAmount(100),
+            UnpaddedBytesAmount(100),
+        ) {
             Ok(None) => (),
             _ => panic!(),
         }
 
         // piece is over max
-        match compute_destination_sector_id(&staged_sectors, 100, 101) {
+        match compute_destination_sector_id(
+            &staged_sectors,
+            UnpaddedBytesAmount(100),
+            UnpaddedBytesAmount(101),
+        ) {
             Err(_) => (),
             _ => panic!(),
         }

filecoin-proofs/src/api/sector_builder/helpers/add_piece.rs

@@ -4,11 +4,12 @@ use crate::api::sector_builder::metadata::StagedSectorMetadata;
 use crate::api::sector_builder::state::StagedState;
 use crate::api::sector_builder::SectorId;
 use itertools::chain;
+use sector_base::api::bytes_amount::UnpaddedBytesAmount;
 use std::cmp::Reverse;
 
 pub fn get_sectors_ready_for_sealing(
     staged_state: &StagedState,
-    max_user_bytes_per_staged_sector: u64,
+    max_user_bytes_per_staged_sector: UnpaddedBytesAmount,
     max_num_staged_sectors: u8,
     seal_all_staged_sectors: bool,
 ) -> Vec<SectorId> {
@@ -59,7 +60,7 @@ mod tests {
                 sector_id,
                 pieces: vec![PieceMetadata {
                     piece_key: format!("{}", sector_id),
-                    num_bytes,
+                    num_bytes: UnpaddedBytesAmount(num_bytes),
                 }],
                 seal_status,
                 ..Default::default()
@@ -79,9 +80,10 @@ mod tests {
             sectors: m,
         };
 
-        let to_seal: Vec<SectorId> = get_sectors_ready_for_sealing(&state, 127, 10, true)
-            .into_iter()
-            .collect();
+        let to_seal: Vec<SectorId> =
+            get_sectors_ready_for_sealing(&state, UnpaddedBytesAmount(127), 10, true)
+                .into_iter()
+                .collect();
 
         assert_eq!(vec![201 as SectorId, 200 as SectorId], to_seal);
     }
@@ -98,9 +100,10 @@ mod tests {
             sectors: m,
         };
 
-        let to_seal: Vec<SectorId> = get_sectors_ready_for_sealing(&state, 127, 10, false)
-            .into_iter()
-            .collect();
+        let to_seal: Vec<SectorId> =
+            get_sectors_ready_for_sealing(&state, UnpaddedBytesAmount(127), 10, false)
+                .into_iter()
+                .collect();
 
         assert_eq!(vec![200 as SectorId], to_seal);
     }
@@ -119,9 +122,10 @@ mod tests {
             sectors: m,
         };
 
-        let to_seal: Vec<SectorId> = get_sectors_ready_for_sealing(&state, 127, 2, false)
-            .into_iter()
-            .collect();
+        let to_seal: Vec<SectorId> =
+            get_sectors_ready_for_sealing(&state, UnpaddedBytesAmount(127), 2, false)
+                .into_iter()
+                .collect();
 
         assert_eq!(vec![201 as SectorId, 200 as SectorId], to_seal);
     }
@@ -140,9 +144,10 @@ mod tests {
             sectors: m,
         };
 
-        let to_seal: Vec<SectorId> = get_sectors_ready_for_sealing(&state, 127, 4, false)
-            .into_iter()
-            .collect();
+        let to_seal: Vec<SectorId> =
+            get_sectors_ready_for_sealing(&state, UnpaddedBytesAmount(127), 4, false)
+                .into_iter()
+                .collect();
 
         assert_eq!(vec![0; 0], to_seal);
     }
@@ -161,9 +166,10 @@ mod tests {
             sectors: m,
         };
 
-        let to_seal: Vec<SectorId> = get_sectors_ready_for_sealing(&state, 127, 4, false)
-            .into_iter()
-            .collect();
+        let to_seal: Vec<SectorId> =
+            get_sectors_ready_for_sealing(&state, UnpaddedBytesAmount(127), 4, false)
+                .into_iter()
+                .collect();
 
         assert_eq!(vec![0; 0], to_seal);
     }

filecoin-proofs/src/api/sector_builder/helpers/get_sectors_ready_for_sealing.rs

@@ -4,6 +4,7 @@ use crate::api::sector_builder::metadata::sector_id_as_bytes;
 use crate::api::sector_builder::metadata::SealedSectorMetadata;
 use crate::api::sector_builder::WrappedSectorStore;
 use crate::error;
+use sector_base::api::bytes_amount::UnpaddedBytesAmount;
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -47,7 +48,7 @@ fn retrieve_piece_aux<'a>(
     prover_id: &[u8; 31],
     piece_key: &'a str,
     staging_sector_access: &'a str,
-) -> error::Result<(u64, Vec<u8>)> {
+) -> error::Result<(UnpaddedBytesAmount, Vec<u8>)> {
     let (start_offset, num_bytes) = piece_pos(&sealed_sector, piece_key).ok_or_else(|| {
         let msg = format!(
             "piece {} not found in sector {}",
@@ -69,7 +70,8 @@ fn retrieve_piece_aux<'a>(
     if num_bytes_unsealed != num_bytes {
         let s = format!(
             "expected to unseal {} bytes, but unsealed {} bytes",
-            num_bytes, num_bytes_unsealed
+            u64::from(num_bytes),
+            u64::from(num_bytes_unsealed)
         );
 
         return Err(err_unrecov(s).into());
@@ -86,16 +88,23 @@ fn retrieve_piece_aux<'a>(
 
 // Returns a tuple of piece bytes-offset and number-of-bytes in piece if the
 // provided sealed sector contains a matching piece.
-fn piece_pos(sealed_sector: &SealedSectorMetadata, piece_key: &str) -> Option<(u64, u64)> {
+fn piece_pos(
+    sealed_sector: &SealedSectorMetadata,
+    piece_key: &str,
+) -> Option<(u64, UnpaddedBytesAmount)> {
     let (found_piece, start_offset, num_bytes) = sealed_sector.pieces.iter().fold(
-        (false, 0, 0),
+        (false, 0, UnpaddedBytesAmount(0)),
         |(eject, start_offset, num_bytes), item| {
             if eject {
                 (eject, start_offset, num_bytes)
             } else if item.piece_key == piece_key {
                 (true, start_offset, item.num_bytes)
             } else {
-                (false, start_offset + item.num_bytes, item.num_bytes)
+                (
+                    false,
+                    start_offset + u64::from(item.num_bytes),
+                    item.num_bytes,
+                )
             }
         },
     );
@@ -118,31 +127,31 @@ mod tests {
 
         sealed_sector.pieces.push(PieceMetadata {
             piece_key: String::from("x"),
-            num_bytes: 5,
+            num_bytes: UnpaddedBytesAmount(5),
         });
 
         sealed_sector.pieces.push(PieceMetadata {
             piece_key: String::from("y"),
-            num_bytes: 30,
+            num_bytes: UnpaddedBytesAmount(30),
         });
 
         sealed_sector.pieces.push(PieceMetadata {
             piece_key: String::from("z"),
-            num_bytes: 100,
+            num_bytes: UnpaddedBytesAmount(100),
         });
 
         match piece_pos(&sealed_sector, "x") {
-            Some(pair) => assert_eq!(pair, (0, 5)),
+            Some(pair) => assert_eq!(pair, (0, UnpaddedBytesAmount(5))),
             None => panic!(),
         }
 
         match piece_pos(&sealed_sector, "y") {
-            Some(pair) => assert_eq!(pair, (5, 30)),
+            Some(pair) => assert_eq!(pair, (5, UnpaddedBytesAmount(30))),
             None => panic!(),
         }
 
         match piece_pos(&sealed_sector, "z") {
-            Some(pair) => assert_eq!(pair, (35, 100)),
+            Some(pair) => assert_eq!(pair, (35, UnpaddedBytesAmount(100))),
             None => panic!(),
         }
     }

filecoin-proofs/src/api/sector_builder/helpers/retrieve_piece.rs

@@ -3,6 +3,7 @@ use crate::error;
 use crate::serde_big_array::BigArray;
 use byteorder::LittleEndian;
 use byteorder::WriteBytesExt;
+use sector_base::api::bytes_amount::UnpaddedBytesAmount;
 use serde::{Deserialize, Serialize};
 use std::fmt;
 
@@ -30,7 +31,7 @@ pub struct SealedSectorMetadata {
 #[derive(Clone, Serialize, Deserialize, Debug, PartialEq)]
 pub struct PieceMetadata {
     pub piece_key: String,
-    pub num_bytes: u64,
+    pub num_bytes: UnpaddedBytesAmount,
 }
 
 #[derive(Clone, Serialize, Deserialize, Debug, PartialEq)]
@@ -84,8 +85,10 @@ impl Default for SealedSectorMetadata {
     }
 }
 
-pub fn sum_piece_bytes(s: &StagedSectorMetadata) -> u64 {
-    s.pieces.iter().map(|x| x.num_bytes).sum()
+pub fn sum_piece_bytes(s: &StagedSectorMetadata) -> UnpaddedBytesAmount {
+    s.pieces
+        .iter()
+        .fold(UnpaddedBytesAmount(0), |acc, x| acc + x.num_bytes)
 }
 
 pub fn sector_id_as_bytes(sector_id: SectorId) -> error::Result<[u8; 31]> {