Skip to content

Commit 568acce

Browse files
feat: switch to stacked drg
2 parents 64ece87 + bfca8d6 commit 568acce

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+4762
-3435
lines changed

.circleci/config.yml

Lines changed: 33 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1+
version: 2.1
12

2-
version: 2
33
jobs:
44
cargo_fetch:
55
docker:
@@ -13,7 +13,7 @@ jobs:
1313
command: cargo generate-lockfile
1414
- restore_cache:
1515
keys:
16-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
16+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
1717
- run: rustup install $(cat rust-toolchain)
1818
- run: rustup default $(cat rust-toolchain)
1919
- run: rustup component add rustfmt-preview
@@ -27,7 +27,7 @@ jobs:
2727
paths:
2828
- Cargo.lock
2929
- save_cache:
30-
key: cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
30+
key: cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
3131
paths:
3232
- /root/.cargo
3333
- /root/.rustup
@@ -42,7 +42,7 @@ jobs:
4242
at: "."
4343
- restore_cache:
4444
keys:
45-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
45+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
4646
- restore_cache:
4747
keys:
4848
- parameter-cache-{{ .Revision }}
@@ -72,7 +72,7 @@ jobs:
7272
at: "."
7373
- restore_cache:
7474
keys:
75-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
75+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
7676
- parameter-cache-{{ .Revision }}
7777
- run:
7878
name: Test (stable) in release profile
@@ -91,7 +91,7 @@ jobs:
9191
at: "."
9292
- restore_cache:
9393
keys:
94-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
94+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
9595
- parameter-cache-{{ .Revision }}
9696
- run:
9797
name: Test (nightly) mem-trees feature
@@ -105,20 +105,28 @@ jobs:
105105
- image: filecoin/rust:latest
106106
working_directory: /mnt/crate
107107
resource_class: xlarge
108+
parameters:
109+
crate:
110+
type: string
111+
features:
112+
type: string
113+
default: ""
108114
steps:
109115
- checkout
110116
- attach_workspace:
111117
at: "."
112118
- restore_cache:
113119
keys:
114-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
120+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
115121
- parameter-cache-{{ .Revision }}
116122
- run:
117-
name: Test (stable) in release profile
123+
name: Test ignored in release profile
118124
command: |
119-
# We must change directories into `storage-proofs` for the `unchecked-degrees` feature to take effect.
120-
cd storage-proofs
121-
cargo +stable test --verbose --release --all --features unchecked-degrees -- --ignored
125+
cd << parameters.crate >>
126+
cargo test --release << parameters.features >> -- --ignored
127+
environment:
128+
RUST_TEST_THREADS: 1
129+
no_output_timeout: 30m
122130
- save_cache:
123131
key: parameter-cache-{{ .Revision }}
124132
paths:
@@ -135,7 +143,7 @@ jobs:
135143
at: "."
136144
- restore_cache:
137145
keys:
138-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
146+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
139147
- restore_cache:
140148
keys:
141149
- parameter-cache-{{ .Revision }}
@@ -155,7 +163,7 @@ jobs:
155163
at: "."
156164
- restore_cache:
157165
keys:
158-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
166+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
159167
- restore_cache:
160168
keys:
161169
- parameter-cache-{{ .Revision }}
@@ -189,7 +197,7 @@ jobs:
189197
at: "."
190198
- restore_cache:
191199
keys:
192-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
200+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
193201
- restore_cache:
194202
keys:
195203
- parameter-cache-{{ .Revision }}
@@ -245,7 +253,7 @@ jobs:
245253
at: "."
246254
- restore_cache:
247255
keys:
248-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
256+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
249257
- run:
250258
name: Run cargo fmt
251259
command: cargo fmt --all -- --check
@@ -261,7 +269,7 @@ jobs:
261269
at: "."
262270
- restore_cache:
263271
keys:
264-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
272+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
265273
- run:
266274
name: Run cargo clippy
267275
command: cargo clippy --all
@@ -277,7 +285,7 @@ jobs:
277285
at: "."
278286
- restore_cache:
279287
keys:
280-
- cargo-v12-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
288+
- cargo-v13-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
281289
- run:
282290
name: Build (release)
283291
command: cargo +$(cat rust-toolchain) build --release --verbose --all
@@ -356,6 +364,14 @@ workflows:
356364
requires:
357365
- cargo_fetch
358366
- test_ignored_release:
367+
name: test_ignored_release_storage_proofs
368+
crate: "storage-proofs"
369+
features: "--features unchecked-degrees"
370+
requires:
371+
- cargo_fetch
372+
- test_ignored_release:
373+
name: test_ignored_release_filecoin_proofs
374+
crate: "filecoin-proofs"
359375
requires:
360376
- cargo_fetch
361377
- test:

Dockerfile-profile

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ ENV LIBUNWIND_VERSION="0.99-beta"
2727
ENV HOME="/root"
2828
ENV DOWNLOADS=${HOME}/downloads
2929
RUN mkdir -p ${DOWNLOADS}
30-
RUN echo ${DOWNLOADS}
30+
RUN echo ${DOWNLOADS}
3131
WORKDIR ${DOWNLOADS}
3232

3333
RUN wget http://download.savannah.gnu.org/releases/libunwind/libunwind-${LIBUNWIND_VERSION}.tar.gz --output-document ${DOWNLOADS}/libunwind-${LIBUNWIND_VERSION}.tar.gz
@@ -68,14 +68,14 @@ cargo update \
6868
cargo build \
6969
-p filecoin-proofs \
7070
--release \
71-
--example zigzag \
71+
--example stacked \
7272
--features \
7373
cpu-profile \
7474
-Z package-features \
7575
&& \
7676
RUST_BACKTRACE=full \
7777
RUST_LOG=trace \
78-
target/release/examples/zigzag \
78+
target/release/examples/stacked \
7979
--size 1024 \
8080
&& \
81-
pprof target/release/examples/zigzag replicate.profile || bash
81+
pprof target/release/examples/stacked replicate.profile || bash

README.md

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ There are currently four different crates:
1212
Primary Components:
1313
- **PoR** (**_Proof-of-Retrievability_**: Merkle inclusion proof)
1414
- **DrgPoRep** (_Depth Robust Graph_ **_Proof-of-Replication_**)
15-
- **ZigZagDrgPoRep** (implemented as a specialized **LayeredDrgPoRep**)
15+
- **StackedDrgPoRep**
1616
- **PoSt** (Proof-of-Spacetime)
1717

1818

@@ -94,7 +94,7 @@ Note: On macOS you need `gtime` (`brew install gnu-time`), as the built in `time
9494

9595
## Profiling
9696

97-
For development purposes we have an (experimental) support for CPU and memory profiling in Rust through a [`gperftools`](https://github.com/dignifiedquire/rust-gperftools) binding library. These can be enabled though the `cpu-profile` and `heap-profile` features in `filecoin-proofs`. An example setup can be found in this [`Dockerfile`](./Dockerfile-profile) to profile CPU usage for the [`zigzag`](https://github.com/filecoin-project/rust-fil-proofs/blob/e6fa4232404641bb4be1ffc42944d2e734ab9748/filecoin-proofs/examples/zigzag.rs#L40-L61) example.
97+
For development purposes we have an (experimental) support for CPU and memory profiling in Rust through a [`gperftools`](https://github.com/dignifiedquire/rust-gperftools) binding library. These can be enabled though the `cpu-profile` and `heap-profile` features in `filecoin-proofs`. An example setup can be found in this [`Dockerfile`](./Dockerfile-profile) to profile CPU usage for the [`stacked`](https://github.com/filecoin-project/rust-fil-proofs/blob/master/filecoin-proofs/examples/stacked.rs#L40-L61) example.
9898

9999
## Logging
100100

@@ -151,7 +151,7 @@ While replicating and generating the Merkle Trees (MT) for the proof at the same
151151

152152
### Speed
153153

154-
One of the most computational expensive operations during replication (besides the encoding itself) is the generation of the indexes of the (expansion) parents in the ZigZag graph, implemented through a Feistel cipher (used as a pseudorandom permutation). To reduce that time we provide a caching mechanism to generate them only once and reuse them throughout replication (across the different layers). Already built into the system it can be activated with the environmental variable
154+
One of the most computational expensive operations during replication (besides the encoding itself) is the generation of the indexes of the (expansion) parents in the Stacked graph, implemented through a Feistel cipher (used as a pseudorandom permutation). To reduce that time we provide a caching mechanism to generate them only once and reuse them throughout replication (across the different layers). Already built into the system it can be activated with the environmental variable
155155

156156
```
157157
FIL_PROOFS_MAXIMIZE_CACHING=1
@@ -161,15 +161,15 @@ To check that it's working you can inspect the replication log to find `using pa
161161

162162
(You can also verify if the cache is working by inspecting the time each layer takes to encode, `encoding, layer:` in the log, where the first two layers, forward and reverse, will take more time than the rest to populate the cache while the remaining 8 should see a considerable time drop.)
163163

164-
In the most extreme case, to reduce time at the cost of *a lot* of memory consumption you can turn on the feature that stores MTs on memory (`mem-trees`) instead of on disk (the default) to generate them all on RAM and avoid disk I/O (if the HW doesn't have enough RAM to handle the MTs, roughly 20x the sector size, this won't have the desired effect as the OS will start backing them on disk anyway). For example, to run the `zigzag` example with this feature turned on you'd need to indicate so to `cargo`,
164+
In the most extreme case, to reduce time at the cost of *a lot* of memory consumption you can turn on the feature that stores MTs on memory (`mem-trees`) instead of on disk (the default) to generate them all on RAM and avoid disk I/O (if the HW doesn't have enough RAM to handle the MTs, roughly 20x the sector size, this won't have the desired effect as the OS will start backing them on disk anyway). For example, to run the `stacked` example with this feature turned on you'd need to indicate so to `cargo`,
165165

166166
```
167167
# NEEDS TO BE RUN INSIDE `storage-proofs\` directory
168168
# for the `--features` to take effect.
169169
cargo run \
170170
-p filecoin-proofs \
171171
--release \
172-
--example zigzag \
172+
--example stacked \
173173
--features \
174174
mem-trees \
175175
-- \
@@ -188,7 +188,7 @@ The following benchmarks were observed when running replication on 1MiB (1024 ki
188188

189189
```
190190
$ cargo build --bin benchy --release
191-
$ env time -v cargo run --bin benchy --release -- zigzag --size=1024
191+
$ env time -v cargo run --bin benchy --release -- stacked --size=1024
192192
193193
window-size: 16
194194
User time (seconds): 87.82

fil-proofs-tooling/README.md

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@
22

33
This crate contains the following binaries
44

5-
- `benchy` - Can be used to capture ZigZag performance metrics
5+
- `benchy` - Can be used to capture Stacked performance metrics
66
- `micro` - Runs the micro benchmarks written with criterion, parses the output.
77

88
## `benchy`
99

10-
The `benchy` program can (currently) be used to capture ZigZag performance
10+
The `benchy` program can (currently) be used to capture Stacked performance
1111
metrics. Metrics are printed to stdout.
1212

1313
```
14-
$ ./target/release/benchy zigzag --size=1024 | jq '.'
14+
$ ./target/release/benchy stacked --size=1024 | jq '.'
1515
{
1616
"inputs": {
1717
"dataSize": 1048576,
@@ -46,11 +46,11 @@ $ ./target/release/benchy zigzag --size=1024 | jq '.'
4646
}
4747
```
4848

49-
To include information about RAM utilization during ZigZag benchmarking, run
49+
To include information about RAM utilization during Stacked benchmarking, run
5050
`benchy` via its wrapper script:
5151

5252
```
53-
$ ./scripts/benchy.sh zigzag --size=1024 | jq '.'
53+
$ ./scripts/benchy.sh stacked --size=1024 | jq '.'
5454
{
5555
"inputs": {
5656
"dataSize": 1048576,
@@ -90,7 +90,7 @@ To run benchy on a remote server, provide SSH connection information to the
9090
benchy-remote.sh script:
9191

9292
```shell
93-
10:13 $ ./fil-proofs-tooling/scripts/benchy-remote.sh master [email protected] zigzag --size=1 | jq '.'
93+
10:13 $ ./fil-proofs-tooling/scripts/benchy-remote.sh master [email protected] stacked --size=1 | jq '.'
9494
{
9595
"inputs": {
9696
// ...

fil-proofs-tooling/scripts/aggregate-benchmarks.sh

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@
22

33
set -e
44

5-
zigzag_path=$1
5+
stacked_path=$1
66
micro_path=$2
77
hash_constraints_path=$3
88
rational_post_path=$4
99

10-
jq --sort-keys -s '{ benchmarks: { "zigzag-benchmarks": { outputs: { "max-resident-set-size-kb": .[0] } } } } * .[1]' \
11-
<(jq '.["max-resident-set-size-kb"]' $zigzag_path) \
12-
<(jq -s '.[0] * { benchmarks: { "hash-constraints": .[1], "zigzag-benchmarks": .[2], "micro-benchmarks": .[3], "rational-post-benchmarks": .[4] } }' \
10+
jq --sort-keys -s '{ benchmarks: { "stacked-benchmarks": { outputs: { "max-resident-set-size-kb": .[0] } } } } * .[1]' \
11+
<(jq '.["max-resident-set-size-kb"]' $stacked_path) \
12+
<(jq -s '.[0] * { benchmarks: { "hash-constraints": .[1], "stacked-benchmarks": .[2], "micro-benchmarks": .[3], "rational-post-benchmarks": .[4] } }' \
1313
<(jq 'del (.benchmarks)' $micro_path) \
1414
<(jq '.benchmarks' $hash_constraints_path) \
15-
<(jq '.benchmarks' $zigzag_path) \
15+
<(jq '.benchmarks' $stacked_path) \
1616
<(jq '.benchmarks' $micro_path) \
1717
<(jq '.benchmarks' $rational_post_path))

fil-proofs-tooling/src/bin/benchy/main.rs

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,13 @@ use clap::{value_t, App, Arg, SubCommand};
55

66
mod hash_fns;
77
mod rational_post;
8-
mod zigzag;
8+
mod stacked;
99

1010
fn main() {
1111
pretty_env_logger::init_timed();
1212

13-
let zigzag_cmd = SubCommand::with_name("zigzag")
14-
.about("Run zigzag sealing")
13+
let stacked_cmd = SubCommand::with_name("stacked")
14+
.about("Run stacked sealing")
1515
.arg(
1616
Arg::with_name("size")
1717
.required(true)
@@ -87,18 +87,6 @@ fn main() {
8787
Arg::with_name("extract")
8888
.long("extract")
8989
.help("Extract data after proving and verifying.")
90-
)
91-
.arg(
92-
Arg::with_name("taper")
93-
.long("taper")
94-
.help("fraction of challenges by which to taper at each layer")
95-
.default_value("0.0")
96-
)
97-
.arg(
98-
Arg::with_name("taper-layers")
99-
.long("taper-layers")
100-
.help("number of layers to taper")
101-
.takes_value(true)
10290
);
10391

10492
let rational_post_cmd = SubCommand::with_name("rational-post")
@@ -116,17 +104,17 @@ fn main() {
116104

117105
let matches = App::new("benchy")
118106
.version("0.1")
119-
.subcommand(zigzag_cmd)
107+
.subcommand(stacked_cmd)
120108
.subcommand(rational_post_cmd)
121109
.subcommand(hash_cmd)
122110
.get_matches();
123111

124112
match matches.subcommand() {
125-
("zigzag", Some(m)) => {
113+
("stacked", Some(m)) => {
126114
Ok(())
127115
.and_then(|_| {
128116
let layers = value_t!(m, "layers", usize)?;
129-
zigzag::run(zigzag::RunOpts {
117+
stacked::run(stacked::RunOpts {
130118
bench: m.is_present("bench"),
131119
bench_only: m.is_present("bench-only"),
132120
challenges: value_t!(m, "challenges", usize)?,
@@ -140,11 +128,9 @@ fn main() {
140128
no_tmp: m.is_present("no-tmp"),
141129
partitions: value_t!(m, "partitions", usize)?,
142130
size: value_t!(m, "size", usize)?,
143-
taper: value_t!(m, "taper", f64)?,
144-
taper_layers: value_t!(m, "taper-layers", usize).unwrap_or(layers),
145131
})
146132
})
147-
.expect("zigzag failed");
133+
.expect("stacked failed");
148134
}
149135
("rational-post", Some(m)) => {
150136
let sector_size_kibs = value_t!(m, "size", usize)

0 commit comments

Comments
 (0)