Skip to content

Commit 6919d40

Browse files
guillaumemichelguillaumemichel
committed
Buffered SweepingProvider wrapper
1 parent 20e4570 commit 6919d40

File tree

5 files changed

+21
-4
lines changed

5 files changed

+21
-4
lines changed

‎core/node/provider.go‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/libp2p/go-libp2p-kad-dht/fullrt"
2525
dht_pb "github.com/libp2p/go-libp2p-kad-dht/pb"
2626
dhtprovider "github.com/libp2p/go-libp2p-kad-dht/provider"
27+
"github.com/libp2p/go-libp2p-kad-dht/provider/buffered"
2728
rds "github.com/libp2p/go-libp2p-kad-dht/provider/datastore"
2829
ddhtprovider "github.com/libp2p/go-libp2p-kad-dht/provider/dual"
2930
routinghelpers "github.com/libp2p/go-libp2p-routing-helpers"
@@ -303,14 +304,22 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
303304
Repo repo.Repo
304305
}
305306
sweepingReprovider := fx.Provide(func(in providerInput) (DHTProvider, *rds.ResettableKeyStore, error) {
306-
keyStore, err := rds.NewResettableKeyStore(in.Repo.Datastore(),
307+
ds := in.Repo.Datastore()
308+
keyStore, err := rds.NewResettableKeyStore(ds,
307309
rds.WithPrefixBits(16),
308310
rds.WithDatastorePrefix("/provider/keystore"),
309311
rds.WithBatchSize(int(cfg.Reprovider.Sweep.KeyStoreBatchSize.WithDefault(config.DefaultReproviderSweepKeyStoreBatchSize))),
310312
)
311313
if err != nil {
312314
return &NoopProvider{}, nil, err
313315
}
316+
317+
bufferedProviderOpts := []buffered.Option{
318+
buffered.WithBatchSize(1 << 10),
319+
buffered.WithDsName("bprov"),
320+
buffered.WithIdleWriteTime(time.Minute),
321+
}
322+
314323
var impl dhtImpl
315324
switch inDht := in.DHT.(type) {
316325
case *dht.IpfsDHT:
@@ -335,8 +344,7 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
335344
if err != nil {
336345
return nil, nil, err
337346
}
338-
_ = prov
339-
return prov, keyStore, nil
347+
return buffered.New(prov, ds, bufferedProviderOpts...), keyStore, nil
340348
}
341349
case *fullrt.FullRT:
342350
if inDht != nil {
@@ -376,7 +384,10 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
376384
}
377385

378386
prov, err := dhtprovider.New(opts...)
379-
return prov, keyStore, err
387+
if err != nil {
388+
return &NoopProvider{}, nil, err
389+
}
390+
return buffered.New(prov, ds, bufferedProviderOpts...), keyStore, nil
380391
})
381392

382393
type keystoreInput struct {

‎docs/examples/kubo-as-a-library/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ require (
8282
github.com/ipfs/go-ds-leveldb v0.5.2 // indirect
8383
github.com/ipfs/go-ds-measure v0.2.2 // indirect
8484
github.com/ipfs/go-ds-pebble v0.5.1 // indirect
85+
github.com/ipfs/go-dsqueue v0.0.4 // indirect
8586
github.com/ipfs/go-fs-lock v0.1.1 // indirect
8687
github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect
8788
github.com/ipfs/go-ipfs-pq v0.0.3 // indirect

‎docs/examples/kubo-as-a-library/go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,8 @@ github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICk
321321
github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs=
322322
github.com/ipfs/go-ds-pebble v0.5.1 h1:p0FAE0zw9J/3T1VkGB9s98jWmfKmw2t0iEwfMUv8iSQ=
323323
github.com/ipfs/go-ds-pebble v0.5.1/go.mod h1:LsmQx4w+0o9znl4hTxYo1Y2lnBTzNCwc4kNpD3wWXM0=
324+
github.com/ipfs/go-dsqueue v0.0.4 h1:tesq26hKRYPG72Tu9kZKsbsLWp1KBfAxWNQlMyU17tk=
325+
github.com/ipfs/go-dsqueue v0.0.4/go.mod h1:K68ng9BVl+gLr8fqCJKaoXnXqo6MzQ6nV0MhZZFEwg4=
324326
github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw=
325327
github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0=
326328
github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ require (
147147
github.com/huin/goupnp v1.3.0 // indirect
148148
github.com/ipfs/bbloom v0.0.4 // indirect
149149
github.com/ipfs/go-bitfield v1.1.0 // indirect
150+
github.com/ipfs/go-dsqueue v0.0.4 // indirect
150151
github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect
151152
github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
152153
github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,8 @@ github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICk
388388
github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs=
389389
github.com/ipfs/go-ds-pebble v0.5.1 h1:p0FAE0zw9J/3T1VkGB9s98jWmfKmw2t0iEwfMUv8iSQ=
390390
github.com/ipfs/go-ds-pebble v0.5.1/go.mod h1:LsmQx4w+0o9znl4hTxYo1Y2lnBTzNCwc4kNpD3wWXM0=
391+
github.com/ipfs/go-dsqueue v0.0.4 h1:tesq26hKRYPG72Tu9kZKsbsLWp1KBfAxWNQlMyU17tk=
392+
github.com/ipfs/go-dsqueue v0.0.4/go.mod h1:K68ng9BVl+gLr8fqCJKaoXnXqo6MzQ6nV0MhZZFEwg4=
391393
github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw=
392394
github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0=
393395
github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=

0 commit comments

Comments
 (0)