Skip to content

Commit 10cd50a

Browse files
authored
feat: increase routing limits (#154)
* feat: increase routing limits * bitswap_setup: Update boxo and call pqm.Startup * Update changelog for increased routing limits
1 parent 9b0b7f4 commit 10cd50a

File tree

5 files changed

+44
-19
lines changed

5 files changed

+44
-19
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ The following emojis are used to highlight certain changes:
2222
- boxo [v0.24.3](https://github.com/ipfs/boxo/releases/tag/v0.24.3)
2323
- go-libp2p-kad-dht [v0.28.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.28.1)
2424
- passing headers that require authorization but are not authorized now results in an HTTP 401 instead of ignoring those headers
25+
- Bitswap settings: Increased default content-discovery limits, with up to 100 in-flight requests.
2526

2627
### Removed
2728

go.mod

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ require (
1010
github.com/felixge/httpsnoop v1.0.4
1111
github.com/ipfs-shipyard/nopfs v0.0.12
1212
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a
13-
github.com/ipfs/boxo v0.24.3
13+
github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36
1414
github.com/ipfs/go-block-format v0.2.0
1515
github.com/ipfs/go-cid v0.4.1
1616
github.com/ipfs/go-datastore v0.6.0
@@ -78,6 +78,8 @@ require (
7878
github.com/francoispqt/gojay v1.2.13 // indirect
7979
github.com/fsnotify/fsnotify v1.7.0 // indirect
8080
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
81+
github.com/gammazero/chanqueue v1.0.0 // indirect
82+
github.com/gammazero/deque v1.0.0 // indirect
8183
github.com/getsentry/sentry-go v0.27.0 // indirect
8284
github.com/go-logr/logr v1.4.2 // indirect
8385
github.com/go-logr/stdr v1.2.2 // indirect

go.sum

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,10 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
130130
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
131131
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
132132
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
133+
github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
134+
github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
135+
github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
136+
github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
133137
github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
134138
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
135139
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
@@ -256,8 +260,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG
256260
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
257261
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
258262
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
259-
github.com/ipfs/boxo v0.24.3 h1:gldDPOWdM3Rz0v5LkVLtZu7A7gFNvAlWcmxhCqlHR3c=
260-
github.com/ipfs/boxo v0.24.3/go.mod h1:h0DRzOY1IBFDHp6KNvrJLMFdSXTYID0Zf+q7X05JsNg=
263+
github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36 h1:/el1Poab8nY9SBAWxjGEPCHOlRfbuxm39Ewcj7b6w+0=
264+
github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36/go.mod h1:lAoydO+oJhB1e7pUn4ju1Z1fuUIwy+zb0hQXRb/bu2g=
261265
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
262266
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
263267
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
@@ -658,6 +662,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED
658662
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
659663
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
660664
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
665+
github.com/slok/go-http-metrics v0.12.0 h1:mAb7hrX4gB4ItU6NkFoKYdBslafg3o60/HbGBRsKaG8=
666+
github.com/slok/go-http-metrics v0.12.0/go.mod h1:Ee/mdT9BYvGrlGzlClkK05pP2hRHmVbRF9dtUVS8LNA=
661667
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
662668
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
663669
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=

handler_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func TestNoBlockcacheHeader(t *testing.T) {
111111
ctx := context.Background()
112112
// pacify metrics reporting code
113113
ctx = metrics.CtxScope(ctx, "test.bsserver.host")
114-
n := bsnet.NewFromIpfsHost(newHost, nil)
114+
n := bsnet.NewFromIpfsHost(newHost)
115115
bs := bsserver.New(ctx, n, gnd.blockstore)
116116
n.Start(bs)
117117
defer bs.Close()

setup_bitswap.go

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"context"
55
"time"
66

7+
"github.com/ipfs/boxo/routing/providerquerymanager"
8+
79
"github.com/ipfs/boxo/bitswap"
810
bsclient "github.com/ipfs/boxo/bitswap/client"
911
bsnet "github.com/ipfs/boxo/bitswap/network"
@@ -21,7 +23,16 @@ import (
2123

2224
func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routing.ContentRouting, bstore blockstore.Blockstore) exchange.Interface {
2325
bsctx := metri.CtxScope(ctx, "ipfs_bitswap")
24-
bn := bsnet.NewFromIpfsHost(h, cr)
26+
27+
bn := bsnet.NewFromIpfsHost(h)
28+
29+
// Custom query manager with the content router and the host
30+
// and our custom options to overwrite the default.
31+
pqm, err := providerquerymanager.New(ctx, h, cr, providerquerymanager.WithMaxInProcessRequests(100))
32+
if err != nil {
33+
panic(err)
34+
}
35+
pqm.Startup()
2536

2637
// --- Client Options
2738
// bitswap.RebroadcastDelay: default is 1 minute to search for a random
@@ -31,6 +42,14 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi
3142
// bitswap.ProviderSearchDelay: default is 1 second.
3243
providerSearchDelay := 1 * time.Second
3344

45+
// --- Bitswap Client Options
46+
clientOpts := []bsclient.Option{
47+
bsclient.RebroadcastDelay(rebroadcastDelay),
48+
bsclient.ProviderSearchDelay(providerSearchDelay),
49+
bsclient.WithoutDuplicatedBlockStats(),
50+
bsclient.WithDefaultProviderQueryManager(false), // we pass it in manually
51+
}
52+
3453
// If peering and shared cache are both enabled, we initialize both a
3554
// Client and a Server with custom request filter and custom options.
3655
// client+server is more expensive but necessary when deployment requires
@@ -48,31 +67,28 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi
4867
return ok
4968
}
5069

51-
// Initialize client+server
52-
bswap := bitswap.New(bsctx, bn, bstore,
53-
// --- Client Options
54-
bitswap.RebroadcastDelay(rebroadcastDelay),
55-
bitswap.ProviderSearchDelay(providerSearchDelay),
56-
bitswap.WithoutDuplicatedBlockStats(),
70+
// turn bitswap clients option into bitswap options
71+
var opts []bitswap.Option
72+
for _, o := range clientOpts {
73+
opts = append(opts, bitswap.WithClientOption(o))
74+
}
5775

58-
// ---- Server Options
76+
// ---- Server Options
77+
opts = append(opts,
5978
bitswap.WithPeerBlockRequestFilter(peerBlockRequestFilter),
60-
bitswap.ProvideEnabled(false),
6179
// When we don't have a block, don't reply. This reduces processment.
6280
bitswap.SetSendDontHaves(false),
6381
bitswap.WithWantHaveReplaceSize(cfg.BitswapWantHaveReplaceSize),
6482
)
83+
84+
// Initialize client+server
85+
bswap := bitswap.New(bsctx, bn, pqm, bstore, opts...)
6586
bn.Start(bswap)
6687
return &noNotifyExchange{bswap}
6788
}
6889

6990
// By default, rainbow runs with bitswap client alone
70-
bswap := bsclient.New(bsctx, bn, bstore,
71-
// --- Client Options
72-
bsclient.RebroadcastDelay(rebroadcastDelay),
73-
bsclient.ProviderSearchDelay(providerSearchDelay),
74-
bsclient.WithoutDuplicatedBlockStats(),
75-
)
91+
bswap := bsclient.New(bsctx, bn, pqm, bstore, clientOpts...)
7692
bn.Start(bswap)
7793
return bswap
7894
}

0 commit comments

Comments
 (0)