merge origin/master into feat/test-slog-interop

resolved conflicts in go.sum files by keeping slog integration version

Author: lidel

.github/workflows/gateway-conformance.yml

@@ -109,13 +109,13 @@ jobs:
         run: cat output.md >> $GITHUB_STEP_SUMMARY
       - name: Upload HTML report
         if: failure() || success()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: gateway-conformance.html
           path: output.html
       - name: Upload JSON report
         if: failure() || success()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: gateway-conformance.json
           path: output.json
@@ -214,13 +214,13 @@ jobs:
         run: cat output.md >> $GITHUB_STEP_SUMMARY
       - name: Upload HTML report
         if: failure() || success()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: gateway-conformance-libp2p.html
           path: output.html
       - name: Upload JSON report
         if: failure() || success()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: gateway-conformance-libp2p.json
           path: output.json

.github/workflows/gotest.yml

@@ -78,7 +78,7 @@ jobs:
           output: test/unit/gotest.junit.xml
         if: failure() || success()
       - name: Archive the JUnit XML report
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: unit
           path: test/unit/gotest.junit.xml
@@ -91,7 +91,7 @@ jobs:
           output: test/unit/gotest.html
         if: failure() || success()
       - name: Archive the HTML report
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: html
           path: test/unit/gotest.html

.github/workflows/interop.yml

@@ -37,7 +37,7 @@ jobs:
         with:
           go-version-file: 'go.mod'
       - run: make build
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         with:
           name: kubo
           path: cmd/ipfs/ipfs
@@ -49,10 +49,10 @@ jobs:
       run:
         shell: bash
     steps:
-      - uses: actions/setup-node@v5
+      - uses: actions/setup-node@v6
         with:
           node-version: lts/*
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@v6
         with:
           name: kubo
           path: cmd/ipfs
@@ -84,10 +84,10 @@ jobs:
       run:
         shell: bash
     steps:
-      - uses: actions/setup-node@v5
+      - uses: actions/setup-node@v6
         with:
           node-version: 20.x
-      - uses: actions/download-artifact@v5
+      - uses: actions/download-artifact@v6
         with:
           name: kubo
           path: cmd/ipfs

.github/workflows/sharness.yml

@@ -88,7 +88,7 @@ jobs:
           destination: sharness.html
       - name: Upload one-page HTML report
         if: github.repository != 'ipfs/kubo' && (failure() || success())
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: sharness.html
           path: kubo/test/sharness/test-results/sharness.html
@@ -108,7 +108,7 @@ jobs:
           destination: sharness-html/
       - name: Upload full HTML report
         if: github.repository != 'ipfs/kubo' && (failure() || success())
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: sharness-html
           path: kubo/test/sharness/test-results/sharness-html

.github/workflows/sync-release-assets.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: ipfs/start-ipfs-daemon-action@v1
         with:
           args: --init --init-profile=flatfs,server --enable-gc=false
-      - uses: actions/setup-node@v5
+      - uses: actions/setup-node@v6
         with:
           node-version: 14
       - name: Sync the latest 5 github releases

.github/workflows/test-migrations.yml

@@ -77,7 +77,7 @@ jobs:
 
       - name: Upload test results
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: ${{ matrix.os }}-test-results
           path: |

config/provide.go

@@ -16,6 +16,7 @@ const (
 	DefaultProvideDHTInterval                 = 22 * time.Hour // https://github.com/ipfs/kubo/pull/9326
 	DefaultProvideDHTMaxWorkers               = 16             // Unified default for both sweep and legacy providers
 	DefaultProvideDHTSweepEnabled             = false
+	DefaultProvideDHTResumeEnabled            = true
 	DefaultProvideDHTDedicatedPeriodicWorkers = 2
 	DefaultProvideDHTDedicatedBurstWorkers    = 1
 	DefaultProvideDHTMaxProvideConnsPerWorker = 20
@@ -86,6 +87,12 @@ type ProvideDHT struct {
 	// OfflineDelay sets the delay after which the provider switches from Disconnected to Offline state (sweep mode only).
 	// Default: DefaultProvideDHTOfflineDelay
 	OfflineDelay *OptionalDuration `json:",omitempty"`
+
+	// ResumeEnabled controls whether the provider resumes from its previous state on restart.
+	// When enabled, the provider persists its reprovide cycle state and provide queue to the datastore,
+	// and restores them on restart. When disabled, the provider starts fresh on each restart.
+	// Default: true
+	ResumeEnabled Flag `json:",omitempty"`
 }
 
 func ParseProvideStrategy(s string) ProvideStrategy {

core/commands/provide.go

@@ -348,6 +348,7 @@ NOTES:
 			}
 			sectionTitle := func(col int, title string) {
 				if !brief && showHeadings {
+					//nolint:govet // dynamic format string is intentional
 					formatLine(col, title+":")
 				}
 			}
@@ -372,8 +373,8 @@ NOTES:
 			// Queues
 			if all || queues || brief {
 				sectionTitle(1, "Queues")
-				formatLine(1, "%sProvide queue: %s CIDs, %s regions", indent, humanNumber(s.Sweep.Queues.PendingKeyProvides), humanNumber(s.Sweep.Queues.PendingRegionProvides))
-				formatLine(1, "%sReprovide queue: %s regions", indent, humanNumber(s.Sweep.Queues.PendingRegionReprovides))
+				formatLine(1, "%sProvide queue: %s CIDs, %s regions", indent, humanSI(s.Sweep.Queues.PendingKeyProvides, 1), humanSI(s.Sweep.Queues.PendingRegionProvides, 1))
+				formatLine(1, "%sReprovide queue: %s regions", indent, humanSI(s.Sweep.Queues.PendingRegionReprovides, 1))
 				addBlankLine(1)
 			}
 
@@ -413,12 +414,12 @@ NOTES:
 				sectionTitle(0, "Network")
 				formatLine(0, "%sAvg record holders: %s", indent, humanFloatOrNA(s.Sweep.Network.AvgHolders))
 				if !brief {
-					formatLine(0, "%sPeers swept: %s", indent, humanNumber(s.Sweep.Network.Peers))
+					formatLine(0, "%sPeers swept: %s", indent, humanInt(s.Sweep.Network.Peers))
 					formatLine(0, "%sFull keyspace coverage: %t", indent, s.Sweep.Network.CompleteKeyspaceCoverage)
 					if s.Sweep.Network.Peers > 0 {
-						formatLine(0, "%sReachable peers: %s (%s%%)", indent, humanNumber(s.Sweep.Network.Reachable), humanNumber(100*s.Sweep.Network.Reachable/s.Sweep.Network.Peers))
+						formatLine(0, "%sReachable peers: %s (%s%%)", indent, humanInt(s.Sweep.Network.Reachable), humanNumber(100*s.Sweep.Network.Reachable/s.Sweep.Network.Peers))
 					} else {
-						formatLine(0, "%sReachable peers: %s", indent, humanNumber(s.Sweep.Network.Reachable))
+						formatLine(0, "%sReachable peers: %s", indent, humanInt(s.Sweep.Network.Reachable))
 					}
 					formatLine(0, "%sAvg region size: %s", indent, humanFloatOrNA(s.Sweep.Network.AvgRegionSize))
 					formatLine(0, "%sReplication factor: %s", indent, humanNumber(s.Sweep.Network.ReplicationFactor))
@@ -430,15 +431,15 @@ NOTES:
 			if all || operations || brief {
 				sectionTitle(1, "Operations")
 				// Ongoing operations
-				formatLine(1, "%sOngoing provides: %s CIDs, %s regions", indent, humanNumber(s.Sweep.Operations.Ongoing.KeyProvides), humanNumber(s.Sweep.Operations.Ongoing.RegionProvides))
-				formatLine(1, "%sOngoing reprovides: %s CIDs, %s regions", indent, humanNumber(s.Sweep.Operations.Ongoing.KeyReprovides), humanNumber(s.Sweep.Operations.Ongoing.RegionReprovides))
+				formatLine(1, "%sOngoing provides: %s CIDs, %s regions", indent, humanSI(s.Sweep.Operations.Ongoing.KeyProvides, 1), humanSI(s.Sweep.Operations.Ongoing.RegionProvides, 1))
+				formatLine(1, "%sOngoing reprovides: %s CIDs, %s regions", indent, humanSI(s.Sweep.Operations.Ongoing.KeyReprovides, 1), humanSI(s.Sweep.Operations.Ongoing.RegionReprovides, 1))
 				// Past operations summary
 				formatLine(1, "%sTotal CIDs provided: %s", indent, humanNumber(s.Sweep.Operations.Past.KeysProvided))
 				if !brief {
 					formatLine(1, "%sTotal records provided: %s", indent, humanNumber(s.Sweep.Operations.Past.RecordsProvided))
 					formatLine(1, "%sTotal provide errors: %s", indent, humanNumber(s.Sweep.Operations.Past.KeysFailed))
-					formatLine(1, "%sCIDs provided/min: %s", indent, humanFloatOrNA(s.Sweep.Operations.Past.KeysProvidedPerMinute))
-					formatLine(1, "%sCIDs reprovided/min: %s", indent, humanFloatOrNA(s.Sweep.Operations.Past.KeysReprovidedPerMinute))
+					formatLine(1, "%sCIDs provided/min/worker: %s", indent, humanFloatOrNA(s.Sweep.Operations.Past.KeysProvidedPerMinute))
+					formatLine(1, "%sCIDs reprovided/min/worker: %s", indent, humanFloatOrNA(s.Sweep.Operations.Past.KeysReprovidedPerMinute))
 					formatLine(1, "%sRegion reprovide duration: %s", indent, humanDurationOrNA(s.Sweep.Operations.Past.RegionReprovideDuration))
 					formatLine(1, "%sAvg CIDs/reprovide: %s", indent, humanFloatOrNA(s.Sweep.Operations.Past.AvgKeysPerReprovide))
 					formatLine(1, "%sRegions reprovided (last cycle): %s", indent, humanNumber(s.Sweep.Operations.Past.RegionReprovidedLastCycle))
@@ -451,7 +452,7 @@ NOTES:
 			if displayWorkers || brief {
 				availableReservedBurst := max(0, s.Sweep.Workers.DedicatedBurst-s.Sweep.Workers.ActiveBurst)
 				availableReservedPeriodic := max(0, s.Sweep.Workers.DedicatedPeriodic-s.Sweep.Workers.ActivePeriodic)
-				availableFreeWorkers := s.Sweep.Workers.Max - max(s.Sweep.Workers.DedicatedBurst, s.Sweep.Workers.ActiveBurst) - max(s.Sweep.Workers.DedicatedPeriodic, s.Sweep.Workers.ActivePeriodic)
+				availableFreeWorkers := max(0, s.Sweep.Workers.Max-max(s.Sweep.Workers.DedicatedBurst, s.Sweep.Workers.ActiveBurst)-max(s.Sweep.Workers.DedicatedPeriodic, s.Sweep.Workers.ActivePeriodic))
 				availableBurst := availableFreeWorkers + availableReservedBurst
 				availablePeriodic := availableFreeWorkers + availableReservedPeriodic
 
@@ -463,21 +464,21 @@ NOTES:
 					if compactMode {
 						specifyWorkers = ""
 					}
-					formatLine(0, "%sActive%s: %s / %s (max)", indent, specifyWorkers, humanNumber(s.Sweep.Workers.Active), humanNumber(s.Sweep.Workers.Max))
+					formatLine(0, "%sActive%s: %s / %s (max)", indent, specifyWorkers, humanInt(s.Sweep.Workers.Active), humanInt(s.Sweep.Workers.Max))
 					if brief {
 						// Brief mode - show condensed worker info
 						formatLine(0, "%sPeriodic%s: %s active, %s available, %s queued", indent, specifyWorkers,
-							humanNumber(s.Sweep.Workers.ActivePeriodic), humanNumber(availablePeriodic), humanNumber(s.Sweep.Workers.QueuedPeriodic))
+							humanInt(s.Sweep.Workers.ActivePeriodic), humanInt(availablePeriodic), humanInt(s.Sweep.Workers.QueuedPeriodic))
 						formatLine(0, "%sBurst%s: %s active, %s available, %s queued\n", indent, specifyWorkers,
-							humanNumber(s.Sweep.Workers.ActiveBurst), humanNumber(availableBurst), humanNumber(s.Sweep.Workers.QueuedBurst))
+							humanInt(s.Sweep.Workers.ActiveBurst), humanInt(availableBurst), humanInt(s.Sweep.Workers.QueuedBurst))
 					} else {
-						formatLine(0, "%sFree%s: %s", indent, specifyWorkers, humanNumber(availableFreeWorkers))
-						formatLine(0, "%sWorkers stats:%s  %-9s %s", indent, "  ", "Periodic", "Burst")
-						formatLine(0, "%s  %-14s %-9s %s", indent, "Active:", humanNumber(s.Sweep.Workers.ActivePeriodic), humanNumber(s.Sweep.Workers.ActiveBurst))
-						formatLine(0, "%s  %-14s %-9s %s", indent, "Dedicated:", humanNumber(s.Sweep.Workers.DedicatedPeriodic), humanNumber(s.Sweep.Workers.DedicatedBurst))
-						formatLine(0, "%s  %-14s %-9s %s", indent, "Available:", humanNumber(availablePeriodic), humanNumber(availableBurst))
-						formatLine(0, "%s  %-14s %-9s %s", indent, "Queued:", humanNumber(s.Sweep.Workers.QueuedPeriodic), humanNumber(s.Sweep.Workers.QueuedBurst))
-						formatLine(0, "%sMax connections/worker: %s", indent, humanNumber(s.Sweep.Workers.MaxProvideConnsPerWorker))
+						formatLine(0, "%sFree%s: %s", indent, specifyWorkers, humanInt(availableFreeWorkers))
+						formatLine(0, "%s  %-14s %-9s %s", indent, "Workers stats:", "Periodic", "Burst")
+						formatLine(0, "%s  %-14s %-9s %s", indent, "Active:", humanInt(s.Sweep.Workers.ActivePeriodic), humanInt(s.Sweep.Workers.ActiveBurst))
+						formatLine(0, "%s  %-14s %-9s %s", indent, "Dedicated:", humanInt(s.Sweep.Workers.DedicatedPeriodic), humanInt(s.Sweep.Workers.DedicatedBurst))
+						formatLine(0, "%s  %-14s %-9s %s", indent, "Available:", humanInt(availablePeriodic), humanInt(availableBurst))
+						formatLine(0, "%s  %-14s %-9s %s", indent, "Queued:", humanInt(s.Sweep.Workers.QueuedPeriodic), humanInt(s.Sweep.Workers.QueuedBurst))
+						formatLine(0, "%sMax connections/worker: %s", indent, humanInt(s.Sweep.Workers.MaxProvideConnsPerWorker))
 						addBlankLine(0)
 					}
 				}
@@ -559,14 +560,18 @@ func humanFloatOrNA(val float64) string {
 	if val <= 0 {
 		return "N/A"
 	}
-	return fmt.Sprintf("%.1f", val)
+	return humanFull(val, 1)
 }
 
-func humanSI(val float64, decimals int) string {
-	v, unit := humanize.ComputeSI(val)
+func humanSI[T constraints.Float | constraints.Integer](val T, decimals int) string {
+	v, unit := humanize.ComputeSI(float64(val))
 	return fmt.Sprintf("%s%s", humanFull(v, decimals), unit)
 }
 
+func humanInt[T constraints.Integer](val T) string {
+	return humanFull(float64(val), 0)
+}
+
 func humanFull(val float64, decimals int) string {
 	return humanize.CommafWithDigits(val, decimals)
 }

core/node/provider.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ipfs/boxo/provider"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/namespace"
 	"github.com/ipfs/go-datastore/query"
 	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/repo"
@@ -36,14 +37,21 @@ import (
 	"go.uber.org/fx"
 )
 
-// The size of a batch that will be used for calculating average announcement
-// time per CID, inside of boxo/provider.ThroughputReport
-// and in 'ipfs stats provide' report.
-// Used when Provide.DHT.SweepEnabled=false
-const sampledBatchSize = 1000
+const (
+	// The size of a batch that will be used for calculating average announcement
+	// time per CID, inside of boxo/provider.ThroughputReport
+	// and in 'ipfs stats provide' report.
+	// Used when Provide.DHT.SweepEnabled=false
+	sampledBatchSize = 1000
 
-// Datastore key used to store previous reprovide strategy.
-const reprovideStrategyKey = "/reprovideStrategy"
+	// Datastore key used to store previous reprovide strategy.
+	reprovideStrategyKey = "/reprovideStrategy"
+
+	// Datastore namespace prefix for provider data.
+	providerDatastorePrefix = "provider"
+	// Datastore path for the provider keystore.
+	keystoreDatastorePath = "keystore"
+)
 
 // Interval between reprovide queue monitoring checks for slow reprovide alerts.
 // Used when Provide.DHT.SweepEnabled=true
@@ -324,10 +332,10 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
 		Repo repo.Repo
 	}
 	sweepingReprovider := fx.Provide(func(in providerInput) (DHTProvider, *keystore.ResettableKeystore, error) {
-		ds := in.Repo.Datastore()
+		ds := namespace.Wrap(in.Repo.Datastore(), datastore.NewKey(providerDatastorePrefix))
 		ks, err := keystore.NewResettableKeystore(ds,
 			keystore.WithPrefixBits(16),
-			keystore.WithDatastorePath("/provider/keystore"),
+			keystore.WithDatastorePath(keystoreDatastorePath),
 			keystore.WithBatchSize(int(cfg.Provide.DHT.KeystoreBatchSize.WithDefault(config.DefaultProvideDHTKeystoreBatchSize))),
 		)
 		if err != nil {
@@ -370,6 +378,8 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
 			if inDht != nil {
 				prov, err := ddhtprovider.New(inDht,
 					ddhtprovider.WithKeystore(ks),
+					ddhtprovider.WithDatastore(ds),
+					ddhtprovider.WithResumeCycle(cfg.Provide.DHT.ResumeEnabled.WithDefault(config.DefaultProvideDHTResumeEnabled)),
 
 					ddhtprovider.WithReprovideInterval(reprovideInterval),
 					ddhtprovider.WithMaxReprovideDelay(time.Hour),
@@ -403,7 +413,9 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
 		}
 		opts := []dhtprovider.Option{
 			dhtprovider.WithKeystore(ks),
-			dhtprovider.WithPeerID(impl.Host().ID()),
+			dhtprovider.WithDatastore(ds),
+			dhtprovider.WithResumeCycle(cfg.Provide.DHT.ResumeEnabled.WithDefault(config.DefaultProvideDHTResumeEnabled)),
+			dhtprovider.WithHost(impl.Host()),
 			dhtprovider.WithRouter(impl),
 			dhtprovider.WithMessageSender(impl.MessageSender()),
 			dhtprovider.WithSelfAddrs(selfAddrsFunc),
@@ -542,6 +554,9 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
 	}
 	reprovideAlert := fx.Invoke(func(lc fx.Lifecycle, in alertInput) {
 		prov := extractSweepingProvider(in.Provider)
+		if prov == nil {
+			return
+		}
 
 		var (
 			cancel context.CancelFunc
@@ -550,9 +565,6 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
 
 		lc.Append(fx.Hook{
 			OnStart: func(ctx context.Context) error {
-				if prov == nil {
-					return nil
-				}
 				gcCtx, c := context.WithCancel(context.Background())
 				cancel = c
 				go func() {
@@ -562,7 +574,7 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
 					defer ticker.Stop()
 
 					var (
-						queueSize, prevQueueSize         int
+						queueSize, prevQueueSize         int64
 						queuedWorkers, prevQueuedWorkers bool
 						count                            int
 					)
@@ -576,7 +588,7 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
 
 						stats := prov.Stats()
 						queuedWorkers = stats.Workers.QueuedPeriodic > 0
-						queueSize = stats.Queues.PendingRegionReprovides
+						queueSize = int64(stats.Queues.PendingRegionReprovides)
 
 						// Alert if reprovide queue keeps growing and all periodic workers are busy.
 						// Requires consecutiveAlertsThreshold intervals of sustained growth.