Merge pull request #288 from WaberZhuang/main

convertor: option to disable sparse file

Author: yuchen0cc

.github/workflows/ci-basic.yml

@@ -78,10 +78,12 @@ jobs:
       - name: CI - record trace
         shell: bash
         run: |
+          set -x
           /opt/overlaybd/snapshotter/ctr rpull registry.hub.docker.com/overlaybd/redis:6.2.1_obdconv
           echo "[ by record-trace ]"
           /opt/overlaybd/snapshotter/ctr record-trace --runtime "io.containerd.runc.v2" --disable-network-isolation --time 15 registry.hub.docker.com/overlaybd/redis:6.2.1_obdconv registry.hub.docker.com/overlaybd/redis:6.2.1_obdconv_trace
           ctr i ls | grep 6.2.1_obdconv_trace
+          sleep 10s # wait for tcmu device to be removed
           echo "[ by label ]"
           touch /tmp/trace_file
           ctr run -d --snapshotter=overlaybd --snapshotter-label containerd.io/snapshot/overlaybd/record-trace=yes --snapshotter-label containerd.io/snapshot/overlaybd/record-trace-path=/tmp/trace_file registry.hub.docker.com/overlaybd/redis:6.2.1_obdconv demo

cmd/convertor/builder/builder.go

@@ -66,6 +66,9 @@ type BuilderOptions struct {
 	// ConcurrencyLimit limits the number of manifests that can be built at once
 	// 0 means no limit
 	ConcurrencyLimit int
+
+	// disable sparse file when converting overlaybd
+	DisableSparse bool
 }
 
 type graphBuilder struct {
@@ -248,6 +251,7 @@ func (b *graphBuilder) buildOne(ctx context.Context, src v1.Descriptor, tag bool
 	switch b.Engine {
 	case Overlaybd:
 		engine = NewOverlayBDBuilderEngine(engineBase)
+		engine.(*overlaybdBuilderEngine).disableSparse = b.DisableSparse
 	case TurboOCI:
 		engine = NewTurboOCIBuilderEngine(engineBase)
 	}

cmd/convertor/builder/overlaybd_builder.go

@@ -51,6 +51,7 @@ type overlaybdConvertResult struct {
 
 type overlaybdBuilderEngine struct {
 	*builderEngineBase
+	disableSparse   bool
 	overlaybdConfig *sn.OverlayBDBSConfig
 	overlaybdLayers []overlaybdConvertResult
 }
@@ -427,7 +428,10 @@ func (e *overlaybdBuilderEngine) getLayerDir(idx int) string {
 }
 
 func (e *overlaybdBuilderEngine) create(ctx context.Context, dir string, mkfs bool, vsizeGB int) error {
-	opts := []string{"-s", fmt.Sprintf("%d", vsizeGB)}
+	opts := []string{fmt.Sprintf("%d", vsizeGB)}
+	if !e.disableSparse {
+		opts = append(opts, "-s")
+	}
 	if mkfs {
 		opts = append(opts, "--mkfs")
 		logrus.Infof("mkfs for baselayer, vsize: %d GB", vsizeGB)

cmd/convertor/main.go

@@ -48,6 +48,7 @@ var (
 	dbstr            string
 	dbType           string
 	concurrencyLimit int
+	disableSparse    bool
 
 	// certification
 	certDirs    []string
@@ -104,6 +105,7 @@ Version: ` + commitID,
 				NoUpload:         noUpload,
 				DumpManifest:     dumpManifest,
 				ConcurrencyLimit: concurrencyLimit,
+				DisableSparse:    disableSparse,
 			}
 			if overlaybd != "" {
 				logrus.Info("building [Overlaybd - Native]  image...")
@@ -165,6 +167,7 @@ func init() {
 	rootCmd.Flags().StringVar(&dbstr, "db-str", "", "db str for overlaybd conversion")
 	rootCmd.Flags().StringVar(&dbType, "db-type", "", "type of db to use for conversion deduplication. Available: mysql. Default none")
 	rootCmd.Flags().IntVar(&concurrencyLimit, "concurrency-limit", 4, "the number of manifests that can be built at the same time, used for multi-arch images, 0 means no limit")
+	rootCmd.Flags().BoolVar(&disableSparse, "disable-sparse", false, "disable sparse file for overlaybd")
 
 	// certification
 	rootCmd.Flags().StringArrayVar(&certDirs, "cert-dir", nil, "In these directories, root CA should be named as *.crt and client cert should be named as *.cert, *.key")