continued writing

2 files changed, 73 insertions, 20 deletions

diff --git a/cmd/cmd.go b/cmd/cmd.go
index 974770d..d3fc632 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -17,7 +17,7 @@ func runCmd(cCtx *cli.Context) error {
 		return err
 	}
 
-	db, err := badger.Open(files.Options(ctx.Value("dbfile").(string)))
+	db, err := badger.Open(Options(ctx.Value("dbfile").(string)))
 	if err != nil {
 		return err
 	}
@@ -28,34 +28,27 @@ func runCmd(cCtx *cli.Context) error {
 	ctx, stop := signal.NotifyContext(ctx, os.Interrupt)
 	defer stop()
 
+	counter := int64(0)
+
+	ctx = context.WithValue(ctx, "counter", &counter)
+
 	if err = files.IndexFlacs(ctx); err != nil {
 		return err
 	}
 
+	/* db.SetDiscardTs(badger.) */
+
 	if err = files.ReencodeFlacs(ctx); err != nil {
 		return err
 	}
 
-	return nil
+	db.RunValueLogGC(0.2)
 
-	/*
-		test := new(errgroup.Group)
-		test.SetLimit(3)
-		for n := 1; n < 10; n++ {
-			test.Go(func() error {
-				select {
-				case <-ctx.Done():
-					return nil
-				default:
-					fmt.Println("test")
-					time.Sleep(3 * time.Second)
-					return nil
-				}
-			})
-		}
-		_ = test.Wait()
-		fmt.Println("reached")
-		return cCtx.Err() */
+	if err := db.Sync(); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func Start() {
diff --git a/cmd/init.go b/cmd/init.go
index 2459315..63a87e6 100644
--- a/cmd/init.go
+++ b/cmd/init.go
@@ -8,7 +8,10 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"time"
 
+	badger "github.com/dgraph-io/badger/v4"
+	"github.com/dgraph-io/badger/v4/options"
 	"github.com/urfave/cli/v2"
 )
 
@@ -75,3 +78,60 @@ func initArgs(cCtx *cli.Context) (context.Context, error) {
 
 	return context.WithValue(ctx, "encoder", strings.ReplaceAll(strings.Split(string(encoder), " ")[1], "\n", "")), nil
 }
+
+func Options(path string) badger.Options {
+	return badger.Options{
+		Dir:      path,
+		ValueDir: path,
+
+		MemTableSize:        64 << 20,
+		BaseTableSize:       2 << 20,
+		BaseLevelSize:       10 << 20,
+		TableSizeMultiplier: 2,
+		LevelSizeMultiplier: 10,
+		MaxLevels:           7,
+		NumGoroutines:       8,
+		MetricsEnabled:      false,
+
+		NumCompactors:           4, // Run at least 2 compactors. Zero-th compactor prioritizes L0.
+		NumLevelZeroTables:      5,
+		NumLevelZeroTablesStall: 15,
+		NumMemtables:            5,
+		BloomFalsePositive:      0.01,
+		BlockSize:               4 * 1024,
+		SyncWrites:              false,
+		NumVersionsToKeep:       1,
+		CompactL0OnClose:        true,
+		VerifyValueChecksum:     false,
+		Compression:             options.Snappy,
+		BlockCacheSize:          256 << 20,
+		IndexCacheSize:          0,
+
+		// The following benchmarks were done on a 4 KB block size (default block size). The
+		// compression is ratio supposed to increase with increasing compression level but since the
+		// input for compression algorithm is small (4 KB), we don't get significant benefit at
+		// level 3.
+		// NOTE: The benchmarks are with DataDog ZSTD that requires CGO. Hence, no longer valid.
+		// no_compression-16              10	 502848865 ns/op	 165.46 MB/s	-
+		// zstd_compression/level_1-16     7	 739037966 ns/op	 112.58 MB/s	2.93
+		// zstd_compression/level_3-16     7	 756950250 ns/op	 109.91 MB/s	2.72
+		// zstd_compression/level_15-16    1	11135686219 ns/op	   7.47 MB/s	4.38
+		// Benchmark code can be found in table/builder_test.go file
+		ZSTDCompressionLevel: 1,
+
+		// (2^30 - 1)*2 when mmapping < 2^31 - 1, max int32.
+		// -1 so 2*ValueLogFileSize won't overflow on 32-bit systems.
+		ValueLogFileSize: 1<<30 - 1,
+
+		ValueLogMaxEntries: 1000000,
+
+		VLogPercentile: 0.0,
+		ValueThreshold: (1 << 20),
+
+		Logger:                        nil,
+		EncryptionKey:                 []byte{},
+		EncryptionKeyRotationDuration: 10 * 24 * time.Hour, // Default 10 days.
+		DetectConflicts:               true,
+		NamespaceOffset:               -1,
+	}
+}