continued writing

6 files changed, 130 insertions, 119 deletions

diff --git a/cmd/cmd.go b/cmd/cmd.go
index 974770d..d3fc632 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -17,7 +17,7 @@ func runCmd(cCtx *cli.Context) error {
 		return err
 	}
 
-	db, err := badger.Open(files.Options(ctx.Value("dbfile").(string)))
+	db, err := badger.Open(Options(ctx.Value("dbfile").(string)))
 	if err != nil {
 		return err
 	}
@@ -28,34 +28,27 @@ func runCmd(cCtx *cli.Context) error {
 	ctx, stop := signal.NotifyContext(ctx, os.Interrupt)
 	defer stop()
 
+	counter := int64(0)
+
+	ctx = context.WithValue(ctx, "counter", &counter)
+
 	if err = files.IndexFlacs(ctx); err != nil {
 		return err
 	}
 
+	/* db.SetDiscardTs(badger.) */
+
 	if err = files.ReencodeFlacs(ctx); err != nil {
 		return err
 	}
 
-	return nil
+	db.RunValueLogGC(0.2)
 
-	/*
-		test := new(errgroup.Group)
-		test.SetLimit(3)
-		for n := 1; n < 10; n++ {
-			test.Go(func() error {
-				select {
-				case <-ctx.Done():
-					return nil
-				default:
-					fmt.Println("test")
-					time.Sleep(3 * time.Second)
-					return nil
-				}
-			})
-		}
-		_ = test.Wait()
-		fmt.Println("reached")
-		return cCtx.Err() */
+	if err := db.Sync(); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func Start() {
diff --git a/cmd/init.go b/cmd/init.go
index 2459315..63a87e6 100644
--- a/cmd/init.go
+++ b/cmd/init.go
@@ -8,7 +8,10 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"time"
 
+	badger "github.com/dgraph-io/badger/v4"
+	"github.com/dgraph-io/badger/v4/options"
 	"github.com/urfave/cli/v2"
 )
 
@@ -75,3 +78,60 @@ func initArgs(cCtx *cli.Context) (context.Context, error) {
 
 	return context.WithValue(ctx, "encoder", strings.ReplaceAll(strings.Split(string(encoder), " ")[1], "\n", "")), nil
 }
+
+func Options(path string) badger.Options {
+	return badger.Options{
+		Dir:      path,
+		ValueDir: path,
+
+		MemTableSize:        64 << 20,
+		BaseTableSize:       2 << 20,
+		BaseLevelSize:       10 << 20,
+		TableSizeMultiplier: 2,
+		LevelSizeMultiplier: 10,
+		MaxLevels:           7,
+		NumGoroutines:       8,
+		MetricsEnabled:      false,
+
+		NumCompactors:           4, // Run at least 2 compactors. Zero-th compactor prioritizes L0.
+		NumLevelZeroTables:      5,
+		NumLevelZeroTablesStall: 15,
+		NumMemtables:            5,
+		BloomFalsePositive:      0.01,
+		BlockSize:               4 * 1024,
+		SyncWrites:              false,
+		NumVersionsToKeep:       1,
+		CompactL0OnClose:        true,
+		VerifyValueChecksum:     false,
+		Compression:             options.Snappy,
+		BlockCacheSize:          256 << 20,
+		IndexCacheSize:          0,
+
+		// The following benchmarks were done on a 4 KB block size (default block size). The
+		// compression is ratio supposed to increase with increasing compression level but since the
+		// input for compression algorithm is small (4 KB), we don't get significant benefit at
+		// level 3.
+		// NOTE: The benchmarks are with DataDog ZSTD that requires CGO. Hence, no longer valid.
+		// no_compression-16              10	 502848865 ns/op	 165.46 MB/s	-
+		// zstd_compression/level_1-16     7	 739037966 ns/op	 112.58 MB/s	2.93
+		// zstd_compression/level_3-16     7	 756950250 ns/op	 109.91 MB/s	2.72
+		// zstd_compression/level_15-16    1	11135686219 ns/op	   7.47 MB/s	4.38
+		// Benchmark code can be found in table/builder_test.go file
+		ZSTDCompressionLevel: 1,
+
+		// (2^30 - 1)*2 when mmapping < 2^31 - 1, max int32.
+		// -1 so 2*ValueLogFileSize won't overflow on 32-bit systems.
+		ValueLogFileSize: 1<<30 - 1,
+
+		ValueLogMaxEntries: 1000000,
+
+		VLogPercentile: 0.0,
+		ValueThreshold: (1 << 20),
+
+		Logger:                        nil,
+		EncryptionKey:                 []byte{},
+		EncryptionKeyRotationDuration: 10 * 24 * time.Hour, // Default 10 days.
+		DetectConflicts:               true,
+		NamespaceOffset:               -1,
+	}
+}
diff --git a/files/database.go b/files/database.go
index 1ad40aa..5cb8e98 100644
--- a/files/database.go
+++ b/files/database.go
@@ -3,10 +3,9 @@ package files
 import (
 	"context"
 	"encoding/json"
-	"time"
 
 	badger "github.com/dgraph-io/badger/v4"
-	"github.com/dgraph-io/badger/v4/options"
+	progressbar "github.com/schollz/progressbar/v3"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -60,30 +59,35 @@ func (filedata *FileInfo) evaluateFile(database *badger.DB, hashsum []byte, enco
 	return ReencodeNotNeeded
 }
 
-func (filedata *FileInfo) updateFile(database *badger.DB, hashsum []byte) error {
-	return database.Update(func(txn *badger.Txn) error {
-		encoded, err := json.Marshal(filedata)
-		if err != nil {
-			return err
-		}
-		return txn.Set(hashsum, encoded)
-	})
+func (filedata *FileInfo) updateFile(wb *badger.WriteBatch, hashsum []byte) error {
+	encoded, err := json.Marshal(filedata)
+	if err != nil {
+		return err
+	}
+	return wb.Set(hashsum, encoded)
 }
 
 func (filedata *FileInfo) IndexFile(ctx context.Context, hashsum []byte) error {
 	database := ctx.Value("database").(*badger.DB)
 	encoder := ctx.Value("encoder").(string)
+	counter := ctx.Value("counter").(*int64)
 
 	err := filedata.evaluateFile(database, hashsum, encoder)
 	switch err {
 	case FileMoved, ReencodeNotNeeded:
 		filedata.Process = false
 	case ReencodeNeeded, badger.ErrKeyNotFound:
-
+		*counter += 1
 	default:
 		return err
 	}
-	if err := filedata.updateFile(database, hashsum); err != nil {
+
+	wb := database.NewWriteBatch()
+	defer wb.Cancel()
+	if err := filedata.updateFile(wb, hashsum); err != nil {
+		return err
+	}
+	if err := wb.Flush(); err != nil {
 		return err
 	}
 	return nil
@@ -91,6 +95,16 @@ func (filedata *FileInfo) IndexFile(ctx context.Context, hashsum []byte) error {
 
 func ReencodeFlacs(ctx context.Context) error {
 	db := ctx.Value("database").(*badger.DB)
+	bar := progressbar.NewOptions64(
+		*ctx.Value("counter").(*int64),
+		progressbar.OptionSetDescription("Reencoding..."),
+		progressbar.OptionShowCount(),
+	)
+	defer bar.Close()
+
+	wb := db.NewWriteBatch()
+	defer wb.Cancel()
+
 	wg := new(errgroup.Group)
 	wg.SetLimit(4)
 	err := db.View(func(txn *badger.Txn) error {
@@ -100,18 +114,6 @@ func ReencodeFlacs(ctx context.Context) error {
 		defer it.Close()
 
 		for it.Rewind(); it.Valid(); it.Next() {
-			/* item := it.Item()
-			key := item.Key()
-			info, err := decodeDbInfo(item)
-			if err != nil {
-				return err
-			}
-			if err := info.reencodeFile(ctx); err != nil {
-				return err
-			}
-			if err := info.updateFile(db, key); err != nil {
-				return err
-			} */
 			select {
 			case <-ctx.Done():
 				return nil
@@ -122,20 +124,24 @@ func ReencodeFlacs(ctx context.Context) error {
 				if err != nil {
 					return err
 				}
-				wg.Go(func() error {
-					select {
-					case <-ctx.Done():
-						return nil
-					default:
-						if err := info.reencodeFile(ctx); err != nil {
-							return err
-						}
-						if err := info.updateFile(db, key); err != nil {
-							return err
+				if info.Process {
+					wg.Go(func() error {
+						select {
+						case <-ctx.Done():
+							return nil
+						default:
+							if err := info.reencodeFile(ctx); err != nil {
+								return err
+							}
+							if err := info.updateFile(wb, key); err != nil {
+								return err
+							}
+							bar.Add64(1)
+							return nil
 						}
-						return nil
-					}
-				})
+					})
+				}
+
 			}
 		}
 		return nil
@@ -143,60 +149,3 @@ func ReencodeFlacs(ctx context.Context) error {
 	wg.Wait()
 	return err
 }
-
-func Options(path string) badger.Options {
-	return badger.Options{
-		Dir:      path,
-		ValueDir: path,
-
-		MemTableSize:        64 << 20,
-		BaseTableSize:       2 << 20,
-		BaseLevelSize:       10 << 20,
-		TableSizeMultiplier: 2,
-		LevelSizeMultiplier: 10,
-		MaxLevels:           7,
-		NumGoroutines:       8,
-		MetricsEnabled:      true,
-
-		NumCompactors:           4, // Run at least 2 compactors. Zero-th compactor prioritizes L0.
-		NumLevelZeroTables:      5,
-		NumLevelZeroTablesStall: 15,
-		NumMemtables:            5,
-		BloomFalsePositive:      0.01,
-		BlockSize:               4 * 1024,
-		SyncWrites:              false,
-		NumVersionsToKeep:       1,
-		CompactL0OnClose:        false,
-		VerifyValueChecksum:     false,
-		Compression:             options.Snappy,
-		BlockCacheSize:          256 << 20,
-		IndexCacheSize:          0,
-
-		// The following benchmarks were done on a 4 KB block size (default block size). The
-		// compression is ratio supposed to increase with increasing compression level but since the
-		// input for compression algorithm is small (4 KB), we don't get significant benefit at
-		// level 3.
-		// NOTE: The benchmarks are with DataDog ZSTD that requires CGO. Hence, no longer valid.
-		// no_compression-16              10	 502848865 ns/op	 165.46 MB/s	-
-		// zstd_compression/level_1-16     7	 739037966 ns/op	 112.58 MB/s	2.93
-		// zstd_compression/level_3-16     7	 756950250 ns/op	 109.91 MB/s	2.72
-		// zstd_compression/level_15-16    1	11135686219 ns/op	   7.47 MB/s	4.38
-		// Benchmark code can be found in table/builder_test.go file
-		ZSTDCompressionLevel: 1,
-
-		// (2^30 - 1)*2 when mmapping < 2^31 - 1, max int32.
-		// -1 so 2*ValueLogFileSize won't overflow on 32-bit systems.
-		ValueLogFileSize: 1<<30 - 1,
-
-		ValueLogMaxEntries: 1000000,
-
-		VLogPercentile: 0.0,
-		ValueThreshold: (1 << 20),
-
-		Logger:                        nil,
-		EncryptionKey:                 []byte{},
-		EncryptionKeyRotationDuration: 10 * 24 * time.Hour, // Default 10 days.
-		DetectConflicts:               true,
-		NamespaceOffset:               -1,
-	}
-}
diff --git a/files/operations.go b/files/operations.go
index 3da6586..885e31f 100644
--- a/files/operations.go
+++ b/files/operations.go
@@ -18,13 +18,13 @@ import (
 )
 
 func (file *FileInfo) reencodeFile(ctx context.Context) error {
-	/* cmd := exec.Command("flac", "-8", "-f", file.AbsPath)
-	if err := cmd.Run(); err != nil {
-		log.Printf("%s\n", err.Error())
+	out, err := exec.Command("flac", "-8fs", "-j4", file.AbsPath).Output()
+	if err != nil {
+		fmt.Printf("%v\n", out)
 		return err
 	}
-	file.Encoder = ctx.Value("encoder").(string) */
-	fmt.Println(file)
+	file.Encoder = ctx.Value("encoder").(string)
+	file.Process = false
 	return nil
 }
 
diff --git a/go.mod b/go.mod
index e7345d0..59f489a 100644
--- a/go.mod
+++ b/go.mod
@@ -15,6 +15,8 @@ require (
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/mattn/go-colorable v0.1.2 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/otel v1.35.0 // indirect
 	go.opentelemetry.io/otel/metric v1.35.0 // indirect
@@ -30,6 +32,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/dgraph-io/badger/v4 v4.7.0
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	golang.org/x/sync v0.13.0
 )
diff --git a/go.sum b/go.sum
index 1723f5b..03ee3b0 100644
--- a/go.sum
+++ b/go.sum
@@ -32,10 +32,16 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
+github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=