diff options
Diffstat (limited to 'files')
| -rw-r--r-- | files/database.go | 141 | ||||
| -rw-r--r-- | files/operations.go | 97 | ||||
| -rw-r--r-- | files/types.go | 8 |
3 files changed, 152 insertions, 94 deletions
diff --git a/files/database.go b/files/database.go index 5cb8e98..33c2636 100644 --- a/files/database.go +++ b/files/database.go @@ -3,46 +3,48 @@ package files import ( "context" "encoding/json" + "errors" + "log" + "os" + "strings" - badger "github.com/dgraph-io/badger/v4" + "github.com/rosedblabs/rosedb/v2" progressbar "github.com/schollz/progressbar/v3" "golang.org/x/sync/errgroup" ) -func decodeDbInfo(item *badger.Item) (*FileInfo, error) { +func decodeDbInfo(value []byte) (*FileInfo, error) { var info FileInfo - if err := item.Value(func(val []byte) error { - if err := json.Unmarshal(val, &info); err != nil { - return err - } - return nil - }); err != nil { + if err := json.Unmarshal(value, &info); err != nil { return nil, err } return &info, nil } -func getInfoFromDb(database *badger.DB, hashsum []byte) (*FileInfo, error) { - var info *FileInfo - if err := database.View(func(txn *badger.Txn) error { - item, err := txn.Get(hashsum) - if err != nil { - return err - } +func getInfoFromDb(batch *rosedb.Batch, hashsum []byte) (*FileInfo, error) { + exist, err := batch.Exist(hashsum) + if err != nil { + return nil, err + } - info, err = decodeDbInfo(item) - if err != nil { - return err - } - return nil - }); err != nil { + if !exist { + return nil, rosedb.ErrKeyNotFound + } + + value, err := batch.Get(hashsum) + if err != nil { + return nil, err + } + + info, err := decodeDbInfo(value) + if err != nil { return nil, err } - return info, nil + return info, err } -func (filedata *FileInfo) evaluateFile(database *badger.DB, hashsum []byte, encoder string) error { - info, err := getInfoFromDb(database, hashsum) +func (filedata *FileInfo) evaluateFile(batch *rosedb.Batch, hashsum []byte, encoder string) error { + info, err := getInfoFromDb(batch, hashsum) if err != nil { return err } @@ -59,71 +61,79 @@ func (filedata *FileInfo) evaluateFile(database *badger.DB, hashsum []byte, enco return ReencodeNotNeeded } -func (filedata *FileInfo) updateFile(wb *badger.WriteBatch, hashsum []byte) error { +func (filedata *FileInfo) updateFile(batch *rosedb.Batch, hashsum []byte) error { encoded, err := json.Marshal(filedata) if err != nil { return err } - return wb.Set(hashsum, encoded) + return batch.Put(hashsum, encoded) } -func (filedata *FileInfo) IndexFile(ctx context.Context, hashsum []byte) error { - database := ctx.Value("database").(*badger.DB) +func (filedata *FileInfo) IndexFile(ctx context.Context, hashsum []byte, batch *rosedb.Batch) error { encoder := ctx.Value("encoder").(string) counter := ctx.Value("counter").(*int64) - err := filedata.evaluateFile(database, hashsum, encoder) + err := filedata.evaluateFile(batch, hashsum, encoder) + switch err { case FileMoved, ReencodeNotNeeded: filedata.Process = false - case ReencodeNeeded, badger.ErrKeyNotFound: + case ReencodeNeeded, rosedb.ErrKeyNotFound: *counter += 1 default: return err } - wb := database.NewWriteBatch() - defer wb.Cancel() - if err := filedata.updateFile(wb, hashsum); err != nil { - return err - } - if err := wb.Flush(); err != nil { + if err := filedata.updateFile(batch, hashsum); err != nil { return err } + return nil } func ReencodeFlacs(ctx context.Context) error { - db := ctx.Value("database").(*badger.DB) + db := ctx.Value("database").(*rosedb.DB) bar := progressbar.NewOptions64( *ctx.Value("counter").(*int64), progressbar.OptionSetDescription("Reencoding..."), progressbar.OptionShowCount(), ) - defer bar.Close() - wb := db.NewWriteBatch() - defer wb.Cancel() + batch := db.NewBatch(rosedb.DefaultBatchOptions) wg := new(errgroup.Group) wg.SetLimit(4) - err := db.View(func(txn *badger.Txn) error { - opts := badger.DefaultIteratorOptions - opts.PrefetchSize = 10 - it := txn.NewIterator(opts) - defer it.Close() - for it.Rewind(); it.Valid(); it.Next() { - select { - case <-ctx.Done(): - return nil - default: - item := it.Item() - key := item.Key() - info, err := decodeDbInfo(item) - if err != nil { - return err + iterOpts := rosedb.DefaultIteratorOptions + iterOpts.ContinueOnError = true + iter := db.NewIterator(iterOpts) + + for iter.Rewind(); iter.Valid(); iter.Next() { + select { + case <-ctx.Done(): + wg.Wait() + if err := batch.Commit(); err != nil { + return err + } + bar.Exit() + return nil + default: + item := iter.Item() + key := item.Key + info, err := decodeDbInfo(item.Value) + if err != nil { + log.Println(err.Error()) + } + + if !strings.Contains(info.AbsPath, ctx.Value("path").(string)) { + continue + } else if _, err := os.Stat(info.AbsPath); errors.Is(err, os.ErrNotExist) { + if err := batch.Delete(key); err != nil { + log.Println(err.Error()) } + } else if err != nil { + log.Println(err.Error()) + } else { if info.Process { wg.Go(func() error { select { @@ -133,19 +143,30 @@ func ReencodeFlacs(ctx context.Context) error { if err := info.reencodeFile(ctx); err != nil { return err } - if err := info.updateFile(wb, key); err != nil { + if err := batch.Delete(key); err != nil { + return err + } + key, err = getSha256(info.AbsPath) + if err != nil { + return err + } + if err := info.updateFile(batch, key); err != nil { return err } + bar.Add64(1) return nil } }) } - } } - return nil - }) + } + wg.Wait() - return err + if err := batch.Commit(); err != nil { + return err + } + bar.Close() + return nil } diff --git a/files/operations.go b/files/operations.go index 885e31f..6f44900 100644 --- a/files/operations.go +++ b/files/operations.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "io/fs" + "log" "os" "os/exec" "path/filepath" @@ -14,18 +15,38 @@ import ( "time" "github.com/briandowns/spinner" - "golang.org/x/sync/errgroup" + "github.com/rosedblabs/rosedb/v2" ) func (file *FileInfo) reencodeFile(ctx context.Context) error { - out, err := exec.Command("flac", "-8fs", "-j4", file.AbsPath).Output() - if err != nil { - fmt.Printf("%v\n", out) - return err + select { + case <-ctx.Done(): + return nil + default: + args := ctx.Value("flac").([]string) + if args != nil { + args = append(args, file.AbsPath) + cmd := exec.Command("flac", args...) + if err := cmd.Run(); err != nil { + if !strings.Contains(err.Error(), "interrupt") { + log.Println(err.Error()) + return err + } + } + } else { + cmd := exec.Command("flac", "-8f", "-j4", file.AbsPath) + if err := cmd.Run(); err != nil { + if !strings.Contains(err.Error(), "interrupt") { + log.Println(err.Error()) + return err + } + } + } + + file.Encoder = ctx.Value("encoder").(string) + file.Process = false + return nil } - file.Encoder = ctx.Value("encoder").(string) - file.Process = false - return nil } func getEncoderVer(path string) (string, error) { @@ -44,18 +65,11 @@ func getEncoderVer(path string) (string, error) { } } -func getInfoFromFile(path string, info fs.DirEntry) (*FileInfo, error) { +func getInfoFromFile(path string) (*FileInfo, error) { var filedata FileInfo filedata.Process = true - tmp, err := info.Info() - if err != nil { - return nil, err - } - - filedata.Modtime = tmp.ModTime() - abspath, err := filepath.Abs(path) if err != nil { return nil, err @@ -87,14 +101,25 @@ func getSha256(path string) ([]byte, error) { return hash.Sum(nil), nil } +func updateSpinner(spin *spinner.Spinner, counter int64) { + line := fmt.Sprintf(" Indexing flacs...\t %v", counter) + spin.Suffix = line +} + func IndexFlacs(ctx context.Context) error { + counter := ctx.Value("counter").(*int64) + database := ctx.Value("database").(*rosedb.DB) + + batch := database.NewBatch(rosedb.DefaultBatchOptions) + spin := spinner.New(spinner.CharSets[9], 100*time.Millisecond) - spin.Suffix = " Indexing flacs..." + + updateSpinner(spin, *counter) spin.Start() - wg := new(errgroup.Group) - wg.SetLimit(100) + /* wg := new(errgroup.Group) + wg.SetLimit(100) */ if err := filepath.WalkDir(ctx.Value("path").(string), func(path string, info fs.DirEntry, err error) error { select { @@ -103,17 +128,14 @@ func IndexFlacs(ctx context.Context) error { spin.Stop() return filepath.SkipAll default: - if err != nil { - return err - } if !info.IsDir() { if filepath.Ext(path) == ".flac" { - wg.Go(func() error { + /* wg.Go(func() error { select { case <-ctx.Done(): return nil default: - data, err := getInfoFromFile(path, info) + data, err := getInfoFromFile(path) if err != nil { return err } @@ -122,10 +144,22 @@ func IndexFlacs(ctx context.Context) error { if err != nil { return err } - - return data.IndexFile(ctx, hashsum) + updateSpinner(spin, *counter) + return data.IndexFile(ctx, hashsum, batch) } - }) + }) */ + + data, err := getInfoFromFile(path) + if err != nil { + return err + } + + hashsum, err := getSha256(path) + if err != nil { + return err + } + updateSpinner(spin, *counter) + return data.IndexFile(ctx, hashsum, batch) } } @@ -136,8 +170,13 @@ func IndexFlacs(ctx context.Context) error { return err } - wg.Wait() - spin.FinalMSG = "Done indexing flacs\n" + /* wg.Wait() */ + + if err := batch.Commit(); err != nil { + return err + } + spin.FinalMSG = fmt.Sprintf("Done indexing flacs: \t%v to process\n", *counter) spin.Stop() + return nil } diff --git a/files/types.go b/files/types.go index 87ae6e1..7bb2f42 100644 --- a/files/types.go +++ b/files/types.go @@ -2,7 +2,6 @@ package files import ( "errors" - "time" ) var ( @@ -12,8 +11,7 @@ var ( ) type FileInfo struct { - AbsPath string `json:"abspath"` - Modtime time.Time `json:"modtime"` - Encoder string `json:"encoder"` - Process bool `json:"process"` + AbsPath string `json:"abspath"` + Encoder string `json:"encoder"` + Process bool `json:"process"` } |