From 1fea9b682b6ed08b17495374a8ede21b51101faa Mon Sep 17 00:00:00 2001 From: jakka Date: Thu, 19 Jun 2025 14:39:55 +0300 Subject: reworked encoding async logic --- Cargo.lock | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/files.rs | 59 +++++++++++++++++++++++++----------- src/flac.rs | 12 +------- src/main.rs | 1 - 5 files changed, 143 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3df01f5..8bc9a12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,6 +439,19 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -520,6 +533,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "equivalent" version = "1.0.2" @@ -574,6 +593,7 @@ dependencies = [ "flac-bound", "futures-util", "i24", + "indicatif", "libsql", "md-5", "metaflac", @@ -581,6 +601,7 @@ dependencies = [ "symphonia", "tokio", "tokio-util", + "walkdir", ] [[package]] @@ -936,6 +957,22 @@ dependencies = [ "hashbrown 0.15.4", ] +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "futures-core", + "number_prefix", + "portable-atomic", + "tokio", + "unicode-segmentation", + "unicode-width", + "web-time", +] + [[package]] name = "inout" version = "0.1.4" @@ -1275,6 +1312,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.36.7" @@ -1414,6 +1457,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1672,6 +1721,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.27" @@ -2137,6 +2195,18 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" + [[package]] name = "untrusted" version = "0.9.0" @@ -2167,6 +2237,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2249,6 +2329,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -2279,6 +2369,15 @@ dependencies = [ "rustix", ] +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "windows-core" version = "0.61.2" diff --git a/Cargo.toml b/Cargo.toml index d9c7b5a..113ee1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ flac-bound = { version = "0.5.0", features = [ ], default-features = false } futures-util = "0.3.31" i24 = "2.1.0" +indicatif = { version = "0.17.11", features = ["tokio", "improved_unicode", "futures"] } libsql = { version = "0.9.10", features = ["core", "sync"] } md-5 = "0.10.6" metaflac = "0.2.8" @@ -33,4 +34,5 @@ tokio = { version = "1.45.1", features = [ "signal", ] } tokio-util = "0.7.15" +walkdir = "2.5.0" #symphonia = { git = "https://github.com/pdeljanov/Symphonia.git", branch = "dev-0.6", default-features = false, features = ["flac"] } diff --git a/src/files.rs b/src/files.rs index a019cfb..6f2bce2 100644 --- a/src/files.rs +++ b/src/files.rs @@ -2,13 +2,15 @@ use anyhow::{Result, anyhow}; use futures_util::StreamExt; use pin_utils::pin_mut; use std::{ + error::Error, fmt::Display, path::{Path, PathBuf}, time::UNIX_EPOCH, }; use tokio::{fs::read_dir, task::JoinSet}; +use walkdir::WalkDir; -use crate::{db::Database, flac::handle_encode}; +use crate::{db::Database, flac::encode_file}; #[derive(Debug)] pub struct FileError { @@ -36,6 +38,8 @@ impl Display for FileError { } } +impl Error for FileError {} + async fn handle_file(file: impl AsRef, conn: Database) -> Result<()> { match conn.check_file(&file).await { Ok(true) => { @@ -64,11 +68,30 @@ async fn handle_file(file: impl AsRef, conn: Database) -> Result<()> { Ok(()) } +async fn count_flacs(path: impl AsRef) -> u64 { + let mut counter = 0; + let _ = WalkDir::new(path) + .into_iter() + .map(|file| { + let path = file.unwrap().into_path(); + if path.is_file() && path.extension().unwrap() == "flac" { + counter += 1 + } + }) + .collect::>(); + counter +} + pub async fn index_files_recursively(path: impl AsRef, conn: &Database) -> Result<()> { if !path.as_ref().is_dir() { return Err(anyhow!("Invalid root directory")); } let abspath = path.as_ref().canonicalize()?; + + let files = count_flacs(&abspath).await; + + println!("Total flacs:\t{files}"); + let mut tasks = JoinSet::new(); let mut dirs = vec![abspath]; @@ -117,30 +140,30 @@ pub async fn reencode_files(conn: &Database) -> Result<()> { while let Some(Ok(row)) = stream.next().await { if let Some(file) = row.get_value(0)?.as_text() { let filename = Path::new(file).canonicalize()?; - tasks.spawn_blocking(move || handle_encode(filename)); + let newconn = conn.clone(); + tasks.spawn(async move { + let file = filename.clone(); + if let Err(error) = tokio::task::spawn_blocking(move || encode_file(file)).await? { + return Err(anyhow!(FileError::new(&filename, error))); + }; + + if let Err(error) = newconn.update_file(&filename).await { + return Err(anyhow!(FileError::new(&filename, error))); + }; + + Ok(()) + }); } } - let mut update_tasks = JoinSet::new(); - while let Some(task) = tasks.join_next().await { match task { - Ok(Ok(path)) => { - let newconn = conn.clone(); - update_tasks.spawn(async move { newconn.update_file(path).await }); + Ok(Err(error)) => eprintln!("Error encountered:\t{error}"), + Err(error) => eprintln!("Error encountered:\t{error}"), + _ => { counter += 1; - print!("\rReencoded files:\t{counter}") + print!("Reencoded:\t{counter}"); } - Ok(Err(error)) => eprintln!("{error}"), - Err(error) => eprintln!("Error encountered:\t{}", error), - } - } - - while let Some(task) = update_tasks.join_next().await { - match task { - Ok(Err(error)) => eprintln!("{error}"), - Err(error) => eprintln!("Error encountered:\t{}", error), - _ => {} } } diff --git a/src/flac.rs b/src/flac.rs index 42f9d21..97f61b5 100644 --- a/src/flac.rs +++ b/src/flac.rs @@ -15,8 +15,6 @@ use symphonia::core::{ meta::MetadataOptions, }; -use crate::files; - pub const CURRENT_VENDOR: &str = "reference libFLAC 1.5.0 20250211"; type BoxedFormatReader = Box; @@ -281,7 +279,7 @@ fn encode_cycle_32( Ok(hasher.finalize().to_vec()) } -fn encode_file(filename: impl AsRef) -> Result<()> { +pub fn encode_file(filename: impl AsRef) -> Result<()> { let filencoder = FileEncoder::new(filename)?; let mut outf = File::create(filencoder.temp_name())?; @@ -299,14 +297,6 @@ fn encode_file(filename: impl AsRef) -> Result<()> { filencoder.encode(enc) } -pub fn handle_encode(file: impl AsRef) -> Result> { - if let Err(error) = encode_file(&file) { - Err(anyhow!(files::FileError::new(file, error))) - } else { - Ok(file) - } -} - pub fn get_vendor(file: impl AsRef) -> Result { if let Some(vorbis) = Tag::read_from_path(file)?.vorbis_comments() { Ok(vorbis.vendor_string.to_owned()) diff --git a/src/main.rs b/src/main.rs index da04a9e..bb2a716 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,6 @@ fn build_cli() -> Command { Arg::new("doit") .long("doit") .help("Actually reencode files") - .conflicts_with("index") .action(ArgAction::SetTrue), ) .arg( -- cgit v1.3.1