diff options
| -rw-r--r-- | Cargo.lock | 9 | ||||
| -rw-r--r-- | Cargo.toml | 2 | ||||
| -rw-r--r-- | src/db.rs | 19 | ||||
| -rw-r--r-- | src/files.rs | 97 | ||||
| -rw-r--r-- | src/flac.rs | 238 | ||||
| -rw-r--r-- | src/main.rs | 22 |
6 files changed, 208 insertions, 179 deletions
@@ -1512,7 +1512,7 @@ dependencies = [ [[package]] name = "reencoder" -version = "0.0.1" +version = "0.1.0" dependencies = [ "anyhow", "clap", @@ -1765,12 +1765,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "smallvec" @@ -1,6 +1,6 @@ [package] name = "reencoder" -version = "0.0.1" +version = "0.1.0" edition = "2024" repository = "https://github.com/justjakka/reencoder/" license = "BSD-3-Clause" @@ -3,7 +3,6 @@ use directories::BaseDirs; use futures_util::Stream; use libsql::{Builder, Connection, params}; use std::{ - ffi::OsStr, path::Path, time::{Duration, UNIX_EPOCH}, }; @@ -32,8 +31,8 @@ impl Database { Ok(Database(conn)) } - pub async fn insert_file(&self, filename: &impl AsRef<OsStr>) -> Result<()> { - let abs_filename = Path::new(filename).canonicalize()?; + pub async fn insert_file(&self, filename: impl AsRef<Path>) -> Result<()> { + let abs_filename = filename.as_ref().canonicalize()?; let toencode = !matches!(get_vendor(&abs_filename)?.as_str(), CURRENT_VENDOR); let modtime = abs_filename @@ -52,8 +51,8 @@ impl Database { Ok(()) } - pub async fn update_file(&self, filename: &impl AsRef<OsStr>) -> Result<()> { - let abs_filename = Path::new(filename).canonicalize()?; + pub async fn update_file(&self, filename: impl AsRef<Path>) -> Result<()> { + let abs_filename = filename.as_ref().canonicalize()?; let modtime = abs_filename .metadata()? @@ -71,8 +70,8 @@ impl Database { Ok(()) } - pub async fn check_file(&self, filename: &impl AsRef<OsStr>) -> Result<bool> { - let abs_filename = Path::new(filename).canonicalize()?; + pub async fn check_file(&self, filename: impl AsRef<Path>) -> Result<bool> { + let abs_filename = filename.as_ref().canonicalize()?; if let Some(row) = self .0 @@ -87,8 +86,8 @@ impl Database { } } - pub async fn get_modtime(&self, filename: &impl AsRef<OsStr>) -> Result<u64> { - let abs_filename = Path::new(filename).canonicalize()?; + pub async fn get_modtime(&self, filename: impl AsRef<Path>) -> Result<u64> { + let abs_filename = filename.as_ref().canonicalize()?; if let Some(row) = self .0 @@ -125,6 +124,8 @@ impl Database { tasks.join_all().await; + self.0.execute("VACUUM;", ()).await?; + Ok(()) } diff --git a/src/files.rs b/src/files.rs index c0f4b68..743e489 100644 --- a/src/files.rs +++ b/src/files.rs @@ -17,8 +17,11 @@ pub struct FileError { } impl FileError { - pub fn new(file: PathBuf, error: anyhow::Error) -> Self { - FileError { file, error } + pub fn new(file: impl AsRef<Path>, error: anyhow::Error) -> Self { + FileError { + file: file.as_ref().to_path_buf(), + error, + } } } @@ -60,15 +63,17 @@ async fn handle_file(file: PathBuf, conn: Database) -> Result<()> { Ok(()) } -pub async fn index_files_recursively(path: &Path, conn: &Database) -> Result<()> { - if !path.is_dir() { +pub async fn index_files_recursively(path: impl AsRef<Path>, conn: &Database) -> Result<()> { + if !path.as_ref().is_dir() { return Err(anyhow!("Invalid root directory")); } - let abspath = path.canonicalize()?; + let abspath = path.as_ref().canonicalize()?; let mut tasks = JoinSet::new(); let mut dirs = vec![abspath]; + let mut counter: u64 = 0; + while let Some(dir) = dirs.pop() { let mut read_dir = read_dir(dir).await?; @@ -91,37 +96,59 @@ pub async fn index_files_recursively(path: &Path, conn: &Database) -> Result<()> match task { Ok(Err(error)) => eprintln!("{error}"), Err(error) => eprintln!("Error encountered:\t{}", error), - _ => {} + _ => { + counter += 1; + print!("\rParsed files:\t{counter}"); + } } } - Ok(()) } -pub async fn reencode_files(conn: &Database, folderpath: Option<&PathBuf>) -> Result<()> { - let mut nocheck = true; - let mut path = Path::new(""); +fn check_path(folderpath: Option<&PathBuf>) -> (PathBuf, bool) { if let Some(real_path) = folderpath { - nocheck = false; - path = real_path; + (real_path.to_owned(), false) + } else { + (PathBuf::new(), true) } +} + +pub async fn reencode_files(folderpath: Option<&PathBuf>, conn: &Database) -> Result<()> { + let (path, nocheck) = check_path(folderpath); let stream = conn.get_toencode_stream().await?; pin_mut!(stream); let mut tasks = JoinSet::new(); + let mut counter: u64 = 0; + while let Some(Ok(row)) = stream.next().await { if let Some(file) = row.get_value(0)?.as_text() { let filename = Path::new(file).canonicalize()?; - if nocheck || filename.starts_with(path) { + if nocheck || filename.starts_with(&path) { tasks.spawn_blocking(move || handle_encode(filename)); } } } + let mut update_tasks = JoinSet::new(); + while let Some(task) = tasks.join_next().await { match task { + Ok(Ok(path)) => { + let newconn = conn.clone(); + update_tasks.spawn(async move { newconn.update_file(path).await }); + counter += 1; + print!("\rReencoded files:\t{counter}") + } + Ok(Err(error)) => eprintln!("{error}"), + Err(error) => eprintln!("Error encountered:\t{}", error), + } + } + + while let Some(task) = update_tasks.join_next().await { + match task { Ok(Err(error)) => eprintln!("{error}"), Err(error) => eprintln!("Error encountered:\t{}", error), _ => {} @@ -131,13 +158,8 @@ pub async fn reencode_files(conn: &Database, folderpath: Option<&PathBuf>) -> Re Ok(()) } -pub async fn count_reencode_files(conn: &Database, folderpath: Option<&PathBuf>) -> Result<u64> { - let mut nocheck = true; - let mut path = Path::new(""); - if let Some(real_path) = folderpath { - nocheck = false; - path = real_path; - } +pub async fn count_reencode_files(folderpath: Option<&PathBuf>, conn: &Database) -> Result<u64> { + let (path, nocheck) = check_path(folderpath); let mut counter: u64 = 0; let stream = conn.get_toencode_stream().await?; @@ -146,7 +168,7 @@ pub async fn count_reencode_files(conn: &Database, folderpath: Option<&PathBuf>) while let Some(Ok(row)) = stream.next().await { if let Some(file) = row.get_value(0)?.as_text() { let filename = Path::new(file).canonicalize()?; - if nocheck || filename.starts_with(path) { + if nocheck || filename.starts_with(&path) { counter += 1; } } @@ -160,25 +182,24 @@ mod tests { use super::*; #[tokio::test] - async fn test_lots_of_files() { + async fn test_index_lots_of_files() { let conn = Database::new("temp3.db").await.unwrap(); - index_files_recursively(Path::new("/mnt/Music"), &conn) + index_files_recursively(Path::new("./testfiles"), &conn) + .await + .unwrap(); + + std::fs::remove_file("temp3.db").unwrap(); + } + + #[tokio::test] + async fn test_reencode_lots_of_files() { + let conn = Database::new("temp4.db").await.unwrap(); + let path = PathBuf::from("./testfiles"); + index_files_recursively(Path::new("./testfiles"), &conn) .await .unwrap(); - println!( - "\n{}", - conn.0 - .query("SELECT COUNT(DISTINCT path) FROM flacs", ()) - .await - .unwrap() - .next() - .await - .unwrap() - .unwrap() - .get_value(0) - .unwrap() - .as_integer() - .unwrap() - ); + println!("\n{}", count_reencode_files(None, &conn).await.unwrap()); + reencode_files(Some(&path), &conn).await.unwrap(); + std::fs::remove_file("temp4.db").unwrap(); } } diff --git a/src/flac.rs b/src/flac.rs index 64ac6a9..42f9d21 100644 --- a/src/flac.rs +++ b/src/flac.rs @@ -1,10 +1,9 @@ use anyhow::{Result, anyhow}; use flac_bound::{FlacEncoder, WriteWrapper}; use i24::i24; -use md5::{Digest, Md5, Md5Core, digest::core_api::CoreWrapper}; +use md5::{Digest, Md5}; use metaflac::{Block, Tag}; use std::{ - ffi::OsStr, fs::File, path::{Path, PathBuf}, }; @@ -54,13 +53,107 @@ impl Bps { } } +struct FileEncoder { + filename: PathBuf, + streamdata: StreamConfig, + format: BoxedFormatReader, + decoder: BoxedAudioDecoder, +} + +impl FileEncoder { + fn new(file: impl AsRef<Path>) -> Result<Self> { + let (format, decoder, config) = init_decoder(&file)?; + Ok(FileEncoder { + filename: file.as_ref().to_path_buf(), + streamdata: config, + format, + decoder, + }) + } + + fn temp_name(&self) -> PathBuf { + self.filename.clone().with_extension("tmp") + } + + fn encode(self, encoder: FlacEncoder) -> Result<()> { + let filename = self.filename.clone(); + let tempname = self.temp_name(); + + let hash = match self.streamdata.bits_per_sample { + Bps::_16 => encode_cycle_16(self.format, self.decoder, encoder)?, + Bps::_24 => encode_cycle_24(self.format, self.decoder, encoder)?, + Bps::_32 => encode_cycle_32(self.format, self.decoder, encoder)?, + }; + + let tags = Tag::read_from_path(&filename)?; + let mut output = Tag::read_from_path(&tempname)?; + let mut streaminfo = tags.get_streaminfo().unwrap().clone(); + + streaminfo.md5 = hash; + output.set_streaminfo(streaminfo); + + for block in tags.blocks() { + match block { + Block::VorbisComment(comment) => { + for (key, val) in comment.comments.clone() { + output.set_vorbis(key, val); + } + } + Block::StreamInfo(_) => {} + _ => output.push_block(block.clone()), + } + } + + output.write_to_path(&tempname)?; + + std::fs::rename(tempname, filename)?; + + Ok(()) + } +} + +fn init_decoder( + filename: impl AsRef<Path>, +) -> Result<(BoxedFormatReader, BoxedAudioDecoder, StreamConfig)> { + let src = std::fs::File::open(filename)?; + let mss = MediaSourceStream::new(Box::new(src), Default::default()); + let mut hint = Hint::new(); + hint.with_extension("flac"); + + let format_opts: FormatOptions = Default::default(); + let metadata_opts: MetadataOptions = Default::default(); + + let format = symphonia::default::get_probe() + .probe(&hint, mss, format_opts, metadata_opts) + .unwrap(); + + let track = format.default_track(TrackType::Audio).unwrap(); + + let decoder = symphonia::default::get_codecs() + .make_audio_decoder( + track.codec_params.as_ref().unwrap().audio().unwrap(), + &Default::default(), + ) + .unwrap(); + + let params = track.codec_params.as_ref().unwrap().audio().unwrap(); + + let config = StreamConfig { + channels: u32::try_from(params.channels.as_ref().unwrap().count()).unwrap(), + bits_per_sample: Bps::new(params.bits_per_sample.unwrap())?, + sample_rate: params.sample_rate.unwrap(), + }; + + Ok((format, decoder, config)) +} + fn encode_cycle_16( - mut format: Box<dyn FormatReader>, - mut decoder: Box<dyn AudioDecoder + 'static>, + mut format: BoxedFormatReader, + mut decoder: BoxedAudioDecoder, mut encoder: FlacEncoder, - hasher: &mut CoreWrapper<Md5Core>, -) -> Result<()> { +) -> Result<Vec<u8>> { let mut buffer: Vec<i32> = Vec::new(); + let mut hasher = Md5::new(); let track_id = format.default_track(TrackType::Audio).unwrap().id; loop { @@ -96,16 +189,16 @@ fn encode_cycle_16( return Err(anyhow!("Encoding failed:\t{:?}", enc.state())); } - Ok(()) + Ok(hasher.finalize().to_vec()) } fn encode_cycle_24( - mut format: Box<dyn FormatReader>, - mut decoder: Box<dyn AudioDecoder + 'static>, + mut format: BoxedFormatReader, + mut decoder: BoxedAudioDecoder, mut encoder: FlacEncoder, - hasher: &mut CoreWrapper<Md5Core>, -) -> Result<()> { +) -> Result<Vec<u8>> { let mut buffer: Vec<i32> = Vec::new(); + let mut hasher = Md5::new(); let track_id = format.default_track(TrackType::Audio).unwrap().id; loop { @@ -141,16 +234,16 @@ fn encode_cycle_24( return Err(anyhow!("Encoding failed:\t{:?}", enc.state())); } - Ok(()) + Ok(hasher.finalize().to_vec()) } fn encode_cycle_32( - mut format: Box<dyn FormatReader>, - mut decoder: Box<dyn AudioDecoder + 'static>, + mut format: BoxedFormatReader, + mut decoder: BoxedAudioDecoder, mut encoder: FlacEncoder, - hasher: &mut CoreWrapper<Md5Core>, -) -> Result<()> { +) -> Result<Vec<u8>> { let mut buffer: Vec<i32> = Vec::new(); + let mut hasher = Md5::new(); let track_id = format.default_track(TrackType::Audio).unwrap().id; loop { @@ -185,116 +278,41 @@ fn encode_cycle_32( return Err(anyhow!("Encoding failed:\t{:?}", enc.state())); } - Ok(()) -} - -fn init_decoder( - filename: impl AsRef<Path>, -) -> Result<(BoxedFormatReader, BoxedAudioDecoder, StreamConfig)> { - let src = std::fs::File::open(filename)?; - let mss = MediaSourceStream::new(Box::new(src), Default::default()); - let mut hint = Hint::new(); - hint.with_extension("flac"); - - let format_opts: FormatOptions = Default::default(); - let metadata_opts: MetadataOptions = Default::default(); - - let format = symphonia::default::get_probe() - .probe(&hint, mss, format_opts, metadata_opts) - .unwrap(); - - let track = format.default_track(TrackType::Audio).unwrap(); - - let decoder = symphonia::default::get_codecs() - .make_audio_decoder( - track.codec_params.as_ref().unwrap().audio().unwrap(), - &Default::default(), - ) - .unwrap(); - - let params = track.codec_params.as_ref().unwrap().audio().unwrap(); - - let config = StreamConfig { - channels: u32::try_from(params.channels.as_ref().unwrap().count()).unwrap(), - bits_per_sample: Bps::new(params.bits_per_sample.unwrap())?, - sample_rate: params.sample_rate.unwrap(), - }; - - Ok((format, decoder, config)) -} - -fn write_tags( - file: impl AsRef<Path>, - tempname: impl AsRef<Path>, - hasher: impl Digest, -) -> Result<()> { - let tags = Tag::read_from_path(file)?; - let mut output = Tag::read_from_path(&tempname)?; - let mut streaminfo = tags.get_streaminfo().unwrap().clone(); - - streaminfo.md5 = hasher.finalize()[..].to_vec(); - output.set_streaminfo(streaminfo); - - for block in tags.blocks() { - match block { - Block::VorbisComment(comment) => { - for (key, val) in comment.comments.clone() { - output.set_vorbis(key, val); - } - } - Block::StreamInfo(_) => {} - _ => output.push_block(block.clone()), - } - } - - output.write_to_path(&tempname)?; - Ok(()) + Ok(hasher.finalize().to_vec()) } -fn encode_file(filename: impl AsRef<OsStr>) -> Result<()> { - let file = Path::new(&filename); - let tempname = &format!("{}.tmp", file.to_str().unwrap()); - - let (format, decoder, config) = init_decoder(file)?; +fn encode_file(filename: impl AsRef<Path>) -> Result<()> { + let filencoder = FileEncoder::new(filename)?; - let mut outf = File::create(tempname)?; + let mut outf = File::create(filencoder.temp_name())?; let mut outw = WriteWrapper(&mut outf); let enc = FlacEncoder::new() .unwrap() - .channels(config.channels) - .bits_per_sample(config.bits_per_sample.value()) - .sample_rate(config.sample_rate) + .channels(filencoder.streamdata.channels) + .bits_per_sample(filencoder.streamdata.bits_per_sample.value()) + .sample_rate(filencoder.streamdata.sample_rate) .compression_level(8) .verify(false) .init_write(&mut outw) .unwrap(); - let mut hasher = Md5::new(); - - match config.bits_per_sample { - Bps::_16 => encode_cycle_16(format, decoder, enc, &mut hasher), - Bps::_24 => encode_cycle_24(format, decoder, enc, &mut hasher), - Bps::_32 => encode_cycle_32(format, decoder, enc, &mut hasher), - }?; - - write_tags(file, tempname, hasher)?; - - std::fs::rename(tempname, file)?; - - Ok(()) + filencoder.encode(enc) } -pub fn handle_encode(file: PathBuf) -> Result<()> { +pub fn handle_encode(file: impl AsRef<Path>) -> Result<impl AsRef<Path>> { if let Err(error) = encode_file(&file) { Err(anyhow!(files::FileError::new(file, error))) } else { - Ok(()) + Ok(file) } } -pub fn get_vendor(file: &Path) -> Result<String> { - let tag = Tag::read_from_path(file)?; - Ok(tag.vorbis_comments().unwrap().vendor_string.clone()) +pub fn get_vendor(file: impl AsRef<Path>) -> Result<String> { + if let Some(vorbis) = Tag::read_from_path(file)?.vorbis_comments() { + Ok(vorbis.vendor_string.to_owned()) + } else { + Err(anyhow!("Vendor string not found")) + } } #[cfg(test)] @@ -307,7 +325,7 @@ mod tests { let name = "16bit.flac"; let tempname = "16bit.flac.temp"; std::fs::copy(name, tempname).unwrap(); - encode_file(std::path::Path::new(name)).unwrap(); + encode_file(name).unwrap(); let target_md5 = Tag::read_from_path(tempname) .unwrap() .get_streaminfo() @@ -329,7 +347,7 @@ mod tests { let name = "24bit.flac"; let tempname = "24bit.flac.temp"; std::fs::copy(name, tempname).unwrap(); - encode_file(std::path::Path::new(name)).unwrap(); + encode_file(name).unwrap(); let target_md5 = Tag::read_from_path(tempname) .unwrap() .get_streaminfo() @@ -351,7 +369,7 @@ mod tests { let name = "32bit.flac"; let tempname = "32bit.flac.temp"; std::fs::copy(name, tempname).unwrap(); - encode_file(std::path::Path::new(name)).unwrap(); + encode_file(name).unwrap(); let target_md5 = Tag::read_from_path(tempname) .unwrap() .get_streaminfo() diff --git a/src/main.rs b/src/main.rs index d660f0a..5d3835f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,14 +16,6 @@ fn build_cli() -> Command { .value_parser(value_parser!(PathBuf)), ) .arg( - Arg::new("index") - .short('i') - .long("index") - .help("Only index files") - .requires("path") - .action(ArgAction::SetTrue), - ) - .arg( Arg::new("doit") .long("doit") .help("Actually reencode files") @@ -97,13 +89,13 @@ fn main() -> Result<()> { }; let path = args.get_one::<PathBuf>("path"); - if !args.get_flag("index") && !args.get_flag("clean") && !args.get_flag("doit") { - let count = files::count_reencode_files(&conn, path).await.unwrap(); + if path.is_none() && !args.get_flag("clean") && !args.get_flag("doit") { + let count = files::count_reencode_files(path, &conn).await.unwrap(); println!("Files to reencode:\t{count}"); - } - - if args.get_flag("index") { - files::index_files_recursively(path.unwrap(), &conn).await?; + } else if let Some(realpath) = path { + if !args.get_flag("doit") { + files::index_files_recursively(realpath, &conn).await?; + } } if args.get_flag("clean") { @@ -111,7 +103,7 @@ fn main() -> Result<()> { } if args.get_flag("doit") { - files::reencode_files(&conn, path).await?; + files::reencode_files(path, &conn).await?; } Ok::<(), anyhow::Error>(()) })?; |