diff --git a/Cargo.lock b/Cargo.lock index 919613f..c39b2ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -47,7 +62,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -58,7 +73,7 @@ checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", "once_cell", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -67,6 +82,27 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + [[package]] name = "bitflags" version = "2.9.0" @@ -79,6 +115,12 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "cfg-if" version = "1.0.0" @@ -141,7 +183,7 @@ dependencies = [ "libc", "once_cell", "unicode-width", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -163,7 +205,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -181,9 +223,15 @@ dependencies = [ "cfg-if", "libc", "r-efi", - "wasi", + "wasi 0.14.2+wasi-0.2.4", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + [[package]] name = "hashbrown" version = "0.15.3" @@ -221,6 +269,17 @@ dependencies = [ "web-time", ] +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -255,6 +314,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.27" @@ -276,6 +345,26 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -292,6 +381,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -304,6 +402,29 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "pick" version = "0.0.5" @@ -316,6 +437,7 @@ dependencies = [ "regex", "serde", "tempfile", + "tokio", "toml", "tracing", "tracing-subscriber", @@ -358,6 +480,15 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +[[package]] +name = "redox_syscall" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.11.1" @@ -402,6 +533,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rustc-demangle" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" + [[package]] name = "rustix" version = "1.0.7" @@ -412,7 +549,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -424,6 +561,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.219" @@ -462,12 +605,37 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" + [[package]] name = "smallvec" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "strsim" version = "0.11.1" @@ -495,7 +663,7 @@ dependencies = [ "getrandom", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -508,6 +676,37 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tokio" +version = "1.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "slab", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "toml" version = "0.8.22" @@ -651,6 +850,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasi" version = "0.14.2+wasi-0.2.4" @@ -749,7 +954,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -758,6 +963,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.59.0" diff --git a/Cargo.toml b/Cargo.toml index 4e6f418..4930160 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,3 +70,4 @@ walkdir = "2.5.0" tempfile = "3.10.1" anyhow = "1.0.98" indicatif = { version = "0.17.11", features = ["improved_unicode"] } +tokio = { version = "1.46.1", features = ["full"] } diff --git a/src/logging.rs b/src/logging.rs index a0990fb..81c9896 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -140,7 +140,7 @@ impl LoggingPreset { /// log: LogCli, /// } /// ``` -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Clone)] pub struct LogCli { /// Increase verbosity (can be repeated) #[arg(short, action = clap::ArgAction::Count,global = true)] diff --git a/src/main.rs b/src/main.rs index 0768248..30765d7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,10 @@ -use anyhow::Result; +use anyhow::{Error, Result}; use clap::Parser; use indicatif::ProgressIterator; use logging::LogCli; use std::{path::PathBuf, process::ExitCode}; use style::progress_style; +use tokio::task::JoinSet; use tool::PickTool; use tracing::{debug, error, trace}; @@ -15,7 +16,6 @@ pub mod tool; mod prepare; // count size of files to process -// parallelism // capture/print stdout/stderr // workdir vs root // package & auto-build @@ -37,7 +37,7 @@ mod prepare; // warn when no matches /// Pick is a utility that processes files based on pattern matching rules. -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Clone)] #[command(version, about, long_about = None, styles=style::get_styles())] struct Cli { #[command(flatten)] @@ -59,10 +59,6 @@ fn main() -> ExitCode { } fn main_inner() -> Result { - // - // MARK: setup - // - let mut cli = Cli::parse(); cli.manifest = std::path::absolute(&cli.manifest)?; let cli = cli; @@ -81,18 +77,55 @@ fn main_inner() -> Result { let queue = prepare::list_queue(&manifest, &work_dir)?; - #[expect(clippy::unwrap_used)] // Fix later - let bash = manifest.tool.bash.as_ref().unwrap(); - bash.before(&cli.manifest, &manifest.config)?; + let bash = { + #[expect(clippy::unwrap_used)] // Fix later + let bash = manifest.tool.bash.clone().unwrap(); + bash.before(&cli.manifest, &manifest.config)?; - for ctx in queue - .into_iter() - .progress_with_style(progress_style()) - .with_message("Processing") - { - trace!("Running `{}` on {}", ctx.task, ctx.path_rel_str); - bash.run(&cli.manifest, &manifest.config, ctx)?; - } + bash + }; + + let max_tasks = manifest.config.threads.map(|x| x.get()).unwrap_or(1); + + debug!("Starting runtime with {max_tasks} threads"); + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(3) + .max_blocking_threads(max_tasks) + .build() + .unwrap(); + + let rt_cli = cli.clone(); + let rt_manifest = manifest.clone(); + let rt_bash = bash.clone(); + rt.block_on(async move { + let mut js: JoinSet> = JoinSet::new(); + + for ctx in queue + .into_iter() + .progress_with_style(progress_style()) + .with_message("Processing") + { + trace!("Running `{}` on {}", ctx.task, ctx.path_rel_str); + + let cli = rt_cli.clone(); + let manifest = rt_manifest.clone(); + let bash = rt_bash.clone(); + js.spawn_blocking(move || { + bash.run(&cli.manifest, &manifest.config, ctx)?; + return Ok(()); + }); + + if js.len() >= max_tasks { + js.join_next().await.unwrap()??; + } + } + + while let Some(x) = js.join_next().await { + x.unwrap()? + } + + return Ok::<_, Error>(()); + })?; bash.after(&cli.manifest, &manifest.config)?; diff --git a/src/manifest/types.rs b/src/manifest/types.rs index 6f2e3ce..611cedb 100644 --- a/src/manifest/types.rs +++ b/src/manifest/types.rs @@ -1,13 +1,16 @@ use anyhow::Result; use indexmap::IndexMap; use serde::Deserialize; -use std::path::{Path, PathBuf}; +use std::{ + num::NonZero, + path::{Path, PathBuf}, +}; use crate::tool::ToolConfig; use super::rule::FlatPickRule; -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] #[serde(deny_unknown_fields)] pub struct Manifest { pub config: PickConfig, @@ -32,6 +35,9 @@ pub struct PickConfig { #[serde(default = "default_false")] pub process_links: bool, + + #[serde(default)] + pub threads: Option>, } impl PickConfig { diff --git a/src/tool/bash.rs b/src/tool/bash.rs index 438ce14..1473c80 100644 --- a/src/tool/bash.rs +++ b/src/tool/bash.rs @@ -8,7 +8,7 @@ use crate::manifest::types::PickConfig; use super::{PickTool, TaskContext}; -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct ToolBash { #[serde(default)] pub before: Option, diff --git a/src/tool/mod.rs b/src/tool/mod.rs index 9db3b2c..acf0318 100644 --- a/src/tool/mod.rs +++ b/src/tool/mod.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use serde::{Deserialize, de::DeserializeOwned}; +use serde::{de::DeserializeOwned, Deserialize}; use std::{ fmt::Debug, path::{Path, PathBuf}, @@ -41,7 +41,7 @@ pub struct TaskContext { pub path_rel_str: String, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct ToolConfig { #[serde(default)] pub bash: Option,