Mark/pick
1
0
Fork 0
Code Pull Requests Actions Packages Releases Activity

Parallelism

Browse Source
This commit is contained in:
rm-dr
2025-07-13 17:18:25 +00:00
parent 939ccc90fe
commit cd7d5008dc
7 changed files with 286 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
Cargo.lock generated
View File

@ -2,6 +2,21 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "addr2line"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
dependencies = [
"gimli",
]
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aho-corasick"
version = "1.1.3"
@ -47,7 +62,7 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
dependencies = [
"windows-sys",
"windows-sys 0.59.0",
]
[[package]]
@ -58,7 +73,7 @@ checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
dependencies = [
"anstyle",
"once_cell",
"windows-sys",
"windows-sys 0.59.0",
]
[[package]]
@ -67,6 +82,27 @@ version = "1.0.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
[[package]]
name = "autocfg"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "backtrace"
version = "0.3.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002"
dependencies = [
"addr2line",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
"windows-targets",
]
[[package]]
name = "bitflags"
version = "2.9.0"
@ -79,6 +115,12 @@ version = "3.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
[[package]]
name = "bytes"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -141,7 +183,7 @@ dependencies = [
"libc",
"once_cell",
"unicode-width",
"windows-sys",
"windows-sys 0.59.0",
]
[[package]]
@ -163,7 +205,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
dependencies = [
"libc",
"windows-sys",
"windows-sys 0.59.0",
]
[[package]]
@ -181,9 +223,15 @@ dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasi",
"wasi 0.14.2+wasi-0.2.4",
]
[[package]]
name = "gimli"
version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
[[package]]
name = "hashbrown"
version = "0.15.3"
@ -221,6 +269,17 @@ dependencies = [
"web-time",
]
[[package]]
name = "io-uring"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013"
dependencies = [
"bitflags",
"cfg-if",
"libc",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
@ -255,6 +314,16 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "lock_api"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.27"
@ -276,6 +345,26 @@ version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
]
[[package]]
name = "mio"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
dependencies = [
"libc",
"wasi 0.11.1+wasi-snapshot-preview1",
"windows-sys 0.59.0",
]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
@ -292,6 +381,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "object"
version = "0.36.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
version = "1.21.3"
@ -304,6 +402,29 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "parking_lot"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-targets",
]
[[package]]
name = "pick"
version = "0.0.5"
@ -316,6 +437,7 @@ dependencies = [
"regex",
"serde",
"tempfile",
"tokio",
"toml",
"tracing",
"tracing-subscriber",
@ -358,6 +480,15 @@ version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
[[package]]
name = "redox_syscall"
version = "0.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.11.1"
@ -402,6 +533,12 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "rustc-demangle"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f"
[[package]]
name = "rustix"
version = "1.0.7"
@ -412,7 +549,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
"windows-sys 0.59.0",
]
[[package]]
@ -424,6 +561,12 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "serde"
version = "1.0.219"
@ -462,12 +605,37 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
dependencies = [
"libc",
]
[[package]]
name = "slab"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d"
[[package]]
name = "smallvec"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
[[package]]
name = "socket2"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "strsim"
version = "0.11.1"
@ -495,7 +663,7 @@ dependencies = [
"getrandom",
"once_cell",
"rustix",
"windows-sys",
"windows-sys 0.59.0",
]
[[package]]
@ -508,6 +676,37 @@ dependencies = [
"once_cell",
]
[[package]]
name = "tokio"
version = "1.46.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
dependencies = [
"backtrace",
"bytes",
"io-uring",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"slab",
"socket2",
"tokio-macros",
"windows-sys 0.52.0",
]
[[package]]
name = "tokio-macros"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "toml"
version = "0.8.22"
@ -651,6 +850,12 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasi"
version = "0.14.2+wasi-0.2.4"
@ -749,7 +954,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys",
"windows-sys 0.59.0",
]
[[package]]
@ -758,6 +963,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-sys"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-sys"
version = "0.59.0"

1
Cargo.toml
View File

@ -70,3 +70,4 @@ walkdir = "2.5.0"
tempfile = "3.10.1"
anyhow = "1.0.98"
indicatif = { version = "0.17.11", features = ["improved_unicode"] }
tokio = { version = "1.46.1", features = ["full"] }

2
src/logging.rs
View File

@ -140,7 +140,7 @@ impl LoggingPreset {
/// log: LogCli,
/// }
/// ```
#[derive(Parser, Debug)]
#[derive(Parser, Debug, Clone)]
pub struct LogCli {
/// Increase verbosity (can be repeated)
#[arg(short, action = clap::ArgAction::Count,global = true)]

69
src/main.rs
View File

@ -1,9 +1,10 @@
use anyhow::Result;
use anyhow::{Error, Result};
use clap::Parser;
use indicatif::ProgressIterator;
use logging::LogCli;
use std::{path::PathBuf, process::ExitCode};
use style::progress_style;
use tokio::task::JoinSet;
use tool::PickTool;
use tracing::{debug, error, trace};
@ -15,7 +16,6 @@ pub mod tool;
mod prepare;
// count size of files to process
// parallelism
// capture/print stdout/stderr
// workdir vs root
// package & auto-build
@ -37,7 +37,7 @@ mod prepare;
// warn when no matches
/// Pick is a utility that processes files based on pattern matching rules.
#[derive(Parser, Debug)]
#[derive(Parser, Debug, Clone)]
#[command(version, about, long_about = None, styles=style::get_styles())]
struct Cli {
#[command(flatten)]
@ -59,10 +59,6 @@ fn main() -> ExitCode {
}
fn main_inner() -> Result<ExitCode> {
//
// MARK: setup
//
let mut cli = Cli::parse();
cli.manifest = std::path::absolute(&cli.manifest)?;
let cli = cli;
@ -81,18 +77,55 @@ fn main_inner() -> Result<ExitCode> {
let queue = prepare::list_queue(&manifest, &work_dir)?;
#[expect(clippy::unwrap_used)] // Fix later
let bash = manifest.tool.bash.as_ref().unwrap();
bash.before(&cli.manifest, &manifest.config)?;
let bash = {
#[expect(clippy::unwrap_used)] // Fix later
let bash = manifest.tool.bash.clone().unwrap();
bash.before(&cli.manifest, &manifest.config)?;
for ctx in queue
.into_iter()
.progress_with_style(progress_style())
.with_message("Processing")
{
trace!("Running `{}` on {}", ctx.task, ctx.path_rel_str);
bash.run(&cli.manifest, &manifest.config, ctx)?;
}
bash
};
let max_tasks = manifest.config.threads.map(|x| x.get()).unwrap_or(1);
debug!("Starting runtime with {max_tasks} threads");
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(3)
.max_blocking_threads(max_tasks)
.build()
.unwrap();
let rt_cli = cli.clone();
let rt_manifest = manifest.clone();
let rt_bash = bash.clone();
rt.block_on(async move {
let mut js: JoinSet<Result<()>> = JoinSet::new();
for ctx in queue
.into_iter()
.progress_with_style(progress_style())
.with_message("Processing")
{
trace!("Running `{}` on {}", ctx.task, ctx.path_rel_str);
let cli = rt_cli.clone();
let manifest = rt_manifest.clone();
let bash = rt_bash.clone();
js.spawn_blocking(move || {
bash.run(&cli.manifest, &manifest.config, ctx)?;
return Ok(());
});
if js.len() >= max_tasks {
js.join_next().await.unwrap()??;
}
}
while let Some(x) = js.join_next().await {
x.unwrap()?
}
return Ok::<_, Error>(());
})?;
bash.after(&cli.manifest, &manifest.config)?;

10
src/manifest/types.rs
View File

@ -1,13 +1,16 @@
use anyhow::Result;
use indexmap::IndexMap;
use serde::Deserialize;
use std::path::{Path, PathBuf};
use std::{
num::NonZero,
path::{Path, PathBuf},
};
use crate::tool::ToolConfig;
use super::rule::FlatPickRule;
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Clone)]
#[serde(deny_unknown_fields)]
pub struct Manifest {
pub config: PickConfig,
@ -32,6 +35,9 @@ pub struct PickConfig {
#[serde(default = "default_false")]
pub process_links: bool,
#[serde(default)]
pub threads: Option<NonZero<usize>>,
}
impl PickConfig {

2
src/tool/bash.rs
View File

@ -8,7 +8,7 @@ use crate::manifest::types::PickConfig;
use super::{PickTool, TaskContext};
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Clone)]
pub struct ToolBash {
#[serde(default)]
pub before: Option<String>,

4
src/tool/mod.rs
View File

@ -1,5 +1,5 @@
use anyhow::Result;
use serde::{Deserialize, de::DeserializeOwned};
use serde::{de::DeserializeOwned, Deserialize};
use std::{
fmt::Debug,
path::{Path, PathBuf},
@ -41,7 +41,7 @@ pub struct TaskContext {
pub path_rel_str: String,
}
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Clone)]
pub struct ToolConfig {
#[serde(default)]
pub bash: Option<ToolBash>,