Configure server though env
All checks were successful
All checks were successful
This commit is contained in:
91
Cargo.lock
generated
91
Cargo.lock
generated
@@ -721,6 +721,12 @@ dependencies = [
|
|||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dotenvy"
|
||||||
|
version = "0.15.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "downcast-rs"
|
name = "downcast-rs"
|
||||||
version = "2.0.2"
|
version = "2.0.2"
|
||||||
@@ -748,6 +754,15 @@ dependencies = [
|
|||||||
"cfg-if",
|
"cfg-if",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "envy"
|
||||||
|
version = "0.4.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "epub"
|
name = "epub"
|
||||||
version = "1.2.4"
|
version = "1.2.4"
|
||||||
@@ -1584,6 +1599,16 @@ version = "0.4.29"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "loki-api"
|
||||||
|
version = "0.1.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bdc38a304f59a03e6efa3876766a48c70a766a93f88341c3fff4212834b8e327"
|
||||||
|
dependencies = [
|
||||||
|
"prost",
|
||||||
|
"prost-types",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lru"
|
name = "lru"
|
||||||
version = "0.12.5"
|
version = "0.12.5"
|
||||||
@@ -1962,7 +1987,10 @@ dependencies = [
|
|||||||
"anstyle",
|
"anstyle",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"axum",
|
"axum",
|
||||||
|
"base64",
|
||||||
"clap",
|
"clap",
|
||||||
|
"dotenvy",
|
||||||
|
"envy",
|
||||||
"indicatif",
|
"indicatif",
|
||||||
"pile-config",
|
"pile-config",
|
||||||
"pile-dataset",
|
"pile-dataset",
|
||||||
@@ -1970,11 +1998,14 @@ dependencies = [
|
|||||||
"pile-value",
|
"pile-value",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
"toml",
|
"toml",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-indicatif",
|
"tracing-indicatif",
|
||||||
|
"tracing-loki",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
"url",
|
||||||
"utoipa",
|
"utoipa",
|
||||||
"utoipa-swagger-ui",
|
"utoipa-swagger-ui",
|
||||||
]
|
]
|
||||||
@@ -2172,6 +2203,38 @@ dependencies = [
|
|||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "prost"
|
||||||
|
version = "0.13.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"prost-derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "prost-derive"
|
||||||
|
version = "0.13.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"itertools 0.14.0",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.117",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "prost-types"
|
||||||
|
version = "0.13.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16"
|
||||||
|
dependencies = [
|
||||||
|
"prost",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pxfm"
|
name = "pxfm"
|
||||||
version = "0.1.28"
|
version = "0.1.28"
|
||||||
@@ -2697,6 +2760,12 @@ dependencies = [
|
|||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "snap"
|
||||||
|
version = "1.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "socket2"
|
name = "socket2"
|
||||||
version = "0.6.2"
|
version = "0.6.2"
|
||||||
@@ -3289,6 +3358,27 @@ dependencies = [
|
|||||||
"tracing-core",
|
"tracing-core",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tracing-loki"
|
||||||
|
version = "0.2.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ba3beec919fbdf99d719de8eda6adae3281f8a5b71ae40431f44dc7423053d34"
|
||||||
|
dependencies = [
|
||||||
|
"loki-api",
|
||||||
|
"reqwest",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"snap",
|
||||||
|
"tokio",
|
||||||
|
"tokio-stream",
|
||||||
|
"tracing",
|
||||||
|
"tracing-core",
|
||||||
|
"tracing-log",
|
||||||
|
"tracing-serde",
|
||||||
|
"tracing-subscriber",
|
||||||
|
"url",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tracing-serde"
|
name = "tracing-serde"
|
||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
@@ -3415,6 +3505,7 @@ dependencies = [
|
|||||||
"idna",
|
"idna",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"serde",
|
"serde",
|
||||||
|
"serde_derive",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
17
Cargo.toml
17
Cargo.toml
@@ -72,7 +72,7 @@ pile-value = { path = "crates/pile-value" }
|
|||||||
pile-io = { path = "crates/pile-io" }
|
pile-io = { path = "crates/pile-io" }
|
||||||
pile-client = { path = "crates/pile-client" }
|
pile-client = { path = "crates/pile-client" }
|
||||||
|
|
||||||
# Clients & servers
|
# MARK: Clients & servers
|
||||||
tantivy = "0.25.0"
|
tantivy = "0.25.0"
|
||||||
servable = { version = "0.0.7", features = ["image"] }
|
servable = { version = "0.0.7", features = ["image"] }
|
||||||
axum = { version = "0.8.8", features = ["macros", "multipart"] }
|
axum = { version = "0.8.8", features = ["macros", "multipart"] }
|
||||||
@@ -88,14 +88,14 @@ utoipa-swagger-ui = { version = "9.0.2", features = [
|
|||||||
"vendored",
|
"vendored",
|
||||||
] }
|
] }
|
||||||
reqwest = { version = "0.12", features = ["blocking"] }
|
reqwest = { version = "0.12", features = ["blocking"] }
|
||||||
|
tracing-loki = "0.2.6"
|
||||||
|
|
||||||
|
# MARK: Async & Parallelism
|
||||||
# Async & Parallelism
|
|
||||||
tokio = { version = "1.49.0", features = ["full"] }
|
tokio = { version = "1.49.0", features = ["full"] }
|
||||||
tokio-stream = "0.1"
|
tokio-stream = "0.1"
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
|
|
||||||
# CLI & logging
|
# MARK: CLI & logging
|
||||||
tracing = "0.1.44"
|
tracing = "0.1.44"
|
||||||
tracing-subscriber = { version = "0.3.22", features = ["env-filter", "json"] }
|
tracing-subscriber = { version = "0.3.22", features = ["env-filter", "json"] }
|
||||||
indicatif = { version = "0.18.4", features = ["improved_unicode"] }
|
indicatif = { version = "0.18.4", features = ["improved_unicode"] }
|
||||||
@@ -103,7 +103,7 @@ tracing-indicatif = "0.3.14"
|
|||||||
anstyle = "1.0.13"
|
anstyle = "1.0.13"
|
||||||
clap = { version = "4.5.60", features = ["derive"] }
|
clap = { version = "4.5.60", features = ["derive"] }
|
||||||
|
|
||||||
# Serialization & formats
|
# MARK: Serialization & formats
|
||||||
serde = { version = "1.0.228", features = ["derive"] }
|
serde = { version = "1.0.228", features = ["derive"] }
|
||||||
serde_json = "1.0.149"
|
serde_json = "1.0.149"
|
||||||
base64 = "0.22.1"
|
base64 = "0.22.1"
|
||||||
@@ -112,8 +112,10 @@ toml = "1.0.3"
|
|||||||
toml_edit = "0.25.4"
|
toml_edit = "0.25.4"
|
||||||
sha2 = "0.11.0-rc.5"
|
sha2 = "0.11.0-rc.5"
|
||||||
blake3 = "1.8.3"
|
blake3 = "1.8.3"
|
||||||
|
dotenvy = "0.15.7"
|
||||||
|
envy = "0.4.2"
|
||||||
|
|
||||||
# Extractors
|
# MARK: Extractors
|
||||||
pdf = "0.10.0"
|
pdf = "0.10.0"
|
||||||
id3 = "1.16.4"
|
id3 = "1.16.4"
|
||||||
epub = "1.2.2"
|
epub = "1.2.2"
|
||||||
@@ -121,7 +123,7 @@ kamadak-exif = "0.6.1"
|
|||||||
pdfium-render = "0.8"
|
pdfium-render = "0.8"
|
||||||
image = { version = "0.25", default-features = false, features = ["png"] }
|
image = { version = "0.25", default-features = false, features = ["png"] }
|
||||||
|
|
||||||
# Misc helpers
|
# MARK: Misc helpers
|
||||||
thiserror = "2.0.18"
|
thiserror = "2.0.18"
|
||||||
anyhow = "1.0.102"
|
anyhow = "1.0.102"
|
||||||
itertools = "0.14.0"
|
itertools = "0.14.0"
|
||||||
@@ -137,3 +139,4 @@ chrono = "0.4.43"
|
|||||||
parking_lot = "0.12.5"
|
parking_lot = "0.12.5"
|
||||||
rayon = "1.11.0"
|
rayon = "1.11.0"
|
||||||
percent-encoding = "2"
|
percent-encoding = "2"
|
||||||
|
url = { version = "2.5.8", features = ["serde"] }
|
||||||
|
|||||||
@@ -28,3 +28,9 @@ serde_json = { workspace = true }
|
|||||||
axum = { workspace = true }
|
axum = { workspace = true }
|
||||||
utoipa = { workspace = true }
|
utoipa = { workspace = true }
|
||||||
utoipa-swagger-ui = { workspace = true }
|
utoipa-swagger-ui = { workspace = true }
|
||||||
|
url = { workspace = true }
|
||||||
|
tracing-loki = { workspace = true }
|
||||||
|
base64 = { workspace = true }
|
||||||
|
dotenvy = { workspace = true }
|
||||||
|
envy = { workspace = true }
|
||||||
|
thiserror = { workspace = true }
|
||||||
|
|||||||
@@ -20,10 +20,6 @@ use crate::{CliCmd, GlobalContext};
|
|||||||
|
|
||||||
#[derive(Debug, Args)]
|
#[derive(Debug, Args)]
|
||||||
pub struct ServerCommand {
|
pub struct ServerCommand {
|
||||||
/// Address to bind to
|
|
||||||
#[arg(default_value = "0.0.0.0:9000")]
|
|
||||||
addr: String,
|
|
||||||
|
|
||||||
/// The datasets we should serve. Can be repeated.
|
/// The datasets we should serve. Can be repeated.
|
||||||
#[arg(long, short = 'c')]
|
#[arg(long, short = 'c')]
|
||||||
config: Vec<PathBuf>,
|
config: Vec<PathBuf>,
|
||||||
@@ -31,26 +27,18 @@ pub struct ServerCommand {
|
|||||||
/// If provided, do not serve docs
|
/// If provided, do not serve docs
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
no_docs: bool,
|
no_docs: bool,
|
||||||
|
|
||||||
/// If provided, require this bearer token for all requests
|
|
||||||
#[arg(long)]
|
|
||||||
token: Option<String>,
|
|
||||||
|
|
||||||
/// Working directory root
|
|
||||||
#[arg(long, default_value = "./.pile")]
|
|
||||||
workdir: PathBuf,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CliCmd for ServerCommand {
|
impl CliCmd for ServerCommand {
|
||||||
async fn run(
|
async fn run(
|
||||||
self,
|
self,
|
||||||
_ctx: GlobalContext,
|
ctx: GlobalContext,
|
||||||
flag: CancelFlag,
|
flag: CancelFlag,
|
||||||
) -> Result<i32, CancelableTaskError<anyhow::Error>> {
|
) -> Result<i32, CancelableTaskError<anyhow::Error>> {
|
||||||
let datasets = {
|
let datasets = {
|
||||||
let mut datasets = Vec::new();
|
let mut datasets = Vec::new();
|
||||||
for c in &self.config {
|
for c in &self.config {
|
||||||
let ds = Datasets::open(&c, &self.workdir)
|
let ds = Datasets::open(&c, &ctx.config.workdir_root)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("while opening dataset for {}", c.display()))?;
|
.with_context(|| format!("while opening dataset for {}", c.display()))?;
|
||||||
datasets.push(Arc::new(ds));
|
datasets.push(Arc::new(ds));
|
||||||
@@ -59,7 +47,7 @@ impl CliCmd for ServerCommand {
|
|||||||
Arc::new(datasets)
|
Arc::new(datasets)
|
||||||
};
|
};
|
||||||
|
|
||||||
let bearer = BearerToken(self.token.map(Arc::new));
|
let bearer = BearerToken(ctx.config.api_token.clone().map(Arc::new));
|
||||||
|
|
||||||
let mut router = Router::new();
|
let mut router = Router::new();
|
||||||
for d in datasets.iter() {
|
for d in datasets.iter() {
|
||||||
@@ -85,14 +73,14 @@ impl CliCmd for ServerCommand {
|
|||||||
|
|
||||||
let app = router.into_make_service_with_connect_info::<std::net::SocketAddr>();
|
let app = router.into_make_service_with_connect_info::<std::net::SocketAddr>();
|
||||||
|
|
||||||
let listener = match tokio::net::TcpListener::bind(self.addr.clone()).await {
|
let listener = match tokio::net::TcpListener::bind(ctx.config.server_addr.clone()).await {
|
||||||
Ok(x) => x,
|
Ok(x) => x,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
match error.kind() {
|
match error.kind() {
|
||||||
std::io::ErrorKind::AddrInUse => {
|
std::io::ErrorKind::AddrInUse => {
|
||||||
error!(
|
error!(
|
||||||
message = "Cannot bind to address, already in use",
|
message = "Cannot bind to address, already in use",
|
||||||
addr = self.addr
|
addr = ctx.config.server_addr
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
|
|||||||
109
crates/pile/src/config/config.rs
Normal file
109
crates/pile/src/config/config.rs
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
use serde::Deserialize;
|
||||||
|
use std::{num::NonZeroUsize, path::PathBuf};
|
||||||
|
use tracing::info;
|
||||||
|
|
||||||
|
use crate::config::{
|
||||||
|
env::load_env,
|
||||||
|
logging::{LoggingFormat, LoggingInitializer, LoggingPreset, LoggingTarget, LokiConfig},
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Note that the field of this struct are not capitalized.
|
||||||
|
/// Envy is case-insensitive, and expects Rust fields to be snake_case.
|
||||||
|
#[derive(Debug, Deserialize, Clone)]
|
||||||
|
pub struct PileServerConfig {
|
||||||
|
#[serde(flatten)]
|
||||||
|
pub loki: Option<LokiConfig>,
|
||||||
|
|
||||||
|
/// The logging level to run with
|
||||||
|
#[serde(default)]
|
||||||
|
pub loglevel: LoggingPreset,
|
||||||
|
|
||||||
|
#[serde(default)]
|
||||||
|
pub logformat: LoggingFormat,
|
||||||
|
|
||||||
|
/// How many worker threads to use
|
||||||
|
pub threads: Option<NonZeroUsize>,
|
||||||
|
|
||||||
|
/// IP and port to bind to
|
||||||
|
/// Should look like `127.0.0.1:3030`
|
||||||
|
pub server_addr: String,
|
||||||
|
|
||||||
|
pub api_token: Option<String>,
|
||||||
|
pub workdir_root: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for PileServerConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
loki: None,
|
||||||
|
loglevel: LoggingPreset::Debug,
|
||||||
|
logformat: LoggingFormat::Ansi,
|
||||||
|
threads: None,
|
||||||
|
server_addr: "0.0.0.0:3000".into(),
|
||||||
|
api_token: None,
|
||||||
|
workdir_root: "./.pile".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PileServerConfig {
|
||||||
|
pub fn load(with_env: bool, cli_log_level: LoggingPreset) -> Self {
|
||||||
|
let config = match with_env {
|
||||||
|
false => Self::default(),
|
||||||
|
true => {
|
||||||
|
let env = match load_env::<Self>() {
|
||||||
|
Ok(x) => x,
|
||||||
|
|
||||||
|
#[expect(clippy::print_stdout)]
|
||||||
|
Err(err) => {
|
||||||
|
println!("Error while loading .env: {err}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
env.get_config().clone()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
{
|
||||||
|
let res = LoggingInitializer {
|
||||||
|
app_name: "pile-server",
|
||||||
|
loki: config.loki.clone(),
|
||||||
|
preset: if with_env {
|
||||||
|
config.loglevel
|
||||||
|
} else {
|
||||||
|
cli_log_level
|
||||||
|
},
|
||||||
|
target: LoggingTarget::Stderr {
|
||||||
|
format: config.logformat,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
.initialize();
|
||||||
|
|
||||||
|
if let Err(e) = res {
|
||||||
|
#[expect(clippy::print_stderr)]
|
||||||
|
for e in e.chain() {
|
||||||
|
eprintln!("{e}");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
info!(message = "Config loaded", ?config);
|
||||||
|
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn make_runtime(&self) -> tokio::runtime::Runtime {
|
||||||
|
let mut rt = tokio::runtime::Builder::new_multi_thread();
|
||||||
|
rt.enable_all();
|
||||||
|
if let Some(threads) = self.threads {
|
||||||
|
rt.worker_threads(threads.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[expect(clippy::unwrap_used)]
|
||||||
|
let rt = rt.build().unwrap();
|
||||||
|
|
||||||
|
return rt;
|
||||||
|
}
|
||||||
|
}
|
||||||
108
crates/pile/src/config/env.rs
Normal file
108
crates/pile/src/config/env.rs
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
#![expect(dead_code)]
|
||||||
|
|
||||||
|
use serde::de::DeserializeOwned;
|
||||||
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
|
env::VarError,
|
||||||
|
io::ErrorKind,
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
};
|
||||||
|
use thiserror::Error;
|
||||||
|
|
||||||
|
/// An error we might encounter when loading an env
|
||||||
|
#[derive(Debug, Error)]
|
||||||
|
pub enum EnvLoadError {
|
||||||
|
#[error("i/o error")]
|
||||||
|
IOError(#[from] std::io::Error),
|
||||||
|
|
||||||
|
#[error("varerror")]
|
||||||
|
VarError(#[from] VarError),
|
||||||
|
|
||||||
|
#[error("line parse error: `{on_line}` at char {at_char}")]
|
||||||
|
LineParse { on_line: String, at_char: usize },
|
||||||
|
|
||||||
|
#[error("other dotenvy error")]
|
||||||
|
Other(#[from] dotenvy::Error),
|
||||||
|
|
||||||
|
#[error("missing value {0}")]
|
||||||
|
MissingValue(String),
|
||||||
|
|
||||||
|
#[error("parse error: {0}")]
|
||||||
|
OtherParseError(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
pub enum LoadedEnv<T> {
|
||||||
|
/// We loaded config from `.env` and env vars
|
||||||
|
FoundFile { config: T, path: PathBuf },
|
||||||
|
|
||||||
|
/// We could not find `.env` and only loaded env vars
|
||||||
|
OnlyVars(T),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> LoadedEnv<T> {
|
||||||
|
pub fn get_config(&self) -> &T {
|
||||||
|
match self {
|
||||||
|
Self::FoundFile { config, .. } => config,
|
||||||
|
Self::OnlyVars(config) => config,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load the configuration type `T` from the current environment,
|
||||||
|
/// including the `.env` if it exists.
|
||||||
|
#[expect(clippy::wildcard_enum_match_arm)]
|
||||||
|
pub fn load_env<T: DeserializeOwned>() -> Result<LoadedEnv<T>, EnvLoadError> {
|
||||||
|
let env_path = match dotenvy::dotenv() {
|
||||||
|
Ok(path) => Some(path),
|
||||||
|
|
||||||
|
Err(dotenvy::Error::Io(err)) => match err.kind() {
|
||||||
|
ErrorKind::NotFound => None,
|
||||||
|
_ => return Err(EnvLoadError::IOError(err)),
|
||||||
|
},
|
||||||
|
|
||||||
|
Err(dotenvy::Error::EnvVar(err)) => {
|
||||||
|
return Err(EnvLoadError::VarError(err));
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(dotenvy::Error::LineParse(on_line, at_char)) => {
|
||||||
|
return Err(EnvLoadError::LineParse { on_line, at_char });
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(err) => {
|
||||||
|
return Err(EnvLoadError::Other(err));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match envy::from_env::<T>() {
|
||||||
|
Ok(config) => {
|
||||||
|
if let Some(path) = env_path {
|
||||||
|
return Ok(LoadedEnv::FoundFile { path, config });
|
||||||
|
} else {
|
||||||
|
return Ok(LoadedEnv::OnlyVars(config));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(envy::Error::MissingValue(value)) => {
|
||||||
|
return Err(EnvLoadError::MissingValue(value.into()));
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(envy::Error::Custom(message)) => {
|
||||||
|
return Err(EnvLoadError::OtherParseError(message));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load an .env file to a hashmap.
|
||||||
|
///
|
||||||
|
/// This function does not read the current env,
|
||||||
|
/// only parsing vars explicitly declared in the given file.
|
||||||
|
pub fn load_env_dict(p: impl AsRef<Path>) -> Result<HashMap<String, String>, EnvLoadError> {
|
||||||
|
let mut out = HashMap::new();
|
||||||
|
|
||||||
|
for item in dotenvy::from_filename_iter(p)? {
|
||||||
|
let (key, val) = item?;
|
||||||
|
out.insert(key, val);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Ok(out);
|
||||||
|
}
|
||||||
@@ -1,7 +1,13 @@
|
|||||||
|
use anyhow::Result;
|
||||||
use clap::ValueEnum;
|
use clap::ValueEnum;
|
||||||
|
use indicatif::MultiProgress;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use std::{fmt::Display, str::FromStr};
|
use std::{fmt::Display, str::FromStr};
|
||||||
use tracing_subscriber::EnvFilter;
|
use tracing_indicatif::IndicatifWriter;
|
||||||
|
use tracing_subscriber::{
|
||||||
|
EnvFilter, Layer, fmt::MakeWriter, layer::SubscriberExt, util::SubscriberInitExt,
|
||||||
|
};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub enum LogLevel {
|
pub enum LogLevel {
|
||||||
@@ -32,6 +38,7 @@ pub enum LoggingPreset {
|
|||||||
Info,
|
Info,
|
||||||
Debug,
|
Debug,
|
||||||
Trace,
|
Trace,
|
||||||
|
Loki,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct LoggingConfig {
|
pub struct LoggingConfig {
|
||||||
@@ -138,6 +145,203 @@ impl LoggingPreset {
|
|||||||
pile_dataset: LogLevel::Trace,
|
pile_dataset: LogLevel::Trace,
|
||||||
pile_toolbox: LogLevel::Trace,
|
pile_toolbox: LogLevel::Trace,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
Self::Loki => LoggingConfig {
|
||||||
|
other: LogLevel::Warn,
|
||||||
|
extractor: LogLevel::Error,
|
||||||
|
|
||||||
|
pile: LogLevel::Trace,
|
||||||
|
pile_flac: LogLevel::Trace,
|
||||||
|
pile_config: LogLevel::Trace,
|
||||||
|
pile_dataset: LogLevel::Trace,
|
||||||
|
pile_toolbox: LogLevel::Trace,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
// MARK: initializer
|
||||||
|
//
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize, Clone)]
|
||||||
|
pub struct LokiConfig {
|
||||||
|
pub loki_host: Url,
|
||||||
|
pub loki_user: String,
|
||||||
|
pub loki_pass: String,
|
||||||
|
pub loki_node_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Where to print logs
|
||||||
|
#[expect(dead_code)]
|
||||||
|
pub enum LoggingTarget {
|
||||||
|
/// Send logs to stdout
|
||||||
|
Stdout { format: LoggingFormat },
|
||||||
|
|
||||||
|
/// Send logs to stderr
|
||||||
|
Stderr { format: LoggingFormat },
|
||||||
|
|
||||||
|
/// Send logs to an IndicatifWriter.
|
||||||
|
///
|
||||||
|
/// This is the same as Stderr { format: Ansi {color:true} },
|
||||||
|
/// but uses an indicatifwriter with the given multiprogress.
|
||||||
|
Indicatif(MultiProgress),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How to print logs
|
||||||
|
#[derive(Debug, Clone, Copy, Deserialize, Default)]
|
||||||
|
pub enum LoggingFormat {
|
||||||
|
#[default]
|
||||||
|
Ansi,
|
||||||
|
AnsiNoColor,
|
||||||
|
Json,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct LoggingInitializer {
|
||||||
|
pub app_name: &'static str,
|
||||||
|
|
||||||
|
/// If `Some`, send logs to the given loki server
|
||||||
|
pub loki: Option<LokiConfig>,
|
||||||
|
|
||||||
|
/// Log filter for printed logs
|
||||||
|
pub preset: LoggingPreset,
|
||||||
|
|
||||||
|
/// Where to print logs
|
||||||
|
pub target: LoggingTarget,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LoggingInitializer {
|
||||||
|
pub fn initialize(self) -> Result<()> {
|
||||||
|
let mut stderr_ansi_layer = None;
|
||||||
|
let mut stderr_json_layer = None;
|
||||||
|
let mut stdout_ansi_layer = None;
|
||||||
|
let mut stdout_json_layer = None;
|
||||||
|
let mut indicatif_layer = None;
|
||||||
|
match self.target {
|
||||||
|
LoggingTarget::Stderr {
|
||||||
|
format: LoggingFormat::Ansi,
|
||||||
|
} => {
|
||||||
|
stderr_ansi_layer = Some(
|
||||||
|
tracing_subscriber::fmt::Layer::default()
|
||||||
|
.without_time()
|
||||||
|
.with_ansi(true)
|
||||||
|
.with_writer(std::io::stderr)
|
||||||
|
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
LoggingTarget::Stderr {
|
||||||
|
format: LoggingFormat::AnsiNoColor,
|
||||||
|
} => {
|
||||||
|
stderr_ansi_layer = Some(
|
||||||
|
tracing_subscriber::fmt::Layer::default()
|
||||||
|
.without_time()
|
||||||
|
.with_ansi(false)
|
||||||
|
.with_writer(std::io::stderr)
|
||||||
|
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
LoggingTarget::Stderr {
|
||||||
|
format: LoggingFormat::Json,
|
||||||
|
} => {
|
||||||
|
stderr_json_layer = Some(
|
||||||
|
tracing_subscriber::fmt::Layer::default()
|
||||||
|
.without_time()
|
||||||
|
.with_ansi(false)
|
||||||
|
.json()
|
||||||
|
.flatten_event(true)
|
||||||
|
.with_writer(std::io::stderr)
|
||||||
|
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
LoggingTarget::Stdout {
|
||||||
|
format: LoggingFormat::Ansi,
|
||||||
|
} => {
|
||||||
|
stdout_ansi_layer = Some(
|
||||||
|
tracing_subscriber::fmt::Layer::default()
|
||||||
|
.without_time()
|
||||||
|
.with_ansi(true)
|
||||||
|
.with_writer(std::io::stdout)
|
||||||
|
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
LoggingTarget::Stdout {
|
||||||
|
format: LoggingFormat::AnsiNoColor,
|
||||||
|
} => {
|
||||||
|
stdout_ansi_layer = Some(
|
||||||
|
tracing_subscriber::fmt::Layer::default()
|
||||||
|
.without_time()
|
||||||
|
.with_ansi(false)
|
||||||
|
.with_writer(std::io::stdout)
|
||||||
|
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
LoggingTarget::Stdout {
|
||||||
|
format: LoggingFormat::Json,
|
||||||
|
} => {
|
||||||
|
stdout_json_layer = Some(
|
||||||
|
tracing_subscriber::fmt::Layer::default()
|
||||||
|
.without_time()
|
||||||
|
.with_ansi(false)
|
||||||
|
.json()
|
||||||
|
.flatten_event(true)
|
||||||
|
.with_writer(std::io::stdout)
|
||||||
|
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
LoggingTarget::Indicatif(mp) => {
|
||||||
|
let writer: IndicatifWriter<tracing_indicatif::writer::Stderr> =
|
||||||
|
IndicatifWriter::new(mp);
|
||||||
|
|
||||||
|
indicatif_layer = Some(
|
||||||
|
tracing_subscriber::fmt::Layer::default()
|
||||||
|
.without_time()
|
||||||
|
.with_ansi(true)
|
||||||
|
.with_writer(writer.make_writer())
|
||||||
|
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let loki_layer = {
|
||||||
|
if let Some(cfg) = self.loki {
|
||||||
|
use anyhow::Context;
|
||||||
|
use base64::{Engine, prelude::BASE64_STANDARD};
|
||||||
|
|
||||||
|
let basic_auth = format!("{}:{}", cfg.loki_user, cfg.loki_pass);
|
||||||
|
let encoded_basic_auth = BASE64_STANDARD.encode(basic_auth.as_bytes());
|
||||||
|
|
||||||
|
let (layer, task) = tracing_loki::builder()
|
||||||
|
.label("node_name", cfg.loki_node_name)
|
||||||
|
.context("while building loki node_name label")?
|
||||||
|
.label("app", self.app_name)
|
||||||
|
.context("while building loki app label")?
|
||||||
|
.http_header("Authorization", format!("Basic {encoded_basic_auth}"))
|
||||||
|
.context("while building loki header")?
|
||||||
|
.build_url(cfg.loki_host)
|
||||||
|
.context("while building loki layer")?;
|
||||||
|
|
||||||
|
tokio::spawn(task);
|
||||||
|
Some(layer.with_filter::<EnvFilter>(LoggingPreset::Loki.get_config().into()))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
tracing_subscriber::registry()
|
||||||
|
.with(loki_layer)
|
||||||
|
.with(stdout_ansi_layer)
|
||||||
|
.with(stdout_json_layer)
|
||||||
|
.with(stderr_ansi_layer)
|
||||||
|
.with(stderr_json_layer)
|
||||||
|
.with(indicatif_layer)
|
||||||
|
.init();
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,2 +1,6 @@
|
|||||||
mod logging;
|
pub mod env;
|
||||||
pub use logging::*;
|
pub mod logging;
|
||||||
|
|
||||||
|
#[expect(clippy::module_inception)]
|
||||||
|
mod config;
|
||||||
|
pub use config::*;
|
||||||
|
|||||||
@@ -1,15 +1,13 @@
|
|||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use config::LoggingPreset;
|
|
||||||
use indicatif::MultiProgress;
|
use indicatif::MultiProgress;
|
||||||
use pile_toolbox::cancelabletask::CancelableTaskResult;
|
use pile_toolbox::cancelabletask::CancelableTaskResult;
|
||||||
use std::process::ExitCode;
|
use std::process::ExitCode;
|
||||||
use tracing::{error, warn};
|
use tracing::{error, warn};
|
||||||
use tracing_indicatif::{IndicatifWriter, writer::Stderr};
|
|
||||||
use tracing_subscriber::fmt::MakeWriter;
|
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
command::{CliCmd, CliCmdDispatch, SubCommand},
|
command::{CliCmd, CliCmdDispatch, SubCommand},
|
||||||
|
config::{PileServerConfig, logging::LoggingPreset},
|
||||||
signal::start_signal_task,
|
signal::start_signal_task,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -36,17 +34,11 @@ struct Cli {
|
|||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct GlobalContext {
|
pub struct GlobalContext {
|
||||||
pub mp: MultiProgress,
|
pub mp: MultiProgress,
|
||||||
|
pub config: PileServerConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> ExitCode {
|
fn main() -> ExitCode {
|
||||||
#[expect(clippy::unwrap_used)]
|
match main_inner() {
|
||||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
|
||||||
.enable_all()
|
|
||||||
.worker_threads(10)
|
|
||||||
.build()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
match rt.block_on(main_inner()) {
|
|
||||||
Ok(code) => {
|
Ok(code) => {
|
||||||
std::process::exit(code);
|
std::process::exit(code);
|
||||||
}
|
}
|
||||||
@@ -59,7 +51,7 @@ fn main() -> ExitCode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn main_inner() -> Result<i32> {
|
fn main_inner() -> Result<i32> {
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
|
|
||||||
let level_i: i16 = cli.v as i16 - cli.q as i16;
|
let level_i: i16 = cli.v as i16 - cli.q as i16;
|
||||||
@@ -80,17 +72,12 @@ async fn main_inner() -> Result<i32> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mp = MultiProgress::new();
|
let mp = MultiProgress::new();
|
||||||
let writer: IndicatifWriter<Stderr> = IndicatifWriter::new(mp.clone());
|
let config = PileServerConfig::load(matches!(cli.cmd, SubCommand::Server { .. }), level);
|
||||||
|
let rt = config.make_runtime();
|
||||||
|
|
||||||
tracing_subscriber::fmt()
|
let ctx = GlobalContext { mp, config };
|
||||||
.with_env_filter(level.get_config())
|
|
||||||
.without_time()
|
|
||||||
.with_ansi(true)
|
|
||||||
.with_writer(writer.make_writer())
|
|
||||||
.init();
|
|
||||||
|
|
||||||
let ctx = GlobalContext { mp };
|
|
||||||
|
|
||||||
|
let res = rt.block_on(async {
|
||||||
let task = cli.cmd.start(ctx).context("while starting task")?;
|
let task = cli.cmd.start(ctx).context("while starting task")?;
|
||||||
let signal_task = start_signal_task(task.flag().clone());
|
let signal_task = start_signal_task(task.flag().clone());
|
||||||
|
|
||||||
@@ -112,4 +99,7 @@ async fn main_inner() -> Result<i32> {
|
|||||||
Err(err).context("while running task")
|
Err(err).context("while running task")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
res
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,4 +11,14 @@ services:
|
|||||||
- "./x.ignore/books:/data/books:ro"
|
- "./x.ignore/books:/data/books:ro"
|
||||||
- "./pile:/workdir"
|
- "./pile:/workdir"
|
||||||
|
|
||||||
command: "pile server -c /data/books/pile.toml --workdir /workdir 0.0.0.0:7100"
|
environment:
|
||||||
|
SERVER_ADDR: "0.0.0.0:7100"
|
||||||
|
WORKDIR_ROOT: "/workdir"
|
||||||
|
API_TOKEN: "pile_token"
|
||||||
|
THREADS: 8
|
||||||
|
#LOKI_HOST: "http://loki:3100"
|
||||||
|
#LOKI_USER: "user"
|
||||||
|
#LOKI_PASS: "pass"
|
||||||
|
#LOKI_NODE_NAME: "pile"
|
||||||
|
|
||||||
|
command: "pile server -c /data/books/pile.toml"
|
||||||
|
|||||||
Reference in New Issue
Block a user