Configure server though env
All checks were successful
All checks were successful
This commit is contained in:
91
Cargo.lock
generated
91
Cargo.lock
generated
@@ -721,6 +721,12 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dotenvy"
|
||||
version = "0.15.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
|
||||
|
||||
[[package]]
|
||||
name = "downcast-rs"
|
||||
version = "2.0.2"
|
||||
@@ -748,6 +754,15 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "envy"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "epub"
|
||||
version = "1.2.4"
|
||||
@@ -1584,6 +1599,16 @@ version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "loki-api"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bdc38a304f59a03e6efa3876766a48c70a766a93f88341c3fff4212834b8e327"
|
||||
dependencies = [
|
||||
"prost",
|
||||
"prost-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.12.5"
|
||||
@@ -1962,7 +1987,10 @@ dependencies = [
|
||||
"anstyle",
|
||||
"anyhow",
|
||||
"axum",
|
||||
"base64",
|
||||
"clap",
|
||||
"dotenvy",
|
||||
"envy",
|
||||
"indicatif",
|
||||
"pile-config",
|
||||
"pile-dataset",
|
||||
@@ -1970,11 +1998,14 @@ dependencies = [
|
||||
"pile-value",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"toml",
|
||||
"tracing",
|
||||
"tracing-indicatif",
|
||||
"tracing-loki",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"utoipa",
|
||||
"utoipa-swagger-ui",
|
||||
]
|
||||
@@ -2172,6 +2203,38 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost"
|
||||
version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-derive"
|
||||
version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-types"
|
||||
version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16"
|
||||
dependencies = [
|
||||
"prost",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pxfm"
|
||||
version = "0.1.28"
|
||||
@@ -2697,6 +2760,12 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snap"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.6.2"
|
||||
@@ -3289,6 +3358,27 @@ dependencies = [
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-loki"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba3beec919fbdf99d719de8eda6adae3281f8a5b71ae40431f44dc7423053d34"
|
||||
dependencies = [
|
||||
"loki-api",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snap",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-log",
|
||||
"tracing-serde",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-serde"
|
||||
version = "0.2.0"
|
||||
@@ -3415,6 +3505,7 @@ dependencies = [
|
||||
"idna",
|
||||
"percent-encoding",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
17
Cargo.toml
17
Cargo.toml
@@ -72,7 +72,7 @@ pile-value = { path = "crates/pile-value" }
|
||||
pile-io = { path = "crates/pile-io" }
|
||||
pile-client = { path = "crates/pile-client" }
|
||||
|
||||
# Clients & servers
|
||||
# MARK: Clients & servers
|
||||
tantivy = "0.25.0"
|
||||
servable = { version = "0.0.7", features = ["image"] }
|
||||
axum = { version = "0.8.8", features = ["macros", "multipart"] }
|
||||
@@ -88,14 +88,14 @@ utoipa-swagger-ui = { version = "9.0.2", features = [
|
||||
"vendored",
|
||||
] }
|
||||
reqwest = { version = "0.12", features = ["blocking"] }
|
||||
tracing-loki = "0.2.6"
|
||||
|
||||
|
||||
# Async & Parallelism
|
||||
# MARK: Async & Parallelism
|
||||
tokio = { version = "1.49.0", features = ["full"] }
|
||||
tokio-stream = "0.1"
|
||||
async-trait = "0.1"
|
||||
|
||||
# CLI & logging
|
||||
# MARK: CLI & logging
|
||||
tracing = "0.1.44"
|
||||
tracing-subscriber = { version = "0.3.22", features = ["env-filter", "json"] }
|
||||
indicatif = { version = "0.18.4", features = ["improved_unicode"] }
|
||||
@@ -103,7 +103,7 @@ tracing-indicatif = "0.3.14"
|
||||
anstyle = "1.0.13"
|
||||
clap = { version = "4.5.60", features = ["derive"] }
|
||||
|
||||
# Serialization & formats
|
||||
# MARK: Serialization & formats
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.149"
|
||||
base64 = "0.22.1"
|
||||
@@ -112,8 +112,10 @@ toml = "1.0.3"
|
||||
toml_edit = "0.25.4"
|
||||
sha2 = "0.11.0-rc.5"
|
||||
blake3 = "1.8.3"
|
||||
dotenvy = "0.15.7"
|
||||
envy = "0.4.2"
|
||||
|
||||
# Extractors
|
||||
# MARK: Extractors
|
||||
pdf = "0.10.0"
|
||||
id3 = "1.16.4"
|
||||
epub = "1.2.2"
|
||||
@@ -121,7 +123,7 @@ kamadak-exif = "0.6.1"
|
||||
pdfium-render = "0.8"
|
||||
image = { version = "0.25", default-features = false, features = ["png"] }
|
||||
|
||||
# Misc helpers
|
||||
# MARK: Misc helpers
|
||||
thiserror = "2.0.18"
|
||||
anyhow = "1.0.102"
|
||||
itertools = "0.14.0"
|
||||
@@ -137,3 +139,4 @@ chrono = "0.4.43"
|
||||
parking_lot = "0.12.5"
|
||||
rayon = "1.11.0"
|
||||
percent-encoding = "2"
|
||||
url = { version = "2.5.8", features = ["serde"] }
|
||||
|
||||
@@ -28,3 +28,9 @@ serde_json = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
utoipa = { workspace = true }
|
||||
utoipa-swagger-ui = { workspace = true }
|
||||
url = { workspace = true }
|
||||
tracing-loki = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
dotenvy = { workspace = true }
|
||||
envy = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
@@ -20,10 +20,6 @@ use crate::{CliCmd, GlobalContext};
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
pub struct ServerCommand {
|
||||
/// Address to bind to
|
||||
#[arg(default_value = "0.0.0.0:9000")]
|
||||
addr: String,
|
||||
|
||||
/// The datasets we should serve. Can be repeated.
|
||||
#[arg(long, short = 'c')]
|
||||
config: Vec<PathBuf>,
|
||||
@@ -31,26 +27,18 @@ pub struct ServerCommand {
|
||||
/// If provided, do not serve docs
|
||||
#[arg(long)]
|
||||
no_docs: bool,
|
||||
|
||||
/// If provided, require this bearer token for all requests
|
||||
#[arg(long)]
|
||||
token: Option<String>,
|
||||
|
||||
/// Working directory root
|
||||
#[arg(long, default_value = "./.pile")]
|
||||
workdir: PathBuf,
|
||||
}
|
||||
|
||||
impl CliCmd for ServerCommand {
|
||||
async fn run(
|
||||
self,
|
||||
_ctx: GlobalContext,
|
||||
ctx: GlobalContext,
|
||||
flag: CancelFlag,
|
||||
) -> Result<i32, CancelableTaskError<anyhow::Error>> {
|
||||
let datasets = {
|
||||
let mut datasets = Vec::new();
|
||||
for c in &self.config {
|
||||
let ds = Datasets::open(&c, &self.workdir)
|
||||
let ds = Datasets::open(&c, &ctx.config.workdir_root)
|
||||
.await
|
||||
.with_context(|| format!("while opening dataset for {}", c.display()))?;
|
||||
datasets.push(Arc::new(ds));
|
||||
@@ -59,7 +47,7 @@ impl CliCmd for ServerCommand {
|
||||
Arc::new(datasets)
|
||||
};
|
||||
|
||||
let bearer = BearerToken(self.token.map(Arc::new));
|
||||
let bearer = BearerToken(ctx.config.api_token.clone().map(Arc::new));
|
||||
|
||||
let mut router = Router::new();
|
||||
for d in datasets.iter() {
|
||||
@@ -85,14 +73,14 @@ impl CliCmd for ServerCommand {
|
||||
|
||||
let app = router.into_make_service_with_connect_info::<std::net::SocketAddr>();
|
||||
|
||||
let listener = match tokio::net::TcpListener::bind(self.addr.clone()).await {
|
||||
let listener = match tokio::net::TcpListener::bind(ctx.config.server_addr.clone()).await {
|
||||
Ok(x) => x,
|
||||
Err(error) => {
|
||||
match error.kind() {
|
||||
std::io::ErrorKind::AddrInUse => {
|
||||
error!(
|
||||
message = "Cannot bind to address, already in use",
|
||||
addr = self.addr
|
||||
addr = ctx.config.server_addr
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
|
||||
109
crates/pile/src/config/config.rs
Normal file
109
crates/pile/src/config/config.rs
Normal file
@@ -0,0 +1,109 @@
|
||||
use serde::Deserialize;
|
||||
use std::{num::NonZeroUsize, path::PathBuf};
|
||||
use tracing::info;
|
||||
|
||||
use crate::config::{
|
||||
env::load_env,
|
||||
logging::{LoggingFormat, LoggingInitializer, LoggingPreset, LoggingTarget, LokiConfig},
|
||||
};
|
||||
|
||||
/// Note that the field of this struct are not capitalized.
|
||||
/// Envy is case-insensitive, and expects Rust fields to be snake_case.
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct PileServerConfig {
|
||||
#[serde(flatten)]
|
||||
pub loki: Option<LokiConfig>,
|
||||
|
||||
/// The logging level to run with
|
||||
#[serde(default)]
|
||||
pub loglevel: LoggingPreset,
|
||||
|
||||
#[serde(default)]
|
||||
pub logformat: LoggingFormat,
|
||||
|
||||
/// How many worker threads to use
|
||||
pub threads: Option<NonZeroUsize>,
|
||||
|
||||
/// IP and port to bind to
|
||||
/// Should look like `127.0.0.1:3030`
|
||||
pub server_addr: String,
|
||||
|
||||
pub api_token: Option<String>,
|
||||
pub workdir_root: PathBuf,
|
||||
}
|
||||
|
||||
impl Default for PileServerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
loki: None,
|
||||
loglevel: LoggingPreset::Debug,
|
||||
logformat: LoggingFormat::Ansi,
|
||||
threads: None,
|
||||
server_addr: "0.0.0.0:3000".into(),
|
||||
api_token: None,
|
||||
workdir_root: "./.pile".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PileServerConfig {
|
||||
pub fn load(with_env: bool, cli_log_level: LoggingPreset) -> Self {
|
||||
let config = match with_env {
|
||||
false => Self::default(),
|
||||
true => {
|
||||
let env = match load_env::<Self>() {
|
||||
Ok(x) => x,
|
||||
|
||||
#[expect(clippy::print_stdout)]
|
||||
Err(err) => {
|
||||
println!("Error while loading .env: {err}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
env.get_config().clone()
|
||||
}
|
||||
};
|
||||
|
||||
{
|
||||
let res = LoggingInitializer {
|
||||
app_name: "pile-server",
|
||||
loki: config.loki.clone(),
|
||||
preset: if with_env {
|
||||
config.loglevel
|
||||
} else {
|
||||
cli_log_level
|
||||
},
|
||||
target: LoggingTarget::Stderr {
|
||||
format: config.logformat,
|
||||
},
|
||||
}
|
||||
.initialize();
|
||||
|
||||
if let Err(e) = res {
|
||||
#[expect(clippy::print_stderr)]
|
||||
for e in e.chain() {
|
||||
eprintln!("{e}");
|
||||
}
|
||||
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
info!(message = "Config loaded", ?config);
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
pub fn make_runtime(&self) -> tokio::runtime::Runtime {
|
||||
let mut rt = tokio::runtime::Builder::new_multi_thread();
|
||||
rt.enable_all();
|
||||
if let Some(threads) = self.threads {
|
||||
rt.worker_threads(threads.into());
|
||||
}
|
||||
|
||||
#[expect(clippy::unwrap_used)]
|
||||
let rt = rt.build().unwrap();
|
||||
|
||||
return rt;
|
||||
}
|
||||
}
|
||||
108
crates/pile/src/config/env.rs
Normal file
108
crates/pile/src/config/env.rs
Normal file
@@ -0,0 +1,108 @@
|
||||
#![expect(dead_code)]
|
||||
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
env::VarError,
|
||||
io::ErrorKind,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use thiserror::Error;
|
||||
|
||||
/// An error we might encounter when loading an env
|
||||
#[derive(Debug, Error)]
|
||||
pub enum EnvLoadError {
|
||||
#[error("i/o error")]
|
||||
IOError(#[from] std::io::Error),
|
||||
|
||||
#[error("varerror")]
|
||||
VarError(#[from] VarError),
|
||||
|
||||
#[error("line parse error: `{on_line}` at char {at_char}")]
|
||||
LineParse { on_line: String, at_char: usize },
|
||||
|
||||
#[error("other dotenvy error")]
|
||||
Other(#[from] dotenvy::Error),
|
||||
|
||||
#[error("missing value {0}")]
|
||||
MissingValue(String),
|
||||
|
||||
#[error("parse error: {0}")]
|
||||
OtherParseError(String),
|
||||
}
|
||||
|
||||
pub enum LoadedEnv<T> {
|
||||
/// We loaded config from `.env` and env vars
|
||||
FoundFile { config: T, path: PathBuf },
|
||||
|
||||
/// We could not find `.env` and only loaded env vars
|
||||
OnlyVars(T),
|
||||
}
|
||||
|
||||
impl<T> LoadedEnv<T> {
|
||||
pub fn get_config(&self) -> &T {
|
||||
match self {
|
||||
Self::FoundFile { config, .. } => config,
|
||||
Self::OnlyVars(config) => config,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Load the configuration type `T` from the current environment,
|
||||
/// including the `.env` if it exists.
|
||||
#[expect(clippy::wildcard_enum_match_arm)]
|
||||
pub fn load_env<T: DeserializeOwned>() -> Result<LoadedEnv<T>, EnvLoadError> {
|
||||
let env_path = match dotenvy::dotenv() {
|
||||
Ok(path) => Some(path),
|
||||
|
||||
Err(dotenvy::Error::Io(err)) => match err.kind() {
|
||||
ErrorKind::NotFound => None,
|
||||
_ => return Err(EnvLoadError::IOError(err)),
|
||||
},
|
||||
|
||||
Err(dotenvy::Error::EnvVar(err)) => {
|
||||
return Err(EnvLoadError::VarError(err));
|
||||
}
|
||||
|
||||
Err(dotenvy::Error::LineParse(on_line, at_char)) => {
|
||||
return Err(EnvLoadError::LineParse { on_line, at_char });
|
||||
}
|
||||
|
||||
Err(err) => {
|
||||
return Err(EnvLoadError::Other(err));
|
||||
}
|
||||
};
|
||||
|
||||
match envy::from_env::<T>() {
|
||||
Ok(config) => {
|
||||
if let Some(path) = env_path {
|
||||
return Ok(LoadedEnv::FoundFile { path, config });
|
||||
} else {
|
||||
return Ok(LoadedEnv::OnlyVars(config));
|
||||
}
|
||||
}
|
||||
|
||||
Err(envy::Error::MissingValue(value)) => {
|
||||
return Err(EnvLoadError::MissingValue(value.into()));
|
||||
}
|
||||
|
||||
Err(envy::Error::Custom(message)) => {
|
||||
return Err(EnvLoadError::OtherParseError(message));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Load an .env file to a hashmap.
|
||||
///
|
||||
/// This function does not read the current env,
|
||||
/// only parsing vars explicitly declared in the given file.
|
||||
pub fn load_env_dict(p: impl AsRef<Path>) -> Result<HashMap<String, String>, EnvLoadError> {
|
||||
let mut out = HashMap::new();
|
||||
|
||||
for item in dotenvy::from_filename_iter(p)? {
|
||||
let (key, val) = item?;
|
||||
out.insert(key, val);
|
||||
}
|
||||
|
||||
return Ok(out);
|
||||
}
|
||||
@@ -1,7 +1,13 @@
|
||||
use anyhow::Result;
|
||||
use clap::ValueEnum;
|
||||
use indicatif::MultiProgress;
|
||||
use serde::Deserialize;
|
||||
use std::{fmt::Display, str::FromStr};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_indicatif::IndicatifWriter;
|
||||
use tracing_subscriber::{
|
||||
EnvFilter, Layer, fmt::MakeWriter, layer::SubscriberExt, util::SubscriberInitExt,
|
||||
};
|
||||
use url::Url;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub enum LogLevel {
|
||||
@@ -32,6 +38,7 @@ pub enum LoggingPreset {
|
||||
Info,
|
||||
Debug,
|
||||
Trace,
|
||||
Loki,
|
||||
}
|
||||
|
||||
pub struct LoggingConfig {
|
||||
@@ -138,6 +145,203 @@ impl LoggingPreset {
|
||||
pile_dataset: LogLevel::Trace,
|
||||
pile_toolbox: LogLevel::Trace,
|
||||
},
|
||||
|
||||
Self::Loki => LoggingConfig {
|
||||
other: LogLevel::Warn,
|
||||
extractor: LogLevel::Error,
|
||||
|
||||
pile: LogLevel::Trace,
|
||||
pile_flac: LogLevel::Trace,
|
||||
pile_config: LogLevel::Trace,
|
||||
pile_dataset: LogLevel::Trace,
|
||||
pile_toolbox: LogLevel::Trace,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// MARK: initializer
|
||||
//
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct LokiConfig {
|
||||
pub loki_host: Url,
|
||||
pub loki_user: String,
|
||||
pub loki_pass: String,
|
||||
pub loki_node_name: String,
|
||||
}
|
||||
|
||||
/// Where to print logs
|
||||
#[expect(dead_code)]
|
||||
pub enum LoggingTarget {
|
||||
/// Send logs to stdout
|
||||
Stdout { format: LoggingFormat },
|
||||
|
||||
/// Send logs to stderr
|
||||
Stderr { format: LoggingFormat },
|
||||
|
||||
/// Send logs to an IndicatifWriter.
|
||||
///
|
||||
/// This is the same as Stderr { format: Ansi {color:true} },
|
||||
/// but uses an indicatifwriter with the given multiprogress.
|
||||
Indicatif(MultiProgress),
|
||||
}
|
||||
|
||||
/// How to print logs
|
||||
#[derive(Debug, Clone, Copy, Deserialize, Default)]
|
||||
pub enum LoggingFormat {
|
||||
#[default]
|
||||
Ansi,
|
||||
AnsiNoColor,
|
||||
Json,
|
||||
}
|
||||
|
||||
pub struct LoggingInitializer {
|
||||
pub app_name: &'static str,
|
||||
|
||||
/// If `Some`, send logs to the given loki server
|
||||
pub loki: Option<LokiConfig>,
|
||||
|
||||
/// Log filter for printed logs
|
||||
pub preset: LoggingPreset,
|
||||
|
||||
/// Where to print logs
|
||||
pub target: LoggingTarget,
|
||||
}
|
||||
|
||||
impl LoggingInitializer {
|
||||
pub fn initialize(self) -> Result<()> {
|
||||
let mut stderr_ansi_layer = None;
|
||||
let mut stderr_json_layer = None;
|
||||
let mut stdout_ansi_layer = None;
|
||||
let mut stdout_json_layer = None;
|
||||
let mut indicatif_layer = None;
|
||||
match self.target {
|
||||
LoggingTarget::Stderr {
|
||||
format: LoggingFormat::Ansi,
|
||||
} => {
|
||||
stderr_ansi_layer = Some(
|
||||
tracing_subscriber::fmt::Layer::default()
|
||||
.without_time()
|
||||
.with_ansi(true)
|
||||
.with_writer(std::io::stderr)
|
||||
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||
)
|
||||
}
|
||||
|
||||
LoggingTarget::Stderr {
|
||||
format: LoggingFormat::AnsiNoColor,
|
||||
} => {
|
||||
stderr_ansi_layer = Some(
|
||||
tracing_subscriber::fmt::Layer::default()
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_writer(std::io::stderr)
|
||||
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||
)
|
||||
}
|
||||
|
||||
LoggingTarget::Stderr {
|
||||
format: LoggingFormat::Json,
|
||||
} => {
|
||||
stderr_json_layer = Some(
|
||||
tracing_subscriber::fmt::Layer::default()
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.json()
|
||||
.flatten_event(true)
|
||||
.with_writer(std::io::stderr)
|
||||
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||
)
|
||||
}
|
||||
|
||||
LoggingTarget::Stdout {
|
||||
format: LoggingFormat::Ansi,
|
||||
} => {
|
||||
stdout_ansi_layer = Some(
|
||||
tracing_subscriber::fmt::Layer::default()
|
||||
.without_time()
|
||||
.with_ansi(true)
|
||||
.with_writer(std::io::stdout)
|
||||
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||
)
|
||||
}
|
||||
|
||||
LoggingTarget::Stdout {
|
||||
format: LoggingFormat::AnsiNoColor,
|
||||
} => {
|
||||
stdout_ansi_layer = Some(
|
||||
tracing_subscriber::fmt::Layer::default()
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_writer(std::io::stdout)
|
||||
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||
)
|
||||
}
|
||||
|
||||
LoggingTarget::Stdout {
|
||||
format: LoggingFormat::Json,
|
||||
} => {
|
||||
stdout_json_layer = Some(
|
||||
tracing_subscriber::fmt::Layer::default()
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.json()
|
||||
.flatten_event(true)
|
||||
.with_writer(std::io::stdout)
|
||||
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||
)
|
||||
}
|
||||
|
||||
LoggingTarget::Indicatif(mp) => {
|
||||
let writer: IndicatifWriter<tracing_indicatif::writer::Stderr> =
|
||||
IndicatifWriter::new(mp);
|
||||
|
||||
indicatif_layer = Some(
|
||||
tracing_subscriber::fmt::Layer::default()
|
||||
.without_time()
|
||||
.with_ansi(true)
|
||||
.with_writer(writer.make_writer())
|
||||
.with_filter::<EnvFilter>(self.preset.get_config().into()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
let loki_layer = {
|
||||
if let Some(cfg) = self.loki {
|
||||
use anyhow::Context;
|
||||
use base64::{Engine, prelude::BASE64_STANDARD};
|
||||
|
||||
let basic_auth = format!("{}:{}", cfg.loki_user, cfg.loki_pass);
|
||||
let encoded_basic_auth = BASE64_STANDARD.encode(basic_auth.as_bytes());
|
||||
|
||||
let (layer, task) = tracing_loki::builder()
|
||||
.label("node_name", cfg.loki_node_name)
|
||||
.context("while building loki node_name label")?
|
||||
.label("app", self.app_name)
|
||||
.context("while building loki app label")?
|
||||
.http_header("Authorization", format!("Basic {encoded_basic_auth}"))
|
||||
.context("while building loki header")?
|
||||
.build_url(cfg.loki_host)
|
||||
.context("while building loki layer")?;
|
||||
|
||||
tokio::spawn(task);
|
||||
Some(layer.with_filter::<EnvFilter>(LoggingPreset::Loki.get_config().into()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(loki_layer)
|
||||
.with(stdout_ansi_layer)
|
||||
.with(stdout_json_layer)
|
||||
.with(stderr_ansi_layer)
|
||||
.with(stderr_json_layer)
|
||||
.with(indicatif_layer)
|
||||
.init();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,2 +1,6 @@
|
||||
mod logging;
|
||||
pub use logging::*;
|
||||
pub mod env;
|
||||
pub mod logging;
|
||||
|
||||
#[expect(clippy::module_inception)]
|
||||
mod config;
|
||||
pub use config::*;
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
use config::LoggingPreset;
|
||||
use indicatif::MultiProgress;
|
||||
use pile_toolbox::cancelabletask::CancelableTaskResult;
|
||||
use std::process::ExitCode;
|
||||
use tracing::{error, warn};
|
||||
use tracing_indicatif::{IndicatifWriter, writer::Stderr};
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
use crate::{
|
||||
command::{CliCmd, CliCmdDispatch, SubCommand},
|
||||
config::{PileServerConfig, logging::LoggingPreset},
|
||||
signal::start_signal_task,
|
||||
};
|
||||
|
||||
@@ -36,17 +34,11 @@ struct Cli {
|
||||
#[derive(Clone)]
|
||||
pub struct GlobalContext {
|
||||
pub mp: MultiProgress,
|
||||
pub config: PileServerConfig,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
#[expect(clippy::unwrap_used)]
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.worker_threads(10)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
match rt.block_on(main_inner()) {
|
||||
match main_inner() {
|
||||
Ok(code) => {
|
||||
std::process::exit(code);
|
||||
}
|
||||
@@ -59,7 +51,7 @@ fn main() -> ExitCode {
|
||||
}
|
||||
}
|
||||
|
||||
async fn main_inner() -> Result<i32> {
|
||||
fn main_inner() -> Result<i32> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
let level_i: i16 = cli.v as i16 - cli.q as i16;
|
||||
@@ -80,36 +72,34 @@ async fn main_inner() -> Result<i32> {
|
||||
}
|
||||
|
||||
let mp = MultiProgress::new();
|
||||
let writer: IndicatifWriter<Stderr> = IndicatifWriter::new(mp.clone());
|
||||
let config = PileServerConfig::load(matches!(cli.cmd, SubCommand::Server { .. }), level);
|
||||
let rt = config.make_runtime();
|
||||
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(level.get_config())
|
||||
.without_time()
|
||||
.with_ansi(true)
|
||||
.with_writer(writer.make_writer())
|
||||
.init();
|
||||
let ctx = GlobalContext { mp, config };
|
||||
|
||||
let ctx = GlobalContext { mp };
|
||||
let res = rt.block_on(async {
|
||||
let task = cli.cmd.start(ctx).context("while starting task")?;
|
||||
let signal_task = start_signal_task(task.flag().clone());
|
||||
|
||||
let task = cli.cmd.start(ctx).context("while starting task")?;
|
||||
let signal_task = start_signal_task(task.flag().clone());
|
||||
match task.join().await {
|
||||
Ok(CancelableTaskResult::Finished(Ok(code))) => Ok(code),
|
||||
Ok(CancelableTaskResult::Cancelled) => {
|
||||
signal_task.abort();
|
||||
warn!("Task cancelled successfully");
|
||||
Ok(1)
|
||||
}
|
||||
|
||||
match task.join().await {
|
||||
Ok(CancelableTaskResult::Finished(Ok(code))) => Ok(code),
|
||||
Ok(CancelableTaskResult::Cancelled) => {
|
||||
signal_task.abort();
|
||||
warn!("Task cancelled successfully");
|
||||
Ok(1)
|
||||
Err(err) => {
|
||||
signal_task.abort();
|
||||
Err(err).context("while joining task")
|
||||
}
|
||||
|
||||
Ok(CancelableTaskResult::Finished(Err(err))) => {
|
||||
signal_task.abort();
|
||||
Err(err).context("while running task")
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Err(err) => {
|
||||
signal_task.abort();
|
||||
Err(err).context("while joining task")
|
||||
}
|
||||
|
||||
Ok(CancelableTaskResult::Finished(Err(err))) => {
|
||||
signal_task.abort();
|
||||
Err(err).context("while running task")
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
@@ -11,4 +11,14 @@ services:
|
||||
- "./x.ignore/books:/data/books:ro"
|
||||
- "./pile:/workdir"
|
||||
|
||||
command: "pile server -c /data/books/pile.toml --workdir /workdir 0.0.0.0:7100"
|
||||
environment:
|
||||
SERVER_ADDR: "0.0.0.0:7100"
|
||||
WORKDIR_ROOT: "/workdir"
|
||||
API_TOKEN: "pile_token"
|
||||
THREADS: 8
|
||||
#LOKI_HOST: "http://loki:3100"
|
||||
#LOKI_USER: "user"
|
||||
#LOKI_PASS: "pass"
|
||||
#LOKI_NODE_NAME: "pile"
|
||||
|
||||
command: "pile server -c /data/books/pile.toml"
|
||||
|
||||
Reference in New Issue
Block a user