Configure server though env
All checks were successful
CI / Typos (push) Successful in 21s
CI / Build and test (push) Successful in 2m33s
CI / Clippy (push) Successful in 3m12s
Docker / build-and-push (push) Successful in 5m35s
CI / Build and test (all features) (push) Successful in 7m1s

This commit is contained in:
2026-03-26 20:40:51 -07:00
parent 256af68382
commit 5807733e62
10 changed files with 579 additions and 66 deletions

91
Cargo.lock generated
View File

@@ -721,6 +721,12 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "dotenvy"
version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
[[package]]
name = "downcast-rs"
version = "2.0.2"
@@ -748,6 +754,15 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "envy"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965"
dependencies = [
"serde",
]
[[package]]
name = "epub"
version = "1.2.4"
@@ -1584,6 +1599,16 @@ version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "loki-api"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdc38a304f59a03e6efa3876766a48c70a766a93f88341c3fff4212834b8e327"
dependencies = [
"prost",
"prost-types",
]
[[package]]
name = "lru"
version = "0.12.5"
@@ -1962,7 +1987,10 @@ dependencies = [
"anstyle",
"anyhow",
"axum",
"base64",
"clap",
"dotenvy",
"envy",
"indicatif",
"pile-config",
"pile-dataset",
@@ -1970,11 +1998,14 @@ dependencies = [
"pile-value",
"serde",
"serde_json",
"thiserror",
"tokio",
"toml",
"tracing",
"tracing-indicatif",
"tracing-loki",
"tracing-subscriber",
"url",
"utoipa",
"utoipa-swagger-ui",
]
@@ -2172,6 +2203,38 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prost"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-derive"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "prost-types"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16"
dependencies = [
"prost",
]
[[package]]
name = "pxfm"
version = "0.1.28"
@@ -2697,6 +2760,12 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "snap"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
[[package]]
name = "socket2"
version = "0.6.2"
@@ -3289,6 +3358,27 @@ dependencies = [
"tracing-core",
]
[[package]]
name = "tracing-loki"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3beec919fbdf99d719de8eda6adae3281f8a5b71ae40431f44dc7423053d34"
dependencies = [
"loki-api",
"reqwest",
"serde",
"serde_json",
"snap",
"tokio",
"tokio-stream",
"tracing",
"tracing-core",
"tracing-log",
"tracing-serde",
"tracing-subscriber",
"url",
]
[[package]]
name = "tracing-serde"
version = "0.2.0"
@@ -3415,6 +3505,7 @@ dependencies = [
"idna",
"percent-encoding",
"serde",
"serde_derive",
]
[[package]]

View File

@@ -72,7 +72,7 @@ pile-value = { path = "crates/pile-value" }
pile-io = { path = "crates/pile-io" }
pile-client = { path = "crates/pile-client" }
# Clients & servers
# MARK: Clients & servers
tantivy = "0.25.0"
servable = { version = "0.0.7", features = ["image"] }
axum = { version = "0.8.8", features = ["macros", "multipart"] }
@@ -88,14 +88,14 @@ utoipa-swagger-ui = { version = "9.0.2", features = [
"vendored",
] }
reqwest = { version = "0.12", features = ["blocking"] }
tracing-loki = "0.2.6"
# Async & Parallelism
# MARK: Async & Parallelism
tokio = { version = "1.49.0", features = ["full"] }
tokio-stream = "0.1"
async-trait = "0.1"
# CLI & logging
# MARK: CLI & logging
tracing = "0.1.44"
tracing-subscriber = { version = "0.3.22", features = ["env-filter", "json"] }
indicatif = { version = "0.18.4", features = ["improved_unicode"] }
@@ -103,7 +103,7 @@ tracing-indicatif = "0.3.14"
anstyle = "1.0.13"
clap = { version = "4.5.60", features = ["derive"] }
# Serialization & formats
# MARK: Serialization & formats
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.149"
base64 = "0.22.1"
@@ -112,8 +112,10 @@ toml = "1.0.3"
toml_edit = "0.25.4"
sha2 = "0.11.0-rc.5"
blake3 = "1.8.3"
dotenvy = "0.15.7"
envy = "0.4.2"
# Extractors
# MARK: Extractors
pdf = "0.10.0"
id3 = "1.16.4"
epub = "1.2.2"
@@ -121,7 +123,7 @@ kamadak-exif = "0.6.1"
pdfium-render = "0.8"
image = { version = "0.25", default-features = false, features = ["png"] }
# Misc helpers
# MARK: Misc helpers
thiserror = "2.0.18"
anyhow = "1.0.102"
itertools = "0.14.0"
@@ -137,3 +139,4 @@ chrono = "0.4.43"
parking_lot = "0.12.5"
rayon = "1.11.0"
percent-encoding = "2"
url = { version = "2.5.8", features = ["serde"] }

View File

@@ -28,3 +28,9 @@ serde_json = { workspace = true }
axum = { workspace = true }
utoipa = { workspace = true }
utoipa-swagger-ui = { workspace = true }
url = { workspace = true }
tracing-loki = { workspace = true }
base64 = { workspace = true }
dotenvy = { workspace = true }
envy = { workspace = true }
thiserror = { workspace = true }

View File

@@ -20,10 +20,6 @@ use crate::{CliCmd, GlobalContext};
#[derive(Debug, Args)]
pub struct ServerCommand {
/// Address to bind to
#[arg(default_value = "0.0.0.0:9000")]
addr: String,
/// The datasets we should serve. Can be repeated.
#[arg(long, short = 'c')]
config: Vec<PathBuf>,
@@ -31,26 +27,18 @@ pub struct ServerCommand {
/// If provided, do not serve docs
#[arg(long)]
no_docs: bool,
/// If provided, require this bearer token for all requests
#[arg(long)]
token: Option<String>,
/// Working directory root
#[arg(long, default_value = "./.pile")]
workdir: PathBuf,
}
impl CliCmd for ServerCommand {
async fn run(
self,
_ctx: GlobalContext,
ctx: GlobalContext,
flag: CancelFlag,
) -> Result<i32, CancelableTaskError<anyhow::Error>> {
let datasets = {
let mut datasets = Vec::new();
for c in &self.config {
let ds = Datasets::open(&c, &self.workdir)
let ds = Datasets::open(&c, &ctx.config.workdir_root)
.await
.with_context(|| format!("while opening dataset for {}", c.display()))?;
datasets.push(Arc::new(ds));
@@ -59,7 +47,7 @@ impl CliCmd for ServerCommand {
Arc::new(datasets)
};
let bearer = BearerToken(self.token.map(Arc::new));
let bearer = BearerToken(ctx.config.api_token.clone().map(Arc::new));
let mut router = Router::new();
for d in datasets.iter() {
@@ -85,14 +73,14 @@ impl CliCmd for ServerCommand {
let app = router.into_make_service_with_connect_info::<std::net::SocketAddr>();
let listener = match tokio::net::TcpListener::bind(self.addr.clone()).await {
let listener = match tokio::net::TcpListener::bind(ctx.config.server_addr.clone()).await {
Ok(x) => x,
Err(error) => {
match error.kind() {
std::io::ErrorKind::AddrInUse => {
error!(
message = "Cannot bind to address, already in use",
addr = self.addr
addr = ctx.config.server_addr
);
}
_ => {

View File

@@ -0,0 +1,109 @@
use serde::Deserialize;
use std::{num::NonZeroUsize, path::PathBuf};
use tracing::info;
use crate::config::{
env::load_env,
logging::{LoggingFormat, LoggingInitializer, LoggingPreset, LoggingTarget, LokiConfig},
};
/// Note that the field of this struct are not capitalized.
/// Envy is case-insensitive, and expects Rust fields to be snake_case.
#[derive(Debug, Deserialize, Clone)]
pub struct PileServerConfig {
#[serde(flatten)]
pub loki: Option<LokiConfig>,
/// The logging level to run with
#[serde(default)]
pub loglevel: LoggingPreset,
#[serde(default)]
pub logformat: LoggingFormat,
/// How many worker threads to use
pub threads: Option<NonZeroUsize>,
/// IP and port to bind to
/// Should look like `127.0.0.1:3030`
pub server_addr: String,
pub api_token: Option<String>,
pub workdir_root: PathBuf,
}
impl Default for PileServerConfig {
fn default() -> Self {
Self {
loki: None,
loglevel: LoggingPreset::Debug,
logformat: LoggingFormat::Ansi,
threads: None,
server_addr: "0.0.0.0:3000".into(),
api_token: None,
workdir_root: "./.pile".into(),
}
}
}
impl PileServerConfig {
pub fn load(with_env: bool, cli_log_level: LoggingPreset) -> Self {
let config = match with_env {
false => Self::default(),
true => {
let env = match load_env::<Self>() {
Ok(x) => x,
#[expect(clippy::print_stdout)]
Err(err) => {
println!("Error while loading .env: {err}");
std::process::exit(1);
}
};
env.get_config().clone()
}
};
{
let res = LoggingInitializer {
app_name: "pile-server",
loki: config.loki.clone(),
preset: if with_env {
config.loglevel
} else {
cli_log_level
},
target: LoggingTarget::Stderr {
format: config.logformat,
},
}
.initialize();
if let Err(e) = res {
#[expect(clippy::print_stderr)]
for e in e.chain() {
eprintln!("{e}");
}
std::process::exit(1);
}
}
info!(message = "Config loaded", ?config);
return config;
}
pub fn make_runtime(&self) -> tokio::runtime::Runtime {
let mut rt = tokio::runtime::Builder::new_multi_thread();
rt.enable_all();
if let Some(threads) = self.threads {
rt.worker_threads(threads.into());
}
#[expect(clippy::unwrap_used)]
let rt = rt.build().unwrap();
return rt;
}
}

View File

@@ -0,0 +1,108 @@
#![expect(dead_code)]
use serde::de::DeserializeOwned;
use std::{
collections::HashMap,
env::VarError,
io::ErrorKind,
path::{Path, PathBuf},
};
use thiserror::Error;
/// An error we might encounter when loading an env
#[derive(Debug, Error)]
pub enum EnvLoadError {
#[error("i/o error")]
IOError(#[from] std::io::Error),
#[error("varerror")]
VarError(#[from] VarError),
#[error("line parse error: `{on_line}` at char {at_char}")]
LineParse { on_line: String, at_char: usize },
#[error("other dotenvy error")]
Other(#[from] dotenvy::Error),
#[error("missing value {0}")]
MissingValue(String),
#[error("parse error: {0}")]
OtherParseError(String),
}
pub enum LoadedEnv<T> {
/// We loaded config from `.env` and env vars
FoundFile { config: T, path: PathBuf },
/// We could not find `.env` and only loaded env vars
OnlyVars(T),
}
impl<T> LoadedEnv<T> {
pub fn get_config(&self) -> &T {
match self {
Self::FoundFile { config, .. } => config,
Self::OnlyVars(config) => config,
}
}
}
/// Load the configuration type `T` from the current environment,
/// including the `.env` if it exists.
#[expect(clippy::wildcard_enum_match_arm)]
pub fn load_env<T: DeserializeOwned>() -> Result<LoadedEnv<T>, EnvLoadError> {
let env_path = match dotenvy::dotenv() {
Ok(path) => Some(path),
Err(dotenvy::Error::Io(err)) => match err.kind() {
ErrorKind::NotFound => None,
_ => return Err(EnvLoadError::IOError(err)),
},
Err(dotenvy::Error::EnvVar(err)) => {
return Err(EnvLoadError::VarError(err));
}
Err(dotenvy::Error::LineParse(on_line, at_char)) => {
return Err(EnvLoadError::LineParse { on_line, at_char });
}
Err(err) => {
return Err(EnvLoadError::Other(err));
}
};
match envy::from_env::<T>() {
Ok(config) => {
if let Some(path) = env_path {
return Ok(LoadedEnv::FoundFile { path, config });
} else {
return Ok(LoadedEnv::OnlyVars(config));
}
}
Err(envy::Error::MissingValue(value)) => {
return Err(EnvLoadError::MissingValue(value.into()));
}
Err(envy::Error::Custom(message)) => {
return Err(EnvLoadError::OtherParseError(message));
}
};
}
/// Load an .env file to a hashmap.
///
/// This function does not read the current env,
/// only parsing vars explicitly declared in the given file.
pub fn load_env_dict(p: impl AsRef<Path>) -> Result<HashMap<String, String>, EnvLoadError> {
let mut out = HashMap::new();
for item in dotenvy::from_filename_iter(p)? {
let (key, val) = item?;
out.insert(key, val);
}
return Ok(out);
}

View File

@@ -1,7 +1,13 @@
use anyhow::Result;
use clap::ValueEnum;
use indicatif::MultiProgress;
use serde::Deserialize;
use std::{fmt::Display, str::FromStr};
use tracing_subscriber::EnvFilter;
use tracing_indicatif::IndicatifWriter;
use tracing_subscriber::{
EnvFilter, Layer, fmt::MakeWriter, layer::SubscriberExt, util::SubscriberInitExt,
};
use url::Url;
#[derive(Debug, Default)]
pub enum LogLevel {
@@ -32,6 +38,7 @@ pub enum LoggingPreset {
Info,
Debug,
Trace,
Loki,
}
pub struct LoggingConfig {
@@ -138,6 +145,203 @@ impl LoggingPreset {
pile_dataset: LogLevel::Trace,
pile_toolbox: LogLevel::Trace,
},
Self::Loki => LoggingConfig {
other: LogLevel::Warn,
extractor: LogLevel::Error,
pile: LogLevel::Trace,
pile_flac: LogLevel::Trace,
pile_config: LogLevel::Trace,
pile_dataset: LogLevel::Trace,
pile_toolbox: LogLevel::Trace,
},
}
}
}
//
// MARK: initializer
//
#[derive(Debug, Deserialize, Clone)]
pub struct LokiConfig {
pub loki_host: Url,
pub loki_user: String,
pub loki_pass: String,
pub loki_node_name: String,
}
/// Where to print logs
#[expect(dead_code)]
pub enum LoggingTarget {
/// Send logs to stdout
Stdout { format: LoggingFormat },
/// Send logs to stderr
Stderr { format: LoggingFormat },
/// Send logs to an IndicatifWriter.
///
/// This is the same as Stderr { format: Ansi {color:true} },
/// but uses an indicatifwriter with the given multiprogress.
Indicatif(MultiProgress),
}
/// How to print logs
#[derive(Debug, Clone, Copy, Deserialize, Default)]
pub enum LoggingFormat {
#[default]
Ansi,
AnsiNoColor,
Json,
}
pub struct LoggingInitializer {
pub app_name: &'static str,
/// If `Some`, send logs to the given loki server
pub loki: Option<LokiConfig>,
/// Log filter for printed logs
pub preset: LoggingPreset,
/// Where to print logs
pub target: LoggingTarget,
}
impl LoggingInitializer {
pub fn initialize(self) -> Result<()> {
let mut stderr_ansi_layer = None;
let mut stderr_json_layer = None;
let mut stdout_ansi_layer = None;
let mut stdout_json_layer = None;
let mut indicatif_layer = None;
match self.target {
LoggingTarget::Stderr {
format: LoggingFormat::Ansi,
} => {
stderr_ansi_layer = Some(
tracing_subscriber::fmt::Layer::default()
.without_time()
.with_ansi(true)
.with_writer(std::io::stderr)
.with_filter::<EnvFilter>(self.preset.get_config().into()),
)
}
LoggingTarget::Stderr {
format: LoggingFormat::AnsiNoColor,
} => {
stderr_ansi_layer = Some(
tracing_subscriber::fmt::Layer::default()
.without_time()
.with_ansi(false)
.with_writer(std::io::stderr)
.with_filter::<EnvFilter>(self.preset.get_config().into()),
)
}
LoggingTarget::Stderr {
format: LoggingFormat::Json,
} => {
stderr_json_layer = Some(
tracing_subscriber::fmt::Layer::default()
.without_time()
.with_ansi(false)
.json()
.flatten_event(true)
.with_writer(std::io::stderr)
.with_filter::<EnvFilter>(self.preset.get_config().into()),
)
}
LoggingTarget::Stdout {
format: LoggingFormat::Ansi,
} => {
stdout_ansi_layer = Some(
tracing_subscriber::fmt::Layer::default()
.without_time()
.with_ansi(true)
.with_writer(std::io::stdout)
.with_filter::<EnvFilter>(self.preset.get_config().into()),
)
}
LoggingTarget::Stdout {
format: LoggingFormat::AnsiNoColor,
} => {
stdout_ansi_layer = Some(
tracing_subscriber::fmt::Layer::default()
.without_time()
.with_ansi(false)
.with_writer(std::io::stdout)
.with_filter::<EnvFilter>(self.preset.get_config().into()),
)
}
LoggingTarget::Stdout {
format: LoggingFormat::Json,
} => {
stdout_json_layer = Some(
tracing_subscriber::fmt::Layer::default()
.without_time()
.with_ansi(false)
.json()
.flatten_event(true)
.with_writer(std::io::stdout)
.with_filter::<EnvFilter>(self.preset.get_config().into()),
)
}
LoggingTarget::Indicatif(mp) => {
let writer: IndicatifWriter<tracing_indicatif::writer::Stderr> =
IndicatifWriter::new(mp);
indicatif_layer = Some(
tracing_subscriber::fmt::Layer::default()
.without_time()
.with_ansi(true)
.with_writer(writer.make_writer())
.with_filter::<EnvFilter>(self.preset.get_config().into()),
)
}
}
let loki_layer = {
if let Some(cfg) = self.loki {
use anyhow::Context;
use base64::{Engine, prelude::BASE64_STANDARD};
let basic_auth = format!("{}:{}", cfg.loki_user, cfg.loki_pass);
let encoded_basic_auth = BASE64_STANDARD.encode(basic_auth.as_bytes());
let (layer, task) = tracing_loki::builder()
.label("node_name", cfg.loki_node_name)
.context("while building loki node_name label")?
.label("app", self.app_name)
.context("while building loki app label")?
.http_header("Authorization", format!("Basic {encoded_basic_auth}"))
.context("while building loki header")?
.build_url(cfg.loki_host)
.context("while building loki layer")?;
tokio::spawn(task);
Some(layer.with_filter::<EnvFilter>(LoggingPreset::Loki.get_config().into()))
} else {
None
}
};
tracing_subscriber::registry()
.with(loki_layer)
.with(stdout_ansi_layer)
.with(stdout_json_layer)
.with(stderr_ansi_layer)
.with(stderr_json_layer)
.with(indicatif_layer)
.init();
Ok(())
}
}

View File

@@ -1,2 +1,6 @@
mod logging;
pub use logging::*;
pub mod env;
pub mod logging;
#[expect(clippy::module_inception)]
mod config;
pub use config::*;

View File

@@ -1,15 +1,13 @@
use anyhow::{Context, Result};
use clap::Parser;
use config::LoggingPreset;
use indicatif::MultiProgress;
use pile_toolbox::cancelabletask::CancelableTaskResult;
use std::process::ExitCode;
use tracing::{error, warn};
use tracing_indicatif::{IndicatifWriter, writer::Stderr};
use tracing_subscriber::fmt::MakeWriter;
use crate::{
command::{CliCmd, CliCmdDispatch, SubCommand},
config::{PileServerConfig, logging::LoggingPreset},
signal::start_signal_task,
};
@@ -36,17 +34,11 @@ struct Cli {
#[derive(Clone)]
pub struct GlobalContext {
pub mp: MultiProgress,
pub config: PileServerConfig,
}
fn main() -> ExitCode {
#[expect(clippy::unwrap_used)]
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.worker_threads(10)
.build()
.unwrap();
match rt.block_on(main_inner()) {
match main_inner() {
Ok(code) => {
std::process::exit(code);
}
@@ -59,7 +51,7 @@ fn main() -> ExitCode {
}
}
async fn main_inner() -> Result<i32> {
fn main_inner() -> Result<i32> {
let cli = Cli::parse();
let level_i: i16 = cli.v as i16 - cli.q as i16;
@@ -80,36 +72,34 @@ async fn main_inner() -> Result<i32> {
}
let mp = MultiProgress::new();
let writer: IndicatifWriter<Stderr> = IndicatifWriter::new(mp.clone());
let config = PileServerConfig::load(matches!(cli.cmd, SubCommand::Server { .. }), level);
let rt = config.make_runtime();
tracing_subscriber::fmt()
.with_env_filter(level.get_config())
.without_time()
.with_ansi(true)
.with_writer(writer.make_writer())
.init();
let ctx = GlobalContext { mp, config };
let ctx = GlobalContext { mp };
let res = rt.block_on(async {
let task = cli.cmd.start(ctx).context("while starting task")?;
let signal_task = start_signal_task(task.flag().clone());
let task = cli.cmd.start(ctx).context("while starting task")?;
let signal_task = start_signal_task(task.flag().clone());
match task.join().await {
Ok(CancelableTaskResult::Finished(Ok(code))) => Ok(code),
Ok(CancelableTaskResult::Cancelled) => {
signal_task.abort();
warn!("Task cancelled successfully");
Ok(1)
}
match task.join().await {
Ok(CancelableTaskResult::Finished(Ok(code))) => Ok(code),
Ok(CancelableTaskResult::Cancelled) => {
signal_task.abort();
warn!("Task cancelled successfully");
Ok(1)
Err(err) => {
signal_task.abort();
Err(err).context("while joining task")
}
Ok(CancelableTaskResult::Finished(Err(err))) => {
signal_task.abort();
Err(err).context("while running task")
}
}
});
Err(err) => {
signal_task.abort();
Err(err).context("while joining task")
}
Ok(CancelableTaskResult::Finished(Err(err))) => {
signal_task.abort();
Err(err).context("while running task")
}
}
res
}

View File

@@ -11,4 +11,14 @@ services:
- "./x.ignore/books:/data/books:ro"
- "./pile:/workdir"
command: "pile server -c /data/books/pile.toml --workdir /workdir 0.0.0.0:7100"
environment:
SERVER_ADDR: "0.0.0.0:7100"
WORKDIR_ROOT: "/workdir"
API_TOKEN: "pile_token"
THREADS: 8
#LOKI_HOST: "http://loki:3100"
#LOKI_USER: "user"
#LOKI_PASS: "pass"
#LOKI_NODE_NAME: "pile"
command: "pile server -c /data/books/pile.toml"