From 3bc66ddc48c6946745d90d82ae2ed07d5875608f Mon Sep 17 00:00:00 2001 From: rm-dr <96270320+rm-dr@users.noreply.github.com> Date: Wed, 15 Apr 2026 09:23:52 -0700 Subject: [PATCH] Split server into crate --- Cargo.lock | 27 ++++-- Cargo.toml | 3 +- crates/pile-client/Cargo.toml | 4 +- crates/pile-client/src/lib.rs | 2 +- crates/pile-dataset/Cargo.toml | 14 --- crates/pile-dataset/src/lib.rs | 3 - crates/pile-dataset/src/serve/mod.rs | 92 ------------------- crates/pile-serve/Cargo.toml | 28 ++++++ .../serve => pile-serve/src}/config_schema.rs | 3 +- .../src/serve => pile-serve/src}/extract.rs | 3 +- .../src/serve => pile-serve/src}/items.rs | 3 +- crates/pile-serve/src/lib.rs | 89 ++++++++++++++++++ .../src/serve => pile-serve/src}/lookup.rs | 3 +- .../src/serve => pile-serve/src}/schema.rs | 3 +- .../serve => pile-serve/src}/schema_field.rs | 3 +- crates/pile/Cargo.toml | 9 +- crates/pile/src/command/serve.rs | 3 +- crates/pile/src/command/server.rs | 6 +- 18 files changed, 161 insertions(+), 137 deletions(-) delete mode 100644 crates/pile-dataset/src/serve/mod.rs create mode 100644 crates/pile-serve/Cargo.toml rename crates/{pile-dataset/src/serve => pile-serve/src}/config_schema.rs (95%) rename crates/{pile-dataset/src/serve => pile-serve/src}/extract.rs (99%) rename crates/{pile-dataset/src/serve => pile-serve/src}/items.rs (98%) create mode 100644 crates/pile-serve/src/lib.rs rename crates/{pile-dataset/src/serve => pile-serve/src}/lookup.rs (98%) rename crates/{pile-dataset/src/serve => pile-serve/src}/schema.rs (99%) rename crates/{pile-dataset/src/serve => pile-serve/src}/schema_field.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index ad16b74..0a51efd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1994,6 +1994,7 @@ dependencies = [ "indicatif", "pile-config", "pile-dataset", + "pile-serve", "pile-toolbox", "pile-value", "serde", @@ -2016,7 +2017,7 @@ version = "0.0.2" dependencies = [ "axum", "bytes", - "pile-dataset", + "pile-serve", "reqwest", "serde", "thiserror", @@ -2037,23 +2038,17 @@ dependencies = [ name = "pile-dataset" version = "0.0.2" dependencies = [ - "axum", "chrono", - "percent-encoding", "pile-config", "pile-toolbox", "pile-value", "regex", - "serde", "serde_json", "tantivy", "thiserror", "tokio", - "tokio-util", "toml", "tracing", - "utoipa", - "utoipa-swagger-ui", ] [[package]] @@ -2077,6 +2072,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "pile-serve" +version = "0.0.2" +dependencies = [ + "axum", + "percent-encoding", + "pile-config", + "pile-dataset", + "pile-value", + "serde", + "serde_json", + "tokio", + "tokio-util", + "tracing", + "utoipa", + "utoipa-swagger-ui", +] + [[package]] name = "pile-toolbox" version = "0.0.2" diff --git a/Cargo.toml b/Cargo.toml index 859af9a..a27ef3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,7 @@ pile-dataset = { path = "crates/pile-dataset" } pile-value = { path = "crates/pile-value" } pile-io = { path = "crates/pile-io" } pile-client = { path = "crates/pile-client" } +pile-serve = { path = "crates/pile-serve" } # MARK: Clients & servers tantivy = "0.25.0" @@ -87,7 +88,7 @@ utoipa-swagger-ui = { version = "9.0.2", features = [ "debug-embed", "vendored", ] } -reqwest = { version = "0.12", features = ["blocking"] } +reqwest = { version = "0.12", features = ["blocking", "json", "stream"] } tracing-loki = "0.2.6" # MARK: Async & Parallelism diff --git a/crates/pile-client/Cargo.toml b/crates/pile-client/Cargo.toml index 474c222..a497840 100644 --- a/crates/pile-client/Cargo.toml +++ b/crates/pile-client/Cargo.toml @@ -8,9 +8,9 @@ edition = { workspace = true } workspace = true [dependencies] -pile-dataset = { workspace = true, features = ["axum"] } +pile-serve = { workspace = true } -reqwest = { version = "0.12", features = ["json", "stream"] } +reqwest = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } bytes = { workspace = true } diff --git a/crates/pile-client/src/lib.rs b/crates/pile-client/src/lib.rs index 74d35c2..2c12a1f 100644 --- a/crates/pile-client/src/lib.rs +++ b/crates/pile-client/src/lib.rs @@ -8,7 +8,7 @@ use serde::Deserialize; use thiserror::Error; use tracing::{trace, warn}; -pub use pile_dataset::serve::{ +pub use pile_serve::{ ApiValue, FieldSpec, FieldsResponse, ItemsResponse, LookupRequest, LookupResponse, SchemaResponse, }; diff --git a/crates/pile-dataset/Cargo.toml b/crates/pile-dataset/Cargo.toml index 8655072..9177d36 100644 --- a/crates/pile-dataset/Cargo.toml +++ b/crates/pile-dataset/Cargo.toml @@ -20,21 +20,7 @@ chrono = { workspace = true } toml = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } -tokio-util = { version = "0.7", features = ["io"] } - -serde = { workspace = true, optional = true } -axum = { workspace = true, optional = true } -percent-encoding = { workspace = true, optional = true } -utoipa = { workspace = true, optional = true } -utoipa-swagger-ui = { workspace = true, optional = true } [features] default = [] pdfium = ["pile-value/pdfium"] -axum = [ - "dep:axum", - "dep:utoipa", - "dep:utoipa-swagger-ui", - "dep:serde", - "dep:percent-encoding", -] diff --git a/crates/pile-dataset/src/lib.rs b/crates/pile-dataset/src/lib.rs index b27410c..e79ed35 100644 --- a/crates/pile-dataset/src/lib.rs +++ b/crates/pile-dataset/src/lib.rs @@ -2,6 +2,3 @@ mod dataset; pub use dataset::{Dataset, DatasetError, Datasets}; pub mod index; - -#[cfg(feature = "axum")] -pub mod serve; diff --git a/crates/pile-dataset/src/serve/mod.rs b/crates/pile-dataset/src/serve/mod.rs deleted file mode 100644 index a3740a9..0000000 --- a/crates/pile-dataset/src/serve/mod.rs +++ /dev/null @@ -1,92 +0,0 @@ -use axum::{ - Router, - routing::{get, post}, -}; -use std::sync::Arc; -use utoipa::OpenApi; -use utoipa_swagger_ui::SwaggerUi; - -use crate::Datasets; - -mod lookup; -pub use lookup::*; - -mod extract; -pub use extract::*; - -mod items; -pub use items::*; - -mod config_schema; -pub use config_schema::*; - -mod schema_field; -pub use schema_field::*; - -mod schema; -pub use schema::*; - -#[derive(OpenApi)] -#[openapi( - tags(), - paths( - lookup, - get_extract, - items_list, - config_schema, - schema_field, - schema_all - ), - components(schemas( - LookupRequest, - LookupResponse, - LookupResult, - ExtractQuery, - ItemsQuery, - ItemsResponse, - ItemRef - )) -)] -pub(crate) struct Api; - -impl Datasets { - #[inline] - pub fn router(self: Arc, with_docs: bool) -> Router<()> { - self.router_prefix(with_docs, None) - } - - #[inline] - pub fn router_prefix(self: Arc, with_docs: bool, prefix: Option<&str>) -> Router<()> { - let mut router = Router::new() - .route("/lookup", post(lookup)) - .route("/extract", get(get_extract)) - .route("/items", get(items_list)) - .route("/config/schema", get(config_schema)) - .route("/schema", get(schema_all)) - .route("/schema/{field}", get(schema_field)) - .with_state(self.clone()); - - if let Some(prefix) = prefix { - router = Router::new().nest(prefix, router); - } - - if with_docs { - let docs_path = match prefix { - None => "/docs".into(), - Some(prefix) => format!("{prefix}/docs"), - }; - - let api = Api::openapi(); - let api = match prefix { - None => api, - Some(prefix) => utoipa::openapi::OpenApi::default().nest(prefix, api), - }; - - let docs = - SwaggerUi::new(docs_path.clone()).url(format!("{}/openapi.json", docs_path), api); - - router = router.merge(docs); - } - router - } -} diff --git a/crates/pile-serve/Cargo.toml b/crates/pile-serve/Cargo.toml new file mode 100644 index 0000000..2ede2b2 --- /dev/null +++ b/crates/pile-serve/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "pile-serve" +version = { workspace = true } +rust-version = { workspace = true } +edition = { workspace = true } + +[lints] +workspace = true + +[dependencies] +pile-config = { workspace = true } +pile-value = { workspace = true } +pile-dataset = { workspace = true } + +serde_json = { workspace = true } +tracing = { workspace = true } +tokio = { workspace = true } +tokio-util = { version = "0.7", features = ["io"] } + +serde = { workspace = true } +axum = { workspace = true } +percent-encoding = { workspace = true } +utoipa = { workspace = true } +utoipa-swagger-ui = { workspace = true } + +[features] +default = [] +pdfium = ["pile-value/pdfium"] diff --git a/crates/pile-dataset/src/serve/config_schema.rs b/crates/pile-serve/src/config_schema.rs similarity index 95% rename from crates/pile-dataset/src/serve/config_schema.rs rename to crates/pile-serve/src/config_schema.rs index 483e1ed..3e01e58 100644 --- a/crates/pile-dataset/src/serve/config_schema.rs +++ b/crates/pile-serve/src/config_schema.rs @@ -4,12 +4,11 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; +use pile_dataset::Datasets; use std::{collections::HashMap, sync::Arc}; pub use pile_config::FieldSpec; -use crate::Datasets; - pub type FieldsResponse = HashMap; /// Retrieve this dataset's schema. diff --git a/crates/pile-dataset/src/serve/extract.rs b/crates/pile-serve/src/extract.rs similarity index 99% rename from crates/pile-dataset/src/serve/extract.rs rename to crates/pile-serve/src/extract.rs index 6d909a6..c86acd9 100644 --- a/crates/pile-dataset/src/serve/extract.rs +++ b/crates/pile-serve/src/extract.rs @@ -7,6 +7,7 @@ use axum::{ }; use percent_encoding::percent_decode_str; use pile_config::{Label, objectpath::ObjectPath}; +use pile_dataset::Datasets; use pile_value::{ extract::traits::ExtractState, value::{BinaryPileValue, PileValue}, @@ -17,8 +18,6 @@ use tokio_util::io::ReaderStream; use tracing::debug; use utoipa::ToSchema; -use crate::Datasets; - #[derive(Deserialize, ToSchema)] pub struct ExtractQuery { source: String, diff --git a/crates/pile-dataset/src/serve/items.rs b/crates/pile-serve/src/items.rs similarity index 98% rename from crates/pile-dataset/src/serve/items.rs rename to crates/pile-serve/src/items.rs index c411604..0c12a1e 100644 --- a/crates/pile-dataset/src/serve/items.rs +++ b/crates/pile-serve/src/items.rs @@ -4,13 +4,12 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; +use pile_dataset::Datasets; use serde::{Deserialize, Serialize}; use std::sync::Arc; use tracing::debug; use utoipa::ToSchema; -use crate::Datasets; - #[derive(Deserialize, ToSchema)] pub struct ItemsQuery { #[serde(default)] diff --git a/crates/pile-serve/src/lib.rs b/crates/pile-serve/src/lib.rs new file mode 100644 index 0000000..81bc574 --- /dev/null +++ b/crates/pile-serve/src/lib.rs @@ -0,0 +1,89 @@ +use axum::{ + Router, + routing::{get, post}, +}; +use pile_dataset::Datasets; +use std::sync::Arc; +use utoipa::OpenApi; +use utoipa_swagger_ui::SwaggerUi; + +mod lookup; +pub use lookup::*; + +mod extract; +pub use extract::*; + +mod items; +pub use items::*; + +mod config_schema; +pub use config_schema::*; + +mod schema_field; +pub use schema_field::*; + +mod schema; +pub use schema::*; + +#[derive(OpenApi)] +#[openapi( + tags(), + paths( + lookup, + get_extract, + items_list, + config_schema, + schema_field, + schema_all + ), + components(schemas( + LookupRequest, + LookupResponse, + LookupResult, + ExtractQuery, + ItemsQuery, + ItemsResponse, + ItemRef + )) +)] +pub(crate) struct Api; + +#[inline] +pub fn router(ds: Arc, with_docs: bool) -> Router<()> { + router_prefix(ds, with_docs, None) +} + +#[inline] +pub fn router_prefix(ds: Arc, with_docs: bool, prefix: Option<&str>) -> Router<()> { + let mut router = Router::new() + .route("/lookup", post(lookup)) + .route("/extract", get(get_extract)) + .route("/items", get(items_list)) + .route("/config/schema", get(config_schema)) + .route("/schema", get(schema_all)) + .route("/schema/{field}", get(schema_field)) + .with_state(ds.clone()); + + if let Some(prefix) = prefix { + router = Router::new().nest(prefix, router); + } + + if with_docs { + let docs_path = match prefix { + None => "/docs".into(), + Some(prefix) => format!("{prefix}/docs"), + }; + + let api = Api::openapi(); + let api = match prefix { + None => api, + Some(prefix) => utoipa::openapi::OpenApi::default().nest(prefix, api), + }; + + let docs = + SwaggerUi::new(docs_path.clone()).url(format!("{}/openapi.json", docs_path), api); + + router = router.merge(docs); + } + router +} diff --git a/crates/pile-dataset/src/serve/lookup.rs b/crates/pile-serve/src/lookup.rs similarity index 98% rename from crates/pile-dataset/src/serve/lookup.rs rename to crates/pile-serve/src/lookup.rs index 780ee83..3ce59d7 100644 --- a/crates/pile-dataset/src/serve/lookup.rs +++ b/crates/pile-serve/src/lookup.rs @@ -4,13 +4,12 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; +use pile_dataset::Datasets; use serde::{Deserialize, Serialize}; use std::{sync::Arc, time::Instant}; use tracing::debug; use utoipa::ToSchema; -use crate::Datasets; - #[derive(Serialize, Deserialize, ToSchema, Debug)] pub struct LookupRequest { pub query: String, diff --git a/crates/pile-dataset/src/serve/schema.rs b/crates/pile-serve/src/schema.rs similarity index 99% rename from crates/pile-dataset/src/serve/schema.rs rename to crates/pile-serve/src/schema.rs index 16dbea8..50c3479 100644 --- a/crates/pile-dataset/src/serve/schema.rs +++ b/crates/pile-serve/src/schema.rs @@ -5,13 +5,12 @@ use axum::{ response::{IntoResponse, Response}, }; use pile_config::Label; +use pile_dataset::Datasets; use pile_value::{extract::traits::ExtractState, value::PileValue}; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, sync::Arc}; use utoipa::IntoParams; -use crate::Datasets; - #[derive(Deserialize, IntoParams)] pub struct SchemaQuery { source: String, diff --git a/crates/pile-dataset/src/serve/schema_field.rs b/crates/pile-serve/src/schema_field.rs similarity index 99% rename from crates/pile-dataset/src/serve/schema_field.rs rename to crates/pile-serve/src/schema_field.rs index 65cb042..a32103a 100644 --- a/crates/pile-dataset/src/serve/schema_field.rs +++ b/crates/pile-serve/src/schema_field.rs @@ -6,6 +6,7 @@ use axum::{ response::{IntoResponse, Response}, }; use pile_config::Label; +use pile_dataset::Datasets; use pile_value::{ extract::traits::ExtractState, value::{BinaryPileValue, PileValue}, @@ -16,8 +17,6 @@ use tokio_util::io::ReaderStream; use tracing::debug; use utoipa::IntoParams; -use crate::Datasets; - #[derive(Deserialize, IntoParams)] pub struct SchemaFieldQuery { source: String, diff --git a/crates/pile/Cargo.toml b/crates/pile/Cargo.toml index fada178..8572fcb 100644 --- a/crates/pile/Cargo.toml +++ b/crates/pile/Cargo.toml @@ -9,8 +9,9 @@ workspace = true [dependencies] pile-toolbox = { workspace = true } -pile-dataset = { workspace = true, features = ["axum", "pdfium"] } -pile-value = { workspace = true, features = ["pdfium"] } +pile-dataset = { workspace = true } +pile-serve = { workspace = true } +pile-value = { workspace = true } pile-config = { workspace = true } tracing = { workspace = true } @@ -34,3 +35,7 @@ base64 = { workspace = true } dotenvy = { workspace = true } envy = { workspace = true } thiserror = { workspace = true } + +[features] +default = ["pdfium"] +pdfium = ["pile-dataset/pdfium", "pile-serve/pdfium", "pile-value/pdfium"] diff --git a/crates/pile/src/command/serve.rs b/crates/pile/src/command/serve.rs index df4462f..04f8cef 100644 --- a/crates/pile/src/command/serve.rs +++ b/crates/pile/src/command/serve.rs @@ -57,8 +57,7 @@ impl CliCmd for ServeCommand { })?; } - let app = Arc::new(ds) - .router(true) + let app = pile_serve::router(Arc::new(ds), true) .into_make_service_with_connect_info::(); let listener = match tokio::net::TcpListener::bind(self.addr.clone()).await { diff --git a/crates/pile/src/command/server.rs b/crates/pile/src/command/server.rs index 9f6e4e8..1060e82 100644 --- a/crates/pile/src/command/server.rs +++ b/crates/pile/src/command/server.rs @@ -106,7 +106,11 @@ impl CliCmd for ServerCommand { let mut router = Router::new(); for d in datasets.iter() { let prefix = format!("/{}", d.config.dataset.name); - router = router.merge(d.clone().router_prefix(!self.no_docs, Some(&prefix))) + router = router.merge(pile_serve::router_prefix( + d.clone(), + !self.no_docs, + Some(&prefix), + )) } router = router.merge(