Split server into crate

Cargo.lock

@@ -1994,6 +1994,7 @@ dependencies = [
  "indicatif",
  "pile-config",
  "pile-dataset",
+ "pile-serve",
  "pile-toolbox",
  "pile-value",
  "serde",
@@ -2016,7 +2017,7 @@ version = "0.0.2"
 dependencies = [
  "axum",
  "bytes",
- "pile-dataset",
+ "pile-serve",
  "reqwest",
  "serde",
  "thiserror",
@@ -2037,23 +2038,17 @@ dependencies = [
 name = "pile-dataset"
 version = "0.0.2"
 dependencies = [
- "axum",
  "chrono",
- "percent-encoding",
  "pile-config",
  "pile-toolbox",
  "pile-value",
  "regex",
- "serde",
  "serde_json",
  "tantivy",
  "thiserror",
  "tokio",
- "tokio-util",
  "toml",
  "tracing",
- "utoipa",
- "utoipa-swagger-ui",
 ]
 
 [[package]]
@@ -2077,6 +2072,24 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "pile-serve"
+version = "0.0.2"
+dependencies = [
+ "axum",
+ "percent-encoding",
+ "pile-config",
+ "pile-dataset",
+ "pile-value",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tokio-util",
+ "tracing",
+ "utoipa",
+ "utoipa-swagger-ui",
+]
+
 [[package]]
 name = "pile-toolbox"
 version = "0.0.2"

Cargo.toml

@@ -71,6 +71,7 @@ pile-dataset = { path = "crates/pile-dataset" }
 pile-value = { path = "crates/pile-value" }
 pile-io = { path = "crates/pile-io" }
 pile-client = { path = "crates/pile-client" }
+pile-serve = { path = "crates/pile-serve" }
 
 # MARK: Clients & servers
 tantivy = "0.25.0"
@@ -87,7 +88,7 @@ utoipa-swagger-ui = { version = "9.0.2", features = [
 	"debug-embed",
 	"vendored",
 ] }
-reqwest = { version = "0.12", features = ["blocking"] }
+reqwest = { version = "0.12", features = ["blocking", "json", "stream"] }
 tracing-loki = "0.2.6"
 
 # MARK: Async & Parallelism

crates/pile-client/Cargo.toml

@@ -8,9 +8,9 @@ edition = { workspace = true }
 workspace = true
 
 [dependencies]
-pile-dataset = { workspace = true, features = ["axum"] }
+pile-serve = { workspace = true }
 
-reqwest = { version = "0.12", features = ["json", "stream"] }
+reqwest = { workspace = true }
 serde = { workspace = true }
 thiserror = { workspace = true }
 bytes = { workspace = true }

crates/pile-client/src/lib.rs

@@ -8,7 +8,7 @@ use serde::Deserialize;
 use thiserror::Error;
 use tracing::{trace, warn};
 
-pub use pile_dataset::serve::{
+pub use pile_serve::{
 	ApiValue, FieldSpec, FieldsResponse, ItemsResponse, LookupRequest, LookupResponse,
 	SchemaResponse,
 };

crates/pile-dataset/Cargo.toml

@@ -20,21 +20,7 @@ chrono = { workspace = true }
 toml = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
-tokio-util = { version = "0.7", features = ["io"] }
-
-serde = { workspace = true, optional = true }
-axum = { workspace = true, optional = true }
-percent-encoding = { workspace = true, optional = true }
-utoipa = { workspace = true, optional = true }
-utoipa-swagger-ui = { workspace = true, optional = true }
 
 [features]
 default = []
 pdfium = ["pile-value/pdfium"]
-axum = [
-	"dep:axum",
-	"dep:utoipa",
-	"dep:utoipa-swagger-ui",
-	"dep:serde",
-	"dep:percent-encoding",
-]

crates/pile-dataset/src/lib.rs

@@ -2,6 +2,3 @@ mod dataset;
 pub use dataset::{Dataset, DatasetError, Datasets};
 
 pub mod index;
-
-#[cfg(feature = "axum")]
-pub mod serve;

crates/pile-dataset/src/serve/mod.rs

@@ -1,92 +0,0 @@
-use axum::{
-	Router,
-	routing::{get, post},
-};
-use std::sync::Arc;
-use utoipa::OpenApi;
-use utoipa_swagger_ui::SwaggerUi;
-
-use crate::Datasets;
-
-mod lookup;
-pub use lookup::*;
-
-mod extract;
-pub use extract::*;
-
-mod items;
-pub use items::*;
-
-mod config_schema;
-pub use config_schema::*;
-
-mod schema_field;
-pub use schema_field::*;
-
-mod schema;
-pub use schema::*;
-
-#[derive(OpenApi)]
-#[openapi(
-	tags(),
-	paths(
-		lookup,
-		get_extract,
-		items_list,
-		config_schema,
-		schema_field,
-		schema_all
-	),
-	components(schemas(
-		LookupRequest,
-		LookupResponse,
-		LookupResult,
-		ExtractQuery,
-		ItemsQuery,
-		ItemsResponse,
-		ItemRef
-	))
-)]
-pub(crate) struct Api;
-
-impl Datasets {
-	#[inline]
-	pub fn router(self: Arc<Self>, with_docs: bool) -> Router<()> {
-		self.router_prefix(with_docs, None)
-	}
-
-	#[inline]
-	pub fn router_prefix(self: Arc<Self>, with_docs: bool, prefix: Option<&str>) -> Router<()> {
-		let mut router = Router::new()
-			.route("/lookup", post(lookup))
-			.route("/extract", get(get_extract))
-			.route("/items", get(items_list))
-			.route("/config/schema", get(config_schema))
-			.route("/schema", get(schema_all))
-			.route("/schema/{field}", get(schema_field))
-			.with_state(self.clone());
-
-		if let Some(prefix) = prefix {
-			router = Router::new().nest(prefix, router);
-		}
-
-		if with_docs {
-			let docs_path = match prefix {
-				None => "/docs".into(),
-				Some(prefix) => format!("{prefix}/docs"),
-			};
-
-			let api = Api::openapi();
-			let api = match prefix {
-				None => api,
-				Some(prefix) => utoipa::openapi::OpenApi::default().nest(prefix, api),
-			};
-
-			let docs =
-				SwaggerUi::new(docs_path.clone()).url(format!("{}/openapi.json", docs_path), api);
-
-			router = router.merge(docs);
-		}
-		router
-	}
-}

crates/pile-serve/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "pile-serve"
+version = { workspace = true }
+rust-version = { workspace = true }
+edition = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+pile-config = { workspace = true }
+pile-value = { workspace = true }
+pile-dataset = { workspace = true }
+
+serde_json = { workspace = true }
+tracing = { workspace = true }
+tokio = { workspace = true }
+tokio-util = { version = "0.7", features = ["io"] }
+
+serde = { workspace = true }
+axum = { workspace = true }
+percent-encoding = { workspace = true }
+utoipa = { workspace = true }
+utoipa-swagger-ui = { workspace = true }
+
+[features]
+default = []
+pdfium = ["pile-value/pdfium"]

crates/pile-serve/src/config_schema.rs

@@ -4,12 +4,11 @@ use axum::{
 	http::StatusCode,
 	response::{IntoResponse, Response},
 };
+use pile_dataset::Datasets;
 use std::{collections::HashMap, sync::Arc};
 
 pub use pile_config::FieldSpec;
 
-use crate::Datasets;
-
 pub type FieldsResponse = HashMap<String, FieldSpec>;
 
 /// Retrieve this dataset's schema.

crates/pile-serve/src/extract.rs

@@ -7,6 +7,7 @@ use axum::{
 };
 use percent_encoding::percent_decode_str;
 use pile_config::{Label, objectpath::ObjectPath};
+use pile_dataset::Datasets;
 use pile_value::{
 	extract::traits::ExtractState,
 	value::{BinaryPileValue, PileValue},
@@ -17,8 +18,6 @@ use tokio_util::io::ReaderStream;
 use tracing::debug;
 use utoipa::ToSchema;
 
-use crate::Datasets;
-
 #[derive(Deserialize, ToSchema)]
 pub struct ExtractQuery {
 	source: String,

crates/pile-serve/src/items.rs

@@ -4,13 +4,12 @@ use axum::{
 	http::StatusCode,
 	response::{IntoResponse, Response},
 };
+use pile_dataset::Datasets;
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use tracing::debug;
 use utoipa::ToSchema;
 
-use crate::Datasets;
-
 #[derive(Deserialize, ToSchema)]
 pub struct ItemsQuery {
 	#[serde(default)]

crates/pile-serve/src/lib.rs

@@ -0,0 +1,89 @@
+use axum::{
+	Router,
+	routing::{get, post},
+};
+use pile_dataset::Datasets;
+use std::sync::Arc;
+use utoipa::OpenApi;
+use utoipa_swagger_ui::SwaggerUi;
+
+mod lookup;
+pub use lookup::*;
+
+mod extract;
+pub use extract::*;
+
+mod items;
+pub use items::*;
+
+mod config_schema;
+pub use config_schema::*;
+
+mod schema_field;
+pub use schema_field::*;
+
+mod schema;
+pub use schema::*;
+
+#[derive(OpenApi)]
+#[openapi(
+	tags(),
+	paths(
+		lookup,
+		get_extract,
+		items_list,
+		config_schema,
+		schema_field,
+		schema_all
+	),
+	components(schemas(
+		LookupRequest,
+		LookupResponse,
+		LookupResult,
+		ExtractQuery,
+		ItemsQuery,
+		ItemsResponse,
+		ItemRef
+	))
+)]
+pub(crate) struct Api;
+
+#[inline]
+pub fn router(ds: Arc<Datasets>, with_docs: bool) -> Router<()> {
+	router_prefix(ds, with_docs, None)
+}
+
+#[inline]
+pub fn router_prefix(ds: Arc<Datasets>, with_docs: bool, prefix: Option<&str>) -> Router<()> {
+	let mut router = Router::new()
+		.route("/lookup", post(lookup))
+		.route("/extract", get(get_extract))
+		.route("/items", get(items_list))
+		.route("/config/schema", get(config_schema))
+		.route("/schema", get(schema_all))
+		.route("/schema/{field}", get(schema_field))
+		.with_state(ds.clone());
+
+	if let Some(prefix) = prefix {
+		router = Router::new().nest(prefix, router);
+	}
+
+	if with_docs {
+		let docs_path = match prefix {
+			None => "/docs".into(),
+			Some(prefix) => format!("{prefix}/docs"),
+		};
+
+		let api = Api::openapi();
+		let api = match prefix {
+			None => api,
+			Some(prefix) => utoipa::openapi::OpenApi::default().nest(prefix, api),
+		};
+
+		let docs =
+			SwaggerUi::new(docs_path.clone()).url(format!("{}/openapi.json", docs_path), api);
+
+		router = router.merge(docs);
+	}
+	router
+}

crates/pile-serve/src/lookup.rs

@@ -4,13 +4,12 @@ use axum::{
 	http::StatusCode,
 	response::{IntoResponse, Response},
 };
+use pile_dataset::Datasets;
 use serde::{Deserialize, Serialize};
 use std::{sync::Arc, time::Instant};
 use tracing::debug;
 use utoipa::ToSchema;
 
-use crate::Datasets;
-
 #[derive(Serialize, Deserialize, ToSchema, Debug)]
 pub struct LookupRequest {
 	pub query: String,

crates/pile-serve/src/schema.rs

@@ -5,13 +5,12 @@ use axum::{
 	response::{IntoResponse, Response},
 };
 use pile_config::Label;
+use pile_dataset::Datasets;
 use pile_value::{extract::traits::ExtractState, value::PileValue};
 use serde::{Deserialize, Serialize};
 use std::{collections::HashMap, sync::Arc};
 use utoipa::IntoParams;
 
-use crate::Datasets;
-
 #[derive(Deserialize, IntoParams)]
 pub struct SchemaQuery {
 	source: String,

crates/pile-serve/src/schema_field.rs

@@ -6,6 +6,7 @@ use axum::{
 	response::{IntoResponse, Response},
 };
 use pile_config::Label;
+use pile_dataset::Datasets;
 use pile_value::{
 	extract::traits::ExtractState,
 	value::{BinaryPileValue, PileValue},
@@ -16,8 +17,6 @@ use tokio_util::io::ReaderStream;
 use tracing::debug;
 use utoipa::IntoParams;
 
-use crate::Datasets;
-
 #[derive(Deserialize, IntoParams)]
 pub struct SchemaFieldQuery {
 	source: String,

crates/pile/Cargo.toml

@@ -9,8 +9,9 @@ workspace = true
 
 [dependencies]
 pile-toolbox = { workspace = true }
-pile-dataset = { workspace = true, features = ["axum", "pdfium"] }
-pile-value = { workspace = true, features = ["pdfium"] }
+pile-dataset = { workspace = true }
+pile-serve = { workspace = true }
+pile-value = { workspace = true }
 pile-config = { workspace = true }
 
 tracing = { workspace = true }
@@ -34,3 +35,7 @@ base64 = { workspace = true }
 dotenvy = { workspace = true }
 envy = { workspace = true }
 thiserror = { workspace = true }
+
+[features]
+default = ["pdfium"]
+pdfium = ["pile-dataset/pdfium", "pile-serve/pdfium", "pile-value/pdfium"]

crates/pile/src/command/serve.rs

@@ -57,8 +57,7 @@ impl CliCmd for ServeCommand {
 				})?;
 		}
 
-		let app = Arc::new(ds)
-			.router(true)
+		let app = pile_serve::router(Arc::new(ds), true)
 			.into_make_service_with_connect_info::<std::net::SocketAddr>();
 
 		let listener = match tokio::net::TcpListener::bind(self.addr.clone()).await {

crates/pile/src/command/server.rs

@@ -106,7 +106,11 @@ impl CliCmd for ServerCommand {
 		let mut router = Router::new();
 		for d in datasets.iter() {
 			let prefix = format!("/{}", d.config.dataset.name);
-			router = router.merge(d.clone().router_prefix(!self.no_docs, Some(&prefix)))
+			router = router.merge(pile_serve::router_prefix(
+				d.clone(),
+				!self.no_docs,
+				Some(&prefix),
+			))
 		}
 
 		router = router.merge(