to_json tweak

crates/pile-dataset/src/extract/mod.rs

@@ -45,6 +45,21 @@ pub trait ObjectExtractor: Send + Sync {
 	/// `Self::field` must return [Some] for all these keys
 	/// and [None] for all others.
 	async fn fields(&self) -> Result<Vec<Label>, std::io::Error>;
+
+	/// Convert this to a JSON value.
+	async fn to_json(&self) -> Result<serde_json::Value, std::io::Error> {
+		let keys = self.fields().await?;
+		let mut map = serde_json::Map::new();
+		for k in &keys {
+			let v = match self.field(k).await? {
+				Some(x) => x,
+				None => continue,
+			};
+			map.insert(k.to_string(), Box::pin(v.to_json()).await?);
+		}
+
+		Ok(serde_json::Value::Object(map))
+	}
 }
 
 /// An attachment that extracts metadata from an [Item].
@@ -63,6 +78,22 @@ pub trait ListExtractor: Send + Sync {
 	async fn is_empty(&self) -> Result<bool, std::io::Error> {
 		Ok(self.len().await? == 0)
 	}
+
+	/// Convert this list to a JSON value.
+	async fn to_json(&self) -> Result<serde_json::Value, std::io::Error> {
+		let len = self.len().await?;
+		let mut list = Vec::with_capacity(len);
+		for i in 0..len {
+			#[expect(clippy::expect_used)]
+			let v = self
+				.get(i)
+				.await?
+				.expect("value must be present according to length");
+			list.push(Box::pin(v.to_json()).await?);
+		}
+
+		Ok(serde_json::Value::Array(list))
+	}
 }
 
 pub struct MetaExtractor {

crates/pile-dataset/src/extract/pdf/pdf_pages.rs

@@ -92,4 +92,13 @@ impl ListExtractor for PdfPagesExtractor {
 			}
 		}
 	}
+
+	// Override, extracting all pages is very slow,
+	// and we can't display binary in json anyway
+	async fn to_json(&self) -> Result<serde_json::Value, std::io::Error> {
+		Ok(serde_json::Value::String(format!(
+			"<PdfPages ({} pages)>",
+			self.len().await?
+		)))
+	}
 }

crates/pile-dataset/src/value.rs

@@ -188,19 +188,7 @@ impl PileValue {
 				Value::Object(map)
 			}
 
-			Self::ListExtractor(e) => {
+			Self::ListExtractor(e) => e.to_json().await?,
-				let len = e.len().await?;
-				let mut list = Vec::with_capacity(len);
-				for i in 0..len {
-					#[expect(clippy::expect_used)]
-					let v = e.get(i)
-						.await?
-						.expect("value must be present according to length");
-					list.push(Box::pin(v.to_json()).await?);
-				}
-
-				Value::Array(list)
-			}
 		})
 	}
 }