Filter by mime

crates/pile-dataset/src/index/index_fts.rs

@@ -1,4 +1,4 @@
-use pile_config::{ConfigToml, DatasetFts, Label, objectpath::ObjectPath};
+use pile_config::{ConfigToml, DatasetFts, Label};
 use pile_value::{
 	extract::traits::ExtractState,
 	value::{Item, PileValue},
@@ -10,7 +10,7 @@ use tantivy::{
 	query::QueryParser,
 	schema::{self, Schema, Value as TantivyValue},
 };
-use tracing::warn;
+use tracing::{debug, trace, warn};
 
 #[derive(Debug, Clone)]
 pub struct FtsLookupResult {
@@ -79,12 +79,18 @@ impl DbFtsIndex {
 
 		let mut empty = true;
 		for name in self.fts_cfg().fields.keys() {
-			let vals = self.get_field(state, &item, name).await?;
+			let x = self.get_field(state, &item, name).await?;
-			let field = self.schema.get_field(name)?;
 
-			for v in vals {
+			let val = match x {
-				empty = false;
+				Some(x) => x,
-				doc.add_text(field, v);
+				None => continue,
+			};
+
+			empty = false;
+
+			let field = self.schema.get_field(name);
+			if let Ok(field) = field {
+				doc.add_text(field, val);
 			}
 		}
 
@@ -100,29 +106,110 @@ impl DbFtsIndex {
 		state: &ExtractState,
 		extractor: &PileValue,
 		field_name: &Label,
-	) -> Result<Vec<String>, std::io::Error> {
+	) -> Result<Option<String>, std::io::Error> {
 		let field = match self.cfg.schema.get(field_name) {
 			Some(x) => x,
 			None => {
 				warn!("Unknown field {field_name:?}");
-				return Ok(Vec::new());
+				return Ok(None);
 			}
 		};
 
 		// Try paths in order, using the first value we find
-		for path in field.path.as_slice() {
+		'outer: for path in field.path.as_slice() {
 			let val = match extractor.query(state, path).await? {
 				Some(x) => x,
-				None => continue,
+				None => return Ok(None),
 			};
 
-			let val = val_to_string(state, &val, path, field_name).await?;
+			let mut val = match val {
-			if !val.is_empty() {
+				PileValue::Null => {
-				return Ok(val);
+					trace!(
+						message = "Skipping field, is null",
+						field = field_name.to_string(),
+						?path,
+						// value = ?val
+					);
+					continue;
+				}
+				x => x.clone(),
+			};
+
+			loop {
+				val = match val {
+					PileValue::String(x) => return Ok(Some(x.to_string())),
+					PileValue::U64(x) => return Ok(Some(x.to_string())),
+					PileValue::I64(x) => return Ok(Some(x.to_string())),
+
+					PileValue::Array(x) => {
+						if x.len() == 1 {
+							x[0].clone()
+						} else if x.len() > 1 {
+							debug!(
+								message = "Skipping field, is array with more than one element",
+								field = field_name.to_string(),
+								?path,
+							);
+							continue 'outer;
+						} else {
+							debug!(
+								message = "Skipping field, is empty array",
+								field = field_name.to_string(),
+								?path,
+							);
+							continue 'outer;
+						}
+					}
+
+					PileValue::Null => {
+						trace!(
+							message = "Skipping field, is null",
+							field = field_name.to_string(),
+							?path,
+						);
+						continue 'outer;
+					}
+
+					PileValue::ObjectExtractor(_) => {
+						trace!(
+							message = "Skipping field, is object",
+							field = field_name.to_string(),
+							?path,
+						);
+						continue 'outer;
+					}
+
+					PileValue::Item(_) => {
+						trace!(
+							message = "Skipping field, is item",
+							field = field_name.to_string(),
+							?path,
+						);
+						continue 'outer;
+					}
+
+					PileValue::ListExtractor(_) => {
+						trace!(
+							message = "Skipping field, is ListExtractor",
+							field = field_name.to_string(),
+							?path,
+						);
+						continue 'outer;
+					}
+
+					PileValue::Blob { .. } => {
+						trace!(
+							message = "Skipping field, is blob",
+							field = field_name.to_string(),
+							?path,
+						);
+						continue 'outer;
+					}
+				}
 			}
 		}
 
-		return Ok(Vec::new());
+		return Ok(None);
 	}
 
 	/// Run the given query on this table's ftx index.
@@ -211,42 +298,3 @@ impl DbFtsIndex {
 		return Ok(out);
 	}
 }
-
-async fn val_to_string(
-	state: &ExtractState,
-	val: &PileValue,
-	path: &ObjectPath,
-	field_name: &str,
-) -> Result<Vec<String>, std::io::Error> {
-	match val {
-		PileValue::String(x) => return Ok(vec![x.to_string()]),
-		PileValue::U64(x) => return Ok(vec![x.to_string()]),
-		PileValue::I64(x) => return Ok(vec![x.to_string()]),
-
-		PileValue::Array(x) => {
-			let mut out = Vec::new();
-			for x in x.iter() {
-				out.extend(Box::pin(val_to_string(state, x, path, field_name)).await?);
-			}
-			return Ok(out);
-		}
-
-		#[expect(clippy::unwrap_used)]
-		PileValue::ListExtractor(x) => {
-			let mut out = Vec::new();
-			let len = x.len(state).await?;
-			for i in 0..len {
-				let v = x.get(state, i).await?;
-				out.extend(Box::pin(val_to_string(state, &v.unwrap(), path, field_name)).await?);
-			}
-			return Ok(out);
-		}
-
-		PileValue::Null => {}
-		PileValue::ObjectExtractor(_) => {}
-		PileValue::Item(_) => {}
-		PileValue::Blob { .. } => {}
-	}
-
-	return Ok(Vec::new());
-}

crates/pile-value/src/extract/item/id3.rs

@@ -58,15 +58,13 @@ impl Id3Extractor {
 
 		let mut output: HashMap<Label, Vec<PileValue>> = HashMap::new();
 		for frame in tag.frames() {
-			if let Some(texts) = frame.content().text_values() {
+			if let Some(text) = frame.content().text() {
 				let name = frame_id_to_field(frame.id());
 				if let Some(key) = Label::new(name) {
-					for text in texts {
+					output
-						output
+						.entry(key)
-							.entry(key.clone())
+						.or_default()
-							.or_default()
+						.push(PileValue::String(Arc::new(text.into())));
-							.push(PileValue::String(Arc::new(text.into())));
-					}
 				}
 			}
 		}

crates/pile-value/src/extract/item/json.rs

@@ -1,87 +0,0 @@
-use pile_config::Label;
-use std::{
-	collections::HashMap,
-	sync::{Arc, OnceLock},
-};
-
-use crate::{
-	extract::traits::{ExtractState, ObjectExtractor},
-	value::{AsyncReader, Item, PileValue},
-};
-
-fn json_to_pile(value: serde_json::Value) -> PileValue {
-	match value {
-		serde_json::Value::Null => PileValue::Null,
-		serde_json::Value::Bool(b) => PileValue::String(Arc::new(b.to_string().into())),
-		serde_json::Value::Number(n) => PileValue::String(Arc::new(n.to_string().into())),
-		serde_json::Value::String(s) => PileValue::String(Arc::new(s.into())),
-		serde_json::Value::Array(a) => {
-			PileValue::Array(Arc::new(a.into_iter().map(json_to_pile).collect()))
-		}
-		serde_json::Value::Object(_) => PileValue::Null,
-	}
-}
-
-pub struct JsonExtractor {
-	item: Item,
-	output: OnceLock<HashMap<Label, PileValue>>,
-}
-
-impl JsonExtractor {
-	pub fn new(item: &Item) -> Self {
-		Self {
-			item: item.clone(),
-			output: OnceLock::new(),
-		}
-	}
-
-	async fn get_inner(&self) -> Result<&HashMap<Label, PileValue>, std::io::Error> {
-		if let Some(x) = self.output.get() {
-			return Ok(x);
-		}
-
-		let mut reader = self.item.read().await?;
-		let bytes = reader.read_to_end().await?;
-		let json: serde_json::Value = match serde_json::from_slice(&bytes) {
-			Ok(x) => x,
-			Err(_) => return Ok(self.output.get_or_init(HashMap::new)),
-		};
-
-		let output: HashMap<Label, PileValue> = match json {
-			serde_json::Value::Object(map) => map
-				.into_iter()
-				.filter_map(|(k, v)| Label::new(&k).map(|label| (label, json_to_pile(v))))
-				.collect(),
-			_ => HashMap::new(),
-		};
-
-		return Ok(self.output.get_or_init(|| output));
-	}
-}
-
-#[async_trait::async_trait]
-impl ObjectExtractor for JsonExtractor {
-	async fn field(
-		&self,
-		state: &ExtractState,
-		name: &Label,
-		args: Option<&str>,
-	) -> Result<Option<PileValue>, std::io::Error> {
-		if args.is_some() {
-			return Ok(None);
-		}
-
-		if !state.ignore_mime
-			&& (self.item.mime().type_() != mime::APPLICATION
-				&& self.item.mime().type_() != mime::TEXT)
-		{
-			return Ok(None);
-		}
-
-		Ok(self.get_inner().await?.get(name).cloned())
-	}
-
-	async fn fields(&self) -> Result<Vec<Label>, std::io::Error> {
-		Ok(self.get_inner().await?.keys().cloned().collect())
-	}
-}

crates/pile-value/src/extract/item/mod.rs

@@ -18,9 +18,6 @@ pub use exif::*;
 mod pdf;
 pub use pdf::*;
 
-mod json;
-pub use json::*;
-
 mod toml;
 use pile_config::Label;
 pub use toml::*;
@@ -69,10 +66,6 @@ impl ItemExtractor {
 					Label::new("pdf").unwrap(),
 					PileValue::ObjectExtractor(Arc::new(PdfExtractor::new(item))),
 				),
-				(
-					Label::new("json").unwrap(),
-					PileValue::ObjectExtractor(Arc::new(JsonExtractor::new(item))),
-				),
 				(
 					Label::new("toml").unwrap(),
 					PileValue::ObjectExtractor(Arc::new(TomlExtractor::new(item))),
@@ -108,7 +101,6 @@ impl ObjectExtractor for ItemExtractor {
 			Label::new("epub").unwrap(),
 			Label::new("exif").unwrap(),
 			Label::new("pdf").unwrap(),
-			Label::new("json").unwrap(),
 			Label::new("sidecar").unwrap(),
 		]);
 	}

crates/pile-value/src/source/dir.rs

@@ -47,20 +47,14 @@ impl DataSource for Arc<DirDataSource> {
 			source: Arc::clone(self),
 			mime: mime_guess::from_path(&key).first_or_octet_stream(),
 			path: key.clone(),
-			sidecar: self
+			sidecar: self.sidecars.then(|| {
-				.sidecars
+				Box::new(Item::File {
-				.then(|| {
+					source: Arc::clone(self),
-					let sidecar_path = key.with_extension("toml");
+					mime: mime_guess::from_path(key.with_extension("toml")).first_or_octet_stream(),
-					sidecar_path.is_file().then(|| {
+					path: key.with_extension("toml"),
-						Box::new(Item::File {
+					sidecar: None,
-							source: Arc::clone(self),
-							mime: mime_guess::from_path(&sidecar_path).first_or_octet_stream(),
-							path: sidecar_path,
-							sidecar: None,
-						})
-					})
 				})
-				.flatten(),
+			}),
 		}));
 	}
 
@@ -97,21 +91,15 @@ impl DataSource for Arc<DirDataSource> {
 						mime: mime_guess::from_path(&path).first_or_octet_stream(),
 						path: path.clone(),
 
-						sidecar: source
+						sidecar: source.sidecars.then(|| {
-							.sidecars
+							Box::new(Item::File {
-							.then(|| {
+								source: Arc::clone(&source),
-								let sidecar_path = path.with_extension("toml");
+								mime: mime_guess::from_path(path.with_extension("toml"))
-								sidecar_path.is_file().then(|| {
+									.first_or_octet_stream(),
-									Box::new(Item::File {
+								path: path.with_extension("toml"),
-										source: Arc::clone(&source),
+								sidecar: None,
-										mime: mime_guess::from_path(&sidecar_path)
-											.first_or_octet_stream(),
-										path: sidecar_path,
-										sidecar: None,
-									})
-								})
 							})
-							.flatten(),
+						}),
 					},
 				};
 

crates/pile/src/command/annotate.rs

@@ -77,16 +77,18 @@ impl CliCmd for AnnotateCommand {
 						};
 
 						let item = PileValue::Item(item.clone());
-						let vals =
+						let Some(value) = index
-							index
+							.get_field(&state, &item, &field)
-								.get_field(&state, &item, &field)
+							.await
-								.await
+							.with_context(|| {
-								.with_context(|| {
+								format!("while extracting field from {}", path.display())
-									format!("while extracting field from {}", path.display())
+							})?
-								})?;
+						else {
+							continue;
+						};
 
 						// TODO: implement sidecar writing
-						let _ = (&dest_path, &vals);
+						let _ = (&dest_path, &value);
 						todo!("write_sidecar not yet implemented");
 
 						#[expect(unreachable_code)]