Extractor refactor, S3 support

crates/pile-dataset/src/extract/toml.rs

@@ -0,0 +1,66 @@
+use pile_config::Label;
+use std::{collections::HashMap, sync::OnceLock};
+
+use crate::{Item, PileValue, extract::Extractor};
+
+fn toml_to_pile(value: toml::Value) -> PileValue<'static> {
+	match value {
+		toml::Value::String(s) => PileValue::String(s.into()),
+		toml::Value::Integer(i) => PileValue::String(i.to_string().into()),
+		toml::Value::Float(f) => PileValue::String(f.to_string().into()),
+		toml::Value::Boolean(b) => PileValue::String(b.to_string().into()),
+		toml::Value::Datetime(d) => PileValue::String(d.to_string().into()),
+		toml::Value::Array(a) => PileValue::Array(a.into_iter().map(toml_to_pile).collect()),
+		toml::Value::Table(_) => PileValue::Null,
+	}
+}
+
+pub struct TomlExtractor<'a> {
+	item: &'a Item,
+	output: OnceLock<HashMap<Label, PileValue<'a>>>,
+}
+
+impl<'a> TomlExtractor<'a> {
+	pub fn new(item: &'a Item) -> Self {
+		Self {
+			item,
+			output: OnceLock::new(),
+		}
+	}
+
+	async fn get_inner(&self) -> Result<&HashMap<Label, PileValue<'a>>, std::io::Error> {
+		if let Some(x) = self.output.get() {
+			return Ok(x);
+		}
+
+		let bytes = self.item.read().await?.read_to_end().await?;
+		let toml: toml::Value = match toml::from_slice(&bytes) {
+			Ok(x) => x,
+			Err(_) => return Ok(self.output.get_or_init(HashMap::new)),
+		};
+
+		let output: HashMap<Label, PileValue<'_>> = match toml {
+			toml::Value::Table(t) => t
+				.into_iter()
+				.filter_map(|(k, v)| Label::new(&k).map(|label| (label, toml_to_pile(v))))
+				.collect(),
+			_ => HashMap::new(),
+		};
+
+		return Ok(self.output.get_or_init(|| output));
+	}
+}
+
+#[async_trait::async_trait]
+impl Extractor for TomlExtractor<'_> {
+	async fn field<'a>(
+		&'a self,
+		name: &Label,
+	) -> Result<Option<&'a PileValue<'a>>, std::io::Error> {
+		Ok(self.get_inner().await?.get(name))
+	}
+
+	async fn fields(&self) -> Result<Vec<Label>, std::io::Error> {
+		Ok(self.get_inner().await?.keys().cloned().collect())
+	}
+}