Read files incrementally

crates/pile-dataset/src/extract/exif.rs

@@ -1,8 +1,8 @@
 use pile_config::Label;
-use std::{collections::HashMap, io::Cursor, sync::OnceLock};
+use std::{collections::HashMap, io::BufReader, sync::OnceLock};
 use tracing::debug;
 
-use crate::{Item, PileValue, extract::Extractor};
+use crate::{Item, PileValue, SyncReadBridge, extract::Extractor};
 
 pub struct ExifExtractor<'a> {
 	item: &'a Item,
@@ -22,10 +22,29 @@ impl<'a> ExifExtractor<'a> {
 			return Ok(x);
 		}
 
-		let bytes = self.item.read().await?.read_to_end().await?;
-		let mut cursor = Cursor::new(bytes);
+		let reader = SyncReadBridge::new_current(self.item.read().await?);
+		let raw_fields = tokio::task::spawn_blocking(move || {
+			let mut br = BufReader::new(reader);
+			let exif = exif::Reader::new()
+				.read_from_container(&mut br)
+				.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?;
 
-		let exif = match exif::Reader::new().read_from_container(&mut cursor) {
+			let fields: Vec<(String, String)> = exif
+				.fields()
+				.map(|f| {
+					(
+						f.tag.to_string(),
+						f.display_value().with_unit(&exif).to_string(),
+					)
+				})
+				.collect();
+
+			Ok::<_, std::io::Error>(fields)
+		})
+		.await
+		.map_err(std::io::Error::other)?;
+
+		let raw_fields = match raw_fields {
 			Ok(x) => x,
 			Err(error) => {
 				debug!(message = "Could not process exif", ?error, key = ?self.item.key());
@@ -35,23 +54,22 @@ impl<'a> ExifExtractor<'a> {
 
 		let mut output: HashMap<Label, PileValue<'a>> = HashMap::new();
 
-		for field in exif.fields() {
-			let Some(label) = tag_to_label(&field.tag) else {
+		for (tag_name, value) in raw_fields {
+			let Some(label) = tag_to_label(&tag_name) else {
 				continue;
 			};
 			// First occurrence wins (PRIMARY IFD comes before THUMBNAIL)
-			output.entry(label).or_insert_with(|| {
-				PileValue::String(field.display_value().with_unit(&exif).to_string().into())
-			});
+			output
+				.entry(label)
+				.or_insert_with(|| PileValue::String(value.into()));
 		}
 
 		return Ok(self.output.get_or_init(|| output));
 	}
 }
 
-fn tag_to_label(tag: &exif::Tag) -> Option<Label> {
+fn tag_to_label(tag: &str) -> Option<Label> {
 	let sanitized: String = tag
-		.to_string()
 		.chars()
 		.map(|c| if c == ' ' { '_' } else { c })
 		.filter(|c| Label::VALID_CHARS.contains(*c))