Add S3 encryption

crates/pile-dataset/Cargo.toml

@@ -11,6 +11,7 @@ workspace = true
 pile-config = { workspace = true }
 pile-toolbox = { workspace = true }
 pile-value = { workspace = true }
+pile-io = { workspace = true }
 
 serde_json = { workspace = true }
 tantivy = { workspace = true }

crates/pile-dataset/src/dataset.rs

@@ -5,7 +5,7 @@ use pile_config::{
 use pile_toolbox::cancelabletask::{CancelFlag, CancelableTaskError};
 use pile_value::{
 	extract::traits::ExtractState,
-	source::{DataSource, DirDataSource, S3DataSource, misc::path_ts_earliest},
+	source::{DataSource, DirDataSource, S3DataSource, misc::path_ts_earliest, string_to_key},
 	value::{Item, PileValue},
 };
 use serde_json::Value;
@@ -135,11 +135,14 @@ impl Datasets {
 					region,
 					credentials,
 					pattern,
+					encryption_key,
 				} => {
 					if !enabled {
 						continue;
 					}
 
+					let encryption_key = encryption_key.as_ref().map(|x| string_to_key(x));
+
 					match S3DataSource::new(
 						label,
 						bucket.clone(),
@@ -148,6 +151,7 @@ impl Datasets {
 						region.clone(),
 						credentials,
 						pattern.clone(),
+						encryption_key,
 					)
 					.await
 					{
@@ -236,11 +240,14 @@ impl Datasets {
 					region,
 					credentials,
 					pattern,
+					encryption_key,
 				} => {
 					if !enabled {
 						continue;
 					}
 
+					let encryption_key = encryption_key.as_ref().map(|x| string_to_key(x));
+
 					match S3DataSource::new(
 						label,
 						bucket.clone(),
@@ -249,6 +256,7 @@ impl Datasets {
 						region.clone(),
 						credentials,
 						pattern.clone(),
+						encryption_key,
 					)
 					.await
 					{
@@ -349,7 +357,7 @@ impl Datasets {
 					index_writer.add_document(doc).map_err(DatasetError::from)?;
 					total += 1;
 					if logged_at.elapsed().as_secs() >= 5 {
-						debug!("Indexed {total} documents so far");
+						debug!("Indexed {total} documents");
 						logged_at = Instant::now();
 					}
 				}

crates/pile-dataset/src/serve/item.rs

@@ -5,7 +5,7 @@ use axum::{
 	response::{IntoResponse, Response},
 };
 use pile_config::Label;
-use pile_value::value::{AsyncReader, AsyncSeekReader};
+use pile_io::{AsyncReader, AsyncSeekReader};
 use serde::Deserialize;
 use std::{io::SeekFrom, sync::Arc, time::Instant};
 use tokio::sync::mpsc;