Reorganize S3 clients

crates/pile-value/Cargo.toml

@@ -31,7 +31,6 @@ image = { workspace = true, optional = true }
 id3 = { workspace = true }
 tokio = { workspace = true }
 async-trait = { workspace = true }
-aws-sdk-s3 = { workspace = true }
 mime = { workspace = true }
 mime_guess = { workspace = true }
 

crates/pile-value/src/source/s3.rs

@@ -1,9 +1,9 @@
-use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
 use chrono::{DateTime, Utc};
 use pile_config::{
-	Label, S3Credentials,
+	Label,
 	pattern::{GroupPattern, GroupSegment},
 };
+use pile_io::S3Client;
 use smartstring::{LazyCompact, SmartString};
 use std::{
 	collections::{HashMap, HashSet},
@@ -19,9 +19,9 @@ use crate::{
 #[derive(Debug)]
 pub struct S3DataSource {
 	pub name: Label,
-	pub bucket: SmartString<LazyCompact>,
+	pub client: Arc<S3Client>,
+
 	pub prefix: Option<SmartString<LazyCompact>>,
-	pub client: Arc<aws_sdk_s3::Client>,
 	pub pattern: GroupPattern,
 	pub encryption_key: Option<[u8; 32]>,
 	pub index: OnceLock<HashMap<SmartString<LazyCompact>, Item>>,
@@ -30,40 +30,30 @@ pub struct S3DataSource {
 impl S3DataSource {
 	pub async fn new(
 		name: &Label,
-		bucket: String,
-		prefix: Option<String>,
-		endpoint: Option<String>,
-		region: String,
-		credentials: &S3Credentials,
+		bucket: &str,
+		prefix: Option<&str>,
+		endpoint: Option<&str>,
+		region: &str,
+		access_key_id: &str,
+		secret_access_key: &str,
+		cache_limit_bytes: usize,
 		pattern: GroupPattern,
 		encryption_key: Option<[u8; 32]>,
 	) -> Result<Arc<Self>, std::io::Error> {
-		let client = {
-			let creds = Credentials::new(
-				&credentials.access_key_id,
-				&credentials.secret_access_key,
-				None,
-				None,
-				"pile",
-			);
-
-			let mut s3_config = aws_sdk_s3::config::Builder::new()
-				.behavior_version(BehaviorVersion::latest())
-				.region(Region::new(region))
-				.credentials_provider(creds);
-
-			if let Some(ep) = endpoint {
-				s3_config = s3_config.endpoint_url(ep).force_path_style(true);
-			}
-
-			aws_sdk_s3::Client::from_conf(s3_config.build())
-		};
+		let client = S3Client::new(
+			bucket,
+			endpoint,
+			region,
+			access_key_id,
+			secret_access_key,
+			cache_limit_bytes,
+		)
+		.await;
 
 		let source = Arc::new(Self {
 			name: name.clone(),
-			bucket: bucket.into(),
+			client,
 			prefix: prefix.map(|x| x.into()),
-			client: Arc::new(client),
 			pattern,
 			encryption_key,
 			index: OnceLock::new(),
@@ -78,9 +68,10 @@ impl S3DataSource {
 
 		loop {
 			let mut req = source
+				.client
 				.client
 				.list_objects_v2()
-				.bucket(source.bucket.as_str());
+				.bucket(source.client.bucket());
 
 			if let Some(prefix) = &source.prefix {
 				req = req.prefix(prefix.as_str());
@@ -191,7 +182,11 @@ impl DataSource for Arc<S3DataSource> {
 		let mut continuation_token: Option<String> = None;
 
 		loop {
-			let mut req = self.client.list_objects_v2().bucket(self.bucket.as_str());
+			let mut req = self
+				.client
+				.client
+				.list_objects_v2()
+				.bucket(self.client.bucket());
 
 			if let Some(prefix) = &self.prefix {
 				req = req.prefix(prefix.as_str());

crates/pile-value/src/value/item.rs

@@ -1,6 +1,6 @@
 use mime::Mime;
 use pile_config::Label;
-use pile_io::{ChaChaReaderAsync, S3Reader, SyncReadBridge};
+use pile_io::{SyncReadBridge, chacha::ChaChaReaderv1Async};
 use smartstring::{LazyCompact, SmartString};
 use std::{collections::HashMap, fs::File, path::PathBuf, sync::Arc};
 
@@ -59,39 +59,13 @@ impl Item {
 					}
 				};
 
-				let head = source
-					.client
-					.head_object()
-					.bucket(source.bucket.as_str())
-					.key(full_key.as_str())
-					.send()
-					.await
-					.map_err(std::io::Error::other)?;
-
-				let size = head.content_length().unwrap_or(0) as u64;
+				let reader = source.client.get(&full_key).await?;
 
 				match source.encryption_key {
-					None => ItemReader::S3(S3Reader {
-						client: source.client.clone(),
-						bucket: source.bucket.clone(),
-						key: full_key,
-						cursor: 0,
-						size,
-					}),
-
-					Some(enc_key) => ItemReader::EncryptedS3(
-						ChaChaReaderAsync::new(
-							S3Reader {
-								client: source.client.clone(),
-								bucket: source.bucket.clone(),
-								key: full_key,
-								cursor: 0,
-								size,
-							},
-							enc_key,
-						)
-						.await?,
-					),
+					None => ItemReader::S3(reader),
+					Some(enc_key) => {
+						ItemReader::EncryptedS3(ChaChaReaderv1Async::new(reader, enc_key).await?)
+					}
 				}
 			}
 		})

crates/pile-value/src/value/readers.rs

@@ -1,4 +1,4 @@
-use pile_io::{AsyncReader, AsyncSeekReader, ChaChaReaderAsync, S3Reader};
+use pile_io::{AsyncReader, AsyncSeekReader, S3Reader, chacha::ChaChaReaderv1Async};
 use std::{fs::File, io::Seek};
 
 //
@@ -8,7 +8,7 @@ use std::{fs::File, io::Seek};
 pub enum ItemReader {
 	File(File),
 	S3(S3Reader),
-	EncryptedS3(ChaChaReaderAsync<S3Reader>),
+	EncryptedS3(ChaChaReaderv1Async<S3Reader>),
 }
 
 impl AsyncReader for ItemReader {