Reorganize S3 clients

crates/pile-value/src/source/s3.rs

@@ -1,9 +1,9 @@
-use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
 use chrono::{DateTime, Utc};
 use pile_config::{
-	Label, S3Credentials,
+	Label,
 	pattern::{GroupPattern, GroupSegment},
 };
+use pile_io::S3Client;
 use smartstring::{LazyCompact, SmartString};
 use std::{
 	collections::{HashMap, HashSet},
@@ -19,9 +19,9 @@ use crate::{
 #[derive(Debug)]
 pub struct S3DataSource {
 	pub name: Label,
-	pub bucket: SmartString<LazyCompact>,
+	pub client: Arc<S3Client>,
+
 	pub prefix: Option<SmartString<LazyCompact>>,
-	pub client: Arc<aws_sdk_s3::Client>,
 	pub pattern: GroupPattern,
 	pub encryption_key: Option<[u8; 32]>,
 	pub index: OnceLock<HashMap<SmartString<LazyCompact>, Item>>,
@@ -30,40 +30,30 @@ pub struct S3DataSource {
 impl S3DataSource {
 	pub async fn new(
 		name: &Label,
-		bucket: String,
-		prefix: Option<String>,
-		endpoint: Option<String>,
-		region: String,
-		credentials: &S3Credentials,
+		bucket: &str,
+		prefix: Option<&str>,
+		endpoint: Option<&str>,
+		region: &str,
+		access_key_id: &str,
+		secret_access_key: &str,
+		cache_limit_bytes: usize,
 		pattern: GroupPattern,
 		encryption_key: Option<[u8; 32]>,
 	) -> Result<Arc<Self>, std::io::Error> {
-		let client = {
-			let creds = Credentials::new(
-				&credentials.access_key_id,
-				&credentials.secret_access_key,
-				None,
-				None,
-				"pile",
-			);
-
-			let mut s3_config = aws_sdk_s3::config::Builder::new()
-				.behavior_version(BehaviorVersion::latest())
-				.region(Region::new(region))
-				.credentials_provider(creds);
-
-			if let Some(ep) = endpoint {
-				s3_config = s3_config.endpoint_url(ep).force_path_style(true);
-			}
-
-			aws_sdk_s3::Client::from_conf(s3_config.build())
-		};
+		let client = S3Client::new(
+			bucket,
+			endpoint,
+			region,
+			access_key_id,
+			secret_access_key,
+			cache_limit_bytes,
+		)
+		.await;
 
 		let source = Arc::new(Self {
 			name: name.clone(),
-			bucket: bucket.into(),
+			client,
 			prefix: prefix.map(|x| x.into()),
-			client: Arc::new(client),
 			pattern,
 			encryption_key,
 			index: OnceLock::new(),
@@ -78,9 +68,10 @@ impl S3DataSource {
 
 		loop {
 			let mut req = source
+				.client
 				.client
 				.list_objects_v2()
-				.bucket(source.bucket.as_str());
+				.bucket(source.client.bucket());
 
 			if let Some(prefix) = &source.prefix {
 				req = req.prefix(prefix.as_str());
@@ -191,7 +182,11 @@ impl DataSource for Arc<S3DataSource> {
 		let mut continuation_token: Option<String> = None;
 
 		loop {
-			let mut req = self.client.list_objects_v2().bucket(self.bucket.as_str());
+			let mut req = self
+				.client
+				.client
+				.list_objects_v2()
+				.bucket(self.client.bucket());
 
 			if let Some(prefix) = &self.prefix {
 				req = req.prefix(prefix.as_str());