Remove S3 + encryption

crates/pile-dataset/src/dataset.rs

@@ -5,7 +5,7 @@ use pile_config::{
 use pile_toolbox::cancelabletask::{CancelFlag, CancelableTaskError};
 use pile_value::{
 	extract::traits::ExtractState,
-	source::{DataSource, DirDataSource, S3DataSource, misc::path_ts_earliest, string_to_key},
+	source::{DataSource, DirDataSource, misc::path_ts_earliest},
 	value::{Item, PileValue},
 };
 use serde_json::Value;
@@ -33,31 +33,27 @@ pub enum DatasetError {
 // MARK: Dataset enum
 //
 
-/// An opened data source — either a local filesystem directory or an S3 bucket.
+/// An opened data source
 pub enum Dataset {
 	Dir(Arc<DirDataSource>),
-	S3(Arc<S3DataSource>),
 }
 
 impl Dataset {
 	pub fn len(&self) -> usize {
 		match self {
 			Self::Dir(ds) => ds.len(),
-			Self::S3(ds) => ds.len(),
 		}
 	}
 
 	pub async fn get(&self, key: &str) -> Option<Item> {
 		match self {
 			Self::Dir(ds) => ds.get(key).await.ok().flatten(),
-			Self::S3(ds) => ds.get(key).await.ok().flatten(),
 		}
 	}
 
 	pub fn iter(&self) -> Box<dyn Iterator<Item = &Item> + Send + '_> {
 		match self {
 			Self::Dir(ds) => Box::new(ds.iter()),
-			Self::S3(ds) => Box::new(ds.iter()),
 		}
 	}
 
@@ -68,14 +64,12 @@ impl Dataset {
 	) -> Box<dyn Iterator<Item = &Item> + Send + '_> {
 		match self {
 			Self::Dir(ds) => Box::new(ds.iter_page(offset, limit)),
-			Self::S3(ds) => Box::new(ds.iter_page(offset, limit)),
 		}
 	}
 
 	pub async fn latest_change(&self) -> Result<Option<DateTime<Utc>>, std::io::Error> {
 		match self {
 			Self::Dir(ds) => ds.latest_change().await,
-			Self::S3(ds) => ds.latest_change().await,
 		}
 	}
 }
@@ -148,46 +142,6 @@ impl Datasets {
 						),
 					);
 				}
-
-				Source::S3 {
-					enabled,
-					bucket,
-					prefix,
-					endpoint,
-					region,
-					credentials,
-					pattern,
-					encryption_key,
-				} => {
-					let target = match enabled {
-						true => &mut sources,
-						false => &mut disabled_sources,
-					};
-
-					let encryption_key = encryption_key.as_ref().map(|x| string_to_key(x));
-
-					match S3DataSource::new(
-						label,
-						bucket,
-						prefix.as_ref().map(|x| x.as_str()),
-						endpoint.as_ref().map(|x| x.as_str()),
-						region,
-						&credentials.access_key_id,
-						&credentials.secret_access_key,
-						10_000_000,
-						pattern.clone(),
-						encryption_key,
-					)
-					.await
-					{
-						Ok(ds) => {
-							target.insert(label.clone(), Dataset::S3(ds));
-						}
-						Err(err) => {
-							warn!("Could not open S3 source {label}: {err}");
-						}
-					}
-				}
 			}
 		}
 
@@ -259,46 +213,6 @@ impl Datasets {
 						),
 					);
 				}
-
-				Source::S3 {
-					enabled,
-					bucket,
-					prefix,
-					endpoint,
-					region,
-					credentials,
-					pattern,
-					encryption_key,
-				} => {
-					let target = match enabled {
-						true => &mut sources,
-						false => &mut disabled_sources,
-					};
-
-					let encryption_key = encryption_key.as_ref().map(|x| string_to_key(x));
-
-					match S3DataSource::new(
-						label,
-						bucket,
-						prefix.as_ref().map(|x| x.as_str()),
-						endpoint.as_ref().map(|x| x.as_str()),
-						region,
-						&credentials.access_key_id,
-						&credentials.secret_access_key,
-						10_000_000,
-						pattern.clone(),
-						encryption_key,
-					)
-					.await
-					{
-						Ok(ds) => {
-							target.insert(label.clone(), Dataset::S3(ds));
-						}
-						Err(err) => {
-							warn!("Could not open S3 source {label}: {err}");
-						}
-					}
-				}
 			}
 		}