Mark/pile
1
0
Fork 0
Code Pull Requests Actions Packages Releases Activity

Reorganize S3 clients

Browse Source
This commit is contained in:
rm-dr
2026-03-23 21:09:22 -07:00
parent 5da81679be
commit 76d38d48c5
16 changed files with 310 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
crates/pile-dataset/src/dataset.rs
View File

@@ -81,6 +81,7 @@ pub struct Datasets {
pub config: ConfigToml,
pub sources: HashMap<Label, Dataset>,
pub disabled_sources: HashMap<Label, Dataset>,
}
impl Datasets {
@@ -114,6 +115,8 @@ impl Datasets {
};
let mut sources = HashMap::new();
let mut disabled_sources = HashMap::new();
for (label, source) in &config.dataset.source {
match source {
Source::Filesystem {
@@ -121,11 +124,12 @@ impl Datasets {
path,
pattern,
} => {
if !enabled {
continue;
}
let target = match enabled {
true => &mut sources,
false => &mut disabled_sources,
};
sources.insert(
target.insert(
label.clone(),
Dataset::Dir(
DirDataSource::new(label, path_parent.join(path), pattern.clone())
@@ -144,26 +148,29 @@ impl Datasets {
pattern,
encryption_key,
} => {
if !enabled {
continue;
}
let target = match enabled {
true => &mut sources,
false => &mut disabled_sources,
};
let encryption_key = encryption_key.as_ref().map(|x| string_to_key(x));
match S3DataSource::new(
label,
bucket.clone(),
prefix.clone(),
endpoint.clone(),
region.clone(),
credentials,
bucket,
prefix.as_ref().map(|x| x.as_str()),
endpoint.as_ref().map(|x| x.as_str()),
region,
&credentials.access_key_id,
&credentials.secret_access_key,
10_000_000,
pattern.clone(),
encryption_key,
)
.await
{
Ok(ds) => {
sources.insert(label.clone(), Dataset::S3(ds));
target.insert(label.clone(), Dataset::S3(ds));
}
Err(err) => {
warn!("Could not open S3 source {label}: {err}");
@@ -179,6 +186,7 @@ impl Datasets {
path_parent,
config,
sources,
disabled_sources,
});
}
@@ -219,6 +227,7 @@ impl Datasets {
.join(config.dataset.name.as_str());
let mut sources = HashMap::new();
let mut disabled_sources = HashMap::new();
for (label, source) in &config.dataset.source {
match source {
Source::Filesystem {
@@ -226,11 +235,12 @@ impl Datasets {
path,
pattern,
} => {
if !enabled {
continue;
}
let target = match enabled {
true => &mut sources,
false => &mut disabled_sources,
};
sources.insert(
target.insert(
label.clone(),
Dataset::Dir(
DirDataSource::new(label, path_parent.join(path), pattern.clone())
@@ -249,26 +259,29 @@ impl Datasets {
pattern,
encryption_key,
} => {
if !enabled {
continue;
}
let target = match enabled {
true => &mut sources,
false => &mut disabled_sources,
};
let encryption_key = encryption_key.as_ref().map(|x| string_to_key(x));
match S3DataSource::new(
label,
bucket.clone(),
prefix.clone(),
endpoint.clone(),
region.clone(),
credentials,
bucket,
prefix.as_ref().map(|x| x.as_str()),
endpoint.as_ref().map(|x| x.as_str()),
region,
&credentials.access_key_id,
&credentials.secret_access_key,
10_000_000,
pattern.clone(),
encryption_key,
)
.await
{
Ok(ds) => {
sources.insert(label.clone(), Dataset::S3(ds));
target.insert(label.clone(), Dataset::S3(ds));
}
Err(err) => {
warn!("Could not open S3 source {label}: {err}");
@@ -284,6 +297,7 @@ impl Datasets {
path_parent,
config,
sources,
disabled_sources,
});
}