Mark/pile
1
0
Fork 0
Code Actions Releases Activity

Extractor rewrite

Browse Source
This commit is contained in:
rm-dr
2026-03-11 10:12:36 -07:00
parent b789255ea9
commit 4868a96b1e
51 changed files with 661 additions and 687 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
crates/pile-config/src/lib.rs
View File

@@ -1,9 +1,6 @@
use serde::Deserialize;
use std::{collections::HashMap, fmt::Debug, path::PathBuf};
mod post;
pub use post::*;
mod misc;
pub use misc::*;
@@ -40,10 +37,6 @@ pub struct DatasetConfig {
/// Where to find this field
pub source: HashMap<Label, Source>,
/// How to post-process this field
#[serde(default)]
pub post: Vec<FieldSpecPost>,
}
#[derive(Debug, Clone, Deserialize)]
@@ -100,10 +93,6 @@ pub struct FieldSpec {
/// How to find this field in a data entry
pub path: Vec<ObjectPath>,
/// How to post-process this field
#[serde(default)]
pub post: Vec<FieldSpecPost>,
}
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]

18
crates/pile-config/src/post.rs
View File

@@ -1,18 +0,0 @@
use serde::Deserialize;
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
#[serde(untagged)]
pub enum FieldSpecPost {
TrimSuffix { trim_suffix: String },
TrimPrefix { trim_prefix: String },
SetCase { case: Case },
Join { join: String },
NotEmpty { notempty: bool },
}
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum Case {
Lower,
Upper,
}

22
crates/pile-dataset/Cargo.toml
View File

@@ -10,37 +10,23 @@ workspace = true
[dependencies]
pile-config = { workspace = true }
pile-toolbox = { workspace = true }
pile-flac = { workspace = true }
pile-value = { workspace = true }
serde_json = { workspace = true }
itertools = { workspace = true }
walkdir = { workspace = true }
tantivy = { workspace = true }
tracing = { workspace = true }
chrono = { workspace = true }
toml = { workspace = true }
thiserror = { workspace = true }
smartstring = { workspace = true }
blake3 = { workspace = true }
epub = { workspace = true }
kamadak-exif = { workspace = true }
pdf = { workspace = true }
pdfium-render = { workspace = true, optional = true }
image = { workspace = true, optional = true }
id3 = { workspace = true }
tokio = { workspace = true }
tokio-stream = { workspace = true }
async-trait = { workspace = true }
aws-sdk-s3 = { workspace = true }
mime = { workspace = true }
mime_guess = { workspace = true }
serde = { workspace = true }
serde = { workspace = true, optional = true }
axum = { workspace = true, optional = true }
utoipa = { workspace = true, optional = true }
utoipa-swagger-ui = { workspace = true, optional = true }
[features]
default = []
pdfium = ["dep:pdfium-render", "dep:image"]
axum = ["dep:axum", "dep:utoipa", "dep:utoipa-swagger-ui"]
pdfium = ["pile-value/pdfium"]
axum = ["dep:axum", "dep:utoipa", "dep:utoipa-swagger-ui", "dep:serde"]

19
crates/pile-dataset/src/dataset.rs
View File

@@ -1,6 +1,10 @@
use chrono::{DateTime, Utc};
use pile_config::{ConfigToml, Label, Source, objectpath::ObjectPath};
use pile_toolbox::cancelabletask::{CancelFlag, CancelableTaskError};
use pile_value::{
source::{DataSource, DirDataSource, S3DataSource, misc::path_ts_earliest},
value::{Item, PileValue},
};
use serde_json::Value;
use std::{collections::HashMap, io::ErrorKind, path::PathBuf, sync::Arc, time::Instant};
use tantivy::{Executor, Index, IndexWriter, TantivyError, collector::TopDocs};
@@ -9,13 +13,7 @@ use tokio::task::JoinSet;
use tokio_stream::{StreamExt, wrappers::ReceiverStream};
use tracing::{debug, info, trace, warn};
use crate::{
DataSource, Item, PileValue,
extract::MetaExtractor,
index::{DbFtsIndex, FtsLookupResult},
path_ts_earliest,
source::{DirDataSource, S3DataSource},
};
use crate::index::{DbFtsIndex, FtsLookupResult};
#[derive(Debug, Error)]
pub enum DatasetError {
@@ -183,11 +181,12 @@ impl Datasets {
let Some(item) = self.get(source, key).await else {
return Ok(None);
};
let extractor = MetaExtractor::new(&item);
let root = PileValue::ObjectExtractor(Arc::new(extractor));
let Some(value) = root.query(path).await? else {
let item = PileValue::Item(item);
let Some(value) = item.query(path).await? else {
return Ok(None);
};
Ok(Some(value.to_json().await?))
}

165
crates/pile-dataset/src/extract/mod.rs
View File

@@ -1,165 +0,0 @@
use pile_config::Label;
use std::{collections::HashMap, sync::Arc};
mod flac;
pub use flac::*;
mod id3;
pub use id3::*;
mod fs;
pub use fs::*;
mod epub;
pub use epub::*;
mod exif;
pub use exif::*;
mod pdf;
pub use pdf::*;
mod toml;
pub use toml::*;
mod map;
pub use map::*;
mod sidecar;
pub use sidecar::*;
use crate::{Item, PileValue};
/// An attachment that extracts metadata from an [Item].
///
/// Metadata is exposed as an immutable map of {label: value},
/// much like a json object.
#[async_trait::async_trait]
pub trait ObjectExtractor: Send + Sync {
/// Get the field at `name` from `item`.
/// - returns `None` if `name` is not a valid field
/// - returns `Some(Null)` if `name` is not available
async fn field(&self, name: &pile_config::Label) -> Result<Option<PileValue>, std::io::Error>;
/// Return all fields in this extractor.
/// `Self::field` must return [Some] for all these keys
/// and [None] for all others.
async fn fields(&self) -> Result<Vec<Label>, std::io::Error>;
/// Convert this to a JSON value.
async fn to_json(&self) -> Result<serde_json::Value, std::io::Error> {
let keys = self.fields().await?;
let mut map = serde_json::Map::new();
for k in &keys {
let v = match self.field(k).await? {
Some(x) => x,
None => continue,
};
map.insert(k.to_string(), Box::pin(v.to_json()).await?);
}
Ok(serde_json::Value::Object(map))
}
}
/// An attachment that extracts metadata from an [Item].
///
/// Metadata is exposed as an immutable list of values.
#[async_trait::async_trait]
pub trait ListExtractor: Send + Sync {
/// Get the item at index `idx`.
/// Indices start at zero, and must be consecutive.
/// - returns `None` if `idx` is out of range
/// - returns `Some(Null)` if `None` is at `idx`
async fn get(&self, idx: usize) -> Result<Option<PileValue>, std::io::Error>;
async fn len(&self) -> Result<usize, std::io::Error>;
async fn is_empty(&self) -> Result<bool, std::io::Error> {
Ok(self.len().await? == 0)
}
/// Convert this list to a JSON value.
async fn to_json(&self) -> Result<serde_json::Value, std::io::Error> {
let len = self.len().await?;
let mut list = Vec::with_capacity(len);
for i in 0..len {
#[expect(clippy::expect_used)]
let v = self
.get(i)
.await?
.expect("value must be present according to length");
list.push(Box::pin(v.to_json()).await?);
}
Ok(serde_json::Value::Array(list))
}
}
pub struct MetaExtractor {
inner: MapExtractor,
}
impl MetaExtractor {
#[expect(clippy::unwrap_used)]
pub fn new(item: &Item) -> Self {
let inner = MapExtractor {
inner: HashMap::from([
(
Label::new("flac").unwrap(),
crate::PileValue::ObjectExtractor(Arc::new(FlacExtractor::new(item))),
),
(
Label::new("id3").unwrap(),
crate::PileValue::ObjectExtractor(Arc::new(Id3Extractor::new(item))),
),
(
Label::new("fs").unwrap(),
crate::PileValue::ObjectExtractor(Arc::new(FsExtractor::new(item))),
),
(
Label::new("epub").unwrap(),
crate::PileValue::ObjectExtractor(Arc::new(EpubExtractor::new(item))),
),
(
Label::new("exif").unwrap(),
crate::PileValue::ObjectExtractor(Arc::new(ExifExtractor::new(item))),
),
(
Label::new("pdf").unwrap(),
crate::PileValue::ObjectExtractor(Arc::new(PdfExtractor::new(item))),
),
(
Label::new("toml").unwrap(),
crate::PileValue::ObjectExtractor(Arc::new(TomlExtractor::new(item))),
),
(
Label::new("sidecar").unwrap(),
crate::PileValue::ObjectExtractor(Arc::new(SidecarExtractor::new(item))),
),
]),
};
Self { inner }
}
}
#[async_trait::async_trait]
impl ObjectExtractor for MetaExtractor {
async fn field(&self, name: &pile_config::Label) -> Result<Option<PileValue>, std::io::Error> {
self.inner.field(name).await
}
#[expect(clippy::unwrap_used)]
async fn fields(&self) -> Result<Vec<Label>, std::io::Error> {
return Ok(vec![
Label::new("flac").unwrap(),
Label::new("id3").unwrap(),
Label::new("fs").unwrap(),
Label::new("epub").unwrap(),
Label::new("exif").unwrap(),
Label::new("pdf").unwrap(),
Label::new("sidecar").unwrap(),
]);
}
}

95
crates/pile-dataset/src/extract/pdf/pdf_cover.rs
View File

@@ -1,95 +0,0 @@
use image::ImageFormat;
use pdfium_render::prelude::*;
use pile_config::Label;
use std::{
collections::HashMap,
io::{BufReader, Cursor},
sync::{Arc, OnceLock},
};
use tracing::trace;
use crate::{Item, PileValue, SyncReadBridge, extract::ObjectExtractor};
pub struct PdfCoverExtractor {
item: Item,
output: OnceLock<HashMap<Label, PileValue>>,
}
impl PdfCoverExtractor {
pub fn new(item: &Item) -> Self {
Self {
item: item.clone(),
output: OnceLock::new(),
}
}
async fn get_inner(&self) -> Result<&HashMap<Label, PileValue>, std::io::Error> {
if let Some(x) = self.output.get() {
return Ok(x);
}
let reader = SyncReadBridge::new_current(self.item.read().await?);
let cover = tokio::task::spawn_blocking(move || {
let mut bytes = Vec::new();
std::io::Read::read_to_end(&mut BufReader::new(reader), &mut bytes)?;
let pdfium = Pdfium::default();
let document = pdfium
.load_pdf_from_byte_slice(&bytes, None)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?;
let render_config = PdfRenderConfig::new().set_target_width(1024);
let page = document
.pages()
.get(0)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?;
let image = page
.render_with_config(&render_config)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?
.as_image();
let mut png_bytes = Vec::new();
image
.write_to(&mut Cursor::new(&mut png_bytes), ImageFormat::Png)
.map_err(|e| std::io::Error::other(e.to_string()))?;
Ok::<_, std::io::Error>(png_bytes)
})
.await
.map_err(std::io::Error::other)?;
let output = match cover {
Ok(data) => {
#[expect(clippy::unwrap_used)]
let label = Label::new("cover").unwrap();
HashMap::from([(
label,
PileValue::Blob {
mime: mime::IMAGE_PNG,
bytes: Arc::new(data),
},
)])
}
Err(error) => {
trace!(message = "Could not render pdf cover", ?error, key = ?self.item.key());
HashMap::new()
}
};
return Ok(self.output.get_or_init(|| output));
}
}
#[async_trait::async_trait]
impl ObjectExtractor for PdfCoverExtractor {
async fn field(&self, name: &Label) -> Result<Option<PileValue>, std::io::Error> {
Ok(self.get_inner().await?.get(name).cloned())
}
async fn fields(&self) -> Result<Vec<Label>, std::io::Error> {
Ok(self.get_inner().await?.keys().cloned().collect())
}
}

132
crates/pile-dataset/src/index/index_fts.rs
View File

@@ -1,9 +1,6 @@
use itertools::Itertools;
use pile_config::{Case, ConfigToml, DatasetFts, FieldSpecPost, Label};
use std::{
path::PathBuf,
sync::{Arc, LazyLock},
};
use pile_config::{ConfigToml, DatasetFts, Label};
use pile_value::value::{Item, PileValue};
use std::{path::PathBuf, sync::LazyLock};
use tantivy::{
DocAddress, Index, ReloadPolicy, TantivyDocument, TantivyError,
collector::Collector,
@@ -12,8 +9,6 @@ use tantivy::{
};
use tracing::{debug, trace, warn};
use crate::{Item, PileValue, extract::MetaExtractor};
#[derive(Debug, Clone)]
pub struct FtsLookupResult {
pub score: f32,
@@ -76,11 +71,11 @@ impl DbFtsIndex {
doc.add_text(self.schema.get_field("_meta_source")?, item.source_name());
doc.add_text(self.schema.get_field("_meta_key")?, key);
let extractor = PileValue::ObjectExtractor(Arc::new(MetaExtractor::new(item)));
let item = PileValue::Item(item.clone());
let mut empty = true;
for name in self.fts_cfg().fields.keys() {
let x = self.get_field(&extractor, name).await?;
let x = self.get_field(&item, name).await?;
let val = match x {
Some(x) => x,
@@ -135,13 +130,6 @@ impl DbFtsIndex {
x => x.clone(),
};
for post in &field.post {
val = match apply(post, &val) {
Some(x) => x,
None => return Ok(None),
};
}
loop {
val = match val {
PileValue::String(x) => return Ok(Some(x.to_string())),
@@ -186,6 +174,15 @@ impl DbFtsIndex {
continue 'outer;
}
PileValue::Item(_) => {
trace!(
message = "Skipping field, is item",
field = field_name.to_string(),
?path,
);
continue 'outer;
}
PileValue::ListExtractor(_) => {
trace!(
message = "Skipping field, is ListExtractor",
@@ -296,104 +293,3 @@ impl DbFtsIndex {
return Ok(out);
}
}
pub fn apply(post: &FieldSpecPost, val: &PileValue) -> Option<PileValue> {
Some(match post {
FieldSpecPost::NotEmpty { notempty: false } => val.clone(),
FieldSpecPost::NotEmpty { notempty: true } => match val {
PileValue::Null => return None,
PileValue::String(x) if x.is_empty() => return None,
PileValue::Array(x) if x.is_empty() => return None,
x => x.clone(),
},
FieldSpecPost::SetCase { case: Case::Lower } => match val {
PileValue::Null => return None,
PileValue::U64(_) => return None,
PileValue::I64(_) => return None,
PileValue::Blob { .. } => return None,
PileValue::ObjectExtractor(_) => return None,
PileValue::ListExtractor(_) => return None,
PileValue::String(x) => PileValue::String(Arc::new(x.as_str().to_lowercase().into())),
PileValue::Array(x) => PileValue::Array(Arc::new(
x.iter().map(|x| apply(post, x)).collect::<Option<_>>()?,
)),
},
FieldSpecPost::SetCase { case: Case::Upper } => match val {
PileValue::Null => return None,
PileValue::U64(_) => return None,
PileValue::I64(_) => return None,
PileValue::Blob { .. } => return None,
PileValue::ObjectExtractor(_) => return None,
PileValue::ListExtractor(_) => return None,
PileValue::String(x) => PileValue::String(Arc::new(x.as_str().to_uppercase().into())),
PileValue::Array(x) => PileValue::Array(Arc::new(
x.iter()
.map(|x| apply(post, x))
.collect::<Option<Vec<_>>>()?,
)),
},
FieldSpecPost::TrimSuffix { trim_suffix } => match val {
PileValue::Null => return None,
PileValue::U64(_) => return None,
PileValue::I64(_) => return None,
PileValue::Blob { .. } => return None,
PileValue::ObjectExtractor(_) => return None,
PileValue::ListExtractor(_) => return None,
PileValue::String(x) => PileValue::String(Arc::new(
x.strip_suffix(trim_suffix).unwrap_or(x.as_str()).into(),
)),
PileValue::Array(x) => PileValue::Array(Arc::new(
x.iter()
.map(|x| apply(post, x))
.collect::<Option<Vec<_>>>()?,
)),
},
FieldSpecPost::TrimPrefix { trim_prefix } => match val {
PileValue::Null => return None,
PileValue::U64(_) => return None,
PileValue::I64(_) => return None,
PileValue::Blob { .. } => return None,
PileValue::ObjectExtractor(_) => return None,
PileValue::ListExtractor(_) => return None,
PileValue::String(x) => PileValue::String(Arc::new(
x.strip_prefix(trim_prefix).unwrap_or(x.as_str()).into(),
)),
PileValue::Array(x) => PileValue::Array(Arc::new(
x.iter()
.map(|x| apply(post, x))
.collect::<Option<Vec<_>>>()?,
)),
},
FieldSpecPost::Join { join } => match val {
PileValue::Null => return None,
PileValue::U64(_) => return None,
PileValue::I64(_) => return None,
PileValue::Blob { .. } => return None,
PileValue::ObjectExtractor(_) => return None,
PileValue::ListExtractor(_) => return None,
PileValue::String(x) => PileValue::String(x.clone()),
PileValue::Array(x) => PileValue::String(Arc::new(
x.iter()
.map(|x| apply(post, x))
.map(|x| x.and_then(|x| x.as_str().map(|x| x.to_owned())))
.collect::<Option<Vec<_>>>()?
.into_iter()
.join(join)
.into(),
)),
},
})
}

14
crates/pile-dataset/src/lib.rs
View File

@@ -1,21 +1,7 @@
mod traits;
pub use traits::*;
mod misc;
pub use misc::*;
mod dataset;
pub use dataset::{Dataset, DatasetError, Datasets};
mod item;
pub use item::*;
mod value;
pub use value::*;
pub mod extract;
pub mod index;
pub mod source;
#[cfg(feature = "axum")]
pub mod serve;

9
crates/pile-dataset/src/serve/field.rs
View File

@@ -5,12 +5,13 @@ use axum::{
response::{IntoResponse, Response},
};
use pile_config::{Label, objectpath::ObjectPath};
use pile_value::value::PileValue;
use serde::Deserialize;
use std::{sync::Arc, time::Instant};
use tracing::debug;
use utoipa::ToSchema;
use crate::{Datasets, PileValue, extract::MetaExtractor};
use crate::Datasets;
#[derive(Deserialize, ToSchema)]
pub struct FieldQuery {
@@ -61,10 +62,8 @@ pub async fn get_field(
return StatusCode::NOT_FOUND.into_response();
};
let extractor = MetaExtractor::new(&item);
let root: PileValue = PileValue::ObjectExtractor(Arc::new(extractor));
let value = match root.query(&path).await {
let item = PileValue::Item(item);
let value = match item.query(&path).await {
Ok(Some(v)) => v,
Ok(None) => return StatusCode::NOT_FOUND.into_response(),
Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, format!("{e:?}")).into_response(),

3
crates/pile-dataset/src/serve/item.rs
View File

@@ -4,12 +4,13 @@ use axum::{
response::{IntoResponse, Response},
};
use pile_config::Label;
use pile_value::value::AsyncReader;
use serde::Deserialize;
use std::{sync::Arc, time::Instant};
use tracing::debug;
use utoipa::ToSchema;
use crate::{AsyncReader, Datasets};
use crate::Datasets;
#[derive(Deserialize, ToSchema)]
pub struct ItemQuery {

5
crates/pile-dataset/src/source/mod.rs
View File

@@ -1,5 +0,0 @@
mod dir;
pub use dir::*;
mod s3;
pub use s3::*;

36
crates/pile-extractor/Cargo.toml Normal file
View File

@@ -0,0 +1,36 @@
[package]
name = "pile-value"
version = { workspace = true }
rust-version = { workspace = true }
edition = { workspace = true }
[lints]
workspace = true
[dependencies]
pile-config = { workspace = true }
pile-flac = { workspace = true }
serde_json = { workspace = true }
walkdir = { workspace = true }
tracing = { workspace = true }
chrono = { workspace = true }
toml = { workspace = true }
smartstring = { workspace = true }
blake3 = { workspace = true }
epub = { workspace = true }
kamadak-exif = { workspace = true }
pdf = { workspace = true }
pdfium-render = { workspace = true, optional = true }
image = { workspace = true, optional = true }
id3 = { workspace = true }
tokio = { workspace = true }
tokio-stream = { workspace = true }
async-trait = { workspace = true }
aws-sdk-s3 = { workspace = true }
mime = { workspace = true }
mime_guess = { workspace = true }
[features]
default = []
pdfium = ["dep:pdfium-render", "dep:image"]

0
crates/pile-dataset/build.rs → crates/pile-extractor/build.rs
View File

5
crates/pile-dataset/src/extract/epub/epub_meta.rs → crates/pile-extractor/src/extract/item/epub/epub_meta.rs
View File

@@ -6,7 +6,10 @@ use std::{
};
use tracing::trace;
use crate::{Item, PileValue, SyncReadBridge, extract::ObjectExtractor};
use crate::{
extract::traits::ObjectExtractor,
value::{Item, PileValue, SyncReadBridge},
};
pub struct EpubMetaExtractor {
item: Item,

2
crates/pile-dataset/src/extract/epub/epub_text.rs → crates/pile-extractor/src/extract/item/epub/epub_text.rs
View File

@@ -6,7 +6,7 @@ use std::{
};
use tracing::debug;
use crate::{Item, PileValue, SyncReadBridge, extract::ObjectExtractor};
use crate::{value::{Item, PileValue, SyncReadBridge}, extract::traits::ObjectExtractor};
pub struct EpubTextExtractor {
item: Item,

5
crates/pile-dataset/src/extract/epub/mod.rs → crates/pile-extractor/src/extract/item/epub/mod.rs
View File

@@ -7,7 +7,10 @@ pub use epub_meta::*;
mod epub_text;
pub use epub_text::*;
use crate::{Item, PileValue, extract::ObjectExtractor};
use crate::{
extract::traits::ObjectExtractor,
value::{Item, PileValue},
};
pub struct EpubExtractor {
text: Arc<EpubTextExtractor>,

2
crates/pile-dataset/src/extract/exif.rs → crates/pile-extractor/src/extract/item/exif.rs
View File

@@ -6,7 +6,7 @@ use std::{
};
use tracing::trace;
use crate::{Item, PileValue, SyncReadBridge, extract::ObjectExtractor};
use crate::{value::{Item, PileValue, SyncReadBridge}, extract::traits::ObjectExtractor};
pub struct ExifExtractor {
item: Item,

4
crates/pile-dataset/src/extract/flac.rs → crates/pile-extractor/src/extract/item/flac.rs
View File

@@ -8,8 +8,8 @@ use std::{
};
use crate::{
Item, PileValue, SyncReadBridge,
extract::{ListExtractor, ObjectExtractor},
value::{Item, PileValue, SyncReadBridge},
extract::traits::{ListExtractor, ObjectExtractor},
};
pub struct FlacImagesExtractor {

2
crates/pile-dataset/src/extract/fs.rs → crates/pile-extractor/src/extract/item/fs.rs
View File

@@ -5,7 +5,7 @@ use std::{
sync::{Arc, OnceLock},
};
use crate::{Item, PileValue, extract::ObjectExtractor};
use crate::{value::{Item, PileValue}, extract::traits::ObjectExtractor};
pub struct FsExtractor {
item: Item,

5
crates/pile-dataset/src/extract/id3.rs → crates/pile-extractor/src/extract/item/id3.rs
View File

@@ -7,7 +7,10 @@ use std::{
sync::{Arc, OnceLock},
};
use crate::{Item, PileValue, SyncReadBridge, extract::ObjectExtractor};
use crate::{
extract::traits::ObjectExtractor,
value::{Item, PileValue, SyncReadBridge},
};
pub struct Id3Extractor {
item: Item,

99
crates/pile-extractor/src/extract/item/mod.rs Normal file
View File

@@ -0,0 +1,99 @@
mod flac;
use std::{collections::HashMap, sync::Arc};
pub use flac::*;
mod id3;
pub use id3::*;
mod fs;
pub use fs::*;
mod epub;
pub use epub::*;
mod exif;
pub use exif::*;
mod pdf;
pub use pdf::*;
mod toml;
use pile_config::Label;
pub use toml::*;
mod sidecar;
pub use sidecar::*;
use crate::{
extract::{misc::MapExtractor, traits::ObjectExtractor},
value::{Item, PileValue},
};
pub struct ItemExtractor {
inner: MapExtractor,
}
impl ItemExtractor {
#[expect(clippy::unwrap_used)]
pub fn new(item: &Item) -> Self {
let inner = MapExtractor {
inner: HashMap::from([
(
Label::new("flac").unwrap(),
PileValue::ObjectExtractor(Arc::new(FlacExtractor::new(item))),
),
(
Label::new("id3").unwrap(),
PileValue::ObjectExtractor(Arc::new(Id3Extractor::new(item))),
),
(
Label::new("fs").unwrap(),
PileValue::ObjectExtractor(Arc::new(FsExtractor::new(item))),
),
(
Label::new("epub").unwrap(),
PileValue::ObjectExtractor(Arc::new(EpubExtractor::new(item))),
),
(
Label::new("exif").unwrap(),
PileValue::ObjectExtractor(Arc::new(ExifExtractor::new(item))),
),
(
Label::new("pdf").unwrap(),
PileValue::ObjectExtractor(Arc::new(PdfExtractor::new(item))),
),
(
Label::new("toml").unwrap(),
PileValue::ObjectExtractor(Arc::new(TomlExtractor::new(item))),
),
(
Label::new("sidecar").unwrap(),
PileValue::ObjectExtractor(Arc::new(SidecarExtractor::new(item))),
),
]),
};
Self { inner }
}
}
#[async_trait::async_trait]
impl ObjectExtractor for ItemExtractor {
async fn field(&self, name: &pile_config::Label) -> Result<Option<PileValue>, std::io::Error> {
self.inner.field(name).await
}
#[expect(clippy::unwrap_used)]
async fn fields(&self) -> Result<Vec<Label>, std::io::Error> {
return Ok(vec![
Label::new("flac").unwrap(),
Label::new("id3").unwrap(),
Label::new("fs").unwrap(),
Label::new("epub").unwrap(),
Label::new("exif").unwrap(),
Label::new("pdf").unwrap(),
Label::new("sidecar").unwrap(),
]);
}
}

13
crates/pile-dataset/src/extract/pdf/mod.rs → crates/pile-extractor/src/extract/item/pdf/mod.rs
View File

@@ -1,11 +1,6 @@
use pile_config::Label;
use std::sync::Arc;
#[cfg(feature = "pdfium")]
mod pdf_cover;
#[cfg(feature = "pdfium")]
pub use pdf_cover::*;
#[cfg(feature = "pdfium")]
mod pdf_pages;
#[cfg(feature = "pdfium")]
@@ -17,14 +12,12 @@ pub use pdf_meta::*;
mod pdf_text;
pub use pdf_text::*;
use crate::{Item, PileValue, extract::ObjectExtractor};
use crate::{value::{Item, PileValue}, extract::traits::ObjectExtractor};
pub struct PdfExtractor {
text: Arc<PdfTextExtractor>,
meta: Arc<PdfMetaExtractor>,
#[cfg(feature = "pdfium")]
cover: Arc<PdfCoverExtractor>,
#[cfg(feature = "pdfium")]
pages: Arc<PdfPagesExtractor>,
}
@@ -34,8 +27,6 @@ impl PdfExtractor {
text: Arc::new(PdfTextExtractor::new(item)),
meta: Arc::new(PdfMetaExtractor::new(item)),
#[cfg(feature = "pdfium")]
cover: Arc::new(PdfCoverExtractor::new(item)),
#[cfg(feature = "pdfium")]
pages: Arc::new(PdfPagesExtractor::new(item)),
}
}
@@ -48,8 +39,6 @@ impl ObjectExtractor for PdfExtractor {
"text" => self.text.field(name).await,
"meta" => Ok(Some(PileValue::ObjectExtractor(self.meta.clone()))),
#[cfg(feature = "pdfium")]
"cover" => self.cover.field(name).await,
#[cfg(feature = "pdfium")]
"pages" => Ok(Some(PileValue::ListExtractor(self.pages.clone()))),
_ => Ok(None),
}

3
crates/pile-dataset/src/extract/pdf/pdf_meta.rs → crates/pile-extractor/src/extract/item/pdf/pdf_meta.rs
View File

@@ -8,8 +8,7 @@ use std::{
};
use tracing::trace;
use crate::extract::ObjectExtractor;
use crate::{Item, PileValue, SyncReadBridge};
use crate::{extract::traits::ObjectExtractor, value::{Item, PileValue, SyncReadBridge}};
pub struct PdfMetaExtractor {
item: Item,

5
crates/pile-dataset/src/extract/pdf/pdf_pages.rs → crates/pile-extractor/src/extract/item/pdf/pdf_pages.rs
View File

@@ -6,7 +6,10 @@ use std::{
};
use tracing::trace;
use crate::{Item, PileValue, SyncReadBridge, extract::ListExtractor};
use crate::{
extract::traits::ListExtractor,
value::{Item, PileValue, SyncReadBridge},
};
pub struct PdfPagesExtractor {
item: Item,

3
crates/pile-dataset/src/extract/pdf/pdf_text.rs → crates/pile-extractor/src/extract/item/pdf/pdf_text.rs
View File

@@ -8,8 +8,7 @@ use std::{
};
use tracing::trace;
use crate::extract::ObjectExtractor;
use crate::{Item, PileValue, SyncReadBridge};
use crate::{extract::traits::ObjectExtractor, value::{Item, PileValue, SyncReadBridge}};
pub struct PdfTextExtractor {
item: Item,

5
crates/pile-dataset/src/extract/sidecar.rs → crates/pile-extractor/src/extract/item/sidecar.rs
View File

@@ -2,9 +2,10 @@ use pile_config::Label;
use std::sync::OnceLock;
use crate::{
Item, PileValue,
extract::{ObjectExtractor, TomlExtractor},
value::{Item, PileValue},
extract::traits::ObjectExtractor,
};
use super::TomlExtractor;
pub struct SidecarExtractor {
item: Item,

5
crates/pile-dataset/src/extract/toml.rs → crates/pile-extractor/src/extract/item/toml.rs
View File

@@ -4,7 +4,10 @@ use std::{
sync::{Arc, OnceLock},
};
use crate::{AsyncReader, Item, PileValue, extract::ObjectExtractor};
use crate::{
extract::traits::ObjectExtractor,
value::{AsyncReader, Item, PileValue},
};
fn toml_to_pile(value: toml::Value) -> PileValue {
match value {

24
crates/pile-extractor/src/extract/misc/list.rs Normal file
View File

@@ -0,0 +1,24 @@
use std::sync::Arc;
use crate::{extract::traits::ListExtractor, value::PileValue};
pub struct ArrayExtractor {
inner: Arc<Vec<PileValue>>,
}
impl ArrayExtractor {
pub fn new(inner: Arc<Vec<PileValue>>) -> Self {
Self { inner }
}
}
#[async_trait::async_trait]
impl ListExtractor for ArrayExtractor {
async fn get(&self, idx: usize) -> Result<Option<PileValue>, std::io::Error> {
Ok(self.inner.get(idx).cloned())
}
async fn len(&self) -> Result<usize, std::io::Error> {
Ok(self.inner.len())
}
}

12
crates/pile-dataset/src/extract/map.rs → crates/pile-extractor/src/extract/misc/map.rs
View File

@@ -1,10 +1,18 @@
use pile_config::Label;
use std::collections::HashMap;
use crate::{PileValue, extract::ObjectExtractor};
use crate::{extract::traits::ObjectExtractor, value::PileValue};
pub struct MapExtractor {
pub(crate) inner: HashMap<Label, PileValue>,
pub inner: HashMap<Label, PileValue>,
}
impl Default for MapExtractor {
fn default() -> Self {
Self {
inner: HashMap::new(),
}
}
}
#[async_trait::async_trait]

8
crates/pile-extractor/src/extract/misc/mod.rs Normal file
View File

@@ -0,0 +1,8 @@
mod list;
pub use list::*;
mod vec;
pub use vec::*;
mod map;
pub use map::*;

22
crates/pile-extractor/src/extract/misc/vec.rs Normal file
View File

@@ -0,0 +1,22 @@
use crate::{extract::traits::ListExtractor, value::PileValue};
pub struct VecExtractor {
pub inner: Vec<PileValue>,
}
impl Default for VecExtractor {
fn default() -> Self {
Self { inner: Vec::new() }
}
}
#[async_trait::async_trait]
impl ListExtractor for VecExtractor {
async fn get(&self, idx: usize) -> Result<Option<PileValue>, std::io::Error> {
Ok(self.inner.get(idx).cloned())
}
async fn len(&self) -> Result<usize, std::io::Error> {
Ok(self.inner.len())
}
}

4
crates/pile-extractor/src/extract/mod.rs Normal file
View File

@@ -0,0 +1,4 @@
pub mod item;
pub mod misc;
pub mod string;
pub mod traits;

51
crates/pile-extractor/src/extract/string.rs Normal file
View File

@@ -0,0 +1,51 @@
use pile_config::Label;
use smartstring::{LazyCompact, SmartString};
use std::sync::Arc;
use crate::{extract::traits::ObjectExtractor, value::PileValue};
pub struct StringExtractor {
item: Arc<SmartString<LazyCompact>>,
}
impl StringExtractor {
pub fn new(item: &Arc<SmartString<LazyCompact>>) -> Self {
Self { item: item.clone() }
}
}
#[async_trait::async_trait]
impl ObjectExtractor for StringExtractor {
async fn field(&self, name: &Label) -> Result<Option<PileValue>, std::io::Error> {
Ok(match name.as_str() {
"trim" => Some(PileValue::String(Arc::new(
self.item.as_str().trim().into(),
))),
"upper" => Some(PileValue::String(Arc::new(
self.item.as_str().to_lowercase().into(),
))),
"lower" => Some(PileValue::String(Arc::new(
self.item.as_str().to_uppercase().into(),
))),
"nonempty" => Some(match self.item.is_empty() {
true => PileValue::Null,
false => PileValue::String(self.item.clone()),
}),
_ => None,
})
}
#[expect(clippy::unwrap_used)]
async fn fields(&self) -> Result<Vec<Label>, std::io::Error> {
return Ok(vec![
Label::new("trim").unwrap(),
Label::new("upper").unwrap(),
Label::new("lower").unwrap(),
Label::new("nonempty").unwrap(),
]);
}
}

68
crates/pile-extractor/src/extract/traits.rs Normal file
View File

@@ -0,0 +1,68 @@
/// An attachment that extracts metadata from an [Item].
///
/// Metadata is exposed as an immutable map of {label: value},
/// much like a json object.
#[async_trait::async_trait]
pub trait ObjectExtractor: Send + Sync {
/// Get the field at `name` from `item`.
/// - returns `None` if `name` is not a valid field
/// - returns `Some(Null)` if `name` is not available
async fn field(
&self,
name: &pile_config::Label,
) -> Result<Option<crate::value::PileValue>, std::io::Error>;
/// Return all fields in this extractor.
/// `Self::field` must return [Some] for all these keys
/// and [None] for all others.
async fn fields(&self) -> Result<Vec<pile_config::Label>, std::io::Error>;
/// Convert this to a JSON value.
async fn to_json(&self) -> Result<serde_json::Value, std::io::Error> {
let keys = self.fields().await?;
let mut map = serde_json::Map::new();
for k in &keys {
let v = match self.field(k).await? {
Some(x) => x,
None => continue,
};
map.insert(k.to_string(), Box::pin(v.to_json()).await?);
}
Ok(serde_json::Value::Object(map))
}
}
/// An attachment that extracts metadata from an [Item].
///
/// Metadata is exposed as an immutable list of values.
#[async_trait::async_trait]
pub trait ListExtractor: Send + Sync {
/// Get the item at index `idx`.
/// Indices start at zero, and must be consecutive.
/// - returns `None` if `idx` is out of range
/// - returns `Some(Null)` if `None` is at `idx`
async fn get(&self, idx: usize) -> Result<Option<crate::value::PileValue>, std::io::Error>;
async fn len(&self) -> Result<usize, std::io::Error>;
async fn is_empty(&self) -> Result<bool, std::io::Error> {
Ok(self.len().await? == 0)
}
/// Convert this list to a JSON value.
async fn to_json(&self) -> Result<serde_json::Value, std::io::Error> {
let len = self.len().await?;
let mut list = Vec::with_capacity(len);
for i in 0..len {
#[expect(clippy::expect_used)]
let v = self
.get(i)
.await?
.expect("value must be present according to length");
list.push(Box::pin(v.to_json()).await?);
}
Ok(serde_json::Value::Array(list))
}
}

3
crates/pile-extractor/src/lib.rs Normal file
View File

@@ -0,0 +1,3 @@
pub mod extract;
pub mod source;
pub mod value;

5
crates/pile-dataset/src/source/dir.rs → crates/pile-extractor/src/source/dir.rs
View File

@@ -4,7 +4,10 @@ use std::{path::PathBuf, sync::Arc};
use tokio_stream::wrappers::ReceiverStream;
use walkdir::WalkDir;
use crate::{DataSource, Item, path_ts_latest};
use crate::{
source::{DataSource, misc::path_ts_latest},
value::Item,
};
#[derive(Debug)]
pub struct DirDataSource {

0
crates/pile-dataset/src/misc.rs → crates/pile-extractor/src/source/misc.rs
View File

17
crates/pile-dataset/src/traits.rs → crates/pile-extractor/src/source/mod.rs
View File

@@ -1,15 +1,24 @@
mod dir;
pub use dir::*;
mod s3;
pub use s3::*;
pub mod misc;
use chrono::{DateTime, Utc};
use tokio_stream::wrappers::ReceiverStream;
use crate::Item;
/// A read-only set of [Item]s.
pub trait DataSource {
/// Get an item from this datasource
fn get(&self, key: &str) -> impl Future<Output = Result<Option<Item>, std::io::Error>> + Send;
fn get(
&self,
key: &str,
) -> impl Future<Output = Result<Option<crate::value::Item>, std::io::Error>> + Send;
/// Iterate over all items in this source in an arbitrary order
fn iter(&self) -> ReceiverStream<Result<Item, std::io::Error>>;
fn iter(&self) -> ReceiverStream<Result<crate::value::Item, std::io::Error>>;
/// Return the time of the latest change to the data in this source
fn latest_change(

2
crates/pile-dataset/src/source/s3.rs → crates/pile-extractor/src/source/s3.rs
View File

@@ -5,7 +5,7 @@ use smartstring::{LazyCompact, SmartString};
use std::sync::Arc;
use tokio_stream::wrappers::ReceiverStream;
use crate::{DataSource, Item};
use crate::{source::DataSource, value::Item};
#[derive(Debug)]
pub struct S3DataSource {

0
crates/pile-dataset/src/source/s3reader.rs → crates/pile-extractor/src/source/s3reader.rs
View File

105
crates/pile-extractor/src/value/item.rs Normal file
View File

@@ -0,0 +1,105 @@
use mime::Mime;
use smartstring::{LazyCompact, SmartString};
use std::{fs::File, path::PathBuf, sync::Arc};
use crate::{
source::{DirDataSource, S3DataSource},
value::{ItemReader, S3Reader},
};
//
// MARK: item
//
/// A cheaply-clonable pointer to an item in a dataset
#[derive(Debug, Clone)]
pub enum Item {
File {
source: Arc<DirDataSource>,
mime: Mime,
path: PathBuf,
sidecar: Option<Box<Item>>,
},
S3 {
source: Arc<S3DataSource>,
mime: Mime,
key: SmartString<LazyCompact>,
sidecar: Option<Box<Item>>,
},
}
impl Item {
/// Open the item for reading. For S3, performs a HEAD request to determine
/// the object size.
pub async fn read(&self) -> Result<ItemReader, std::io::Error> {
Ok(match self {
Self::File { path, .. } => ItemReader::File(File::open(path)?),
Self::S3 { source, key, .. } => {
let head = source
.client
.head_object()
.bucket(source.bucket.as_str())
.key(key.as_str())
.send()
.await
.map_err(std::io::Error::other)?;
let size = head.content_length().unwrap_or(0) as u64;
ItemReader::S3(S3Reader {
client: source.client.clone(),
bucket: source.bucket.clone(),
key: key.to_owned(),
cursor: 0,
size,
})
}
})
}
pub fn source_name(&self) -> &pile_config::Label {
match self {
Self::File { source, .. } => &source.name,
Self::S3 { source, .. } => &source.name,
}
}
#[expect(clippy::expect_used)]
pub fn key(&self) -> SmartString<LazyCompact> {
match self {
Self::File { path, .. } => path.to_str().expect("path is not utf-8").into(),
Self::S3 { key, .. } => key.clone(),
}
}
pub fn hash(&self) -> Result<blake3::Hash, std::io::Error> {
match self {
Self::File { path, .. } => {
let mut hasher = blake3::Hasher::new();
let mut file = std::fs::File::open(path)?;
std::io::copy(&mut file, &mut hasher)?;
return Ok(hasher.finalize());
}
Self::S3 { .. } => todo!(),
}
}
pub fn mime(&self) -> &Mime {
match self {
Self::File { mime, .. } => mime,
Self::S3 { mime, .. } => mime,
}
}
pub fn sidecar(&self) -> Option<&Self> {
match self {
Self::File { sidecar, .. } => sidecar.as_ref().map(|x| &**x),
Self::S3 { sidecar, .. } => sidecar.as_ref().map(|x| &**x),
}
}
}

8
crates/pile-extractor/src/value/mod.rs Normal file
View File

@@ -0,0 +1,8 @@
mod item;
pub use item::*;
mod readers;
pub use readers::*;
mod value;
pub use value::*;

113
crates/pile-dataset/src/item.rs → crates/pile-extractor/src/value/readers.rs
View File

@@ -1,114 +1,13 @@
use mime::Mime;
use smartstring::{LazyCompact, SmartString};
use std::{
fs::File,
io::{Read, Seek, SeekFrom},
path::PathBuf,
sync::Arc,
};
use tokio::runtime::Handle;
use crate::source::{DirDataSource, S3DataSource};
//
// MARK: item
//
/// A cheaply-clonable pointer to an item in a dataset
#[derive(Debug, Clone)]
pub enum Item {
File {
source: Arc<DirDataSource>,
mime: Mime,
path: PathBuf,
sidecar: Option<Box<Item>>,
},
S3 {
source: Arc<S3DataSource>,
mime: Mime,
key: SmartString<LazyCompact>,
sidecar: Option<Box<Item>>,
},
}
impl Item {
/// Open the item for reading. For S3, performs a HEAD request to determine
/// the object size.
pub async fn read(&self) -> Result<ItemReader, std::io::Error> {
Ok(match self {
Self::File { path, .. } => ItemReader::File(File::open(path)?),
Self::S3 { source, key, .. } => {
let head = source
.client
.head_object()
.bucket(source.bucket.as_str())
.key(key.as_str())
.send()
.await
.map_err(std::io::Error::other)?;
let size = head.content_length().unwrap_or(0) as u64;
ItemReader::S3(S3Reader {
client: source.client.clone(),
bucket: source.bucket.clone(),
key: key.to_owned(),
cursor: 0,
size,
})
}
})
}
pub fn source_name(&self) -> &pile_config::Label {
match self {
Self::File { source, .. } => &source.name,
Self::S3 { source, .. } => &source.name,
}
}
#[expect(clippy::expect_used)]
pub fn key(&self) -> SmartString<LazyCompact> {
match self {
Self::File { path, .. } => path.to_str().expect("path is not utf-8").into(),
Self::S3 { key, .. } => key.clone(),
}
}
pub fn hash(&self) -> Result<blake3::Hash, std::io::Error> {
match self {
Self::File { path, .. } => {
let mut hasher = blake3::Hasher::new();
let mut file = std::fs::File::open(path)?;
std::io::copy(&mut file, &mut hasher)?;
return Ok(hasher.finalize());
}
Self::S3 { .. } => todo!(),
}
}
pub fn mime(&self) -> &Mime {
match self {
Self::File { mime, .. } => mime,
Self::S3 { mime, .. } => mime,
}
}
pub fn sidecar(&self) -> Option<&Self> {
match self {
Self::File { sidecar, .. } => sidecar.as_ref().map(|x| &**x),
Self::S3 { sidecar, .. } => sidecar.as_ref().map(|x| &**x),
}
}
}
//
// MARK: reader
// MARK: traits
//
pub trait AsyncReader: Send {
@@ -210,11 +109,11 @@ impl AsyncSeekReader for ItemReader {
//
pub struct S3Reader {
client: Arc<aws_sdk_s3::Client>,
bucket: SmartString<LazyCompact>,
key: SmartString<LazyCompact>,
cursor: u64,
size: u64,
pub client: Arc<aws_sdk_s3::Client>,
pub bucket: SmartString<LazyCompact>,
pub key: SmartString<LazyCompact>,
pub cursor: u64,
pub size: u64,
}
impl AsyncReader for S3Reader {

131
crates/pile-dataset/src/value.rs → crates/pile-extractor/src/value/value.rs
View File

@@ -4,7 +4,15 @@ use serde_json::{Map, Value};
use smartstring::{LazyCompact, SmartString};
use std::sync::Arc;
use crate::extract::{ListExtractor, ObjectExtractor};
use crate::{
extract::{
item::ItemExtractor,
misc::{ArrayExtractor, MapExtractor, VecExtractor},
string::StringExtractor,
traits::{ListExtractor, ObjectExtractor},
},
value::Item,
};
/// An immutable, cheaply-clonable, lazily-computed value.
/// Very similar to [serde_json::Value].
@@ -30,6 +38,9 @@ pub enum PileValue {
/// A lazily-computed array
ListExtractor(Arc<dyn ListExtractor>),
/// An pointer to an item in this dataset
Item(Item),
}
impl Clone for PileValue {
@@ -46,11 +57,40 @@ impl Clone for PileValue {
mime: mime.clone(),
bytes: bytes.clone(),
},
Self::Item(i) => Self::Item(i.clone()),
}
}
}
impl PileValue {
pub fn object_extractor(&self) -> Arc<dyn ObjectExtractor> {
match self {
Self::Null => Arc::new(MapExtractor::default()),
Self::U64(_) => Arc::new(MapExtractor::default()),
Self::I64(_) => Arc::new(MapExtractor::default()),
Self::Array(_) => Arc::new(MapExtractor::default()),
Self::String(s) => Arc::new(StringExtractor::new(&s)),
Self::Blob { .. } => Arc::new(MapExtractor::default()),
Self::ListExtractor(_) => Arc::new(MapExtractor::default()),
Self::ObjectExtractor(e) => e.clone(),
Self::Item(i) => Arc::new(ItemExtractor::new(i)),
}
}
pub fn list_extractor(&self) -> Arc<dyn ListExtractor> {
match self {
Self::Null => Arc::new(VecExtractor::default()),
Self::U64(_) => Arc::new(VecExtractor::default()),
Self::I64(_) => Arc::new(VecExtractor::default()),
Self::Array(a) => Arc::new(ArrayExtractor::new(a.clone())),
Self::String(_) => Arc::new(VecExtractor::default()),
Self::Blob { .. } => Arc::new(VecExtractor::default()),
Self::ListExtractor(e) => e.clone(),
Self::ObjectExtractor(_) => Arc::new(VecExtractor::default()),
Self::Item(_) => Arc::new(VecExtractor::default()),
}
}
pub async fn query(&self, query: &ObjectPath) -> Result<Option<Self>, std::io::Error> {
let mut out: Option<PileValue> = Some(self.clone());
@@ -58,50 +98,41 @@ impl PileValue {
match s {
PathSegment::Root => out = Some(self.clone()),
PathSegment::Field(field) => {
out = match out {
None => return Ok(None),
Some(Self::Null) => None,
Some(Self::U64(_)) => None,
Some(Self::I64(_)) => None,
Some(Self::Array(_)) => None,
Some(Self::String(_)) => None,
Some(Self::Blob { .. }) => None,
Some(Self::ListExtractor(_)) => None,
Some(Self::ObjectExtractor(e)) => e.field(field).await?,
}
let e = match out.map(|x| x.object_extractor()) {
Some(e) => e,
None => {
out = None;
continue;
}
};
out = e.field(&field).await?;
}
PathSegment::Index(idx) => {
out = match &out {
None => return Ok(None),
Some(Self::Null) => None,
Some(Self::U64(_)) => None,
Some(Self::I64(_)) => None,
Some(Self::Blob { .. }) => None,
Some(Self::Array(v)) => {
let idx = if *idx >= 0 {
usize::try_from(*idx).ok()
} else {
usize::try_from(v.len() as i64 - idx).ok()
};
idx.and_then(|idx| v.get(idx)).cloned()
let e = match out.map(|x| x.list_extractor()) {
Some(e) => e,
None => {
out = None;
continue;
}
Some(Self::String(_)) => None,
Some(Self::ObjectExtractor(_)) => None,
Some(Self::ListExtractor(e)) => {
let idx = if *idx >= 0 {
usize::try_from(*idx).ok()
} else {
usize::try_from(e.len().await? as i64 - idx).ok()
};
};
match idx {
Some(idx) => e.get(idx).await?,
None => None,
}
let idx = if *idx >= 0 {
usize::try_from(*idx).ok()
} else {
usize::try_from(e.len().await? as i64 - idx).ok()
};
let idx = match idx {
Some(idx) => idx,
None => {
out = None;
continue;
}
}
};
out = e.get(idx).await?;
}
}
}
@@ -127,7 +158,8 @@ impl PileValue {
Self::Array(x) => (!x.is_empty()).then(|| Value::Number(1u64.into())),
Self::ListExtractor(x) => (!x.is_empty().await?).then(|| Value::Number(1u64.into())),
Self::ObjectExtractor(e) => {
Self::ObjectExtractor(_) | Self::Item(_) => {
let e = self.object_extractor();
let keys = e.fields().await?;
let mut map = Map::new();
for k in &keys {
@@ -160,22 +192,27 @@ impl PileValue {
Self::Null => Value::Null,
Self::U64(x) => Value::Number((*x).into()),
Self::I64(x) => Value::Number((*x).into()),
Self::String(x) => Value::String(x.to_string()),
// TODO: replace with something meaningful
// TODO: replace with something meaningful?
Self::Blob { mime, bytes } => {
Value::String(format!("<Blob ({mime}, {} bytes)>", bytes.len()))
}
Self::String(x) => Value::String(x.to_string()),
Self::Array(x) => {
#[expect(clippy::expect_used)]
Self::Array(_) | Self::ListExtractor(_) => {
let e = self.list_extractor();
let len = e.len().await?;
let mut arr = Vec::new();
for item in &**x {
arr.push(Box::pin(item.to_json()).await?);
for i in 0..len {
let v = e.get(i).await?.expect("item must be present");
arr.push(Box::pin(v.to_json()).await?);
}
Value::Array(arr)
}
Self::ObjectExtractor(e) => {
Self::ObjectExtractor(_) | Self::Item(_) => {
let e = self.object_extractor();
let keys = e.fields().await?;
let mut map = Map::new();
for k in &keys {
@@ -187,8 +224,6 @@ impl PileValue {
}
Value::Object(map)
}
Self::ListExtractor(e) => e.to_json().await?,
})
}
}

1
crates/pile/Cargo.toml
View File

@@ -10,6 +10,7 @@ workspace = true
[dependencies]
pile-toolbox = { workspace = true }
pile-dataset = { workspace = true, features = ["axum", "pdfium"] }
pile-value = { workspace = true, features = ["pdfium"] }
pile-config = { workspace = true }
tracing = { workspace = true }

14
crates/pile/src/command/annotate.rs
View File

@@ -1,10 +1,12 @@
use anyhow::{Context, Result};
use clap::Args;
use pile_config::{Label, Source};
use pile_dataset::index::DbFtsIndex;
use pile_dataset::source::DirDataSource;
use pile_dataset::{DataSource, Datasets, Item, PileValue, extract::MetaExtractor};
use pile_dataset::{Datasets, index::DbFtsIndex};
use pile_toolbox::cancelabletask::{CancelFlag, CancelableTaskError};
use pile_value::{
source::{DataSource, DirDataSource},
value::{Item, PileValue},
};
use std::{path::PathBuf, sync::Arc};
use tokio_stream::StreamExt;
use tracing::{info, warn};
@@ -72,11 +74,9 @@ impl CliCmd for AnnotateCommand {
continue;
};
let meta = MetaExtractor::new(&item);
let extractor = PileValue::ObjectExtractor(Arc::new(meta));
let item = PileValue::Item(item.clone());
let Some(value) =
index.get_field(&extractor, &field).await.with_context(|| {
index.get_field(&item, &field).await.with_context(|| {
format!("while extracting field from {}", path.display())
})?
else {

10
crates/pile/src/command/fields.rs
View File

@@ -1,9 +1,10 @@
use anyhow::{Context, Result};
use clap::Args;
use pile_dataset::{Datasets, PileValue, extract::MetaExtractor};
use pile_dataset::Datasets;
use pile_toolbox::cancelabletask::{CancelFlag, CancelableTaskError};
use pile_value::value::PileValue;
use serde_json::{Map, Value};
use std::{path::PathBuf, sync::Arc, time::Instant};
use std::{path::PathBuf, time::Instant};
use tokio::task::JoinSet;
use tokio_stream::StreamExt;
use tracing::info;
@@ -93,9 +94,8 @@ impl CliCmd for FieldsCommand {
item_result.with_context(|| format!("while reading source {name}"))?;
let name = name.clone();
join_set.spawn(async move {
let meta = MetaExtractor::new(&item);
let value = PileValue::ObjectExtractor(Arc::new(meta));
let result = value.count_fields().await.with_context(|| {
let item = PileValue::Item(item);
let result = item.count_fields().await.with_context(|| {
format!("while counting fields in source {name}")
})?;
Ok(result.and_then(|v| {

8
crates/pile/src/command/list.rs
View File

@@ -1,8 +1,9 @@
use anyhow::{Context, Result};
use clap::Args;
use pile_config::objectpath::ObjectPath;
use pile_dataset::{Datasets, PileValue, extract::MetaExtractor};
use pile_dataset::Datasets;
use pile_toolbox::cancelabletask::{CancelFlag, CancelableTaskError};
use pile_value::value::PileValue;
use std::{path::PathBuf, str::FromStr, sync::Arc};
use tokio::task::JoinSet;
use tokio_stream::StreamExt;
@@ -79,9 +80,8 @@ impl CliCmd for ListCommand {
let invert = self.invert;
join_set.spawn(async move {
let meta = MetaExtractor::new(&item);
let root = PileValue::ObjectExtractor(Arc::new(meta));
let value = root.query(&path).await?;
let item = PileValue::Item(item);
let value = item.query(&path).await?;
let is_present =
matches!(value, Some(v) if !matches!(v, PileValue::Null));

10
crates/pile/src/command/probe.rs
View File

@@ -1,9 +1,10 @@
use anyhow::{Context, Result};
use clap::Args;
use pile_config::{Label, objectpath::ObjectPath};
use pile_dataset::{Datasets, PileValue, extract::MetaExtractor};
use pile_dataset::Datasets;
use pile_toolbox::cancelabletask::{CancelFlag, CancelableTaskError};
use std::{path::PathBuf, sync::Arc};
use pile_value::value::PileValue;
use std::path::PathBuf;
use crate::{CliCmd, GlobalContext};
@@ -54,9 +55,8 @@ impl CliCmd for ProbeCommand {
anyhow::anyhow!("{:?} not found in source {:?}", self.key, self.source)
})?;
let value = PileValue::ObjectExtractor(Arc::new(MetaExtractor::new(&item)));
value
.to_json()
let item = PileValue::Item(item);
item.to_json()
.await
.with_context(|| format!("while extracting {}", self.key))?
};