Dataset length

crates/pile-dataset/src/dataset.rs

@@ -40,6 +40,13 @@ pub enum Dataset {
 }
 
 impl Dataset {
+	pub fn len(&self) -> usize {
+		match self {
+			Self::Dir(ds) => ds.len(),
+			Self::S3(ds) => ds.len(),
+		}
+	}
+
 	pub async fn get(&self, key: &str) -> Option<Item> {
 		match self {
 			Self::Dir(ds) => ds.get(key).await.ok().flatten(),

crates/pile-value/src/source/dir.rs

@@ -112,6 +112,10 @@ impl DirDataSource {
 }
 
 impl DataSource for Arc<DirDataSource> {
+	fn len(&self) -> usize {
+		self.index.get().expect("index should be initialized").len()
+	}
+
 	#[expect(clippy::expect_used)]
 	async fn get(&self, key: &str) -> Result<Option<Item>, std::io::Error> {
 		return Ok(self

crates/pile-value/src/source/mod.rs

@@ -8,6 +8,9 @@ pub mod misc;
 
 /// A read-only set of [Item]s.
 pub trait DataSource {
+	/// Get the number of items in this source
+	fn len(&self) -> usize;
+
 	/// Get an item from this datasource
 	fn get(
 		&self,

crates/pile-value/src/source/s3.rs

@@ -163,6 +163,10 @@ impl S3DataSource {
 }
 
 impl DataSource for Arc<S3DataSource> {
+	fn len(&self) -> usize {
+		self.index.get().expect("index should be initialized").len()
+	}
+
 	#[expect(clippy::expect_used)]
 	async fn get(&self, key: &str) -> Result<Option<Item>, std::io::Error> {
 		return Ok(self