From 4010d3cc1cc3d89b176eb2fa08e4ffc34ec33161 Mon Sep 17 00:00:00 2001
From: rm-dr <96270320+rm-dr@users.noreply.github.com>
Date: Mon, 23 Mar 2026 14:12:11 -0700
Subject: [PATCH] Dataset length

---
 crates/pile-dataset/src/dataset.rs  | 7 +++++++
 crates/pile-value/src/source/dir.rs | 4 ++++
 crates/pile-value/src/source/mod.rs | 3 +++
 crates/pile-value/src/source/s3.rs  | 4 ++++
 4 files changed, 18 insertions(+)

diff --git a/crates/pile-dataset/src/dataset.rs b/crates/pile-dataset/src/dataset.rs
index e54720f..0fd9756 100644
--- a/crates/pile-dataset/src/dataset.rs
+++ b/crates/pile-dataset/src/dataset.rs
@@ -40,6 +40,13 @@ pub enum Dataset {
 }
 
 impl Dataset {
+	pub fn len(&self) -> usize {
+		match self {
+			Self::Dir(ds) => ds.len(),
+			Self::S3(ds) => ds.len(),
+		}
+	}
+
 	pub async fn get(&self, key: &str) -> Option<Item> {
 		match self {
 			Self::Dir(ds) => ds.get(key).await.ok().flatten(),
diff --git a/crates/pile-value/src/source/dir.rs b/crates/pile-value/src/source/dir.rs
index 06e8505..8654841 100644
--- a/crates/pile-value/src/source/dir.rs
+++ b/crates/pile-value/src/source/dir.rs
@@ -112,6 +112,10 @@ impl DirDataSource {
 }
 
 impl DataSource for Arc<DirDataSource> {
+	fn len(&self) -> usize {
+		self.index.get().expect("index should be initialized").len()
+	}
+
 	#[expect(clippy::expect_used)]
 	async fn get(&self, key: &str) -> Result<Option<Item>, std::io::Error> {
 		return Ok(self
diff --git a/crates/pile-value/src/source/mod.rs b/crates/pile-value/src/source/mod.rs
index 86894d3..b600f30 100644
--- a/crates/pile-value/src/source/mod.rs
+++ b/crates/pile-value/src/source/mod.rs
@@ -8,6 +8,9 @@ pub mod misc;
 
 /// A read-only set of [Item]s.
 pub trait DataSource {
+	/// Get the number of items in this source
+	fn len(&self) -> usize;
+
 	/// Get an item from this datasource
 	fn get(
 		&self,
diff --git a/crates/pile-value/src/source/s3.rs b/crates/pile-value/src/source/s3.rs
index bbe8fa8..191fbfd 100644
--- a/crates/pile-value/src/source/s3.rs
+++ b/crates/pile-value/src/source/s3.rs
@@ -163,6 +163,10 @@ impl S3DataSource {
 }
 
 impl DataSource for Arc<S3DataSource> {
+	fn len(&self) -> usize {
+		self.index.get().expect("index should be initialized").len()
+	}
+
 	#[expect(clippy::expect_used)]
 	async fn get(&self, key: &str) -> Result<Option<Item>, std::io::Error> {
 		return Ok(self