From 97723f56e8d4c68f0c475a77c908199597c3e1dd Mon Sep 17 00:00:00 2001 From: Firelight Flagboy Date: Mon, 29 Jul 2024 15:54:42 +0200 Subject: [PATCH] Add method to list manifests from a workspace --- Cargo.lock | 1 + libparsec/crates/platform_storage/Cargo.toml | 3 +- .../platform_storage/src/native/workspace.rs | 36 +++++++++++++ .../crates/platform_storage/src/web/db.rs | 53 +++++++++++++++++++ .../crates/platform_storage/src/web/model.rs | 9 ++++ .../platform_storage/src/web/workspace.rs | 13 +++++ .../crates/platform_storage/src/workspace.rs | 8 +++ .../platform_storage/tests/unit/workspace.rs | 52 ++++++++++++++++++ 8 files changed, 174 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3e27ed4b338..7b495694689 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2244,6 +2244,7 @@ dependencies = [ "sqlx", "tokio", "wasm-bindgen-test", + "web-sys", ] [[package]] diff --git a/libparsec/crates/platform_storage/Cargo.toml b/libparsec/crates/platform_storage/Cargo.toml index f78968374eb..21039a5b59d 100644 --- a/libparsec/crates/platform_storage/Cargo.toml +++ b/libparsec/crates/platform_storage/Cargo.toml @@ -32,8 +32,9 @@ libsqlite3-sys = { workspace = true, features = ["bundled"] } sqlx = { workspace = true, features = ["sqlite", "runtime-tokio", "macros"] } [target.'cfg(target_arch = "wasm32")'.dependencies] -indexed_db_futures = { workspace = true, features = ["indices"] } +indexed_db_futures = { workspace = true, features = ["indices", "cursors"] } js-sys = { workspace = true } +web-sys = { workspace = true, features = ["IdbKeyRange"] } serde = { workspace = true } serde-wasm-bindgen = { workspace = true } diff --git a/libparsec/crates/platform_storage/src/native/workspace.rs b/libparsec/crates/platform_storage/src/native/workspace.rs index 0bfc627e540..30323736ef5 100644 --- a/libparsec/crates/platform_storage/src/native/workspace.rs +++ b/libparsec/crates/platform_storage/src/native/workspace.rs @@ -4,6 +4,7 @@ // validation work and takes care of handling concurrency issues. // Hence no unique violation should occur under normal circumstances here. +use libparsec_platform_async::stream::{StreamExt, TryStreamExt}; use sqlx::{ sqlite::{SqliteConnectOptions, SqliteJournalMode, SqliteSynchronous}, ConnectOptions, Connection, Row, SqliteConnection, @@ -199,6 +200,14 @@ impl PlatformWorkspaceStorage { db_get_manifest(&mut self.conn, entry_id).await } + pub async fn list_manifests( + &mut self, + offset: u32, + limit: u32, + ) -> anyhow::Result> { + db_list_manifests(&mut self.conn, offset, limit).await + } + pub async fn get_chunk(&mut self, chunk_id: ChunkID) -> anyhow::Result>> { db_get_chunk(&mut self.conn, chunk_id).await } @@ -641,6 +650,33 @@ async fn db_get_manifest( } } +async fn db_list_manifests( + executor: impl sqlx::Executor<'_, Database = sqlx::Sqlite>, + offset: u32, + limit: u32, +) -> anyhow::Result> { + let rows = sqlx::query("SELECT blob FROM vlobs LIMIT ?1 OFFSET ?2") + .bind(limit) + .bind(offset) + .fetch_many(executor); + + let res: anyhow::Result> = rows + .filter_map(|row_res| async { + match row_res { + // `fetch_many` returns a single "left" element for the query result (containing, among others, + // the number of changes), then (what we are actually interested into here) one "right" + // elements for each row in the result. + Ok(sqlx::Either::Left(_res)) => None, + Ok(sqlx::Either::Right(row)) => Some(row.try_get(0).map_err(anyhow::Error::from)), + Err(e) => Some(Err(e.into())), + } + }) + .try_collect() + .await; + + res +} + pub async fn db_get_chunk( executor: impl sqlx::Executor<'_, Database = sqlx::Sqlite>, chunk_id: ChunkID, diff --git a/libparsec/crates/platform_storage/src/web/db.rs b/libparsec/crates/platform_storage/src/web/db.rs index fa4a549ec61..cd1b32d2a13 100644 --- a/libparsec/crates/platform_storage/src/web/db.rs +++ b/libparsec/crates/platform_storage/src/web/db.rs @@ -93,6 +93,59 @@ where .map_err(|e| anyhow::anyhow!("{e:?}")) } +pub(super) async fn list( + tx: &IdbTransaction<'_>, + store: &str, + offset: u32, + limit: u32, +) -> anyhow::Result> +where + V: DeserializeOwned, +{ + use js_sys::Number; + use web_sys::IdbKeyRange; + + // Index start at 1 + let start = offset + 1; + let end = start + limit; + + let range = IdbKeyRange::bound_with_lower_open_and_upper_open( + &Number::from(start), + &Number::from(end), + false, + true, + ) + .map_err(|e| anyhow::anyhow!("{e:?}"))?; + + let store = tx + .object_store(store) + .map_err(|e| anyhow::anyhow!("{e:?}"))?; + + let Some(cursor) = store + .open_cursor_with_range_owned(range) + .map_err(|e| anyhow::anyhow!("{e:?}"))? + .await + .map_err(|e| anyhow::anyhow!("{e:?}"))? + else { + return Ok(Vec::new()); + }; + + cursor + .into_vec(0) + .await + .map_err(|e| anyhow::anyhow!("{e:?}")) + .and_then(|v: Vec<_>| { + v.into_iter() + .map(|key_val| + // TODO: Sad that KeyVal does not provide it's internal value without a reference. + // That reference force us to clone the value, which is not optimal. + // Could be improved if https://github.com/Alorel/rust-indexed-db/issues/39 is fixed + serde_wasm_bindgen::from_value(key_val.value().clone()) + .map_err(|e| anyhow::anyhow!("{e:?}"))) + .collect::>>() + }) +} + pub(super) async fn count( tx: &IdbTransaction<'_>, store: &str, diff --git a/libparsec/crates/platform_storage/src/web/model.rs b/libparsec/crates/platform_storage/src/web/model.rs index 9a6d14a7913..56af502af9d 100644 --- a/libparsec/crates/platform_storage/src/web/model.rs +++ b/libparsec/crates/platform_storage/src/web/model.rs @@ -574,6 +574,15 @@ impl Vlob { super::db::get_all(&tx, Self::STORE).await } + pub(super) async fn list( + conn: &IdbDatabase, + offset: u32, + limit: u32, + ) -> anyhow::Result> { + let tx = Self::read(conn)?; + super::db::list(&tx, Self::STORE, offset, limit).await + } + pub(super) async fn remove(tx: &IdbTransaction<'_>, vlob_id: &Bytes) -> anyhow::Result<()> { let vlob_id = serde_wasm_bindgen::to_value(vlob_id).map_err(|e| anyhow::anyhow!("{e:?}"))?; diff --git a/libparsec/crates/platform_storage/src/web/workspace.rs b/libparsec/crates/platform_storage/src/web/workspace.rs index e1da4b75c49..63a04e5721a 100644 --- a/libparsec/crates/platform_storage/src/web/workspace.rs +++ b/libparsec/crates/platform_storage/src/web/workspace.rs @@ -143,6 +143,19 @@ impl PlatformWorkspaceStorage { ) } + pub async fn list_manifests( + &mut self, + offset: u32, + limit: u32, + ) -> anyhow::Result> { + Vlob::list(&self.conn, offset, limit).await.map(|vlobs| { + vlobs + .into_iter() + .map(|vlob| RawEncryptedManifest::from(vlob.blob)) + .collect() + }) + } + pub async fn get_chunk(&mut self, chunk_id: ChunkID) -> anyhow::Result>> { let transaction = super::model::Chunk::read(&self.conn)?; db_get_chunk(&transaction, chunk_id).await diff --git a/libparsec/crates/platform_storage/src/workspace.rs b/libparsec/crates/platform_storage/src/workspace.rs index 0bf8967dfbc..7fe851aaf9d 100644 --- a/libparsec/crates/platform_storage/src/workspace.rs +++ b/libparsec/crates/platform_storage/src/workspace.rs @@ -142,6 +142,14 @@ impl WorkspaceStorage { self.platform.get_manifest(entry_id).await } + pub async fn list_manifests( + &mut self, + offset: u32, + limit: u32, + ) -> anyhow::Result> { + self.platform.list_manifests(offset, limit).await + } + pub async fn populate_manifest( &mut self, manifest: &UpdateManifestData, diff --git a/libparsec/crates/platform_storage/tests/unit/workspace.rs b/libparsec/crates/platform_storage/tests/unit/workspace.rs index bdad3ceec74..5efd4e675f4 100644 --- a/libparsec/crates/platform_storage/tests/unit/workspace.rs +++ b/libparsec/crates/platform_storage/tests/unit/workspace.rs @@ -325,6 +325,58 @@ async fn get_and_update_manifest(env: &TestbedEnv) { ); } +#[parsec_test(testbed = "minimal")] +async fn list_manifests(env: &TestbedEnv) { + let realm_id = VlobID::from_hex("aa0000000000000000000000000000ee").unwrap(); + let entry1_id = VlobID::from_hex("aa0000000000000000000000000000f1").unwrap(); + let entry2_id = VlobID::from_hex("aa0000000000000000000000000000f2").unwrap(); + let alice = env.local_device("alice@dev1"); + + let mut workspace_storage = + WorkspaceStorage::start(&env.discriminant_dir, &alice, realm_id, u64::MAX) + .await + .unwrap(); + + workspace_storage + .update_manifests( + [ + UpdateManifestData { + entry_id: entry1_id, + encrypted: b"".to_vec(), + need_sync: true, + base_version: 1, + }, + UpdateManifestData { + entry_id: entry2_id, + encrypted: b"".to_vec(), + need_sync: false, + base_version: 2, + }, + ] + .into_iter(), + ) + .await + .unwrap(); + + p_assert_eq!( + workspace_storage.list_manifests(0, 1).await.unwrap(), + [b""] + ); + p_assert_eq!( + workspace_storage.list_manifests(1, 1).await.unwrap(), + [b""] + ); + + p_assert_eq!( + workspace_storage.list_manifests(0, 10).await.unwrap(), + [b"", b""] + ); + p_assert_eq!( + workspace_storage.list_manifests(2, 10).await.unwrap(), + [] as [&[u8]; 0] + ); +} + #[parsec_test(testbed = "minimal")] async fn update_manifests(env: &TestbedEnv) { let realm_id = VlobID::from_hex("aa0000000000000000000000000000ee").unwrap();