diff --git a/Cargo.lock b/Cargo.lock index 6b8ad2a7fc..094f3258a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -235,6 +235,51 @@ version = "1.1.0" source = "registry+/~https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backoff" version = "0.4.0" @@ -618,6 +663,42 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "console-api" +version = "0.5.0" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.10" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures", + "hdrhistogram", + "humantime", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.5" @@ -690,6 +771,15 @@ version = "2.2.0" source = "registry+/~https://github.com/rust-lang/crates.io-index" checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.5.1" @@ -1388,6 +1478,16 @@ version = "0.2.1" source = "registry+/~https://github.com/rust-lang/crates.io-index" checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" +[[package]] +name = "flate2" +version = "1.0.27" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.10.14" @@ -1401,6 +1501,18 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1651,6 +1763,19 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hdrhistogram" +version = "7.5.2" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +dependencies = [ + "base64 0.13.1", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "heck" version = "0.4.1" @@ -1772,6 +1897,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.27" @@ -1810,6 +1941,18 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -1977,13 +2120,14 @@ dependencies = [ "comfy-table", "config", "console", + "console-subscriber", "data-encoding", "derive_more", "dialoguer", "dirs-next", "duct", "ed25519-dalek", - "flume", + "flume 0.11.0", "futures", "genawaiter", "hex", @@ -2008,6 +2152,7 @@ dependencies = [ "quic-rpc", "quinn", "rand", + "rand_chacha", "range-collections", "regex", "rustyline", @@ -2051,7 +2196,7 @@ dependencies = [ "chrono", "data-encoding", "derive_more", - "flume", + "flume 0.11.0", "futures", "genawaiter", "hex", @@ -2156,7 +2301,7 @@ dependencies = [ "derive_more", "duct", "ed25519-dalek", - "flume", + "flume 0.11.0", "futures", "governor", "hex", @@ -2230,7 +2375,7 @@ dependencies = [ "data-encoding", "derive_more", "ed25519-dalek", - "flume", + "flume 0.11.0", "futures", "hex", "iroh-blake3", @@ -2250,6 +2395,7 @@ dependencies = [ "rand_core", "redb", "serde", + "strum 0.25.0", "tempfile", "test-strategy", "thiserror", @@ -2411,6 +2557,12 @@ version = "0.1.10" source = "registry+/~https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md5" version = "0.7.0" @@ -3358,6 +3510,38 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + [[package]] name = "pyo3-build-config" version = "0.19.2" @@ -3392,7 +3576,7 @@ checksum = "6d60c2fc2390baad4b9d41ae9957ae88c3095496f88e252ef50722df8b5b78d7" dependencies = [ "bincode", "educe", - "flume", + "flume 0.10.14", "futures", "pin-project", "quinn", @@ -4546,6 +4730,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synstructure" version = "0.12.6" @@ -4737,9 +4927,20 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.4", "tokio-macros", + "tracing", "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.1.0" @@ -4875,6 +5076,60 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64 0.21.4", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+/~https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" diff --git a/iroh-bytes/Cargo.toml b/iroh-bytes/Cargo.toml index 4376837f23..879dc0fe98 100644 --- a/iroh-bytes/Cargo.toml +++ b/iroh-bytes/Cargo.toml @@ -18,7 +18,7 @@ bytes = { version = "1.4", features = ["serde"] } chrono = "0.4.31" data-encoding = "2.3.3" derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into", "into"] } -flume = "0.10.14" +flume = "0.11" futures = "0.3.25" genawaiter = { version = "0.99.1", features = ["futures03"] } hex = "0.4.3" diff --git a/iroh-gossip/src/net.rs b/iroh-gossip/src/net.rs index b597193f71..d49986cd55 100644 --- a/iroh-gossip/src/net.rs +++ b/iroh-gossip/src/net.rs @@ -24,10 +24,10 @@ pub const GOSSIP_ALPN: &[u8] = b"/iroh-gossip/0"; /// Maximum message size is limited to 1024 bytes. pub const MAX_MESSAGE_SIZE: usize = 1024; -/// Channel capacity for topic subscription broadcast channels (one per topic) -const SUBSCRIBE_ALL_CAP: usize = 64; /// Channel capacity for all subscription broadcast channels (single) -const SUBSCRIBE_TOPIC_CAP: usize = 64; +const SUBSCRIBE_ALL_CAP: usize = 2048; +/// Channel capacity for topic subscription broadcast channels (one per topic) +const SUBSCRIBE_TOPIC_CAP: usize = 2048; /// Channel capacity for the send queue (one per connection) const SEND_QUEUE_CAP: usize = 64; /// Channel capacity for the ToActor message queue (single) @@ -196,24 +196,27 @@ impl Gossip { /// /// Note that this method takes self by value. Usually you would clone the [`Gossip`] handle. /// before. - pub fn subscribe_all(self) -> impl Stream> { + pub fn subscribe_all( + self, + ) -> impl Stream> { Gen::new(|co| async move { - if let Err(cause) = self.subscribe_all0(&co).await { - co.yield_(Err(cause)).await + if let Err(err) = self.subscribe_all0(&co).await { + warn!("subscribe_all produced error: {err:?}"); + co.yield_(Err(broadcast::error::RecvError::Closed)).await } }) } async fn subscribe_all0( &self, - co: &Co>, + co: &Co>, ) -> anyhow::Result<()> { let (tx, rx) = oneshot::channel(); self.send(ToActor::SubscribeAll(tx)).await?; - let mut res = rx.await.map_err(|_| anyhow!("subscribe_tx dropped"))??; + let mut res = rx.await??; loop { - let event = res.recv().await?; - co.yield_(Ok(event)).await; + let event = res.recv().await; + co.yield_(event).await; } } @@ -256,9 +259,9 @@ impl Gossip { /// /// TODO: Optionally resolve to an error once all connection attempts failed. #[derive(Debug)] -pub struct JoinTopicFut(oneshot::Receiver>); +pub struct JoinTopicFut(oneshot::Receiver>); impl Future for JoinTopicFut { - type Output = anyhow::Result<()>; + type Output = anyhow::Result; fn poll( mut self: std::pin::Pin<&mut Self>, @@ -290,7 +293,7 @@ enum ToActor { Join( TopicId, Vec, - #[debug(skip)] oneshot::Sender>, + #[debug(skip)] oneshot::Sender>, ), /// Leave a topic, send disconnect messages and drop all state. Quit(TopicId), @@ -345,10 +348,14 @@ struct Actor { impl Actor { pub async fn run(mut self) -> anyhow::Result<()> { + let mut i = 0; loop { + i += 1; + trace!(?i, "tick"); tokio::select! { biased; msg = self.to_actor_rx.recv() => { + trace!(?i, "tick: to_actor_rx"); match msg { Some(msg) => self.handle_to_actor_msg(msg, Instant::now()).await?, None => { @@ -371,6 +378,7 @@ impl Actor { } } (peer_id, res) = self.dialer.next_conn() => { + trace!(?i, "tick: dialer"); match res { Ok(conn) => { debug!(peer = ?peer_id, "dial successfull"); @@ -382,6 +390,7 @@ impl Actor { } } event = self.in_event_rx.recv() => { + trace!(?i, "tick: in_event_rx"); match event { Some(event) => { self.handle_in_event(event, Instant::now()).await.context("in_event_rx.recv -> handle_in_event")?; @@ -390,6 +399,7 @@ impl Actor { } } drain = self.timers.wait_and_drain() => { + trace!(?i, "tick: timers"); let now = Instant::now(); for (_instant, timer) in drain { self.handle_in_event(InEvent::TimerExpired(timer), now).await.context("timers.drain_expired -> handle_in_event")?; @@ -412,21 +422,24 @@ impl Actor { // Spawn a task for this connection let in_event_tx = self.in_event_tx.clone(); - tokio::spawn(async move { - debug!(peer = ?peer_id, "connection established"); - match connection_loop(peer_id, conn, origin, send_rx, &in_event_tx).await { - Ok(()) => { - debug!(peer = ?peer_id, "connection closed without error") - } - Err(err) => { - debug!(peer = ?peer_id, "connection closed with error {err:?}") + tokio::spawn( + async move { + debug!("connection established"); + match connection_loop(peer_id, conn, origin, send_rx, &in_event_tx).await { + Ok(()) => { + debug!("connection closed without error") + } + Err(err) => { + debug!("connection closed with error {err:?}") + } } + in_event_tx + .send(InEvent::PeerDisconnected(peer_id)) + .await + .ok(); } - in_event_tx - .send(InEvent::PeerDisconnected(peer_id)) - .await - .ok(); - }); + .instrument(error_span!("gossip_conn", peer = %peer_id.fmt_short())), + ); // Forward queued pending sends if let Some(send_queue) = self.pending_sends.remove(&peer_id) { @@ -440,12 +453,13 @@ impl Actor { .await?; if self.state.has_active_peers(&topic_id) { // If the active_view contains at least one peer, reply now - reply.send(Ok(())).ok(); + reply.send(Ok(topic_id)).ok(); } else { // Otherwise, wait for any peer to come up as neighbor. let sub = self.subscribe(topic_id); tokio::spawn(async move { let res = wait_for_neighbor_up(sub).await; + let res = res.map(|_| topic_id); reply.send(res).ok(); }); } @@ -575,7 +589,7 @@ async fn wait_for_neighbor_up(mut sub: broadcast::Receiver) -> anyhow::Re Ok(Event::NeighborUp(_neighbor)) => break Ok(()), Ok(_) | Err(broadcast::error::RecvError::Lagged(_)) => {} Err(broadcast::error::RecvError::Closed) => { - break Err(anyhow!("Failed to join swarm: Gossip actor dropped")) + break Err(anyhow!("Failed to join swarm: channel closed")) } } } diff --git a/iroh-net/Cargo.toml b/iroh-net/Cargo.toml index fdf22f1902..5f87564f18 100644 --- a/iroh-net/Cargo.toml +++ b/iroh-net/Cargo.toml @@ -23,7 +23,7 @@ data-encoding = "2.3.3" der = { version = "0.7", features = ["alloc", "derive"] } derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into", "deref"] } ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"] } -flume = "0.10.14" +flume = "0.11" futures = "0.3.25" governor = "0.6.0" hex = "0.4.3" diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index 686e36b650..90af0d1b43 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -18,7 +18,7 @@ crossbeam = "0.8.2" data-encoding = "2.4.0" derive_more = { version = "1.0.0-beta.1", features = ["debug", "deref", "display", "from", "try_into", "into", "as_ref"] } ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"] } -flume = "0.10" +flume = "0.11" iroh-bytes = { version = "0.7.0", path = "../iroh-bytes" } iroh-metrics = { version = "0.7.0", path = "../iroh-metrics", optional = true } once_cell = "1.18.0" @@ -26,6 +26,7 @@ postcard = { version = "1", default-features = false, features = ["alloc", "use- rand = "0.8.5" rand_core = "0.6.4" serde = { version = "1.0.164", features = ["derive"] } +strum = { version = "0.25", features = ["derive"] } url = "2.4" bytes = "1" parking_lot = "0.12.1" diff --git a/iroh-sync/src/actor.rs b/iroh-sync/src/actor.rs new file mode 100644 index 0000000000..cc6075ea29 --- /dev/null +++ b/iroh-sync/src/actor.rs @@ -0,0 +1,778 @@ +//! This contains an actor spawned on a seperate thread to process replica and store operations. + +use std::{ + collections::{hash_map, HashMap}, + sync::Arc, +}; + +use anyhow::{anyhow, Context, Result}; +use bytes::Bytes; +use iroh_bytes::Hash; +use serde::{Deserialize, Serialize}; +use tokio::sync::oneshot; +use tracing::{debug, error, error_span, trace, warn}; + +use crate::{ + ranger::Message, + store::{self, GetFilter}, + Author, AuthorId, ContentStatus, ContentStatusCallback, Event, Namespace, NamespaceId, + PeerIdBytes, Replica, SignedEntry, SyncOutcome, +}; + +#[derive(derive_more::Debug, derive_more::Display)] +enum Action { + #[display("NewAuthor")] + ImportAuthor { + author: Author, + #[debug("reply")] + reply: oneshot::Sender>, + }, + #[display("NewReplica")] + ImportNamespace { + namespace: Namespace, + #[debug("reply")] + reply: oneshot::Sender>, + }, + #[display("ListAuthors")] + ListAuthors { + #[debug("reply")] + reply: flume::Sender>, + }, + #[display("ListReplicas")] + ListReplicas { + #[debug("reply")] + reply: flume::Sender>, + }, + #[display("Replica({}, {})", _0.fmt_short(), _1)] + Replica(NamespaceId, ReplicaAction), + #[display("Shutdown")] + Shutdown, +} + +#[derive(derive_more::Debug, strum::Display)] +enum ReplicaAction { + Open { + #[debug("reply")] + reply: oneshot::Sender>, + opts: OpenOpts, + }, + Close { + #[debug("reply")] + reply: oneshot::Sender>, + }, + GetState { + #[debug("reply")] + reply: oneshot::Sender>, + }, + SetSync { + sync: bool, + #[debug("reply")] + reply: oneshot::Sender>, + }, + Subscribe { + sender: flume::Sender, + #[debug("reply")] + reply: oneshot::Sender>, + }, + Unsubscribe { + sender: flume::Sender, + #[debug("reply")] + reply: oneshot::Sender>, + }, + InsertLocal { + author: AuthorId, + key: Bytes, + hash: Hash, + len: u64, + #[debug("reply")] + reply: oneshot::Sender>, + }, + DeletePrefix { + author: AuthorId, + key: Bytes, + #[debug("reply")] + reply: oneshot::Sender>, + }, + InsertRemote { + entry: SignedEntry, + from: PeerIdBytes, + content_status: ContentStatus, + #[debug("reply")] + reply: oneshot::Sender>, + }, + SyncInitialMessage { + #[debug("reply")] + reply: oneshot::Sender>>, + }, + SyncProcessMessage { + message: Message, + from: PeerIdBytes, + state: SyncOutcome, + #[debug("reply")] + reply: oneshot::Sender>, SyncOutcome)>>, + }, + GetSyncPeers { + #[debug("reply")] + reply: oneshot::Sender>>>, + }, + RegisterUsefulPeer { + peer: PeerIdBytes, + #[debug("reply")] + reply: oneshot::Sender>, + }, + GetOne { + author: AuthorId, + key: Bytes, + reply: oneshot::Sender>>, + }, + GetMany { + filter: GetFilter, + reply: flume::Sender>, + }, + DropReplica { + reply: oneshot::Sender>, + }, + ExportSecretKey { + reply: oneshot::Sender>, + }, +} + +/// The state for an open replica. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct OpenState { + /// Whether to accept sync requests for this replica. + pub sync: bool, + /// How many event subscriptions are open + pub subscribers: usize, + /// By how many handles the replica is currently held open + pub handles: usize, +} + +#[derive(Debug)] +struct OpenReplica { + replica: Replica, + handles: usize, + sync: bool, +} + +/// The [`SyncHandle`] is the handle to a thread that runs replica and store operations. +#[derive(Debug, Clone)] +pub struct SyncHandle { + tx: flume::Sender, +} + +/// Options when opening a replica. +#[derive(Debug, Default)] +pub struct OpenOpts { + /// Set to true to set sync state to true. + pub sync: bool, + /// Optionally subscribe to replica events. + pub subscribe: Option>, +} +impl OpenOpts { + /// Set sync state to true. + pub fn sync(mut self) -> Self { + self.sync = true; + self + } + /// Subscribe to replica events. + pub fn subscribe(mut self, subscribe: flume::Sender) -> Self { + self.subscribe = Some(subscribe); + self + } +} + +#[allow(missing_docs)] +impl SyncHandle { + /// Spawn a sync actor and return a handle. + pub fn spawn( + store: S, + content_status_callback: Option, + me: String, + ) -> SyncHandle { + const ACTION_CAP: usize = 1024; + let (action_tx, action_rx) = flume::bounded(ACTION_CAP); + let mut actor = Actor { + store, + states: Default::default(), + action_rx, + content_status_callback, + }; + std::thread::spawn(move || { + let span = error_span!("sync", %me); + let _enter = span.enter(); + + if let Err(err) = actor.run() { + error!("Sync actor closed with error: {err:?}"); + } + }); + SyncHandle { tx: action_tx } + } + + pub async fn open(&self, namespace: NamespaceId, opts: OpenOpts) -> Result<()> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::Open { reply, opts }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn close(&self, namespace: NamespaceId) -> Result { + let (reply, rx) = oneshot::channel(); + self.send_replica(namespace, ReplicaAction::Close { reply }) + .await?; + rx.await? + } + + pub async fn subscribe( + &self, + namespace: NamespaceId, + sender: flume::Sender, + ) -> Result<()> { + let (reply, rx) = oneshot::channel(); + self.send_replica(namespace, ReplicaAction::Subscribe { sender, reply }) + .await?; + rx.await? + } + + pub async fn unsubscribe( + &self, + namespace: NamespaceId, + sender: flume::Sender, + ) -> Result<()> { + let (reply, rx) = oneshot::channel(); + self.send_replica(namespace, ReplicaAction::Unsubscribe { sender, reply }) + .await?; + rx.await? + } + + pub async fn set_sync(&self, namespace: NamespaceId, sync: bool) -> Result<()> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::SetSync { sync, reply }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn insert_local( + &self, + namespace: NamespaceId, + author: AuthorId, + key: Bytes, + hash: Hash, + len: u64, + ) -> Result<()> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::InsertLocal { + author, + key, + hash, + len, + reply, + }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn delete_prefix( + &self, + namespace: NamespaceId, + author: AuthorId, + key: Bytes, + ) -> Result { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::DeletePrefix { author, key, reply }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn insert_remote( + &self, + namespace: NamespaceId, + entry: SignedEntry, + from: PeerIdBytes, + content_status: ContentStatus, + ) -> Result<()> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::InsertRemote { + entry, + from, + content_status, + reply, + }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn sync_initial_message( + &self, + namespace: NamespaceId, + ) -> Result> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::SyncInitialMessage { reply }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn sync_process_message( + &self, + namespace: NamespaceId, + message: Message, + from: PeerIdBytes, + state: SyncOutcome, + ) -> Result<(Option>, SyncOutcome)> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::SyncProcessMessage { + reply, + message, + from, + state, + }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn get_sync_peers(&self, namespace: NamespaceId) -> Result>> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::GetSyncPeers { reply }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn register_useful_peer( + &self, + namespace: NamespaceId, + peer: PeerIdBytes, + ) -> Result<()> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::RegisterUsefulPeer { reply, peer }; + self.send_replica(namespace, action).await?; + rx.await? + } + + // TODO: it would be great if this could be a sync method... + pub async fn get_many( + &self, + namespace: NamespaceId, + filter: GetFilter, + reply: flume::Sender>, + ) -> Result<()> { + let action = ReplicaAction::GetMany { filter, reply }; + self.send_replica(namespace, action).await?; + Ok(()) + } + + pub async fn get_one( + &self, + namespace: NamespaceId, + author: AuthorId, + key: Bytes, + ) -> Result> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::GetOne { author, key, reply }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn drop_replica(&self, namespace: NamespaceId) -> Result<()> { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::DropReplica { reply }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn export_secret_key(&self, namespace: NamespaceId) -> Result { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::ExportSecretKey { reply }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn get_state(&self, namespace: NamespaceId) -> Result { + let (reply, rx) = oneshot::channel(); + let action = ReplicaAction::GetState { reply }; + self.send_replica(namespace, action).await?; + rx.await? + } + + pub async fn shutdown(&self) { + self.send(Action::Shutdown).await.ok(); + } + + pub async fn list_authors(&self, reply: flume::Sender>) -> Result<()> { + self.send(Action::ListAuthors { reply }).await + } + + pub async fn list_replicas(&self, reply: flume::Sender>) -> Result<()> { + self.send(Action::ListReplicas { reply }).await + } + + pub async fn import_author(&self, author: Author) -> Result { + let (reply, rx) = oneshot::channel(); + self.send(Action::ImportAuthor { author, reply }).await?; + rx.await? + } + + pub async fn import_namespace(&self, namespace: Namespace) -> Result { + let (reply, rx) = oneshot::channel(); + self.send(Action::ImportNamespace { namespace, reply }) + .await?; + rx.await? + } + + async fn send(&self, action: Action) -> Result<()> { + self.tx + .send_async(action) + .await + .context("sending to iroh_sync actor failed")?; + Ok(()) + } + async fn send_replica(&self, namespace: NamespaceId, action: ReplicaAction) -> Result<()> { + self.send(Action::Replica(namespace, action)).await?; + Ok(()) + } +} + +struct Actor { + store: S, + states: OpenReplicas, + action_rx: flume::Receiver, + content_status_callback: Option, +} + +impl Actor { + fn run(&mut self) -> Result<()> { + while let Ok(action) = self.action_rx.recv() { + trace!(%action, "tick"); + let is_shutdown = matches!(action, Action::Shutdown); + if self.on_action(action).is_err() { + warn!("failed to send reply: receiver dropped"); + } + if is_shutdown { + break; + } + } + trace!("shutdown"); + Ok(()) + } + + fn on_action(&mut self, action: Action) -> Result<(), SendReplyError> { + match action { + Action::Shutdown => { + self.close_all(); + Ok(()) + } + Action::ImportAuthor { author, reply } => { + let id = author.id(); + send_reply(reply, self.store.import_author(author).map(|_| id)) + } + Action::ImportNamespace { namespace, reply } => { + let id = namespace.id(); + send_reply(reply, self.store.import_namespace(namespace).map(|_| id)) + } + Action::ListAuthors { reply } => iter_to_channel( + reply, + self.store + .list_authors() + .map(|a| a.map(|a| a.map(|a| a.id()))), + ), + Action::ListReplicas { reply } => iter_to_channel(reply, self.store.list_namespaces()), + Action::Replica(namespace, action) => self.on_replica_action(namespace, action), + } + } + + fn on_replica_action( + &mut self, + namespace: NamespaceId, + action: ReplicaAction, + ) -> Result<(), SendReplyError> { + match action { + ReplicaAction::Open { reply, opts } => { + let res = self.open(namespace, opts); + send_reply(reply, res) + } + ReplicaAction::Close { reply } => { + let res = self.close(namespace); + // ignore errors when no receiver is present for close + reply.send(Ok(res)).ok(); + Ok(()) + } + ReplicaAction::Subscribe { sender, reply } => send_reply_with(reply, self, |this| { + let replica = this.states.replica(&namespace)?; + replica.subscribe(sender); + Ok(()) + }), + ReplicaAction::Unsubscribe { sender, reply } => send_reply_with(reply, self, |this| { + let replica = this.states.replica(&namespace)?; + replica.unsubscribe(&sender); + drop(sender); + Ok(()) + }), + ReplicaAction::SetSync { sync, reply } => send_reply_with(reply, self, |this| { + let state = this.states.get_mut(&namespace)?; + state.sync = sync; + Ok(()) + }), + ReplicaAction::InsertLocal { + author, + key, + hash, + len, + reply, + } => send_reply_with(reply, self, |this| { + let author = get_author(&this.store, &author)?; + let replica = this.states.replica(&namespace)?; + replica.insert(&key, &author, hash, len)?; + Ok(()) + }), + ReplicaAction::DeletePrefix { author, key, reply } => { + send_reply_with(reply, self, |this| { + let author = get_author(&this.store, &author)?; + let replica = this.states.replica(&namespace)?; + let res = replica.delete_prefix(&key, &author)?; + Ok(res) + }) + } + ReplicaAction::InsertRemote { + entry, + from, + content_status, + reply, + } => send_reply_with(reply, self, move |this| { + let replica = this.states.replica_if_syncing(&namespace)?; + replica.insert_remote_entry(entry, from, content_status)?; + Ok(()) + }), + + ReplicaAction::SyncInitialMessage { reply } => { + send_reply_with(reply, self, move |this| { + let replica = this.states.replica_if_syncing(&namespace)?; + let res = replica.sync_initial_message()?; + Ok(res) + }) + } + ReplicaAction::SyncProcessMessage { + message, + from, + mut state, + reply, + } => send_reply_with(reply, self, move |this| { + let replica = this.states.replica_if_syncing(&namespace)?; + let res = replica.sync_process_message(message, from, &mut state)?; + Ok((res, state)) + }), + ReplicaAction::GetSyncPeers { reply } => send_reply_with(reply, self, move |this| { + this.states.ensure_open(&namespace)?; + let peers = this.store.get_sync_peers(&namespace)?; + Ok(peers.map(|iter| iter.collect())) + }), + ReplicaAction::RegisterUsefulPeer { peer, reply } => { + let res = self.store.register_useful_peer(namespace, peer); + send_reply(reply, res) + } + ReplicaAction::GetOne { author, key, reply } => { + send_reply_with(reply, self, move |this| { + this.states.ensure_open(&namespace)?; + this.store.get_one(namespace, author, key) + }) + } + ReplicaAction::GetMany { filter, reply } => { + let iter = self + .states + .ensure_open(&namespace) + .and_then(|_| self.store.get_many(namespace, filter)); + iter_to_channel(reply, iter) + } + ReplicaAction::DropReplica { reply } => send_reply_with(reply, self, |this| { + this.close(namespace); + this.store.remove_replica(&namespace) + }), + ReplicaAction::ExportSecretKey { reply } => { + let res = self.states.replica(&namespace).map(|r| r.secret_key()); + send_reply(reply, res) + } + ReplicaAction::GetState { reply } => send_reply_with(reply, self, move |this| { + let state = this.states.get_mut(&namespace)?; + Ok(OpenState { + handles: state.handles, + sync: state.sync, + subscribers: state.replica.subscribers_count(), + }) + }), + } + } + + fn close(&mut self, namespace: NamespaceId) -> bool { + let on_close_cb = |replica| self.store.close_replica(replica); + self.states.close_with(namespace, on_close_cb) + } + + fn close_all(&mut self) { + let on_close_cb = |replica| self.store.close_replica(replica); + self.states.close_all_with(on_close_cb); + } + + fn open(&mut self, namespace: NamespaceId, opts: OpenOpts) -> Result<()> { + let open_cb = || { + let mut replica = self.store.open_replica(&namespace)?; + if let Some(cb) = &self.content_status_callback { + replica.set_content_status_callback(Arc::clone(cb)); + } + Ok(replica) + }; + self.states.open_with(namespace, opts, open_cb) + } +} + +struct OpenReplicas(HashMap>); + +// We need a manual impl here because the derive won't work unless we'd restrict to S: Default. +impl Default for OpenReplicas { + fn default() -> Self { + Self(Default::default()) + } +} +impl OpenReplicas { + fn replica(&mut self, namespace: &NamespaceId) -> Result<&mut Replica> { + self.get_mut(namespace).map(|state| &mut state.replica) + } + + fn get_mut(&mut self, namespace: &NamespaceId) -> Result<&mut OpenReplica> { + self.0.get_mut(namespace).context("replica not open") + } + + fn replica_if_syncing(&mut self, namespace: &NamespaceId) -> Result<&mut Replica> { + let state = self.get_mut(namespace)?; + if !state.sync { + Err(anyhow!("sync is not enabled for replica")) + } else { + Ok(&mut state.replica) + } + } + + fn is_open(&self, namespace: &NamespaceId) -> bool { + self.0.contains_key(namespace) + } + + fn ensure_open(&self, namespace: &NamespaceId) -> Result<()> { + match self.is_open(namespace) { + true => Ok(()), + false => Err(anyhow!("replica not open")), + } + } + fn open_with( + &mut self, + namespace: NamespaceId, + opts: OpenOpts, + open_cb: impl Fn() -> Result>, + ) -> Result<()> { + match self.0.entry(namespace) { + hash_map::Entry::Vacant(e) => { + let mut replica = open_cb()?; + if let Some(sender) = opts.subscribe { + replica.subscribe(sender); + } + debug!(namespace = %namespace.fmt_short(), "open"); + let state = OpenReplica { + replica, + sync: opts.sync, + handles: 1, + }; + e.insert(state); + } + hash_map::Entry::Occupied(mut e) => { + let state = e.get_mut(); + state.handles += 1; + state.sync = state.sync || opts.sync; + if let Some(sender) = opts.subscribe { + state.replica.subscribe(sender); + } + } + } + Ok(()) + } + fn close_with( + &mut self, + namespace: NamespaceId, + on_close: impl Fn(Replica), + ) -> bool { + match self.0.entry(namespace) { + hash_map::Entry::Vacant(_e) => { + warn!(namespace = %namespace.fmt_short(), "received close request for closed replica"); + true + } + hash_map::Entry::Occupied(mut e) => { + let state = e.get_mut(); + state.handles = state.handles.wrapping_sub(1); + if state.handles == 0 { + let (_, state) = e.remove_entry(); + debug!(namespace = %namespace.fmt_short(), "close"); + on_close(state.replica); + true + } else { + false + } + } + } + } + + fn close_all_with(&mut self, on_close: impl Fn(Replica)) { + for (_namespace, state) in self.0.drain() { + on_close(state.replica) + } + } +} + +fn iter_to_channel( + channel: flume::Sender>, + iter: Result>>, +) -> Result<(), SendReplyError> { + match iter { + Err(err) => channel.send(Err(err)).map_err(send_reply_error)?, + Ok(iter) => { + for item in iter { + channel.send(item).map_err(send_reply_error)?; + } + } + } + Ok(()) +} + +fn get_author(store: &S, id: &AuthorId) -> Result { + store.get_author(id)?.context("author not found") +} + +#[derive(Debug)] +struct SendReplyError; + +fn send_reply(sender: oneshot::Sender, value: T) -> Result<(), SendReplyError> { + sender.send(value).map_err(send_reply_error) +} + +fn send_reply_with( + sender: oneshot::Sender>, + this: &mut Actor, + f: impl FnOnce(&mut Actor) -> Result, +) -> Result<(), SendReplyError> { + sender.send(f(this)).map_err(send_reply_error) +} + +fn send_reply_error(_err: T) -> SendReplyError { + SendReplyError +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn open_close() -> anyhow::Result<()> { + let store = store::memory::Store::default(); + let sync = SyncHandle::spawn(store, None, "foo".into()); + let namespace = Namespace::new(&mut rand::rngs::OsRng {}); + sync.import_namespace(namespace.clone()).await?; + sync.open(namespace.id(), Default::default()).await?; + let (tx, rx) = flume::bounded(10); + sync.subscribe(namespace.id(), tx).await?; + sync.close(namespace.id()).await?; + assert!(rx.recv_async().await.is_err()); + Ok(()) + } +} diff --git a/iroh-sync/src/keys.rs b/iroh-sync/src/keys.rs index d58d723d2e..7e2067097b 100644 --- a/iroh-sync/src/keys.rs +++ b/iroh-sync/src/keys.rs @@ -338,7 +338,6 @@ pub(super) mod base32 { let len = bytes.as_ref().len().min(10); let mut text = data_encoding::BASE32_NOPAD.encode(&bytes.as_ref()[..len]); text.make_ascii_lowercase(); - text.push('…'); text } /// Parse from a base32 string into a byte array @@ -413,6 +412,12 @@ impl AuthorId { pub fn into_public_key(&self) -> Result { AuthorPublicKey::from_bytes(&self.0) } + + /// Convert to a base32 string limited to the first 10 bytes for a friendly string + /// representation of the key. + pub fn fmt_short(&self) -> String { + base32::fmt_short(self.0) + } } impl NamespaceId { @@ -442,6 +447,12 @@ impl NamespaceId { pub fn into_public_key(&self) -> Result { NamespacePublicKey::from_bytes(&self.0) } + + /// Convert to a base32 string limited to the first 10 bytes for a friendly string + /// representation of the key. + pub fn fmt_short(&self) -> String { + base32::fmt_short(self.0) + } } impl From<&[u8; 32]> for NamespaceId { diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index 391b6b09b7..285c1203c1 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -29,6 +29,7 @@ //! [paper]: https://arxiv.org/abs/2212.13567 #![deny(missing_docs, rustdoc::broken_intra_doc_links)] +pub mod actor; mod keys; #[cfg(feature = "metrics")] pub mod metrics; diff --git a/iroh-sync/src/net.rs b/iroh-sync/src/net.rs index 1d81f9f941..6e2c10e2ec 100644 --- a/iroh-sync/src/net.rs +++ b/iroh-sync/src/net.rs @@ -1,16 +1,18 @@ //! Network implementation of the iroh-sync protocol -use std::future::Future; +use std::{ + future::Future, + time::{Duration, Instant}, +}; use iroh_net::{key::PublicKey, magic_endpoint::get_peer_id, MagicEndpoint, PeerAddr}; use serde::{Deserialize, Serialize}; -use tracing::debug; +use tracing::{debug, error_span, trace, Instrument}; use crate::{ - net::codec::{run_alice, run_bob}, - store, - sync::Replica, - NamespaceId, + actor::SyncHandle, + net::codec::{run_alice, BobState}, + NamespaceId, SyncOutcome, }; #[cfg(feature = "metrics")] @@ -24,22 +26,27 @@ pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; mod codec; /// Connect to a peer and sync a replica -pub async fn connect_and_sync( +pub async fn connect_and_sync( endpoint: &MagicEndpoint, - doc: &Replica, + sync: &SyncHandle, + namespace: NamespaceId, peer: PeerAddr, -) -> Result<(), ConnectError> { +) -> Result { + let t_start = Instant::now(); let peer_id = peer.peer_id; - debug!(?peer_id, "sync[dial]: connect"); - let namespace = doc.namespace(); + trace!("connect"); let connection = endpoint .connect(peer, SYNC_ALPN) .await .map_err(ConnectError::connect)?; - debug!(?peer_id, ?namespace, "sync[dial]: connected"); + let (mut send_stream, mut recv_stream) = connection.open_bi().await.map_err(ConnectError::connect)?; - let res = run_alice::(&mut send_stream, &mut recv_stream, doc, peer_id).await; + + let t_connect = t_start.elapsed(); + debug!(?t_connect, "connected"); + + let res = run_alice(&mut send_stream, &mut recv_stream, sync, namespace, peer_id).await; send_stream.finish().await.map_err(ConnectError::close)?; recv_stream @@ -54,23 +61,59 @@ pub async fn connect_and_sync( inc!(Metrics, sync_via_connect_failure); } - debug!(?peer_id, ?namespace, ?res, "sync[dial]: done"); - res + let t_process = t_start.elapsed() - t_connect; + match &res { + Ok(res) => { + debug!( + ?t_connect, + ?t_process, + sent = %res.num_sent, + recv = %res.num_recv, + "done, ok" + ); + } + Err(err) => { + debug!(?t_connect, ?t_process, ?err, "done, failed"); + } + } + + let outcome = res?; + + let timings = Timings { + connect: t_connect, + process: t_process, + }; + + let res = SyncFinished { + namespace, + peer: peer_id, + outcome, + timings, + }; + + Ok(res) } -/// What to do with incoming sync requests -pub type AcceptOutcome = Result::Instance>, AbortReason>; +/// Whether we want to accept or reject an incoming sync request. +#[derive(Debug, Clone)] +pub enum AcceptOutcome { + /// Accept the sync request. + Allow, + /// Decline the sync request + Reject(AbortReason), +} /// Handle an iroh-sync connection and sync all shared documents in the replica store. -pub async fn handle_connection( +pub async fn handle_connection( + sync: SyncHandle, connecting: quinn::Connecting, accept_cb: F, -) -> Result<(NamespaceId, PublicKey), AcceptError> +) -> Result where - S: store::Store, F: Fn(NamespaceId, PublicKey) -> Fut, - Fut: Future>>, + Fut: Future, { + let t_start = Instant::now(); let connection = connecting.await.map_err(AcceptError::connect)?; let peer = get_peer_id(&connection) .await @@ -79,9 +122,18 @@ where .accept_bi() .await .map_err(|e| AcceptError::open(peer, e))?; - debug!(?peer, "sync[accept]: handle"); - let res = run_bob::(&mut send_stream, &mut recv_stream, accept_cb, peer).await; + let t_connect = t_start.elapsed(); + let span = error_span!("accept", peer = %peer.fmt_short(), namespace = tracing::field::Empty); + span.in_scope(|| { + debug!(?t_connect, "connection established"); + }); + + let mut state = BobState::new(peer); + let res = state + .run(&mut send_stream, &mut recv_stream, sync, accept_cb) + .instrument(span.clone()) + .await; #[cfg(feature = "metrics")] if res.is_ok() { @@ -90,10 +142,8 @@ where inc!(Metrics, sync_via_accept_failure); } - let namespace = match &res { - Ok(namespace) => Some(*namespace), - Err(err) => err.namespace(), - }; + let namespace = state.namespace(); + let outcome = state.into_outcome(); send_stream .finish() @@ -103,11 +153,59 @@ where .read_to_end(0) .await .map_err(|error| AcceptError::close(peer, namespace, error))?; + + let t_process = t_start.elapsed() - t_connect; + span.in_scope(|| match &res { + Ok(_res) => { + debug!( + ?t_connect, + ?t_process, + sent = %outcome.num_sent, + recv = %outcome.num_recv, + "done, ok" + ); + } + Err(err) => { + debug!(?t_connect, ?t_process, ?err, "done, failed"); + } + }); + let namespace = res?; - debug!(?peer, ?namespace, "sync[accept]: done"); + let timings = Timings { + connect: t_connect, + process: t_process, + }; + let res = SyncFinished { + namespace, + outcome, + peer, + timings, + }; + + Ok(res) +} + +/// Details of a finished sync operation. +#[derive(Debug, Clone)] +pub struct SyncFinished { + /// The namespace that was synced. + pub namespace: NamespaceId, + /// The peer we syned with. + pub peer: PublicKey, + /// The outcome of the sync operation + pub outcome: SyncOutcome, + /// The time this operation took + pub timings: Timings, +} - Ok((namespace, peer)) +/// Time a sync operation took +#[derive(Debug, Default, Clone)] +pub struct Timings { + /// Time to establish connection + pub connect: Duration, + /// Time to run sync exchange + pub process: Duration, } /// Errors that may occur on handling incoming sync connections. @@ -165,9 +263,6 @@ pub enum ConnectError { /// The remote peer aborted the sync request. #[error("Remote peer aborted sync: {0:?}")] RemoteAbort(AbortReason), - /// We cancelled the operation - #[error("Cancelled")] - Cancelled, /// Failed to run sync #[error("Failed to sync")] Sync { @@ -186,9 +281,11 @@ pub enum ConnectError { #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] pub enum AbortReason { /// Namespace is not avaiable. - NotAvailable, + NotFound, /// We are already syncing this namespace. AlreadySyncing, + /// We experienced an error while trying to provide the requested resource + InternalServerError, } impl AcceptError { diff --git a/iroh-sync/src/net/codec.rs b/iroh-sync/src/net/codec.rs index d085b71652..8fae754079 100644 --- a/iroh-sync/src/net/codec.rs +++ b/iroh-sync/src/net/codec.rs @@ -8,11 +8,12 @@ use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_stream::StreamExt; use tokio_util::codec::{Decoder, Encoder, FramedRead, FramedWrite}; -use tracing::trace; +use tracing::{debug, trace, Span}; use crate::{ + actor::SyncHandle, net::{AbortReason, AcceptError, AcceptOutcome, ConnectError}, - store, NamespaceId, Replica, + NamespaceId, SyncOutcome, }; #[derive(Debug, Default)] @@ -88,23 +89,27 @@ enum Message { } /// Runs the initiator side of the sync protocol. -pub(super) async fn run_alice( +pub(super) async fn run_alice( writer: &mut W, reader: &mut R, - alice: &Replica, - other_peer_id: PublicKey, -) -> Result<(), ConnectError> { - let other_peer_id = *other_peer_id.as_bytes(); + handle: &SyncHandle, + namespace: NamespaceId, + peer: PublicKey, +) -> Result { + let peer_bytes = *peer.as_bytes(); let mut reader = FramedRead::new(reader, SyncCodec); let mut writer = FramedWrite::new(writer, SyncCodec); + let mut progress = Some(SyncOutcome::default()); + // Init message - let init_message = Message::Init { - namespace: alice.namespace(), - message: alice.sync_initial_message().map_err(ConnectError::sync)?, - }; - trace!("alice -> bob: {:#?}", init_message); + let message = handle + .sync_initial_message(namespace) + .await + .map_err(ConnectError::sync)?; + let init_message = Message::Init { namespace, message }; + trace!("send init message"); writer .send(init_message) .await @@ -118,11 +123,15 @@ pub(super) async fn run_alice { - if let Some(msg) = alice - .sync_process_message(msg, other_peer_id) - .map_err(ConnectError::sync)? - { - trace!("alice -> bob: {:#?}", msg); + trace!("recv process message"); + let current_progress = progress.take().unwrap(); + let (reply, next_progress) = handle + .sync_process_message(namespace, msg, peer_bytes, current_progress) + .await + .map_err(ConnectError::sync)?; + progress = Some(next_progress); + if let Some(msg) = reply { + trace!("send process message"); writer .send(Message::Sync(msg)) .await @@ -137,67 +146,81 @@ pub(super) async fn run_alice( +#[cfg(test)] +pub(super) async fn run_bob( writer: &mut W, reader: &mut R, + handle: SyncHandle, accept_cb: F, - other_peer_id: PublicKey, -) -> Result + peer: PublicKey, +) -> Result<(NamespaceId, SyncOutcome), AcceptError> where - S: store::Store, R: AsyncRead + Unpin, W: AsyncWrite + Unpin, F: Fn(NamespaceId, PublicKey) -> Fut, - Fut: Future>>, + Fut: Future, { - let mut state = BobState::::new(other_peer_id); - state.run(writer, reader, accept_cb).await + let mut state = BobState::new(peer); + let namespace = state.run(writer, reader, handle, accept_cb).await?; + Ok((namespace, state.into_outcome())) } -struct BobState { - replica: Option>, +/// State for the receiver side of the sync protocol. +pub struct BobState { + namespace: Option, peer: PublicKey, + progress: Option, } -impl BobState { +impl BobState { + /// Create a new state for a single connection. pub fn new(peer: PublicKey) -> Self { Self { peer, - replica: None, + namespace: None, + progress: Some(Default::default()), } } - pub fn fail(&self, reason: impl Into) -> AcceptError { + fn fail(&self, reason: impl Into) -> AcceptError { AcceptError::sync(self.peer, self.namespace(), reason.into()) } - async fn run( + /// Handle connection and run to end. + pub async fn run( &mut self, writer: W, reader: R, + sync: SyncHandle, accept_cb: F, ) -> Result where R: AsyncRead + Unpin, W: AsyncWrite + Unpin, F: Fn(NamespaceId, PublicKey) -> Fut, - Fut: Future>>, + Fut: Future, { let mut reader = FramedRead::new(reader, SyncCodec); let mut writer = FramedWrite::new(writer, SyncCodec); while let Some(msg) = reader.next().await { let msg = msg.map_err(|e| self.fail(e))?; - let next = match (msg, self.replica.as_ref()) { + let next = match (msg, self.namespace.as_ref()) { (Message::Init { namespace, message }, None) => { + Span::current() + .record("namespace", tracing::field::display(&namespace.fmt_short())); + trace!("recv init message"); let accept = accept_cb(namespace, self.peer).await; - let accept = accept.map_err(|e| self.fail(e))?; - let replica = match accept { - Ok(replica) => replica, - Err(reason) => { + match accept { + AcceptOutcome::Allow => { + trace!("allow request"); + } + AcceptOutcome::Reject(reason) => { + debug!(?reason, "reject request"); writer .send(Message::Abort { reason }) .await @@ -208,15 +231,24 @@ impl BobState { reason, }); } - }; - trace!(?namespace, peer = ?self.peer, "run_bob: recv initial message {message:#?}"); - let next = replica.sync_process_message(message, *self.peer.as_bytes()); - self.replica = Some(replica); + } + let last_progress = self.progress.take().unwrap(); + let next = sync + .sync_process_message( + namespace, + message, + *self.peer.as_bytes(), + last_progress, + ) + .await; + self.namespace = Some(namespace); next } - (Message::Sync(msg), Some(replica)) => { - trace!(namespace = ?replica.namespace(), peer = ?self.peer, "run_bob: recv {msg:#?}"); - replica.sync_process_message(msg, *self.peer.as_bytes()) + (Message::Sync(msg), Some(namespace)) => { + trace!("recv process message"); + let last_progress = self.progress.take().unwrap(); + sync.sync_process_message(*namespace, msg, *self.peer.as_bytes(), last_progress) + .await } (Message::Init { .. }, Some(_)) => { return Err(self.fail(anyhow!("double init message"))) @@ -224,14 +256,15 @@ impl BobState { (Message::Sync(_), None) => { return Err(self.fail(anyhow!("unexpected sync message before init"))) } - (Message::Abort { reason }, _) => { - return Err(self.fail(anyhow!("unexpected abort message ({reason:?})"))) + (Message::Abort { .. }, _) => { + return Err(self.fail(anyhow!("unexpected sync abort message"))) } }; - let next = next.map_err(|e| self.fail(e))?; - match next { + let (reply, progress) = next.map_err(|e| self.fail(e))?; + self.progress = Some(progress); + match reply { Some(msg) => { - trace!(namespace = ?self.namespace(), peer = ?self.peer, "run_bob: send {msg:#?}"); + trace!("send process message"); writer .send(Message::Sync(msg)) .await @@ -241,21 +274,28 @@ impl BobState { } } - trace!(namespace = ?self.namespace().unwrap(), peer = ?self.peer, "run_bob: finished"); + trace!("done"); self.namespace() .ok_or_else(|| self.fail(anyhow!("Stream closed before init message"))) } - fn namespace(&self) -> Option { - self.replica.as_ref().map(|r| r.namespace()).to_owned() + /// Get the namespace that is synced, if available. + pub fn namespace(&self) -> Option { + self.namespace + } + + /// Consume self and get the [`SyncOutcome`] for this connection. + pub fn into_outcome(self) -> SyncOutcome { + self.progress.unwrap() } } #[cfg(test)] mod tests { use crate::{ - store::{GetFilter, Store}, + actor::OpenOpts, + store::{self, GetFilter, Store}, sync::Namespace, AuthorId, }; @@ -272,25 +312,25 @@ mod tests { let alice_peer_id = SecretKey::from_bytes(&[1u8; 32]).public(); let bob_peer_id = SecretKey::from_bytes(&[2u8; 32]).public(); - let alice_replica_store = store::memory::Store::default(); + let alice_store = store::memory::Store::default(); // For now uses same author on both sides. - let author = alice_replica_store.new_author(&mut rng).unwrap(); + let author = alice_store.new_author(&mut rng).unwrap(); let namespace = Namespace::new(&mut rng); - let alice_replica = alice_replica_store.new_replica(namespace.clone()).unwrap(); + let mut alice_replica = alice_store.new_replica(namespace.clone()).unwrap(); alice_replica .hash_and_insert("hello bob", &author, "from alice") .unwrap(); - let bob_replica_store = store::memory::Store::default(); - let bob_replica = bob_replica_store.new_replica(namespace.clone()).unwrap(); + let bob_store = store::memory::Store::default(); + let mut bob_replica = bob_store.new_replica(namespace.clone()).unwrap(); bob_replica .hash_and_insert("hello alice", &author, "from bob") .unwrap(); assert_eq!( - bob_replica_store + bob_store .get_many(bob_replica.namespace(), GetFilter::All) .unwrap() .collect::>>() @@ -299,7 +339,7 @@ mod tests { 1 ); assert_eq!( - alice_replica_store + alice_store .get_many(alice_replica.namespace(), GetFilter::All) .unwrap() .collect::>>() @@ -308,33 +348,42 @@ mod tests { 1 ); + // close the replicas because now the async actor will take over + alice_store.close_replica(alice_replica); + bob_store.close_replica(bob_replica); + let (alice, bob) = tokio::io::duplex(64); let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); - let replica = alice_replica.clone(); + let alice_handle = SyncHandle::spawn(alice_store.clone(), None, "alice".to_string()); + alice_handle + .open(namespace.id(), OpenOpts::default().sync()) + .await?; + let namespace_id = namespace.id(); + let alice_handle2 = alice_handle.clone(); let alice_task = tokio::task::spawn(async move { - run_alice::( + run_alice( &mut alice_writer, &mut alice_reader, - &replica, + &alice_handle2, + namespace_id, bob_peer_id, ) .await }); let (mut bob_reader, mut bob_writer) = tokio::io::split(bob); - let bob_replica_store_task = bob_replica_store.clone(); + let bob_handle = SyncHandle::spawn(bob_store.clone(), None, "bob".to_string()); + bob_handle + .open(namespace.id(), OpenOpts::default().sync()) + .await?; + let bob_handle2 = bob_handle.clone(); let bob_task = tokio::task::spawn(async move { - run_bob::( + run_bob( &mut bob_writer, &mut bob_reader, - |namespace, _| { - futures::future::ready( - bob_replica_store_task - .open_replica(&namespace) - .map(|r| r.ok_or(AbortReason::NotAvailable)), - ) - }, + bob_handle2, + |_namespace, _peer| futures::future::ready(AcceptOutcome::Allow), alice_peer_id, ) .await @@ -343,9 +392,12 @@ mod tests { alice_task.await??; bob_task.await??; + alice_handle.shutdown().await; + bob_handle.shutdown().await; + assert_eq!( - bob_replica_store - .get_many(bob_replica.namespace(), GetFilter::All) + bob_store + .get_many(namespace.id(), GetFilter::All) .unwrap() .collect::>>() .unwrap() @@ -353,8 +405,8 @@ mod tests { 2 ); assert_eq!( - alice_replica_store - .get_many(alice_replica.namespace(), GetFilter::All) + alice_store + .get_many(namespace.id(), GetFilter::All) .unwrap() .collect::>>() .unwrap() @@ -387,7 +439,7 @@ mod tests { fn insert_messages( mut rng: impl CryptoRngCore, store: &S, - replica: &Replica, + replica: &mut crate::sync::Replica, num_authors: usize, msgs_per_author: usize, key_value_fn: impl Fn(&AuthorId, usize) -> (String, String), @@ -430,8 +482,8 @@ mod tests { async fn test_sync_many_authors(alice_store: S, bob_store: S) -> Result<()> { let num_messages = &[1, 2, 5, 10]; let num_authors = &[2, 3, 4, 5, 10]; - let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(99); + for num_messages in num_messages { for num_authors in num_authors { println!( @@ -444,12 +496,11 @@ mod tests { let mut all_messages = vec![]; - let alice_replica = alice_store.new_replica(namespace.clone()).unwrap(); - + let mut alice_replica = alice_store.new_replica(namespace.clone()).unwrap(); let alice_messages = insert_messages( &mut rng, &alice_store, - &alice_replica, + &mut alice_replica, *num_authors, *num_messages, |author, i| { @@ -461,11 +512,11 @@ mod tests { ); all_messages.extend_from_slice(&alice_messages); - let bob_replica = bob_store.new_replica(namespace.clone()).unwrap(); + let mut bob_replica = bob_store.new_replica(namespace.clone()).unwrap(); let bob_messages = insert_messages( &mut rng, &bob_store, - &bob_replica, + &mut bob_replica, *num_authors, *num_messages, |author, i| { @@ -479,67 +530,80 @@ mod tests { all_messages.sort(); - let res = get_messages(&alice_store, alice_replica.namespace()); + let res = get_messages(&alice_store, namespace.id()); assert_eq!(res, alice_messages); - let res = get_messages(&bob_store, bob_replica.namespace()); + let res = get_messages(&bob_store, namespace.id()); assert_eq!(res, bob_messages); + // replicas can be opened only once so close the replicas before spawning the + // actors + alice_store.close_replica(alice_replica); + let alice_handle = + SyncHandle::spawn(alice_store.clone(), None, "alice".to_string()); + + bob_store.close_replica(bob_replica); + let bob_handle = SyncHandle::spawn(bob_store.clone(), None, "bob".to_string()); + run_sync( - &alice_store, + alice_handle.clone(), alice_node_pubkey, - &bob_store, + bob_handle.clone(), bob_node_pubkey, namespace.id(), ) .await?; - let res = get_messages(&bob_store, bob_replica.namespace()); + let res = get_messages(&bob_store, namespace.id()); assert_eq!(res.len(), all_messages.len()); assert_eq!(res, all_messages); - let res = get_messages(&bob_store, bob_replica.namespace()); + let res = get_messages(&bob_store, namespace.id()); assert_eq!(res.len(), all_messages.len()); assert_eq!(res, all_messages); + + alice_handle.shutdown().await; + bob_handle.shutdown().await; } } + Ok(()) } - async fn run_sync( - alice_store: &S, + async fn run_sync( + alice_handle: SyncHandle, alice_node_pubkey: PublicKey, - bob_store: &S, + bob_handle: SyncHandle, bob_node_pubkey: PublicKey, namespace: NamespaceId, ) -> Result<()> { + alice_handle + .open(namespace, OpenOpts::default().sync()) + .await?; + bob_handle + .open(namespace, OpenOpts::default().sync()) + .await?; let (alice, bob) = tokio::io::duplex(1024); let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); - let alice_replica = alice_store.open_replica(&namespace)?.unwrap(); let alice_task = tokio::task::spawn(async move { - run_alice::( + run_alice( &mut alice_writer, &mut alice_reader, - &alice_replica, + &alice_handle, + namespace, bob_node_pubkey, ) .await }); let (mut bob_reader, mut bob_writer) = tokio::io::split(bob); - let bob_store = bob_store.clone(); let bob_task = tokio::task::spawn(async move { - run_bob::( + run_bob( &mut bob_writer, &mut bob_reader, - |namespace, _| { - futures::future::ready( - bob_store - .open_replica(&namespace) - .map(|r| r.ok_or(AbortReason::NotAvailable)), - ) - }, + bob_handle, + |_namespace, _peer| futures::future::ready(AcceptOutcome::Allow), alice_node_pubkey, ) .await @@ -572,8 +636,8 @@ mod tests { let alice_node_pubkey = SecretKey::generate_with_rng(&mut rng).public(); let bob_node_pubkey = SecretKey::generate_with_rng(&mut rng).public(); let namespace = Namespace::new(&mut rng); - let alice_replica = alice_store.new_replica(namespace.clone()).unwrap(); - let bob_replica = bob_store.new_replica(namespace.clone()).unwrap(); + let mut alice_replica = alice_store.new_replica(namespace.clone()).unwrap(); + let mut bob_replica = bob_store.new_replica(namespace.clone()).unwrap(); let author = alice_store.new_author(&mut rng)?; bob_store.import_author(author.clone())?; @@ -600,25 +664,34 @@ mod tests { vec![(author.id(), key.clone(), hash_bob)] ); + alice_store.close_replica(alice_replica); + bob_store.close_replica(bob_replica); + + let alice_handle = SyncHandle::spawn(alice_store.clone(), None, "alice".to_string()); + let bob_handle = SyncHandle::spawn(bob_store.clone(), None, "bob".to_string()); + run_sync( - &alice_store, + alice_handle.clone(), alice_node_pubkey, - &bob_store, + bob_handle.clone(), bob_node_pubkey, namespace.id(), ) .await?; assert_eq!( - get_messages(&alice_store, alice_replica.namespace()), + get_messages(&alice_store, namespace.id()), vec![(author.id(), key.clone(), hash_bob)] ); assert_eq!( - get_messages(&bob_store, bob_replica.namespace()), + get_messages(&bob_store, namespace.id()), vec![(author.id(), key.clone(), hash_bob)] ); + alice_handle.shutdown().await; + bob_handle.shutdown().await; + Ok(()) } } diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs index 446b7613bc..ef6c791670 100644 --- a/iroh-sync/src/ranger.rs +++ b/iroh-sync/src/ranger.rs @@ -212,6 +212,14 @@ impl Message { pub fn parts(&self) -> &[MessagePart] { &self.parts } + + pub fn values(&self) -> impl Iterator { + self.parts().iter().filter_map(|p| p.values()).flatten() + } + + pub fn value_count(&self) -> usize { + self.values().count() + } } pub trait Store: Sized { @@ -327,17 +335,23 @@ where /// It must return true if the entry is valid and should be stored, and false otherwise /// (which means the entry will be dropped and not stored). /// + /// `on_insert_cb` is called for each entry that was actually inserted into the store (so not + /// for entries which validated, but are not inserted because they are older than one of their + /// prefixes). + /// /// `content_status_cb` is called for each outgoing entry about to be sent to the remote. /// It must return a [`ContentStatus`], which will be sent to the remote with the entry. - pub fn process_message( + pub fn process_message( &mut self, message: Message, validate_cb: F, - content_status_cb: F2, + mut on_insert_cb: F2, + content_status_cb: F3, ) -> Result>, S::Error> where F: Fn(&S, &E, ContentStatus) -> bool, - F2: Fn(&S, &E) -> ContentStatus, + F2: FnMut(&S, E, ContentStatus), + F3: Fn(&S, &E) -> ContentStatus, { let mut out = Vec::new(); @@ -397,7 +411,11 @@ where // Store incoming values for (entry, content_status) in values { if validate_cb(&self.store, &entry, content_status) { - self.put(entry)?; + // TODO: Get rid of the clone? + let outcome = self.put(entry.clone())?; + if let InsertOutcome::Inserted { .. } = outcome { + on_insert_cb(&self.store, entry, content_status); + } } } @@ -603,6 +621,7 @@ where } /// The outcome of a [`Store::put`] operation. +#[derive(Debug)] pub enum InsertOutcome { /// The entry was not inserted because a newer entry for its key or a /// prefix of its key exists. @@ -1333,12 +1352,22 @@ mod tests { alice_to_bob.push(msg.clone()); if let Some(msg) = bob - .process_message(msg, &bob_validate_cb, |_, _| ContentStatus::Complete) + .process_message( + msg, + &bob_validate_cb, + |_, _, _| (), + |_, _| ContentStatus::Complete, + ) .unwrap() { bob_to_alice.push(msg.clone()); next_to_bob = alice - .process_message(msg, &alice_validate_cb, |_, _| ContentStatus::Complete) + .process_message( + msg, + &alice_validate_cb, + |_, _, _| (), + |_, _| ContentStatus::Complete, + ) .unwrap(); } } diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index fd14a55227..32f615fc39 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -25,6 +25,20 @@ pub(crate) const PEERS_PER_DOC_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::ne None => panic!("this is clearly non zero"), }; +/// Error return from [`Store::open_replica`] +#[derive(Debug, thiserror::Error)] +pub enum OpenError { + /// The replica was already opened. + #[error("Replica is already open")] + AlreadyOpen, + /// The replica does not exist. + #[error("Replica not found")] + NotFound, + /// Other error while opening the replica. + #[error("{0}")] + Other(#[from] anyhow::Error), +} + /// Abstraction over the different available storage solutions. pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// The specialized instance scoped to a `Namespace`. @@ -56,7 +70,14 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { Self: 'a; /// Create a new replica for `namespace` and persist in this store. - fn new_replica(&self, namespace: Namespace) -> Result>; + fn new_replica(&self, namespace: Namespace) -> Result> { + let id = namespace.id(); + self.import_namespace(namespace)?; + self.open_replica(&id).map_err(Into::into) + } + + /// Import a new replica namespace. + fn import_namespace(&self, namespace: Namespace) -> Result<()>; /// List all replica namespaces in this store. fn list_namespaces(&self) -> Result>; @@ -65,18 +86,18 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// /// Store implementers must ensure that only a single instance of [`Replica`] is created per /// namespace. On subsequent calls, a clone of that singleton instance must be returned. - fn open_replica(&self, namespace: &NamespaceId) -> Result>>; + fn open_replica(&self, namespace: &NamespaceId) -> Result, OpenError>; /// Close a replica. - /// - /// This removes the event subscription from the replica, if active, and removes the replica - /// instance from the store's cache. - fn close_replica(&self, namespace: &NamespaceId); + fn close_replica(&self, replica: Replica); /// Remove a replica. /// /// Completely removes a replica and deletes both the namespace private key and all document /// entries. + /// + /// Note that a replica has to be closed before it can be removed. The store has to enforce + /// that a replica cannot be removed while it is still open. fn remove_replica(&self, namespace: &NamespaceId) -> Result<()>; /// Create a new author key and persist it in the store. diff --git a/iroh-sync/src/store/fs.rs b/iroh-sync/src/store/fs.rs index e608080449..d92d47322d 100644 --- a/iroh-sync/src/store/fs.rs +++ b/iroh-sync/src/store/fs.rs @@ -1,8 +1,8 @@ //! On disk storage for replicas. -use std::{cmp::Ordering, collections::HashMap, path::Path, sync::Arc}; +use std::{cmp::Ordering, collections::HashSet, path::Path, sync::Arc}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use derive_more::From; use ed25519_dalek::{SignatureError, VerifyingKey}; use iroh_bytes::Hash; @@ -22,13 +22,13 @@ use crate::{ AuthorId, NamespaceId, PeerIdBytes, }; -use super::{pubkeys::MemPublicKeyStore, PublicKeyStore}; +use super::{pubkeys::MemPublicKeyStore, OpenError, PublicKeyStore}; /// Manages the replicas and authors for an instance. #[derive(Debug, Clone)] pub struct Store { db: Arc, - replicas: Arc>>>, + open_replicas: Arc>>, pubkeys: MemPublicKeyStore, } @@ -53,6 +53,7 @@ const NAMESPACES_TABLE: TableDefinition<&[u8; 32], &[u8; 32]> = // Value: // (u64, [u8; 32], [u8; 32], u64, [u8; 32]) // # (timestamp, signature_namespace, signature_author, len, hash) +const RECORDS_TABLE: TableDefinition = TableDefinition::new("records-1"); type RecordsId<'a> = (&'a [u8; 32], &'a [u8; 32], &'a [u8]); type RecordsValue<'a> = (u64, &'a [u8; 64], &'a [u8; 64], u64, &'a [u8; 32]); @@ -60,8 +61,6 @@ type RecordsRange<'a> = TableRange<'a, RecordsId<'static>, RecordsValue<'static> type RecordsTable<'a> = ReadOnlyTable<'a, RecordsId<'static>, RecordsValue<'static>>; type DbResult = Result; -const RECORDS_TABLE: TableDefinition = TableDefinition::new("records-1"); - /// Number of seconds elapsed since [`std::time::SystemTime::UNIX_EPOCH`]. Used to register the /// last time a peer was useful in a document. // NOTE: resolution is nanoseconds, stored as a u64 since this covers ~500years from unix epoch, @@ -92,7 +91,7 @@ impl Store { Ok(Store { db: Arc::new(db), - replicas: Default::default(), + open_replicas: Default::default(), pubkeys: Default::default(), }) } @@ -129,26 +128,33 @@ impl super::Store for Store { type NamespaceIter<'a> = std::vec::IntoIter>; type PeersIter<'a> = std::vec::IntoIter; - fn open_replica(&self, namespace_id: &NamespaceId) -> Result>> { - if let Some(replica) = self.replicas.read().get(namespace_id) { - return Ok(Some(replica.clone())); + fn open_replica( + &self, + namespace_id: &NamespaceId, + ) -> Result, OpenError> { + if self.open_replicas.read().contains(namespace_id) { + return Err(OpenError::AlreadyOpen); } - let read_tx = self.db.begin_read()?; - let namespace_table = read_tx.open_table(NAMESPACES_TABLE)?; - let Some(namespace) = namespace_table.get(namespace_id.as_bytes())? else { - return Ok(None); + let read_tx = self.db.begin_read().map_err(anyhow::Error::from)?; + let namespace_table = read_tx + .open_table(NAMESPACES_TABLE) + .map_err(anyhow::Error::from)?; + let Some(namespace) = namespace_table + .get(namespace_id.as_bytes()) + .map_err(anyhow::Error::from)? + else { + return Err(OpenError::NotFound); }; let namespace = Namespace::from_bytes(namespace.value()); let replica = Replica::new(namespace, StoreInstance::new(*namespace_id, self.clone())); - self.replicas.write().insert(*namespace_id, replica.clone()); - Ok(Some(replica)) + self.open_replicas.write().insert(*namespace_id); + Ok(replica) } - fn close_replica(&self, namespace_id: &NamespaceId) { - if let Some(replica) = self.replicas.write().remove(namespace_id) { - replica.close(); - } + fn close_replica(&self, mut replica: Replica) { + self.open_replicas.write().remove(&replica.namespace()); + replica.close(); } fn list_namespaces(&self) -> Result> { @@ -196,19 +202,15 @@ impl super::Store for Store { Ok(authors.into_iter()) } - fn new_replica(&self, namespace: Namespace) -> Result> { - let id = namespace.id(); + fn import_namespace(&self, namespace: Namespace) -> Result<()> { self.insert_namespace(namespace.clone())?; - - let replica = Replica::new(namespace, StoreInstance::new(id, self.clone())); - - self.replicas.write().insert(id, replica.clone()); - Ok(replica) + Ok(()) } fn remove_replica(&self, namespace: &NamespaceId) -> Result<()> { - self.close_replica(namespace); - self.replicas.write().remove(namespace); + if self.open_replicas.read().contains(namespace) { + return Err(anyhow!("replica is not closed")); + } let start = range_start(namespace); let end = range_end(namespace); let write_tx = self.db.begin_write()?; @@ -549,6 +551,7 @@ impl crate::ranger::Store for StoreInstance { fn put(&mut self, e: SignedEntry) -> Result<()> { let write_tx = self.store.db.begin_write()?; { + // insert into record table let mut record_table = write_tx.open_table(RECORDS_TABLE)?; let key = ( &e.id().namespace().to_bytes(), @@ -916,7 +919,7 @@ mod tests { let author = store.new_author(&mut rand::thread_rng())?; let namespace = Namespace::new(&mut rand::thread_rng()); - let replica = store.new_replica(namespace)?; + let mut replica = store.new_replica(namespace)?; // test author prefix relation for all-255 keys let key1 = vec![255, 255]; @@ -947,12 +950,9 @@ mod tests { let author = store.new_author(&mut rand::thread_rng())?; let namespace = Namespace::new(&mut rand::thread_rng()); let replica = store.new_replica(namespace.clone())?; - - let replica_back = store.open_replica(&namespace.id())?.unwrap(); - assert_eq!( - replica.namespace().as_bytes(), - replica_back.namespace().as_bytes() - ); + store.close_replica(replica); + let replica = store.open_replica(&namespace.id())?; + assert_eq!(replica.namespace(), namespace.id()); let author_back = store.get_author(&author.id())?.unwrap(); assert_eq!(author.to_bytes(), author_back.to_bytes(),); diff --git a/iroh-sync/src/store/memory.rs b/iroh-sync/src/store/memory.rs index baf6742528..17d564bc07 100644 --- a/iroh-sync/src/store/memory.rs +++ b/iroh-sync/src/store/memory.rs @@ -1,12 +1,12 @@ //! In memory storage for replicas. use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, HashMap, HashSet}, convert::Infallible, sync::Arc, }; -use anyhow::Result; +use anyhow::{anyhow, Result}; use ed25519_dalek::{SignatureError, VerifyingKey}; use iroh_bytes::Hash; use parking_lot::{RwLock, RwLockReadGuard}; @@ -17,14 +17,15 @@ use crate::{ AuthorId, NamespaceId, PeerIdBytes, Record, }; -use super::{pubkeys::MemPublicKeyStore, PublicKeyStore}; +use super::{pubkeys::MemPublicKeyStore, OpenError, PublicKeyStore}; type SyncPeersCache = Arc>>>; /// Manages the replicas and authors for an instance. #[derive(Debug, Clone, Default)] pub struct Store { - replicas: Arc>>>, + open_replicas: Arc>>, + namespaces: Arc>>, authors: Arc>>, /// Stores records by namespace -> identifier + timestamp replica_records: Arc>, @@ -46,21 +47,29 @@ impl super::Store for Store { type NamespaceIter<'a> = std::vec::IntoIter>; type PeersIter<'a> = std::vec::IntoIter; - fn open_replica(&self, namespace: &NamespaceId) -> Result>> { - let replicas = &*self.replicas.read(); - Ok(replicas.get(namespace).cloned()) + fn open_replica(&self, id: &NamespaceId) -> Result, OpenError> { + if self.open_replicas.read().contains(id) { + return Err(OpenError::AlreadyOpen); + } + let namespace = { + let namespaces = self.namespaces.read(); + let namespace = namespaces.get(id).ok_or(OpenError::NotFound)?; + namespace.clone() + }; + let replica = Replica::new(namespace, ReplicaStoreInstance::new(*id, self.clone())); + self.open_replicas.write().insert(*id); + Ok(replica) } - fn close_replica(&self, namespace_id: &NamespaceId) { - if let Some(replica) = self.replicas.read().get(namespace_id) { - replica.close(); - } + fn close_replica(&self, mut replica: Replica) { + self.open_replicas.write().remove(&replica.namespace()); + replica.close(); } fn list_namespaces(&self) -> Result> { // TODO: avoid collect? Ok(self - .replicas + .namespaces .read() .keys() .cloned() @@ -91,19 +100,17 @@ impl super::Store for Store { .into_iter()) } - fn new_replica(&self, namespace: Namespace) -> Result> { - let id = namespace.id(); - let replica = Replica::new(namespace, ReplicaStoreInstance::new(id, self.clone())); - self.replicas - .write() - .insert(replica.namespace(), replica.clone()); - Ok(replica) + fn import_namespace(&self, namespace: Namespace) -> Result<()> { + self.namespaces.write().insert(namespace.id(), namespace); + Ok(()) } fn remove_replica(&self, namespace: &NamespaceId) -> Result<()> { - self.close_replica(namespace); - self.replicas.write().remove(namespace); + if self.open_replicas.read().contains(namespace) { + return Err(anyhow!("replica is not closed")); + } self.replica_records.write().remove(namespace); + self.namespaces.write().remove(namespace); Ok(()) } diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index e85c4626fa..878d0f81cb 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -9,10 +9,7 @@ use std::{ cmp::Ordering, fmt::Debug, - sync::{ - atomic::{self, AtomicBool}, - Arc, - }, + sync::Arc, time::{Duration, SystemTime}, }; @@ -23,8 +20,6 @@ use derive_more::Deref; #[cfg(feature = "metrics")] use iroh_metrics::{inc, inc_by}; -use parking_lot::RwLock; - use ed25519_dalek::{Signature, SignatureError}; use iroh_bytes::Hash; use serde::{Deserialize, Serialize}; @@ -53,6 +48,19 @@ pub type PeerIdBytes = [u8; 32]; /// Value is 10 minutes. pub const MAX_TIMESTAMP_FUTURE_SHIFT: u64 = 10 * 60 * Duration::from_secs(1).as_millis() as u64; +/// Callback that may be set on a replica to determine the availability status for a content hash. +pub type ContentStatusCallback = Arc ContentStatus + Send + Sync + 'static>; + +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub enum Event { + Insert { + namespace: NamespaceId, + origin: InsertOrigin, + entry: SignedEntry, + }, +} + /// Whether an entry was inserted locally or by a remote peer. #[derive(Debug, Clone)] pub enum InsertOrigin { @@ -78,30 +86,49 @@ pub enum ContentStatus { Missing, } -/// Local representation of a mutable, synchronizable key-value store. -#[derive(derive_more::Debug, Clone)] -pub struct Replica + PublicKeyStore> { - inner: Arc>, +/// Outcome of a sync operation. +#[derive(Debug, Clone, Default)] +pub struct SyncOutcome { + /// Number of entries we received. + pub num_recv: usize, + /// Number of entries we sent. + pub num_sent: usize, } -#[derive(derive_more::Debug)] -struct InnerReplica + PublicKeyStore> { - namespace: Namespace, - peer: RwLock>, - #[allow(clippy::type_complexity)] - on_insert_sender: RwLock>>, - - #[allow(clippy::type_complexity)] - #[debug("ContentStatusCallback")] - content_status_cb: RwLock ContentStatus + Send + Sync + 'static>>>, - - closed: AtomicBool, +#[derive(Debug, Default)] +struct Subscribers(Vec>); +impl Subscribers { + pub fn subscribe(&mut self, sender: flume::Sender) { + self.0.push(sender) + } + pub fn unsubscribe(&mut self, sender: &flume::Sender) { + self.0.retain(|s| !s.same_channel(sender)); + } + pub fn send(&mut self, event: Event) { + self.0.retain(|sender| sender.send(event.clone()).is_ok()) + } + pub fn len(&self) -> usize { + self.0.len() + } + pub fn send_with(&mut self, f: impl FnOnce() -> Event) { + if !self.0.is_empty() { + self.send(f()) + } + } + pub fn clear(&mut self) { + self.0.clear() + } } -#[derive(Debug, Serialize, Deserialize)] -struct ReplicaData { - entries: Vec, +/// Local representation of a mutable, synchronizable key-value store. +#[derive(derive_more::Debug)] +pub struct Replica + PublicKeyStore> { namespace: Namespace, + peer: Peer, + subscribers: Subscribers, + #[debug("ContentStatusCallback")] + content_status_cb: Option, + closed: bool, } impl + PublicKeyStore + 'static> Replica { @@ -109,65 +136,56 @@ impl + PublicKeyStore + 'static> Replica { // TODO: make read only replicas possible pub fn new(namespace: Namespace, store: S) -> Self { Replica { - inner: Arc::new(InnerReplica { - namespace, - peer: RwLock::new(Peer::from_store(store)), - on_insert_sender: RwLock::new(None), - content_status_cb: RwLock::new(None), - closed: AtomicBool::new(false), - }), + namespace, + peer: Peer::from_store(store), + subscribers: Default::default(), + // on_insert_sender: RwLock::new(None), + content_status_cb: None, + closed: false, } } /// Mark the replica as closed, prohibiting any further operations. /// /// This method is not public. Use [store::Store::close_replica] instead. - pub(crate) fn close(&self) { - self.unsubscribe(); - self.inner.closed.store(true, atomic::Ordering::Release); + pub(crate) fn close(&mut self) { + self.subscribers.clear(); + self.closed = true; } - /// Subscribe to insert events. + /// Subcribe to insert events. /// - /// Only one subscription can be active at a time. If a previous subscription was created, this - /// will return `None`. - /// - /// When subscribing to a replica, you must ensure that the returned [`flume::Receiver`] is + /// When subscribing to a replica, you must ensure that the corresponding [`flume::Receiver`] is /// received from in a loop. If not receiving, local and remote inserts will hang waiting for /// the receiver to be received from. - // TODO: Allow to clear a previous subscription? - pub fn subscribe(&self) -> Option> { - let mut on_insert_sender = self.inner.on_insert_sender.write(); - match &*on_insert_sender { - Some(_sender) => None, - None => { - let (s, r) = flume::bounded(16); // TODO: should this be configurable? - *on_insert_sender = Some(s); - Some(r) - } - } + pub fn subscribe(&mut self, sender: flume::Sender) { + self.subscribers.subscribe(sender) + } + + /// Explicitly unsubscribe a sender. + /// + /// Simply dropping the receiver is fine too. If you cloned a single sender to subscribe to + /// multiple replicas, you can use this method to explicitly unsubscribe the sender from + /// this replica without having to drop the receiver. + pub fn unsubscribe(&mut self, sender: &flume::Sender) { + self.subscribers.unsubscribe(sender) } - /// Remove the subscription. - pub fn unsubscribe(&self) -> bool { - self.inner.on_insert_sender.write().take().is_some() + /// Get the number of current event subscribers. + pub fn subscribers_count(&self) -> usize { + self.subscribers.len() } /// Set the content status callback. /// /// Only one callback can be active at a time. If a previous callback was registered, this /// will return `false`. - pub fn set_content_status_callback( - &self, - cb: Box ContentStatus + Send + Sync + 'static>, - ) -> bool { - let mut content_status_cb = self.inner.content_status_cb.write(); - match &*content_status_cb { - Some(_cb) => false, - None => { - *content_status_cb = Some(cb); - true - } + pub fn set_content_status_callback(&mut self, cb: ContentStatusCallback) -> bool { + if self.content_status_cb.is_some() { + false + } else { + self.content_status_cb = Some(cb); + true } } @@ -185,7 +203,7 @@ impl + PublicKeyStore + 'static> Replica { /// manually, it must be closed via [`store::Store::close_replica`] or /// [`store::Store::remove_replica`] pub fn closed(&self) -> bool { - self.inner.closed.load(atomic::Ordering::Acquire) + self.closed } /// Insert a new record at the given key. @@ -196,7 +214,7 @@ impl + PublicKeyStore + 'static> Replica { /// Returns the number of entries removed as a consequence of this insertion, /// or an error either if the entry failed to validate or if a store operation failed. pub fn insert( - &self, + &mut self, key: impl AsRef<[u8]>, author: &Author, hash: Hash, @@ -209,7 +227,7 @@ impl + PublicKeyStore + 'static> Replica { let id = RecordIdentifier::new(self.namespace(), author.id(), key); let record = Record::new_current(hash, len); let entry = Entry::new(id, record); - let signed_entry = entry.sign(&self.inner.namespace, author); + let signed_entry = entry.sign(&self.namespace, author); self.insert_entry(signed_entry, InsertOrigin::Local) } @@ -220,14 +238,14 @@ impl + PublicKeyStore + 'static> Replica { /// /// Returns the number of entries deleted. pub fn delete_prefix( - &self, + &mut self, prefix: impl AsRef<[u8]>, author: &Author, ) -> Result> { self.ensure_open()?; let id = RecordIdentifier::new(self.namespace(), author.id(), prefix); let entry = Entry::new_empty(id); - let signed_entry = entry.sign(&self.inner.namespace, author); + let signed_entry = entry.sign(&self.namespace, author); self.insert_entry(signed_entry, InsertOrigin::Local) } @@ -239,7 +257,7 @@ impl + PublicKeyStore + 'static> Replica { /// Returns the number of entries removed as a consequence of this insertion, /// or an error if the entry failed to validate or if a store operation failed. pub fn insert_remote_entry( - &self, + &mut self, entry: SignedEntry, received_from: PeerIdBytes, content_status: ContentStatus, @@ -257,38 +275,25 @@ impl + PublicKeyStore + 'static> Replica { /// /// Returns the number of entries removed as a consequence of this insertion. fn insert_entry( - &self, + &mut self, entry: SignedEntry, origin: InsertOrigin, ) -> Result> { - let expected_namespace = self.namespace(); + let namespace = self.namespace(); #[cfg(feature = "metrics")] let len = entry.content_len(); - let mut peer = self.inner.peer.write(); - let store = peer.store(); - validate_entry( - system_time_now(), - store, - expected_namespace, - &entry, - &origin, - )?; + let store = self.peer.store(); + validate_entry(system_time_now(), store, namespace, &entry, &origin)?; - let outcome = peer.put(entry.clone()).map_err(InsertError::Store)?; + let outcome = self.peer.put(entry.clone()).map_err(InsertError::Store)?; let removed_count = match outcome { InsertOutcome::Inserted { removed } => removed, InsertOutcome::NotInserted => return Err(InsertError::NewerEntryExists), }; - drop(peer); - - if let Some(sender) = self.inner.on_insert_sender.read().as_ref() { - sender.send((origin.clone(), entry)).ok(); - } - #[cfg(feature = "metrics")] { match origin { @@ -303,6 +308,12 @@ impl + PublicKeyStore + 'static> Replica { } } + self.subscribers.send(Event::Insert { + namespace, + origin, + entry, + }); + Ok(removed_count) } @@ -311,7 +322,7 @@ impl + PublicKeyStore + 'static> Replica { /// This does not store the content, just the record of it. /// Returns the calculated hash. pub fn hash_and_insert( - &self, + &mut self, key: impl AsRef<[u8]>, author: &Author, data: impl AsRef<[u8]>, @@ -325,50 +336,60 @@ impl + PublicKeyStore + 'static> Replica { /// Get the identifier for an entry in this replica. pub fn id(&self, key: impl AsRef<[u8]>, author: &Author) -> RecordIdentifier { - RecordIdentifier::new(self.inner.namespace.id(), author.id(), key) + RecordIdentifier::new(self.namespace.id(), author.id(), key) } /// Create the initial message for the set reconciliation flow with a remote peer. - pub fn sync_initial_message( - &self, - ) -> Result, anyhow::Error> { + pub fn sync_initial_message(&self) -> anyhow::Result> { self.ensure_open()?; - self.inner.peer.read().initial_message().map_err(Into::into) + self.peer.initial_message().map_err(Into::into) } /// Process a set reconciliation message from a remote peer. /// /// Returns the next message to be sent to the peer, if any. pub fn sync_process_message( - &self, + &mut self, message: crate::ranger::Message, from_peer: PeerIdBytes, + state: &mut SyncOutcome, ) -> Result>, anyhow::Error> { self.ensure_open()?; - let expected_namespace = self.namespace(); + let my_namespace = self.namespace(); let now = system_time_now(); + + // update state with incoming data. + state.num_recv += message.value_count(); + + // let subscribers = std::rc::Rc::new(&mut self.subscribers); + // l let reply = self - .inner .peer - .write() .process_message( message, + // validate callback: validate incoming entries, and send to on_insert channel |store, entry, content_status| { let origin = InsertOrigin::Sync { from: from_peer, content_status, }; - if validate_entry(now, store, expected_namespace, entry, &origin).is_ok() { - if let Some(sender) = self.inner.on_insert_sender.read().as_ref() { - sender.send((origin, entry.clone())).ok(); - } - true - } else { - false - } + validate_entry(now, store, my_namespace, entry, &origin).is_ok() + }, + // on_insert callback: is called when an entry was actually inserted in the store + |_store, entry, content_status| { + // We use `send_with` to only clone the entry if we have active subcriptions. + self.subscribers.send_with(|| Event::Insert { + namespace: my_namespace, + origin: InsertOrigin::Sync { + from: from_peer, + content_status, + }, + entry: entry.clone(), + }) }, + // content_status callback: get content status for outgoing entries |_store, entry| { - if let Some(cb) = self.inner.content_status_cb.read().as_ref() { + if let Some(cb) = self.content_status_cb.as_ref() { cb(entry.content_hash()) } else { ContentStatus::Missing @@ -377,18 +398,22 @@ impl + PublicKeyStore + 'static> Replica { ) .map_err(Into::into)?; + // update state with outgoing data. + if let Some(ref reply) = reply { + state.num_sent += reply.value_count(); + } + Ok(reply) } /// Get the namespace identifier for this [`Replica`]. pub fn namespace(&self) -> NamespaceId { - self.inner.namespace.id() + self.namespace.id() } /// Get the byte represenation of the [`Namespace`] key for this replica. - // TODO: Why return [u8; 32] and not `Namespace` here? - pub fn secret_key(&self) -> [u8; 32] { - self.inner.namespace.to_bytes() + pub fn secret_key(&self) -> Namespace { + self.namespace.clone() } } @@ -962,7 +987,7 @@ mod tests { use crate::{ ranger::{Range, Store as _}, - store::{self, GetFilter, Store}, + store::{self, GetFilter, OpenError, Store}, }; use super::*; @@ -996,7 +1021,7 @@ mod tests { let signed_entry = entry.sign(&myspace, &alice); signed_entry.verify(&()).expect("failed to verify"); - let my_replica = store.new_replica(myspace)?; + let mut my_replica = store.new_replica(myspace)?; for i in 0..10 { my_replica.hash_and_insert( format!("/{i}"), @@ -1104,12 +1129,9 @@ mod tests { .collect::>()?; assert_eq!(entries.len(), 12); - let replica = store.open_replica(&my_replica.namespace())?.unwrap(); // Get Range of all should return all latest - let entries_second: Vec<_> = replica - .inner + let entries_second: Vec<_> = my_replica .peer - .read() .store() .get_range(Range::new( RecordIdentifier::default(), @@ -1196,7 +1218,7 @@ mod tests { for i in 0..n_replicas { let namespace = Namespace::new(&mut rng); let author = store.new_author(&mut rng)?; - let replica = store.new_replica(namespace)?; + let mut replica = store.new_replica(namespace)?; for j in 0..n_entries { let key = format!("{j}"); let data = format!("{i}:{j}"); @@ -1320,7 +1342,7 @@ mod tests { fn test_timestamps(store: S) -> Result<()> { let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(1); let namespace = Namespace::new(&mut rng); - let replica = store.new_replica(namespace.clone())?; + let mut replica = store.new_replica(namespace.clone())?; let author = store.new_author(&mut rng)?; let key = b"hello"; @@ -1381,17 +1403,22 @@ mod tests { let mut rng = rand::thread_rng(); let author = Author::new(&mut rng); let myspace = Namespace::new(&mut rng); - let alice = alice_store.new_replica(myspace.clone())?; + let mut alice = alice_store.new_replica(myspace.clone())?; for el in &alice_set { alice.hash_and_insert(el, &author, el.as_bytes())?; } - let bob = bob_store.new_replica(myspace.clone())?; + let mut bob = bob_store.new_replica(myspace.clone())?; for el in &bob_set { bob.hash_and_insert(el, &author, el.as_bytes())?; } - sync::(&alice, &bob)?; + let (alice_out, bob_out) = sync::(&mut alice, &mut bob)?; + + assert_eq!(alice_out.num_sent, 2); + assert_eq!(bob_out.num_recv, 2); + assert_eq!(alice_out.num_recv, 6); + assert_eq!(bob_out.num_sent, 6); check_entries(&alice_store, &myspace.id(), &author, &alice_set)?; check_entries(&alice_store, &myspace.id(), &author, &bob_set)?; @@ -1426,8 +1453,8 @@ mod tests { let mut rng = rand::thread_rng(); let author = Author::new(&mut rng); let namespace = Namespace::new(&mut rng); - let alice = alice_store.new_replica(namespace.clone())?; - let bob = bob_store.new_replica(namespace.clone())?; + let mut alice = alice_store.new_replica(namespace.clone())?; + let mut bob = bob_store.new_replica(namespace.clone())?; let key = b"key"; let alice_value = b"alice"; @@ -1435,7 +1462,7 @@ mod tests { let _alice_hash = alice.hash_and_insert(key, &author, alice_value)?; // system time increased - sync should overwrite let bob_hash = bob.hash_and_insert(key, &author, bob_value)?; - sync::(&alice, &bob)?; + sync::(&mut alice, &mut bob)?; assert_eq!( get_content_hash(&alice_store, namespace.id(), author.id(), key)?, Some(bob_hash) @@ -1449,7 +1476,7 @@ mod tests { // system time increased - sync should overwrite let _bob_hash_2 = bob.hash_and_insert(key, &author, bob_value)?; let alice_hash_2 = alice.hash_and_insert(key, &author, alice_value_2)?; - sync::(&alice, &bob)?; + sync::(&mut alice, &mut bob)?; assert_eq!( get_content_hash(&alice_store, namespace.id(), author.id(), key)?, Some(alice_hash_2) @@ -1468,7 +1495,7 @@ mod tests { let store = store::memory::Store::default(); let author = Author::new(&mut rng); let namespace = Namespace::new(&mut rng); - let replica = store.new_replica(namespace.clone())?; + let mut replica = store.new_replica(namespace.clone())?; let key = b"hi"; let t = system_time_now(); @@ -1511,7 +1538,7 @@ mod tests { let mut rng = rand::thread_rng(); let alice = Author::new(&mut rng); let myspace = Namespace::new(&mut rng); - let replica = store.new_replica(myspace.clone())?; + let mut replica = store.new_replica(myspace.clone())?; let hash = Hash::new(b""); let res = replica.insert(b"foo", &alice, hash, 0); assert!(matches!(res, Err(InsertError::EntryIsEmpty))); @@ -1538,7 +1565,7 @@ mod tests { let mut rng = rand::thread_rng(); let alice = Author::new(&mut rng); let myspace = Namespace::new(&mut rng); - let replica = store.new_replica(myspace.clone())?; + let mut replica = store.new_replica(myspace.clone())?; let hash1 = replica.hash_and_insert(b"foobar", &alice, b"hello")?; let hash2 = replica.hash_and_insert(b"fooboo", &alice, b"world")?; @@ -1587,17 +1614,17 @@ mod tests { let mut rng = rand::thread_rng(); let author = Author::new(&mut rng); let myspace = Namespace::new(&mut rng); - let alice = alice_store.new_replica(myspace.clone())?; + let mut alice = alice_store.new_replica(myspace.clone())?; for el in &alice_set { alice.hash_and_insert(el, &author, el.as_bytes())?; } - let bob = bob_store.new_replica(myspace.clone())?; + let mut bob = bob_store.new_replica(myspace.clone())?; for el in &bob_set { bob.hash_and_insert(el, &author, el.as_bytes())?; } - sync::(&alice, &bob)?; + sync::(&mut alice, &mut bob)?; check_entries(&alice_store, &myspace.id(), &author, &alice_set)?; check_entries(&alice_store, &myspace.id(), &author, &bob_set)?; @@ -1606,7 +1633,7 @@ mod tests { alice.delete_prefix("foo", &author)?; bob.hash_and_insert("fooz", &author, "fooz".as_bytes())?; - sync::(&alice, &bob)?; + sync::(&mut alice, &mut bob)?; check_entries(&alice_store, &myspace.id(), &author, &["fog", "fooz"])?; check_entries(&bob_store, &myspace.id(), &author, &["fog", "fooz"])?; @@ -1631,7 +1658,7 @@ mod tests { let mut rng = rand::thread_rng(); let namespace = Namespace::new(&mut rng); let author = Author::new(&mut rng); - let replica = store.new_replica(namespace.clone())?; + let mut replica = store.new_replica(namespace.clone())?; // insert entry let hash = replica.hash_and_insert(b"foo", &author, b"bar")?; @@ -1641,26 +1668,22 @@ mod tests { assert_eq!(res.len(), 1); // remove replica + let res = store.remove_replica(&namespace.id()); + // may not remove replica while still open; + assert!(res.is_err()); + store.close_replica(replica); store.remove_replica(&namespace.id())?; let res = store .get_many(namespace.id(), GetFilter::All)? .collect::>(); assert_eq!(res.len(), 0); - // may not insert on removed replica - let res = replica.insert(b"foo", &author, hash, 3); - assert!(matches!(res, Err(InsertError::Closed))); - let res = store - .get_many(namespace.id(), GetFilter::All)? - .collect::>(); - assert_eq!(res.len(), 0); - // may not reopen removed replica - let res = store.open_replica(&namespace.id())?; - assert!(res.is_none()); + let res = store.open_replica(&namespace.id()); + assert!(matches!(res, Err(OpenError::NotFound))); // may recreate replica - let replica = store.new_replica(namespace.clone())?; + let mut replica = store.new_replica(namespace.clone())?; replica.insert(b"foo", &author, hash, 3)?; let res = store .get_many(namespace.id(), GetFilter::All)? @@ -1687,7 +1710,7 @@ mod tests { let mut rng = rand::thread_rng(); let author = Author::new(&mut rng); let namespace = Namespace::new(&mut rng); - let replica = store.new_replica(namespace.clone())?; + let mut replica = store.new_replica(namespace.clone())?; let edgecases = [0u8, 1u8, 255u8]; let prefixes = [0u8, 255u8]; @@ -1748,7 +1771,7 @@ mod tests { let mut rng = rand::thread_rng(); let author = Author::new(&mut rng); let namespace = Namespace::new(&mut rng); - let replica = store.new_replica(namespace.clone())?; + let mut replica = store.new_replica(namespace.clone())?; let hash = Hash::new(b"foo"); let len = 3; @@ -1770,6 +1793,74 @@ mod tests { Ok(()) } + /// This tests that no events are emitted for entries received during sync which are obsolete + /// (too old) by the time they are actually inserted in the store. + #[test] + fn test_replica_no_wrong_remote_insert_events() -> Result<()> { + let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(1); + let store1 = store::memory::Store::default(); + let store2 = store::memory::Store::default(); + let peer1 = [1u8; 32]; + let peer2 = [2u8; 32]; + let mut state1 = SyncOutcome::default(); + let mut state2 = SyncOutcome::default(); + + let author = Author::new(&mut rng); + let namespace = Namespace::new(&mut rng); + let mut replica1 = store1.new_replica(namespace.clone())?; + let mut replica2 = store2.new_replica(namespace.clone())?; + + let (events1_sender, events1) = flume::bounded(32); + let (events2_sender, events2) = flume::bounded(32); + + replica1.subscribe(events1_sender); + replica2.subscribe(events2_sender); + + replica1.hash_and_insert(b"foo", &author, b"init")?; + + let from1 = replica1.sync_initial_message()?; + let from2 = replica2 + .sync_process_message(from1, peer1, &mut state2) + .unwrap() + .unwrap(); + let from1 = replica1 + .sync_process_message(from2, peer2, &mut state1) + .unwrap() + .unwrap(); + // now we will receive the entry from rpelica1. we will insert a newer entry now, while the + // sync is already running. this means the entry from replica1 will be rejected. we make + // sure that no InsertRemote event is emitted for this entry. + replica2.hash_and_insert(b"foo", &author, b"update")?; + let from2 = replica2 + .sync_process_message(from1, peer1, &mut state2) + .unwrap(); + assert!(from2.is_none()); + let events1 = events1.drain().collect::>(); + let events2 = events2.drain().collect::>(); + assert_eq!(events1.len(), 1); + assert_eq!(events2.len(), 1); + assert!(matches!( + events1[0], + Event::Insert { + origin: InsertOrigin::Local, + .. + } + )); + assert!(matches!( + events2[0], + Event::Insert { + origin: InsertOrigin::Local, + .. + } + )); + assert_eq!(state1.num_sent, 1); + assert_eq!(state1.num_recv, 0); + assert_eq!(state2.num_sent, 0); + assert_eq!(state2.num_recv, 1); + + Ok(()) + } + fn assert_keys(store: &S, namespace: NamespaceId, mut expected: Vec>) { expected.sort(); assert_eq!(expected, get_keys_sorted(store, namespace)); @@ -1811,11 +1902,13 @@ mod tests { } fn sync( - alice: &Replica, - bob: &Replica, - ) -> Result<()> { + alice: &mut Replica, + bob: &mut Replica, + ) -> Result<(SyncOutcome, SyncOutcome)> { let alice_peer_id = [1u8; 32]; let bob_peer_id = [2u8; 32]; + let mut alice_state = SyncOutcome::default(); + let mut bob_state = SyncOutcome::default(); // Sync alice - bob let mut next_to_bob = Some(alice.sync_initial_message()?); let mut rounds = 0; @@ -1823,11 +1916,13 @@ mod tests { assert!(rounds < 100, "too many rounds"); rounds += 1; println!("round {}", rounds); - if let Some(msg) = bob.sync_process_message(msg, alice_peer_id)? { - next_to_bob = alice.sync_process_message(msg, bob_peer_id)? + if let Some(msg) = bob.sync_process_message(msg, alice_peer_id, &mut bob_state)? { + next_to_bob = alice.sync_process_message(msg, bob_peer_id, &mut alice_state)? } } - Ok(()) + assert_eq!(alice_state.num_sent, bob_state.num_recv); + assert_eq!(alice_state.num_recv, bob_state.num_sent); + Ok((alice_state, bob_state)) } fn check_entries( @@ -1836,9 +1931,8 @@ mod tests { author: &Author, set: &[&str], ) -> Result<()> { - let replica = store.open_replica(namespace)?.unwrap(); for el in set { - store.get_one(replica.namespace(), author.id(), el)?; + store.get_one(*namespace, author.id(), el)?; } Ok(()) } diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 470f13d0d1..db39f4aaab 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -18,7 +18,7 @@ bao-tree = { version = "0.9.1", features = ["tokio_fsm"], default-features = fal bytes = "1" data-encoding = "2.4.0" derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } -flume = "0.10.14" +flume = "0.11" futures = "0.3.25" genawaiter = { version = "0.99", default-features = false, features = ["futures03"] } hex = { version = "0.4.3" } @@ -75,16 +75,17 @@ cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic metrics = ["iroh-metrics"] flat-db = ["iroh-bytes/flat-db"] test = [] -example-sync = ["cli"] [dev-dependencies] anyhow = { version = "1", features = ["backtrace"] } bytes = "1" +console-subscriber = "0.1.10" duct = "0.13.6" genawaiter = { version = "0.99", features = ["futures03"] } iroh-test = { path = "../iroh-test" } nix = "0.26.2" proptest = "1.2.0" +rand_chacha = "0.3.1" regex = { version = "1.7.1", features = ["std"] } testdir = "0.8" tokio = { version = "1", features = ["macros", "io-util", "rt"] } @@ -106,10 +107,6 @@ required-features = [] name = "hello-world" required-features = [] -[[example]] -name = "sync" -required-features = ["example-sync"] - [[example]] name = "rpc" required-features = ["clap"] diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs deleted file mode 100644 index 4485559dbf..0000000000 --- a/iroh/examples/sync.rs +++ /dev/null @@ -1,1062 +0,0 @@ -//! Live edit a p2p document -//! -//! By default a new peer id is created when starting the example. To reuse your identity, -//! set the `--secret-key` CLI flag with the secret key printed on a previous invocation. -//! -//! Run with: -//! $ cargo run --example sync --features=example-sync -//! Then follow the instructions printed -//! -//! You can use this with a local DERP server. To do so, run -//! `cargo run --bin derper -- --dev` -//! and then set the `-d http://localhost:3340` flag on this example. - -use std::{ - collections::HashSet, fmt, net::SocketAddr, path::PathBuf, str::FromStr, sync::Arc, - time::Instant, -}; - -use anyhow::{anyhow, bail}; -use bytes::Bytes; -use clap::{CommandFactory, FromArgMatches, Parser}; -use futures::StreamExt; -use indicatif::HumanBytes; -use iroh::{ - downloader::Downloader, - sync_engine::{LiveEvent, SyncEngine, SYNC_ALPN}, -}; -use iroh_bytes::util::runtime; -use iroh_bytes::{ - store::{ImportMode, Map, MapEntry, Store as BaoStore}, - util::{progress::IgnoreProgressSender, BlobFormat}, -}; -use iroh_gossip::{ - net::{Gossip, GOSSIP_ALPN}, - proto::TopicId, -}; -use iroh_io::AsyncSliceReaderExt; -use iroh_net::{ - derp::{DerpMap, DerpMode}, - key::SecretKey, - magic_endpoint::get_alpn, - MagicEndpoint, PeerAddr, -}; -use iroh_sync::{ - store::{self, GetFilter, Store as _}, - sync::{Author, Entry, Namespace, Replica, SignedEntry}, -}; -use once_cell::sync::OnceCell; -use serde::{Deserialize, Serialize}; -use tokio::{ - io::AsyncWriteExt, - sync::{mpsc, oneshot}, - task::JoinHandle, -}; -use tracing::warn; -use tracing_subscriber::{EnvFilter, Registry}; -use url::Url; - -use iroh_bytes_handlers::IrohBytesHandlers; - -const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; - -type Doc = Replica<::Instance>; - -#[derive(Parser, Debug)] -struct Args { - /// Secret key for this node - #[clap(long)] - secret_key: Option, - /// Path to a data directory where blobs will be persisted - #[clap(short, long)] - storage_path: Option, - /// Set a custom DERP server. By default, the DERP server hosted by n0 will be used. - #[clap(short, long)] - derp: Option, - /// Disable DERP completeley - #[clap(long)] - no_derp: bool, - /// Set your nickname - #[clap(short, long)] - name: Option, - /// Set the bind port for our socket. By default, a random port will be used. - #[clap(short, long, default_value = "0")] - bind_port: u16, - /// Bind address on which to serve Prometheus metrics - #[clap(long)] - metrics_addr: Option, - #[clap(subcommand)] - command: Command, -} - -#[derive(Parser, Debug)] -enum Command { - Open { doc_name: String }, - Join { ticket: String }, -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let args = Args::parse(); - run(args).await -} - -pub fn init_metrics_collection( - metrics_addr: Option, -) -> Option> { - iroh::metrics::try_init_metrics_collection().ok(); - // doesn't start the server if the address is None - if let Some(metrics_addr) = metrics_addr { - return Some(tokio::spawn(async move { - if let Err(e) = iroh_metrics::metrics::start_metrics_server(metrics_addr).await { - eprintln!("Failed to start metrics server: {e}"); - } - })); - } - tracing::info!("Metrics server not started, no address provided"); - None -} - -async fn run(args: Args) -> anyhow::Result<()> { - // setup logging - let log_filter = init_logging(); - - let metrics_fut = init_metrics_collection(args.metrics_addr); - - // parse or generate our secret_key - let secret_key = match args.secret_key { - None => SecretKey::generate(), - Some(key) => SecretKey::from_str(&key)?, - }; - println!("> our secret key: {}", secret_key); - - // configure our derp map - let derp_mode = match (args.no_derp, args.derp) { - (false, None) => DerpMode::Default, - (false, Some(url)) => DerpMode::Custom(DerpMap::from_url(url, 0)), - (true, None) => DerpMode::Disabled, - (true, Some(_)) => bail!("You cannot set --no-derp and --derp at the same time"), - }; - println!("> using DERP servers: {}", fmt_derp_mode(&derp_mode)); - - // build our magic endpoint and the gossip protocol - let (endpoint, gossip) = { - // init a cell that will hold our gossip handle to be used in endpoint callbacks - let gossip_cell: OnceCell = OnceCell::new(); - // init a channel that will emit once the initial endpoints of our local node are discovered - let (initial_endpoints_tx, mut initial_endpoints_rx) = mpsc::channel(1); - // build the magic endpoint - let endpoint = MagicEndpoint::builder() - .secret_key(secret_key.clone()) - .alpns(vec![ - GOSSIP_ALPN.to_vec(), - SYNC_ALPN.to_vec(), - iroh_bytes::protocol::ALPN.to_vec(), - ]) - .derp_mode(derp_mode) - .on_endpoints({ - let gossip_cell = gossip_cell.clone(); - Box::new(move |endpoints| { - // send our updated endpoints to the gossip protocol to be sent as PeerData to peers - if let Some(gossip) = gossip_cell.get() { - gossip.update_endpoints(endpoints).ok(); - } - // trigger oneshot on the first endpoint update - initial_endpoints_tx.try_send(endpoints.to_vec()).ok(); - }) - }) - .bind(args.bind_port) - .await?; - - // initialize the gossip protocol - let gossip = Gossip::from_endpoint(endpoint.clone(), Default::default()); - // insert into the gossip cell to be used in the endpoint callbacks above - gossip_cell.set(gossip.clone()).unwrap(); - - // wait for a first endpoint update so that we know about at least one of our addrs - let initial_endpoints = initial_endpoints_rx.recv().await.unwrap(); - // pass our initial endpoints to the gossip protocol so that they can be announced to peers - gossip.update_endpoints(&initial_endpoints)?; - (endpoint, gossip) - }; - println!("> our peer id: {}", endpoint.peer_id()); - - let (topic, peers) = match &args.command { - Command::Open { doc_name } => { - let topic: TopicId = iroh_bytes::Hash::new(doc_name.as_bytes()).into(); - println!( - "> opening document {doc_name} as namespace {} and waiting for peers to join us...", - fmt_hash(topic.as_bytes()) - ); - (topic, vec![]) - } - Command::Join { ticket } => { - let Ticket { topic, peers } = Ticket::from_str(ticket)?; - println!("> joining topic {topic} and connecting to {peers:?}",); - (topic, peers) - } - }; - - let our_ticket = { - // add our local endpoints to the ticket and print it for others to join - let mut peers = peers.clone(); - peers.push(endpoint.my_addr().await?); - Ticket { peers, topic } - }; - println!("> ticket to join us: {our_ticket}"); - - // unwrap our storage path or default to temp - let storage_path = args.storage_path.unwrap_or_else(|| { - let name = format!("iroh-sync-{}", endpoint.peer_id()); - let dir = std::env::temp_dir().join(name); - if !dir.exists() { - std::fs::create_dir_all(&dir).expect("failed to create temp dir"); - } - dir - }); - println!("> storage directory: {storage_path:?}"); - - // create a runtime that can spawn tasks on a local-thread executors (to support !Send futures) - let rt = iroh_bytes::util::runtime::Handle::from_current(num_cpus::get())?; - - // create a doc store for the iroh-sync docs - let author = Author::from_bytes(&secret_key.to_bytes()); - let docs_path = storage_path.join("docs.db"); - let docs = iroh_sync::store::fs::Store::new(&docs_path)?; - - // create a bao store for the iroh-bytes blobs - let blob_path = storage_path.join("blobs"); - std::fs::create_dir_all(&blob_path)?; - let db = iroh_bytes::store::flat::Store::load(&blob_path, &blob_path, &blob_path, &rt).await?; - - // create the live syncer - let downloader = Downloader::new(db.clone(), endpoint.clone(), rt.clone()).await; - let live_sync = SyncEngine::spawn( - rt.clone(), - endpoint.clone(), - gossip.clone(), - docs.clone(), - db.clone(), - downloader, - ); - - // construct the state that is passed to the endpoint loop and from there cloned - // into to the connection handler task for incoming connections. - let state = Arc::new(State { - gossip: gossip.clone(), - bytes: IrohBytesHandlers::new(rt.clone(), db.clone()), - sync: live_sync.clone(), - }); - - // spawn our endpoint loop that forwards incoming connections - rt.main().spawn(endpoint_loop(endpoint.clone(), state)); - - // open our document and add to the live syncer - let namespace = Namespace::from_bytes(topic.as_bytes()); - println!("> opening doc {}", fmt_hash(namespace.id().as_bytes())); - let doc = match docs.open_replica(&namespace.id()) { - Ok(Some(doc)) => doc, - Err(_) | Ok(None) => docs.new_replica(namespace)?, - }; - live_sync.start_sync(doc.namespace(), peers.clone()).await?; - - // spawn an repl thread that reads stdin and parses each line as a `Cmd` command - let (cmd_tx, mut cmd_rx) = mpsc::channel(1); - std::thread::spawn(move || repl_loop(cmd_tx).expect("input loop crashed")); - // process commands in a loop - println!("> ready to accept commands"); - println!("> type `help` for a list of commands"); - - let current_watch: Arc>> = - Arc::new(tokio::sync::Mutex::new(None)); - - let watch = current_watch.clone(); - let mut doc_events = live_sync - .doc_subscribe(iroh::rpc_protocol::DocSubscribeRequest { - doc_id: doc.namespace(), - }) - .await; - rt.main().spawn(async move { - while let Some(Ok(event)) = doc_events.next().await { - let matcher = watch.lock().await; - if let Some(matcher) = &*matcher { - match event.event { - LiveEvent::ContentReady { .. } | LiveEvent::SyncFinished { .. } => {} - LiveEvent::InsertLocal { entry } | LiveEvent::InsertRemote { entry, .. } => { - let key = entry.id().key(); - if key.starts_with(matcher.as_bytes()) { - println!("change: {}", fmt_entry(&entry)); - } - } - _ => {} - } - } - } - }); - - let repl_state = ReplState { - rt, - store: docs, - author, - doc, - db, - ticket: our_ticket, - log_filter, - current_watch, - }; - - loop { - // wait for a command from the input repl thread - let Some((cmd, to_repl_tx)) = cmd_rx.recv().await else { - break; - }; - // exit command: break early - if let Cmd::Exit = cmd { - to_repl_tx.send(ToRepl::Exit).ok(); - break; - } - - // handle the command, but select against Ctrl-C signal so that commands can be aborted - tokio::select! { - biased; - _ = tokio::signal::ctrl_c() => { - println!("> aborted"); - } - res = repl_state.handle_command(cmd) => if let Err(err) = res { - println!("> error: {err}"); - }, - }; - // notify to the repl that we want to get the next command - to_repl_tx.send(ToRepl::Continue).ok(); - } - - // exit: cancel the sync and store blob database and document - if let Err(err) = live_sync.shutdown().await { - println!("> syncer closed with error: {err:?}"); - } - if let Some(metrics_fut) = metrics_fut { - metrics_fut.abort(); - drop(metrics_fut); - } - - Ok(()) -} - -struct ReplState { - rt: runtime::Handle, - store: store::fs::Store, - author: Author, - doc: Doc, - db: iroh_bytes::store::flat::Store, - ticket: Ticket, - log_filter: LogLevelReload, - current_watch: Arc>>, -} - -impl ReplState { - async fn handle_command(&self, cmd: Cmd) -> anyhow::Result<()> { - match cmd { - Cmd::Set { key, value } => { - let value = value.into_bytes(); - let len = value.len(); - let tag = self.db.import_bytes(value.into(), BlobFormat::Raw).await?; - self.doc - .insert(key, &self.author, *tag.hash(), len as u64)?; - } - Cmd::Get { - key, - print_content, - prefix, - } => { - let entries = if prefix { - self.store.get_many( - self.doc.namespace(), - GetFilter::Prefix(key.as_bytes().to_vec()), - )? - } else { - self.store.get_many( - self.doc.namespace(), - GetFilter::Key(key.as_bytes().to_vec()), - )? - }; - for entry in entries { - let entry = entry?; - println!("{}", fmt_entry(entry.entry())); - if print_content { - println!("{}", fmt_content(&self.db, &entry).await); - } - } - } - Cmd::Watch { key } => { - println!("watching key: '{key}'"); - self.current_watch.lock().await.replace(key); - } - Cmd::WatchCancel => match self.current_watch.lock().await.take() { - Some(key) => { - println!("canceled watching key: '{key}'"); - } - None => { - println!("no watch active"); - } - }, - Cmd::Ls { prefix } => { - let entries = match prefix { - None => self.store.get_many(self.doc.namespace(), GetFilter::All)?, - Some(prefix) => self.store.get_many( - self.doc.namespace(), - GetFilter::Prefix(prefix.as_bytes().to_vec()), - )?, - }; - let mut count = 0; - for entry in entries { - let entry = entry?; - count += 1; - println!("{}", fmt_entry(entry.entry()),); - } - println!("> {} entries", count); - } - Cmd::Ticket => { - println!("Ticket: {}", self.ticket); - } - Cmd::Log { directive } => { - let next_filter = EnvFilter::from_str(&directive)?; - self.log_filter.modify(|layer| *layer = next_filter)?; - } - Cmd::Stats => get_stats(), - Cmd::Fs(cmd) => self.handle_fs_command(cmd).await?, - Cmd::Hammer { - prefix, - threads, - count, - size, - mode, - } => { - println!( - "> Hammering with prefix \"{prefix}\" for {threads} x {count} messages of size {size} bytes in {mode} mode", - mode = format!("{mode:?}").to_lowercase() - ); - let start = Instant::now(); - let mut handles: Vec>> = Vec::new(); - match mode { - HammerMode::Set => { - let mut bytes = vec![0; size]; - // TODO: Add a flag to fill content differently per entry to be able to - // test downloading too - bytes.fill(97); - for t in 0..threads { - let prefix = prefix.clone(); - let doc = self.doc.clone(); - let bytes = bytes.clone(); - let db = self.db.clone(); - let author = self.author.clone(); - let handle = self.rt.main().spawn(async move { - for i in 0..count { - let value = - String::from_utf8(bytes.clone()).unwrap().into_bytes(); - let len = value.len(); - let key = format!("{}/{}/{}", prefix, t, i); - let tag = - db.import_bytes(value.into(), BlobFormat::Raw).await?; - doc.insert(key, &author, *tag.hash(), len as u64)?; - } - Ok(count) - }); - handles.push(handle); - } - } - HammerMode::Get => { - for t in 0..threads { - let prefix = prefix.clone(); - let doc = self.doc.clone(); - let store = self.store.clone(); - let handle = self.rt.main().spawn(async move { - let mut read = 0; - for i in 0..count { - let key = format!("{}/{}/{}", prefix, t, i); - let entries = store.get_many( - doc.namespace(), - GetFilter::Key(key.as_bytes().to_vec()), - )?; - for entry in entries { - let entry = entry?; - let _content = fmt_content_simple(&doc, &entry); - read += 1; - } - } - Ok(read) - }); - handles.push(handle); - } - } - } - - let mut total_count = 0; - for result in futures::future::join_all(handles).await { - // Check that no errors ocurred and count rows inserted/read - total_count += result??; - } - - let diff = start.elapsed().as_secs_f64(); - println!( - "> Hammering done in {diff:.2}s for {total_count} messages with total of {size}", - size = HumanBytes(total_count as u64 * size as u64), - ); - } - Cmd::Exit => {} - } - Ok(()) - } - - async fn handle_fs_command(&self, cmd: FsCmd) -> anyhow::Result<()> { - match cmd { - FsCmd::ImportFile { file_path, key } => { - let file_path = canonicalize_path(&file_path)?.canonicalize()?; - let (tag, len) = self - .db - .import_file( - file_path.clone(), - ImportMode::Copy, - BlobFormat::Raw, - IgnoreProgressSender::default(), - ) - .await?; - let hash = *tag.hash(); - self.doc.insert(key, &self.author, hash, len)?; - println!( - "> imported {file_path:?}: {} ({})", - fmt_hash(hash), - HumanBytes(len) - ); - } - FsCmd::ImportDir { - dir_path, - mut key_prefix, - } => { - if key_prefix.ends_with('/') { - key_prefix.pop(); - } - let root = canonicalize_path(&dir_path)?.canonicalize()?; - let files = walkdir::WalkDir::new(&root).into_iter(); - // TODO: parallelize - for file in files { - let file = file?; - if file.file_type().is_file() { - let relative = file.path().strip_prefix(&root)?.to_string_lossy(); - if relative.is_empty() { - warn!("invalid file path: {:?}", file.path()); - continue; - } - let key = format!("{key_prefix}/{relative}"); - let (tag, len) = self - .db - .import_file( - file.path().into(), - ImportMode::Copy, - BlobFormat::Raw, - IgnoreProgressSender::default(), - ) - .await?; - let hash = *tag.hash(); - self.doc.insert(key, &self.author, hash, len)?; - println!( - "> imported {relative}: {} ({})", - fmt_hash(hash), - HumanBytes(len) - ); - } - } - } - FsCmd::ExportDir { - mut key_prefix, - dir_path, - } => { - if !key_prefix.ends_with('/') { - key_prefix.push('/'); - } - let root = canonicalize_path(&dir_path)?; - println!("> exporting {key_prefix} to {root:?}"); - let entries = self.store.get_many( - self.doc.namespace(), - GetFilter::Prefix(key_prefix.as_bytes().to_vec()), - )?; - let mut checked_dirs = HashSet::new(); - for entry in entries { - let entry = entry?; - let key = entry.entry().id().key(); - let relative = String::from_utf8(key[key_prefix.len()..].to_vec())?; - let len = entry.entry().record().content_len(); - let blob = self.db.get(&entry.content_hash()); - if let Some(blob) = blob { - let mut reader = blob.data_reader().await?; - let path = root.join(&relative); - let parent = path.parent().unwrap(); - if !checked_dirs.contains(parent) { - tokio::fs::create_dir_all(&parent).await?; - checked_dirs.insert(parent.to_owned()); - } - let mut file = tokio::fs::File::create(&path).await?; - copy(&mut reader, &mut file).await?; - println!( - "> exported {} to {path:?} ({})", - fmt_hash(entry.content_hash()), - HumanBytes(len) - ); - } - } - } - FsCmd::ExportFile { key, file_path } => { - let path = canonicalize_path(&file_path)?; - // TODO: Fix - let entry = self - .store - .get_many( - self.doc.namespace(), - GetFilter::Key(key.as_bytes().to_vec()), - )? - .next(); - if let Some(entry) = entry { - let entry = entry?; - println!("> exporting {key} to {path:?}"); - let parent = path.parent().ok_or_else(|| anyhow!("Invalid path"))?; - tokio::fs::create_dir_all(&parent).await?; - let mut file = tokio::fs::File::create(&path).await?; - let blob = self - .db - .get(&entry.content_hash()) - .ok_or_else(|| anyhow!(format!("content for {key} is not available")))?; - let mut reader = blob.data_reader().await?; - copy(&mut reader, &mut file).await?; - } else { - println!("> key not found, abort"); - } - } - } - - Ok(()) - } -} - -#[derive(Parser, Debug)] -pub enum Cmd { - /// Set an entry - Set { - /// Key to the entry (parsed as UTF-8 string). - key: String, - /// Content to store for this entry (parsed as UTF-8 string) - value: String, - }, - /// Get entries by key - /// - /// Shows the author, content hash and content length for all entries for this key. - Get { - /// Key to the entry (parsed as UTF-8 string). - key: String, - /// Print the value (but only if it is valid UTF-8 and smaller than 1MB) - #[clap(short = 'c', long)] - print_content: bool, - /// Match the key as prefix, not an exact match. - #[clap(short = 'p', long)] - prefix: bool, - }, - /// List entries. - Ls { - /// Optionally list only entries whose key starts with PREFIX. - prefix: Option, - }, - - /// Import from and export to the local file system. - #[clap(subcommand)] - Fs(FsCmd), - - /// Print the ticket with which other peers can join our document. - Ticket, - /// Change the log level - Log { - /// The log level or log filtering directive - /// - /// Valid log levels are: "trace", "debug", "info", "warn", "error" - /// - /// You can also set one or more filtering directives to enable more fine-grained log - /// filtering. The supported filtering directives and their semantics are documented here: - /// https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives - /// - /// To disable logging completely, set to the empty string (via empty double quotes: ""). - #[clap(verbatim_doc_comment)] - directive: String, - }, - /// Watch for changes. - Watch { - /// The key to watch. - key: String, - }, - /// Cancels any running watch command. - WatchCancel, - /// Show stats about the current session - Stats, - /// Hammer time - stress test with the hammer - Hammer { - /// The hammer mode - #[clap(value_enum)] - mode: HammerMode, - /// The key prefix - prefix: String, - /// The number of threads to use (each thread will create it's own replica) - #[clap(long, short, default_value = "2")] - threads: usize, - /// The number of entries to create - #[clap(long, short, default_value = "1000")] - count: usize, - /// The size of each entry in Bytes - #[clap(long, short, default_value = "1024")] - size: usize, - }, - /// Quit - Exit, -} - -#[derive(Clone, Debug, clap::ValueEnum)] -pub enum HammerMode { - /// Create entries - Set, - /// Read entries - Get, -} - -#[derive(Parser, Debug)] -pub enum FsCmd { - /// Import a file system directory into the document. - ImportDir { - /// The file system path to import recursively - dir_path: String, - /// The key prefix to apply to the document keys - key_prefix: String, - }, - /// Import a file into the document. - ImportFile { - /// The path to the file - file_path: String, - /// The key in the document - key: String, - }, - /// Export a part of the document into a file system directory - ExportDir { - /// The key prefix to filter on - key_prefix: String, - /// The file system path to export to - dir_path: String, - }, - /// Import a file into the document. - ExportFile { - /// The key in the document - key: String, - /// The path to the file - file_path: String, - }, -} - -impl FromStr for Cmd { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - let args = shell_words::split(s)?; - let matches = Cmd::command() - .multicall(true) - .subcommand_required(true) - .try_get_matches_from(args)?; - let cmd = Cmd::from_arg_matches(&matches)?; - Ok(cmd) - } -} - -#[derive(Debug)] -struct State { - gossip: Gossip, - bytes: IrohBytesHandlers, - sync: SyncEngine, -} - -async fn endpoint_loop( - endpoint: MagicEndpoint, - state: Arc>, -) -> anyhow::Result<()> { - while let Some(conn) = endpoint.accept().await { - let state = state.clone(); - tokio::spawn(async move { - if let Err(err) = handle_connection(conn, state).await { - println!("> connection closed, reason: {err}"); - } - }); - } - Ok(()) -} - -async fn handle_connection( - mut conn: quinn::Connecting, - state: Arc>, -) -> anyhow::Result<()> { - let alpn = get_alpn(&mut conn).await?; - println!("> incoming connection with alpn {alpn}"); - match alpn.as_bytes() { - GOSSIP_ALPN => state.gossip.handle_connection(conn.await?).await, - SYNC_ALPN => state.sync.handle_connection(conn).await, - alpn if alpn == iroh_bytes::protocol::ALPN => state.bytes.handle_connection(conn).await, - _ => bail!("ignoring connection: unsupported ALPN protocol"), - } -} - -#[derive(Debug)] -enum ToRepl { - Continue, - Exit, -} - -fn repl_loop(cmd_tx: mpsc::Sender<(Cmd, oneshot::Sender)>) -> anyhow::Result<()> { - use rustyline::{error::ReadlineError, Config, DefaultEditor}; - let mut rl = DefaultEditor::with_config(Config::builder().check_cursor_position(true).build())?; - loop { - // prepare a channel to receive a signal from the main thread when a command completed - let (to_repl_tx, to_repl_rx) = oneshot::channel(); - let readline = rl.readline(">> "); - match readline { - Ok(line) if line.is_empty() => continue, - Ok(line) => { - rl.add_history_entry(line.as_str())?; - match Cmd::from_str(&line) { - Ok(cmd) => cmd_tx.blocking_send((cmd, to_repl_tx))?, - Err(err) => { - println!("{err}"); - continue; - } - }; - } - Err(ReadlineError::Interrupted | ReadlineError::Eof) => { - cmd_tx.blocking_send((Cmd::Exit, to_repl_tx))?; - } - Err(ReadlineError::WindowResized) => continue, - Err(err) => return Err(err.into()), - } - // wait for reply from main thread - match to_repl_rx.blocking_recv()? { - ToRepl::Continue => continue, - ToRepl::Exit => break, - } - } - Ok(()) -} - -fn get_stats() { - let Ok(stats) = iroh::metrics::get_metrics() else { - println!("metrics collection is disabled"); - return; - }; - for (name, details) in stats.iter() { - println!( - "{:23} : {:>6} ({})", - name, details.value, details.description - ); - } -} - -#[derive(Debug, Serialize, Deserialize)] -struct Ticket { - topic: TopicId, - peers: Vec, -} -impl Ticket { - /// Deserializes from bytes. - fn from_bytes(bytes: &[u8]) -> anyhow::Result { - postcard::from_bytes(bytes).map_err(Into::into) - } - /// Serializes to bytes. - pub fn to_bytes(&self) -> Vec { - postcard::to_stdvec(self).expect("postcard::to_stdvec is infallible") - } -} - -/// Serializes to base32. -impl fmt::Display for Ticket { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let encoded = self.to_bytes(); - let mut text = data_encoding::BASE32_NOPAD.encode(&encoded); - text.make_ascii_lowercase(); - write!(f, "{text}") - } -} - -/// Deserializes from base32. -impl FromStr for Ticket { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - let bytes = data_encoding::BASE32_NOPAD.decode(s.to_ascii_uppercase().as_bytes())?; - let slf = Self::from_bytes(&bytes)?; - Ok(slf) - } -} - -type LogLevelReload = tracing_subscriber::reload::Handle; -fn init_logging() -> LogLevelReload { - use tracing_subscriber::{filter, fmt, prelude::*, reload}; - let filter = filter::EnvFilter::from_default_env(); - let (filter, reload_handle) = reload::Layer::new(filter); - tracing_subscriber::registry() - .with(filter) - .with(fmt::Layer::default()) - .init(); - reload_handle -} - -// helpers - -fn fmt_entry(entry: &Entry) -> String { - let id = entry.id(); - let key = std::str::from_utf8(id.key()).unwrap_or(""); - let author = fmt_hash(id.author().as_bytes()); - let hash = entry.record().content_hash(); - let hash = fmt_hash(hash.as_bytes()); - let len = HumanBytes(entry.record().content_len()); - format!("@{author}: {key} = {hash} ({len})",) -} - -async fn fmt_content_simple(_doc: &Doc, entry: &SignedEntry) -> String { - let len = entry.entry().record().content_len(); - format!("<{}>", HumanBytes(len)) -} - -async fn fmt_content(db: &B, entry: &SignedEntry) -> String { - let len = entry.entry().record().content_len(); - if len > MAX_DISPLAY_CONTENT_LEN { - format!("<{}>", HumanBytes(len)) - } else { - match read_content(db, entry).await { - Err(err) => format!(""), - Ok(content) => match String::from_utf8(content.into()) { - Ok(str) => str, - Err(_err) => format!("", HumanBytes(len)), - }, - } - } -} - -async fn read_content(db: &B, entry: &SignedEntry) -> anyhow::Result { - let data = db - .get(&entry.content_hash()) - .ok_or_else(|| anyhow!("not found"))? - .data_reader() - .await? - .read_to_end() - .await?; - Ok(data) -} -fn fmt_hash(hash: impl AsRef<[u8]>) -> String { - let mut text = data_encoding::BASE32_NOPAD.encode(hash.as_ref()); - text.make_ascii_lowercase(); - format!("{}…{}", &text[..5], &text[(text.len() - 2)..]) -} -fn fmt_derp_mode(derp_mode: &DerpMode) -> String { - match derp_mode { - DerpMode::Disabled => "None".to_string(), - DerpMode::Default => "Default Derp servers".to_string(), - DerpMode::Custom(map) => map - .regions() - .flat_map(|region| region.nodes.iter().map(|node| node.url.to_string())) - .collect::>() - .join(", "), - } -} -fn canonicalize_path(path: &str) -> anyhow::Result { - let path = PathBuf::from(shellexpand::tilde(&path).to_string()); - Ok(path) -} - -/// Copy from a [`iroh_io::AsyncSliceReader`] into a [`tokio::io::AsyncWrite`] -/// -/// TODO: move to iroh-io or iroh-bytes -async fn copy( - mut reader: impl iroh_io::AsyncSliceReader, - mut writer: impl tokio::io::AsyncWrite + Unpin, -) -> anyhow::Result<()> { - // this is the max chunk size. - // will only allocate this much if the resource behind the reader is at least this big. - let chunk_size = 1024 * 16; - let mut pos = 0u64; - loop { - let chunk = reader.read_at(pos, chunk_size).await?; - if chunk.is_empty() { - break; - } - writer.write_all(&chunk).await?; - pos += chunk.len() as u64; - } - Ok(()) -} - -/// handlers for iroh_bytes connections -mod iroh_bytes_handlers { - use std::sync::Arc; - - use futures::{future::BoxFuture, FutureExt}; - use iroh_bytes::{ - protocol::RequestToken, - provider::{EventSender, RequestAuthorizationHandler}, - }; - - #[derive(Debug, Clone)] - pub struct IrohBytesHandlers { - db: iroh_bytes::store::flat::Store, - rt: iroh_bytes::util::runtime::Handle, - event_sender: NoopEventSender, - auth_handler: Arc, - } - impl IrohBytesHandlers { - pub fn new( - rt: iroh_bytes::util::runtime::Handle, - db: iroh_bytes::store::flat::Store, - ) -> Self { - Self { - db, - rt, - event_sender: NoopEventSender, - auth_handler: Arc::new(NoopRequestAuthorizationHandler), - } - } - pub async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { - iroh_bytes::provider::handle_connection( - conn, - self.db.clone(), - self.event_sender.clone(), - self.auth_handler.clone(), - self.rt.clone(), - ) - .await; - Ok(()) - } - } - - #[derive(Debug, Clone)] - struct NoopEventSender; - impl EventSender for NoopEventSender { - fn send(&self, _event: iroh_bytes::provider::Event) -> BoxFuture<()> { - async {}.boxed() - } - } - #[derive(Debug)] - struct NoopRequestAuthorizationHandler; - impl RequestAuthorizationHandler for NoopRequestAuthorizationHandler { - fn authorize( - &self, - token: Option, - _request: &iroh_bytes::protocol::Request, - ) -> BoxFuture<'static, anyhow::Result<()>> { - async move { - if let Some(token) = token { - anyhow::bail!( - "no authorization handler defined, but token was provided: {:?}", - token - ); - } - Ok(()) - } - .boxed() - } - } -} diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 2392bf1d8d..f6508cd2c4 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -7,6 +7,8 @@ use std::io::{self, Cursor}; use std::path::PathBuf; use std::pin::Pin; use std::result::Result as StdResult; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use std::task::{Context, Poll}; use anyhow::{anyhow, Result}; @@ -18,7 +20,9 @@ use iroh_bytes::store::ValidateProgress; use iroh_bytes::Hash; use iroh_bytes::{BlobFormat, Tag}; use iroh_net::{key::PublicKey, magic_endpoint::ConnectionInfo, PeerAddr}; +use iroh_sync::actor::OpenState; use iroh_sync::{store::GetFilter, AuthorId, Entry, NamespaceId}; +use quic_rpc::message::RpcMsg; use quic_rpc::{RpcClient, ServiceConnection}; use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; use tokio_util::io::{ReaderStream, StreamReader}; @@ -29,15 +33,15 @@ use crate::rpc_protocol::{ BlobAddStreamUpdate, BlobDeleteBlobRequest, BlobDownloadRequest, BlobListCollectionsRequest, BlobListCollectionsResponse, BlobListIncompleteRequest, BlobListIncompleteResponse, BlobListRequest, BlobListResponse, BlobReadRequest, BlobReadResponse, BlobValidateRequest, - CounterStats, DeleteTagRequest, DocCreateRequest, DocDelRequest, DocDelResponse, - DocDropRequest, DocGetManyRequest, DocGetOneRequest, DocImportRequest, DocInfoRequest, - DocLeaveRequest, DocListRequest, DocSetHashRequest, DocSetRequest, DocShareRequest, - DocStartSyncRequest, DocSubscribeRequest, DocTicket, GetProgress, ListTagsRequest, - ListTagsResponse, NodeConnectionInfoRequest, NodeConnectionInfoResponse, - NodeConnectionsRequest, NodeShutdownRequest, NodeStatsRequest, NodeStatusRequest, - NodeStatusResponse, ProviderService, SetTagOption, ShareMode, WrapOption, + CounterStats, DeleteTagRequest, DocCloseRequest, DocCreateRequest, DocDelRequest, + DocDelResponse, DocDropRequest, DocGetManyRequest, DocGetOneRequest, DocImportRequest, + DocLeaveRequest, DocListRequest, DocOpenRequest, DocSetHashRequest, DocSetRequest, + DocShareRequest, DocStartSyncRequest, DocStatusRequest, DocSubscribeRequest, DocTicket, + GetProgress, ListTagsRequest, ListTagsResponse, NodeConnectionInfoRequest, + NodeConnectionInfoResponse, NodeConnectionsRequest, NodeShutdownRequest, NodeStatsRequest, + NodeStatusRequest, NodeStatusResponse, ProviderService, SetTagOption, ShareMode, WrapOption, }; -use crate::sync_engine::{LiveEvent, LiveStatus}; +use crate::sync_engine::LiveEvent; pub mod mem; #[cfg(feature = "cli")] @@ -134,10 +138,7 @@ where /// Create a new document. pub async fn create(&self) -> Result> { let res = self.rpc.rpc(DocCreateRequest {}).await??; - let doc = Doc { - id: res.id, - rpc: self.rpc.clone(), - }; + let doc = Doc::new(self.rpc.clone(), res.id); Ok(doc) } @@ -154,10 +155,7 @@ where /// Import a document from a ticket and join all peers in the ticket. pub async fn import(&self, ticket: DocTicket) -> Result> { let res = self.rpc.rpc(DocImportRequest(ticket)).await??; - let doc = Doc { - id: res.doc_id, - rpc: self.rpc.clone(), - }; + let doc = Doc::new(self.rpc.clone(), res.doc_id); Ok(doc) } @@ -168,14 +166,9 @@ where } /// Get a [`Doc`] client for a single document. Return None if the document cannot be found. - pub async fn get(&self, id: NamespaceId) -> Result>> { - if let Err(_err) = self.rpc.rpc(DocInfoRequest { doc_id: id }).await? { - return Ok(None); - } - let doc = Doc { - id, - rpc: self.rpc.clone(), - }; + pub async fn open(&self, id: NamespaceId) -> Result>> { + self.rpc.rpc(DocOpenRequest { doc_id: id }).await??; + let doc = Doc::new(self.rpc.clone(), id); Ok(Some(doc)) } } @@ -523,34 +516,82 @@ impl AsyncRead for BlobReader { /// Document handle #[derive(Debug, Clone)] -pub struct Doc { +pub struct Doc>(Arc>); + +#[derive(Debug)] +struct DocInner> { id: NamespaceId, rpc: RpcClient, + closed: AtomicBool, +} + +impl Drop for DocInner +where + C: ServiceConnection, +{ + fn drop(&mut self) { + let doc_id = self.id; + let rpc = self.rpc.clone(); + tokio::task::spawn(async move { + rpc.rpc(DocCloseRequest { doc_id }).await.ok(); + }); + } } impl Doc where C: ServiceConnection, { + fn new(rpc: RpcClient, id: NamespaceId) -> Self { + Self(Arc::new(DocInner { + rpc, + id, + closed: AtomicBool::new(false), + })) + } + + async fn rpc(&self, msg: M) -> Result + where + M: RpcMsg, + { + let res = self.0.rpc.rpc(msg).await?; + Ok(res) + } + /// Get the document id of this doc. pub fn id(&self) -> NamespaceId { - self.id + self.0.id + } + + /// Close the document. + pub async fn close(&self) -> Result<()> { + self.0.closed.store(true, Ordering::Release); + self.rpc(DocCloseRequest { doc_id: self.id() }).await??; + Ok(()) + } + + fn ensure_open(&self) -> Result<()> { + if self.0.closed.load(Ordering::Acquire) { + Err(anyhow!("document is closed")) + } else { + Ok(()) + } } /// Set the content of a key to a byte array. pub async fn set_bytes( &self, author_id: AuthorId, - key: Vec, - value: Vec, + key: impl Into, + value: impl Into, ) -> Result { + self.ensure_open()?; let res = self - .rpc .rpc(DocSetRequest { - doc_id: self.id, + doc_id: self.id(), author_id, - key, - value, + key: key.into(), + value: value.into(), }) .await??; Ok(res.entry.content_hash()) @@ -560,30 +601,32 @@ where pub async fn set_hash( &self, author_id: AuthorId, - key: Vec, + key: impl Into, hash: Hash, size: u64, ) -> Result<()> { - self.rpc - .rpc(DocSetHashRequest { - doc_id: self.id, - author_id, - key, - hash, - size, - }) - .await??; + self.ensure_open()?; + self.rpc(DocSetHashRequest { + doc_id: self.id(), + author_id, + key: key.into(), + hash, + size, + }) + .await??; Ok(()) } /// Read the content of an [`Entry`] as a streaming [`BlobReader`]. pub async fn read(&self, entry: &Entry) -> Result { - BlobReader::from_rpc(&self.rpc, entry.content_hash()).await + self.ensure_open()?; + BlobReader::from_rpc(&self.0.rpc, entry.content_hash()).await } /// Read all content of an [`Entry`] into a buffer. pub async fn read_to_bytes(&self, entry: &Entry) -> Result { - BlobReader::from_rpc(&self.rpc, entry.content_hash()) + self.ensure_open()?; + BlobReader::from_rpc(&self.0.rpc, entry.content_hash()) .await? .read_to_bytes() .await @@ -595,13 +638,13 @@ where /// entries whose key starts with or is equal to the given `prefix`. /// /// Returns the number of entries deleted. - pub async fn del(&self, author_id: AuthorId, prefix: Vec) -> Result { + pub async fn del(&self, author_id: AuthorId, prefix: impl Into) -> Result { + self.ensure_open()?; let res = self - .rpc .rpc(DocDelRequest { - doc_id: self.id, + doc_id: self.id(), author_id, - prefix, + prefix: prefix.into(), }) .await??; let DocDelResponse { removed } = res; @@ -609,13 +652,13 @@ where } /// Get the latest entry for a key and author. - pub async fn get_one(&self, author: AuthorId, key: Vec) -> Result> { + pub async fn get_one(&self, author: AuthorId, key: impl Into) -> Result> { + self.ensure_open()?; let res = self - .rpc .rpc(DocGetOneRequest { author, - key, - doc_id: self.id, + key: key.into(), + doc_id: self.id(), }) .await??; Ok(res.entry.map(|entry| entry.into())) @@ -623,10 +666,12 @@ where /// Get entries. pub async fn get_many(&self, filter: GetFilter) -> Result>> { + self.ensure_open()?; let stream = self + .0 .rpc .server_streaming(DocGetManyRequest { - doc_id: self.id, + doc_id: self.id(), filter, }) .await?; @@ -635,10 +680,10 @@ where /// Share this document with peers over a ticket. pub async fn share(&self, mode: ShareMode) -> anyhow::Result { + self.ensure_open()?; let res = self - .rpc .rpc(DocShareRequest { - doc_id: self.id, + doc_id: self.id(), mode, }) .await??; @@ -647,10 +692,10 @@ where /// Start to sync this document with a list of peers. pub async fn start_sync(&self, peers: Vec) -> Result<()> { + self.ensure_open()?; let _res = self - .rpc .rpc(DocStartSyncRequest { - doc_id: self.id, + doc_id: self.id(), peers, }) .await??; @@ -659,22 +704,26 @@ where /// Stop the live sync for this document. pub async fn leave(&self) -> Result<()> { - let _res = self.rpc.rpc(DocLeaveRequest { doc_id: self.id }).await??; + self.ensure_open()?; + let _res = self.rpc(DocLeaveRequest { doc_id: self.id() }).await??; Ok(()) } /// Subscribe to events for this document. pub async fn subscribe(&self) -> anyhow::Result>> { + self.ensure_open()?; let stream = self + .0 .rpc - .server_streaming(DocSubscribeRequest { doc_id: self.id }) + .server_streaming(DocSubscribeRequest { doc_id: self.id() }) .await?; Ok(flatten(stream).map_ok(|res| res.event).map_err(Into::into)) } /// Get status info for this document - pub async fn status(&self) -> anyhow::Result { - let res = self.rpc.rpc(DocInfoRequest { doc_id: self.id }).await??; + pub async fn status(&self) -> anyhow::Result { + self.ensure_open()?; + let res = self.rpc(DocStatusRequest { doc_id: self.id() }).await??; Ok(res.status) } } diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 6b835cc322..5db8bd914d 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -521,7 +521,6 @@ impl DocCommands { LiveEvent::NeighborDown(peer) => { println!("neighbor peer down: {peer:?}"); } - LiveEvent::Closed => println!("document closed"), } } } @@ -550,7 +549,7 @@ impl DocCommands { async fn get_doc(iroh: &Iroh, env: &ConsoleEnv, id: Option) -> anyhow::Result { iroh.docs - .get(env.doc(id)?) + .open(env.doc(id)?) .await? .context("Document not found") } diff --git a/iroh/src/downloader.rs b/iroh/src/downloader.rs index f3251906fa..2e4b1f0392 100644 --- a/iroh/src/downloader.rs +++ b/iroh/src/downloader.rs @@ -29,6 +29,10 @@ use std::{ collections::{hash_map::Entry, HashMap, VecDeque}, num::NonZeroUsize, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; use bao_tree::ChunkRanges; @@ -37,7 +41,7 @@ use iroh_bytes::{protocol::RangeSpecSeq, store::Store, Hash, HashAndFormat, Temp use iroh_net::{key::PublicKey, MagicEndpoint}; use tokio::sync::{mpsc, oneshot}; use tokio_util::{sync::CancellationToken, time::delay_queue}; -use tracing::{debug, error_span, trace, Instrument}; +use tracing::{debug, error_span, trace, warn, Instrument}; mod get; mod invariants; @@ -207,10 +211,10 @@ impl std::future::Future for DownloadHandle { } /// Handle for the download services. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct Downloader { /// Next id to use for a download intent. - next_id: Id, + next_id: Arc, /// Channel to communicate with the service. msg_tx: mpsc::Sender, } @@ -238,13 +242,15 @@ impl Downloader { service.run().instrument(error_span!("downloader", %me)) }; rt.local_pool().spawn_pinned(create_future); - Self { next_id: 0, msg_tx } + Self { + next_id: Arc::new(AtomicU64::new(0)), + msg_tx, + } } /// Queue a download. pub async fn queue(&mut self, kind: DownloadKind, peers: Vec) -> DownloadHandle { - let id = self.next_id; - self.next_id = self.next_id.wrapping_add(1); + let id = self.next_id.fetch_add(1, Ordering::SeqCst); let (sender, receiver) = oneshot::channel(); let handle = DownloadHandle { @@ -888,7 +894,7 @@ impl, D: Dialer> Service { let next_peer = self.get_best_candidate(kind.hash()); self.schedule_request(kind, remaining_retries, next_peer, intents); } else { - debug!(%peer, ?kind, %reason, "download failed"); + warn!(%peer, ?kind, %reason, "download failed"); for sender in intents.into_values() { let _ = sender.send(Err(anyhow::anyhow!("download ran out of attempts"))); } diff --git a/iroh/src/downloader/get.rs b/iroh/src/downloader/get.rs index 92d5698de7..cde856fc52 100644 --- a/iroh/src/downloader/get.rs +++ b/iroh/src/downloader/get.rs @@ -223,9 +223,6 @@ pub async fn get( BlobFormat::Raw => get_blob(db, conn, &hash).await, BlobFormat::HashSeq => get_hash_seq(db, conn, &hash).await, }; - if let Err(e) = stats.as_ref() { - tracing::error!("get failed: {e:?}"); - } Ok((stats?, tt)) } diff --git a/iroh/src/downloader/test.rs b/iroh/src/downloader/test.rs index 94feb55572..66caebba7c 100644 --- a/iroh/src/downloader/test.rs +++ b/iroh/src/downloader/test.rs @@ -27,7 +27,10 @@ impl Downloader { service.run().await }); - Downloader { next_id: 0, msg_tx } + Downloader { + next_id: Arc::new(AtomicU64::new(0)), + msg_tx, + } } } diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 17286f698d..d36e4aa1f7 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -1480,16 +1480,28 @@ fn handle_rpc_request { chan.rpc(msg, handler, |handler, req| async move { - handler.inner.sync.author_create(req) + handler.inner.sync.author_create(req).await }) .await } AuthorImport(_msg) => { todo!() } - DocInfo(msg) => { + DocOpen(msg) => { chan.rpc(msg, handler, |handler, req| async move { - handler.inner.sync.doc_info(req).await + handler.inner.sync.doc_open(req).await + }) + .await + } + DocClose(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.doc_close(req).await + }) + .await + } + DocStatus(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.doc_status(req).await }) .await } @@ -1501,7 +1513,7 @@ fn handle_rpc_request { chan.rpc(msg, handler, |handler, req| async move { - handler.inner.sync.doc_create(req) + handler.inner.sync.doc_create(req).await }) .await } diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 865d339940..5aa44a8612 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -20,6 +20,7 @@ use iroh_net::{ }; use iroh_sync::{ + actor::OpenState, store::GetFilter, sync::{NamespaceId, SignedEntry}, AuthorId, @@ -32,7 +33,7 @@ use serde::{Deserialize, Serialize}; pub use iroh_bytes::{provider::AddProgress, store::ValidateProgress, util::RpcResult}; -use crate::sync_engine::{LiveEvent, LiveStatus}; +use crate::sync_engine::LiveEvent; /// A 32-byte key or token pub type KeyBytes = [u8; 32]; @@ -568,23 +569,53 @@ pub struct DocShareResponse(pub DocTicket); /// Get info on a document #[derive(Serialize, Deserialize, Debug)] -pub struct DocInfoRequest { +pub struct DocStatusRequest { /// The document id pub doc_id: NamespaceId, } -impl RpcMsg for DocInfoRequest { - type Response = RpcResult; +impl RpcMsg for DocStatusRequest { + type Response = RpcResult; } -/// Response to [`DocInfoRequest`] +/// Response to [`DocStatusRequest`] // TODO: actually provide info #[derive(Serialize, Deserialize, Debug)] -pub struct DocInfoResponse { +pub struct DocStatusResponse { /// Live sync status - pub status: LiveStatus, + pub status: OpenState, } +/// Open a document +#[derive(Serialize, Deserialize, Debug)] +pub struct DocOpenRequest { + /// The document id + pub doc_id: NamespaceId, +} + +impl RpcMsg for DocOpenRequest { + type Response = RpcResult; +} + +/// Response to [`DocOpenRequest`] +#[derive(Serialize, Deserialize, Debug)] +pub struct DocOpenResponse {} + +/// Open a document +#[derive(Serialize, Deserialize, Debug)] +pub struct DocCloseRequest { + /// The document id + pub doc_id: NamespaceId, +} + +impl RpcMsg for DocCloseRequest { + type Response = RpcResult; +} + +/// Response to [`DocCloseRequest`] +#[derive(Serialize, Deserialize, Debug)] +pub struct DocCloseResponse {} + /// Start to sync a doc with peers. #[derive(Serialize, Deserialize, Debug)] pub struct DocStartSyncRequest { @@ -640,11 +671,11 @@ pub struct DocSetRequest { /// Author of this entry. pub author_id: AuthorId, /// Key of this entry. - pub key: Vec, + pub key: Bytes, /// Value of this entry. // TODO: Allow to provide the hash directly // TODO: Add a way to provide content as stream - pub value: Vec, + pub value: Bytes, } impl RpcMsg for DocSetRequest { @@ -666,7 +697,7 @@ pub struct DocDelRequest { /// Author of this entry. pub author_id: AuthorId, /// Prefix to delete. - pub prefix: Vec, + pub prefix: Bytes, } impl RpcMsg for DocDelRequest { @@ -688,7 +719,7 @@ pub struct DocSetHashRequest { /// Author of this entry. pub author_id: AuthorId, /// Key of this entry. - pub key: Vec, + pub key: Bytes, /// Hash of this entry. pub hash: Hash, /// Size of this entry. @@ -733,7 +764,7 @@ pub struct DocGetOneRequest { /// The document id pub doc_id: NamespaceId, /// Key - pub key: Vec, + pub key: Bytes, /// Author pub author: AuthorId, } @@ -863,7 +894,9 @@ pub enum ProviderRequest { DeleteTag(DeleteTagRequest), ListTags(ListTagsRequest), - DocInfo(DocInfoRequest), + DocOpen(DocOpenRequest), + DocClose(DocCloseRequest), + DocStatus(DocStatusRequest), DocList(DocListRequest), DocCreate(DocCreateRequest), DocDrop(DocDropRequest), @@ -906,7 +939,9 @@ pub enum ProviderResponse { ListTags(ListTagsResponse), DeleteTag(RpcResult<()>), - DocInfo(RpcResult), + DocOpen(RpcResult), + DocClose(RpcResult), + DocStatus(RpcResult), DocList(RpcResult), DocCreate(RpcResult), DocDrop(RpcResult), diff --git a/iroh/src/sync_engine.rs b/iroh/src/sync_engine.rs index ffb9e428a9..fb459397f6 100644 --- a/iroh/src/sync_engine.rs +++ b/iroh/src/sync_engine.rs @@ -2,65 +2,147 @@ //! //! [`iroh_sync::Replica`] is also called documents here. -use anyhow::anyhow; -use iroh_bytes::{store::Store as BaoStore, util::runtime::Handle}; +use std::sync::Arc; + +use anyhow::Result; +use futures::{ + future::{BoxFuture, FutureExt, Shared}, + Stream, TryStreamExt, +}; +use iroh_bytes::{store::EntryStatus, util::runtime::Handle, Hash}; use iroh_gossip::net::Gossip; -use iroh_net::{MagicEndpoint, PeerAddr}; +use iroh_net::{key::PublicKey, MagicEndpoint, PeerAddr}; use iroh_sync::{ - store::Store, - sync::{Author, AuthorId, NamespaceId, Replica}, + actor::SyncHandle, store::Store, sync::NamespaceId, ContentStatus, ContentStatusCallback, + Entry, InsertOrigin, }; +use serde::{Deserialize, Serialize}; +use tokio::sync::{mpsc, oneshot}; +use tokio_stream::StreamExt; +use tracing::{error, error_span, Instrument}; use crate::downloader::Downloader; +mod gossip; mod live; pub mod rpc; +use gossip::GossipActor; +use live::{LiveActor, ToLiveActor}; + +pub use self::live::{Origin, SyncEvent}; pub use iroh_sync::net::SYNC_ALPN; -pub use live::*; -/// The SyncEngine contains the [`LiveSync`] handle, and keeps a copy of the store and endpoint. +/// Capacity of the channel for the [`ToLiveActor`] messages. +const ACTOR_CHANNEL_CAP: usize = 64; +/// Capacity for the channels for [`SyncEngine::subscribe`]. +const SUBSCRIBE_CHANNEL_CAP: usize = 256; + +/// The sync engine coordinates actors that manage open documents, set-reconciliation syncs with +/// peers and a gossip swarm for each syncing document. /// /// The RPC methods dealing with documents and sync operate on the `SyncEngine`, with method /// implementations in [rpc]. -#[derive(Debug, Clone)] +#[derive(derive_more::Debug, Clone)] pub struct SyncEngine { pub(crate) rt: Handle, - pub(crate) store: S, pub(crate) endpoint: MagicEndpoint, - pub(crate) live: LiveSync, + pub(crate) sync: SyncHandle, + to_live_actor: mpsc::Sender, + tasks_fut: Shared>, + #[debug("ContentStatusCallback")] + content_status_cb: ContentStatusCallback, + + // TODO: + // After the latest refactoring we don't need the store here anymore because all interactions + // go over the [`SyncHandle`]. Removing the store removes the `S: Store` generic from the + // `SyncEngine`, in turn removing the `S: Store` generic from [`iroh::node::Node`]. Yay! + // As this changes the code in many lines, I'd defer it to a follwup. + _store: S, } impl SyncEngine { /// Start the sync engine. /// - /// This will spawn a background task for the [`LiveSync`]. When documents are added to the - /// engine with [`Self::start_sync`], then new entries inserted locally will be sent to peers - /// through iroh-gossip. - /// - /// The engine will also register for [`Replica::subscribe`] events to download content for new - /// entries from peers. - pub fn spawn( + /// This will spawn two tokio tasks for the live sync coordination and gossip actors, and a + /// thread for the [`iroh_sync::actor::SyncHandle`]. + pub fn spawn( rt: Handle, endpoint: MagicEndpoint, gossip: Gossip, - store: S, + replica_store: S, bao_store: B, downloader: Downloader, ) -> Self { - let live = LiveSync::spawn( - rt.clone(), + let (live_actor_tx, to_live_actor_recv) = mpsc::channel(ACTOR_CHANNEL_CAP); + let (to_gossip_actor, to_gossip_actor_recv) = mpsc::channel(ACTOR_CHANNEL_CAP); + let me = endpoint.peer_id().fmt_short(); + + let content_status_cb = { + let bao_store = bao_store.clone(); + Arc::new(move |hash| entry_to_content_status(bao_store.contains(&hash))) + }; + let sync = SyncHandle::spawn( + replica_store.clone(), + Some(content_status_cb.clone()), + me.clone(), + ); + + let mut actor = LiveActor::new( + sync.clone(), endpoint.clone(), - store.clone(), - gossip, + gossip.clone(), bao_store, + downloader.clone(), + to_live_actor_recv, + live_actor_tx.clone(), + to_gossip_actor, + ); + let mut gossip_actor = GossipActor::new( + to_gossip_actor_recv, + sync.clone(), + gossip, downloader, + live_actor_tx.clone(), + ); + let live_actor_task = rt.main().spawn( + async move { + if let Err(err) = actor.run().await { + error!("sync actor failed: {err:?}"); + } + } + .instrument(error_span!("sync", %me)), ); + let gossip_actor_task = rt.main().spawn( + async move { + if let Err(err) = gossip_actor.run().await { + error!("gossip recv actor failed: {err:?}"); + } + } + .instrument(error_span!("sync", %me)), + ); + let tasks_fut = async move { + if let Err(err) = live_actor_task.await { + error!("Error while joining actor task: {err:?}"); + } + gossip_actor_task.abort(); + if let Err(err) = gossip_actor_task.await { + if !err.is_cancelled() { + error!("Error while joining gossip recv task task: {err:?}"); + } + } + } + .boxed() + .shared(); + Self { - live, - store, rt, endpoint, + sync, + to_live_actor: live_actor_tx, + tasks_fut, + content_status_cb, + _store: replica_store, } } @@ -68,40 +150,178 @@ impl SyncEngine { /// /// If `peers` is non-empty, it will both do an initial set-reconciliation sync with each peer, /// and join an iroh-gossip swarm with these peers to receive and broadcast document updates. - pub async fn start_sync( + pub async fn start_sync(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { + let (reply, reply_rx) = oneshot::channel(); + self.to_live_actor + .send(ToLiveActor::StartSync { + namespace, + peers, + reply, + }) + .await?; + reply_rx.await??; + Ok(()) + } + + /// Join and sync with a set of peers for a document that is already syncing. + pub async fn join_peers(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { + let (reply, reply_rx) = oneshot::channel(); + self.to_live_actor + .send(ToLiveActor::JoinPeers { + namespace, + peers, + reply, + }) + .await?; + reply_rx.await??; + Ok(()) + } + + /// Stop the live sync for a document and leave the gossip swarm. + /// + /// If `kill_subscribers` is true, all existing event subscribers will be dropped. This means + /// they will receive `None` and no further events in case of rejoining the document. + pub async fn leave(&self, namespace: NamespaceId, kill_subscribers: bool) -> Result<()> { + let (reply, reply_rx) = oneshot::channel(); + self.to_live_actor + .send(ToLiveActor::Leave { + namespace, + kill_subscribers, + reply, + }) + .await?; + reply_rx.await??; + Ok(()) + } + + /// Subscribe to replica and sync progress events. + pub fn subscribe( &self, namespace: NamespaceId, - peers: Vec, - ) -> anyhow::Result<()> { - self.live.start_sync(namespace, peers).await + ) -> impl Stream> + Unpin + 'static { + let content_status_cb = self.content_status_cb.clone(); + + // Create a future that sends channel senders to the respective actors. + // We clone `self` so that the future does not capture any lifetimes. + let this = self.clone(); + let fut = async move { + // Subscribe to insert events from the replica. + let replica_events = { + let (s, r) = flume::bounded(SUBSCRIBE_CHANNEL_CAP); + this.sync.subscribe(namespace, s).await?; + r.into_stream() + .map(move |ev| LiveEvent::from_replica_event(ev, &content_status_cb)) + }; + + // Subscribe to events from the [`live::Actor`]. + let sync_events = { + let (s, r) = flume::bounded(SUBSCRIBE_CHANNEL_CAP); + let (reply, reply_rx) = oneshot::channel(); + this.to_live_actor + .send(ToLiveActor::Subscribe { + namespace, + sender: s, + reply, + }) + .await?; + reply_rx.await??; + r.into_stream().map(|event| Ok(LiveEvent::from(event))) + }; + + // Merge the two receivers into a single stream. + let stream = replica_events.merge(sync_events); + // We need type annotations for the error type here. + Result::<_, anyhow::Error>::Ok(stream) + }; + + // Flatten the future into a single stream. If the future errors, the error will be + // returned from the first call to [`Stream::next`]. + // We first pin the future so that the resulting stream is `Unpin`. + Box::pin(fut).into_stream().try_flatten() } - /// Stop syncing a document. - pub async fn leave(&self, namespace: NamespaceId, force_close: bool) -> anyhow::Result<()> { - self.live.leave(namespace, force_close).await + /// Handle an incoming iroh-sync connection. + pub async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { + self.to_live_actor + .send(ToLiveActor::HandleConnection { conn }) + .await?; + Ok(()) } /// Shutdown the sync engine. - pub async fn shutdown(&self) -> anyhow::Result<()> { - self.live.shutdown().await + pub async fn shutdown(&self) -> Result<()> { + self.to_live_actor.send(ToLiveActor::Shutdown).await?; + self.tasks_fut.clone().await; + Ok(()) } +} - /// Get a [`Replica`] from the store, returning an error if the replica does not exist. - pub fn get_replica(&self, id: &NamespaceId) -> anyhow::Result> { - self.store - .open_replica(id)? - .ok_or_else(|| anyhow!("doc not found")) +pub(crate) fn entry_to_content_status(entry: EntryStatus) -> ContentStatus { + match entry { + EntryStatus::Complete => ContentStatus::Complete, + EntryStatus::Partial => ContentStatus::Incomplete, + EntryStatus::NotFound => ContentStatus::Missing, } +} + +/// Events informing about actions of the live sync progres. +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, strum::Display)] +#[allow(clippy::large_enum_variant)] +pub enum LiveEvent { + /// A local insertion. + InsertLocal { + /// The inserted entry. + entry: Entry, + }, + /// Received a remote insert. + InsertRemote { + /// The peer that sent us the entry. + from: PublicKey, + /// The inserted entry. + entry: Entry, + /// If the content is available at the local node + content_status: ContentStatus, + }, + /// The content of an entry was downloaded and is now available at the local node + ContentReady { + /// The content hash of the newly available entry content + hash: Hash, + }, + /// We have a new neighbor in the swarm. + NeighborUp(PublicKey), + /// We lost a neighbor in the swarm. + NeighborDown(PublicKey), + /// A set-reconciliation sync finished. + SyncFinished(SyncEvent), +} - /// Get an [`Author`] from the store, returning an error if the replica does not exist. - pub fn get_author(&self, id: &AuthorId) -> anyhow::Result { - self.store - .get_author(id)? - .ok_or_else(|| anyhow!("author not found")) +impl From for LiveEvent { + fn from(ev: live::Event) -> Self { + match ev { + live::Event::ContentReady { hash } => Self::ContentReady { hash }, + live::Event::NeighborUp(peer) => Self::NeighborUp(peer), + live::Event::NeighborDown(peer) => Self::NeighborDown(peer), + live::Event::SyncFinished(ev) => Self::SyncFinished(ev), + } } +} - /// Handle an incoming iroh-sync connection. - pub async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { - self.live.handle_connection(conn).await +impl LiveEvent { + fn from_replica_event( + ev: iroh_sync::Event, + content_status_cb: &ContentStatusCallback, + ) -> Result { + Ok(match ev { + iroh_sync::Event::Insert { origin, entry, .. } => match origin { + InsertOrigin::Local => Self::InsertLocal { + entry: entry.into(), + }, + InsertOrigin::Sync { from, .. } => Self::InsertRemote { + content_status: content_status_cb(entry.content_hash()), + entry: entry.into(), + from: PublicKey::from_bytes(&from)?, + }, + }, + }) } } diff --git a/iroh/src/sync_engine/gossip.rs b/iroh/src/sync_engine/gossip.rs new file mode 100644 index 0000000000..ce5f9a2e8d --- /dev/null +++ b/iroh/src/sync_engine/gossip.rs @@ -0,0 +1,206 @@ +use std::{collections::HashSet, future::Future, pin::Pin}; + +use anyhow::{anyhow, Context, Result}; +use futures::{ + stream::{FuturesUnordered, StreamExt}, + FutureExt, +}; +use iroh_gossip::{ + net::{Event, Gossip}, + proto::TopicId, +}; +use iroh_net::key::PublicKey; +use iroh_sync::{actor::SyncHandle, ContentStatus, NamespaceId}; +use tokio::sync::{broadcast::error::RecvError, mpsc}; +use tracing::{debug, error, trace}; + +use super::live::{Op, ToLiveActor}; +use crate::downloader::{Downloader, PeerRole}; + +#[derive(strum::Display, Debug)] +pub enum ToGossipActor { + Shutdown, + Join { + namespace: NamespaceId, + peers: Vec, + }, + Leave { + namespace: NamespaceId, + }, +} + +type JoinFut = Pin)> + Send + 'static>>; + +/// This actor subscribes to all gossip events. When receiving entries, they are inserted in the +/// replica (if open). Other events are forwarded to the main actor to be handled there. +pub struct GossipActor { + inbox: mpsc::Receiver, + sync: SyncHandle, + gossip: Gossip, + downloader: Downloader, + to_sync_actor: mpsc::Sender, + joined: HashSet, + want_join: HashSet, + pending_joins: FuturesUnordered, +} + +impl GossipActor { + pub fn new( + inbox: mpsc::Receiver, + sync: SyncHandle, + gossip: Gossip, + downloader: Downloader, + to_sync_actor: mpsc::Sender, + ) -> Self { + Self { + inbox, + sync, + gossip, + downloader, + to_sync_actor, + joined: Default::default(), + want_join: Default::default(), + pending_joins: Default::default(), + } + } + pub async fn run(&mut self) -> anyhow::Result<()> { + let mut gossip_events = self.gossip.clone().subscribe_all(); + let mut i = 0; + loop { + i += 1; + trace!(?i, "tick wait"); + tokio::select! { + next = gossip_events.next() => { + trace!(?i, "tick: gossip_event"); + if let Err(err) = self.on_gossip_event(next).await { + error!("gossip actor died: {err:?}"); + return Err(err); + } + }, + msg = self.inbox.recv() => { + let msg = msg.context("to_actor closed")?; + trace!(%msg, ?i, "tick: to_actor"); + if !self.on_actor_message(msg).await.context("on_actor_message")? { + break; + } + } + res = self.pending_joins.next(), if !self.pending_joins.is_empty() => { + trace!(?i, "tick: pending_joins"); + let (namespace, res) = res.context("pending_joins closed")?; + match res { + Ok(_topic) => { + debug!(namespace = %namespace.fmt_short(), "joined gossip"); + self.joined.insert(namespace); + }, + Err(err) => { + if self.want_join.contains(&namespace) { + error!(?namespace, ?err, "failed to join gossip"); + } + } + } + } + + } + } + Ok(()) + } + + async fn on_actor_message(&mut self, msg: ToGossipActor) -> anyhow::Result { + match msg { + ToGossipActor::Shutdown => { + for namespace in self.joined.iter() { + self.gossip.quit((*namespace).into()).await.ok(); + } + return Ok(false); + } + ToGossipActor::Join { namespace, peers } => { + // join gossip for the topic to receive and send message + let fut = self + .gossip + .join(namespace.into(), peers) + .await? + .map(move |res| (namespace, res)) + .boxed(); + self.want_join.insert(namespace); + self.pending_joins.push(fut); + } + ToGossipActor::Leave { namespace } => { + self.gossip.quit(namespace.into()).await?; + self.joined.remove(&namespace); + self.want_join.remove(&namespace); + } + } + Ok(true) + } + async fn on_gossip_event( + &mut self, + event: Option>, + ) -> Result<()> { + let (topic, event) = match event { + Some(Ok(event)) => event, + None => return Err(anyhow!("Gossip event channel closed")), + Some(Err(err)) => match err { + RecvError::Lagged(n) => { + error!("GossipActor too slow (lagged by {n}) - dropping gossip event"); + return Ok(()); + } + RecvError::Closed => { + return Err(anyhow!("Gossip event channel closed")); + } + }, + }; + let namespace: NamespaceId = topic.as_bytes().into(); + if let Err(err) = self.on_gossip_event_inner(namespace, event).await { + error!(namespace = %namespace.fmt_short(), ?err, "Failed to process gossip event"); + } + Ok(()) + } + + async fn on_gossip_event_inner(&mut self, namespace: NamespaceId, event: Event) -> Result<()> { + match event { + Event::Received(msg) => { + let op: Op = postcard::from_bytes(&msg.content)?; + match op { + Op::Put(entry) => { + debug!(peer = %msg.delivered_from.fmt_short(), namespace = %namespace.fmt_short(), "received entry via gossip"); + // Insert the entry into our replica. + // If the message was broadcast with neighbor scope, or is received + // directly from the author, we assume that the content is available at + // that peer. Otherwise we don't. + // The download is not triggered here, but in the `on_replica_event` + // handler for the `InsertRemote` event. + let content_status = match msg.scope.is_direct() { + true => ContentStatus::Complete, + false => ContentStatus::Missing, + }; + let from = *msg.delivered_from.as_bytes(); + self.sync + .insert_remote(namespace, entry, from, content_status) + .await?; + } + Op::ContentReady(hash) => { + // Inform the downloader that we now know that this peer has the content + // for this hash. + self.downloader + .peers_have(hash, vec![(msg.delivered_from, PeerRole::Provider).into()]) + .await; + } + } + } + // A new neighbor appeared in the gossip swarm. Try to sync with it directly. + // [Self::sync_with_peer] will check to not resync with peers synced previously in the + // same session. TODO: Maybe this is too broad and leads to too many sync requests. + Event::NeighborUp(peer) => { + self.to_sync_actor + .send(ToLiveActor::NeighborUp { namespace, peer }) + .await?; + } + Event::NeighborDown(peer) => { + self.to_sync_actor + .send(ToLiveActor::NeighborDown { namespace, peer }) + .await?; + } + } + Ok(()) + } +} diff --git a/iroh/src/sync_engine/live.rs b/iroh/src/sync_engine/live.rs index a34f16d6d3..6ab18dc41b 100644 --- a/iroh/src/sync_engine/live.rs +++ b/iroh/src/sync_engine/live.rs @@ -1,41 +1,33 @@ +#![allow(missing_docs)] + use std::{ collections::{HashMap, HashSet}, - sync::{atomic::AtomicU64, Arc}, time::SystemTime, }; use crate::downloader::{DownloadKind, Downloader, PeerRole}; -use anyhow::{anyhow, bail, Result}; -use flume::r#async::RecvStream; +use anyhow::{Context, Result}; use futures::{ - future::{BoxFuture, Shared}, - stream::{BoxStream, FuturesUnordered, StreamExt}, - FutureExt, TryFutureExt, -}; -use iroh_bytes::{store::EntryStatus, util::runtime::Handle, Hash}; -use iroh_gossip::{ - net::{Event, Gossip}, - proto::TopicId, + future::BoxFuture, + stream::{FuturesUnordered, StreamExt}, + FutureExt, }; +use iroh_bytes::{store::EntryStatus, Hash}; +use iroh_gossip::{net::Gossip, proto::TopicId}; use iroh_net::{key::PublicKey, MagicEndpoint, PeerAddr}; use iroh_sync::{ + actor::{OpenOpts, SyncHandle}, net::{ connect_and_sync, handle_connection, AbortReason, AcceptError, AcceptOutcome, ConnectError, + SyncFinished, }, - store, - sync::{Entry, InsertOrigin, NamespaceId, Replica, SignedEntry}, + ContentStatus, InsertOrigin, NamespaceId, SignedEntry, }; use serde::{Deserialize, Serialize}; -use tokio::{ - sync::{self, mpsc, oneshot}, - task::JoinError, -}; -use tokio_util::sync::CancellationToken; -use tracing::{debug, error, error_span, warn, Instrument}; +use tokio::sync::{self, mpsc, oneshot}; +use tracing::{debug, error, instrument, trace, warn, Instrument, Span}; -pub use iroh_sync::ContentStatus; - -const CHANNEL_CAP: usize = 8; +use super::gossip::ToGossipActor; /// An iroh-sync operation /// @@ -51,27 +43,15 @@ pub enum Op { #[derive(Debug, Clone)] enum SyncState { None, - Dialing(CancellationToken), + Dialing, Accepting, Finished, Failed, } -/// Sync status for a document -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct LiveStatus { - /// Whether this document is in the live sync - pub active: bool, - /// Number of event listeners registered - pub subscriptions: u64, -} - -#[derive(derive_more::Debug)] -enum ToActor { - Status { - namespace: NamespaceId, - s: sync::oneshot::Sender>, - }, +/// Messages to the sync actor +#[derive(derive_more::Debug, strum::Display)] +pub enum ToLiveActor { StartSync { namespace: NamespaceId, peers: Vec, @@ -81,24 +61,22 @@ enum ToActor { JoinPeers { namespace: NamespaceId, peers: Vec, + #[debug("onsehot::Sender")] + reply: sync::oneshot::Sender>, }, Leave { namespace: NamespaceId, - /// If true removes all active client subscriptions. - force_remove: bool, + kill_subscribers: bool, + #[debug("onsehot::Sender")] + reply: sync::oneshot::Sender>, }, Shutdown, Subscribe { namespace: NamespaceId, - #[debug("cb")] - cb: OnLiveEventCallback, + #[debug("sender")] + sender: flume::Sender, #[debug("oneshot::Sender")] - s: sync::oneshot::Sender>, - }, - Unsubscribe { - namespace: NamespaceId, - token: RemovalToken, - s: sync::oneshot::Sender, + reply: sync::oneshot::Sender>, }, HandleConnection { conn: quinn::Connecting, @@ -107,41 +85,22 @@ enum ToActor { namespace: NamespaceId, peer: PublicKey, #[debug("oneshot::Sender")] - reply: sync::oneshot::Sender>, + reply: sync::oneshot::Sender, + }, + NeighborUp { + namespace: NamespaceId, + peer: PublicKey, + }, + NeighborDown { + namespace: NamespaceId, + peer: PublicKey, }, } -/// Whether to keep a live event callback active. -#[derive(Debug)] -pub enum KeepCallback { - /// Keep active - Keep, - /// Drop this callback - Drop, -} - -/// Callback used for tracking [`LiveEvent`]s. -pub type OnLiveEventCallback = - Box BoxFuture<'static, KeepCallback> + Send + Sync + 'static>; - /// Events informing about actions of the live sync progres. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, strum::Display)] #[allow(clippy::large_enum_variant)] -pub enum LiveEvent { - /// A local insertion. - InsertLocal { - /// The inserted entry. - entry: Entry, - }, - /// Received a remote insert. - InsertRemote { - /// The peer that sent us the entry. - from: PublicKey, - /// The inserted entry. - entry: Entry, - /// If the content is available at the local node - content_status: ContentStatus, - }, +pub enum Event { /// The content of an entry was downloaded and is now available at the local node ContentReady { /// The content hash of the newly available entry content @@ -153,332 +112,217 @@ pub enum LiveEvent { NeighborDown(PublicKey), /// A set-reconciliation sync finished. SyncFinished(SyncEvent), - /// The document was closed. No further events will be emitted. - Closed, } -fn entry_to_content_status(entry: EntryStatus) -> ContentStatus { - match entry { - EntryStatus::Complete => ContentStatus::Complete, - EntryStatus::Partial => ContentStatus::Incomplete, - EntryStatus::NotFound => ContentStatus::Missing, - } -} - -/// Handle to a running live sync actor -#[derive(Debug, Clone)] -pub struct LiveSync { - to_actor_tx: mpsc::Sender>, - task: Shared>>>, -} - -impl LiveSync { - /// Start the live sync. - /// - /// This spawn a background actor to handle gossip events and forward operations over broadcast - /// messages. - pub fn spawn( - rt: Handle, - endpoint: MagicEndpoint, - replica_store: S, - gossip: Gossip, - bao_store: B, - downloader: Downloader, - ) -> Self { - let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); - let me = endpoint.peer_id().fmt_short(); - let mut actor = Actor::new( - endpoint, - gossip, - bao_store, - downloader, - replica_store, - to_actor_rx, - to_actor_tx.clone(), - ); - let task = rt.main().spawn( - async move { - if let Err(err) = actor.run().await { - error!("live sync failed: {err:?}"); - } - } - .instrument(error_span!("sync", %me)), - ); - let handle = LiveSync { - to_actor_tx, - task: task.map_err(Arc::new).boxed().shared(), - }; - handle - } - - /// Cancel the live sync. - pub async fn shutdown(&self) -> Result<()> { - self.to_actor_tx.send(ToActor::::Shutdown).await?; - self.task.clone().await?; - Ok(()) - } - - /// Start to sync a document with a set of peers, also joining the gossip swarm for that - /// document. - pub async fn start_sync(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { - let (reply, reply_rx) = oneshot::channel(); - self.to_actor_tx - .send(ToActor::::StartSync { - namespace, - peers, - reply, - }) - .await?; - reply_rx.await??; - Ok(()) - } - - /// Join and sync with a set of peers for a document that is already syncing. - pub async fn join_peers(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { - self.to_actor_tx - .send(ToActor::::JoinPeers { namespace, peers }) - .await?; - Ok(()) - } - - /// Stop the live sync for a document. - /// - /// This will leave the gossip swarm for this document. - pub async fn leave(&self, namespace: NamespaceId, force_remove: bool) -> Result<()> { - self.to_actor_tx - .send(ToActor::::Leave { - namespace, - force_remove, - }) - .await?; - Ok(()) - } - - /// Subscribes `cb` to events on this `namespace`. - pub async fn subscribe(&self, namespace: NamespaceId, cb: F) -> Result - where - F: Fn(LiveEvent) -> BoxFuture<'static, KeepCallback> + Send + Sync + 'static, - { - let (s, r) = sync::oneshot::channel(); - self.to_actor_tx - .send(ToActor::::Subscribe { - namespace, - cb: Box::new(cb), - s, - }) - .await?; - let token = r.await??; - Ok(token) - } - - /// Unsubscribes `token` to events on this `namespace`. - /// Returns `true` if a callback was found - pub async fn unsubscribe(&self, namespace: NamespaceId, token: RemovalToken) -> Result { - let (s, r) = sync::oneshot::channel(); - self.to_actor_tx - .send(ToActor::::Unsubscribe { - namespace, - token, - s, - }) - .await?; - let token = r.await?; - Ok(token) - } - - /// Get status for a document - pub async fn status(&self, namespace: NamespaceId) -> Result> { - let (s, r) = sync::oneshot::channel(); - self.to_actor_tx - .send(ToActor::::Status { namespace, s }) - .await?; - let status = r.await?; - Ok(status) - } - - /// Handle an incoming iroh-sync connection. - pub async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { - self.to_actor_tx - .send(ToActor::::HandleConnection { conn }) - .await?; - Ok(()) - } -} +type SyncConnectFut = BoxFuture< + 'static, + ( + NamespaceId, + PublicKey, + SyncReason, + Result, + ), +>; +type SyncAcceptFut = BoxFuture<'static, Result>; // Currently peers might double-sync in both directions. -struct Actor { +pub struct LiveActor { + /// Receiver for actor messages. + inbox: mpsc::Receiver, + sync: SyncHandle, endpoint: MagicEndpoint, gossip: Gossip, bao_store: B, downloader: Downloader, - replica_store: S, - - /// Set of replicas that we opened for sync or event subscriptions. - open_replicas: HashSet, - /// Set of replicas that are actively syncing. - syncing_replicas: HashSet, - - /// Events from replicas. - replica_events: futures::stream::SelectAll>, - /// Events from gossip. - gossip_events: BoxStream<'static, Result<(TopicId, Event)>>, - + replica_events_tx: flume::Sender, + replica_events_rx: flume::Receiver, /// Last state of sync for a replica with a peer. sync_state: HashMap<(NamespaceId, PublicKey), SyncState>, - /// Receiver for actor messages. - to_actor_rx: mpsc::Receiver>, /// Send messages to self. /// Note: Must not be used in methods called from `Self::run` directly to prevent deadlocks. /// Only clone into newly spawned tasks. - to_actor_tx: mpsc::Sender>, + sync_actor_tx: mpsc::Sender, + gossip_actor_tx: mpsc::Sender, /// Running sync futures (from connect). #[allow(clippy::type_complexity)] - running_sync_connect: FuturesUnordered< - BoxFuture<'static, (NamespaceId, PublicKey, SyncReason, Result<(), ConnectError>)>, - >, + running_sync_connect: FuturesUnordered, /// Running sync futures (from accept). - running_sync_accept: - FuturesUnordered>>, + running_sync_accept: FuturesUnordered, /// Runnning download futures. pending_downloads: FuturesUnordered>>, - /// Running gossip join futures. - pending_joins: FuturesUnordered)>>, - /// External subscriptions to replica events. - event_subscriptions: HashMap>, - /// Next [`RemovalToken`] for external replica event subscriptions. - event_removal_id: AtomicU64, + // Subscribers to actor events + subscribers: SubscribersMap, + is_syncing: HashSet, } -/// Token needed to remove inserted callbacks. -#[derive(Debug, Clone, Copy)] -pub struct RemovalToken(u64); - -impl Actor { +impl LiveActor { + /// Create the live actor. + #[allow(clippy::too_many_arguments)] pub fn new( + sync: SyncHandle, endpoint: MagicEndpoint, gossip: Gossip, bao_store: B, downloader: Downloader, - replica_store: S, - to_actor_rx: mpsc::Receiver>, - to_actor_tx: mpsc::Sender>, + inbox: mpsc::Receiver, + sync_actor_tx: mpsc::Sender, + gossip_actor_tx: mpsc::Sender, ) -> Self { - let gossip_events = gossip.clone().subscribe_all().boxed(); - + let (replica_events_tx, replica_events_rx) = flume::bounded(1024); Self { - gossip, + inbox, + sync, + replica_events_rx, + replica_events_tx, endpoint, + gossip, bao_store, downloader, - replica_store, - syncing_replicas: Default::default(), - open_replicas: Default::default(), - to_actor_rx, - to_actor_tx, + sync_actor_tx, + gossip_actor_tx, sync_state: Default::default(), running_sync_connect: Default::default(), running_sync_accept: Default::default(), - pending_joins: Default::default(), - replica_events: Default::default(), - gossip_events, - event_subscriptions: Default::default(), - event_removal_id: Default::default(), + subscribers: Default::default(), pending_downloads: Default::default(), + is_syncing: Default::default(), } } - async fn run(&mut self) -> Result<()> { + /// Run the actor loop. + pub async fn run(&mut self) -> Result<()> { + let res = self.run_inner().await; + if let Err(err) = self.shutdown().await { + error!(?err, "Error during shutdown"); + } + res + } + + async fn run_inner(&mut self) -> Result<()> { + let mut i = 0; loop { + i += 1; + trace!(?i, "tick wait"); tokio::select! { biased; - msg = self.to_actor_rx.recv() => { - match msg { - // received shutdown signal, or livesync handle was dropped: - // break loop and exit - Some(ToActor::Shutdown) | None => { - self.shutdown().await?; - break; - } - Some(ToActor::StartSync { namespace, peers, reply }) => { - let res = self.start_sync(namespace, peers).await; - reply.send(res).ok(); - }, - Some(ToActor::Leave { namespace, force_remove }) => { - self.leave(namespace, force_remove).await?; - } - Some(ToActor::JoinPeers { namespace, peers }) => { - self.join_peers(namespace, peers).await?; - }, - Some(ToActor::Subscribe { namespace, cb, s }) => { - let result = self.subscribe(namespace, cb).await; - s.send(result).ok(); - }, - Some(ToActor::Unsubscribe { namespace, token, s }) => { - let result = self.unsubscribe(namespace, token).await; - s.send(result).ok(); - }, - Some(ToActor::Status { namespace , s }) => { - let result = self.status(namespace).await; - s.send(result).ok(); - }, - Some(ToActor::HandleConnection { conn }) => { - self.handle_connection(conn).await; - }, - Some(ToActor::AcceptSyncRequest { namespace, peer, reply }) => { - let outcome = self.accept_sync_request(namespace, peer); - reply.send(outcome).ok(); - }, - }; - } - // new gossip message - Some(event) = self.gossip_events.next() => { - let (topic, event) = event?; - if let Err(err) = self.on_gossip_event(topic, event).await { - error!("Failed to process gossip event: {err:?}"); + msg = self.inbox.recv() => { + let msg = msg.context("to_actor closed")?; + trace!(?i, %msg, "tick: to_actor"); + if !self.on_actor_message(msg).await.context("on_actor_message")? { + break; } - }, - Some((origin, entry)) = self.replica_events.next() => { - if let Err(err) = self.on_replica_event(origin, entry).await { - error!("Failed to process replica event: {err:?}"); + } + event = self.replica_events_rx.recv_async() => { + trace!(?i, "tick: replica_event"); + let event = event.context("replica_events closed")?; + if let Err(err) = self.on_replica_event(event).await { + error!(?err, "Failed to process replica event"); } } - Some((namespace, peer, reason, res)) = self.running_sync_connect.next() => { - self.on_sync_via_connect_finished(namespace, peer, reason, res).await; + res = self.running_sync_connect.next(), if !self.running_sync_connect.is_empty() => { + trace!(?i, "tick: on_sync_via_connect_finished"); + let (namespace, peer, reason, res) = res.context("running_sync_connect closed")?; + if let Err(err) = self.on_sync_via_connect_finished(namespace, peer, reason, res).await { + error!(namespace = %namespace.fmt_short(), ?err, "Failed to process outgoing sync request"); + } } - Some(res) = self.running_sync_accept.next() => { - self.on_sync_via_accept_finished(res).await; - } - Some((namespace, res)) = self.pending_joins.next() => { - if let Err(err) = res { - error!(?namespace, %err, "failed to join gossip"); - } else { - debug!(?namespace, "joined gossip"); + res = self.running_sync_accept.next(), if !self.running_sync_accept.is_empty() => { + trace!(?i, "tick: on_sync_via_accept_finished"); + let res = res.context("running_sync_accept closed")?; + if let Err(err) = self.on_sync_via_accept_finished(res).await { + error!(?err, "Failed to process incoming sync request"); } - // TODO: maintain some join state } - Some(res) = self.pending_downloads.next() => { + res = self.pending_downloads.next(), if !self.pending_downloads.is_empty() => { + trace!(?i, "tick: pending_downloads"); + let res = res.context("pending_downloads closed")?; if let Some((namespace, hash)) = res { - if let Some(subs) = self.event_subscriptions.get_mut(&namespace) { - let event = LiveEvent::ContentReady { hash }; - notify_all(subs, event).await; - } + self.subscribers.send(&namespace, Event::ContentReady { hash }).await; // Inform our neighbors that we have new content ready. let op = Op::ContentReady(hash); let message = postcard::to_stdvec(&op)?.into(); - self.gossip.broadcast_neighbors(namespace.into(), message).await?; + if self.is_syncing(&namespace) { + self.gossip.broadcast_neighbors(namespace.into(), message).await?; + } } } } } + debug!("close (shutdown)"); Ok(()) } + async fn on_actor_message(&mut self, msg: ToLiveActor) -> anyhow::Result { + match msg { + ToLiveActor::Shutdown => { + return Ok(false); + } + ToLiveActor::NeighborUp { namespace, peer } => { + debug!(peer = %peer.fmt_short(), namespace = %namespace.fmt_short(), "neighbor up"); + self.sync_with_peer(namespace, peer, SyncReason::NewNeighbor); + self.subscribers + .send(&namespace, Event::NeighborUp(peer)) + .await; + } + ToLiveActor::NeighborDown { namespace, peer } => { + debug!(peer = %peer.fmt_short(), namespace = %namespace.fmt_short(), "neighbor down"); + self.subscribers + .send(&namespace, Event::NeighborDown(peer)) + .await; + } + ToLiveActor::StartSync { + namespace, + peers, + reply, + } => { + let res = self.start_sync(namespace, peers).await; + reply.send(res).ok(); + } + ToLiveActor::Leave { + namespace, + kill_subscribers, + reply, + } => { + let res = self.leave(namespace, kill_subscribers).await; + reply.send(res).ok(); + } + ToLiveActor::JoinPeers { + namespace, + peers, + reply, + } => { + let res = self.join_peers(namespace, peers).await; + reply.send(res).ok(); + } + ToLiveActor::Subscribe { + namespace, + sender, + reply, + } => { + self.subscribers.subscribe(namespace, sender); + reply.send(Ok(())).ok(); + } + ToLiveActor::HandleConnection { conn } => { + self.handle_connection(conn).await; + } + ToLiveActor::AcceptSyncRequest { + namespace, + peer, + reply, + } => { + let outcome = self.accept_sync_request(namespace, peer); + reply.send(outcome).ok(); + } + }; + Ok(true) + } + fn set_sync_state(&mut self, namespace: NamespaceId, peer: PublicKey, state: SyncState) { self.sync_state.insert((namespace, peer), state); } @@ -489,98 +333,65 @@ impl Actor { .unwrap_or(SyncState::None) } - fn get_replica_if_syncing(&self, namespace: &NamespaceId) -> Option> { - if !self.syncing_replicas.contains(namespace) { - None - } else { - match self.replica_store.open_replica(namespace) { - Ok(replica) => replica, - Err(err) => { - warn!("Failed to get previously opened replica from the store: {err:?}"); - None - } - } - } - } - + #[instrument("connect", skip_all, fields(peer = %peer.fmt_short(), namespace = %namespace.fmt_short()))] fn sync_with_peer(&mut self, namespace: NamespaceId, peer: PublicKey, reason: SyncReason) { - let Some(replica) = self.get_replica_if_syncing(&namespace) else { + if !self.is_syncing(&namespace) { return; - }; + } // Do not initiate the sync if we are already syncing or did previously sync successfully. // TODO: Track finished time and potentially re-run sync on finished state if enough time // passed. match self.get_sync_state(namespace, peer) { - SyncState::Accepting | SyncState::Dialing(_) | SyncState::Finished => { - return; - } + // never run two syncs at the same time + SyncState::Accepting | SyncState::Dialing => return, + // always rerun if we failed or did not run yet SyncState::Failed | SyncState::None => {} + // if we finished previously, only re-run if explicitly requested. + SyncState::Finished => return, }; - - let cancel = CancellationToken::new(); - self.set_sync_state(namespace, peer, SyncState::Dialing(cancel.clone())); - let fut = { - let endpoint = self.endpoint.clone(); - let replica = replica.clone(); - async move { - debug!(?peer, ?namespace, ?reason, "sync[dial]: start"); - let fut = connect_and_sync::(&endpoint, &replica, PeerAddr::new(peer)); - let res = tokio::select! { - biased; - _ = cancel.cancelled() => Err(ConnectError::Cancelled), - res = fut => res - }; - (namespace, peer, reason, res) - } - .boxed() - }; + debug!(?reason, last_state = ?self.get_sync_state(namespace, peer), "start"); + + self.set_sync_state(namespace, peer, SyncState::Dialing); + let endpoint = self.endpoint.clone(); + let sync = self.sync.clone(); + let fut = async move { + let res = connect_and_sync(&endpoint, &sync, namespace, PeerAddr::new(peer)).await; + (namespace, peer, reason, res) + } + .instrument(Span::current()) + .boxed(); self.running_sync_connect.push(fut); } async fn shutdown(&mut self) -> anyhow::Result<()> { - // we have to clone the list of replicas here to reuse the Self::leave code. - let namespaces = self.open_replicas.iter().cloned().collect::>(); - for namespace in namespaces { - if let Err(err) = self.leave(namespace, true).await { - warn!(?namespace, "Error while closing: {err:?}"); - } - } + // cancel all subscriptions + self.subscribers.clear(); + // shutdown gossip actor + self.gossip_actor_tx + .send(ToGossipActor::Shutdown) + .await + .ok(); + // shutdown sync thread + self.sync.shutdown().await; Ok(()) } - async fn status(&mut self, namespace: NamespaceId) -> Option { - let exists = self - .replica_store - .open_replica(&namespace) - .ok() - .flatten() - .is_some(); - if !exists { - return None; - } - let active = self.syncing_replicas.contains(&namespace); - let subscriptions = self - .event_subscriptions - .get(&namespace) - .map(|map| map.len() as u64) - .unwrap_or(0); - self.maybe_close_replica(namespace); - Some(LiveStatus { - active, - subscriptions, - }) - } - async fn start_sync(&mut self, namespace: NamespaceId, mut peers: Vec) -> Result<()> { - self.ensure_open(namespace)?; - self.syncing_replicas.insert(namespace); + // update state to allow sync + if !self.is_syncing(&namespace) { + let opts = OpenOpts::default() + .sync() + .subscribe(self.replica_events_tx.clone()); + self.sync.open(namespace, opts).await?; + self.is_syncing.insert(namespace); + } // add the peers stored for this document - match self.replica_store.get_sync_peers(&namespace) { + match self.sync.get_sync_peers(namespace).await { Ok(None) => { // no peers for this document } Ok(Some(known_useful_peers)) => { - let as_peer_addr = known_useful_peers.filter_map(|peer_id_bytes| { + let as_peer_addr = known_useful_peers.into_iter().filter_map(|peer_id_bytes| { // peers are stored as bytes, don't fail the operation if they can't be // decoded: simply ignore the peer match PublicKey::from_bytes(&peer_id_bytes) { @@ -602,88 +413,28 @@ impl Actor { Ok(()) } - /// Open a replica, if not yet in our set of open replicas. - fn ensure_open(&mut self, namespace: NamespaceId) -> anyhow::Result<()> { - if !self.open_replicas.contains(&namespace) { - let Some(replica) = self.replica_store.open_replica(&namespace)? else { - bail!("Replica not found"); - }; - - // setup event subscription. - let events = replica - .subscribe() - .ok_or_else(|| anyhow::anyhow!("trying to subscribe twice to the same replica"))?; - self.replica_events.push(events.into_stream()); - - // setup content status callback - let bao_store = self.bao_store.clone(); - let content_status_cb = - Box::new(move |hash| entry_to_content_status(bao_store.contains(&hash))); - replica.set_content_status_callback(content_status_cb); - - self.open_replicas.insert(namespace); - } - Ok(()) - } - - /// Close a replica if we don't need it anymore. - /// - /// This closes only if both of the following conditions are met: - /// * The replica is not in the set of actively synced replicas - /// * There are no external event subscriptions for this replica - /// - /// Closing a replica will remove all event subscriptions. - fn maybe_close_replica(&mut self, namespace: NamespaceId) { - if !self.open_replicas.contains(&namespace) - || self.syncing_replicas.contains(&namespace) - || self.event_subscriptions.contains_key(&namespace) - { - return; - } - self.replica_store.close_replica(&namespace); - self.open_replicas.remove(&namespace); - } - - async fn subscribe( + async fn leave( &mut self, namespace: NamespaceId, - cb: OnLiveEventCallback, - ) -> anyhow::Result { - self.ensure_open(namespace)?; - let subs = self.event_subscriptions.entry(namespace).or_default(); - let removal_id = self - .event_removal_id - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - subs.insert(removal_id, cb); - Ok(RemovalToken(removal_id)) - } - - /// Returns `true` if a callback was found and removed - async fn unsubscribe(&mut self, namespace: NamespaceId, token: RemovalToken) -> bool { - if let Some(subs) = self.event_subscriptions.get_mut(&namespace) { - let res = subs.remove(&token.0).is_some(); - if subs.is_empty() { - self.event_subscriptions.remove(&namespace); - } - self.maybe_close_replica(namespace); - return res; - } - - false - } - - async fn leave(&mut self, namespace: NamespaceId, force_remove: bool) -> anyhow::Result<()> { - if self.syncing_replicas.remove(&namespace) { - self.gossip.quit(namespace.into()).await?; - self.sync_state.retain(|(n, _peer), _value| *n != namespace); + kill_subscribers: bool, + ) -> anyhow::Result<()> { + // self.subscribers.remove(&namespace); + if self.is_syncing.remove(&namespace) { + self.sync_state + .retain(|(cur_namespace, _peer), _state| cur_namespace != &namespace); + self.sync.set_sync(namespace, false).await?; + self.sync + .unsubscribe(namespace, self.replica_events_tx.clone()) + .await?; + self.sync.close(namespace).await?; + self.gossip_actor_tx + .send(ToGossipActor::Leave { namespace }) + .await + .context("gossip actor failure")?; } - if force_remove { - let subs = self.event_subscriptions.remove(&namespace); - if let Some(mut subs) = subs { - notify_all(&mut subs, LiveEvent::Closed).await; - } + if kill_subscribers { + self.subscribers.remove(&namespace); } - self.maybe_close_replica(namespace); Ok(()) } @@ -694,26 +445,21 @@ impl Actor { ) -> anyhow::Result<()> { let peer_ids: Vec = peers.iter().map(|p| p.peer_id).collect(); - // add addresses of initial peers to our endpoint address book + // add addresses of peers to our endpoint address book for peer in peers.into_iter() { let peer_id = peer.peer_id; if let Err(err) = self.endpoint.add_peer_addr(peer).await { - warn!(peer = ?peer_id, "failed to add known addrs: {err:?}"); + warn!(peer = %peer_id.fmt_short(), "failed to add known addrs: {err:?}"); } } - // join gossip for the topic to receive and send message - self.pending_joins.push({ - let peer_ids = peer_ids.clone(); - let gossip = self.gossip.clone(); - async move { - match gossip.join(namespace.into(), peer_ids).await { - Err(err) => (namespace, Err(err)), - Ok(fut) => (namespace, fut.await), - } - } - .boxed() - }); + // tell gossip to join + self.gossip_actor_tx + .send(ToGossipActor::Join { + namespace, + peers: peer_ids.clone(), + }) + .await?; // trigger initial sync with initial peers for peer in peer_ids { @@ -722,58 +468,45 @@ impl Actor { Ok(()) } + #[instrument("connect", skip_all, fields(peer = %peer.fmt_short(), namespace = %namespace.fmt_short()))] async fn on_sync_via_connect_finished( &mut self, namespace: NamespaceId, peer: PublicKey, reason: SyncReason, - result: Result<(), ConnectError>, - ) { + result: Result, + ) -> Result<()> { match result { Err(ConnectError::RemoteAbort(AbortReason::AlreadySyncing)) => { - debug!( - ?peer, - ?namespace, - ?reason, - "sync[dial]: remote abort, already syncing" - ); + debug!(?reason, "remote abort, already syncing"); + Ok(()) } - Err(ConnectError::Cancelled) => { - // In case the remote aborted with already running: do nothing - debug!( - ?peer, - ?namespace, - ?reason, - "sync[dial]: cancelled, already syncing" - ); - } - Err(err) => { - self.on_sync_finished(namespace, peer, Origin::Connect(reason), Err(err.into())) - .await; - } - Ok(()) => { - self.on_sync_finished(namespace, peer, Origin::Connect(reason), Ok(())) - .await; + res => { + self.on_sync_finished( + namespace, + peer, + Origin::Connect(reason), + res.map_err(Into::into), + ) + .await } } } + #[instrument("accept", skip_all, fields(peer = %fmt_accept_peer(&res), namespace = %fmt_accept_namespace(&res)))] async fn on_sync_via_accept_finished( &mut self, - res: Result<(NamespaceId, PublicKey), AcceptError>, - ) { + res: Result, + ) -> Result<()> { match res { - Ok((namespace, peer)) => { - self.on_sync_finished(namespace, peer, Origin::Accept, Ok(())) - .await; + Ok(state) => { + self.on_sync_finished(state.namespace, state.peer, Origin::Accept, Ok(state)) + .await } - Err(AcceptError::Abort { - peer, - namespace, - reason, - }) if reason == AbortReason::AlreadySyncing => { + Err(AcceptError::Abort { reason, .. }) if reason == AbortReason::AlreadySyncing => { // In case we aborted the sync: do nothing (our outgoing sync is in progress) - debug!(?peer, ?namespace, ?reason, "sync[accept]: aborted by us"); + debug!(?reason, "aborted by us"); + Ok(()) } Err(err) => { if let (Some(peer), Some(namespace)) = (err.peer(), err.namespace()) { @@ -783,9 +516,11 @@ impl Actor { Origin::Accept, Err(anyhow::Error::from(err)), ) - .await; + .await?; + Ok(()) } else { - debug!("sync[accept]: failed {err:?}"); + debug!(?err, "failed before reading the first message"); + Err(err.into()) } } } @@ -796,127 +531,73 @@ impl Actor { namespace: NamespaceId, peer: PublicKey, origin: Origin, - result: anyhow::Result<()>, - ) { + result: Result, + ) -> Result<()> { // debug log the result, warn in case of errors - match (&origin, &result) { - (Origin::Accept, Ok(())) => debug!(?peer, ?namespace, "sync[accept]: done"), - (Origin::Connect(reason), Ok(())) => { - debug!(?peer, ?namespace, ?reason, "sync[dial]: done") - } - (Origin::Accept, Err(err)) => warn!(?peer, ?namespace, ?err, "sync[accept]: failed"), - (Origin::Connect(reason), Err(err)) => { - warn!(?peer, ?namespace, ?err, ?reason, "sync[dial]: failed") - } - } let state = match result { - Ok(_) => { + Ok(ref details) => { + debug!( + sent = %details.outcome.num_sent, + recv = %details.outcome.num_recv, + t_connect = ?details.timings.connect, + t_process = ?details.timings.process, + "sync finish ok", + ); + // register the peer as useful for the document if let Err(e) = self - .replica_store + .sync .register_useful_peer(namespace, *peer.as_bytes()) + .await { debug!(%e, "failed to register peer for document") } + SyncState::Finished } - Err(_) => SyncState::Failed, + Err(ref err) => { + warn!(?origin, ?err, "sync failed"); + + SyncState::Failed + } }; + self.set_sync_state(namespace, peer, state); - let event = SyncEvent { + + let ev = SyncEvent { namespace, peer, origin, - result: result.map_err(|err| format!("{err:?}")), + result: result + .as_ref() + .map(|_| ()) + .map_err(|err| format!("{err:?}")), finished: SystemTime::now(), }; - let subs = self.event_subscriptions.get_mut(&event.namespace); - if let Some(subs) = subs { - notify_all(subs, LiveEvent::SyncFinished(event)).await; - } + self.subscribers + .send(&namespace, Event::SyncFinished(ev)) + .await; + Ok(()) } - async fn on_gossip_event(&mut self, topic: TopicId, event: Event) -> Result<()> { - let namespace: NamespaceId = topic.as_bytes().into(); - let Some(replica) = self.get_replica_if_syncing(&namespace) else { - return Err(anyhow!("Doc {namespace:?} is not active")); - }; - match event { - // We received a gossip message. Try to insert it into our replica. - Event::Received(msg) => { - let op: Op = postcard::from_bytes(&msg.content)?; - match op { - Op::Put(entry) => { - debug!(peer = ?msg.delivered_from, ?namespace, "received entry via gossip"); - // Insert the entry into our replica. - // If the message was broadcast with neighbor scope, or is received - // directly from the author, we assume that the content is available at - // that peer. Otherwise we don't. - // The download is not triggered here, but in the `on_replica_event` - // handler for the `InsertRemote` event. - let content_status = match msg.scope.is_direct() { - true => ContentStatus::Complete, - false => ContentStatus::Missing, - }; - replica.insert_remote_entry( - entry, - *msg.delivered_from.as_bytes(), - content_status, - )?; - } - Op::ContentReady(hash) => { - // Inform the downloader that we now know that this peer has the content - // for this hash. - self.downloader - .peers_have(hash, vec![(msg.delivered_from, PeerRole::Provider).into()]) - .await; - } - } - } - // A new neighbor appeared in the gossip swarm. Try to sync with it directly. - // [Self::sync_with_peer] will check to not resync with peers synced previously in the - // same session. TODO: Maybe this is too broad and leads to too many sync requests. - Event::NeighborUp(peer) => { - debug!(?peer, ?namespace, "neighbor up"); - self.sync_with_peer(namespace, peer, SyncReason::NewNeighbor); - if let Some(subs) = self.event_subscriptions.get_mut(&namespace) { - notify_all(subs, LiveEvent::NeighborUp(peer)).await; - } - } - Event::NeighborDown(peer) => { - debug!(?peer, ?namespace, "neighbor down"); - if let Some(subs) = self.event_subscriptions.get_mut(&namespace) { - notify_all(subs, LiveEvent::NeighborDown(peer)).await; - } - } - } - Ok(()) + fn is_syncing(&self, namespace: &NamespaceId) -> bool { + self.is_syncing.contains(namespace) } - async fn on_replica_event( - &mut self, - origin: InsertOrigin, - signed_entry: SignedEntry, - ) -> Result<()> { - let namespace = signed_entry.namespace(); + async fn on_replica_event(&mut self, event: iroh_sync::Event) -> Result<()> { + let iroh_sync::Event::Insert { + namespace, + origin, + entry: signed_entry, + } = event; let topic = TopicId::from_bytes(*namespace.as_bytes()); - let subs = self.event_subscriptions.get_mut(&namespace); match origin { InsertOrigin::Local => { - let entry = signed_entry.entry().clone(); - // A new entry was inserted locally. Broadcast a gossip message. - let op = Op::Put(signed_entry); - let message = postcard::to_stdvec(&op)?.into(); - debug!(?namespace, "broadcast new entry"); - self.gossip.broadcast(topic, message).await?; - - // Notify subscribers about the event - if let Some(subs) = subs { - let event = LiveEvent::InsertLocal { - entry: entry.clone(), - }; - notify_all(subs, event).await; + if self.is_syncing(&namespace) { + let op = Op::Put(signed_entry.clone()); + let message = postcard::to_stdvec(&op)?.into(); + self.gossip.broadcast(topic, message).await?; } } InsertOrigin::Sync { @@ -930,6 +611,8 @@ impl Actor { // A new entry was inserted from initial sync or gossip. Queue downloading the // content. let entry_status = self.bao_store.contains(&hash); + + // TODO: Make downloads configurable. if matches!(entry_status, EntryStatus::NotFound | EntryStatus::Partial) { let role = match content_status { ContentStatus::Complete => PeerRole::Provider, @@ -947,43 +630,42 @@ impl Actor { .boxed(); self.pending_downloads.push(fut); } - - // Notify subscribers about the event - if let Some(subs) = subs { - let event = LiveEvent::InsertRemote { - from, - entry: entry.clone(), - content_status: entry_to_content_status(entry_status), - }; - notify_all(subs, event).await; - } } } Ok(()) } + #[instrument("accept", skip_all)] pub async fn handle_connection(&mut self, conn: quinn::Connecting) { - let to_actor_tx = self.to_actor_tx.clone(); - let request_replica_cb = move |namespace, peer| { + let to_actor_tx = self.sync_actor_tx.clone(); + let accept_request_cb = move |namespace, peer| { let to_actor_tx = to_actor_tx.clone(); async move { let (reply_tx, reply_rx) = oneshot::channel(); to_actor_tx - .send(ToActor::AcceptSyncRequest { + .send(ToLiveActor::AcceptSyncRequest { namespace, peer, reply: reply_tx, }) .await .ok(); - reply_rx.await.map_err(anyhow::Error::from) + match reply_rx.await { + Ok(outcome) => outcome, + Err(err) => { + warn!( + "accept request callback failed to retrieve reply from actor: {err:?}" + ); + AcceptOutcome::Reject(AbortReason::InternalServerError) + } + } } .boxed() }; - debug!("sync[accept] incoming connection"); - let fut = - async move { handle_connection::(conn, request_replica_cb).await }.boxed(); + debug!("incoming connection"); + let sync = self.sync.clone(); + let fut = async move { handle_connection(sync, conn, accept_request_cb).await }.boxed(); self.running_sync_accept.push(fut); } @@ -991,33 +673,45 @@ impl Actor { &mut self, namespace: NamespaceId, peer: PublicKey, - ) -> AcceptOutcome { - let Some(replica) = self.get_replica_if_syncing(&namespace) else { - return Err(AbortReason::NotAvailable); + ) -> AcceptOutcome { + if !self.is_syncing(&namespace) { + return AcceptOutcome::Reject(AbortReason::NotFound); }; match self.get_sync_state(namespace, peer) { SyncState::None | SyncState::Failed | SyncState::Finished => { self.set_sync_state(namespace, peer, SyncState::Accepting); - Ok(replica.clone()) + AcceptOutcome::Allow } - SyncState::Accepting => Err(AbortReason::AlreadySyncing), + SyncState::Accepting => AcceptOutcome::Reject(AbortReason::AlreadySyncing), // Incoming sync request while we are dialing ourselves. // In this case, compare the binary representations of our and the other node's peer id // to deterministically decide which of the two concurrent connections will succeed. - SyncState::Dialing(cancel) => { - if peer.as_bytes() > self.endpoint.peer_id().as_bytes() { - cancel.cancel(); + SyncState::Dialing => match expected_sync_direction(&self.endpoint.peer_id(), &peer) { + SyncDirection::Accept => { self.set_sync_state(namespace, peer, SyncState::Accepting); - Ok(replica.clone()) - } else { - Err(AbortReason::AlreadySyncing) + AcceptOutcome::Allow } - } + SyncDirection::Dial => AcceptOutcome::Reject(AbortReason::AlreadySyncing), + }, } } } -/// Outcome of a sync operation +#[derive(Debug)] +enum SyncDirection { + Accept, + Dial, +} + +fn expected_sync_direction(self_peer_id: &PublicKey, other_peer_id: &PublicKey) -> SyncDirection { + if self_peer_id.as_bytes() > other_peer_id.as_bytes() { + SyncDirection::Accept + } else { + SyncDirection::Dial + } +} + +/// Event emitted when a sync operation completes #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] pub struct SyncEvent { /// Namespace that was synced @@ -1052,15 +746,70 @@ pub enum Origin { Accept, } -async fn notify_all(subs: &mut HashMap, event: LiveEvent) { - let res = futures::future::join_all( - subs.iter() - .map(|(idx, sub)| sub(event.clone()).map(|res| (*idx, res))), - ) - .await; - for (idx, res) in res { - if matches!(res, KeepCallback::Drop) { - subs.remove(&idx); +#[derive(Debug, Default)] +struct SubscribersMap(HashMap); + +impl SubscribersMap { + fn subscribe(&mut self, namespace: NamespaceId, sender: flume::Sender) { + self.0.entry(namespace).or_default().subscribe(sender); + } + + async fn send(&mut self, namespace: &NamespaceId, event: Event) -> bool { + let Some(subscribers) = self.0.get_mut(namespace) else { + return false; + }; + + if !subscribers.send(event).await { + self.0.remove(namespace); + } + true + } + + fn remove(&mut self, namespace: &NamespaceId) { + self.0.remove(namespace); + } + + fn clear(&mut self) { + self.0.clear(); + } +} + +#[derive(Debug, Default)] +struct Subscribers(Vec>); + +impl Subscribers { + fn subscribe(&mut self, sender: flume::Sender) { + self.0.push(sender) + } + + async fn send(&mut self, event: Event) -> bool { + let futs = self.0.iter().map(|sender| sender.send_async(event.clone())); + let res = futures::future::join_all(futs).await; + for (i, res) in res.into_iter().enumerate() { + if res.is_err() { + self.0.remove(i); + } } + !self.0.is_empty() + } +} + +fn fmt_accept_peer(res: &Result) -> String { + match res { + Ok(res) => res.peer.fmt_short(), + Err(err) => err + .peer() + .map(|x| x.fmt_short()) + .unwrap_or_else(|| "unknown".to_string()), + } +} + +fn fmt_accept_namespace(res: &Result) -> String { + match res { + Ok(res) => res.namespace.fmt_short(), + Err(err) => err + .namespace() + .map(|x| x.fmt_short()) + .unwrap_or_else(|| "unknown".to_string()), } } diff --git a/iroh/src/sync_engine/rpc.rs b/iroh/src/sync_engine/rpc.rs index 9533fd1c45..373d21ae8c 100644 --- a/iroh/src/sync_engine/rpc.rs +++ b/iroh/src/sync_engine/rpc.rs @@ -1,27 +1,24 @@ //! This module contains an impl block on [`SyncEngine`] with handlers for RPC requests use anyhow::anyhow; -use futures::{FutureExt, Stream}; -use iroh_bytes::{ - store::Store as BaoStore, - util::{BlobFormat, RpcError}, -}; -use iroh_sync::{store::Store, sync::Namespace}; -use itertools::Itertools; -use rand::rngs::OsRng; +use futures::Stream; +use iroh_bytes::{store::Store as BaoStore, util::BlobFormat}; +use iroh_sync::{store::Store, sync::Namespace, Author}; +use tokio_stream::StreamExt; use crate::{ rpc_protocol::{ AuthorCreateRequest, AuthorCreateResponse, AuthorListRequest, AuthorListResponse, - DocCreateRequest, DocCreateResponse, DocDelRequest, DocDelResponse, DocDropRequest, - DocDropResponse, DocGetManyRequest, DocGetManyResponse, DocGetOneRequest, - DocGetOneResponse, DocImportRequest, DocImportResponse, DocInfoRequest, DocInfoResponse, - DocLeaveRequest, DocLeaveResponse, DocListRequest, DocListResponse, DocSetHashRequest, - DocSetHashResponse, DocSetRequest, DocSetResponse, DocShareRequest, DocShareResponse, - DocStartSyncRequest, DocStartSyncResponse, DocSubscribeRequest, DocSubscribeResponse, - DocTicket, RpcResult, ShareMode, + DocCloseRequest, DocCloseResponse, DocCreateRequest, DocCreateResponse, DocDelRequest, + DocDelResponse, DocDropRequest, DocDropResponse, DocGetManyRequest, DocGetManyResponse, + DocGetOneRequest, DocGetOneResponse, DocImportRequest, DocImportResponse, DocLeaveRequest, + DocLeaveResponse, DocListRequest, DocListResponse, DocOpenRequest, DocOpenResponse, + DocSetHashRequest, DocSetHashResponse, DocSetRequest, DocSetResponse, DocShareRequest, + DocShareResponse, DocStartSyncRequest, DocStartSyncResponse, DocStatusRequest, + DocStatusResponse, DocSubscribeRequest, DocSubscribeResponse, DocTicket, RpcResult, + ShareMode, }, - sync_engine::{KeepCallback, LiveStatus, SyncEngine}, + sync_engine::SyncEngine, }; /// Capacity for the flume channels to forward sync store iterators to async RPC streams. @@ -29,9 +26,13 @@ const ITER_CHANNEL_CAP: usize = 64; #[allow(missing_docs)] impl SyncEngine { - pub fn author_create(&self, _req: AuthorCreateRequest) -> RpcResult { + pub async fn author_create( + &self, + _req: AuthorCreateRequest, + ) -> RpcResult { // TODO: pass rng - let author = self.store.new_author(&mut rand::rngs::OsRng {})?; + let author = Author::new(&mut rand::rngs::OsRng {}); + self.sync.import_author(author.clone()).await?; Ok(AuthorCreateResponse { author_id: author.id(), }) @@ -42,73 +43,76 @@ impl SyncEngine { _req: AuthorListRequest, ) -> impl Stream> { let (tx, rx) = flume::bounded(ITER_CHANNEL_CAP); - let store = self.store.clone(); - self.rt.main().spawn_blocking(move || { - let ite = store.list_authors(); - let ite = inline_result(ite).map_ok(|author| AuthorListResponse { - author_id: author.id(), - }); - for entry in ite { - if let Err(_err) = tx.send(entry) { - break; - } + let sync = self.sync.clone(); + // we need to spawn a task to send our request to the sync handle, because the method + // itself must be sync. + self.rt.main().spawn(async move { + let tx2 = tx.clone(); + if let Err(err) = sync.list_authors(tx).await { + tx2.send_async(Err(err)).await.ok(); } }); - rx.into_stream() + rx.into_stream().map(|r| { + r.map(|author_id| AuthorListResponse { author_id }) + .map_err(Into::into) + }) } - pub fn doc_create(&self, _req: DocCreateRequest) -> RpcResult { - let doc = self.store.new_replica(Namespace::new(&mut OsRng {}))?; - Ok(DocCreateResponse { - id: doc.namespace(), - }) + pub async fn doc_create(&self, _req: DocCreateRequest) -> RpcResult { + let namespace = Namespace::new(&mut rand::rngs::OsRng {}); + self.sync.import_namespace(namespace.clone()).await?; + self.sync.open(namespace.id(), Default::default()).await?; + Ok(DocCreateResponse { id: namespace.id() }) } pub async fn doc_drop(&self, req: DocDropRequest) -> RpcResult { let DocDropRequest { doc_id } = req; - let _replica = self.get_replica(&doc_id)?; self.leave(doc_id, true).await?; - self.store.remove_replica(&doc_id)?; + self.sync.drop_replica(doc_id).await?; Ok(DocDropResponse {}) } pub fn doc_list(&self, _req: DocListRequest) -> impl Stream> { let (tx, rx) = flume::bounded(ITER_CHANNEL_CAP); - let store = self.store.clone(); - self.rt.main().spawn_blocking(move || { - let ite = store.list_namespaces(); - let ite = inline_result(ite).map_ok(|id| DocListResponse { id }); - for entry in ite { - if let Err(_err) = tx.send(entry) { - break; - } + let sync = self.sync.clone(); + // we need to spawn a task to send our request to the sync handle, because the method + // itself must be sync. + self.rt.main().spawn(async move { + let tx2 = tx.clone(); + if let Err(err) = sync.list_replicas(tx).await { + tx2.send_async(Err(err)).await.ok(); } }); rx.into_stream() + .map(|r| r.map(|id| DocListResponse { id }).map_err(Into::into)) } - pub async fn doc_info(&self, req: DocInfoRequest) -> RpcResult { - let _replica = self.get_replica(&req.doc_id)?; - let status = self.live.status(req.doc_id).await?; - let status = status.unwrap_or(LiveStatus { - active: false, - subscriptions: 0, - }); - Ok(DocInfoResponse { status }) + pub async fn doc_open(&self, req: DocOpenRequest) -> RpcResult { + self.sync.open(req.doc_id, Default::default()).await?; + Ok(DocOpenResponse {}) + } + + pub async fn doc_close(&self, req: DocCloseRequest) -> RpcResult { + self.sync.close(req.doc_id).await?; + Ok(DocCloseResponse {}) + } + + pub async fn doc_status(&self, req: DocStatusRequest) -> RpcResult { + let status = self.sync.get_state(req.doc_id).await?; + Ok(DocStatusResponse { status }) } pub async fn doc_share(&self, req: DocShareRequest) -> RpcResult { - self.start_sync(req.doc_id, vec![]).await?; let me = self.endpoint.my_addr().await?; - let replica = self.get_replica(&req.doc_id)?; let key = match req.mode { ShareMode::Read => { // TODO: support readonly docs // *replica.namespace().as_bytes() return Err(anyhow!("creating read-only shares is not yet supported").into()); } - ShareMode::Write => replica.secret_key(), + ShareMode::Write => self.sync.export_secret_key(req.doc_id).await?.to_bytes(), }; + self.start_sync(req.doc_id, vec![]).await?; Ok(DocShareResponse(DocTicket { key, peers: vec![me], @@ -119,42 +123,20 @@ impl SyncEngine { &self, req: DocSubscribeRequest, ) -> impl Stream> { - let (s, r) = flume::bounded(64); - let s2 = s.clone(); - let res = self - .live - .subscribe(req.doc_id, { - move |event| { - let s = s.clone(); - async move { - // Send event over the channel, unsubscribe if the channel is closed. - match s.send_async(Ok(DocSubscribeResponse { event })).await { - Err(_err) => KeepCallback::Drop, - Ok(()) => KeepCallback::Keep, - } - } - .boxed() - } - }) - .await; - match res { - Err(err) => { - s2.send_async(Err(err.into())).await.ok(); - } - Ok(_token) => {} - }; - r.into_stream() + let stream = self.subscribe(req.doc_id); + stream.map(|res| { + res.map(|event| DocSubscribeResponse { event }) + .map_err(Into::into) + }) } pub async fn doc_import(&self, req: DocImportRequest) -> RpcResult { let DocImportRequest(DocTicket { key, peers }) = req; - // TODO: support read-only docs - // if let Ok(namespace) = match NamespaceId::from_bytes(&key) {}; let namespace = Namespace::from_bytes(&key); - let id = namespace.id(); - let replica = self.store.new_replica(namespace)?; - self.start_sync(replica.namespace(), peers).await?; - Ok(DocImportResponse { doc_id: id }) + let doc_id = self.sync.import_namespace(namespace).await?; + self.sync.open(doc_id, Default::default()).await?; + self.start_sync(doc_id, peers).await?; + Ok(DocImportResponse { doc_id }) } pub async fn doc_start_sync( @@ -168,7 +150,6 @@ impl SyncEngine { pub async fn doc_leave(&self, req: DocLeaveRequest) -> RpcResult { let DocLeaveRequest { doc_id } = req; - let _replica = self.get_replica(&doc_id)?; self.leave(doc_id, false).await?; Ok(DocLeaveResponse {}) } @@ -184,18 +165,15 @@ impl SyncEngine { key, value, } = req; - let replica = self.get_replica(&doc_id)?; - let author = self.get_author(&author_id)?; let len = value.len(); - let tag = bao_store - .import_bytes(value.into(), BlobFormat::Raw) + let tag = bao_store.import_bytes(value, BlobFormat::Raw).await?; + self.sync + .insert_local(doc_id, author_id, key.clone(), *tag.hash(), len as u64) .await?; - replica - .insert(&key, &author, *tag.hash(), len as u64) - .map_err(anyhow::Error::from)?; let entry = self - .store - .get_one(replica.namespace(), author.id(), &key)? + .sync + .get_one(doc_id, author_id, key) + .await? .ok_or_else(|| anyhow!("failed to get entry after insertion"))?; Ok(DocSetResponse { entry }) } @@ -206,11 +184,7 @@ impl SyncEngine { author_id, prefix, } = req; - let replica = self.get_replica(&doc_id)?; - let author = self.get_author(&author_id)?; - let removed = replica - .delete_prefix(prefix, &author) - .map_err(anyhow::Error::from)?; + let removed = self.sync.delete_prefix(doc_id, author_id, prefix).await?; Ok(DocDelResponse { removed }) } @@ -222,11 +196,9 @@ impl SyncEngine { hash, size, } = req; - let replica = self.get_replica(&doc_id)?; - let author = self.get_author(&author_id)?; - replica - .insert(key, &author, hash, size) - .map_err(anyhow::Error::from)?; + self.sync + .insert_local(doc_id, author_id, key.clone(), hash, size) + .await?; Ok(DocSetHashResponse {}) } @@ -236,17 +208,19 @@ impl SyncEngine { ) -> impl Stream> { let DocGetManyRequest { doc_id, filter } = req; let (tx, rx) = flume::bounded(ITER_CHANNEL_CAP); - let store = self.store.clone(); - self.rt.main().spawn_blocking(move || { - let ite = store.get_many(doc_id, filter); - let ite = inline_result(ite).map_ok(|entry| DocGetManyResponse { entry }); - for entry in ite { - if let Err(_err) = tx.send(entry) { - break; - } + let sync = self.sync.clone(); + // we need to spawn a task to send our request to the sync handle, because the method + // itself must be sync. + self.rt.main().spawn(async move { + let tx2 = tx.clone(); + if let Err(err) = sync.get_many(doc_id, filter, tx).await { + tx2.send_async(Err(err)).await.ok(); } }); - rx.into_stream() + rx.into_stream().map(|r| { + r.map(|entry| DocGetManyResponse { entry }) + .map_err(Into::into) + }) } pub async fn doc_get_one(&self, req: DocGetOneRequest) -> RpcResult { @@ -255,17 +229,7 @@ impl SyncEngine { author, key, } = req; - let replica = self.get_replica(&doc_id)?; - let entry = self.store.get_one(replica.namespace(), author, key)?; + let entry = self.sync.get_one(doc_id, author, key).await?; Ok(DocGetOneResponse { entry }) } } - -fn inline_result( - ite: Result>>, impl Into>, -) -> impl Iterator> { - match ite { - Ok(ite) => itertools::Either::Left(ite.map(|item| item.map_err(|err| err.into()))), - Err(err) => itertools::Either::Right(Some(Err(err.into())).into_iter()), - } -} diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 6fe28723b0..9ac88d15f2 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -1,25 +1,33 @@ -use std::{net::SocketAddr, time::Duration}; +use std::{ + future::Future, + net::SocketAddr, + sync::Arc, + time::{Duration, Instant}, +}; -use anyhow::{anyhow, bail, Result}; -use futures::{Stream, StreamExt, TryStreamExt}; +use anyhow::{anyhow, bail, Context, Result}; +use bytes::Bytes; +use futures::{Stream, StreamExt}; use iroh::{ client::mem::Doc, node::{Builder, Node}, rpc_protocol::ShareMode, sync_engine::{LiveEvent, SyncEvent}, }; -use iroh_net::key::PublicKey; +use iroh_net::key::{PublicKey, SecretKey}; use quic_rpc::transport::misc::DummyServerEndpoint; +use rand::{CryptoRng, Rng, SeedableRng}; use tracing::{debug, info}; use tracing_subscriber::{prelude::*, EnvFilter}; -use iroh_bytes::util::runtime; +use iroh_bytes::{util::runtime, Hash}; +use iroh_net::derp::DerpMode; use iroh_sync::{ store::{self, GetFilter}, - ContentStatus, NamespaceId, + AuthorId, ContentStatus, Entry, NamespaceId, }; -const LIMIT: Duration = Duration::from_secs(15); +const TIMEOUT: Duration = Duration::from_secs(60); /// Pick up the tokio runtime from the thread local and add a /// thread per core runtime. @@ -30,38 +38,58 @@ fn test_runtime() -> runtime::Handle { fn test_node( rt: runtime::Handle, addr: SocketAddr, + secret_key: SecretKey, ) -> Builder { let db = iroh_bytes::store::mem::Store::new(rt.clone()); let store = iroh_sync::store::memory::Store::default(); - Node::builder(db, store).runtime(&rt).bind_addr(addr) + Node::builder(db, store) + .secret_key(secret_key) + .derp_mode(DerpMode::Disabled) + .runtime(&rt) + .bind_addr(addr) } -async fn spawn_node( +// The function is not `async fn` so that we can take a `&mut` borrow on the `rng` without +// capturing that `&mut` lifetime in the returned future. This allows to call it in a loop while +// still collecting the futures before awaiting them alltogether (see [`spawn_nodes`]) +fn spawn_node( rt: runtime::Handle, i: usize, -) -> anyhow::Result> { - let node = test_node(rt, "127.0.0.1:0".parse()?); - let node = node.spawn().await?; - info!("spawned node {i} {:?}", node.peer_id()); - Ok(node) + rng: &mut (impl CryptoRng + Rng), +) -> impl Future>> + + 'static { + let secret_key = SecretKey::generate_with_rng(rng); + async move { + let node = test_node(rt, "127.0.0.1:0".parse()?, secret_key); + let node = node.spawn().await?; + info!(?i, me = %node.peer_id().fmt_short(), "node spawned"); + Ok(node) + } } async fn spawn_nodes( rt: runtime::Handle, n: usize, + mut rng: &mut (impl CryptoRng + Rng), ) -> anyhow::Result>> { - futures::future::join_all((0..n).map(|i| spawn_node(rt.clone(), i))) - .await - .into_iter() - .collect() + let mut futs = vec![]; + for i in 0..n { + futs.push(spawn_node(rt.clone(), i, &mut rng)); + } + futures::future::join_all(futs).await.into_iter().collect() +} + +pub fn test_rng(seed: &[u8]) -> rand_chacha::ChaCha12Rng { + rand_chacha::ChaCha12Rng::from_seed(*Hash::new(seed).as_bytes()) } /// This tests the simplest scenario: A node connects to another node, and performs sync. #[tokio::test] async fn sync_simple() -> Result<()> { setup_logging(); + let mut rng = test_rng(b"sync_simple"); let rt = test_runtime(); - let nodes = spawn_nodes(rt, 2).await?; + let nodes = spawn_nodes(rt, 2, &mut rng).await?; let clients = nodes.iter().map(|node| node.client()).collect::>(); // create doc on node0 @@ -82,25 +110,29 @@ async fn sync_simple() -> Result<()> { let doc1 = clients[1].docs.import(ticket.clone()).await?; let mut events1 = doc1.subscribe().await?; info!("node1: assert 4 events"); - assert_each_unordered( - collect_some(&mut events1, 4, LIMIT).await?, + assert_next_unordered( + &mut events1, + TIMEOUT, vec![ Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer0)), Box::new(move |e| matches!(e, LiveEvent::InsertRemote { from, .. } if *from == peer0 )), Box::new(move |e| match_sync_finished(e, peer0, doc_id)), Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash0)), ], - ); + ) + .await; assert_latest(&doc1, b"k1", b"v1").await; info!("node0: assert 2 events"); - assert_each_unordered( - collect_some(&mut events0, 2, LIMIT).await?, + assert_next_unordered( + &mut events0, + TIMEOUT, vec![ Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer1)), Box::new(move |e| match_sync_finished(e, peer1, doc_id)), ], - ); + ) + .await; for node in nodes { node.shutdown(); @@ -111,9 +143,10 @@ async fn sync_simple() -> Result<()> { /// Test subscribing to replica events (without sync) #[tokio::test] async fn sync_subscribe_no_sync() -> Result<()> { + let mut rng = test_rng(b"sync_subscribe"); setup_logging(); let rt = test_runtime(); - let node = spawn_node(rt, 0).await?; + let node = spawn_node(rt, 0, &mut rng).await?; let client = node.client(); let doc = client.docs.create().await?; let mut sub = doc.subscribe().await?; @@ -128,12 +161,101 @@ async fn sync_subscribe_no_sync() -> Result<()> { Ok(()) } +#[tokio::test] +async fn sync_gossip_bulk() -> Result<()> { + let n_entries: usize = std::env::var("N_ENTRIES") + .map(|x| x.parse().expect("N_ENTRIES must be a number")) + .unwrap_or(1000); + let mut rng = test_rng(b"sync_gossip_bulk"); + setup_logging(); + + let rt = test_runtime(); + let nodes = spawn_nodes(rt.clone(), 2, &mut rng).await?; + let clients = nodes.iter().map(|node| node.client()).collect::>(); + + let _peer0 = nodes[0].peer_id(); + let author0 = clients[0].authors.create().await?; + let doc0 = clients[0].docs.create().await?; + let mut ticket = doc0.share(ShareMode::Write).await?; + // unset peers to not yet start sync + let peers = ticket.peers.clone(); + ticket.peers = vec![]; + let doc1 = clients[1].docs.import(ticket).await?; + let mut events = doc1.subscribe().await?; + + // create entries for initial sync. + let now = Instant::now(); + let value = b"foo"; + for i in 0..n_entries { + let key = format!("init/{i}"); + doc0.set_bytes(author0, key.as_bytes().to_vec(), value.to_vec()) + .await?; + } + let elapsed = now.elapsed(); + info!( + "insert took {elapsed:?} for {n_entries} ({:?} per entry)", + elapsed / n_entries as u32 + ); + + let now = Instant::now(); + let mut count = 0; + doc0.start_sync(vec![]).await?; + doc1.start_sync(peers).await?; + while let Some(event) = events.next().await { + let event = event?; + if matches!(event, LiveEvent::InsertRemote { .. }) { + count += 1; + } + if count == n_entries { + break; + } + } + let elapsed = now.elapsed(); + info!( + "initial sync took {elapsed:?} for {n_entries} ({:?} per entry)", + elapsed / n_entries as u32 + ); + + // publish another 1000 entries + let mut count = 0; + let value = b"foo"; + let now = Instant::now(); + for i in 0..n_entries { + let key = format!("gossip/{i}"); + doc0.set_bytes(author0, key.as_bytes().to_vec(), value.to_vec()) + .await?; + } + let elapsed = now.elapsed(); + info!( + "insert took {elapsed:?} for {n_entries} ({:?} per entry)", + elapsed / n_entries as u32 + ); + + while let Some(event) = events.next().await { + let event = event?; + if matches!(event, LiveEvent::InsertRemote { .. }) { + count += 1; + } + if count == n_entries { + break; + } + } + let elapsed = now.elapsed(); + info!( + "gossip recv took {elapsed:?} for {n_entries} ({:?} per entry)", + elapsed / n_entries as u32 + ); + + Ok(()) +} + /// This tests basic sync and gossip with 3 peers. #[tokio::test] async fn sync_full_basic() -> Result<()> { + let mut rng = test_rng(b"sync_full_basic"); setup_logging(); let rt = test_runtime(); - let mut nodes = spawn_nodes(rt.clone(), 2).await?; + let mut nodes = spawn_nodes(rt.clone(), 2, &mut rng).await?; let mut clients = nodes.iter().map(|node| node.client()).collect::>(); // peer0: create doc and ticket @@ -165,24 +287,28 @@ async fn sync_full_basic() -> Result<()> { info!("peer1: wait for 4 events (for sync and join with peer0)"); let mut events1 = doc1.subscribe().await?; - assert_each_unordered( - collect_some(&mut events1, 4, LIMIT).await?, + assert_next_unordered( + &mut events1, + TIMEOUT, vec![ Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer0)), Box::new(move |e| matches!(e, LiveEvent::InsertRemote { from, .. } if *from == peer0 )), Box::new(move |e| match_sync_finished(e, peer0, doc_id)), Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash0)), ], - ); + ) + .await; info!("peer0: wait for 2 events (join & accept sync finished from peer1)"); - assert_each_unordered( - collect_some(&mut events0, 2, LIMIT).await?, + assert_next_unordered( + &mut events0, + TIMEOUT, vec![ Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer1)), Box::new(move |e| match_sync_finished(e, peer1, doc_id)), ], - ); + ) + .await; info!("peer1: insert entry"); let key1 = b"k2"; @@ -200,15 +326,16 @@ async fn sync_full_basic() -> Result<()> { // peer0: assert events for entry received via gossip info!("peer0: wait for 2 events (gossip'ed entry from peer1)"); - assert_each_unordered( - collect_some(&mut events0, 2, LIMIT).await?, + assert_next_unordered( + &mut events0, + TIMEOUT, vec![ Box::new( move |e| matches!(e, LiveEvent::InsertRemote { from, content_status: ContentStatus::Missing, .. } if *from == peer1), ), Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash1)), ], - ); + ).await; assert_latest(&doc0, key1, value1).await; // Note: If we could check gossip messages directly here (we can't easily), we would notice @@ -217,16 +344,17 @@ async fn sync_full_basic() -> Result<()> { // our gossip implementation does not allow us to filter message receivers this way. info!("peer2: spawn"); - nodes.push(spawn_node(rt.clone(), nodes.len()).await?); + nodes.push(spawn_node(rt.clone(), nodes.len(), &mut rng).await?); clients.push(nodes.last().unwrap().client()); let doc2 = clients[2].docs.import(ticket).await?; let peer2 = nodes[2].peer_id(); let mut events2 = doc2.subscribe().await?; info!("peer2: wait for 8 events (from sync with peers)"); - let actual = collect_some(&mut events2, 8, LIMIT).await?; - assert_each_unordered( - actual, + assert_next_unordered_with_optionals( + &mut events2, + TIMEOUT, + // required events vec![ // 2 NeighborUp events Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer0)), @@ -245,27 +373,40 @@ async fn sync_full_basic() -> Result<()> { Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash0)), Box::new(move |e| matches!(e, LiveEvent::ContentReady { hash } if *hash == hash1)), ], - ); + // optional events + // it may happen that we run sync two times against our two peers: + // if the first sync (as a result of us joining the peer manually through the ticket) completes + // before the peer shows up as a neighbor, we run sync again for the NeighborUp event. + vec![ + // 2 SyncFinished events + Box::new(move |e| match_sync_finished(e, peer0, doc_id)), + Box::new(move |e| match_sync_finished(e, peer1, doc_id)), + ] + ).await; assert_latest(&doc2, b"k1", b"v1").await; assert_latest(&doc2, b"k2", b"v2").await; info!("peer0: wait for 2 events (join & accept sync finished from peer2)"); - assert_each_unordered( - collect_some(&mut events0, 2, LIMIT).await?, + assert_next_unordered( + &mut events0, + TIMEOUT, vec![ Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer2)), Box::new(move |e| match_sync_finished(e, peer2, doc_id)), ], - ); + ) + .await; info!("peer1: wait for 2 events (join & accept sync finished from peer2)"); - assert_each_unordered( - collect_some(&mut events1, 2, LIMIT).await?, + assert_next_unordered( + &mut events1, + TIMEOUT, vec![ Box::new(move |e| matches!(e, LiveEvent::NeighborUp(peer) if *peer == peer2)), Box::new(move |e| match_sync_finished(e, peer2, doc_id)), ], - ); + ) + .await; info!("shutdown"); for node in nodes { @@ -276,34 +417,103 @@ async fn sync_full_basic() -> Result<()> { } #[tokio::test] -async fn sync_subscribe_stop() -> Result<()> { +async fn sync_open_close() -> Result<()> { + let mut rng = test_rng(b"sync_subscribe_stop_close"); + setup_logging(); + let rt = test_runtime(); + let node = spawn_node(rt, 0, &mut rng).await?; + let client = node.client(); + + let doc = client.docs.create().await?; + let status = doc.status().await?; + assert_eq!(status.handles, 1); + + let doc2 = client.docs.open(doc.id()).await?.unwrap(); + let status = doc2.status().await?; + assert_eq!(status.handles, 2); + + doc.close().await?; + assert!(doc.status().await.is_err()); + + let status = doc2.status().await?; + assert_eq!(status.handles, 1); + + Ok(()) +} + +#[tokio::test] +async fn sync_subscribe_stop_close() -> Result<()> { + let mut rng = test_rng(b"sync_subscribe_stop_close"); setup_logging(); let rt = test_runtime(); - let node = spawn_node(rt, 0).await?; + let node = spawn_node(rt, 0, &mut rng).await?; let client = node.client(); let doc = client.docs.create().await?; let author = client.authors.create().await?; - doc.start_sync(vec![]).await?; let status = doc.status().await?; - assert!(status.active); - assert_eq!(status.subscriptions, 0); + assert_eq!(status.subscribers, 0); + assert_eq!(status.handles, 1); + assert!(!status.sync); + + doc.start_sync(vec![]).await?; + let status = doc.status().await?; + assert!(status.sync); + assert_eq!(status.handles, 2); + assert_eq!(status.subscribers, 1); let sub = doc.subscribe().await?; let status = doc.status().await?; - assert_eq!(status.subscriptions, 1); + assert_eq!(status.subscribers, 2); drop(sub); - + // trigger an event that makes the actor check if the event channels are still connected doc.set_bytes(author, b"x".to_vec(), b"x".to_vec()).await?; let status = doc.status().await?; - assert_eq!(status.subscriptions, 0); + assert_eq!(status.subscribers, 1); - node.shutdown(); + doc.leave().await?; + let status = doc.status().await?; + assert_eq!(status.subscribers, 0); + assert_eq!(status.handles, 1); + assert!(!status.sync); Ok(()) } +#[derive(Debug, Ord, Eq, PartialEq, PartialOrd, Clone)] +struct ExpectedEntry { + author: AuthorId, + key: String, + value: String, +} + +impl PartialEq for ExpectedEntry { + fn eq(&self, other: &Entry) -> bool { + self.key.as_bytes() == other.key() + && Hash::new(&self.value) == other.content_hash() + && self.author == other.author() + } +} +impl PartialEq<(Entry, Bytes)> for ExpectedEntry { + fn eq(&self, (entry, content): &(Entry, Bytes)) -> bool { + self.key.as_bytes() == entry.key() + && Hash::new(&self.value) == entry.content_hash() + && self.author == entry.author() + && self.value.as_bytes() == content.as_ref() + } +} +impl PartialEq for Entry { + fn eq(&self, other: &ExpectedEntry) -> bool { + other.eq(self) + } +} +impl PartialEq for (Entry, Bytes) { + fn eq(&self, other: &ExpectedEntry) -> bool { + other.eq(self) + } +} + #[tokio::test] async fn doc_delete() -> Result<()> { let rt = test_runtime(); @@ -340,9 +550,10 @@ async fn doc_delete() -> Result<()> { #[tokio::test] async fn sync_drop_doc() -> Result<()> { + let mut rng = test_rng(b"sync_drop_doc"); setup_logging(); let rt = test_runtime(); - let node = spawn_node(rt, 0).await?; + let node = spawn_node(rt, 0, &mut rng).await?; let client = node.client(); let doc = client.docs.create().await?; @@ -361,10 +572,8 @@ async fn sync_drop_doc() -> Result<()> { .set_bytes(author, b"foo".to_vec(), b"bar".to_vec()) .await; assert!(res.is_err()); - let res = client.docs.get(doc.id()).await?; - assert!(res.is_none()); - let ev = sub.next().await; - assert!(matches!(ev, Some(Ok(LiveEvent::Closed)))); + let res = client.docs.open(doc.id()).await; + assert!(res.is_err()); let ev = sub.next().await; assert!(ev.is_none()); @@ -406,67 +615,95 @@ async fn next(mut stream: impl Stream> + Un event } -/// Collect the next n elements of a [`TryStream`] -/// -/// If `timeout` is exceeded before n elements are collected an error is returned. -async fn collect_some( - mut stream: impl Stream> + Unpin, - n: usize, - timeout: Duration, -) -> Result> { - let mut res = Vec::with_capacity(n); - let sleep = tokio::time::sleep(timeout); - tokio::pin!(sleep); - while res.len() < n { - tokio::select! { - () = &mut sleep => { - bail!("Failed to collect {n} elements in {timeout:?} (collected only {})", res.len()); - }, - event = stream.try_next() => { - let event = event?; - match event { - None => bail!("stream ended after {} items, but expected {n}", res.len()), - Some(event) => res.push(event), - } - } +#[allow(clippy::type_complexity)] +fn apply_matchers(item: &T, matchers: &mut Vec bool>>) -> bool { + for i in 0..matchers.len() { + if matchers[i](item) { + let _ = matchers.remove(i); + return true; } } - Ok(res) + false } -/// Assert that each item in the iterator is matched by one of the functions in `fns`. +/// Receive `matchers.len()` elements from a stream and assert that each element matches one of the +/// functions in `matchers`. /// -/// The iterator must yield exactly as many elements as are in the function list. -/// Order is not imporant. Once a function matched an item, it is removed from the function list. +/// Order of the matchers is not relevant. +/// +/// Returns all received events. #[allow(clippy::type_complexity)] -fn assert_each_unordered( - items: impl IntoIterator, - mut fns: Vec bool>>, -) { - let len = fns.len(); - let iter = items.into_iter(); - for item in iter { - if fns.is_empty() { - panic!("iterator is longer than expected length of {len}"); - } - let mut ok = false; - for i in 0..fns.len() { - if fns[i](&item) { - ok = true; - let _ = fns.remove(i); +async fn assert_next_unordered( + stream: impl Stream> + Unpin, + timeout: Duration, + matchers: Vec bool>>, +) -> Vec { + assert_next_unordered_with_optionals(stream, timeout, matchers, vec![]).await +} + +/// Receive between `min` and `max` elements from the stream and assert that each element matches +/// either one of the matchers in `required_matchers` or in `optional_matchers`. +/// +/// Order of the matchers is not relevant. +/// +/// Will return an error if: +/// * Any element fails to match one of the required or optional matchers +/// * More than `max` elements were received, but not all required matchers were used yet +/// * The timeout completes before all required matchers were used +/// +/// Returns all received events. +#[allow(clippy::type_complexity)] +async fn assert_next_unordered_with_optionals( + mut stream: impl Stream> + Unpin, + timeout: Duration, + mut required_matchers: Vec bool>>, + mut optional_matchers: Vec bool>>, +) -> Vec { + let max = required_matchers.len() + optional_matchers.len(); + let required = required_matchers.len(); + // we have to use a mutex because rustc is not intelligent enough to realize + // that the mutable borrow terminates when the future completes + let events = Arc::new(parking_lot::Mutex::new(vec![])); + let fut = async { + while let Some(event) = stream.next().await { + let event = event.context("failed to read from stream")?; + let len = { + let mut events = events.lock(); + events.push(event.clone()); + events.len() + }; + if !apply_matchers(&event, &mut required_matchers) + && !apply_matchers(&event, &mut optional_matchers) + { + bail!("Event didn't match any matcher: {event:?}"); + } + if required_matchers.is_empty() || len == max { break; } } - if !ok { - panic!("no rule matched item {item:?}"); + if !required_matchers.is_empty() { + bail!( + "Matched only {} of {required} required matchers", + required - required_matchers.len() + ); } - } - if !fns.is_empty() { - panic!( - "expected {len} elements but stream stopped after {}", - len - fns.len() + Ok(()) + }; + tokio::pin!(fut); + let res = tokio::time::timeout(timeout, fut) + .await + .map_err(|_| anyhow!("Timeout reached ({timeout:?})")) + .and_then(|res| res); + let events = events.lock().clone(); + if let Err(err) = &res { + println!("Received events: {events:#?}"); + println!( + "Received {} events, expected between {required} and {max}", + events.len() ); + panic!("Failed to receive or match all events: {err:?}"); } + events } /// Asserts that the event is a [`LiveEvent::SyncFinished`] and that the contained [`SyncEvent`]