dojo-client partial world syncer (#630)

* Add deps

* Add entity storage trait

* Add syncer struct

* Add sync example project

* update

* update cairo test

Cargo.toml

@@ -2,6 +2,7 @@
 
 resolver = "2"
 
+exclude = [ "examples/client" ]
 members = [
   "crates/dojo-client",
   "crates/dojo-lang",

crates/dojo-client/Cargo.toml

@@ -6,13 +6,14 @@ version = "0.1.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-trait.workspace = true
 dojo-types = { path = "../dojo-types" }
-dojo-world = { path = "../dojo-world" }
 starknet-crypto.workspace = true
 starknet.workspace = true
 thiserror.workspace = true
+tokio = { version = "1.16", features = [ "sync", "time" ], default-features = false }
 
 [dev-dependencies]
 camino.workspace = true
 dojo-test-utils = { path = "../dojo-test-utils", features = [ "build-examples" ] }
-tokio.workspace = true
+dojo-world = { path = "../dojo-world" }

crates/dojo-client/src/contract/system_test.rs

@@ -44,5 +44,5 @@ async fn test_system() {
 
     let position = position_component.entity(vec![account.address()], block_id).await.unwrap();
 
-    assert_eq!(position, vec![1_u8.into(), 1_u8.into()]);
+    assert_eq!(position, vec![11_u8.into(), 11_u8.into()]);
 }

crates/dojo-client/src/lib.rs

@@ -1 +1,3 @@
 pub mod contract;
+pub mod storage;
+pub mod sync;

crates/dojo-client/src/storage.rs

@@ -0,0 +1,35 @@
+use async_trait::async_trait;
+use starknet::macros::short_string;
+use starknet_crypto::{poseidon_hash_many, FieldElement};
+
+// TODO: is this low level enough?
+/// Low level storage interface
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait EntityStorage {
+    type Error;
+
+    /// This function mimic `world::set_entity` of `dojo-core`
+    async fn set(
+        &mut self,
+        component: FieldElement,
+        keys: Vec<FieldElement>,
+        values: Vec<FieldElement>,
+    ) -> Result<(), Self::Error>;
+
+    /// This function mimic `world::entity` of `dojo-core`
+    async fn get(
+        &self,
+        component: FieldElement,
+        keys: Vec<FieldElement>,
+        length: usize,
+    ) -> Result<Vec<FieldElement>, Self::Error>;
+}
+
+pub fn component_storage_base_address(
+    component: FieldElement,
+    keys: &[FieldElement],
+) -> FieldElement {
+    let id = poseidon_hash_many(keys);
+    poseidon_hash_many(&[short_string!("dojo_storage"), component, id])
+}

crates/dojo-client/src/sync.rs

@@ -0,0 +1,135 @@
+use std::sync::Arc;
+use std::time::Duration;
+
+use starknet::core::types::{BlockId, BlockTag};
+use starknet::core::utils::cairo_short_string_to_felt;
+use starknet::providers::Provider;
+use starknet_crypto::FieldElement;
+use thiserror::Error;
+use tokio::sync::RwLock;
+use tokio::time::{Instant, Interval};
+
+use crate::contract::component::{ComponentError, ComponentReader};
+use crate::contract::world::WorldContractReader;
+use crate::storage::EntityStorage;
+
+#[derive(Debug, Error)]
+pub enum SyncerError<S, P> {
+    #[error(transparent)]
+    Component(ComponentError<P>),
+    #[error(transparent)]
+    Storage(S),
+}
+
+/// Request to sync a component of an entity.
+/// This struct will be used to construct an [EntityReader].
+pub struct EntityComponentReq {
+    /// Component name
+    pub component: String,
+    /// The entity keys
+    pub keys: Vec<FieldElement>,
+}
+
+/// A type which wraps a [ComponentReader] for reading a component value of an entity.
+pub struct EntityReader<'a, P: Provider + Sync> {
+    /// Component name
+    component: String,
+    /// The entity keys
+    keys: Vec<FieldElement>,
+    /// Component reader
+    reader: ComponentReader<'a, P>,
+}
+
+pub struct WorldPartialSyncer<'a, S: EntityStorage, P: Provider + Sync> {
+    // We wrap it in an Arc<RwLock> to allow sharing the storage between threads.
+    /// Storage to store the synced entity component values.
+    storage: Arc<RwLock<S>>,
+    /// Client for reading the World contract.
+    world_reader: &'a WorldContractReader<'a, P>,
+    /// The entity components to sync.
+    entity_components_to_sync: Vec<EntityComponentReq>,
+    /// The interval to run the syncing loop.
+    interval: Interval,
+}
+
+impl<'a, S, P> WorldPartialSyncer<'a, S, P>
+where
+    S: EntityStorage + Send + Sync,
+    P: Provider + Sync + 'static,
+{
+    const DEFAULT_INTERVAL: Duration = Duration::from_secs(1);
+
+    pub fn new(
+        storage: Arc<RwLock<S>>,
+        world_reader: &'a WorldContractReader<'a, P>,
+        entities: Vec<EntityComponentReq>,
+    ) -> WorldPartialSyncer<'a, S, P> {
+        Self {
+            world_reader,
+            storage,
+            entity_components_to_sync: entities,
+            interval: tokio::time::interval_at(
+                Instant::now() + Self::DEFAULT_INTERVAL,
+                Self::DEFAULT_INTERVAL,
+            ),
+        }
+    }
+
+    pub fn with_interval(mut self, milisecond: u64) -> Self {
+        let interval = Duration::from_millis(milisecond);
+        self.interval = tokio::time::interval_at(Instant::now() + interval, interval);
+        self
+    }
+
+    /// Starts the syncing process.
+    /// This function will run forever.
+    pub async fn start(&mut self) -> Result<(), SyncerError<S::Error, P::Error>> {
+        let entity_readers = self.entity_readers().await?;
+
+        loop {
+            self.interval.tick().await;
+
+            for reader in &entity_readers {
+                let values = reader
+                    .reader
+                    .entity(reader.keys.clone(), BlockId::Tag(BlockTag::Pending))
+                    .await
+                    .map_err(SyncerError::Component)?;
+
+                self.storage
+                    .write()
+                    .await
+                    .set(
+                        cairo_short_string_to_felt(&reader.component).unwrap(),
+                        reader.keys.clone(),
+                        values,
+                    )
+                    .await
+                    .map_err(SyncerError::Storage)?
+            }
+        }
+    }
+
+    /// Get the entity reader for every requested component to sync.
+    async fn entity_readers(
+        &self,
+    ) -> Result<Vec<EntityReader<'a, P>>, SyncerError<S::Error, P::Error>> {
+        let mut entity_readers = Vec::new();
+
+        for i in &self.entity_components_to_sync {
+            let comp_reader = self
+                .world_reader
+                .component(&i.component, BlockId::Tag(BlockTag::Pending))
+                .await
+                .map_err(SyncerError::Component)?;
+
+            entity_readers.push(EntityReader {
+                component: i.component.clone(),
+                keys: i.keys.clone(),
+                reader: comp_reader,
+            });
+        }
+
+        Ok(entity_readers)
+    }
+}

examples/client/.gitignore

@@ -0,0 +1,2 @@
+target
+Cargo.lock

examples/client/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+edition = "2021"
+name = "partial-syncer-example"
+version = "0.1.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1.68"
+dojo-client = { path = "../../crates/dojo-client" }
+starknet = "0.4.0"
+tokio = { version = "1.11.0", features = [ "full" ] }
+url = "2.2.2"

examples/client/src/main.rs

@@ -0,0 +1,109 @@
+//! This example demonstrates how to use the `WorldPartialSyncer` to sync a single entity from the
+//! world contract. The entity is synced to an in-memory storage, and then the storage is queried
+//! for the entity's component.
+//!
+//! This uses the example project under `examples/ecs`.
+//!
+//! To run this example, you must first migrate the `ecs` example project with `--name ohayo`
+
+use std::sync::Arc;
+use std::time::Duration;
+use std::vec;
+
+use dojo_client::contract::world::{WorldContract, WorldContractReader};
+use dojo_client::storage::EntityStorage;
+use dojo_client::sync::{EntityComponentReq, WorldPartialSyncer};
+use starknet::accounts::SingleOwnerAccount;
+use starknet::core::types::FieldElement;
+use starknet::core::utils::cairo_short_string_to_felt;
+use starknet::providers::jsonrpc::HttpTransport;
+use starknet::providers::{JsonRpcClient, Provider};
+use starknet::signers::{LocalWallet, SigningKey};
+use storage::InMemoryStorage;
+use tokio::select;
+use tokio::sync::RwLock;
+use url::Url;
+
+mod storage;
+
+async fn run_execute_system_loop() {
+    let world_address = FieldElement::from_hex_be(
+        "0x2886968c76e33e66d2206ad75f57c931bafed9b784294d3649a950e0cd3e973",
+    )
+    .unwrap();
+
+    let provider =
+        JsonRpcClient::new(HttpTransport::new(Url::parse("http://localhost:5050").unwrap()));
+
+    let chain_id = provider.chain_id().await.unwrap();
+
+    let account = SingleOwnerAccount::new(
+        provider,
+        LocalWallet::from_signing_key(SigningKey::from_secret_scalar(
+            FieldElement::from_hex_be(
+                "0x0300001800000000300000180000000000030000000000003006001800006600",
+            )
+            .unwrap(),
+        )),
+        FieldElement::from_hex_be(
+            "0x03ee9e18edc71a6df30ac3aca2e0b02a198fbce19b7480a63a0d71cbd76652e0",
+        )
+        .unwrap(),
+        chain_id,
+    );
+
+    let world = WorldContract::new(world_address, &account);
+
+    loop {
+        let _ = world.execute("spawn", vec![]).await.unwrap();
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+}
+
+async fn run_read_storage_loop(storage: Arc<RwLock<InMemoryStorage>>, keys: Vec<FieldElement>) {
+    loop {
+        // Read the entity's position directly from the storage.
+        let values = storage
+            .read()
+            .await
+            .get(cairo_short_string_to_felt("Position").unwrap(), keys.clone(), 2)
+            .await
+            .unwrap();
+
+        println!("Position x {:#x} y {:#x}", values[0], values[1]);
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    let client =
+        JsonRpcClient::new(HttpTransport::new(Url::parse("http://localhost:5050").unwrap()));
+
+    let world_address = FieldElement::from_hex_be(
+        "0x2886968c76e33e66d2206ad75f57c931bafed9b784294d3649a950e0cd3e973",
+    )
+    .unwrap();
+    let keys: Vec<FieldElement> = vec![
+        FieldElement::from_hex_be(
+            "0x3ee9e18edc71a6df30ac3aca2e0b02a198fbce19b7480a63a0d71cbd76652e0",
+        )
+        .unwrap(),
+    ];
+
+    let storage = Arc::new(RwLock::new(InMemoryStorage::new()));
+
+    let world_reader = WorldContractReader::new(world_address, &client);
+    let mut syncer = WorldPartialSyncer::new(
+        Arc::clone(&storage),
+        &world_reader,
+        vec![EntityComponentReq { component: String::from("Position"), keys: keys.clone() }],
+    )
+    .with_interval(2000);
+
+    select! {
+        _ = run_execute_system_loop() => {}
+        _ = syncer.start() => {}
+        _ = run_read_storage_loop(storage, keys) => {}
+    }
+}

examples/client/src/storage.rs

@@ -0,0 +1,52 @@
+use std::collections::HashMap;
+
+use async_trait::async_trait;
+use dojo_client::storage::{component_storage_base_address, EntityStorage};
+use starknet::core::types::FieldElement;
+
+/// Simple in memory implementation of [EntityStorage]
+pub struct InMemoryStorage {
+    /// storage key -> Component value
+    pub inner: HashMap<FieldElement, FieldElement>,
+}
+
+impl InMemoryStorage {
+    pub fn new() -> Self {
+        Self { inner: HashMap::new() }
+    }
+}
+
+// Example implementation of [EntityStorage]
+#[async_trait]
+impl EntityStorage for InMemoryStorage {
+    type Error = ();
+
+    async fn set(
+        &mut self,
+        component: FieldElement,
+        keys: Vec<FieldElement>,
+        values: Vec<FieldElement>,
+    ) -> Result<(), Self::Error> {
+        let base_address = component_storage_base_address(component, &keys);
+        for (offset, value) in values.into_iter().enumerate() {
+            self.inner.insert(base_address + offset.into(), value);
+        }
+        Ok(())
+    }
+
+    async fn get(
+        &self,
+        component: FieldElement,
+        keys: Vec<FieldElement>,
+        length: usize,
+    ) -> Result<Vec<FieldElement>, Self::Error> {
+        let base_address = component_storage_base_address(component, &keys);
+        let mut values = Vec::with_capacity(length);
+        for i in 0..length {
+            let address = base_address + i.into();
+            let value = self.inner.get(&address).cloned();
+            values.push(value.unwrap_or(FieldElement::ZERO));
+        }
+        Ok(values)
+    }
+}