GO-4012 merge

Makefile

@@ -339,7 +339,7 @@ endif
 ### Tantivy Section
 
 REPO := anyproto/tantivy-go
-VERSION := v0.0.9
+VERSION := v0.1.0
 OUTPUT_DIR := deps/libs
 SHA_FILE = tantivity_sha256.txt
 
@@ -363,28 +363,31 @@ define remove_arch
 	rm -f $(OUTPUT_DIR)/$(1)
 endef
 
-download-tantivy: $(TANTIVY_LIBS)
+remove-libs:
+	@rm -rf deps/libs/*
+
+download-tantivy: remove-libs $(TANTIVY_LIBS)
 
 $(TANTIVY_LIBS):
 	@mkdir -p $(OUTPUT_DIR)/$(shell echo $@ | cut -d'.' -f1)
 	$(call download_tantivy_lib,$@)
 	@tar -C $(OUTPUT_DIR)/$(shell echo $@ | cut -d'.' -f1) -xvzf $(OUTPUT_DIR)/$@
 
 download-tantivy-all-force: download-tantivy
-	@rm -f $(SHA_FILE)
+	rm -f $(SHA_FILE)
 	@for file in $(TANTIVY_LIBS); do \
 		echo "SHA256 $(OUTPUT_DIR)/$$file" ; \
 		shasum -a 256 $(OUTPUT_DIR)/$$file | awk '{print $$1 "  " "'$(OUTPUT_DIR)/$$file'" }' >> $(SHA_FILE); \
 	done
+	@rm -rf deps/libs/*.tar.gz
 	@echo "SHA256 checksums generated."
 
 download-tantivy-all: download-tantivy
-	@rm -rf /deps/libs/*
 	@echo "Validating SHA256 checksums..."
 	@shasum -a 256 -c $(SHA_FILE) --status || { echo "Hash mismatch detected."; exit 1; }
 	@echo "All files are valid."
+	@rm -rf deps/libs/*.tar.gz
 
-download-tantivy-local:
-	@rm -rf /deps/libs/*
+download-tantivy-local: remove-libs
 	@mkdir -p $(OUTPUT_DIR)
 	@cp -r $(TANTIVY_GO_PATH)/libs/ $(OUTPUT_DIR)

core/anytype/bootstrap.go

@@ -25,6 +25,7 @@ import (
 	"github.com/anyproto/any-sync/nodeconf"
 	"github.com/anyproto/any-sync/nodeconf/nodeconfstore"
 	"github.com/anyproto/any-sync/util/crypto"
+	"github.com/anyproto/any-sync/util/syncqueues"
 	"go.uber.org/zap"
 
 	"github.com/anyproto/any-sync/nameservice/nameserviceclient"
@@ -217,6 +218,7 @@ func Bootstrap(a *app.App, components ...app.Component) {
 		Register(nodeconfsource.New()).
 		Register(nodeconfstore.New()).
 		Register(nodeconf.New()).
+		Register(syncqueues.New()).
 		Register(peerstore.New()).
 		Register(storage.New()).
 		Register(secureservice.New()).
@@ -228,10 +230,11 @@ func Bootstrap(a *app.App, components ...app.Component) {
 		Register(yamux.New()).
 		Register(quic.New()).
 		Register(clientserver.New()).
-		Register(streampool.New()).
 		Register(coordinatorclient.New()).
 		Register(nodeclient.New()).
 		Register(credentialprovider.New()).
+		Register(spacecore.NewStreamOpener()).
+		Register(streampool.New()).
 		Register(commonspace.New()).
 		Register(aclclient.NewAclJoiningClient()).
 		Register(virtualspaceservice.New()).

core/anytype/config/config.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 
 	"github.com/anyproto/any-sync/app"
+	"github.com/anyproto/any-sync/net/streampool"
+
 	//nolint:misspell
 	"github.com/anyproto/any-sync/commonspace/config"
 	"github.com/anyproto/any-sync/metric"
@@ -389,6 +391,14 @@ func (c *Config) GetNodeConfStorePath() string {
 	return filepath.Join(c.RepoPath, "nodeconf")
 }
 
+func (c *Config) GetStreamConfig() streampool.StreamConfig {
+	return streampool.StreamConfig{
+		SendQueueSize:    300,
+		DialQueueWorkers: 4,
+		DialQueueSize:    300,
+	}
+}
+
 func (c *Config) GetYamux() yamux.Config {
 	return yamux.Config{
 		ListenAddrs:     []string{},

core/block/editor/chatobject/chatobject_test.go

@@ -121,7 +121,6 @@ func TestEditMessage(t *testing.T) {
 	// Edit
 	editedMessage := givenMessage()
 	editedMessage.Message.Text = "edited text!"
-	editedMessage.Reactions.Reactions["🥰"].Ids = []string{"identity1"}
 
 	changeId = "messageId2"
 	fx.source.EXPECT().PushStoreChange(mock.Anything, mock.Anything).RunAndReturn(applyToStore(changeId))
@@ -147,6 +146,10 @@ func TestEditMessage(t *testing.T) {
 	assert.Equal(t, want, got)
 }
 
+func TestToggleReaction(t *testing.T) {
+	// TODO Implement
+}
+
 func applyToStore(changeId string) func(ctx context.Context, params source.PushStoreChangeParams) (string, error) {
 	return func(ctx context.Context, params source.PushStoreChangeParams) (string, error) {
 		tx, err := params.State.NewTx(ctx)

core/block/editor/state/position.go

@@ -176,37 +176,20 @@ func (s *State) wrapToRow(opId string, parent, b simple.Block) (row simple.Block
 	}
 	// do not need to remove from cache
 	parent.Model().ChildrenIds[pos] = row.Model().Id
-	s.addCacheIds(parent.Model(), row.Model().Id)
 	return
 }
 
 func (s *State) setChildrenIds(parent *model.Block, childrenIds []string) {
 	parent.ChildrenIds = childrenIds
-	s.addCacheIds(parent, childrenIds...)
 }
 
 // do not use this method outside of normalization
 func (s *State) SetChildrenIds(parent *model.Block, childrenIds []string) {
 	s.setChildrenIds(parent, childrenIds)
 }
 
-// do not use this method outside of normalization
-func (s *State) RemoveFromCache(childrenIds []string) {
-	s.removeFromCache(childrenIds...)
-}
-
 func (s *State) removeChildren(parent *model.Block, childrenId string) {
 	parent.ChildrenIds = slice.RemoveMut(parent.ChildrenIds, childrenId)
-	s.removeFromCache(childrenId)
-}
-
-func (s *State) addCacheIds(parent *model.Block, childrenIds ...string) {
-	if s.isParentIdsCacheEnabled {
-		cache := s.getParentIdsCache()
-		for _, childId := range childrenIds {
-			cache[childId] = parent.Id
-		}
-	}
 }
 
 func (s *State) prependChildrenIds(block *model.Block, ids ...string) {

core/block/editor/state/state.go

@@ -135,8 +135,6 @@ type State struct {
 	groupId                  string
 	noObjectType             bool
 	originalCreatedTimestamp int64 // pass here from snapshots when importing objects
-	parentIdsCache           map[string]string
-	isParentIdsCacheEnabled  bool
 }
 
 func (s *State) MigrationVersion() uint32 {
@@ -215,21 +213,12 @@ func (s *State) Set(b simple.Block) {
 	if !s.Exists(b.Model().Id) {
 		s.Add(b)
 	} else {
-		s.removeFromCache(s.Pick(b.Model().Id).Model().ChildrenIds...)
 		s.setChildrenIds(b.Model(), b.Model().ChildrenIds)
 		s.blocks[b.Model().Id] = b
 		s.blockInit(b)
 	}
 }
 
-func (s *State) removeFromCache(ids ...string) {
-	if s.isParentIdsCacheEnabled {
-		for _, id := range ids {
-			delete(s.parentIdsCache, id)
-		}
-	}
-}
-
 func (s *State) Get(id string) (b simple.Block) {
 	if b = s.blocks[id]; b != nil {
 		return
@@ -265,7 +254,6 @@ func (s *State) CleanupBlock(id string) bool {
 	)
 	for t != nil {
 		if _, ok = t.blocks[id]; ok {
-			s.removeFromCache(id)
 			delete(t.blocks, id)
 			return true
 		}
@@ -323,81 +311,16 @@ func (s *State) HasParent(id, parentId string) bool {
 }
 
 func (s *State) PickParentOf(id string) (res simple.Block) {
-	var cacheFound simple.Block
-	if s.isParentIdsCacheEnabled {
-		cache := s.getParentIdsCache()
-		if parentId, ok := cache[id]; ok {
-			cacheFound = s.Pick(parentId)
-		}
-		// if cacheFound != nil {
-		// 	rootId := s.RootId()
-		// 	topParentId := cacheFound.Model().Id
-		// 	for topParentId != rootId {
-		// 		if nextId, ok := cache[topParentId]; ok {
-		// 			topParentId = nextId
-		// 		} else {
-		// 			cacheFound = nil
-		// 			break
-		// 		}
-		// 	}
-		// }
-		// restore this code after checking if cache is working correctly
-		// return
-	}
-
 	s.Iterate(func(b simple.Block) bool {
 		if slice.FindPos(b.Model().ChildrenIds, id) != -1 {
 			res = b
 			return false
 		}
 		return true
 	})
-
-	// remove this code after checking if cache is working correctly
-	if s.isParentIdsCacheEnabled && res != cacheFound {
-		// var cacheFoundId, resFoundId string
-		// if cacheFound != nil {
-		// 	cacheFoundId = cacheFound.Model().Id
-		// }
-		// if res != nil {
-		// 	resFoundId = res.Model().Id
-		// }
-		// log.With("id", id).
-		// 	With("cacheFoundId", cacheFoundId).
-		// 	With("resFoundId", resFoundId).
-		// 	With("objId", s.RootId()).
-		// 	Warn("discrepancy in state parent search")
-	}
-
 	return
 }
 
-func (s *State) ResetParentIdsCache() {
-	s.parentIdsCache = nil
-	s.isParentIdsCacheEnabled = false
-}
-
-func (s *State) EnableParentIdsCache() bool {
-	if s.isParentIdsCacheEnabled {
-		return true
-	}
-	s.isParentIdsCacheEnabled = true
-	return false
-}
-
-func (s *State) getParentIdsCache() map[string]string {
-	if s.parentIdsCache == nil {
-		s.parentIdsCache = make(map[string]string)
-		s.Iterate(func(block simple.Block) bool {
-			for _, id := range block.Model().ChildrenIds {
-				s.parentIdsCache[id] = block.Model().Id
-			}
-			return true
-		})
-	}
-	return s.parentIdsCache
-}
-
 func (s *State) IsChild(parentId, childId string) bool {
 	for {
 		parent := s.PickParentOf(childId)
@@ -502,12 +425,6 @@ func ApplyStateFastOne(s *State) (msgs []simple.EventMessage, action undo.Action
 }
 
 func (s *State) apply(fast, one, withLayouts bool) (msgs []simple.EventMessage, action undo.Action, err error) {
-	alreadyEnabled := s.EnableParentIdsCache()
-	defer func() {
-		if !alreadyEnabled {
-			s.ResetParentIdsCache()
-		}
-	}()
 	if s.parent != nil && (s.parent.parent != nil || fast) {
 		s.intermediateApply()
 		if one {

core/block/editor/table/block.go

@@ -250,7 +250,6 @@ func (tb Table) normalizeColumns() {
 	}
 
 	if invalidFound {
-		tb.s.RemoveFromCache(toRemove)
 		tb.s.SetChildrenIds(tb.Columns(), colIds)
 	}
 }
@@ -292,7 +291,6 @@ func (tb Table) normalizeRows() {
 	}
 
 	if invalidFound {
-		tb.s.RemoveFromCache(toRemove)
 		tb.s.SetChildrenIds(tb.Rows(), rowIds)
 	}
 }

core/block/editor/table/table.go

@@ -290,7 +290,6 @@ func (tb Table) MoveBlocksUnderTheTable(ids ...string) {
 		log.Errorf("failed to find table block '%s' among children of block '%s'", tb.block.Model().Id, parent.Model().Id)
 		return
 	}
-	tb.s.RemoveFromCache(ids)
 	tb.s.SetChildrenIds(parent.Model(), slice.Insert(children, pos+1, ids...))
 }
 

core/block/object/treemanager/treemanager.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/anyproto/any-sync/app"
 	"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
+	"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
 	"github.com/anyproto/any-sync/commonspace/object/treemanager"
 	"go.uber.org/zap"
 
@@ -79,6 +80,16 @@ func (m *treeManager) GetTree(ctx context.Context, spaceId, id string) (tr objec
 	return sb.Tree(), nil
 }
 
+func (m *treeManager) ValidateAndPutTree(ctx context.Context, spaceId string, payload treestorage.TreeStorageCreatePayload) error {
+	// TODO: this should be better done inside cache
+	spc, err := m.spaceService.Get(ctx, spaceId)
+	if err != nil {
+		return err
+	}
+	_, err = spc.TreeBuilder().PutTree(ctx, payload, nil)
+	return err
+}
+
 func (m *treeManager) MarkTreeDeleted(ctx context.Context, spaceId, treeId string) error {
 	err := m.onDelete(domain.FullID{
 		SpaceID:  spaceId,

core/block/object/treesyncer/treesyncer.go

@@ -221,8 +221,8 @@ func (t *treeSyncer) requestTree(peerId, id string) {
 }
 
 func (t *treeSyncer) updateTree(p peer.Peer, id string) {
-	peerId := p.Id()
 	log := log.With(zap.String("treeId", id), zap.String("spaceId", t.spaceId))
+	peerId := p.Id()
 	ctx := peer.CtxWithPeerId(t.mainCtx, peerId)
 	tr, err := t.treeManager.GetTree(ctx, t.spaceId, id)
 	if err != nil {

core/block/service.go

@@ -524,7 +524,13 @@ func (s *Service) SetSource(ctx session.Context, req pb.RpcObjectSetSourceReques
 			return true
 		})
 		st.SetDetailAndBundledRelation(bundle.RelationKeySetOf, pbtypes.StringList(req.Source))
-		return sb.Apply(st, smartblock.NoRestrictions)
+
+		flags := internalflag.NewFromState(st)
+		// set with source is no longer empty
+		flags.Remove(model.InternalFlag_editorDeleteEmpty)
+		flags.AddToState(st)
+
+		return sb.Apply(st, smartblock.NoRestrictions, smartblock.KeepInternalFlags)
 	})
 }