GO-4159 get rid of private info in errors from search

core/block/editor/state/state.go

@@ -1088,7 +1088,7 @@ func (s *State) Snippet() string {
 		}
 		return true
 	})
-	return textutil.Truncate(builder.String(), snippetMaxSize)
+	return textutil.TruncateEllipsized(builder.String(), snippetMaxSize)
 }
 
 func (s *State) FileRelationKeys() []string {

core/block/export/export.go

@@ -1066,7 +1066,7 @@ func (fn *namer) Get(path, hash, title, ext string) (name string) {
 		return name
 	}
 	title = slug.Make(strings.TrimSuffix(title, ext))
-	name = text.Truncate(title, fileLenLimit)
+	name = text.TruncateEllipsized(title, fileLenLimit)
 	name = strings.TrimSuffix(name, text.TruncateEllipsis)
 	var (
 		i = 0

pkg/lib/localstore/ftsearch/ftsearchbleve.go

@@ -28,7 +28,7 @@ import (
 const (
 	CName  = "fts"
 	ftsDir = "fts"
-	ftsVer = "5"
+	ftsVer = "6"
 
 	fieldTitle        = "Title"
 	fieldText         = "Text"

pkg/lib/localstore/ftsearch/ftsearchtantivy.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/anyproto/anytype-heart/core/wallet"
 	"github.com/anyproto/anytype-heart/metrics"
+	"github.com/anyproto/anytype-heart/util/text"
 )
 
 func TantivyNew() FTSearch {
@@ -180,12 +181,7 @@ func (f *ftSearchTantivy) Run(context.Context) error {
 		return err
 	}
 
-	err = index.RegisterTextAnalyzerEdgeNgram(tantivy.TokenizerEdgeNgram, 1, 5, 100)
-	if err != nil {
-		return err
-	}
-
-	err = index.RegisterTextAnalyzerNgram(tantivy.TokenizerNgram, 1, 5, false)
+	err = index.RegisterTextAnalyzerNgram(tantivy.TokenizerNgram, 3, 5, false)
 	if err != nil {
 		return err
 	}
@@ -266,14 +262,16 @@ func (f *ftSearchTantivy) BatchIndex(ctx context.Context, docs []SearchDoc, dele
 
 func (f *ftSearchTantivy) Search(spaceIds []string, highlightFormatter HighlightFormatter, query string) (results search.DocumentMatchCollection, err error) {
 	spaceIdsQuery := getSpaceIdsQuery(spaceIds)
-	if spaceIdsQuery == "" {
-		query = escapeQuery(query)
-	} else {
-		query = fmt.Sprintf("%s AND %s", spaceIdsQuery, escapeQuery(query))
+	query = prepareQuery(query)
+	if query == "" {
+		return nil, nil
+	}
+	if spaceIdsQuery != "" {
+		query = fmt.Sprintf("%s AND %s", spaceIdsQuery, query)
 	}
 	result, err := f.index.Search(query, 100, true, fieldId, fieldSpace, fieldTitle, fieldText)
 	if err != nil {
-		return nil, err
+		return nil, wrapError(err)
 	}
 	p := f.parserPool.Get()
 	defer f.parserPool.Put(p)
@@ -309,7 +307,16 @@ func (f *ftSearchTantivy) Search(spaceIds []string, highlightFormatter Highlight
 	)
 }
 
+func wrapError(err error) error {
+	errStr := err.Error()
+	if strings.Contains(errStr, "Syntax Error:") {
+		return fmt.Errorf("invalid query")
+	}
+	return err
+}
+
 func getSpaceIdsQuery(ids []string) string {
+	ids = lo.Filter(ids, func(item string, index int) bool { return item != "" })
 	if len(ids) == 0 || lo.EveryBy(ids, func(id string) bool { return id == "" }) {
 		return ""
 	}
@@ -346,16 +353,21 @@ func (f *ftSearchTantivy) cleanupBleve() {
 	_ = os.RemoveAll(filepath.Join(f.rootPath, ftsDir))
 }
 
-func escapeQuery(query string) string {
+func prepareQuery(query string) string {
+	query = text.Truncate(query, 100, "")
+	query = strings.ToLower(query)
+	query = strings.TrimSpace(query)
 	var escapedQuery strings.Builder
 
 	for _, char := range query {
-		if _, found := specialChars[char]; found {
-			escapedQuery.WriteRune(' ')
+		if _, found := specialChars[char]; !found {
+			escapedQuery.WriteRune(char)
 		}
-		escapedQuery.WriteRune(char)
 	}
 
 	resultQuery := escapedQuery.String()
+	if resultQuery == "" {
+		return resultQuery
+	}
 	return "(\"" + resultQuery + "\" OR " + resultQuery + ")"
 }

pkg/lib/localstore/ftsearch/ftsearchtantivy_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/anyproto/any-sync/app"
@@ -391,3 +392,43 @@ func assertMultiSpace(t *testing.T, tmpDir string) {
 
 	_ = ft.Close(nil)
 }
+
+func TestEscapeQuery(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{strings.Repeat("a", 99) + " aa", `("` + strings.Repeat("a", 99) + `" OR ` + strings.Repeat("a", 99) + `)`},
+		{`""`, ``},
+		{"simpleQuery", `("simplequery" OR simplequery)`},
+		{"with+special^chars", `("withspecialchars" OR withspecialchars)`},
+		{"text`with:brackets{}", `("textwithbrackets" OR textwithbrackets)`},
+		{"escaped[]symbols()", `("escapedsymbols" OR escapedsymbols)`},
+		{"multiple!!special~~", `("multiplespecial" OR multiplespecial)`},
+	}
+
+	for _, test := range tests {
+		actual := prepareQuery(test.input)
+		if actual != test.expected {
+			t.Errorf("For input '%s', expected '%s', but got '%s'", test.input, test.expected, actual)
+		}
+	}
+}
+
+// Tests
+func TestGetSpaceIdsQuery(t *testing.T) {
+	// Test with empty slice of ids
+	assert.Equal(t, "", getSpaceIdsQuery([]string{}))
+
+	// Test with slice containing only empty strings
+	assert.Equal(t, "", getSpaceIdsQuery([]string{"", "", ""}))
+
+	// Test with a single id
+	assert.Equal(t, "(SpaceID:123)", getSpaceIdsQuery([]string{"123"}))
+
+	// Test with multiple ids
+	assert.Equal(t, "(SpaceID:123 OR SpaceID:456 OR SpaceID:789)", getSpaceIdsQuery([]string{"123", "456", "789"}))
+
+	// Test with some empty ids
+	assert.Equal(t, "(SpaceID:123 OR SpaceID:789)", getSpaceIdsQuery([]string{"123", "", "789"}))
+}

util/text/text.go

@@ -7,8 +7,12 @@ import (
 
 const TruncateEllipsis = " …"
 
-func Truncate(text string, length int) string {
-	length = length - UTF16RuneCountString(TruncateEllipsis)
+func TruncateEllipsized(text string, length int) string {
+	return Truncate(text, length, TruncateEllipsis)
+}
+
+func Truncate(text string, length int, ending string) string {
+	length -= UTF16RuneCountString(ending)
 	if UTF16RuneCountString(text) <= length {
 		return text
 	}
@@ -30,7 +34,7 @@ func Truncate(text string, length int) string {
 				endTextPos = lastWordIndex
 			}
 			out := utf16Text[0:endTextPos]
-			return UTF16ToStr(out) + TruncateEllipsis
+			return UTF16ToStr(out) + ending
 		}
 	}
 	return UTF16ToStr(utf16Text)

util/text/text_test.go

@@ -65,7 +65,7 @@ func TestTruncate(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			actual := Truncate(test.text, test.length)
+			actual := TruncateEllipsized(test.text, test.length)
 			assert.Equal(t, test.expected, actual)
 		})
 	}