From a8d5b51790b0346a7cd6567c693b4bf6f4fb403c Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 30 Aug 2024 10:49:26 +0200 Subject: [PATCH 1/2] GO-4000: Fix options filter --- pkg/lib/database/filter.go | 79 ++++++++++++++++++++++++--------- pkg/lib/database/filter_test.go | 36 +++------------ 2 files changed, 64 insertions(+), 51 deletions(-) diff --git a/pkg/lib/database/filter.go b/pkg/lib/database/filter.go index eeefb54b9f..a75ea78959 100644 --- a/pkg/lib/database/filter.go +++ b/pkg/lib/database/filter.go @@ -187,20 +187,13 @@ func makeFilterByCondition(spaceID string, rawFilter *model.BlockContentDataview if err != nil { return nil, ErrValueMustBeListSupporting } - return FilterOptionsEqual{ - Key: rawFilter.RelationKey, - Value: list, - Options: optionsToMap(spaceID, rawFilter.RelationKey, store), - }, nil + return newFilterOptionsEqual(&fastjson.Arena{}, rawFilter.RelationKey, list, optionsToMap(spaceID, rawFilter.RelationKey, store)), nil case model.BlockContentDataviewFilter_NotExactIn: list, err := pbtypes.ValueListWrapper(rawFilter.Value) if err != nil { return nil, ErrValueMustBeListSupporting } - return FilterNot{FilterOptionsEqual{ - Key: rawFilter.RelationKey, - Value: list, - }}, nil + return FilterNot{newFilterOptionsEqual(&fastjson.Arena{}, rawFilter.RelationKey, list, optionsToMap(spaceID, rawFilter.RelationKey, store))}, nil case model.BlockContentDataviewFilter_Exists: return FilterExists{ Key: rawFilter.RelationKey, @@ -604,13 +597,29 @@ func (l FilterAllIn) AnystoreFilter() query.Filter { return query.And(conds) } +func newFilterOptionsEqual(arena *fastjson.Arena, key string, value *types.ListValue, Options map[string]string) *FilterOptionsEqual { + f := &FilterOptionsEqual{ + arena: arena, + Key: key, + Value: value, + Options: Options, + } + f.compileValueFilter() + return f +} + type FilterOptionsEqual struct { + arena *fastjson.Arena + Key string Value *types.ListValue Options map[string]string + + // valueFilter is precompiled filter without key selector + valueFilter query.Filter } -func (exIn FilterOptionsEqual) FilterObject(g *types.Struct) bool { +func (exIn *FilterOptionsEqual) FilterObject(g *types.Struct) bool { val := pbtypes.Get(g, exIn.Key) if val == nil { return false @@ -650,20 +659,48 @@ func (exIn FilterOptionsEqual) FilterObject(g *types.Struct) bool { return true } -func (exIn FilterOptionsEqual) AnystoreFilter() query.Filter { - path := []string{exIn.Key} +func (exIn *FilterOptionsEqual) Ok(v *fastjson.Value) bool { + defer exIn.arena.Reset() + + arr := v.GetArray(exIn.Key) + // Just fall back to precompiled filter + if len(arr) == 0 { + return exIn.valueFilter.Ok(v.Get(exIn.Key)) + } + + // Discard deleted options + optionList := exIn.arena.NewArray() + var i int + for _, arrVal := range arr { + optionId := string(arrVal.GetStringBytes()) + _, ok := exIn.Options[optionId] + if ok { + optionList.SetArrayItem(i, exIn.arena.NewString(optionId)) + i++ + } + } + return exIn.valueFilter.Ok(optionList) +} + +func (exIn *FilterOptionsEqual) compileValueFilter() { conds := make([]query.Filter, 0, len(exIn.Value.GetValues())+1) - conds = append(conds, query.Key{ - Path: path, - Filter: query.Size{Size: int64(len(exIn.Value.GetValues()))}, - }) + conds = append(conds, query.Size{Size: int64(len(exIn.Value.GetValues()))}) for _, v := range exIn.Value.GetValues() { - conds = append(conds, query.Key{ - Path: path, - Filter: query.NewComp(query.CompOpEq, scalarPbValueToAny(v)), - }) + conds = append(conds, query.NewComp(query.CompOpEq, scalarPbValueToAny(v))) } - return query.And(conds) + exIn.valueFilter = query.And(conds) +} + +func (exIn *FilterOptionsEqual) IndexBounds(fieldName string, bs query.Bounds) (bounds query.Bounds) { + return bs +} + +func (exIn *FilterOptionsEqual) AnystoreFilter() query.Filter { + return exIn +} + +func (exIn *FilterOptionsEqual) String() string { + return "{}" } func optionsToMap(spaceID string, key string, store ObjectStore) map[string]string { diff --git a/pkg/lib/database/filter_test.go b/pkg/lib/database/filter_test.go index 02bb204693..93b3496c54 100644 --- a/pkg/lib/database/filter_test.go +++ b/pkg/lib/database/filter_test.go @@ -544,56 +544,32 @@ func TestFilterOptionsEqual(t *testing.T) { "optionId3": "3", } t.Run("one option, ok", func(t *testing.T) { - eq := FilterOptionsEqual{ - Key: "k", - Options: optionIdToName, - Value: pbtypes.StringList([]string{"optionId1"}).GetListValue(), - } + eq := newFilterOptionsEqual(&fastjson.Arena{}, "k", pbtypes.StringList([]string{"optionId1"}).GetListValue(), optionIdToName) obj := &types.Struct{Fields: map[string]*types.Value{"k": pbtypes.StringList([]string{"optionId1"})}} assertFilter(t, eq, obj, true) }) t.Run("two options, ok", func(t *testing.T) { - eq := FilterOptionsEqual{ - Key: "k", - Options: optionIdToName, - Value: pbtypes.StringList([]string{"optionId1", "optionId3"}).GetListValue(), - } + eq := newFilterOptionsEqual(&fastjson.Arena{}, "k", pbtypes.StringList([]string{"optionId1", "optionId3"}).GetListValue(), optionIdToName) obj := &types.Struct{Fields: map[string]*types.Value{"k": pbtypes.StringList([]string{"optionId1", "optionId3"})}} assertFilter(t, eq, obj, true) }) t.Run("two options, ok, not existing options are discarded", func(t *testing.T) { - eq := FilterOptionsEqual{ - Key: "k", - Options: optionIdToName, - Value: pbtypes.StringList([]string{"optionId1", "optionId3"}).GetListValue(), - } + eq := newFilterOptionsEqual(&fastjson.Arena{}, "k", pbtypes.StringList([]string{"optionId1", "optionId3"}).GetListValue(), optionIdToName) obj := &types.Struct{Fields: map[string]*types.Value{"k": pbtypes.StringList([]string{"optionId1", "optionId3", "optionId7000"})}} assertFilter(t, eq, obj, true) }) t.Run("two options, not ok", func(t *testing.T) { - eq := FilterOptionsEqual{ - Key: "k", - Options: optionIdToName, - Value: pbtypes.StringList([]string{"optionId1", "optionId2"}).GetListValue(), - } + eq := newFilterOptionsEqual(&fastjson.Arena{}, "k", pbtypes.StringList([]string{"optionId1", "optionId2"}).GetListValue(), optionIdToName) obj := &types.Struct{Fields: map[string]*types.Value{"k": pbtypes.StringList([]string{"optionId1", "optionId3"})}} assertFilter(t, eq, obj, false) }) t.Run("two options, not ok, because object has 1 option", func(t *testing.T) { - eq := FilterOptionsEqual{ - Key: "k", - Options: optionIdToName, - Value: pbtypes.StringList([]string{"optionId1", "optionId2"}).GetListValue(), - } + eq := newFilterOptionsEqual(&fastjson.Arena{}, "k", pbtypes.StringList([]string{"optionId1", "optionId2"}).GetListValue(), optionIdToName) obj := &types.Struct{Fields: map[string]*types.Value{"k": pbtypes.StringList([]string{"optionId1"})}} assertFilter(t, eq, obj, false) }) t.Run("two options, not ok, because object has 3 options", func(t *testing.T) { - eq := FilterOptionsEqual{ - Key: "k", - Options: optionIdToName, - Value: pbtypes.StringList([]string{"optionId1", "optionId2"}).GetListValue(), - } + eq := newFilterOptionsEqual(&fastjson.Arena{}, "k", pbtypes.StringList([]string{"optionId1", "optionId2"}).GetListValue(), optionIdToName) obj := &types.Struct{Fields: map[string]*types.Value{"k": pbtypes.StringList([]string{"optionId1", "optionId2", "optionId3"})}} assertFilter(t, eq, obj, false) }) From 44cf10d7a0f4e7f05c121893cc832c25d0ef8f32 Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 30 Aug 2024 12:00:46 +0200 Subject: [PATCH 2/2] GO-4000: Fix linter --- pkg/lib/database/filter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/lib/database/filter.go b/pkg/lib/database/filter.go index a75ea78959..bda65e83d5 100644 --- a/pkg/lib/database/filter.go +++ b/pkg/lib/database/filter.go @@ -597,12 +597,12 @@ func (l FilterAllIn) AnystoreFilter() query.Filter { return query.And(conds) } -func newFilterOptionsEqual(arena *fastjson.Arena, key string, value *types.ListValue, Options map[string]string) *FilterOptionsEqual { +func newFilterOptionsEqual(arena *fastjson.Arena, key string, value *types.ListValue, options map[string]string) *FilterOptionsEqual { f := &FilterOptionsEqual{ arena: arena, Key: key, Value: value, - Options: Options, + Options: options, } f.compileValueFilter() return f