diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 226630d4..0aa98fc7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,7 +42,7 @@ jobs: uses: actions/checkout@v4 - name: run unit tests run: | - go test ./test/... -config badger.yml + go test ./test/... -config pebble.yml badgerTest: @@ -217,7 +217,7 @@ jobs: run: | # start grip server chmod +x grip - ./grip server --rpc-port 18202 --http-port 18201 --config ./test/badger-auth.yml & + ./grip server --rpc-port 18202 --http-port 18201 --config ./test/pebble-auth.yml & sleep 5 # simple auth # run tests without credentials, should fail @@ -228,7 +228,7 @@ jobs: echo "Got expected auth error" fi # run specialized role based tests - make test-authorization ARGS="--grip_config_file_path test/badger-auth.yml" + make test-authorization ARGS="--grip_config_file_path test/pebble-auth.yml" diff --git a/config/config.go b/config/config.go index db684f29..4ab86693 100644 --- a/config/config.go +++ b/config/config.go @@ -129,6 +129,10 @@ func TestifyConfig(c *Config) { a := "grip.db." + rand d.Badger = &a } + if d.Pebble != nil { + a := "grip.db." + rand + d.Pebble = &a + } if d.MongoDB != nil { d.MongoDB.DBName = "gripdb-" + rand } diff --git a/engine/core/processors_has.go b/engine/core/processors_has.go index 756da192..ba1f25ab 100644 --- a/engine/core/processors_has.go +++ b/engine/core/processors_has.go @@ -6,6 +6,7 @@ import ( "github.com/bmeg/grip/engine/logic" "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/util/setcmp" ) //////////////////////////////////////////////////////////////////////////////// @@ -50,7 +51,7 @@ func (h *HasLabel) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe out <- t continue } - if contains(labels, t.GetCurrent().Get().Label) { + if setcmp.ContainsString(labels, t.GetCurrent().Get().Label) { out <- t } } @@ -106,7 +107,7 @@ func (h *HasID) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, o out <- t continue } - if contains(ids, t.GetCurrentID()) { + if setcmp.ContainsString(ids, t.GetCurrentID()) { out <- t } } diff --git a/engine/core/util.go b/engine/core/util.go index 4ec66f79..2cc616ef 100644 --- a/engine/core/util.go +++ b/engine/core/util.go @@ -8,15 +8,6 @@ func debug(i ...interface{}) { pretty.Println(i...) } -func contains(a []string, v string) bool { - for _, i := range a { - if i == v { - return true - } - } - return false -} - func dedupStringSlice(s []string) []string { seen := make(map[string]struct{}, len(s)) j := 0 diff --git a/gripql/inspect/inspect.go b/gripql/inspect/inspect.go index e4355601..da4911d9 100644 --- a/gripql/inspect/inspect.go +++ b/gripql/inspect/inspect.go @@ -22,15 +22,6 @@ func arrayEq(a, b []string) bool { return true } -func contains(a []string, n string) bool { - for _, c := range a { - if c == n { - return true - } - } - return false -} - // PipelineSteps create an array, the same length at stmts that labels the // step id for each of the GraphStatements func PipelineSteps(stmts []*gripql.GraphStatement) []string { diff --git a/kvgraph/graph.go b/kvgraph/graph.go index 8739f2d3..e18a89bd 100644 --- a/kvgraph/graph.go +++ b/kvgraph/graph.go @@ -11,20 +11,12 @@ import ( "github.com/bmeg/grip/kvi" "github.com/bmeg/grip/kvindex" "github.com/bmeg/grip/log" + "github.com/bmeg/grip/util/setcmp" "google.golang.org/protobuf/proto" multierror "github.com/hashicorp/go-multierror" ) -func contains(a []string, v string) bool { - for _, i := range a { - if i == v { - return true - } - } - return false -} - // GetTimestamp returns the update timestamp func (kgdb *KVInterfaceGDB) GetTimestamp() string { return kgdb.kvg.ts.Get(kgdb.graph) @@ -199,6 +191,10 @@ func (kgdb *KVInterfaceGDB) DelEdge(eid string) error { if err := kgdb.kvg.kv.Delete(dkey); err != nil { return err } + if err := kgdb.kvg.idx.RemoveDoc(eid); err != nil { + return err + } + kgdb.kvg.ts.Touch(kgdb.graph) return nil } @@ -235,11 +231,17 @@ func (kgdb *KVInterfaceGDB) DelVertex(id string) error { if err := tx.Delete(vid); err != nil { return err } + for _, k := range delKeys { if err := tx.Delete(k); err != nil { return err } } + + if err := kgdb.kvg.idx.RemoveDoc(kvindex.FieldKeyParse(vid)); err != nil { + return err + } + kgdb.kvg.ts.Touch(kgdb.graph) return nil }) @@ -378,7 +380,7 @@ func (kgdb *KVInterfaceGDB) GetOutChannel(ctx context.Context, reqChan chan gdbi for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { keyValue := it.Key() _, _, dst, _, label, etype := SrcEdgeKeyParse(keyValue) - if len(edgeLabels) == 0 || contains(edgeLabels, label) { + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { vkey := VertexKey(kgdb.graph, dst) if etype == edgeSingle { vertexChan <- elementData{ @@ -456,7 +458,7 @@ func (kgdb *KVInterfaceGDB) GetInChannel(ctx context.Context, reqChan chan gdbi. for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { keyValue := it.Key() _, src, _, _, label, _ := DstEdgeKeyParse(keyValue) - if len(edgeLabels) == 0 || contains(edgeLabels, label) { + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { vkey := VertexKey(kgdb.graph, src) dataValue, err := it.Get(vkey) if err == nil { @@ -506,7 +508,7 @@ func (kgdb *KVInterfaceGDB) GetOutEdgeChannel(ctx context.Context, reqChan chan for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { keyValue := it.Key() _, src, dst, eid, label, edgeType := SrcEdgeKeyParse(keyValue) - if len(edgeLabels) == 0 || contains(edgeLabels, label) { + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { if edgeType == edgeSingle { e := gdbi.Edge{} if load { @@ -563,7 +565,7 @@ func (kgdb *KVInterfaceGDB) GetInEdgeChannel(ctx context.Context, reqChan chan g for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { keyValue := it.Key() _, src, dst, eid, label, edgeType := DstEdgeKeyParse(keyValue) - if len(edgeLabels) == 0 || contains(edgeLabels, label) { + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { if edgeType == edgeSingle { e := gdbi.Edge{} if load { diff --git a/kvgraph/test/index_test.go b/kvgraph/test/index_test.go index 0062f0e8..92d2a4aa 100644 --- a/kvgraph/test/index_test.go +++ b/kvgraph/test/index_test.go @@ -7,6 +7,7 @@ import ( "context" "github.com/bmeg/grip/kvindex" + "github.com/bmeg/grip/util/setcmp" ) var docs = `[ @@ -61,7 +62,7 @@ func TestFieldListing(t *testing.T) { count := 0 for _, field := range idx.ListFields() { - if !contains(newFields, field) { + if !setcmp.ContainsString(newFields, field) { t.Errorf("Bad field return: %s", field) } count++ @@ -89,7 +90,7 @@ func TestLoadDoc(t *testing.T) { count := 0 for d := range idx.GetTermMatch(context.Background(), "v.label", "Person", -1) { - if !contains(personDocs, d) { + if !setcmp.ContainsString(personDocs, d) { t.Errorf("Bad doc return: %s", d) } count++ @@ -100,7 +101,7 @@ func TestLoadDoc(t *testing.T) { count = 0 for d := range idx.GetTermMatch(context.Background(), "v.data.firstName", "Bob", -1) { - if !contains(bobDocs, d) { + if !setcmp.ContainsString(bobDocs, d) { t.Errorf("Bad doc return: %s", d) } count++ @@ -128,7 +129,7 @@ func TestTermEnum(t *testing.T) { count := 0 for d := range idx.FieldTerms("v.data.lastName") { count++ - if !contains(lastNames, d.(string)) { + if !setcmp.ContainsString(lastNames, d.(string)) { t.Errorf("Bad term return: %s", d) } } @@ -139,7 +140,7 @@ func TestTermEnum(t *testing.T) { count = 0 for d := range idx.FieldTerms("v.data.firstName") { count++ - if !contains(firstNames, d.(string)) { + if !setcmp.ContainsString(firstNames, d.(string)) { t.Errorf("Bad term return: %s", d) } } @@ -166,7 +167,7 @@ func TestTermCount(t *testing.T) { count := 0 for d := range idx.FieldStringTermCounts("v.data.lastName") { count++ - if !contains(lastNames, d.String) { + if !setcmp.ContainsString(lastNames, d.String) { t.Errorf("Bad term return: %s", d.String) } if d.String == "Smith" { @@ -182,7 +183,7 @@ func TestTermCount(t *testing.T) { count = 0 for d := range idx.FieldTermCounts("v.data.firstName") { count++ - if !contains(firstNames, d.String) { + if !setcmp.ContainsString(firstNames, d.String) { t.Errorf("Bad term return: %s", d.String) } } @@ -212,7 +213,7 @@ func TestDocDelete(t *testing.T) { count := 0 for d := range idx.FieldStringTermCounts("v.data.lastName") { count++ - if !contains(lastNames, d.String) { + if !setcmp.ContainsString(lastNames, d.String) { t.Errorf("Bad term return: %s", d.String) } if d.String == "Smith" { @@ -233,7 +234,7 @@ func TestDocDelete(t *testing.T) { count = 0 for d := range idx.FieldStringTermCounts("v.data.lastName") { count++ - if !contains(lastNames, d.String) { + if !setcmp.ContainsString(lastNames, d.String) { t.Errorf("Bad term return: %s", d.String) } if d.String == "Smith" { diff --git a/kvgraph/test/main_test.go b/kvgraph/test/main_test.go index 3cd61715..01f56fda 100644 --- a/kvgraph/test/main_test.go +++ b/kvgraph/test/main_test.go @@ -28,15 +28,6 @@ func resetKVInterface() { } } -func contains(a []string, v string) bool { - for _, i := range a { - if i == v { - return true - } - } - return false -} - func TestMain(m *testing.M) { var err error var exit = 1 diff --git a/kvi/pebbledb/pebble_store.go b/kvi/pebbledb/pebble_store.go index b215183b..d239c741 100644 --- a/kvi/pebbledb/pebble_store.go +++ b/kvi/pebbledb/pebble_store.go @@ -69,27 +69,8 @@ func (pdb *PebbleKV) Delete(id []byte) error { // DeletePrefix deletes all elements in kvstore that begin with prefix `id` func (pdb *PebbleKV) DeletePrefix(prefix []byte) error { - deleteBlockSize := 10000 - for found := true; found; { - found = false - wb := make([][]byte, 0, deleteBlockSize) - it, err := pdb.db.NewIter(&pebble.IterOptions{LowerBound: prefix}) - if err != nil { - return err - } - for ; it.Valid() && bytes.HasPrefix(it.Key(), prefix) && len(wb) < deleteBlockSize-1; it.Next() { - wb = append(wb, copyBytes(it.Key())) - } - it.Close() - for _, i := range wb { - err := pdb.db.Delete(i, nil) - if err != nil { - return err - } - found = true - } - } - return nil + nextPrefix := append(prefix, 0xFF) + return pdb.db.DeleteRange(prefix, nextPrefix, nil) } // HasKey returns true if the key is exists in kvstore diff --git a/kvindex/entries.go b/kvindex/entries.go index 5859f185..4636c332 100644 --- a/kvindex/entries.go +++ b/kvindex/entries.go @@ -4,6 +4,8 @@ import ( "encoding/binary" "fmt" "math" + + "github.com/bmeg/grip/util/setcmp" ) // TermType defines in a term is a Number or a String @@ -37,18 +39,19 @@ func fieldScan(docID string, doc map[string]interface{}, fieldPrefix string, fie if containsPrefix(f, fields) { if x, ok := v.(map[string]interface{}); ok { fieldScan(docID, x, fmt.Sprintf("%s.%s", fieldPrefix, k), fields, out) - } else if contains(f, fields) { + } else if setcmp.ContainsString(fields, f) { out <- newEntry(docID, f, v) } } } } -func mapDig(i map[string]interface{}, path []string) interface{} { +// Given a list of fields (graphs), return term (label) of doc (graph element) if it exists on field +func getTermOnField(i map[string]interface{}, path []string) interface{} { if x, ok := i[path[0]]; ok { if len(path) > 1 { if y, ok := x.(map[string]interface{}); ok { - return mapDig(y, path[1:]) + return getTermOnField(y, path[1:]) } } else { return x diff --git a/kvindex/keys.go b/kvindex/keys.go index a0dc5942..a9155735 100644 --- a/kvindex/keys.go +++ b/kvindex/keys.go @@ -6,21 +6,25 @@ import ( // Fields // key: f | field +// Known as 'graph' in grip // val: var idxFieldPrefix = []byte("f") // Terms -// key: t | field | TermType | term +// key: t | field | TermType' +// Known as 'label' in grip // val: count var idxTermPrefix = []byte("t") // Entries // key: i | field | TermType | term | docid +// What links the graph + label to the doc via an id // val: var idxEntryPrefix = []byte("i") // Docs // key: d | docid +// Known as 'data' in grip // val: Doc entry list var idxDocPrefix = []byte("D") @@ -75,7 +79,7 @@ func EntryPrefix(field string) []byte { return bytes.Join([][]byte{idxEntryPrefix, []byte(field), {}}, []byte{0}) } -// EntryTypePrefix get prefix for all entries for a single field +// EntryTypePrefix get prefix for all entries for a single field an a single type func EntryTypePrefix(field string, ttype TermType) []byte { return bytes.Join([][]byte{idxEntryPrefix, []byte(field), {byte(ttype)}, {}}, []byte{0}) } diff --git a/kvindex/kvindex.go b/kvindex/kvindex.go index 35762f49..6bcdf63d 100644 --- a/kvindex/kvindex.go +++ b/kvindex/kvindex.go @@ -16,15 +16,6 @@ import ( const bufferSize = 1000 -func contains(c string, s []string) bool { - for _, i := range s { - if c == i { - return true - } - } - return false -} - func containsPrefix(c string, s []string) bool { for _, i := range s { if strings.HasPrefix(i, c) { @@ -64,8 +55,12 @@ func (idx *KVIndex) RemoveField(path string) error { fk := FieldKey(path) fkt := TermPrefix(path) ed := EntryPrefix(path) + dk := DocKey(path) + idx.KV.DeletePrefix(fkt) idx.KV.DeletePrefix(ed) + idx.KV.DeletePrefix(dk) + delete(idx.Fields, path) return idx.KV.Delete(fk) } @@ -101,19 +96,19 @@ func (idx *KVIndex) AddDocTx(tx kvi.KVBulkWrite, docID string, doc map[string]in docKey := DocKey(docID) for field, p := range idx.Fields { - x := mapDig(doc, p) - if x != nil { - term, t := GetTermBytes(x) + term := getTermOnField(doc, p) + if term != nil { + termBytes, t := GetTermBytes(term) switch t { case TermString, TermNumber: - entryKey := EntryKey(field, t, term, docID) + entryKey := EntryKey(field, t, termBytes, docID) err := tx.Set(entryKey, []byte{}) if err != nil { return fmt.Errorf("failed to set entry key %s: %v", entryKey, err) } sdoc.Entries = append(sdoc.Entries, entryKey) - termKey := TermKey(field, t, term) + termKey := TermKey(field, t, termBytes) //set the term count to 0 to invalidate it. Later on, if other code trying //to get the term count will have to recount //previously, it was a set(get+1), but for bulk loading, its better diff --git a/test/main_test.go b/test/main_test.go index 36a71899..e1673a0b 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -17,6 +17,8 @@ import ( _ "github.com/bmeg/grip/kvi/badgerdb" // import so badger will register itself _ "github.com/bmeg/grip/kvi/boltdb" // import so bolt will register itself _ "github.com/bmeg/grip/kvi/leveldb" // import so level will register itself + _ "github.com/bmeg/grip/kvi/pebbledb" // import so pebble will register itself + "github.com/bmeg/grip/mongo" "github.com/bmeg/grip/psql" "github.com/bmeg/grip/util" @@ -93,7 +95,7 @@ func TestMain(m *testing.M) { return } } else { - conf.AddBadgerDefault() + conf.AddPebbleDefault() } config.TestifyConfig(conf) @@ -116,6 +118,11 @@ func TestMain(m *testing.M) { defer func() { os.RemoveAll(*dbconfig.Badger) }() + } else if dbconfig.Pebble != nil { + gdb, err = kvgraph.NewKVGraphDB("pebble", *dbconfig.Pebble) + defer func() { + os.RemoveAll(*dbconfig.Pebble) + }() } else if dbconfig.Bolt != nil { gdb, err = kvgraph.NewKVGraphDB("bolt", *dbconfig.Bolt) defer func() { @@ -159,6 +166,33 @@ func TestMain(m *testing.M) { } } + // After deleting graph, docs, entries, fields should no longer exist in doc + err = gdb.DeleteGraph("test-graph") + err = gdb.AddGraph("test-graph") + if err != nil { + fmt.Println("Error: failed to add graph:", err) + return + } + db, err = gdb.Graph("test-graph") + if err != nil { + fmt.Println("Error: failed to connect to graph:", err) + return + } + + afterVertexLabels, _ := db.ListVertexLabels() + afterEdgeLabels, _ := db.ListEdgeLabels() + fmt.Printf("afterEdgeLabels: %s afterVertexLabels: %s\n", afterEdgeLabels, afterVertexLabels) + if len(afterVertexLabels) != 0 || len(afterEdgeLabels) != 0 { + panic(fmt.Errorf("afterEdgeLabels: %s or afterVertexLabels: %s are not empty\n", afterEdgeLabels, afterVertexLabels)) + } + + if dbname != "existing-sql" { + err = setupGraph() + if err != nil { + fmt.Println("Error: setting up graph:", err) + return + } + } // run tests exit = m.Run() } diff --git a/test/badger-auth.yml b/test/pebble-auth.yml similarity index 85% rename from test/badger-auth.yml rename to test/pebble-auth.yml index 560931e1..f7c9757b 100644 --- a/test/badger-auth.yml +++ b/test/pebble-auth.yml @@ -1,8 +1,8 @@ -Default: badger +Default: pebble Drivers: - badger: - Badger: grip-badger.db + pebble: + Pebble: grip-pebble.db Server: Accounts: diff --git a/test/badger-proxy-auth.yml b/test/pebble-proxy-auth.yml similarity index 80% rename from test/badger-proxy-auth.yml rename to test/pebble-proxy-auth.yml index 5862c84b..8b8c0403 100644 --- a/test/badger-proxy-auth.yml +++ b/test/pebble-proxy-auth.yml @@ -1,8 +1,8 @@ -Default: badger +Default: pebble Drivers: - badger: - Badger: grip-badger.db + pebble: + Pebble: grip-pebble.db Server: Accounts: