Skip to content

Commit

Permalink
Fixed relevancy ordering for Full-Text
Browse files Browse the repository at this point in the history
  • Loading branch information
@Hydrocharged
Hydrocharged committed Aug 22, 2023
1 parent 967c530 commit 26116f8
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 3 deletions.
1 change: 0 additions & 1 deletion enginetest/evaluation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ func TestScriptWithEngine(t *testing.T, e *sqle.Engine, harness Harness, script
}
}
ctx := NewContext(harness).WithQuery(statement)
//ctx = NewContext(harness).WithQuery(statement)
RunQueryWithContext(t, e, harness, ctx, statement)
}

Expand Down
123 changes: 123 additions & 0 deletions enginetest/queries/fulltext_queries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,129 @@ var FulltextTests = []ScriptTest{
},
},
},
{ // We should not have many relevancy tests since the values are subject to change if/when the algorithm gets updated
Name: "Relevancy Ordering",
SetUpScript: []string{
"CREATE TABLE test (pk INT PRIMARY KEY, doc TEXT, FULLTEXT idx (doc)) COLLATE=utf8mb4_general_ci;",
"INSERT INTO test VALUES (2, 'g hhhh aaaab ooooo aaaa'), (1, 'bbbb ff cccc ddd eee'), (4, 'AAAA aaaa aaaac aaaa Aaaa aaaa'), (3, 'aaaA ff j kkkk llllllll');",
},
Assertions: []ScriptTestAssertion{
{
Query: "SELECT MATCH(doc) AGAINST('aaaa') AS relevance FROM test ORDER BY relevance DESC;",
Expected: []sql.Row{
{float32(5.9636202)},
{float32(4.0278959)},
{float32(3.3721533)},
{float32(0)},
},
},
{
Query: "SELECT MATCH(doc) AGAINST('aaaa') AS relevance, pk FROM test ORDER BY relevance DESC;",
Expected: []sql.Row{
{float32(5.9636202), int32(4)},
{float32(4.0278959), int32(2)},
{float32(3.3721533), int32(3)},
{float32(0), int32(1)},
},
},
{
Query: "SELECT pk, MATCH(doc) AGAINST('aaaa') AS relevance FROM test ORDER BY relevance ASC;",
Expected: []sql.Row{
{int32(1), float32(0)},
{int32(3), float32(3.3721533)},
{int32(2), float32(4.0278959)},
{int32(4), float32(5.9636202)},
},
},
{
Query: "SELECT pk, doc, MATCH(doc) AGAINST('aaaa') AS relevance FROM test ORDER BY relevance DESC;",
Expected: []sql.Row{
{int32(4), "AAAA aaaa aaaac aaaa Aaaa aaaa", float32(5.9636202)},
{int32(2), "g hhhh aaaab ooooo aaaa", float32(4.0278959)},
{int32(3), "aaaA ff j kkkk llllllll", float32(3.3721533)},
{int32(1), "bbbb ff cccc ddd eee", float32(0)},
},
},
{
Query: "SELECT pk, doc, MATCH(doc) AGAINST('aaaa') AS relevance FROM test ORDER BY relevance ASC;",
Expected: []sql.Row{
{int32(1), "bbbb ff cccc ddd eee", float32(0)},
{int32(3), "aaaA ff j kkkk llllllll", float32(3.3721533)},
{int32(2), "g hhhh aaaab ooooo aaaa", float32(4.0278959)},
{int32(4), "AAAA aaaa aaaac aaaa Aaaa aaaa", float32(5.9636202)},
},
},
{
Query: "SELECT pk FROM test ORDER BY MATCH(doc) AGAINST('aaaa') DESC;",
Expected: []sql.Row{
{int32(4)},
{int32(2)},
{int32(3)},
{int32(1)},
},
},
{
Query: "SELECT pk, doc FROM test ORDER BY MATCH(doc) AGAINST('aaaa') ASC;",
Expected: []sql.Row{
{int32(1), "bbbb ff cccc ddd eee"},
{int32(3), "aaaA ff j kkkk llllllll"},
{int32(2), "g hhhh aaaab ooooo aaaa"},
{int32(4), "AAAA aaaa aaaac aaaa Aaaa aaaa"},
},
},
{
Query: "SELECT 1 FROM test ORDER BY MATCH(doc) AGAINST('aaaa') DESC;",
Expected: []sql.Row{
{int32(1)},
{int32(1)},
{int32(1)},
{int32(1)},
},
},
{
Query: "SELECT pk, MATCH(doc) AGAINST('aaaa') AS relevance FROM test HAVING relevance > 4 ORDER BY relevance DESC;",
Expected: []sql.Row{
{int32(4), float32(5.9636202)},
{int32(2), float32(4.0278959)},
},
},
{ // Test with an added column to ensure that unnecessary columns do not affect the results
Query: "ALTER TABLE test ADD COLUMN extracol INT DEFAULT 7;",
Expected: []sql.Row{{types.NewOkResult(0)}},
},
{
Query: "SELECT pk FROM test ORDER BY MATCH(doc) AGAINST('aaaa') DESC;",
Expected: []sql.Row{
{int32(4)},
{int32(2)},
{int32(3)},
{int32(1)},
},
},
{ // Drop the primary key to ensure that results are still consistent without a primary key
Query: "ALTER TABLE test DROP PRIMARY KEY;",
Expected: []sql.Row{{types.NewOkResult(0)}},
},
{
Query: "SELECT pk FROM test ORDER BY MATCH(doc) AGAINST('aaaa') ASC;",
Expected: []sql.Row{
{int32(1)},
{int32(3)},
{int32(2)},
{int32(4)},
},
},
{
Query: "SELECT pk, MATCH(doc) AGAINST('aaaa') AS relevance FROM test ORDER BY relevance DESC;",
Expected: []sql.Row{
{int32(4), float32(5.9636202)},
{int32(2), float32(4.0278959)},
{int32(3), float32(3.3721533)},
{int32(1), float32(0)},
},
},
},
},
{
Name: "CREATE INDEX before insertions",
SetUpScript: []string{
Expand Down
5 changes: 5 additions & 0 deletions sql/expression/matchagainst.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type MatchAgainst struct {
RowCountTable sql.IndexAddressableTable

once sync.Once
expectedRowLen int
evaluatedString string
parser fulltext.DefaultParser
docCountIndex sql.Index
Expand All @@ -69,6 +70,7 @@ func NewMatchAgainst(columns []sql.Expression, expr sql.Expression, searchModifi
DocCountTable: nil,
GlobalCountTable: nil,
RowCountTable: nil,
expectedRowLen: 0,
}
}

Expand All @@ -82,6 +84,7 @@ func (expr *MatchAgainst) Children() []sql.Expression {

// Eval implements sql.Expression
func (expr *MatchAgainst) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
row = row[:expr.expectedRowLen]
switch expr.SearchModifier {
case fulltext.SearchModifier_NaturalLanguage:
return expr.inNaturalLanguageMode(ctx, row)
Expand Down Expand Up @@ -157,6 +160,7 @@ func (expr *MatchAgainst) WithChildren(children ...sql.Expression) (sql.Expressi
DocCountTable: expr.DocCountTable,
GlobalCountTable: expr.GlobalCountTable,
RowCountTable: expr.RowCountTable,
expectedRowLen: expr.expectedRowLen,
}, nil
}

Expand All @@ -174,6 +178,7 @@ func (expr *MatchAgainst) WithInfo(parent, config, position, docCount, globalCou
DocCountTable: docCount,
GlobalCountTable: globalCount,
RowCountTable: rowCount,
expectedRowLen: len(parent.Schema()),
}
}

Expand Down
3 changes: 1 addition & 2 deletions sql/fulltext/fulltext_editor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -545,14 +545,13 @@ func (TableEditor) updateGlobalCount(ctx *sql.Context, ie IndexEditors, word str
}
// Our new count is 1, so we need to create a new entry
return ie.GlobalCount.Editor.Insert(ctx, sql.Row{word, uint64(1)})

} else if len(rows) != 1 {
return fmt.Errorf("somehow there are duplicate entries within the Full-Text global count table")
}
row := rows[0]
oldCount := row[len(row)-1].(uint64)
// First we'll delete the existing row
if err = ie.GlobalCount.Editor.Delete(ctx, sql.Row{word, oldCount}); err != nil {
if err = ie.GlobalCount.Editor.Delete(ctx, row); err != nil {
return err
}
// If we're incrementing, then we can add 1 to the old count
Expand Down

0 comments on commit 26116f8

Please sign in to comment.