From 35cd2c9fa49a8951116b4542643025f31ad99ad8 Mon Sep 17 00:00:00 2001 From: Daylon Wilkins Date: Tue, 15 Aug 2023 07:04:20 -0700 Subject: [PATCH] Even more Full-Text fixes --- enginetest/queries/fulltext_queries.go | 204 ++++++++++++++++++++++++- sql/analyzer/apply_foreign_keys.go | 4 +- sql/analyzer/match_against.go | 3 + sql/expression/matchagainst.go | 33 +++- sql/fulltext/fulltext.go | 98 ++++++++++++ sql/plan/alter_check.go | 6 +- sql/plan/alter_foreign_key.go | 12 +- sql/plan/create_index.go | 2 +- sql/plan/ddl.go | 2 +- sql/plan/delete.go | 6 +- sql/plan/update.go | 12 +- sql/rowexec/ddl_iters.go | 9 +- sql/rowexec/dml.go | 6 + 13 files changed, 360 insertions(+), 37 deletions(-) diff --git a/enginetest/queries/fulltext_queries.go b/enginetest/queries/fulltext_queries.go index cd502d8739..2ef0a433c8 100644 --- a/enginetest/queries/fulltext_queries.go +++ b/enginetest/queries/fulltext_queries.go @@ -16,6 +16,7 @@ package queries import ( "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/types" ) @@ -245,6 +246,106 @@ var FulltextTests = []ScriptTest{ }, }, }, + { + Name: "Basic UPDATE and DELETE checks", + SetUpScript: []string{ + "CREATE TABLE test (pk BIGINT UNSIGNED PRIMARY KEY, v1 VARCHAR(200), v2 VARCHAR(200), FULLTEXT idx (v1, v2));", + "INSERT INTO test VALUES (1, 'abc', 'def pqr'), (2, 'ghi', 'jkl'), (3, 'mno', 'mno'), (4, 'stu vwx', 'xyz zyx yzx'), (5, 'ghs', 'mno shg');", + }, + Assertions: []ScriptTestAssertion{ + { + Query: "SELECT * FROM test WHERE MATCH(v1, v2) AGAINST ('ghi');", + Expected: []sql.Row{{uint64(2), "ghi", "jkl"}}, + }, + { + Query: "UPDATE test SET v1 = 'rgb' WHERE pk = 2;", + Expected: []sql.Row{{types.OkResult{RowsAffected: 1, Info: plan.UpdateInfo{Matched: 1, Updated: 1}}}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v1, v2) AGAINST ('ghi');", + Expected: []sql.Row{}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v1, v2) AGAINST ('rgb');", + Expected: []sql.Row{{uint64(2), "rgb", "jkl"}}, + }, + { + Query: "UPDATE test SET v2 = 'mno' WHERE pk = 2;", + Expected: []sql.Row{{types.OkResult{RowsAffected: 1, Info: plan.UpdateInfo{Matched: 1, Updated: 1}}}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v1, v2) AGAINST ('mno');", + Expected: []sql.Row{{uint64(2), "rgb", "mno"}, {uint64(3), "mno", "mno"}, {uint64(5), "ghs", "mno shg"}}, + }, + { + Query: "DELETE FROM test WHERE pk = 3;", + Expected: []sql.Row{{types.NewOkResult(1)}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v1, v2) AGAINST ('mno');", + Expected: []sql.Row{{uint64(2), "rgb", "mno"}, {uint64(5), "ghs", "mno shg"}}, + }, + }, + }, + { + Name: "Collation handling", + SetUpScript: []string{ + "CREATE TABLE test1 (pk BIGINT UNSIGNED PRIMARY KEY, v1 VARCHAR(200) COLLATE utf8mb4_0900_bin, v2 VARCHAR(200) COLLATE utf8mb4_0900_bin, FULLTEXT idx (v1, v2));", + "CREATE TABLE test2 (pk BIGINT UNSIGNED PRIMARY KEY, v1 VARCHAR(200) COLLATE utf8mb4_0900_ai_ci, v2 VARCHAR(200) COLLATE utf8mb4_0900_ai_ci, FULLTEXT idx (v1, v2));", + "INSERT INTO test1 VALUES (1, 'abc', 'def pqr'), (2, 'ghi', 'jkl'), (3, 'mno', 'mno'), (4, 'stu vwx', 'xyz zyx yzx'), (5, 'ghs', 'mno shg');", + "INSERT INTO test2 VALUES (1, 'abc', 'def pqr'), (2, 'ghi', 'jkl'), (3, 'mno', 'mno'), (4, 'stu vwx', 'xyz zyx yzx'), (5, 'ghs', 'mno shg');", + }, + Assertions: []ScriptTestAssertion{ + { + Query: "SELECT * FROM test1 WHERE MATCH(v1, v2) AGAINST ('ghi');", + Expected: []sql.Row{{uint64(2), "ghi", "jkl"}}, + }, + { + Query: "SELECT * FROM test1 WHERE MATCH(v2, v1) AGAINST ('jkl') = 0;", + Expected: []sql.Row{{uint64(1), "abc", "def pqr"}, {uint64(3), "mno", "mno"}, {uint64(4), "stu vwx", "xyz zyx yzx"}, {uint64(5), "ghs", "mno shg"}}, + }, + { + Query: "SELECT * FROM test1 WHERE MATCH(v2, v1) AGAINST ('jkl mno') AND pk = 3;", + Expected: []sql.Row{{uint64(3), "mno", "mno"}}, + }, + { + Query: "SELECT * FROM test1 WHERE MATCH(v1, v2) AGAINST ('GHI');", + Expected: []sql.Row{}, + }, + { + Query: "SELECT * FROM test1 WHERE MATCH(v2, v1) AGAINST ('JKL') = 0;", + Expected: []sql.Row{{uint64(1), "abc", "def pqr"}, {uint64(2), "ghi", "jkl"}, {uint64(3), "mno", "mno"}, {uint64(4), "stu vwx", "xyz zyx yzx"}, {uint64(5), "ghs", "mno shg"}}, + }, + { + Query: "SELECT * FROM test1 WHERE MATCH(v2, v1) AGAINST ('JKL MNO') AND pk = 3;", + Expected: []sql.Row{}, + }, + { + Query: "SELECT * FROM test2 WHERE MATCH(v1, v2) AGAINST ('ghi');", + Expected: []sql.Row{{uint64(2), "ghi", "jkl"}}, + }, + { + Query: "SELECT * FROM test2 WHERE MATCH(v2, v1) AGAINST ('jkl') = 0;", + Expected: []sql.Row{{uint64(1), "abc", "def pqr"}, {uint64(3), "mno", "mno"}, {uint64(4), "stu vwx", "xyz zyx yzx"}, {uint64(5), "ghs", "mno shg"}}, + }, + { + Query: "SELECT * FROM test2 WHERE MATCH(v2, v1) AGAINST ('jkl mno') AND pk = 3;", + Expected: []sql.Row{{uint64(3), "mno", "mno"}}, + }, + { + Query: "SELECT * FROM test2 WHERE MATCH(v1, v2) AGAINST ('GHI');", + Expected: []sql.Row{{uint64(2), "ghi", "jkl"}}, + }, + { + Query: "SELECT * FROM test2 WHERE MATCH(v2, v1) AGAINST ('JKL') = 0;", + Expected: []sql.Row{{uint64(1), "abc", "def pqr"}, {uint64(3), "mno", "mno"}, {uint64(4), "stu vwx", "xyz zyx yzx"}, {uint64(5), "ghs", "mno shg"}}, + }, + { + Query: "SELECT * FROM test2 WHERE MATCH(v2, v1) AGAINST ('JKL MNO') AND pk = 3;", + Expected: []sql.Row{{uint64(3), "mno", "mno"}}, + }, + }, + }, { Name: "CREATE INDEX before insertions", SetUpScript: []string{ @@ -444,13 +545,69 @@ var FulltextTests = []ScriptTest{ { Name: "ALTER TABLE DROP COLUMN used by index", SetUpScript: []string{ - "CREATE TABLE test (pk BIGINT UNSIGNED PRIMARY KEY, v1 VARCHAR(200), v2 VARCHAR(200), FULLTEXT idx (v1, v2));", - "INSERT INTO test VALUES (1, 'abc', 'def pqr'), (2, 'ghi', 'jkl'), (3, 'mno', 'mno'), (4, 'stu vwx', 'xyz zyx yzx'), (5, 'ghs', 'mno shg');", + "CREATE TABLE test (pk BIGINT UNSIGNED PRIMARY KEY, v1 VARCHAR(200), v2 VARCHAR(200), v3 VARCHAR(200), FULLTEXT idx1 (v1, v2), FULLTEXT idx2 (v2), FULLTEXT idx3 (v2, v3));", + "INSERT INTO test VALUES (1, 'abc', 'def', 'ghi');", }, Assertions: []ScriptTestAssertion{ { - Query: "ALTER TABLE test DROP COLUMN v2;", - ExpectedErr: sql.ErrFullTextMissingColumn, + Query: "SELECT * FROM test WHERE MATCH(v1, v2) AGAINST ('abc');", + Expected: []sql.Row{{uint64(1), "abc", "def", "ghi"}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v2) AGAINST ('def');", + Expected: []sql.Row{{uint64(1), "abc", "def", "ghi"}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v2, v3) AGAINST ('ghi');", + Expected: []sql.Row{{uint64(1), "abc", "def", "ghi"}}, + }, + { + Query: "SHOW CREATE TABLE test;", + Expected: []sql.Row{{"test", "CREATE TABLE `test` (\n `pk` bigint unsigned NOT NULL,\n `v1` varchar(200),\n `v2` varchar(200),\n `v3` varchar(200),\n PRIMARY KEY (`pk`),\n FULLTEXT KEY `idx1` (`v1`,`v2`),\n FULLTEXT KEY `idx2` (`v2`),\n FULLTEXT KEY `idx3` (`v2`,`v3`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "ALTER TABLE test DROP COLUMN v2;", + Expected: []sql.Row{{types.NewOkResult(0)}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v1, v2) AGAINST ('abc');", + ExpectedErr: sql.ErrColumnNotFound, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v2) AGAINST ('def');", + ExpectedErr: sql.ErrColumnNotFound, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v2, v3) AGAINST ('ghi');", + ExpectedErr: sql.ErrColumnNotFound, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v1) AGAINST ('abc');", + Expected: []sql.Row{{uint64(1), "abc", "ghi"}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v3) AGAINST ('ghi');", + Expected: []sql.Row{{uint64(1), "abc", "ghi"}}, + }, + { + Query: "SHOW CREATE TABLE test;", + Expected: []sql.Row{{"test", "CREATE TABLE `test` (\n `pk` bigint unsigned NOT NULL,\n `v1` varchar(200),\n `v3` varchar(200),\n PRIMARY KEY (`pk`),\n FULLTEXT KEY `idx1` (`v1`),\n FULLTEXT KEY `idx3` (`v3`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "ALTER TABLE test DROP COLUMN v3;", + Expected: []sql.Row{{types.NewOkResult(0)}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v1) AGAINST ('abc');", + Expected: []sql.Row{{uint64(1), "abc"}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v3) AGAINST ('ghi');", + ExpectedErr: sql.ErrColumnNotFound, + }, + { + Query: "SHOW CREATE TABLE test;", + Expected: []sql.Row{{"test", "CREATE TABLE `test` (\n `pk` bigint unsigned NOT NULL,\n `v1` varchar(200),\n PRIMARY KEY (`pk`),\n FULLTEXT KEY `idx1` (`v1`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, }, }, }, @@ -489,7 +646,7 @@ var FulltextTests = []ScriptTest{ }, }, { - Name: "ALTER TABLE DROP PRIMARY KEY", + Name: "ALTER TABLE DROP TABLE", SetUpScript: []string{ "CREATE TABLE test (pk BIGINT UNSIGNED PRIMARY KEY, v1 VARCHAR(200), v2 VARCHAR(200), FULLTEXT idx (v1, v2));", "INSERT INTO test VALUES (1, 'abc', 'def pqr'), (2, 'ghi', 'jkl'), (3, 'mno', 'mno'), (4, 'stu vwx', 'xyz zyx yzx'), (5, 'ghs', 'mno shg');", @@ -501,6 +658,27 @@ var FulltextTests = []ScriptTest{ }, }, }, + { + Name: "TRUNCATE TABLE", + SetUpScript: []string{ + "CREATE TABLE test (pk BIGINT UNSIGNED PRIMARY KEY, v1 VARCHAR(200), v2 VARCHAR(200), FULLTEXT idx (v1, v2));", + "INSERT INTO test VALUES (1, 'abc', 'def pqr'), (2, 'ghi', 'jkl'), (3, 'mno', 'mno'), (4, 'stu vwx', 'xyz zyx yzx'), (5, 'ghs', 'mno shg');", + }, + Assertions: []ScriptTestAssertion{ + { + Query: "SELECT * FROM test WHERE MATCH(v1, v2) AGAINST ('ghi');", + Expected: []sql.Row{{uint64(2), "ghi", "jkl"}}, + }, + { + Query: "TRUNCATE TABLE test;", + Expected: []sql.Row{{types.NewOkResult(5)}}, + }, + { + Query: "SELECT * FROM test WHERE MATCH(v1, v2) AGAINST ('ghi');", + Expected: []sql.Row{}, + }, + }, + }, { Name: "No prefix needed for TEXT columns", Assertions: []ScriptTestAssertion{ @@ -648,4 +826,20 @@ var FulltextTests = []ScriptTest{ }, }, }, + { + Name: "Foreign keys ignore Full-Text indexes", + SetUpScript: []string{ + "CREATE TABLE parent (pk BIGINT, v1 VARCHAR(200), FULLTEXT idx (v1));", + }, + Assertions: []ScriptTestAssertion{ + { + Query: "CREATE TABLE child1 (pk BIGINT, v1 VARCHAR(200), FULLTEXT idx (v1), CONSTRAINT fk FOREIGN KEY (v1) REFERENCES parent(v1));", + ExpectedErr: sql.ErrForeignKeyMissingReferenceIndex, + }, + { + Query: "CREATE TABLE child2 (pk BIGINT, v1 VARCHAR(200), INDEX idx (v1), CONSTRAINT fk FOREIGN KEY (v1) REFERENCES parent(v1));", + ExpectedErr: sql.ErrForeignKeyMissingReferenceIndex, + }, + }, + }, } diff --git a/sql/analyzer/apply_foreign_keys.go b/sql/analyzer/apply_foreign_keys.go index 10f8bbd267..20691c2796 100644 --- a/sql/analyzer/apply_foreign_keys.go +++ b/sql/analyzer/apply_foreign_keys.go @@ -263,7 +263,7 @@ func getForeignKeyReferences(ctx *sql.Context, a *Analyzer, tbl sql.ForeignKeyTa } } - parentIndex, ok, err := plan.FindIndexWithPrefix(ctx, parentTbl, fk.ParentColumns, true) + parentIndex, ok, err := plan.FindFKIndexWithPrefix(ctx, parentTbl, fk.ParentColumns, true) if err != nil { return nil, err } @@ -363,7 +363,7 @@ func getForeignKeyRefActions(ctx *sql.Context, a *Analyzer, tbl sql.ForeignKeyTa } } - childIndex, ok, err := plan.FindIndexWithPrefix(ctx, childTbl, fk.Columns, false) + childIndex, ok, err := plan.FindFKIndexWithPrefix(ctx, childTbl, fk.Columns, false) if err != nil { return nil, err } diff --git a/sql/analyzer/match_against.go b/sql/analyzer/match_against.go index f115de18a9..957367768d 100644 --- a/sql/analyzer/match_against.go +++ b/sql/analyzer/match_against.go @@ -59,6 +59,9 @@ func processMatchAgainst(ctx *sql.Context, matchAgainstExpr *expression.MatchAga if !ok { return nil, transform.NewTree, fmt.Errorf("cannot use MATCH ... AGAINST ... on a table that does not declare indexes") } + if _, ok = indexedTbl.(sql.StatisticsTable); !ok { + return nil, transform.NewTree, fmt.Errorf("cannot use MATCH ... AGAINST ... on a table that does not implement sql.StatisticsTable") + } // Verify the indexes that have been set ftIndex := matchAgainstExpr.GetIndex() diff --git a/sql/expression/matchagainst.go b/sql/expression/matchagainst.go index fdb90bcc01..2b4609bcf2 100644 --- a/sql/expression/matchagainst.go +++ b/sql/expression/matchagainst.go @@ -16,6 +16,7 @@ package expression import ( "fmt" + "math" "strings" "sync" @@ -49,6 +50,7 @@ type MatchAgainst struct { docCountIndex sql.Index globalCountIndex sql.Index rowCountIndex sql.Index + parentRowCount uint64 } var _ sql.Expression = (*MatchAgainst)(nil) @@ -267,6 +269,12 @@ func (expr *MatchAgainst) inNaturalLanguageMode(ctx *sql.Context, row sql.Row) ( err = nErr return } + // Load the number of rows from the parent table, since it's used in the relevancy calculation + expr.parentRowCount, nErr = expr.ParentTable.(sql.StatisticsTable).RowCount(ctx) + if nErr != nil { + err = nErr + return + } }) if err != nil { return 0, err @@ -310,13 +318,18 @@ func (expr *MatchAgainst) inNaturalLanguageMode(ctx *sql.Context, row sql.Row) ( if err != nil { return 0, err } - // This did not match, so we continue if len(docCountRows) == 0 { + // This did not match, so we continue continue } else if len(docCountRows) > 1 { return 0, fmt.Errorf("somehow there are duplicate entries within the Full-Text doc count table") } docCountRow := docCountRows[0] + docCount := float64(docCountRow[len(docCountRow)-1].(uint64)) + if docCount == 0 { + // We've got an empty document count, so the word does not match (so it should have been deleted) + continue + } // Otherwise, we've found a match, so we'll grab the global count as well lookup = sql.IndexLookup{Ranges: []sql.Range{ @@ -363,17 +376,21 @@ func (expr *MatchAgainst) inNaturalLanguageMode(ctx *sql.Context, row sql.Row) ( rowCountRow := rowCountRows[0] // Calculate the relevancy (partially based on an old MySQL implementation) - //TODO: use an actual algorithm with a good distribution, however we're focusing on correctly returned results for now - docCount := float32(docCountRow[len(docCountRow)-1].(uint64)) - globalCount := float32(globalCountRow[len(globalCountRow)-1].(uint64)) - uniqueWords := float32(rowCountRow[2].(uint64)) - fp := docCount / globalCount - sp := 1 + 1/(1+0.115*uniqueWords) - accumulatedRelevancy += fp * sp + // https://web.archive.org/web/20220122170304/http://dev.mysql.com/doc/internals/en/full-text-search.html + globalCount := float64(globalCountRow[len(globalCountRow)-1].(uint64)) + uniqueWords := float64(rowCountRow[2].(uint64)) + base := math.Log(docCount) + 1 + normFactor := uniqueWords / (1 + 0.115*uniqueWords) + globalMult := math.Log(float64(expr.parentRowCount)/globalCount) + 1 + accumulatedRelevancy += float32(base * normFactor * globalMult) } if err != nil { return 0, err } + // Due to how we handle floating to bool conversion, we need to add 0.5 if the result is positive + if accumulatedRelevancy > 0 { + accumulatedRelevancy += 0.5 + } // Return the accumulated relevancy from all of the parsed words return accumulatedRelevancy, nil } diff --git a/sql/fulltext/fulltext.go b/sql/fulltext/fulltext.go index 8334136931..6b9adddeb6 100644 --- a/sql/fulltext/fulltext.go +++ b/sql/fulltext/fulltext.go @@ -396,6 +396,101 @@ func RebuildTables(ctx *sql.Context, tbl sql.IndexAddressableTable, db Database) return CreateFulltextIndexes(ctx, db, tbl, predeterminedNames, indexDefs...) } +// DropColumnFromTables removes the given column from all of the Full-Text indexes, which will trigger a rebuild if the +// index spans multiple columns, but will trigger a deletion if the index spans that single column. The column name is +// case-insensitive. +func DropColumnFromTables(ctx *sql.Context, tbl sql.IndexAddressableTable, db Database, colName string) error { + // Check the interfaces on the parameters + dropper, ok := db.(sql.TableDropper) + if !ok { + return sql.ErrIncompleteFullTextIntegration.New() + } + idxAlterable, ok := tbl.(sql.IndexAlterableTable) + if !ok { + return sql.ErrIncompleteFullTextIntegration.New() + } + + lowercaseColName := strings.ToLower(colName) + configTableReuse := make(map[string]bool) + predeterminedNames := make(map[string]IndexTableNames) + var indexDefs []sql.IndexDef + + // Load the indexes to search for Full-Text indexes + indexes, err := tbl.GetIndexes(ctx) + if err != nil { + return err + } + for _, index := range indexes { + // Skip all non-Full-Text indexes + if !index.IsFullText() { + continue + } + // Store the index definition so that we may recreate it below + ftIndex := index.(Index) + tableNames, err := ftIndex.FullTextTableNames(ctx) + if err != nil { + return err + } + predeterminedNames[ftIndex.ID()] = tableNames + // Iterate over the columns to search for the given column + exprs := ftIndex.Expressions() + var indexCols []sql.IndexColumn + for _, expr := range exprs { + exprColName := strings.TrimPrefix(expr, ftIndex.Table()+".") + // Skip this column if it matches our given column + if strings.ToLower(exprColName) == lowercaseColName { + continue + } + indexCols = append(indexCols, sql.IndexColumn{ + Name: exprColName, + Length: 0, + }) + } + if len(indexCols) > 0 { + // This index will continue to exist, so we want to preserve the config table + indexDefs = append(indexDefs, sql.IndexDef{ + Name: ftIndex.ID(), + Columns: indexCols, + Constraint: sql.IndexConstraint_Fulltext, + Storage: sql.IndexUsing_Default, + Comment: ftIndex.Comment(), + }) + configTableReuse[tableNames.Config] = true + } else { + // This index will be deleted, so we should delete the config table if no other indexes will reuse the table + if _, ok := configTableReuse[tableNames.Config]; !ok { + configTableReuse[tableNames.Config] = false + } + } + // We delete all tables besides the config table + if err = dropper.DropTable(ctx, tableNames.Position); err != nil { + return err + } + if err = dropper.DropTable(ctx, tableNames.DocCount); err != nil { + return err + } + if err = dropper.DropTable(ctx, tableNames.GlobalCount); err != nil { + return err + } + if err = dropper.DropTable(ctx, tableNames.RowCount); err != nil { + return err + } + // Finally we'll drop the index + if err = idxAlterable.DropIndex(ctx, ftIndex.ID()); err != nil { + return err + } + } + // Delete all orphaned config tables + for configTableName, reused := range configTableReuse { + if !reused { + if err = dropper.DropTable(ctx, configTableName); err != nil { + return err + } + } + } + return CreateFulltextIndexes(ctx, db, tbl, predeterminedNames, indexDefs...) +} + // CreateFulltextIndexes creates and populates Full-Text indexes on the target table. func CreateFulltextIndexes(ctx *sql.Context, database Database, parent sql.Table, predeterminedNames map[string]IndexTableNames, indexes ...sql.IndexDef) error { @@ -417,6 +512,9 @@ func CreateFulltextIndexes(ctx *sql.Context, database Database, parent sql.Table if _, ok = fulltextAlterable.(sql.IndexAddressableTable); !ok { return sql.ErrFullTextNotSupported.New() } + if _, ok = fulltextAlterable.(sql.StatisticsTable); !ok { + return sql.ErrFullTextNotSupported.New() + } tblSch := parent.Schema() // Grab the key columns, which we will share among all indexes diff --git a/sql/plan/alter_check.go b/sql/plan/alter_check.go index 7d04e01a89..7539932e61 100644 --- a/sql/plan/alter_check.go +++ b/sql/plan/alter_check.go @@ -93,7 +93,7 @@ func (c *CreateCheck) WithChildren(children ...sql.Node) (sql.Node, error) { // CheckPrivileges implements the interface sql.Node. func (c *CreateCheck) CheckPrivileges(ctx *sql.Context, opChecker sql.PrivilegedOperationChecker) bool { - db := getDatabase(c.Child) + db := GetDatabase(c.Child) return opChecker.UserHasPrivileges(ctx, sql.NewPrivilegedOperation(CheckPrivilegeNameForDatabase(db), getTableName(c.Child), "", sql.PrivilegeType_Alter)) } @@ -125,7 +125,7 @@ func (p *DropCheck) WithChildren(children ...sql.Node) (sql.Node, error) { // CheckPrivileges implements the interface sql.Node. func (p *DropCheck) CheckPrivileges(ctx *sql.Context, opChecker sql.PrivilegedOperationChecker) bool { - db := getDatabase(p.Child) + db := GetDatabase(p.Child) return opChecker.UserHasPrivileges(ctx, sql.NewPrivilegedOperation(CheckPrivilegeNameForDatabase(db), getTableName(p.Child), "", sql.PrivilegeType_Alter)) } @@ -199,7 +199,7 @@ func (d DropConstraint) WithChildren(children ...sql.Node) (sql.Node, error) { // CheckPrivileges implements the interface sql.Node. func (d *DropConstraint) CheckPrivileges(ctx *sql.Context, opChecker sql.PrivilegedOperationChecker) bool { - db := getDatabase(d.Child) + db := GetDatabase(d.Child) return opChecker.UserHasPrivileges(ctx, sql.NewPrivilegedOperation(CheckPrivilegeNameForDatabase(db), getTableName(d.Child), "", sql.PrivilegeType_Alter)) } diff --git a/sql/plan/alter_foreign_key.go b/sql/plan/alter_foreign_key.go index 0f16d8795d..e2002e08e3 100644 --- a/sql/plan/alter_foreign_key.go +++ b/sql/plan/alter_foreign_key.go @@ -183,7 +183,7 @@ func ResolveForeignKey(ctx *sql.Context, tbl sql.ForeignKeyTable, refTbl sql.For } // Ensure that a suitable index exists on the referenced table, and check the declaring table for a suitable index. - refTblIndex, ok, err := FindIndexWithPrefix(ctx, refTbl, fkDef.ParentColumns, true) + refTblIndex, ok, err := FindFKIndexWithPrefix(ctx, refTbl, fkDef.ParentColumns, true) if err != nil { return err } @@ -235,7 +235,7 @@ func ResolveForeignKey(ctx *sql.Context, tbl sql.ForeignKeyTable, refTbl sql.For } } - _, ok, err := FindIndexWithPrefix(ctx, tbl, fkDef.Columns, false) + _, ok, err := FindFKIndexWithPrefix(ctx, tbl, fkDef.Columns, false) if err != nil { return err } @@ -430,8 +430,8 @@ func FindForeignKeyColMapping( return indexPositions, appendTypes, nil } -// FindIndexWithPrefix returns an index that has the given columns as a prefix. The returned index is deterministic and -// follows the given rules, from the highest priority in descending order: +// FindFKIndexWithPrefix returns an index that has the given columns as a prefix, with the index intended for use with +// foreign keys. The returned index is deterministic and follows the given rules, from the highest priority in descending order: // // 1. Columns exactly match the index // 2. Columns match as much of the index prefix as possible @@ -447,7 +447,7 @@ func FindForeignKeyColMapping( // If `useExtendedIndexes` is true, then this will include any implicit primary keys that were not explicitly defined on // the index. Some operations only consider explicitly indexed columns, while others also consider any implicit primary // keys as well, therefore this is a boolean to control the desired behavior. -func FindIndexWithPrefix(ctx *sql.Context, tbl sql.IndexAddressableTable, prefixCols []string, useExtendedIndexes bool, ignoredIndexes ...string) (sql.Index, bool, error) { +func FindFKIndexWithPrefix(ctx *sql.Context, tbl sql.IndexAddressableTable, prefixCols []string, useExtendedIndexes bool, ignoredIndexes ...string) (sql.Index, bool, error) { type idxWithLen struct { sql.Index colLen int @@ -466,7 +466,7 @@ func FindIndexWithPrefix(ctx *sql.Context, tbl sql.IndexAddressableTable, prefix // https://dev.mysql.com/doc/refman/8.0/en/create-table-foreign-keys.html#:~:text=Index%20prefixes%20on%20foreign%20key%20columns%20are%20not%20supported. // Ignore spatial indexes; MySQL will not pick them as the underlying secondary index for foreign keys for _, idx := range indexes { - if len(idx.PrefixLengths()) > 0 || idx.IsSpatial() { + if len(idx.PrefixLengths()) > 0 || idx.IsSpatial() || idx.IsFullText() { ignoredIndexesMap[strings.ToLower(idx.ID())] = struct{}{} } } diff --git a/sql/plan/create_index.go b/sql/plan/create_index.go index 783ceb38c0..efb5218730 100644 --- a/sql/plan/create_index.go +++ b/sql/plan/create_index.go @@ -135,7 +135,7 @@ func (c *CreateIndex) WithChildren(children ...sql.Node) (sql.Node, error) { // CheckPrivileges implements the interface sql.Node. func (c *CreateIndex) CheckPrivileges(ctx *sql.Context, opChecker sql.PrivilegedOperationChecker) bool { - db := getDatabase(c.Table) + db := GetDatabase(c.Table) return opChecker.UserHasPrivileges(ctx, sql.NewPrivilegedOperation(CheckPrivilegeNameForDatabase(db), getTableName(c.Table), "", sql.PrivilegeType_Index)) } diff --git a/sql/plan/ddl.go b/sql/plan/ddl.go index 2a7d216c3e..784d59e90b 100644 --- a/sql/plan/ddl.go +++ b/sql/plan/ddl.go @@ -703,7 +703,7 @@ func (d *DropTable) WithChildren(children ...sql.Node) (sql.Node, error) { // CheckPrivileges implements the interface sql.Node. func (d *DropTable) CheckPrivileges(ctx *sql.Context, opChecker sql.PrivilegedOperationChecker) bool { for _, tbl := range d.Tables { - db := getDatabase(tbl) + db := GetDatabase(tbl) if !opChecker.UserHasPrivileges(ctx, sql.NewPrivilegedOperation(CheckPrivilegeNameForDatabase(db), getTableName(tbl), "", sql.PrivilegeType_Drop)) { return false diff --git a/sql/plan/delete.go b/sql/plan/delete.go index 2f53325934..a790a2e346 100644 --- a/sql/plan/delete.go +++ b/sql/plan/delete.go @@ -88,11 +88,11 @@ func (p *DeleteFrom) Resolved() bool { // DB returns the database being deleted from. |Database| is used by another interface we implement. func (p *DeleteFrom) DB() sql.Database { - return getDatabase(p.Child) + return GetDatabase(p.Child) } func (p *DeleteFrom) Database() string { - database := getDatabase(p.Child) + database := GetDatabase(p.Child) if database == nil { return "" } @@ -120,7 +120,7 @@ func (p *DeleteFrom) CheckPrivileges(ctx *sql.Context, opChecker sql.PrivilegedO return false } - db := getDatabase(target) + db := GetDatabase(target) checkName := CheckPrivilegeNameForDatabase(db) op := sql.NewPrivilegedOperation(checkName, deletable.Name(), "", sql.PrivilegeType_Delete) if opChecker.UserHasPrivileges(ctx, op) == false { diff --git a/sql/plan/update.go b/sql/plan/update.go index 2ba49914d6..ac9d6dfec4 100644 --- a/sql/plan/update.go +++ b/sql/plan/update.go @@ -90,11 +90,11 @@ func getUpdatableTable(t sql.Table) (sql.UpdatableTable, error) { } } -// getDatabase returns the first database found in the node tree given -func getDatabase(node sql.Node) sql.Database { +// GetDatabase returns the first database found in the node tree given +func GetDatabase(node sql.Node) sql.Database { switch node := node.(type) { case *IndexedTableAccess: - return getDatabase(node.ResolvedTable) + return GetDatabase(node.ResolvedTable) case *ResolvedTable: return node.Database case *UnresolvedTable: @@ -102,7 +102,7 @@ func getDatabase(node sql.Node) sql.Database { } for _, child := range node.Children() { - return getDatabase(child) + return GetDatabase(child) } return nil @@ -110,11 +110,11 @@ func getDatabase(node sql.Node) sql.Database { // DB returns the database being updated. |Database| is already used by another interface we implement. func (u *Update) DB() sql.Database { - return getDatabase(u.Child) + return GetDatabase(u.Child) } func (u *Update) Database() string { - db := getDatabase(u.Child) + db := GetDatabase(u.Child) if db == nil { return "" } diff --git a/sql/rowexec/ddl_iters.go b/sql/rowexec/ddl_iters.go index 9472031d31..17bc76e9ee 100644 --- a/sql/rowexec/ddl_iters.go +++ b/sql/rowexec/ddl_iters.go @@ -1565,6 +1565,11 @@ func (i *dropColumnIter) Next(ctx *sql.Context) (sql.Row, error) { // Full-Text indexes will need to be rebuilt hasFullText := hasFullText(ctx, i.alterable) + if hasFullText { + if err := fulltext.DropColumnFromTables(ctx, i.alterable.(sql.IndexAddressableTable), i.d.Db.(fulltext.Database), i.d.Column); err != nil { + return nil, err + } + } // drop constraints that reference the dropped column cat, ok := i.alterable.(sql.CheckAlterableTable) @@ -1929,7 +1934,7 @@ func (b *BaseBuilder) executeAlterIndex(ctx *sql.Context, n *plan.AlterIndex) er return err } for _, fk := range fks { - _, ok, err := plan.FindIndexWithPrefix(ctx, fkTable, fk.Columns, false, n.IndexName) + _, ok, err := plan.FindFKIndexWithPrefix(ctx, fkTable, fk.Columns, false, n.IndexName) if err != nil { return err } @@ -1943,7 +1948,7 @@ func (b *BaseBuilder) executeAlterIndex(ctx *sql.Context, n *plan.AlterIndex) er return err } for _, parentFk := range parentFks { - _, ok, err := plan.FindIndexWithPrefix(ctx, fkTable, parentFk.ParentColumns, true, n.IndexName) + _, ok, err := plan.FindFKIndexWithPrefix(ctx, fkTable, parentFk.ParentColumns, true, n.IndexName) if err != nil { return err } diff --git a/sql/rowexec/dml.go b/sql/rowexec/dml.go index 4fde6bd2f1..e8b92614b0 100644 --- a/sql/rowexec/dml.go +++ b/sql/rowexec/dml.go @@ -368,6 +368,12 @@ func (b *BaseBuilder) buildTruncate(ctx *sql.Context, n *plan.Truncate, row sql. break } } + // If we've got Full-Text indexes, then we also need to clear those tables + if hasFullText(ctx, truncatable) { + if err = rebuildFullText(ctx, truncatable.Name(), plan.GetDatabase(n.Child)); err != nil { + return nil, err + } + } return sql.RowsToRowIter(sql.NewRow(types.NewOkResult(removed))), nil }