diff --git a/cmd/rebuild_indexes.go b/cmd/rebuild_indexes.go new file mode 100644 index 0000000000000..900347151c84c --- /dev/null +++ b/cmd/rebuild_indexes.go @@ -0,0 +1,97 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "fmt" + "os" + + "code.gitea.io/gitea/modules/indexer" + "code.gitea.io/gitea/modules/indexer/issues" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/private" + "code.gitea.io/gitea/modules/setting" + + "github.com/urfave/cli" +) + +// CmdRebuildIndexes represents the available rebuild-indexes sub-command +var CmdRebuildIndexes = cli.Command{ + Name: "rebuild-indexes", + Usage: "Rebuild text indexes", + Description: "This command rebuilds text indexes for issues and repositories", + Action: runRebuildIndexes, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "repositories", + Usage: "Rebuild text indexes for repository content", + }, + cli.BoolFlag{ + Name: "issues", + Usage: "Rebuild text indexes for issues", + }, + cli.BoolFlag{ + Name: "all", + Usage: "Rebuild all text indexes", + }, + }, +} + +func runRebuildIndexes(ctx *cli.Context) error { + setting.NewContext() + setting.NewServices() + + var rebuildIssues bool + var rebuildRepositories bool + + if ctx.IsSet("repositories") || ctx.IsSet("all") { + rebuildRepositories = true + } + + if ctx.IsSet("issues") || ctx.IsSet("all") { + rebuildIssues = true + } + + if !rebuildIssues && !rebuildRepositories { + fmt.Printf("At least one of --repositories, --issues or --all must be used\n") + return nil + } + + if rebuildRepositories && !setting.Indexer.RepoIndexerEnabled { + fmt.Printf("Repository indexes are not enabled\n") + rebuildRepositories = false + } + + if rebuildIssues && setting.Indexer.IssueType != "bleve" { + log.ColorFprintf(os.Stdout, "Issue index type '%s' does not support or does not require rebuilding\n", setting.Indexer.IssueType) + rebuildIssues = false + } + + if rebuildRepositories { + attemptRebuild("Rebuild repository indexes", private.RebuildRepoIndex, indexer.DropRepoIndex) + } + + if rebuildIssues { + attemptRebuild("Rebuild issue indexes", private.RebuildIssueIndex, issues.DropIssueIndex) + } + + fmt.Println("Rebuild done or in process.") + return nil +} + +func attemptRebuild(msg string, onlineRebuild func() error, offlineDrop func() error) { + log.Info(msg) + fmt.Printf("%s: attempting through Gitea API...\n", msg) + if err := onlineRebuild(); err != nil { + // FIXME: there's no good way of knowing if Gitea is running + log.ColorFprintf(os.Stdout, "Error (disregard if it's a connection error): %v\n", err) + // Attempt a direct delete + fmt.Printf("Gitea seems to be down; marking index files for recycling the next time Gitea runs.\n") + if err := offlineDrop(); err != nil { + log.ColorFprintf(os.Stdout, "Internal error: %v\n", err) + log.Fatal("Rebuild indexes: %v", err) + } + } +} diff --git a/main.go b/main.go index 30dbf2766224c..de2208a4b6766 100644 --- a/main.go +++ b/main.go @@ -68,6 +68,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.CmdMigrate, cmd.CmdKeys, cmd.CmdConvert, + cmd.CmdRebuildIndexes, } // Now adjust these commands to add our global configuration options diff --git a/models/repo_indexer.go b/models/repo_indexer.go index b842a1c87f229..85849b33d2c4a 100644 --- a/models/repo_indexer.go +++ b/models/repo_indexer.go @@ -8,6 +8,7 @@ import ( "fmt" "strconv" "strings" + "sync" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" @@ -27,6 +28,10 @@ type RepoIndexerStatus struct { CommitSha string `xorm:"VARCHAR(40)"` } +var ( + rebuildLock sync.Mutex +) + func (repo *Repository) getIndexerStatus() error { if repo.IndexerStatus != nil { return nil @@ -104,6 +109,8 @@ func populateRepoIndexerAsynchronously() error { // populateRepoIndexer populate the repo indexer with pre-existing data. This // should only be run when the indexer is created for the first time. func populateRepoIndexer(maxRepoID int64) { + rebuildLock.Lock() + defer rebuildLock.Unlock() log.Info("Populating the repo indexer with existing repositories") // start with the maximum existing repo ID and work backwards, so that we // don't include repos that are created after gitea starts; such repos will @@ -376,3 +383,26 @@ func addOperationToQueue(op repoIndexerOperation) { }() } } + +func rebuildRepoIndex() error { + // Make sure no other build is currently running + rebuildLock.Lock() + defer rebuildLock.Unlock() + if err := indexer.DropRepoIndex(); err != nil { + return err + } + // This could abort with Fatal() + indexer.InitRepoIndexer(nil) + return nil +} + +// RebuildRepoIndex deletes and rebuilds text indexes for repositories +func RebuildRepoIndex() error { + if !setting.Indexer.RepoIndexerEnabled { + return nil + } + if err := rebuildRepoIndex(); err != nil { + return err + } + return populateRepoIndexerAsynchronously() +} diff --git a/modules/indexer/indexer.go b/modules/indexer/indexer.go index 29261c693b592..ae331686df75f 100644 --- a/modules/indexer/indexer.go +++ b/modules/indexer/indexer.go @@ -8,8 +8,6 @@ import ( "os" "strconv" - "code.gitea.io/gitea/modules/setting" - "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/analysis/token/unicodenorm" "github.com/blevesearch/bleve/index/upsidedown" @@ -47,7 +45,7 @@ const maxBatchSize = 16 // updates and bleve version updates. If index needs to be created (or // re-created), returns (nil, nil) func openIndexer(path string, latestVersion int) (bleve.Index, error) { - _, err := os.Stat(setting.Indexer.IssuePath) + _, err := os.Stat(path) if err != nil && os.IsNotExist(err) { return nil, nil } else if err != nil { diff --git a/modules/indexer/issues/bleve.go b/modules/indexer/issues/bleve.go index 36279198b86b0..d2e51eabfcf07 100644 --- a/modules/indexer/issues/bleve.go +++ b/modules/indexer/issues/bleve.go @@ -248,3 +248,10 @@ func (b *BleveIndexer) Search(keyword string, repoID int64, limit, start int) (* } return &ret, nil } + +// Drop marks the index for rebuilding by invalidating its version number +func (b *BleveIndexer) Drop(path string) (bool, error) { + return true, rupture.WriteIndexMetadata(path, &rupture.IndexMetadata{ + Version: -1, + }) +} diff --git a/modules/indexer/issues/db.go b/modules/indexer/issues/db.go index 6e7f0c1a6e1f0..4da260aa1dc80 100644 --- a/modules/indexer/issues/db.go +++ b/modules/indexer/issues/db.go @@ -43,3 +43,8 @@ func (db *DBIndexer) Search(kw string, repoID int64, limit, start int) (*SearchR } return &result, nil } + +// Drop dummy function +func (db *DBIndexer) Drop(path string) (bool, error) { + return false, nil +} diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index df8bfd6305912..0443744d16b30 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -6,6 +6,7 @@ package issues import ( "fmt" + "sync" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" @@ -43,12 +44,14 @@ type Indexer interface { Index(issue []*IndexerData) error Delete(ids ...int64) error Search(kw string, repoID int64, limit, start int) (*SearchResult, error) + Drop(path string) (bool, error) } var ( // issueIndexerQueue queue of issue ids to be updated issueIndexerQueue Queue issueIndexer Indexer + issueRebuildLock sync.Mutex ) // InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until @@ -121,6 +124,8 @@ func InitIssueIndexer(syncReindex bool) error { // populateIssueIndexer populate the issue indexer with issue data func populateIssueIndexer() { + issueRebuildLock.Lock() + defer issueRebuildLock.Unlock() for page := 1; ; page++ { repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{ Page: page, @@ -138,26 +143,32 @@ func populateIssueIndexer() { } for _, repo := range repos { - is, err := models.Issues(&models.IssuesOptions{ - RepoIDs: []int64{repo.ID}, - IsClosed: util.OptionalBoolNone, - IsPull: util.OptionalBoolNone, - }) - if err != nil { - log.Error("Issues: %v", err) - continue - } - if err = models.IssueList(is).LoadDiscussComments(); err != nil { - log.Error("LoadComments: %v", err) - continue - } - for _, issue := range is { - UpdateIssueIndexer(issue) - } + _ = populateIssueIndexerRepo(repo.ID) } } } +// populateIssueIndexerRepo populate the issue indexer with issue data for one repo +func populateIssueIndexerRepo(repoID int64) error { + is, err := models.Issues(&models.IssuesOptions{ + RepoIDs: []int64{repoID}, + IsClosed: util.OptionalBoolNone, + IsPull: util.OptionalBoolNone, + }) + if err != nil { + log.Error("Issues: %v", err) + return err + } + if err = models.IssueList(is).LoadDiscussComments(); err != nil { + log.Error("LoadComments: %v", err) + return err + } + for _, issue := range is { + UpdateIssueIndexer(issue) + } + return nil +} + // UpdateIssueIndexer add/update an issue to the issue indexer func UpdateIssueIndexer(issue *models.Issue) { var comments []string @@ -206,3 +217,41 @@ func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) { } return issueIDs, nil } + +func rebuildIssueIndex() (bool, error) { + // Make sure no other build is currently running + issueRebuildLock.Lock() + defer issueRebuildLock.Unlock() + + canrebuild, err := issueIndexer.Drop(setting.Indexer.IssuePath) + if !canrebuild || err != nil { + return false, err + } + _, err = issueIndexer.Init() + if err != nil { + return false, err + } + return true, nil +} + +// RebuildIssueIndex deletes and rebuilds text indexes for a repo +func RebuildIssueIndex() error { + // Drop the index in a protected func + repopulate, err := rebuildIssueIndex() + if !repopulate || err != nil { + return err + } + // Launch repopulation + go populateIssueIndexer() + + return nil +} + +// DropIssueIndex deletes text indexes for issues; intended to be used from command line +func DropIssueIndex() error { + if issueIndexer != nil { + _, err := issueIndexer.Drop(setting.Indexer.IssuePath) + return err + } + return nil +} diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go index 91ed173aa769a..2afff10455367 100644 --- a/modules/indexer/repo.go +++ b/modules/indexer/repo.go @@ -85,6 +85,9 @@ func InitRepoIndexer(populateIndexer func() error) { if err = createRepoIndexer(setting.Indexer.RepoPath, repoIndexerLatestVersion); err != nil { log.Fatal("CreateRepoIndexer: %v", err) } + if populateIndexer == nil { + return + } if err = populateIndexer(); err != nil { log.Fatal("PopulateRepoIndex: %v", err) } @@ -225,3 +228,10 @@ func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (i } return int64(result.Total), searchResults, nil } + +// DropRepoIndex marks the index for rebuilding by invalidating its version number +func DropRepoIndex() error { + return rupture.WriteIndexMetadata(setting.Indexer.RepoPath, &rupture.IndexMetadata{ + Version: -1, + }) +} diff --git a/modules/private/rebuild_indexes.go b/modules/private/rebuild_indexes.go new file mode 100644 index 0000000000000..36b8abfbfd31e --- /dev/null +++ b/modules/private/rebuild_indexes.go @@ -0,0 +1,47 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package private + +import ( + "fmt" + + "code.gitea.io/gitea/modules/setting" +) + +// RebuildRepoIndex rebuild a repository index +func RebuildRepoIndex() error { + // Ask for running deliver hook and test pull request tasks. + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-repo-index") + resp, err := newInternalRequest(reqURL, "GET").Response() + if err != nil { + return err + } + + defer resp.Body.Close() + + // All 2XX status codes are accepted and others will return an error + if resp.StatusCode/100 != 2 { + return fmt.Errorf("Failed to rebuild repository index: %s", decodeJSONError(resp).Err) + } + return nil +} + +// RebuildIssueIndex rebuild issue index for a repo +func RebuildIssueIndex() error { + // Ask for running deliver hook and test pull request tasks. + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-issue-index") + resp, err := newInternalRequest(reqURL, "GET").Response() + if err != nil { + return err + } + + defer resp.Body.Close() + + // All 2XX status codes are accepted and others will return an error + if resp.StatusCode/100 != 2 { + return fmt.Errorf("Failed to rebuild issue index: %s", decodeJSONError(resp).Err) + } + return nil +} diff --git a/routers/private/internal.go b/routers/private/internal.go index 3a48f5384d8c3..7802ece7eed79 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -81,5 +81,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/hook/post-receive/:owner/:repo", HookPostReceive) m.Get("/serv/none/:keyid", ServNoCommand) m.Get("/serv/command/:keyid/:owner/:repo", ServCommand) + m.Get("/maint/rebuild-repo-index", RebuildRepoIndex) + m.Get("/maint/rebuild-issue-index", RebuildIssueIndex) }, CheckInternalToken) } diff --git a/routers/private/rebuild_indexes.go b/routers/private/rebuild_indexes.go new file mode 100644 index 0000000000000..33bd32cac578e --- /dev/null +++ b/routers/private/rebuild_indexes.go @@ -0,0 +1,39 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package private includes all internal routes. The package name internal is ideal but Golang is not allowed, so we use private as package name instead. +package private + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/indexer/issues" + + "gitea.com/macaron/macaron" +) + +// RebuildRepoIndex rebuilds a repository index +func RebuildRepoIndex(ctx *macaron.Context) { + err := models.RebuildRepoIndex() + if err != nil { + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), + }) + return + } + + ctx.PlainText(200, []byte("success")) +} + +// RebuildIssueIndex rebuilds issue index +func RebuildIssueIndex(ctx *macaron.Context) { + err := issues.RebuildIssueIndex() + if err != nil { + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), + }) + return + } + + ctx.PlainText(200, []byte("success")) +}