Rebuild-indexes cmd

cmd/rebuild_indexes.go

@@ -0,0 +1,124 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/private"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/urfave/cli"
+)
+
+type rebuildStepFunc func(repoID int64) (bool, error)
+
+// CmdRebuildIndexes represents the available rebuild-indexes sub-command
+var CmdRebuildIndexes = cli.Command{
+	Name:        "rebuild-indexes",
+	Usage:       "Rebuild text indexes",
+	Description: "This command rebuilds text indexes for issues and repositories",
+	Action:      runRebuildIndexes,
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "repositories",
+			Usage: "Rebuild text indexes for repository content",
+		},
+		cli.BoolFlag{
+			Name:  "issues",
+			Usage: "Rebuild text indexes for issues",
+		},
+		cli.BoolFlag{
+			Name:  "all",
+			Usage: "Rebuild all text indexes",
+		},
+	},
+}
+
+func runRebuildIndexes(ctx *cli.Context) error {
+	setting.NewContext()
+	setting.NewServices()
+
+	var rebuildIssues bool
+	var rebuildRepositories bool
+
+	if ctx.IsSet("repositories") || ctx.IsSet("all") {
+		rebuildRepositories = true
+		log.Info("Rebuild text indexes for repository content")
+	}
+
+	if ctx.IsSet("issues") || ctx.IsSet("all") {
+		rebuildIssues = true
+		log.Info("Rebuild text indexes for issues")
+	}
+
+	if !rebuildIssues && !rebuildRepositories {
+		fmt.Printf("At least one of --repositories, --issues or --all must be used\n")
+		return nil
+	}
+
+	if !rebuildIssues && !setting.Indexer.RepoIndexerEnabled {
+		fmt.Printf("Repository level text indexes are not enabled\n")
+		return nil
+	}
+
+	if err := initDB(); err != nil {
+		fmt.Printf("Error: %v\n", err)
+		return err
+	}
+
+	for page := 1; ; page++ {
+		repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
+			Page:        page,
+			PageSize:    models.RepositoryListDefaultPageSize,
+			OrderBy:     models.SearchOrderByID,
+			Private:     true,
+			Collaborate: util.OptionalBoolFalse,
+		})
+		if err != nil {
+			log.Error("SearchRepositoryByName: %v", err)
+			return err
+		}
+		if len(repos) == 0 {
+			break
+		}
+
+		for _, repo := range repos {
+			fmt.Printf("Rebuilding text indexes for %s\n", repo.FullName())
+			if rebuildRepositories {
+				if err := rebuildStep(repo.ID, private.RebuildRepoIndex); err != nil {
+					return err
+				}
+			}
+			if rebuildIssues {
+				if err := rebuildStep(repo.ID, private.RebuildIssueIndex); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	fmt.Println("Done")
+	return nil
+}
+
+func rebuildStep(repoID int64, fn rebuildStepFunc) error {
+	for {
+		toobusy, err := fn(repoID)
+		if err != nil {
+			fmt.Printf("Internal error: %v\n", err)
+			return err
+		}
+		if !toobusy {
+			return nil
+		}
+		fmt.Printf("Server too busy; backing off...\n")
+		time.Sleep(1 * time.Second)
+	}
+}

main.go

@@ -68,6 +68,7 @@ arguments - which can alternatively be run by running the subcommand web.`
 		cmd.CmdMigrate,
 		cmd.CmdKeys,
 		cmd.CmdConvert,
+		cmd.CmdRebuildIndexes,
 	}
 	// Now adjust these commands to add our global configuration options
 

models/repo_indexer.go

@@ -360,3 +360,30 @@ func addOperationToQueue(op repoIndexerOperation) {
 		}()
 	}
 }
+
+func isQueueNearFull() bool {
+	qcap := cap(repoIndexerOperationQueue)
+	qlen := len(repoIndexerOperationQueue)
+	if qcap <= 3 {
+		return qlen == qcap
+	}
+	return qcap-qlen < 3
+}
+
+// RebuildRepoIndex deletes and rebuilds text indexes for a repo
+func RebuildRepoIndex(repoID int64) (bool, error) {
+	if isQueueNearFull() {
+		return true, nil
+	}
+	repo := &Repository{ID: repoID}
+	has, err := x.Get(repo)
+	if err != nil {
+		return false, err
+	}
+	if !has {
+		return false, nil
+	}
+	DeleteRepoFromIndexer(repo)
+	UpdateRepoIndexer(repo)
+	return false, nil
+}

modules/indexer/issues/indexer.go

@@ -138,26 +138,32 @@ func populateIssueIndexer() {
 		}
 
 		for _, repo := range repos {
-			is, err := models.Issues(&models.IssuesOptions{
-				RepoIDs:  []int64{repo.ID},
-				IsClosed: util.OptionalBoolNone,
-				IsPull:   util.OptionalBoolNone,
-			})
-			if err != nil {
-				log.Error("Issues: %v", err)
-				continue
-			}
-			if err = models.IssueList(is).LoadDiscussComments(); err != nil {
-				log.Error("LoadComments: %v", err)
-				continue
-			}
-			for _, issue := range is {
-				UpdateIssueIndexer(issue)
-			}
+			_ = populateIssueIndexerRepo(repo.ID)
 		}
 	}
 }
 
+// populateIssueIndexerRepo populate the issue indexer with issue data for one repo
+func populateIssueIndexerRepo(repoID int64) error {
+	is, err := models.Issues(&models.IssuesOptions{
+		RepoIDs:  []int64{repoID},
+		IsClosed: util.OptionalBoolNone,
+		IsPull:   util.OptionalBoolNone,
+	})
+	if err != nil {
+		log.Error("Issues: %v", err)
+		return err
+	}
+	if err = models.IssueList(is).LoadDiscussComments(); err != nil {
+		log.Error("LoadComments: %v", err)
+		return err
+	}
+	for _, issue := range is {
+		UpdateIssueIndexer(issue)
+	}
+	return nil
+}
+
 // UpdateIssueIndexer add/update an issue to the issue indexer
 func UpdateIssueIndexer(issue *models.Issue) {
 	var comments []string
@@ -206,3 +212,21 @@ func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
 	}
 	return issueIDs, nil
 }
+
+// RebuildIssueIndex deletes and rebuilds text indexes for a repo
+func RebuildIssueIndex(repoID int64) (bool, error) {
+	repo, err := models.GetRepositoryByID(repoID)
+	if err != nil {
+		return false, err
+	}
+	if repo == nil {
+		return false, nil
+	}
+	DeleteRepoIssueIndexer(repo)
+	err = populateIssueIndexerRepo(repoID)
+	if err != nil {
+		return false, err
+	}
+	// Currently there's no way to tell if a queue is near full
+	return false, nil
+}

modules/private/rebuild_indexes.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// RebuildRepoIndex rebuild a repository index
+func RebuildRepoIndex(repoID int64) (bool, error) {
+	// Ask for running deliver hook and test pull request tasks.
+	reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-repo-index/%d", repoID)
+	resp, err := newInternalRequest(reqURL, "GET").Response()
+	if err != nil {
+		return false, err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 503 {
+		// Server is too busy; back off a little
+		return true, nil
+	}
+
+	// All 2XX status codes are accepted and others will return an error
+	if resp.StatusCode/100 != 2 {
+		return false, fmt.Errorf("Failed to rebuild indexes for repository: %s", decodeJSONError(resp).Err)
+	}
+	return false, nil
+}
+
+// RebuildIssueIndex rebuild issue index for a repo
+func RebuildIssueIndex(repoID int64) (bool, error) {
+	// Ask for running deliver hook and test pull request tasks.
+	reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-issue-index/%d", repoID)
+	resp, err := newInternalRequest(reqURL, "GET").Response()
+	if err != nil {
+		return false, err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 503 {
+		// Server is too busy; back off a little
+		return true, nil
+	}
+
+	// All 2XX status codes are accepted and others will return an error
+	if resp.StatusCode/100 != 2 {
+		return false, fmt.Errorf("Failed to rebuild indexes for repository: %s", decodeJSONError(resp).Err)
+	}
+	return false, nil
+}

routers/private/internal.go

@@ -81,5 +81,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Get("/hook/post-receive/:owner/:repo", HookPostReceive)
 		m.Get("/serv/none/:keyid", ServNoCommand)
 		m.Get("/serv/command/:keyid/:owner/:repo", ServCommand)
+		m.Get("/maint/rebuild-repo-index/:repoid", RebuildRepoIndex)
+		m.Get("/maint/rebuild-issue-index/:repoid", RebuildIssueIndex)
 	}, CheckInternalToken)
 }

routers/private/rebuild_indexes.go

@@ -0,0 +1,51 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// Package private includes all internal routes. The package name internal is ideal but Golang is not allowed, so we use private as package name instead.
+package private
+
+import (
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/indexer/issues"
+
+	macaron "gopkg.in/macaron.v1"
+)
+
+// RebuildRepoIndex rebuilds a repository index
+func RebuildRepoIndex(ctx *macaron.Context) {
+	repoID := ctx.ParamsInt64(":repoid")
+	toobusy, err := models.RebuildRepoIndex(repoID)
+	if err != nil {
+		ctx.JSON(500, map[string]interface{}{
+			"err": err.Error(),
+		})
+		return
+	}
+
+	if toobusy {
+		ctx.PlainText(503, []byte("too busy"))
+		return
+	}
+
+	ctx.PlainText(200, []byte("success"))
+}
+
+// RebuildIssueIndex rebuilds issue index
+func RebuildIssueIndex(ctx *macaron.Context) {
+	repoID := ctx.ParamsInt64(":repoid")
+	toobusy, err := issues.RebuildIssueIndex(repoID)
+	if err != nil {
+		ctx.JSON(500, map[string]interface{}{
+			"err": err.Error(),
+		})
+		return
+	}
+
+	if toobusy {
+		ctx.PlainText(503, []byte("too busy"))
+		return
+	}
+
+	ctx.PlainText(200, []byte("success"))
+}