perf: speed up repository issue and pull request lists (#40114) (#40132)

Backport #40114 by @silverwind

The repository issue and pull request lists filter `access` by
`repo_id`, which no index covers, so every load scans the whole table.
Issue counts on larger repositories merge three single-column indices,
and the API pull request list loads the latest reviews with `id IN
(SELECT max(id) ... GROUP BY ...)`, which MySQL runs as a scan of every
review, about 1.1s on
https://gitea.com/api/v1/repos/gitea/gitea-mcp/pulls?limit=1.

- Index `access.repo_id`
- Index `team_repo.repo_id`
- Replace the single `repo_id`, `is_closed` and `is_pull` indices with
one `issue` index on `(repo_id, is_closed, is_pull)`
- Join the latest review ids instead of `id IN (subquery)`

Median times in ms on MySQL 8.4 with tables sized like gitea.com (1M
access rows, 500k issues, 200k reviews), for a copy of
https://gitea.com/gitea/gitea-mcp and a repository with 9.7k issues:

| Page | Before | After |
|---|---|---|
| `/{owner}/{repo}/issues` | 107 | 40 |
| `/{owner}/{repo}/pulls` | 106 | 32 |
| `/{owner}/{repo}/pulls?state=closed` | 153 | 35 |
| `/{owner}/{repo}/issues/{index}` | 141 | 60 |
| `/{owner}/{repo}/issues`, 9.7k issues | 303 | 53 |
| `/api/v1/repos/{owner}/{repo}/pulls?limit=50` | 180 | 46 |
| `/api/v1/repos/{owner}/{repo}/pulls?limit=1` | 137 | 16 |
| `/api/v1/repos/{owner}/{repo}/issues`, 9.7k issues | 248 | 43 |

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

models/issues/issue.go

@@ -55,7 +55,7 @@ var ErrIssueAlreadyChanged = util.NewInvalidArgumentErrorf("the issue is already
 // Issue represents an issue or pull request of repository.
 type Issue struct {
 	ID                int64                  `xorm:"pk autoincr"`
-	RepoID            int64                  `xorm:"INDEX UNIQUE(repo_index)"`
+	RepoID            int64                  `xorm:"INDEX(repo_closed_pull) UNIQUE(repo_index)"`
 	Repo              *repo_model.Repository `xorm:"-"`
 	Index             int64                  `xorm:"UNIQUE(repo_index)"` // Index in one repository.
 	PosterID          int64                  `xorm:"INDEX"`
@@ -77,9 +77,9 @@ type Issue struct {
 	AssigneeID        int64            `xorm:"-"`
 	Assignee          *user_model.User `xorm:"-"`
 	isAssigneeLoaded  bool             `xorm:"-"`
-	IsClosed          bool             `xorm:"INDEX"`
+	IsClosed          bool             `xorm:"INDEX(repo_closed_pull)"`
 	IsRead            bool             `xorm:"-"`
-	IsPull            bool             `xorm:"INDEX"` // Indicates whether is a pull request or not.
+	IsPull            bool             `xorm:"INDEX(repo_closed_pull)"` // Indicates whether is a pull request or not.
 	PullRequest       *PullRequest     `xorm:"-"`
 	NumComments       int
 

models/issues/pull_list.go

@@ -335,8 +335,8 @@ func (prs PullRequestList) LoadReviews(ctx context.Context) (ReviewList, error)
 			"reviewer_team_id":   0,
 		}).
 		GroupBy("issue_id, reviewer_id")
-	// Get latest review of each reviewer, sorted in order they were made
-	if err := db.GetEngine(ctx).In("id", subQuery).OrderBy("review.updated_unix ASC").Find(&reviews); err != nil {
+	// Get latest review of each reviewer, sorted in order they were made. Joined because MySQL scans all reviews for "id IN (subquery)", xorm aliases the subquery "review".
+	if err := db.GetEngine(ctx).Table("review").Alias("latest").Join("INNER", subQuery, "latest.id = review.id").OrderBy("latest.updated_unix ASC").Find(&reviews); err != nil {
 		return nil, err
 	}
 
@@ -350,7 +350,7 @@ func (prs PullRequestList) LoadReviews(ctx context.Context) (ReviewList, error)
 		"reviewer_team_id": 0,
 	}).
 		GroupBy("issue_id, reviewer_team_id")
-	if err := db.GetEngine(ctx).In("id", subQueryTeam).OrderBy("review.updated_unix ASC").Find(&teamReviewRequests); err != nil {
+	if err := db.GetEngine(ctx).Table("review").Alias("latest").Join("INNER", subQueryTeam, "latest.id = review.id").OrderBy("latest.updated_unix ASC").Find(&teamReviewRequests); err != nil {
 		return nil, err
 	}
 

models/organization/team_repo.go

@@ -13,12 +13,12 @@ import (
 	"xorm.io/builder"
 )
 
-// TeamRepo represents an team-repository relation.
+// TeamRepo represents an org-team-repository relation.
 type TeamRepo struct {
 	ID     int64 `xorm:"pk autoincr"`
 	OrgID  int64 `xorm:"INDEX"`
 	TeamID int64 `xorm:"UNIQUE(s)"`
-	RepoID int64 `xorm:"UNIQUE(s)"`
+	RepoID int64 `xorm:"UNIQUE(s) INDEX"`
 }
 
 // HasTeamRepo returns true if given repository belongs to team.

models/perm/access/access.go

@@ -25,7 +25,7 @@ import (
 type Access struct {
 	ID     int64 `xorm:"pk autoincr"`
 	UserID int64 `xorm:"UNIQUE(s)"`
-	RepoID int64 `xorm:"UNIQUE(s)"`
+	RepoID int64 `xorm:"UNIQUE(s) INDEX"`
 	Mode   perm.AccessMode
 }
 

models/repo/repo.go

@@ -494,10 +494,9 @@ func (repo *Repository) composeCommonMetas(ctx context.Context) map[string]strin
 		repo.MustOwner(ctx)
 		if repo.Owner.IsOrganization() {
 			teams := make([]string, 0, 5)
-			_ = db.GetEngine(ctx).Table("team_repo").
+			_ = db.GetEngine(ctx).Select("team.lower_name").Table("team_repo").
 				Join("INNER", "team", "team.id = team_repo.team_id").
 				Where("team_repo.repo_id = ?", repo.ID).
-				Select("team.lower_name").
 				OrderBy("team.lower_name").
 				Find(&teams)
 			metas["teams"] = "," + strings.Join(teams, ",") + ","

services/issue/issue.go

@@ -290,7 +290,6 @@ func DeleteIssuesByRepoID(ctx context.Context, repoID int64) (attachmentPaths []
 		issues := make([]*issues_model.Issue, 0, db.DefaultMaxInSize)
 		if err := db.GetEngine(ctx).
 			Where("repo_id = ?", repoID).
-			OrderBy("id").
 			Limit(db.DefaultMaxInSize).
 			Find(&issues); err != nil {
 			return nil, err