From 4a5af4edca3883bcb45056de11b9fd59f9759b9b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 9 Apr 2025 09:34:38 -0700 Subject: [PATCH] Cache GPG keys, emails and users when list commits (#34086) When list commits, some of the commits authors are the same at many situations. But current logic will always fetch the same GPG keys from database. This PR will cache the GPG keys, emails and users for the context so that reducing the database queries. --------- Co-authored-by: wxiaoguang --- models/asymkey/gpg_key.go | 7 +++++++ models/user/user.go | 27 +++++++++++++-------------- modules/cache/context.go | 8 ++++---- modules/cache/context_test.go | 3 ++- modules/cachegroup/cachegroup.go | 12 ++++++++++++ modules/repository/commits.go | 3 ++- routers/private/hook_post_receive.go | 5 ++--- services/asymkey/commit.go | 25 ++++++++++--------------- services/convert/pull.go | 13 +++++++------ services/git/commit.go | 10 ++++++---- 10 files changed, 65 insertions(+), 48 deletions(-) create mode 100644 modules/cachegroup/cachegroup.go diff --git a/models/asymkey/gpg_key.go b/models/asymkey/gpg_key.go index 24c76f7b5cf..220f46ad1d4 100644 --- a/models/asymkey/gpg_key.go +++ b/models/asymkey/gpg_key.go @@ -240,3 +240,10 @@ func DeleteGPGKey(ctx context.Context, doer *user_model.User, id int64) (err err return committer.Commit() } + +func FindGPGKeyWithSubKeys(ctx context.Context, keyID string) ([]*GPGKey, error) { + return db.Find[GPGKey](ctx, FindGPGKeyOptions{ + KeyID: keyID, + IncludeSubKeys: true, + }) +} diff --git a/models/user/user.go b/models/user/user.go index 5989be74f03..100f924cc68 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -1187,29 +1187,28 @@ func GetUsersByEmails(ctx context.Context, emails []string) (map[string]*User, e for _, email := range emailAddresses { userIDs.Add(email.UID) } - users, err := GetUsersMapByIDs(ctx, userIDs.Values()) - if err != nil { - return nil, err - } - results := make(map[string]*User, len(emails)) - for _, email := range emailAddresses { - user := users[email.UID] - if user != nil { - if user.KeepEmailPrivate { - results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user - } else { - results[email.Email] = user + + if len(userIDs) > 0 { + users, err := GetUsersMapByIDs(ctx, userIDs.Values()) + if err != nil { + return nil, err + } + + for _, email := range emailAddresses { + user := users[email.UID] + if user != nil { + results[user.GetEmail()] = user } } } - users = make(map[int64]*User, len(needCheckUserNames)) + users := make(map[int64]*User, len(needCheckUserNames)) if err := db.GetEngine(ctx).In("lower_name", needCheckUserNames.Values()).Find(&users); err != nil { return nil, err } for _, user := range users { - results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user + results[user.GetPlaceholderEmail()] = user } return results, nil } diff --git a/modules/cache/context.go b/modules/cache/context.go index 484cee659a1..85eb9e6790e 100644 --- a/modules/cache/context.go +++ b/modules/cache/context.go @@ -166,15 +166,15 @@ func RemoveContextData(ctx context.Context, tp, key any) { } // GetWithContextCache returns the cache value of the given key in the given context. -func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error) { - v := GetContextData(ctx, cacheGroupKey, cacheTargetID) +func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) { + v := GetContextData(ctx, groupKey, targetKey) if vv, ok := v.(T); ok { return vv, nil } - t, err := f() + t, err := f(ctx, targetKey) if err != nil { return t, err } - SetContextData(ctx, cacheGroupKey, cacheTargetID, t) + SetContextData(ctx, groupKey, targetKey, t) return t, nil } diff --git a/modules/cache/context_test.go b/modules/cache/context_test.go index decb532937c..23dd789dbc2 100644 --- a/modules/cache/context_test.go +++ b/modules/cache/context_test.go @@ -4,6 +4,7 @@ package cache import ( + "context" "testing" "time" @@ -30,7 +31,7 @@ func TestWithCacheContext(t *testing.T) { v = GetContextData(ctx, field, "my_config1") assert.Nil(t, v) - vInt, err := GetWithContextCache(ctx, field, "my_config1", func() (int, error) { + vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) { return 1, nil }) assert.NoError(t, err) diff --git a/modules/cachegroup/cachegroup.go b/modules/cachegroup/cachegroup.go new file mode 100644 index 00000000000..06085f860f7 --- /dev/null +++ b/modules/cachegroup/cachegroup.go @@ -0,0 +1,12 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cachegroup + +const ( + User = "user" + EmailAvatarLink = "email_avatar_link" + UserEmailAddresses = "user_email_addresses" + GPGKeyWithSubKeys = "gpg_key_with_subkeys" + RepoUserPermission = "repo_user_permission" +) diff --git a/modules/repository/commits.go b/modules/repository/commits.go index 16520fb28a5..878fdc16039 100644 --- a/modules/repository/commits.go +++ b/modules/repository/commits.go @@ -13,6 +13,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -131,7 +132,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repo *repo_model func (pc *PushCommits) AvatarLink(ctx context.Context, email string) string { size := avatars.DefaultAvatarPixelSize * setting.Avatar.RenderedSizeFactor - v, _ := cache.GetWithContextCache(ctx, "push_commits", email, func() (string, error) { + v, _ := cache.GetWithContextCache(ctx, cachegroup.EmailAvatarLink, email, func(ctx context.Context, email string) (string, error) { u, err := user_model.GetUserByEmail(ctx, email) if err != nil { if !user_model.IsErrUserNotExist(err) { diff --git a/routers/private/hook_post_receive.go b/routers/private/hook_post_receive.go index 442d0a76c90..8b1e849e7a4 100644 --- a/routers/private/hook_post_receive.go +++ b/routers/private/hook_post_receive.go @@ -14,6 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" @@ -326,9 +327,7 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) { } func loadContextCacheUser(ctx context.Context, id int64) (*user_model.User, error) { - return cache.GetWithContextCache(ctx, "hook_post_receive_user", id, func() (*user_model.User, error) { - return user_model.GetUserByID(ctx, id) - }) + return cache.GetWithContextCache(ctx, cachegroup.User, id, user_model.GetUserByID) } // handlePullRequestMerging handle pull request merging, a pull request action should push at least 1 commit diff --git a/services/asymkey/commit.go b/services/asymkey/commit.go index 5d85be56f1f..105782a93a5 100644 --- a/services/asymkey/commit.go +++ b/services/asymkey/commit.go @@ -11,6 +11,8 @@ import ( asymkey_model "code.gitea.io/gitea/models/asymkey" "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -115,7 +117,7 @@ func ParseCommitWithSignatureCommitter(ctx context.Context, c *git.Commit, commi } } - committerEmailAddresses, _ := user_model.GetEmailAddresses(ctx, committer.ID) + committerEmailAddresses, _ := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committer.ID, user_model.GetEmailAddresses) activated := false for _, e := range committerEmailAddresses { if e.IsActivated && strings.EqualFold(e.Email, c.Committer.Email) { @@ -209,10 +211,9 @@ func checkKeyEmails(ctx context.Context, email string, keys ...*asymkey_model.GP } if key.Verified && key.OwnerID != 0 { if uid != key.OwnerID { - userEmails, _ = user_model.GetEmailAddresses(ctx, key.OwnerID) + userEmails, _ = cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, key.OwnerID, user_model.GetEmailAddresses) uid = key.OwnerID - user = &user_model.User{ID: uid} - _, _ = user_model.GetUser(ctx, user) + user, _ = cache.GetWithContextCache(ctx, cachegroup.User, uid, user_model.GetUserByID) } for _, e := range userEmails { if e.IsActivated && (email == "" || strings.EqualFold(e.Email, email)) { @@ -231,10 +232,7 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s if keyID == "" { return nil } - keys, err := db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{ - KeyID: keyID, - IncludeSubKeys: true, - }) + keys, err := cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, keyID, asymkey_model.FindGPGKeyWithSubKeys) if err != nil { log.Error("GetGPGKeysByKeyID: %v", err) return &asymkey_model.CommitVerification{ @@ -249,10 +247,7 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s for _, key := range keys { var primaryKeys []*asymkey_model.GPGKey if key.PrimaryKeyID != "" { - primaryKeys, err = db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{ - KeyID: key.PrimaryKeyID, - IncludeSubKeys: true, - }) + primaryKeys, err = cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, key.PrimaryKeyID, asymkey_model.FindGPGKeyWithSubKeys) if err != nil { log.Error("GetGPGKeysByKeyID: %v", err) return &asymkey_model.CommitVerification{ @@ -272,8 +267,8 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s Name: name, Email: email, } - if key.OwnerID != 0 { - owner, err := user_model.GetUserByID(ctx, key.OwnerID) + if key.OwnerID > 0 { + owner, err := cache.GetWithContextCache(ctx, cachegroup.User, key.OwnerID, user_model.GetUserByID) if err == nil { signer = owner } else if !user_model.IsErrUserNotExist(err) { @@ -381,7 +376,7 @@ func ParseCommitWithSSHSignature(ctx context.Context, c *git.Commit, committer * } } - committerEmailAddresses, err := user_model.GetEmailAddresses(ctx, committer.ID) + committerEmailAddresses, err := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committer.ID, user_model.GetEmailAddresses) if err != nil { log.Error("GetEmailAddresses: %v", err) } diff --git a/services/convert/pull.go b/services/convert/pull.go index 34c3b1bf9a5..7798bebb08b 100644 --- a/services/convert/pull.go +++ b/services/convert/pull.go @@ -14,6 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" @@ -60,14 +61,14 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u doerID = doer.ID } - const repoDoerPermCacheKey = "repo_doer_perm_cache" - p, err := cache.GetWithContextCache(ctx, repoDoerPermCacheKey, fmt.Sprintf("%d_%d", pr.BaseRepoID, doerID), - func() (access_model.Permission, error) { + repoUserPerm, err := cache.GetWithContextCache(ctx, cachegroup.RepoUserPermission, fmt.Sprintf("%d-%d", pr.BaseRepoID, doerID), + func(ctx context.Context, _ string) (access_model.Permission, error) { return access_model.GetUserRepoPermission(ctx, pr.BaseRepo, doer) - }) + }, + ) if err != nil { log.Error("GetUserRepoPermission[%d]: %v", pr.BaseRepoID, err) - p.AccessMode = perm.AccessModeNone + repoUserPerm.AccessMode = perm.AccessModeNone } apiPullRequest := &api.PullRequest{ @@ -107,7 +108,7 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u Name: pr.BaseBranch, Ref: pr.BaseBranch, RepoID: pr.BaseRepoID, - Repository: ToRepo(ctx, pr.BaseRepo, p), + Repository: ToRepo(ctx, pr.BaseRepo, repoUserPerm), }, Head: &api.PRBranchInfo{ Name: pr.HeadBranch, diff --git a/services/git/commit.go b/services/git/commit.go index 8ab8f3d3690..3faef767829 100644 --- a/services/git/commit.go +++ b/services/git/commit.go @@ -17,7 +17,7 @@ import ( ) // ParseCommitsWithSignature checks if signaute of commits are corresponding to users gpg keys. -func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType, isOwnerMemberCollaborator func(*user_model.User) (bool, error)) ([]*asymkey_model.SignCommit, error) { +func ParseCommitsWithSignature(ctx context.Context, repo *repo_model.Repository, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType) ([]*asymkey_model.SignCommit, error) { newCommits := make([]*asymkey_model.SignCommit, 0, len(oldCommits)) keyMap := map[string]bool{} @@ -47,6 +47,10 @@ func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.Use Verification: asymkey_service.ParseCommitWithSignatureCommitter(ctx, c.Commit, committer), } + isOwnerMemberCollaborator := func(user *user_model.User) (bool, error) { + return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID) + } + _ = asymkey_model.CalculateTrustStatus(signCommit.Verification, repoTrustModel, isOwnerMemberCollaborator, &keyMap) newCommits = append(newCommits, signCommit) @@ -62,11 +66,9 @@ func ConvertFromGitCommit(ctx context.Context, commits []*git.Commit, repo *repo } signedCommits, err := ParseCommitsWithSignature( ctx, + repo, validatedCommits, repo.GetTrustModel(), - func(user *user_model.User) (bool, error) { - return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID) - }, ) if err != nil { return nil, err