Gitea-John71/Gitea
2
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-08-07 18:59:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: git cache (#38763)

Browse Source 
1. always use "last commit cache"
2. correctly build the cache key for any input (SafeCacheKey)
3. fix the git note "last commit cache FIXME" and avoid OOM
This commit is contained in:
wxiaoguang
2026-08-05 00:17:07 +08:00
committed by GitHub
parent 6347a33b34
commit deccd53c24
26 changed files with 213 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/git/commit.go
+1 -4
View File
@@ -66,10 +66,7 @@ func (c *Commit) ParentCount() int {
// GetCommitByPath return the commit of relative path object.
func (c *Commit) GetCommitByPath(ctx context.Context, gitRepo *Repository, relpath string) (*Commit, error) {
if gitRepo.LastCommitCache != nil {
return gitRepo.LastCommitCache.GetCommitByPath(ctx, c.ID.String(), relpath)
}
return gitRepo.getCommitByPathWithID(ctx, c.ID, relpath)
return gitRepo.LastCommitCache.GetCommitByPath(ctx, c.ID, relpath)
}
func (c *Commit) Tree() *Tree {
modules/git/commit_count_cache.go
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"context"
"gitea.dev/modules/cache"
)
func makeCommitsCountCacheKey(repo RepositoryFacade, ref RefName) string {
return cache.SafeCacheKey("git-commits-count:"+repo.GitRepoManagedID(), ref.String())
}
func RemoveCommitsCountCache(repo RepositoryFacade, ref RefName) {
cache.Remove(makeCommitsCountCacheKey(repo, ref))
}
func GetCommitsCountCache(ctx context.Context, repo RepositoryFacade, ref RefName, commit *Commit) (int64, error) {
if commit == nil {
return 0, nil
}
return cache.GetInt64(makeCommitsCountCacheKey(repo, ref), func() (int64, error) {
return CommitsCountOfCommit(ctx, repo, commit.ID.String())
})
}
modules/git/commit_info.go
+3 -9
View File
@@ -60,15 +60,9 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, timeout time.Duration, re
entryNames = append(entryNames, entry.Name())
}
var revs map[string]*Commit
var remainingEntryNames []string
if gitRepo.LastCommitCache != nil {
revs, remainingEntryNames, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryNames, gitRepo.LastCommitCache)
if err != nil {
return nil, nil, err
}
} else {
revs, remainingEntryNames = map[string]*Commit{}, entryNames
revs, remainingEntryNames, err := getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryNames, gitRepo.LastCommitCache)
if err != nil {
return nil, nil, err
}
if len(remainingEntryNames) > 0 {
modules/git/commit_info_nogogit_test.go
+3 -1
View File
@@ -11,6 +11,7 @@ import (
"testing"
"time"
"gitea.dev/modules/git/gitrepo"
"gitea.dev/modules/test"
"gitea.dev/modules/util"
@@ -19,7 +20,8 @@ import (
)
func TestEntries_GetCommitsInfo_ContextErr(t *testing.T) {
repo, err := OpenRepositoryLocal(t.Context(), filepath.Join(testReposDir, "repo1_bare"))
repoPath, _ := filepath.Abs(filepath.Join(testReposDir, "repo1_bare"))
repo, err := OpenRepository(t.Context(), gitrepo.RepositoryManaged("dummy", repoPath))
require.NoError(t, err)
defer repo.Close()
modules/git/commit_info_test.go
+6 -4
View File
@@ -8,6 +8,8 @@ import (
"testing"
"time"
"gitea.dev/modules/git/gitrepo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -126,9 +128,9 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) {
}
func TestEntries_GetCommitsInfo(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo1_bare")
bareRepo1, err := OpenRepositoryLocal(t.Context(), bareRepo1Path)
assert.NoError(t, err)
bareRepo1Path, _ := filepath.Abs(filepath.Join(testReposDir, "repo1_bare"))
bareRepo1, err := OpenRepository(t.Context(), gitrepo.RepositoryManaged("repo1_bare", bareRepo1Path))
require.NoError(t, err)
defer bareRepo1.Close()
testGetCommitsInfo(t, bareRepo1)
@@ -137,7 +139,7 @@ func TestEntries_GetCommitsInfo(t *testing.T) {
if err != nil {
assert.NoError(t, err)
}
clonedRepo1, err := OpenRepositoryLocal(t.Context(), clonedPath)
clonedRepo1, err := OpenRepository(t.Context(), gitrepo.RepositoryManaged("repo1_bare-clone", clonedPath))
if err != nil {
assert.NoError(t, err)
}
modules/git/git.go
+2
View File
@@ -13,6 +13,7 @@ import (
"runtime"
"strings"
"gitea.dev/modules/cache"
"gitea.dev/modules/git/gitcmd"
"gitea.dev/modules/globallock"
"gitea.dev/modules/log"
@@ -191,6 +192,7 @@ func RunGitTests(m interface{ Run() int }) {
}
func runGitTests(m interface{ Run() int }) int {
_ = cache.Init()
gitHomePath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("git-home")
if err != nil {
return testlogger.MainErrorf("unable to create temp dir: %v", err)
modules/git/last_commit_cache.go
+20 -54
View File
@@ -5,103 +5,69 @@ package git
import (
"context"
"crypto/sha256"
"fmt"
"gitea.dev/modules/cache"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
)
func getCacheKey(repoPath, commitID, entryPath string) string {
hashBytes := sha256.Sum256(fmt.Appendf(nil, "%s:%s:%s", repoPath, commitID, entryPath))
return fmt.Sprintf("last_commit:%x", hashBytes)
func getCacheKey(repo RepositoryFacade, commitID, entryPath string) string {
return cache.SafeCacheKey(fmt.Sprintf("git-last-commit:%s:%s", repo.GitRepoManagedID(), commitID), entryPath)
}
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl func() int64
ttlFn func() int64
repo *Repository
commitCache map[string]*Commit
cache cache.StringCache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(count int64, repoPath string, gitRepo *Repository, cache cache.StringCache) *LastCommitCache {
if cache == nil {
return nil
}
if count < setting.CacheService.LastCommit.CommitsCount {
return nil
}
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
ttl: setting.LastCommitCacheTTLSeconds,
cache: cache,
}
}
// Put put the last commit id with commit and entry path
// Put puts the last commit id with commit and entry path
func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
if c == nil || c.cache == nil {
return nil
}
log.Debug("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
return c.cache.Put(getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl())
return c.cache.Put(getCacheKey(c.repo, ref, entryPath), commitID, c.ttlFn())
}
// Get gets the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ctx context.Context, ref, entryPath string) (*Commit, error) {
if c == nil || c.cache == nil {
return nil, nil //nolint:nilnil // return nil when cache is not available
}
commitID, ok := c.cache.Get(getCacheKey(c.repoPath, ref, entryPath))
if !ok || commitID == "" {
lastCommitID, ok := c.cache.Get(getCacheKey(c.repo, ref, entryPath))
if !ok || lastCommitID == "" {
return nil, nil //nolint:nilnil // return nil when cache miss
}
log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, commitID)
if c.commitCache != nil {
if commit, ok := c.commitCache[commitID]; ok {
log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, commitID)
return commit, nil
}
log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, lastCommitID)
if lastCommit, ok := c.commitCache[lastCommitID]; ok {
log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, lastCommitID)
return lastCommit, nil
}
commit, err := c.repo.GetCommit(ctx, commitID)
lastCommit, err := c.repo.GetCommit(ctx, lastCommitID)
if err != nil {
return nil, err
}
if c.commitCache == nil {
c.commitCache = make(map[string]*Commit)
}
c.commitCache[commitID] = commit
return commit, nil
c.commitCache[lastCommitID] = lastCommit
return lastCommit, nil
}
// GetCommitByPath gets the last commit for the entry in the provided commit
func (c *LastCommitCache) GetCommitByPath(ctx context.Context, commitID, entryPath string) (*Commit, error) {
sha, err := NewIDFromString(commitID)
if err != nil {
return nil, err
}
lastCommit, err := c.Get(ctx, sha.String(), entryPath)
func (c *LastCommitCache) GetCommitByPath(ctx context.Context, entryCommitID ObjectID, entryPath string) (*Commit, error) {
entryCommitIDStr := entryCommitID.String()
lastCommit, err := c.Get(ctx, entryCommitIDStr, entryPath)
if err != nil || lastCommit != nil {
return lastCommit, err
}
lastCommit, err = c.repo.getCommitByPathWithID(ctx, sha, entryPath)
lastCommit, err = c.repo.getCommitByPathWithID(ctx, entryCommitID, entryPath)
if err != nil {
return nil, err
}
if err := c.Put(commitID, entryPath, lastCommit.ID.String()); err != nil {
log.Error("Unable to cache %s as the last commit for %q in %s %s. Error %v", lastCommit.ID.String(), entryPath, commitID, c.repoPath, err)
if err := c.Put(entryCommitIDStr, entryPath, lastCommit.ID.String()); err != nil {
log.Error("Unable to cache %s as the last commit for %q in %s %s. Error %v", lastCommit.ID.String(), entryPath, entryCommitID, c.repo.LogString(), err)
}
return lastCommit, nil
modules/git/last_commit_cache_gogit.go
-3
View File
@@ -14,9 +14,6 @@ import (
// CacheCommit will cache the commit from the gitRepository
func (c *Commit) CacheCommit(ctx context.Context, gitRepo *Repository) error {
if gitRepo.LastCommitCache == nil {
return nil
}
commitNodeIndex, closer := gitRepo.CommitNodeIndex()
defer closer()
modules/git/last_commit_cache_nogogit.go
-3
View File
@@ -11,9 +11,6 @@ import (
// CacheCommit will cache the commit from the gitRepository
func (c *Commit) CacheCommit(ctx context.Context, gitRepo *Repository) error {
if gitRepo.LastCommitCache == nil {
return nil
}
return c.recursiveCache(ctx, gitRepo, c.Tree(), "", 1)
}
modules/git/notes.go
+55 -65
View File
@@ -5,10 +5,10 @@ package git
import (
"context"
"io"
"path"
"strings"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
)
// NotesRef is the git ref where Gitea will look for git-notes data.
@@ -17,83 +17,73 @@ const NotesRef = "refs/notes/commits"
// Note stores information about a note created using git-notes.
type Note struct {
Message []byte
Commit *Commit
refCommit *Commit
BlobMessage CommitMessage // if the blob is too large, the message will be truncated
BlobSize int64
TreePath string
}
// GetNote retrieves the git-notes data for a given commit.
// FIXME: Add LastCommitCache support
func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) error {
log.Trace("Searching for git note corresponding to the commit %q in the repository %q", commitID, repo.LogString())
notes, err := repo.GetCommit(ctx, NotesRef)
func GetNote(ctx context.Context, repo *Repository, commitID string) (*Note, error) {
noteCommit, err := repo.GetCommit(ctx, NotesRef)
if err != nil {
if IsErrNotExist(err) {
return err
}
log.Error("Unable to get commit from ref %q. Error: %v", NotesRef, err)
return err
return nil, err
}
path := ""
tree := notes.Tree()
log.Trace("Found tree with ID %q while searching for git note corresponding to the commit %q", tree.ID, commitID)
// A note for a commit is stored in a blob in the notes commit tree, with the path being the commit ID.
// The path can be "FullCommitID" or a fanout path like "ab/cdef...." or "ab/cd/ef.....".
tree := noteCommit.Tree()
entryName := commitID
var entry *TreeEntry
originalCommitID := commitID
for len(commitID) > 2 {
entry, err = tree.GetTreeEntryByPath(ctx, repo, commitID)
var treePathBuf strings.Builder
for len(entryName) > 2 {
entry, err = tree.GetTreeEntryByPath(ctx, repo, entryName)
if err == nil {
path += commitID
treePathBuf.WriteString(entryName)
break
}
if IsErrNotExist(err) {
tree, err = tree.SubTree(ctx, repo, commitID[0:2])
path += commitID[0:2] + "/"
commitID = commitID[2:]
}
if err != nil {
// Err may have been updated by the SubTree we need to recheck if it's again an ErrNotExist
if !IsErrNotExist(err) {
log.Error("Unable to find git note corresponding to the commit %q. Error: %v", originalCommitID, err)
} else if IsErrNotExist(err) {
fanoutDir, fanoutName := entryName[0:2], entryName[2:]
tree, err = tree.SubTree(ctx, repo, fanoutDir)
if err != nil {
return nil, err
}
return err
treePathBuf.WriteString(fanoutDir)
treePathBuf.WriteByte('/')
entryName = fanoutName
} else {
return nil, err
}
}
if entry == nil {
return nil, ErrNotExist{ID: commitID}
}
treePath := treePathBuf.String()
blob := entry.Blob(repo)
dataRc, err := blob.DataAsync(ctx)
note := &Note{TreePath: treePath, refCommit: noteCommit}
note.BlobMessage.MessageRaw, err = blob.GetBlobContent(ctx, setting.UI.MaxDisplayFileSize)
if err != nil {
log.Error("Unable to read blob with ID %q. Error: %v", blob.ID, err)
return err
return nil, err
}
closed := false
defer func() {
if !closed {
_ = dataRc.Close()
}
}()
d, err := io.ReadAll(dataRc)
if err != nil {
log.Error("Unable to read blob with ID %q. Error: %v", blob.ID, err)
return err
}
_ = dataRc.Close()
closed = true
note.Message = d
treePath := ""
if idx := strings.LastIndex(path, "/"); idx > -1 {
treePath = path[:idx]
path = path[idx+1:]
}
lastCommits, err := GetLastCommitForPaths(ctx, repo, notes, treePath, []string{path})
if err != nil {
log.Error("Unable to get the commit for the path %q. Error: %v", treePath, err)
return err
}
note.Commit = lastCommits[path]
return nil
note.BlobSize = blob.Size(ctx) // it should be called after the get blob content, then the "size" is cached
return note, nil
}
func GetNoteWithLastCommit(ctx context.Context, repo *Repository, commitID string) (*Note, *Commit, error) {
note, err := GetNote(ctx, repo, commitID)
if err != nil {
return nil, nil, err
}
parentPath, entryName := path.Split(note.TreePath)
parentPath = strings.Trim(parentPath, "/")
lastCommits, err := GetLastCommitForPaths(ctx, repo, note.refCommit, parentPath, []string{entryName})
if err != nil {
return nil, nil, err
}
lastCommit := lastCommits[entryName]
if lastCommit == nil {
return nil, nil, ErrNotExist{ID: commitID}
}
return note, lastCommit, nil
}
modules/git/notes_test.go
+27 -32
View File
@@ -7,45 +7,40 @@ import (
"path/filepath"
"testing"
"gitea.dev/modules/git/gitrepo"
"github.com/stretchr/testify/assert"
)
func TestGetNotes(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo1_bare")
bareRepo1, err := OpenRepositoryLocal(t.Context(), bareRepo1Path)
assert.NoError(t, err)
defer bareRepo1.Close()
note := Note{}
err = GetNote(t.Context(), bareRepo1, "95bb4d39648ee7e325106df01a621c530863a653", &note)
assert.NoError(t, err)
assert.Equal(t, []byte("Note contents\n"), note.Message)
assert.Equal(t, "Vladimir Panteleev", note.Commit.Author.Name)
}
func TestGetNestedNotes(t *testing.T) {
repoPath := filepath.Join(testReposDir, "repo3_notes")
repo, err := OpenRepositoryLocal(t.Context(), repoPath)
func TestGetNote(t *testing.T) {
repo, err := OpenRepositoryLocal(t.Context(), filepath.Join(testReposDir, "repo1_bare"))
assert.NoError(t, err)
defer repo.Close()
note := Note{}
err = GetNote(t.Context(), repo, "3e668dbfac39cbc80a9ff9c61eb565d944453ba4", &note)
note, err := GetNote(t.Context(), repo, "95bb4d39648ee7e325106df01a621c530863a653")
assert.NoError(t, err)
assert.Equal(t, []byte("Note 2"), note.Message)
err = GetNote(t.Context(), repo, "ba0a96fa63532d6c5087ecef070b0250ed72fa47", &note)
assert.NoError(t, err)
assert.Equal(t, []byte("Note 1"), note.Message)
}
assert.Equal(t, "Note contents\n", note.BlobMessage.MessageUTF8())
assert.EqualValues(t, len(note.BlobMessage.MessageRaw), note.BlobSize)
func TestGetNonExistentNotes(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo1_bare")
bareRepo1, err := OpenRepositoryLocal(t.Context(), bareRepo1Path)
assert.NoError(t, err)
defer bareRepo1.Close()
note := Note{}
err = GetNote(t.Context(), bareRepo1, "non_existent_sha", &note)
assert.Error(t, err)
_, err = GetNote(t.Context(), repo, "non_existent_sha")
assert.ErrorAs(t, err, &ErrNotExist{})
}
func TestGetNoteNestedWithCache(t *testing.T) {
repoPath, _ := filepath.Abs(filepath.Join(testReposDir, "repo3_notes"))
repo, err := OpenRepository(t.Context(), gitrepo.RepositoryManaged("repo3_notes", repoPath))
assert.NoError(t, err)
defer repo.Close()
note, lastCommit, err := GetNoteWithLastCommit(t.Context(), repo, "ba0a96fa63532d6c5087ecef070b0250ed72fa47")
assert.NoError(t, err)
assert.Equal(t, "Note 1", note.BlobMessage.MessageUTF8())
assert.Equal(t, "ba0a96fa63532d6c5087ecef070b0250ed72fa47", note.TreePath)
assert.Equal(t, "Filip Navara", lastCommit.Author.Name)
note, lastCommit, err = GetNoteWithLastCommit(t.Context(), repo, "3e668dbfac39cbc80a9ff9c61eb565d944453ba4")
assert.NoError(t, err)
assert.Equal(t, "Note 2", note.BlobMessage.MessageUTF8())
assert.Equal(t, "3e/66/8dbfac39cbc80a9ff9c61eb565d944453ba4", note.TreePath)
assert.Equal(t, "Filip Navara", lastCommit.Author.Name)
}
modules/git/repo.go
+6
View File
@@ -16,6 +16,7 @@ import (
"sync"
"time"
"gitea.dev/modules/cache"
"gitea.dev/modules/git/gitcmd"
"gitea.dev/modules/git/gitrepo"
"gitea.dev/modules/proxy"
@@ -71,6 +72,11 @@ func OpenRepository(catFileBatchCtx context.Context, repo RepositoryFacade) (*Re
gitRepo := &Repository{
RepositoryBase: RepositoryBase{tagCache: newObjectCache[*Tag](), repoFacade: repo, catFileBatchCtx: catFileBatchCtx},
}
gitRepo.RepositoryBase.LastCommitCache = &LastCommitCache{
repo: gitRepo,
ttlFn: setting.LastCommitCacheTTLSeconds,
cache: cache.GetCache(),
}
if err = openRepositoryInternal(gitRepo); err != nil {
return nil, err
}