Gitea-John71/Gitea
2
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-08-01 06:00:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: retry file remove/rename when a file is busy and clean up os detection (#38588)

Browse Source 
Rename functions "util.Remove" (remove.go) to "util.RemoveWithRetry"
(file_retry.go) and add comments to clarify their behaviors, also add
tests.

Refactor callers: when no concurrent access (cmd cli, migration, app
init, test), use "os.Xxx" directly.

More details are in `modules/util/file_retry.go`

By the way, clean up OS (windows) detection, make FileURLToPath test
always run
This commit is contained in:
wxiaoguang
2026-07-23 22:31:29 +08:00
committed by GitHub
parent c4fc4d363b
commit e992b0a7cc
32 changed files with 225 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/util/file_retry.go
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package util
import (
"errors"
"os"
"syscall"
"time"
)
// On Windows, when a file or directory is in use (opened), the file or directory is not able to be removed or renamed.
// When renaming or removing a local git repository directory:
// * the "cat-batch" git process might be running in a goroutine
// * there can be a data-race between the "cat-batch" git process cancel+exit and the repo rename
// So we need to retry the rename/remove operation for a few times when the "cat-batch" git process is exiting.
// ref: https://github.com/go-gitea/gitea/issues/16427, https://github.com/go-gitea/gitea/issues/16475, https://github.com/go-gitea/gitea/pull/16479
// Also some similar problems when removing a file, e.g.: https://github.com/go-gitea/gitea/issues/12339
//
// Usually, if no concurrent access to a file, use "os.Xxx", otherwise, use "util.XxxWithRetry"
func retryWhenFileBusyInternal(count int, delay time.Duration, f func() error) (err error) {
const errWindowsSharingViolationError = syscall.Errno(32)
for range count {
err = f()
if err == nil {
break
}
isErrBusy := errors.Is(err, syscall.EBUSY) || errors.Is(err, syscall.ENOTEMPTY) || errors.Is(err, syscall.EPERM) || errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE)
isErrBusy = isErrBusy || (isOSWindows && errors.Is(err, errWindowsSharingViolationError))
if !isErrBusy {
break
}
time.Sleep(delay)
}
return err
}
func retryWhenFileBusy(f func() error) (err error) {
return retryWhenFileBusyInternal(5, 100*time.Millisecond, f)
}
func RemoveWithRetry(path string) error {
return retryWhenFileBusy(func() error {
return os.Remove(path)
})
}
func RemoveAllWithRetry(path string) error {
return retryWhenFileBusy(func() error {
return os.RemoveAll(path)
})
}
func RenameWithRetry(oldpath, newpath string) error {
return retryWhenFileBusy(func() error {
return os.Rename(oldpath, newpath)
})
}
modules/util/file_retry_test.go
+44
View File
@@ -0,0 +1,44 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package util
import (
"io"
"os"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestRetryWhenFileBusy(t *testing.T) {
var callCount int
testErrPath := &os.PathError{Op: "test", Path: "test", Err: syscall.EBUSY}
fn := func() error {
if callCount == 2 {
return nil
}
callCount++
return testErrPath
}
callCount = 0
err := retryWhenFileBusyInternal(1, time.Millisecond, fn)
assert.Equal(t, testErrPath, err)
assert.Equal(t, 1, callCount)
callCount = 0
err = retryWhenFileBusyInternal(10, time.Millisecond, fn)
assert.NoError(t, err)
assert.Equal(t, 2, callCount)
callCount = 0
err = retryWhenFileBusyInternal(10, time.Millisecond, func() error {
callCount++
return io.ErrUnexpectedEOF
})
assert.Equal(t, io.ErrUnexpectedEOF, err)
assert.Equal(t, 1, callCount)
}
modules/util/path.go
+22 -24
View File
@@ -10,8 +10,6 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"strings"
)
@@ -77,7 +75,7 @@ const filepathSeparator = string(os.PathSeparator)
func FilePathJoinAbs(base string, sub ...string) string {
// POSIX filesystem can have `\` in file names. Windows: `\` and `/` are both used for path separators
// to keep the behavior consistent, we do not allow `\` in file names, replace all `\` with `/`
if !isOSWindows() {
if !isOSWindows {
base = strings.ReplaceAll(base, "\\", filepathSeparator)
}
if !filepath.IsAbs(base) {
@@ -94,7 +92,7 @@ func FilePathJoinAbs(base string, sub ...string) string {
if s == "" {
continue
}
if isOSWindows() {
if isOSWindows {
elems = append(elems, filepath.Clean(filepathSeparator+s))
} else {
elems = append(elems, filepath.Clean(filepathSeparator+strings.ReplaceAll(s, "\\", filepathSeparator)))
@@ -245,30 +243,30 @@ func ListDirRecursively(rootDir string, opts *ListDirOptions) (res []string, err
return res, nil
}
func isOSWindows() bool {
return runtime.GOOS == "windows"
}
func fileURLToPathInternal(u *url.URL, isWindows bool) (string, error) {
if u.Scheme != "file" {
return "", errors.New("URL scheme is not 'file': " + u.String())
}
if !isWindows {
return u.Path, nil
}
var driveLetterRegexp = regexp.MustCompile("/[A-Za-z]:/")
// If it is a Windows absolute path with drive letter "/C:/dir", strip off the leading slash.
if !strings.HasPrefix(u.Path, "/") || len(u.Path) < 3 {
return u.Path, nil
}
winPath := u.Path[1:]
first := winPath[0]
if ('a' <= first && first <= 'z' || 'A' <= first && first <= 'Z') && winPath[1] == ':' {
return winPath, nil
}
return u.Path, nil
}
// FileURLToPath extracts the path information from a file://... url.
// It returns an error only if the URL is not a file URL.
func FileURLToPath(u *url.URL) (string, error) {
if u.Scheme != "file" {
return "", errors.New("URL scheme is not 'file': " + u.String())
}
path := u.Path
if !isOSWindows() {
return path, nil
}
// If it looks like there's a Windows drive letter at the beginning, strip off the leading slash.
if driveLetterRegexp.MatchString(path) {
return path[1:], nil
}
return path, nil
return fileURLToPathInternal(u, isOSWindows)
}
// HomeDir returns path of '~'(in Linux) on Windows,
@@ -277,7 +275,7 @@ func HomeDir() (home string, err error) {
// TODO: some users run Gitea with mismatched uid and "HOME=xxx" (they set HOME=xxx by environment manually)
// TODO: when running gitea as a sub command inside git, the HOME directory is not the user's home directory
// so at the moment we can not use `user.Current().HomeDir`
if isOSWindows() {
if isOSWindows {
home = os.Getenv("USERPROFILE")
if home == "" {
home = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
modules/util/path_test.go
+22 -20
View File
@@ -7,7 +7,6 @@ import (
"net/url"
"os"
"path/filepath"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
@@ -18,43 +17,46 @@ func TestFileURLToPath(t *testing.T) {
cases := []struct {
url string
expected string
haserror bool
invalid bool
windows bool
}{
// case 0
{
url: "",
haserror: true,
url: "",
invalid: true,
},
// case 1
{
url: "http://test.io",
haserror: true,
url: "http://test.io",
invalid: true,
},
// case 2
{
url: "file:///path",
expected: "/path",
},
// case 3
{
url: "file:///C:/path",
expected: "C:/path",
windows: true,
},
{
url: "file:///z:/path",
expected: "z:/path",
windows: true,
},
{
url: "file:///path",
expected: "/path",
windows: true,
},
}
for n, c := range cases {
if c.windows && runtime.GOOS != "windows" {
continue
}
for _, c := range cases {
u, _ := url.Parse(c.url)
p, err := FileURLToPath(u)
if c.haserror {
assert.Error(t, err, "case %d: should return error", n)
p, err := fileURLToPathInternal(u, c.windows)
if c.invalid {
assert.Error(t, err, "case %s: should return error", c.url)
} else {
assert.NoError(t, err, "case %d: should not return error", n)
assert.Equal(t, c.expected, p, "case %d: should be equal", n)
assert.NoError(t, err, "case %s: should not return error", c.url)
assert.Equal(t, c.expected, p, "case %s: should be equal", c.url)
}
}
}
@@ -180,7 +182,7 @@ func TestCleanPath(t *testing.T) {
}
// for POSIX only, but the result is similar on Windows, because the first element must be an absolute path
if isOSWindows() {
if isOSWindows {
cases = []struct {
elems []string
expected string
modules/util/remove.go
-104
View File
@@ -1,104 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package util
import (
"os"
"runtime"
"syscall"
"time"
)
const windowsSharingViolationError syscall.Errno = 32
// Remove removes the named file or (empty) directory with at most 5 attempts.
func Remove(name string) error {
var err error
for range 5 {
err = os.Remove(name)
if err == nil {
break
}
unwrapped := err.(*os.PathError).Err
if unwrapped == syscall.EBUSY || unwrapped == syscall.ENOTEMPTY || unwrapped == syscall.EPERM || unwrapped == syscall.EMFILE || unwrapped == syscall.ENFILE {
// try again
<-time.After(100 * time.Millisecond)
continue
}
if unwrapped == windowsSharingViolationError && runtime.GOOS == "windows" {
// try again
<-time.After(100 * time.Millisecond)
continue
}
if unwrapped == syscall.ENOENT {
// it's already gone
return nil
}
}
return err
}
// RemoveAll removes the named file or (empty) directory with at most 5 attempts.
func RemoveAll(name string) error {
var err error
for range 5 {
err = os.RemoveAll(name)
if err == nil {
break
}
unwrapped := err.(*os.PathError).Err
if unwrapped == syscall.EBUSY || unwrapped == syscall.ENOTEMPTY || unwrapped == syscall.EPERM || unwrapped == syscall.EMFILE || unwrapped == syscall.ENFILE {
// try again
<-time.After(100 * time.Millisecond)
continue
}
if unwrapped == windowsSharingViolationError && runtime.GOOS == "windows" {
// try again
<-time.After(100 * time.Millisecond)
continue
}
if unwrapped == syscall.ENOENT {
// it's already gone
return nil
}
}
return err
}
// Rename renames (moves) oldpath to newpath with at most 5 attempts.
func Rename(oldpath, newpath string) error {
var err error
for i := range 5 {
err = os.Rename(oldpath, newpath)
if err == nil {
break
}
unwrapped := err.(*os.LinkError).Err
if unwrapped == syscall.EBUSY || unwrapped == syscall.ENOTEMPTY || unwrapped == syscall.EPERM || unwrapped == syscall.EMFILE || unwrapped == syscall.ENFILE {
// try again
<-time.After(100 * time.Millisecond)
continue
}
if unwrapped == windowsSharingViolationError && runtime.GOOS == "windows" {
// try again
<-time.After(100 * time.Millisecond)
continue
}
if i == 0 && os.IsNotExist(err) {
return err
}
if unwrapped == syscall.ENOENT {
// it's already gone
return nil
}
}
return err
}
modules/util/rotatingfilewriter/writer.go
+6 -4
View File
@@ -152,7 +152,7 @@ func (rfw *RotatingFileWriter) DoRotate() error {
return err
}
if err := util.Rename(fd.Name(), fname); err != nil {
if err := util.RenameWithRetry(fd.Name(), fname); err != nil {
return err
}
@@ -203,12 +203,12 @@ func compressOldFile(fname string, compressionLevel int) error {
if err != nil {
_ = zw.Close()
_ = fw.Close()
_ = util.Remove(fname + ".gz")
_ = util.RemoveWithRetry(fname + ".gz")
return fmt.Errorf("compressOldFile: failed to write to gz file: %w", err)
}
_ = reader.Close()
err = util.Remove(fname)
err = util.RemoveWithRetry(fname)
if err != nil {
return fmt.Errorf("compressOldFile: failed to delete old file: %w", err)
}
@@ -235,7 +235,9 @@ func deleteOldFiles(dir, prefix string, removeBefore time.Time) {
}
if info.ModTime().Before(removeBefore) {
if strings.HasPrefix(filepath.Base(path), prefix) {
return util.Remove(path)
if err = util.RemoveWithRetry(path); err != nil && !os.IsNotExist(err) {
errorf("deleteOldFiles: failed to delete old file %s: %v", path, err)
}
}
}
return nil
modules/util/runtime.go
+2
View File
@@ -5,6 +5,8 @@ package util
import "runtime"
const isOSWindows = runtime.GOOS == "windows"
func CallerFuncName(optSkipParent ...int) string {
pc := make([]uintptr, 1)
skipParent := 0