Gitea-John71/Gitea
2
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-08-24 17:14:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(actions)!: add RUN_RETENTION_DAYS to delete old action runs (#38855)

Browse Source 
Gitea keeps completed Actions runs forever. Artifacts and logs expire on
their own schedule, but the run rows never go away, so `action_run` and
its child tables grow without bound.

Adds `RUN_RETENTION_DAYS` to delete completed runs along with their
jobs, tasks and anything the earlier expiries left behind. It defaults
to 400 days, matching how long GitHub keeps run history browsable. A
dedicated `cleanup_action_runs` cron task performs the cleanup, so
admins can schedule it separately from the nightly artifact and log
sweep.

`0` now means "keep forever" for all three retention settings, where
`LOG_RETENTION_DAYS` and `ARTIFACT_RETENTION_DAYS` previously took it
literally and deleted everything at the next sweep.

Docs: https://gitea.com/gitea/docs/pulls/502

----

## ⚠️ BREAKING ⚠️

`RUN_RETENTION_DAYS` defaults to 400, so completed runs older than that
are deleted when the cron task next runs at midnight. Set
`RUN_RETENTION_DAYS = 0` before upgrading to keep all runs.

---------

Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: silverwind <me@silverwind.io>
This commit is contained in:
Federico A. Corazza
2026-08-22 23:32:59 +02:00
committed by GitHub
parent e6af4c341c
commit 1fa6465efd
14 changed files with 399 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
routers/api/actions/artifacts.go
+9 -20
View File
@@ -75,9 +75,11 @@ import (
"gitea.dev/modules/httplib"
"gitea.dev/modules/json"
"gitea.dev/modules/log"
"gitea.dev/modules/optional"
"gitea.dev/modules/reqctx"
"gitea.dev/modules/setting"
"gitea.dev/modules/storage"
"gitea.dev/modules/timeutil"
"gitea.dev/modules/util"
"gitea.dev/modules/web"
web_types "gitea.dev/modules/web/types"
@@ -245,25 +247,12 @@ func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) {
return
}
// get upload file size
fileRealTotalSize := getUploadFileSize(ctx)
// get artifact retention days
expiredDays := setting.Actions.ArtifactRetentionDays
if queryRetentionDays := ctx.Req.URL.Query().Get("retentionDays"); queryRetentionDays != "" {
var err error
expiredDays, err = strconv.ParseInt(queryRetentionDays, 10, 64)
if err != nil {
log.Error("Error parse retention days: %v", err)
ctx.HTTPError(http.StatusBadRequest, "Error parse retention days")
return
}
var expiry optional.Option[timeutil.TimeStamp]
if days := ctx.FormOptionalInt64("retentionDays"); days.Has() {
expiry = optional.Some(timeutil.TimeStampNow().Add(timeutil.Day * days.Value()))
}
log.Debug("[artifact] upload chunk, name: %s, path: %s, size: %d, retention days: %d",
artifactName, artifactPath, fileRealTotalSize, expiredDays)
// create or get artifact with name and path
artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays)
artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiry)
if err != nil {
log.Error("Error create or get artifact: %v", err)
ctx.HTTPError(http.StatusInternalServerError, "Error create or get artifact")
@@ -288,7 +277,7 @@ func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) {
artifact.FileSize = fileRealTotalSize
artifact.FileCompressedSize = chunksTotalSize
artifact.ContentEncodingOrType = ctx.Req.Header.Get("Content-Encoding")
if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
if err := actions.UpdateArtifact(ctx, artifact, "file_size", "file_compressed_size", "content_encoding"); err != nil {
log.Error("Error update artifact: %v", err)
ctx.HTTPError(http.StatusInternalServerError, "Error update artifact")
return
@@ -349,7 +338,7 @@ func (ar artifactRoutes) listArtifacts(ctx *ArtifactContext) {
artifacts, err := actions.FindReadableArtifacts(ctx, actions.FindArtifactsOptions{
RunID: runID,
RunAttemptIDs: attemptIDs,
Status: int(actions.ArtifactStatusUploadConfirmed),
Status: actions.ArtifactStatusUploadConfirmed,
})
if err != nil {
log.Error("Error getting artifacts: %v", err)
@@ -421,7 +410,7 @@ func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) {
RunID: runID,
RunAttemptIDs: attemptIDs,
ArtifactName: itemPath,
Status: int(actions.ArtifactStatusUploadConfirmed),
Status: actions.ArtifactStatusUploadConfirmed,
})
if err != nil {
log.Error("Error getting artifacts: %v", err)
routers/api/actions/artifacts_chunks.go
+19 -14
View File
@@ -279,9 +279,18 @@ func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID, ru
log.Debug("artifact %d chunks not found", art.ID)
continue
}
if err := mergeChunksForArtifact(ctx, chunks, st, art, ""); err != nil {
storagePath, err := mergeChunksForArtifact(chunks, st, art, "")
if err != nil {
return err
}
if storagePath == "" {
continue
}
art.StoragePath = storagePath
art.Status = actions.ArtifactStatusUploadConfirmed
if err := actions.UpdateArtifact(ctx, art, "storage_path", "status"); err != nil {
return fmt.Errorf("update artifact error: %v", err)
}
}
return nil
}
@@ -297,7 +306,9 @@ func generateArtifactStoragePath(artifact *actions.ActionArtifact) string {
return fmt.Sprintf("%d/%d/%d.%s", artifact.RunID%255, artifact.ID%255, time.Now().UnixNano(), extension)
}
func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st storage.ObjectStorage, artifact *actions.ActionArtifact, checksum string) error {
// mergeChunksForArtifact merges the uploaded chunks into one stored object and returns its path.
// An empty path means the chunks are not complete yet and nothing was merged.
func mergeChunksForArtifact(chunks []*chunkFileItem, st storage.ObjectStorage, artifact *actions.ActionArtifact, checksum string) (string, error) {
sort.Slice(chunks, func(i, j int) bool {
return chunks[i].Start < chunks[j].Start
})
@@ -316,7 +327,7 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
// if the last chunk.End + 1 is not equal to chunk.ChunkLength, means chunks are not uploaded completely
if startAt+1 != artifact.FileCompressedSize {
log.Debug("[artifact] chunks are not uploaded completely, artifact_id: %d", artifact.ID)
return nil
return "", nil
}
// use multiReader
readers := make([]io.Reader, 0, len(allChunks))
@@ -331,7 +342,7 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
var readCloser io.ReadCloser
var err error
if readCloser, err = st.Open(c.Path); err != nil {
return fmt.Errorf("open chunk error: %v, %s", err, c.Path)
return "", fmt.Errorf("open chunk error: %v, %s", err, c.Path)
}
readers = append(readers, readCloser)
}
@@ -351,10 +362,10 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
storagePath := generateArtifactStoragePath(artifact)
written, err := st.Save(storagePath, mergedReader, artifact.FileCompressedSize)
if err != nil {
return fmt.Errorf("save merged file error: %v", err)
return "", fmt.Errorf("save merged file error: %v", err)
}
if written != artifact.FileCompressedSize {
return errors.New("merged file size is not equal to chunk length")
return "", errors.New("merged file size is not equal to chunk length")
}
defer func() {
@@ -371,7 +382,7 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
rawChecksum := hashSha256.Sum(nil)
actualChecksum := hex.EncodeToString(rawChecksum)
if !strings.HasSuffix(checksum, actualChecksum) {
return fmt.Errorf("update artifact error checksum is invalid %v vs %v", checksum, actualChecksum)
return "", fmt.Errorf("update artifact error checksum is invalid %v vs %v", checksum, actualChecksum)
}
}
@@ -384,11 +395,5 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
}
}
artifact.StoragePath = storagePath
artifact.Status = actions.ArtifactStatusUploadConfirmed
if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
return fmt.Errorf("update artifact error: %v", err)
}
return nil
return storagePath, nil
}
routers/api/actions/artifactsv4.go
+24 -12
View File
@@ -107,8 +107,10 @@ import (
actions_module "gitea.dev/modules/actions"
"gitea.dev/modules/httplib"
"gitea.dev/modules/log"
"gitea.dev/modules/optional"
"gitea.dev/modules/setting"
"gitea.dev/modules/storage"
"gitea.dev/modules/timeutil"
"gitea.dev/modules/util"
"gitea.dev/modules/web"
"gitea.dev/services/actions"
@@ -332,9 +334,9 @@ func (r *artifactV4Routes) createArtifact(ctx *ArtifactContext) {
artifactName := req.Name
retentionDays := setting.Actions.ArtifactRetentionDays
var expiry optional.Option[timeutil.TimeStamp]
if req.ExpiresAt != nil {
retentionDays = int64(time.Until(req.ExpiresAt.AsTime()).Hours() / 24)
expiry = optional.Some(timeutil.TimeStamp(req.ExpiresAt.AsTime().Unix()))
}
encoding := req.GetMimeType().GetValue()
// Validate media type
@@ -347,7 +349,7 @@ func (r *artifactV4Routes) createArtifact(ctx *ArtifactContext) {
fileName = artifactName + ".zip"
}
// create or get artifact with name and path
artifact, err := actions_model.CreateArtifact(ctx, ctx.ActionTask, artifactName, fileName, retentionDays)
artifact, err := actions_model.CreateArtifact(ctx, ctx.ActionTask, artifactName, fileName, expiry)
if err != nil {
log.Error("Error create or get artifact: %v", err)
ctx.HTTPError(http.StatusInternalServerError, "Error create or get artifact")
@@ -383,7 +385,8 @@ func (r *artifactV4Routes) createArtifact(ctx *ArtifactContext) {
}
}
if err := actions_model.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
if err := actions_model.UpdateArtifact(ctx, artifact,
"content_encoding", "file_size", "file_compressed_size", "storage_path", "status"); err != nil {
log.Error("Error UpdateArtifactByID: %v", err)
ctx.HTTPError(http.StatusInternalServerError, "Error UpdateArtifactByID")
return
@@ -418,7 +421,7 @@ func (r *artifactV4Routes) uploadArtifact(ctx *ArtifactContext) {
}
artifact.FileCompressedSize += uploadedLength
artifact.FileSize += uploadedLength
if err := actions_model.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
if err := actions_model.UpdateArtifact(ctx, artifact, "file_size", "file_compressed_size"); err != nil {
log.Error("Error UpdateArtifactByID: %v", err)
ctx.HTTPError(http.StatusInternalServerError, "Error UpdateArtifactByID")
return
@@ -450,10 +453,6 @@ type BlockList struct {
Latest []string `xml:"Latest"`
}
type Latest struct {
Value string `xml:",chardata"`
}
func (r *artifactV4Routes) readBlockList(runID, artifactID int64) (*BlockList, error) {
blockListName := fmt.Sprintf("%s/%d-%d-blocklist", makeTmpPathNameV4(runID), runID, artifactID)
s, err := r.fs.Open(blockListName)
@@ -531,11 +530,24 @@ func (r *artifactV4Routes) finalizeDefaultArtifact(ctx *ArtifactContext, req *Fi
return
}
if err := mergeChunksForArtifact(ctx, chunks, r.fs, artifact, req.GetHash().GetValue()); err != nil {
storagePath, err := mergeChunksForArtifact(chunks, r.fs, artifact, req.GetHash().GetValue())
if err != nil {
log.Error("Error merge chunks: %v", err)
ctx.HTTPError(http.StatusInternalServerError, "Error merge chunks")
return
}
if storagePath == "" {
return
}
artifact.StoragePath = storagePath
artifact.Status = actions_model.ArtifactStatusUploadConfirmed
if err := actions_model.UpdateArtifact(ctx, artifact,
"storage_path", "status", "file_size", "file_compressed_size"); err != nil {
log.Error("Error UpdateArtifact: %v", err)
ctx.HTTPError(http.StatusInternalServerError, "Error UpdateArtifact")
return
}
}
func (r *artifactV4Routes) finalizeAzureServeDirect(ctx *ArtifactContext, req *FinalizeArtifactRequest, artifact *actions_model.ActionArtifact) {
@@ -583,7 +595,7 @@ func (r *artifactV4Routes) finalizeAzureServeDirect(ctx *ArtifactContext, req *F
artifact.FileSize = actualLength
artifact.FileCompressedSize = actualLength
artifact.Status = actions_model.ArtifactStatusUploadConfirmed
if err := actions_model.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
if err := actions_model.UpdateArtifact(ctx, artifact, "file_size", "file_compressed_size", "status"); err != nil {
log.Error("Error UpdateArtifactByID: %v", err)
ctx.HTTPError(http.StatusInternalServerError, "Error UpdateArtifactByID")
return
@@ -608,7 +620,7 @@ func (r *artifactV4Routes) listArtifacts(ctx *ArtifactContext) {
artifacts, err := actions_model.FindReadableArtifacts(ctx, actions_model.FindArtifactsOptions{
RunID: runID,
RunAttemptIDs: attemptIDs,
Status: int(actions_model.ArtifactStatusUploadConfirmed),
Status: actions_model.ArtifactStatusUploadConfirmed,
FinalizedArtifactsV4: true,
})
if err != nil {