mirror of
https://github.com/go-gitea/gitea.git
synced 2026-08-24 18:24:44 +00:00
feat(actions)!: add RUN_RETENTION_DAYS to delete old action runs (#38855)
Gitea keeps completed Actions runs forever. Artifacts and logs expire on their own schedule, but the run rows never go away, so `action_run` and its child tables grow without bound. Adds `RUN_RETENTION_DAYS` to delete completed runs along with their jobs, tasks and anything the earlier expiries left behind. It defaults to 400 days, matching how long GitHub keeps run history browsable. A dedicated `cleanup_action_runs` cron task performs the cleanup, so admins can schedule it separately from the nightly artifact and log sweep. `0` now means "keep forever" for all three retention settings, where `LOG_RETENTION_DAYS` and `ARTIFACT_RETENTION_DAYS` previously took it literally and deleted everything at the next sweep. Docs: https://gitea.com/gitea/docs/pulls/502 ---- ## ⚠️ BREAKING ⚠️ `RUN_RETENTION_DAYS` defaults to 400, so completed runs older than that are deleted when the cron task next runs at midnight. Set `RUN_RETENTION_DAYS = 0` before upgrading to keep all runs. --------- Co-authored-by: bircni <bircni@icloud.com> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: silverwind <me@silverwind.io>
This commit is contained in:
committed by
GitHub
parent
e6af4c341c
commit
1fa6465efd
+77
-22
@@ -5,7 +5,6 @@ package actions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -54,7 +53,7 @@ func cleanExpiredArtifacts(taskCtx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info("Found %d expired artifacts", len(artifacts))
|
||||
log.Info("Found %d expired Actions artifacts", len(artifacts))
|
||||
for _, artifact := range artifacts {
|
||||
if err := actions_model.SetArtifactExpired(taskCtx, artifact.ID); err != nil {
|
||||
log.Error("Cannot set artifact %d expired: %v", artifact.ID, err)
|
||||
@@ -64,7 +63,7 @@ func cleanExpiredArtifacts(taskCtx context.Context) error {
|
||||
log.Error("Cannot delete artifact %d: %v", artifact.ID, err)
|
||||
// go on
|
||||
}
|
||||
log.Info("Artifact %d is deleted (due to expiration)", artifact.ID)
|
||||
log.Info("Actions artifact %d is deleted (due to expiration)", artifact.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -78,7 +77,7 @@ func cleanNeedDeleteArtifacts(taskCtx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info("Found %d artifacts pending deletion", len(artifacts))
|
||||
log.Info("Found %d Actions artifacts pending deletion", len(artifacts))
|
||||
for _, artifact := range artifacts {
|
||||
if err := actions_model.SetArtifactDeleted(taskCtx, artifact.ID); err != nil {
|
||||
log.Error("Cannot set artifact %d deleted: %v", artifact.ID, err)
|
||||
@@ -88,10 +87,10 @@ func cleanNeedDeleteArtifacts(taskCtx context.Context) error {
|
||||
log.Error("Cannot delete artifact %d: %v", artifact.ID, err)
|
||||
// go on
|
||||
}
|
||||
log.Info("Artifact %d is deleted (due to pending deletion)", artifact.ID)
|
||||
log.Info("Actions artifact %d is deleted (due to pending deletion)", artifact.ID)
|
||||
}
|
||||
if len(artifacts) < deleteArtifactBatchSize {
|
||||
log.Debug("No more artifacts pending deletion")
|
||||
log.Debug("No more Actions artifacts pending deletion")
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -109,6 +108,10 @@ func removeTaskLog(ctx context.Context, task *actions_model.ActionTask) {
|
||||
|
||||
// CleanupExpiredLogs removes logs which are older than the configured retention time
|
||||
func CleanupExpiredLogs(ctx context.Context) error {
|
||||
if setting.Actions.LogRetentionDays <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
olderThan := timeutil.TimeStampNow().AddDuration(-time.Duration(setting.Actions.LogRetentionDays) * 24 * time.Hour)
|
||||
|
||||
count := 0
|
||||
@@ -134,7 +137,7 @@ func CleanupExpiredLogs(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Removed %d logs", count)
|
||||
log.Info("Removed %d expired Actions logs", count)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -146,13 +149,8 @@ func CleanupEphemeralRunners(ctx context.Context) error {
|
||||
Where(builder.Eq{"`action_runner`.`ephemeral`": true}).
|
||||
And(builder.NotIn("`action_task`.`status`", actions_model.StatusWaiting, actions_model.StatusRunning, actions_model.StatusBlocked, actions_model.StatusCancelling))
|
||||
b := builder.Delete(builder.In("id", subQuery)).From("`action_runner`")
|
||||
res, err := db.GetEngine(ctx).Exec(b)
|
||||
if err != nil {
|
||||
return fmt.Errorf("find runners: %w", err)
|
||||
}
|
||||
affected, _ := res.RowsAffected()
|
||||
log.Info("Removed %d runners", affected)
|
||||
return nil
|
||||
_, err := db.GetEngine(ctx).Exec(b)
|
||||
return err
|
||||
}
|
||||
|
||||
// CleanupEphemeralRunnersByPickedTaskOfRepo removes all ephemeral runners that have active/finished tasks on the given repository
|
||||
@@ -162,19 +160,15 @@ func CleanupEphemeralRunnersByPickedTaskOfRepo(ctx context.Context, repoID int64
|
||||
Join("INNER", "`action_task`", "`action_task`.`runner_id` = `action_runner`.`id`").
|
||||
Where(builder.And(builder.Eq{"`action_runner`.`ephemeral`": true}, builder.Eq{"`action_task`.`repo_id`": repoID}))
|
||||
b := builder.Delete(builder.In("id", subQuery)).From("`action_runner`")
|
||||
res, err := db.GetEngine(ctx).Exec(b)
|
||||
if err != nil {
|
||||
return fmt.Errorf("find runners: %w", err)
|
||||
}
|
||||
affected, _ := res.RowsAffected()
|
||||
log.Info("Removed %d runners", affected)
|
||||
return nil
|
||||
_, err := db.GetEngine(ctx).Exec(b)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteRun deletes workflow run, including all logs and artifacts.
|
||||
func DeleteRun(ctx context.Context, run *actions_model.ActionRun) error {
|
||||
if !run.Status.IsDone() {
|
||||
return errors.New("run is not done")
|
||||
// callers guarantee a terminal status, but in production delete it anyway
|
||||
setting.PanicInDevOrTesting("DeleteRun called on non-terminal run %d with status %s", run.ID, run.Status)
|
||||
}
|
||||
|
||||
repoID := run.RepoID
|
||||
@@ -262,9 +256,70 @@ func DeleteRun(ctx context.Context, run *actions_model.ActionRun) error {
|
||||
}
|
||||
for _, art := range artifacts {
|
||||
if err := storage.ActionsArtifacts.Delete(art.StoragePath); err != nil {
|
||||
// don't return any error since the database records have been deleted
|
||||
log.Error("remove artifact file %q: %v", art.StoragePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var cleanupOldRunsBatchSize = 50
|
||||
|
||||
// CleanupOldRuns deletes completed runs older than RUN_RETENTION_DAYS, along with everything under them.
|
||||
func CleanupOldRuns(ctx context.Context) error {
|
||||
if setting.Actions.RunRetentionDays <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
olderThan := timeutil.TimeStampNow().AddDuration(-time.Duration(setting.Actions.RunRetentionDays) * 24 * time.Hour)
|
||||
|
||||
doneStatuses := []actions_model.Status{
|
||||
actions_model.StatusSuccess,
|
||||
actions_model.StatusFailure,
|
||||
actions_model.StatusCancelled,
|
||||
actions_model.StatusSkipped,
|
||||
}
|
||||
|
||||
total, err := cleanupOldRuns(ctx, olderThan, doneStatuses, DeleteRun)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("Deleted %d old Actions runs before %s", total, olderThan.Format(time.RFC3339))
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupOldRuns(ctx context.Context, olderThan timeutil.TimeStamp, doneStatuses []actions_model.Status, deleteRun func(context.Context, *actions_model.ActionRun) error) (int, error) {
|
||||
total := 0
|
||||
failed := container.Set[int64]{} // skipping these stops the outer loop refetching them forever
|
||||
for {
|
||||
runs, err := actions_model.FindOldestRuns(ctx, doneStatuses, olderThan, cleanupOldRunsBatchSize)
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("FindOldestRuns: %w", err)
|
||||
}
|
||||
|
||||
realDeleted := 0
|
||||
for _, run := range runs {
|
||||
if failed.Contains(run.ID) {
|
||||
continue
|
||||
}
|
||||
if err := deleteRun(ctx, run); err != nil {
|
||||
setting.PanicInDevOrTesting("failed to delete old action run %d: %v", run.ID, err)
|
||||
failed.Add(run.ID)
|
||||
continue
|
||||
}
|
||||
total++
|
||||
realDeleted++
|
||||
log.Trace("Deleted old action run %d (created at %s)", run.ID, run.Created.AsTime())
|
||||
}
|
||||
|
||||
if realDeleted == 0 {
|
||||
if len(runs) != 0 {
|
||||
log.Error("Too many actions runs are unable to delete, please figure out and fix the failures")
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package actions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
actions_model "gitea.dev/models/actions"
|
||||
"gitea.dev/models/db"
|
||||
repo_model "gitea.dev/models/repo"
|
||||
"gitea.dev/models/unittest"
|
||||
"gitea.dev/modules/container"
|
||||
"gitea.dev/modules/optional"
|
||||
"gitea.dev/modules/setting"
|
||||
"gitea.dev/modules/test"
|
||||
"gitea.dev/modules/timeutil"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func insertCleanupRun(t *testing.T, index int64, status actions_model.Status, created timeutil.TimeStamp) *actions_model.ActionRun {
|
||||
t.Helper()
|
||||
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
|
||||
run := &actions_model.ActionRun{
|
||||
Title: "cleanup-run", RepoID: repo.ID, OwnerID: repo.OwnerID, WorkflowID: "test.yaml", Index: index,
|
||||
TriggerUserID: 1, Ref: "refs/heads/main",
|
||||
CommitSHA: "c2d72f548424103f01ee1dc02889c1e2bff816b0", Event: "push", TriggerEvent: "push",
|
||||
Status: status,
|
||||
}
|
||||
require.NoError(t, db.Insert(t.Context(), run))
|
||||
// XORM's "created" tag ignores explicit updates to the column even with NoAutoTime, so backdate it with raw SQL.
|
||||
_, err := db.GetEngine(t.Context()).Exec("UPDATE action_run SET created = ? WHERE id = ?", created, run.ID)
|
||||
require.NoError(t, err)
|
||||
return run
|
||||
}
|
||||
|
||||
func deleteAllRuns(t *testing.T) {
|
||||
t.Helper()
|
||||
_, err := db.GetEngine(t.Context()).Exec("DELETE FROM action_run")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestCleanupOldRuns(t *testing.T) {
|
||||
require.NoError(t, unittest.PrepareTestDatabase())
|
||||
defer test.MockVariableValue(&setting.Actions.RunRetentionDays, 30)()
|
||||
|
||||
now := timeutil.TimeStampNow()
|
||||
old := now.AddDuration(-40 * 24 * time.Hour)
|
||||
deleteAllRuns(t)
|
||||
|
||||
t.Run("disabled retention is a no-op", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&setting.Actions.RunRetentionDays, 0)()
|
||||
run := insertCleanupRun(t, 2001, actions_model.StatusSuccess, old)
|
||||
require.NoError(t, CleanupOldRuns(t.Context()))
|
||||
unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRun{ID: run.ID})
|
||||
})
|
||||
|
||||
t.Run("deletes old done runs, keeps recent and in-progress runs", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&cleanupOldRunsBatchSize, 3)() // also test the batch
|
||||
oldSuccess := insertCleanupRun(t, 2002, actions_model.StatusSuccess, old)
|
||||
oldFailure := insertCleanupRun(t, 2003, actions_model.StatusFailure, old)
|
||||
recent := insertCleanupRun(t, 2004, actions_model.StatusSuccess, now.AddDuration(-24*time.Hour))
|
||||
oldRunning := insertCleanupRun(t, 2005, actions_model.StatusRunning, old)
|
||||
oldCanceled := insertCleanupRun(t, 2006, actions_model.StatusCancelled, old)
|
||||
oldSkipped := insertCleanupRun(t, 2007, actions_model.StatusSkipped, old)
|
||||
|
||||
require.NoError(t, CleanupOldRuns(t.Context()))
|
||||
|
||||
unittest.AssertNotExistsBean(t, &actions_model.ActionRun{ID: oldSuccess.ID})
|
||||
unittest.AssertNotExistsBean(t, &actions_model.ActionRun{ID: oldFailure.ID})
|
||||
unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRun{ID: recent.ID})
|
||||
unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRun{ID: oldRunning.ID})
|
||||
unittest.AssertNotExistsBean(t, &actions_model.ActionRun{ID: oldCanceled.ID})
|
||||
unittest.AssertNotExistsBean(t, &actions_model.ActionRun{ID: oldSkipped.ID})
|
||||
})
|
||||
t.Run("error during deleting", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&cleanupOldRunsBatchSize, 3)()
|
||||
defer test.MockVariableValue(&setting.IsInTesting, false)() // skip the panic
|
||||
deleteAllRuns(t)
|
||||
|
||||
for i := range int64(6) {
|
||||
insertCleanupRun(t, 3000+i, actions_model.StatusSuccess, old)
|
||||
}
|
||||
var deletedIndices []int64
|
||||
deleteRun := func(ctx context.Context, run *actions_model.ActionRun) error {
|
||||
if run.Index%2 == 0 {
|
||||
return errors.New("some error")
|
||||
}
|
||||
deletedIndices = append(deletedIndices, run.Index)
|
||||
_, err := db.DeleteByID[actions_model.ActionRun](ctx, run.ID)
|
||||
return err
|
||||
}
|
||||
total, err := cleanupOldRuns(t.Context(), now, []actions_model.Status{actions_model.StatusSuccess}, deleteRun)
|
||||
require.NoError(t, err)
|
||||
// 3000/3002/3004 keep failing and fill up the batch, so the loop stops after 3001 and 3003
|
||||
assert.Equal(t, 2, total)
|
||||
assert.Equal(t, []int64{3001, 3003}, deletedIndices)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCleanupRetentionZeroKeepsForever(t *testing.T) {
|
||||
require.NoError(t, unittest.PrepareTestDatabase())
|
||||
defer test.MockVariableValue(&setting.Actions.LogRetentionDays, 0)()
|
||||
defer test.MockVariableValue(&setting.Actions.ArtifactRetentionDays, 0)()
|
||||
|
||||
liveLogs := unittest.Cond("stopped > 0 AND log_expired = ?", false)
|
||||
|
||||
t.Run("logs", func(t *testing.T) {
|
||||
before := unittest.GetCount(t, &actions_model.ActionTask{}, liveLogs)
|
||||
require.Positive(t, before)
|
||||
require.NoError(t, CleanupExpiredLogs(t.Context()))
|
||||
assert.Equal(t, before, unittest.GetCount(t, &actions_model.ActionTask{}, liveLogs))
|
||||
})
|
||||
|
||||
t.Run("artifacts", func(t *testing.T) {
|
||||
task := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionTask{ID: 47})
|
||||
art, err := actions_model.CreateArtifact(t.Context(), task, "never-expires", "a.txt", optional.None[timeutil.TimeStamp]())
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, art.ExpiredUnix)
|
||||
|
||||
// a workflow-requested expiry is honored, only the instance default may mean never
|
||||
asked, err := actions_model.CreateArtifact(t.Context(), task, "client-asked", "b.txt", optional.Some(timeutil.TimeStampNow()))
|
||||
require.NoError(t, err)
|
||||
assert.Positive(t, asked.ExpiredUnix)
|
||||
|
||||
// re-uploading refreshes the expiry and returns it
|
||||
reuploaded, err := actions_model.CreateArtifact(t.Context(), task, "client-asked", "b.txt", optional.None[timeutil.TimeStamp]())
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, reuploaded.ExpiredUnix)
|
||||
|
||||
// a past expiry must stay reapable, not land on the sentinel
|
||||
past, err := actions_model.CreateArtifact(t.Context(), task, "long-gone", "c.txt", optional.Some(timeutil.TimeStamp(-1000)))
|
||||
require.NoError(t, err)
|
||||
assert.Positive(t, past.ExpiredUnix)
|
||||
|
||||
_, err = db.GetEngine(t.Context()).In("id", art.ID, past.ID).Cols("status").
|
||||
Update(&actions_model.ActionArtifact{Status: actions_model.ArtifactStatusUploadConfirmed})
|
||||
require.NoError(t, err)
|
||||
expiring, err := actions_model.ListNeedExpiredArtifacts(t.Context())
|
||||
require.NoError(t, err)
|
||||
ids := container.FilterSlice(expiring, func(a *actions_model.ActionArtifact) (int64, bool) { return a.ID, true })
|
||||
assert.NotContains(t, ids, art.ID)
|
||||
assert.Contains(t, ids, past.ID)
|
||||
})
|
||||
}
|
||||
@@ -28,7 +28,7 @@ var (
|
||||
|
||||
func taskPickLimiter() chan struct{} {
|
||||
taskPickSemOnce.Do(func() {
|
||||
taskPickSem = make(chan struct{}, setting.Actions.MaxConcurrentTaskPicks)
|
||||
taskPickSem = make(chan struct{}, max(1, setting.Actions.MaxConcurrentTaskPicks))
|
||||
})
|
||||
return taskPickSem
|
||||
}
|
||||
|
||||
@@ -69,12 +69,20 @@ func (b *Base) FormBool(key string) bool {
|
||||
// FormOptionalBool returns an optional.Some(true) or optional.Some(false) if the value
|
||||
// for the provided key exists in the form else it returns optional.None[bool]()
|
||||
func (b *Base) FormOptionalBool(key string) optional.Option[bool] {
|
||||
value := b.Req.FormValue(key)
|
||||
if len(value) == 0 {
|
||||
s := b.Req.FormValue(key)
|
||||
if s == "" {
|
||||
return optional.None[bool]()
|
||||
}
|
||||
s := b.Req.FormValue(key)
|
||||
v, _ := strconv.ParseBool(s)
|
||||
v = v || strings.EqualFold(s, "on")
|
||||
return optional.Some(v)
|
||||
}
|
||||
|
||||
func (b *Base) FormOptionalInt64(key string) optional.Option[int64] {
|
||||
s := b.Req.FormValue(key)
|
||||
v, err := strconv.ParseInt(s, 10, 64)
|
||||
if s == "" || err != nil {
|
||||
return optional.None[int64]()
|
||||
}
|
||||
return optional.Some(v)
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ func initActionsTasks() {
|
||||
registerCancelAbandonedJobs()
|
||||
registerScheduleTasks()
|
||||
registerActionsCleanup()
|
||||
registerCleanupActionRuns()
|
||||
}
|
||||
|
||||
func registerStopZombieTasks() {
|
||||
@@ -74,3 +75,13 @@ func registerActionsCleanup() {
|
||||
return actions_service.Cleanup(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func registerCleanupActionRuns() {
|
||||
RegisterTaskFatal("cleanup_action_runs", &BaseConfig{
|
||||
Enabled: true,
|
||||
RunAtStart: false,
|
||||
Schedule: "@midnight",
|
||||
}, func(ctx context.Context, _ *user_model.User, _ *BaseConfig) error {
|
||||
return actions_service.CleanupOldRuns(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user